Don't leak file handles

* Update `detsys-ts` for: `Ignore hyphen-sep'd diags (#83)` (`07c7fc924119a8d9879c1c164ae593049d47f648`)

* Wait for the socket to appear

* ...

---------

Co-authored-by: grahamc <76716+grahamc@users.noreply.github.com>
Co-authored-by: Graham Christensen <graham@grahamc.com>

.github/workflows/ci.yml

@@ -3,32 +3,17 @@ name: CI
 on:
   pull_request:
   push:
-    branches: [main]
+    branches: [main, curl-data]
   workflow_dispatch:
 
 jobs:
-  tests:
-    runs-on: ubuntu-22.04
-    needs:
-      - check-dist-up-to-date
-      - install-nix
-      - install-with-non-default-source-inputs
-      - install-no-id-token
-    # NOTE(cole-h): GitHub treats "skipped" as "OK" for the purposes of required checks on branch
-    # protection, so we take advantage of this fact and fail if any of the dependent actions failed,
-    # or "skip" (which is a success for GHA's purposes) if none of them did.
-    if: failure()
-    steps:
-      - name: Dependent checks failed
-        run: exit 1
-
   check-dist-up-to-date:
     name: Check the dist/ folder is up to date
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - uses: DeterminateSystems/flakehub-cache-action@main
       - name: pnpm install
         run: nix develop --command pnpm install
       - name: prettier format
@@ -47,18 +32,12 @@ jobs:
   install-nix:
     name: "Test: ${{ matrix.runner }}${{ matrix.determinate && ' with determinate' || '' }}"
     strategy:
+      fail-fast: false
       matrix:
         runner:
           - ubuntu-latest
-          - nscloud-ubuntu-22.04-amd64-4x16
-          - namespace-profile-default-arm64
-          # - macos-12-large # determinate-nixd is broken on macos-12
-          - macos-13-large
-          - macos-14-large
-          - macos-14-xlarge # arm64
         determinate:
           - true
-          - false
     runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
@@ -73,6 +52,12 @@ jobs:
           backtrace: full
           _internal-strict-mode: true
           determinate: ${{ matrix.determinate }}
+        # - name: Breakpoint if tests failed
+        #   uses: namespacelabs/breakpoint-action@v0
+        #   with:
+        #     duration: 30m
+        #     authorized-users: grahamc
+
       - name: echo $PATH
         run: echo $PATH
 
@@ -142,38 +127,3 @@ jobs:
           cat -n /etc/nix/nix.conf
           nix config show | grep -E "^trusted-users = .*$USER"
           nix config show | grep -E "^use-sqlite-wal = true"
-
-  install-with-non-default-source-inputs:
-    name: Install Nix using non-default source-${{ matrix.inputs.key }}
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        inputs:
-          # https://github.com/DeterminateSystems/nix-installer/blob/v0.18.0
-          - key: url
-            value: https://github.com/DeterminateSystems/nix-installer/releases/download/v0.18.0/nix-installer-x86_64-linux
-            nix-version: "2.21.2"
-          # https://github.com/DeterminateSystems/nix-installer/tree/7011c077ec491da410fbc39f68676b0908b9ce7e
-          - key: revision
-            value: 7011c077ec491da410fbc39f68676b0908b9ce7e
-            nix-version: "2.19.2"
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install with alternative source-${{ matrix.inputs.key }}
-        uses: ./
-        with:
-          source-${{ matrix.inputs.key }}: ${{ matrix.inputs.value }}
-          _internal-strict-mode: true
-      - name: Ensure that the expected Nix version ${{ matrix.inputs.nix-version }} is installed via alternative source-${{ matrix.inputs.key }}
-        run: .github/verify-version.sh ${{ matrix.inputs.nix-version }}
-
-  install-no-id-token:
-    name: Install Nix without an ID token
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./
-        with:
-          _internal-strict-mode: true
-          determinate: true

package.json

@@ -25,26 +25,26 @@
   },
   "homepage": "https://github.com/DeterminateSystems/nix-installer-action#readme",
   "dependencies": {
-    "@actions/core": "^1.10.1",
+    "@actions/core": "^1.11.1",
     "@actions/exec": "^1.1.1",
-    "@actions/github": "^5.1.1",
+    "@actions/github": "^6.0.0",
     "detsys-ts": "github:DeterminateSystems/detsys-ts",
-    "got": "^14.3.0",
+    "got": "^14.4.6",
     "string-argv": "^0.3.2"
   },
   "devDependencies": {
     "@trivago/prettier-plugin-sort-imports": "^4.3.0",
-    "@types/node": "^20.14.0",
+    "@types/node": "^20.17.28",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/eslint-plugin": "^7.12.0",
-    "@vercel/ncc": "^0.38.1",
-    "eslint": "^8.57.0",
-    "eslint-import-resolver-typescript": "^3.6.1",
+    "@typescript-eslint/eslint-plugin": "^7.18.0",
+    "@vercel/ncc": "^0.38.3",
+    "eslint": "^8.57.1",
+    "eslint-import-resolver-typescript": "^3.10.0",
     "eslint-plugin-github": "^4.10.2",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-prettier": "^5.1.3",
-    "prettier": "^3.3.0",
-    "tsup": "^8.1.0",
-    "typescript": "^5.4.5"
+    "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-prettier": "^5.2.5",
+    "prettier": "^3.5.3",
+    "tsup": "^8.4.0",
+    "typescript": "^5.8.2"
   }
 }

src/index.ts

@@ -1,6 +1,6 @@
 import * as actionsCore from "@actions/core";
 import * as actionsExec from "@actions/exec";
-import { access, readFile } from "node:fs/promises";
+import { access, open, readFile, stat } from "node:fs/promises";
 import { join } from "node:path";
 import fs from "node:fs";
 import { userInfo } from "node:os";
@@ -9,6 +9,8 @@ import * as path from "path";
 import { DetSysAction, inputs, platform, stringifyError } from "detsys-ts";
 import { randomUUID } from "node:crypto";
 import got from "got";
+import { setTimeout } from "node:timers/promises";
+import { SpawnOptions, spawn } from "node:child_process";
 
 // Nix installation events
 const EVENT_INSTALL_NIX_FAILURE = "install_nix_failure";
@@ -38,6 +40,10 @@ const FACT_NIX_INSTALLER_PLANNER = "nix_installer_planner";
 // Flags
 const FLAG_DETERMINATE = "--determinate";
 
+// Pre/post state keys
+const STATE_EVENT_LOG = "DETERMINATE_NIXD_EVENT_LOG";
+const STATE_EVENT_PID = "DETERMINATE_NIXD_EVENT_PID";
+
 class NixInstallerAction extends DetSysAction {
   determinate: boolean;
   platform: string;
@@ -120,11 +126,14 @@ class NixInstallerAction extends DetSysAction {
     await this.scienceDebugFly();
     await this.detectAndForceDockerShim();
     await this.install();
+    await this.spewEventLog();
   }
 
   async post(): Promise<void> {
     await this.cleanupDockerShim();
     await this.reportOverall();
+    await this.slurpEventLog();
+    await this.cleanupLogger();
   }
 
   private get isMacOS(): boolean {
@@ -819,10 +828,52 @@ class NixInstallerAction extends DetSysAction {
       }
     }
 
+    const maxDurationSeconds = 120;
+    const delayPerAttemptInMiliseconds = 50;
+    const maxAttempts =
+      (maxDurationSeconds * 1000) / delayPerAttemptInMiliseconds;
+    let didSucceed = false;
+
+    for (let attempt = 0; attempt <= maxAttempts; attempt += 1) {
+      if (await this.doesTheSocketExistYet()) {
+        didSucceed = true;
+        break;
+      }
+
+      await setTimeout(50);
+    }
+
+    if (!didSucceed) {
+      throw new Error("Timed out waiting for the Nix Daemon");
+    }
+
     actionsCore.endGroup();
 
     return;
   }
+
+  async doesTheSocketExistYet(): Promise<boolean> {
+    const socketPath = "/nix/var/nix/daemon-socket/socket";
+    try {
+      await stat(socketPath);
+      return true;
+    } catch (error: unknown) {
+      // eslint-disable-next-line no-undef
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+        actionsCore.debug(`Socket '${socketPath}' does not exist yet`);
+        return false;
+      }
+
+      actionsCore.warning(
+        `Error waiting for the Nix Daemon socket: ${stringifyError(error)}`,
+      );
+      this.recordEvent("docker-shim:wait-for-socket", {
+        exception: stringifyError(error),
+      });
+      throw error;
+    }
+  }
+
   async cleanupDockerShim(): Promise<void> {
     const containerId = actionsCore.getState("docker_shim_container_id");
 
@@ -1061,6 +1112,207 @@ class NixInstallerAction extends DetSysAction {
       );
     }
   }
+
+  private async spewEventLog(): Promise<void> {
+    if (!this.determinate) {
+      return;
+    }
+
+    const logfile = this.getTemporaryName();
+    actionsCore.saveState(STATE_EVENT_LOG, logfile);
+    const stdout = await open(logfile, "a");
+    const stderr = await open(`${logfile}.stderr`, "a");
+
+    actionsCore.debug(`Event log: ${logfile}`);
+
+    const opts: SpawnOptions = {
+      stdio: ["ignore", stdout.fd, stderr.fd],
+      detached: true,
+    };
+
+    // Start the server. Once it is ready, it will notify us via the notification server.
+    const daemon = spawn(
+      "curl",
+      [
+        "--no-buffer",
+        "--unix-socket",
+        "/nix/var/determinate/determinate-nixd.socket",
+        "http://localhost/events",
+      ],
+      opts,
+    );
+
+    actionsCore.saveState(STATE_EVENT_PID, daemon.pid);
+    daemon.unref();
+
+    // Wait a tick in the event loop in order for curl to actually be running
+    await Promise.resolve();
+
+    try {
+      await stdout.close();
+    } catch (error) {
+      actionsCore.info(`Could not close curl's stdout: ${error}`);
+    }
+
+    try {
+      await stderr.close();
+    } catch (error) {
+      actionsCore.info(`Could not close curl's stderr: ${error}`);
+    }
+  }
+
+  private async slurpEventLog(): Promise<void> {
+    if (!this.determinate) {
+      return;
+    }
+
+    try {
+      const logPath = actionsCore.getState(STATE_EVENT_LOG);
+      const events = await readMismatchEvents(logPath);
+
+      // No point doing any more work if there are no mismatch events
+      if (events.length === 0) {
+        actionsCore.debug("No hash mismatches found.");
+        return;
+      }
+
+      const listing = await getFileListing();
+
+      // For each file, search for potentially bad hashes
+      for (const file of listing) {
+        const text = await readFile(file, "utf-8");
+        const lines = text.split("\n");
+
+        for (const [index, line] of lines.entries()) {
+          const lineNumber = index + 1;
+
+          for (const event of events) {
+            const match = line.match(event.search);
+            if (!match) {
+              continue;
+            }
+
+            // Allegedly, match.index is optional, so default to 0
+            const column = (match.index ?? 0) + 1;
+
+            actionsCore.error(`This derivation's hash is \`${event.good}\``, {
+              title: "Determinate Nix detected an incorrect dependency hash.",
+              file,
+              startLine: lineNumber,
+              startColumn: column,
+            });
+          }
+        }
+      }
+    } catch (error) {
+      // Don't hard fail the action if something exploded; this feature is only a nice-to-have
+      actionsCore.warning(`Could not consume hash mismatch logs: ${error}`);
+    }
+  }
+
+  async cleanupLogger(): Promise<void> {
+    const rawPid = actionsCore.getState(STATE_EVENT_PID);
+    const pid = Number(rawPid);
+
+    try {
+      process.kill(pid);
+    } catch (error) {
+      actionsCore.info(`Could not kill pid ${rawPid}: ${error}`);
+    }
+  }
+}
+
+// Fields we're interested in from the source event
+interface MismatchSourceEvent {
+  readonly drv: string;
+  readonly good: string;
+  readonly bad: readonly string[];
+}
+
+// Our augmented event with the RegExp to match against the bad hashes
+interface MismatchEvent extends MismatchSourceEvent {
+  readonly search: RegExp;
+}
+
+async function readMismatchEvents(logPath: string): Promise<MismatchEvent[]> {
+  const prefix = "data: ";
+
+  // Used to deduplicate events (see below)
+  const memo = new Set<string>();
+
+  const events = (await readFile(logPath, "utf-8"))
+    .split(/\n/)
+    .filter((line) => line.startsWith(prefix))
+    .map((line) => {
+      // Note: this currently assumes that all events being ingested are mismatches
+      const json = line.slice(prefix.length);
+      const source = JSON.parse(json) as MismatchSourceEvent;
+
+      // Construct a regular expression to search for any of the hash patterns
+      // (do it here to avoid creating RegExp objects in a loop below)
+      const search = new RegExp(
+        source.bad.map((s) => s.replace(/[+]/g, (ch) => `\\${ch}`)).join("|"),
+      );
+
+      return {
+        ...source,
+        search,
+      } satisfies MismatchEvent;
+    })
+    .filter((event) => {
+      // Deduplicate based on the derivation's store path and list of bad hashes.
+      const key = [event.drv, ...event.bad].join("\0");
+      if (memo.has(key)) {
+        false;
+      }
+
+      memo.add(key);
+      return true;
+    });
+
+  return events;
+}
+
+// Get the list of files with potential hash mismatches (limited currently to *.{nix,json,toml})
+async function getFileListing(): Promise<readonly string[]> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    let length = 0;
+
+    const child = spawn(
+      "git",
+      ["ls-files", "-z", "*.nix", "*.json", "*.toml"],
+      {
+        stdio: ["ignore", "pipe", "inherit"],
+      },
+    );
+
+    child.stdout.on("data", (chunk: Buffer) => {
+      chunks.push(chunk);
+      length += chunk.length;
+    });
+
+    child.stdout.on("end", () => {
+      const lines = Buffer.concat(chunks, length)
+        .toString("utf-8")
+        .replace(/\0$/, "")
+        .split(/\0/);
+
+      resolve(lines);
+    });
+
+    child.stdout.on("error", reject);
+
+    child.on("error", reject);
+    child.on("exit", (code, signal) => {
+      // We should consider rejecting the promise here
+      if (code !== 0) {
+        actionsCore.warning(
+          `git ls-files exited suspiciously code=${code}; signal=${signal}`,
+        );
+      }
+    });
+  });
 }
 
 type ExecuteEnvironment = {