Don't leak file handles

.github/workflows/ci.yml

@@ -3,25 +3,10 @@ name: CI
 on:
   pull_request:
   push:
-    branches: [main]
+    branches: [main, curl-data]
   workflow_dispatch:
 
 jobs:
-  tests:
-    runs-on: ubuntu-22.04
-    needs:
-      - check-dist-up-to-date
-      - install-nix
-      - install-with-non-default-source-inputs
-      - install-no-id-token
-    # NOTE(cole-h): GitHub treats "skipped" as "OK" for the purposes of required checks on branch
-    # protection, so we take advantage of this fact and fail if any of the dependent actions failed,
-    # or "skip" (which is a success for GHA's purposes) if none of them did.
-    if: failure()
-    steps:
-      - name: Dependent checks failed
-        run: exit 1
-
   check-dist-up-to-date:
     name: Check the dist/ folder is up to date
     runs-on: ubuntu-22.04
@@ -51,15 +36,8 @@ jobs:
       matrix:
         runner:
           - ubuntu-latest
-          - nscloud-ubuntu-22.04-amd64-4x16
-          - namespace-profile-default-arm64
-          # - macos-12-large # determinate-nixd is broken on macos-12
-          - macos-13-large
-          - macos-14-large
-          - macos-14-xlarge # arm64
         determinate:
           - true
-          - false
     runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
@@ -74,6 +52,12 @@ jobs:
           backtrace: full
           _internal-strict-mode: true
           determinate: ${{ matrix.determinate }}
+        # - name: Breakpoint if tests failed
+        #   uses: namespacelabs/breakpoint-action@v0
+        #   with:
+        #     duration: 30m
+        #     authorized-users: grahamc
+
       - name: echo $PATH
         run: echo $PATH
 
@@ -143,38 +127,3 @@ jobs:
           cat -n /etc/nix/nix.conf
           nix config show | grep -E "^trusted-users = .*$USER"
           nix config show | grep -E "^use-sqlite-wal = true"
-
-  install-with-non-default-source-inputs:
-    name: Install Nix using non-default source-${{ matrix.inputs.key }}
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        inputs:
-          # https://github.com/DeterminateSystems/nix-installer/blob/v0.18.0
-          - key: url
-            value: https://github.com/DeterminateSystems/nix-installer/releases/download/v0.18.0/nix-installer-x86_64-linux
-            nix-version: "2.21.2"
-          # https://github.com/DeterminateSystems/nix-installer/tree/7011c077ec491da410fbc39f68676b0908b9ce7e
-          - key: revision
-            value: 7011c077ec491da410fbc39f68676b0908b9ce7e
-            nix-version: "2.19.2"
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install with alternative source-${{ matrix.inputs.key }}
-        uses: ./
-        with:
-          source-${{ matrix.inputs.key }}: ${{ matrix.inputs.value }}
-          _internal-strict-mode: true
-      - name: Ensure that the expected Nix version ${{ matrix.inputs.nix-version }} is installed via alternative source-${{ matrix.inputs.key }}
-        run: .github/verify-version.sh ${{ matrix.inputs.nix-version }}
-
-  install-no-id-token:
-    name: Install Nix without an ID token
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./
-        with:
-          _internal-strict-mode: true
-          determinate: true

dist/index.js

@@ -88894,6 +88894,7 @@ function makeOptionsConfident(actionOptions) {
 
 
 
+
 var EVENT_INSTALL_NIX_FAILURE = "install_nix_failure";
 var EVENT_INSTALL_NIX_START = "install_nix_start";
 var EVENT_INSTALL_NIX_SUCCESS = "install_nix_start";
@@ -88910,6 +88911,8 @@ var FACT_IN_ACT = "in_act";
 var FACT_IN_NAMESPACE_SO = "in_namespace_so";
 var FACT_NIX_INSTALLER_PLANNER = "nix_installer_planner";
 var FLAG_DETERMINATE = "--determinate";
+var STATE_EVENT_LOG = "DETERMINATE_NIXD_EVENT_LOG";
+var STATE_EVENT_PID = "DETERMINATE_NIXD_EVENT_PID";
 var NixInstallerAction = class extends DetSysAction {
   constructor() {
     super({
@@ -88956,10 +88959,13 @@ var NixInstallerAction = class extends DetSysAction {
     await this.scienceDebugFly();
     await this.detectAndForceDockerShim();
     await this.install();
+    await this.spewEventLog();
   }
   async post() {
     await this.cleanupDockerShim();
     await this.reportOverall();
+    await this.slurpEventLog();
+    await this.cleanupLogger();
   }
   get isMacOS() {
     return this.runnerOs === "macOS";
@@ -89573,18 +89579,18 @@ ${stderrBuffer}`
     try {
       await (0,promises_namespaceObject.stat)(socketPath);
       return true;
-    } catch (error) {
-      if (error.code === "ENOENT") {
+    } catch (error2) {
+      if (error2.code === "ENOENT") {
         core.debug(`Socket '${socketPath}' does not exist yet`);
         return false;
       }
       core.warning(
-        `Error waiting for the Nix Daemon socket: ${stringifyError(error)}`
+        `Error waiting for the Nix Daemon socket: ${stringifyError(error2)}`
       );
       this.recordEvent("docker-shim:wait-for-socket", {
-        exception: stringifyError(error)
+        exception: stringifyError(error2)
       });
-      throw error;
+      throw error2;
     }
   }
   async cleanupDockerShim() {
@@ -89790,7 +89796,142 @@ ${stderrBuffer}`
       );
     }
   }
+  async spewEventLog() {
+    if (!this.determinate) {
+      return;
+    }
+    const logfile = this.getTemporaryName();
+    core.saveState(STATE_EVENT_LOG, logfile);
+    const stdout = await (0,promises_namespaceObject.open)(logfile, "a");
+    const stderr = await (0,promises_namespaceObject.open)(`${logfile}.stderr`, "a");
+    core.debug(`Event log: ${logfile}`);
+    const opts = {
+      stdio: ["ignore", stdout.fd, stderr.fd],
+      detached: true
+    };
+    const daemon = (0,external_node_child_process_namespaceObject.spawn)(
+      "curl",
+      [
+        "--no-buffer",
+        "--unix-socket",
+        "/nix/var/determinate/determinate-nixd.socket",
+        "http://localhost/events"
+      ],
+      opts
+    );
+    core.saveState(STATE_EVENT_PID, daemon.pid);
+    daemon.unref();
+    await Promise.resolve();
+    try {
+      await stdout.close();
+    } catch (error2) {
+      core.info(`Could not close curl's stdout: ${error2}`);
+    }
+    try {
+      await stderr.close();
+    } catch (error2) {
+      core.info(`Could not close curl's stderr: ${error2}`);
+    }
+  }
+  async slurpEventLog() {
+    if (!this.determinate) {
+      return;
+    }
+    try {
+      const logPath = core.getState(STATE_EVENT_LOG);
+      const events = await readMismatchEvents(logPath);
+      if (events.length === 0) {
+        core.debug("No hash mismatches found.");
+        return;
+      }
+      const listing = await getFileListing();
+      for (const file of listing) {
+        const text = await (0,promises_namespaceObject.readFile)(file, "utf-8");
+        const lines = text.split("\n");
+        for (const [index, line] of lines.entries()) {
+          const lineNumber = index + 1;
+          for (const event of events) {
+            const match = line.match(event.search);
+            if (!match) {
+              continue;
+            }
+            const column = (match.index ?? 0) + 1;
+            core.error(`This derivation's hash is \`${event.good}\``, {
+              title: "Determinate Nix detected an incorrect dependency hash.",
+              file,
+              startLine: lineNumber,
+              startColumn: column
+            });
+          }
+        }
+      }
+    } catch (error2) {
+      core.warning(`Could not consume hash mismatch logs: ${error2}`);
+    }
+  }
+  async cleanupLogger() {
+    const rawPid = core.getState(STATE_EVENT_PID);
+    const pid = Number(rawPid);
+    try {
+      process.kill(pid);
+    } catch (error2) {
+      core.info(`Could not kill pid ${rawPid}: ${error2}`);
+    }
+  }
 };
+async function readMismatchEvents(logPath) {
+  const prefix = "data: ";
+  const memo = /* @__PURE__ */ new Set();
+  const events = (await (0,promises_namespaceObject.readFile)(logPath, "utf-8")).split(/\n/).filter((line) => line.startsWith(prefix)).map((line) => {
+    const json = line.slice(prefix.length);
+    const source = JSON.parse(json);
+    const search = new RegExp(
+      source.bad.map((s) => s.replace(/[+]/g, (ch) => `\\${ch}`)).join("|")
+    );
+    return {
+      ...source,
+      search
+    };
+  }).filter((event) => {
+    const key = [event.drv, ...event.bad].join("\0");
+    if (memo.has(key)) {
+      false;
+    }
+    memo.add(key);
+    return true;
+  });
+  return events;
+}
+async function getFileListing() {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    let length = 0;
+    const child = (0,external_node_child_process_namespaceObject.spawn)(
+      "git",
+      ["ls-files", "-z", "*.nix", "*.json", "*.toml"],
+      {
+        stdio: ["ignore", "pipe", "inherit"]
+      }
+    );
+    child.stdout.on("data", (chunk) => {
+      chunks.push(chunk);
+      length += chunk.length;
+    });
+    child.stdout.on("end", () => {
+      const lines = Buffer.concat(chunks, length).toString("utf-8").replace(/\0$/, "").split(/\0/);
+      resolve(lines);
+    });
+    child.stdout.on("error", reject);
+    child.on("error", reject);
+    child.on("exit", (code, signal) => {
+      if (code !== 0) {
+        core.warning(
+          `git ls-files exited suspiciously code=${code}; signal=${signal}`
+        );
+      }
+    });
+  });
+}
 function main() {
   new NixInstallerAction().execute();
 }

src/index.ts

@@ -1,6 +1,6 @@
 import * as actionsCore from "@actions/core";
 import * as actionsExec from "@actions/exec";
-import { access, readFile, stat } from "node:fs/promises";
+import { access, open, readFile, stat } from "node:fs/promises";
 import { join } from "node:path";
 import fs from "node:fs";
 import { userInfo } from "node:os";
@@ -10,6 +10,7 @@ import { DetSysAction, inputs, platform, stringifyError } from "detsys-ts";
 import { randomUUID } from "node:crypto";
 import got from "got";
 import { setTimeout } from "node:timers/promises";
+import { SpawnOptions, spawn } from "node:child_process";
 
 // Nix installation events
 const EVENT_INSTALL_NIX_FAILURE = "install_nix_failure";
@@ -39,6 +40,10 @@ const FACT_NIX_INSTALLER_PLANNER = "nix_installer_planner";
 // Flags
 const FLAG_DETERMINATE = "--determinate";
 
+// Pre/post state keys
+const STATE_EVENT_LOG = "DETERMINATE_NIXD_EVENT_LOG";
+const STATE_EVENT_PID = "DETERMINATE_NIXD_EVENT_PID";
+
 class NixInstallerAction extends DetSysAction {
   determinate: boolean;
   platform: string;
@@ -121,11 +126,14 @@ class NixInstallerAction extends DetSysAction {
     await this.scienceDebugFly();
     await this.detectAndForceDockerShim();
     await this.install();
+    await this.spewEventLog();
   }
 
   async post(): Promise<void> {
     await this.cleanupDockerShim();
     await this.reportOverall();
+    await this.slurpEventLog();
+    await this.cleanupLogger();
   }
 
   private get isMacOS(): boolean {
@@ -1104,6 +1112,207 @@ class NixInstallerAction extends DetSysAction {
       );
     }
   }
+
+  private async spewEventLog(): Promise<void> {
+    if (!this.determinate) {
+      return;
+    }
+
+    const logfile = this.getTemporaryName();
+    actionsCore.saveState(STATE_EVENT_LOG, logfile);
+    const stdout = await open(logfile, "a");
+    const stderr = await open(`${logfile}.stderr`, "a");
+
+    actionsCore.debug(`Event log: ${logfile}`);
+
+    const opts: SpawnOptions = {
+      stdio: ["ignore", stdout.fd, stderr.fd],
+      detached: true,
+    };
+
+    // Start the server. Once it is ready, it will notify us via the notification server.
+    const daemon = spawn(
+      "curl",
+      [
+        "--no-buffer",
+        "--unix-socket",
+        "/nix/var/determinate/determinate-nixd.socket",
+        "http://localhost/events",
+      ],
+      opts,
+    );
+
+    actionsCore.saveState(STATE_EVENT_PID, daemon.pid);
+    daemon.unref();
+
+    // Wait a tick in the event loop in order for curl to actually be running
+    await Promise.resolve();
+
+    try {
+      await stdout.close();
+    } catch (error) {
+      actionsCore.info(`Could not close curl's stdout: ${error}`);
+    }
+
+    try {
+      await stderr.close();
+    } catch (error) {
+      actionsCore.info(`Could not close curl's stderr: ${error}`);
+    }
+  }
+
+  private async slurpEventLog(): Promise<void> {
+    if (!this.determinate) {
+      return;
+    }
+
+    try {
+      const logPath = actionsCore.getState(STATE_EVENT_LOG);
+      const events = await readMismatchEvents(logPath);
+
+      // No point doing any more work if there are no mismatch events
+      if (events.length === 0) {
+        actionsCore.debug("No hash mismatches found.");
+        return;
+      }
+
+      const listing = await getFileListing();
+
+      // For each file, search for potentially bad hashes
+      for (const file of listing) {
+        const text = await readFile(file, "utf-8");
+        const lines = text.split("\n");
+
+        for (const [index, line] of lines.entries()) {
+          const lineNumber = index + 1;
+
+          for (const event of events) {
+            const match = line.match(event.search);
+            if (!match) {
+              continue;
+            }
+
+            // Allegedly, match.index is optional, so default to 0
+            const column = (match.index ?? 0) + 1;
+
+            actionsCore.error(`This derivation's hash is \`${event.good}\``, {
+              title: "Determinate Nix detected an incorrect dependency hash.",
+              file,
+              startLine: lineNumber,
+              startColumn: column,
+            });
+          }
+        }
+      }
+    } catch (error) {
+      // Don't hard fail the action if something exploded; this feature is only a nice-to-have
+      actionsCore.warning(`Could not consume hash mismatch logs: ${error}`);
+    }
+  }
+
+  async cleanupLogger(): Promise<void> {
+    const rawPid = actionsCore.getState(STATE_EVENT_PID);
+    const pid = Number(rawPid);
+
+    try {
+      process.kill(pid);
+    } catch (error) {
+      actionsCore.info(`Could not kill pid ${rawPid}: ${error}`);
+    }
+  }
+}
+
+// Fields we're interested in from the source event
+interface MismatchSourceEvent {
+  readonly drv: string;
+  readonly good: string;
+  readonly bad: readonly string[];
+}
+
+// Our augmented event with the RegExp to match against the bad hashes
+interface MismatchEvent extends MismatchSourceEvent {
+  readonly search: RegExp;
+}
+
+async function readMismatchEvents(logPath: string): Promise<MismatchEvent[]> {
+  const prefix = "data: ";
+
+  // Used to deduplicate events (see below)
+  const memo = new Set<string>();
+
+  const events = (await readFile(logPath, "utf-8"))
+    .split(/\n/)
+    .filter((line) => line.startsWith(prefix))
+    .map((line) => {
+      // Note: this currently assumes that all events being ingested are mismatches
+      const json = line.slice(prefix.length);
+      const source = JSON.parse(json) as MismatchSourceEvent;
+
+      // Construct a regular expression to search for any of the hash patterns
+      // (do it here to avoid creating RegExp objects in a loop below)
+      const search = new RegExp(
+        source.bad.map((s) => s.replace(/[+]/g, (ch) => `\\${ch}`)).join("|"),
+      );
+
+      return {
+        ...source,
+        search,
+      } satisfies MismatchEvent;
+    })
+    .filter((event) => {
+      // Deduplicate based on the derivation's store path and list of bad hashes.
+      const key = [event.drv, ...event.bad].join("\0");
+      if (memo.has(key)) {
+        false;
+      }
+
+      memo.add(key);
+      return true;
+    });
+
+  return events;
+}
+
+// Get the list of files with potential hash mismatches (limited currently to *.{nix,json,toml})
+async function getFileListing(): Promise<readonly string[]> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    let length = 0;
+
+    const child = spawn(
+      "git",
+      ["ls-files", "-z", "*.nix", "*.json", "*.toml"],
+      {
+        stdio: ["ignore", "pipe", "inherit"],
+      },
+    );
+
+    child.stdout.on("data", (chunk: Buffer) => {
+      chunks.push(chunk);
+      length += chunk.length;
+    });
+
+    child.stdout.on("end", () => {
+      const lines = Buffer.concat(chunks, length)
+        .toString("utf-8")
+        .replace(/\0$/, "")
+        .split(/\0/);
+
+      resolve(lines);
+    });
+
+    child.stdout.on("error", reject);
+
+    child.on("error", reject);
+    child.on("exit", (code, signal) => {
+      // We should consider rejecting the promise here
+      if (code !== 0) {
+        actionsCore.warning(
+          `git ls-files exited suspiciously code=${code}; signal=${signal}`,
+        );
+      }
+    });
+  });
 }
 
 type ExecuteEnvironment = {