feat: add bounded parallel runner for the normal test suite

deno.json

@@ -1,4 +1,8 @@
 {
+  "tasks": {
+    "test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts",
+    "test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts"
+  },
   "fmt": {
     "proseWrap": "never"
   }

tools/test.test.ts

@@ -0,0 +1,701 @@
+import {
+  defaultJobs,
+  discoverRustCommands,
+  execute,
+  installCancellationHandlers,
+  mapConcurrent,
+  parseJobs,
+  parseRustListing,
+  resultSummary,
+  rustTestCommands,
+  shardRustCommand,
+} from "./test.ts";
+import { spawn } from "node:child_process";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+
+function assert(
+  condition: unknown,
+  message = "assertion failed",
+): asserts condition {
+  if (!condition) throw new Error(message);
+}
+
+Deno.test("Cargo artifact selection retains every test target and deduplicates executable reports", () => {
+  const test = (path: string, target: string) => ({
+    reason: "compiler-artifact",
+    package_id: "pkg",
+    executable: path,
+    target: { name: target },
+    profile: { test: true },
+  });
+  const commands = rustTestCommands([
+    test("/repo/target/lib-test", "lib"),
+    test("/repo/target/integration", "integration"),
+    test("/repo/target/integration", "integration"),
+    { ...test("/repo/target/app", "app"), profile: { test: false } },
+    { reason: "build-finished" },
+  ], [{
+    id: "pkg",
+    name: "example",
+    manifest_path: "/repo/crates/example/Cargo.toml",
+  }]);
+  assert(commands.length === 2);
+  assert(commands[0].cwd === "/repo/crates/example");
+  assert(commands[0].env?.CARGO_MANIFEST_DIR === commands[0].cwd);
+  assert(commands[1].name === "example/integration");
+  assert(
+    commands.every((command) => command.args.length === 0),
+    "no filtering or ignore flags",
+  );
+});
+
+Deno.test("missing Cargo package or executables fails instead of claiming an empty suite passed", () => {
+  for (
+    const artifacts of [[], [{
+      reason: "compiler-artifact",
+      package_id: "missing",
+      executable: "/test",
+      profile: { test: true },
+      target: { name: "test" },
+    }]]
+  ) {
+    let failed = false;
+    try {
+      rustTestCommands(artifacts, []);
+    } catch {
+      failed = true;
+    }
+    assert(failed);
+  }
+});
+
+Deno.test("bounded scheduling runs every command once and retains failing results", async () => {
+  let active = 0, maximum = 0;
+  const seen: number[] = [];
+  const results = await mapConcurrent([0, 1, 2, 3, 4], 2, async (item) => {
+    active++;
+    maximum = Math.max(active, maximum);
+    await Promise.resolve();
+    seen.push(item);
+    active--;
+    return item === 2 ? 1 : 0;
+  });
+  assert(maximum === 2);
+  assert(seen.sort().join() === "0,1,2,3,4");
+  assert(results.join() === "0,0,1,0,0");
+  for (const jobs of [0, -1, 1.5]) {
+    let failed = false;
+    try {
+      await mapConcurrent([], jobs, () => Promise.resolve(0));
+    } catch {
+      failed = true;
+    }
+    assert(failed);
+  }
+});
+
+function assertThrows(fn: () => unknown) {
+  let failed = false;
+  try {
+    fn();
+  } catch {
+    failed = true;
+  }
+  assert(failed, "expected failure");
+}
+
+Deno.test("CPU-derived jobs accommodates concurrent suites within the CLI bound", () => {
+  assert(defaultJobs(24) === 32);
+  assert(defaultJobs(128) === 32);
+  assert(defaultJobs(6) === 8);
+  assert(defaultJobs(1) === 2);
+  assert(parseJobs([]) === defaultJobs());
+  assert(parseJobs(["--jobs", "64"]) === 64);
+  assert(parseJobs(["--jobs", "1"]) === 1);
+  for (
+    const args of [
+      ["--jobs", "65"],
+      ["--jobs", "0"],
+      ["--jobs", "1.5"],
+      ["--jobs", "NaN"],
+      ["--jobs"],
+      ["--other", "4"],
+    ]
+  ) {
+    assertThrows(() => parseJobs(args));
+  }
+});
+
+Deno.test("Rust terse listing accepts every name and fails closed on unknown or duplicate lines", () => {
+  assert(parseRustListing("").length === 0);
+  assert(
+    parseRustListing("unit::a: test\r\nunit::bench: benchmark\r\n").join() ===
+      "unit::a,unit::bench",
+  );
+  for (
+    const text of [
+      "unit::a\n",
+      "unit::a: ignored\n",
+      "unit::a: test\n1 test, 0 benchmarks\n",
+      "unit::a: test\nunit::a: test\n",
+      "\n",
+      "unit::a: test\n\n",
+      "\x1b[31munit::a: test\n",
+    ]
+  ) {
+    assertThrows(() => parseRustListing(text));
+  }
+});
+
+function listedNames(count: number) {
+  return Array.from({ length: count }, (_, i) => `unit::case_${i}`);
+}
+
+Deno.test("round-robin shards retain every listed name exactly once at thresholds and the 32-shard cap", () => {
+  const task = {
+    name: "pkg/binary",
+    command: "/test-binary",
+    args: [],
+    cwd: "/pkg",
+    env: { CARGO_MANIFEST_DIR: "/pkg" },
+  };
+  for (const count of [0, 1, 72, 73, 108, 1135, 1152, 1153, 2000]) {
+    const names = parseRustListing(
+      listedNames(count).map((name) => `${name}: test\n`).join(""),
+    );
+    const shards = shardRustCommand(task, names);
+    const assigned = shards.flatMap((shard) => shard.selectedNames);
+    assert(assigned.length === count && new Set(assigned).size === count);
+    assert(
+      [...assigned].sort().join("\n") === [...names].sort().join("\n"),
+      "no names may be excluded",
+    );
+    if (count <= 72) {
+      assert(
+        shards.length === 1 && shards[0].task === task,
+        "small binaries must be unchanged",
+      );
+    } else {
+      const expected = Math.min(32, Math.ceil(count / 36));
+      assert(shards.length === expected);
+      const lengths = shards.map((shard) => shard.selectedNames.length);
+      assert(Math.max(...lengths) - Math.min(...lengths) <= 1);
+      shards.forEach((shard, index) => {
+        assert(
+          shard.selectedNames.join() ===
+            names.filter((_, i) => i % expected === index).join(),
+          "must distribute round-robin",
+        );
+        assert(
+          shard.task.args.slice(0, 3).join() === "--exact,--test-threads,2",
+        );
+        assert(shard.task.args.slice(3).join() === shard.selectedNames.join());
+        assert(
+          !shard.task.args.includes("--ignored") &&
+            !shard.task.args.includes("--include-ignored"),
+        );
+        assert(shard.task.cwd === task.cwd && shard.task.env === task.env);
+      });
+    }
+  }
+  assertThrows(() => shardRustCommand(task, ["same", "same"]));
+});
+
+Deno.test("scheduler drains active workers before propagating a callback error", async () => {
+  let drained = false;
+  let failed = false;
+  try {
+    await mapConcurrent([0, 1], 2, async (item) => {
+      if (!item) throw new Error("callback failed");
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      drained = true;
+    });
+  } catch {
+    failed = true;
+  }
+  assert(failed && drained);
+});
+
+const fixturePath = fileURLToPath(import.meta.url);
+function fixtureArgs(mode: string, output: string) {
+  return [
+    "run",
+    "--allow-read",
+    "--allow-write",
+    "--allow-run",
+    "--allow-env",
+    fixturePath,
+    mode,
+    output,
+  ];
+}
+function fixtureTask(mode: string, output: string) {
+  return {
+    name: `fixture/${mode}`,
+    command: Deno.execPath(),
+    args: fixtureArgs(mode, output),
+    cwd: output,
+  };
+}
+async function waitUntil(
+  check: () => boolean | Promise<boolean>,
+  message: string,
+) {
+  const end = performance.now() + 10_000;
+  while (performance.now() < end) {
+    if (await check()) return;
+    await new Promise((resolve) => setTimeout(resolve, 20));
+  }
+  throw new Error(message);
+}
+function absent(pid: number) {
+  try {
+    process.kill(pid, 0);
+    return false;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ESRCH") return true;
+    throw error;
+  }
+}
+function emergencyKill(pid: number) {
+  try {
+    process.kill(-pid, "SIGKILL");
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
+  }
+}
+
+for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) {
+  Deno.test({
+    name: `execute streams real ${mode} child output and records its exit`,
+    ignore: Deno.build.os !== "linux",
+    async fn() {
+      const output = await Deno.makeTempDir();
+      try {
+        const result = await execute({
+          ...fixtureTask(mode, output),
+          inventoryRef: { binaryIndex: 3, shardIndex: 1 },
+        }, {
+          output,
+          label: mode,
+          timeoutMs: 10_000,
+        });
+        assert(result.code === code, JSON.stringify(result));
+        assert(!result.timedOut);
+        assert(!result.error, result.error);
+        assert(result.stdout.includes("fixture stdout 日本語"));
+        assert(result.stderr.includes("fixture stderr"));
+        assert(
+          await Deno.readTextFile(`${output}/${mode}.stdout`) === result.stdout,
+        );
+        assert(
+          await Deno.readTextFile(`${output}/${mode}.stderr`) === result.stderr,
+        );
+        const summary = JSON.parse(
+          await Deno.readTextFile(`${output}/${mode}.result.json`),
+        );
+        assert(summary.code === code && summary.timedOut === false);
+        assert(summary.log === mode && summary.timeoutMs === 10_000);
+        assert(
+          summary.inventoryRef.binaryIndex === 3 &&
+            summary.inventoryRef.shardIndex === 1,
+          "results must link unambiguously to inventory entries",
+        );
+        assert(
+          !("stdout" in summary),
+          "summary references logs, not full output",
+        );
+        assert(
+          absent(Number(await Deno.readTextFile(`${output}/parent.pid`))),
+          "direct child must be reaped",
+        );
+      } finally {
+        await Deno.remove(output, { recursive: true });
+      }
+    },
+  });
+}
+
+Deno.test({
+  name:
+    "execute preserves live stdout/stderr and times out a group containing a grandchild",
+  ignore: Deno.build.os !== "linux",
+  async fn() {
+    const output = await Deno.makeTempDir();
+    let parent: number | undefined;
+    let finished = false;
+    const running = execute(fixtureTask("tree", output), {
+      output,
+      label: "tree",
+      timeoutMs: 4_000,
+      killGraceMs: 500,
+    }).finally(() => finished = true);
+    try {
+      await waitUntil(async () => {
+        const text = await Deno.readTextFile(`${output}/tree.stdout`);
+        return text.includes("grandchild ready");
+      }, "grandchild did not become ready");
+      parent = Number(await Deno.readTextFile(`${output}/parent.pid`));
+      const grandchild = Number(
+        await Deno.readTextFile(`${output}/grandchild.pid`),
+      );
+      assert(!finished, "logs must be visible before execute resolves");
+      assert(
+        (await Deno.readTextFile(`${output}/tree.stderr`)).includes(
+          "fixture stderr",
+        ),
+      );
+      assert(
+        !absent(parent) && !absent(grandchild),
+        "both processes must actually be running",
+      );
+      const result = await running;
+      assert(result.timedOut && result.code === 124, JSON.stringify(result));
+      assert(!result.error, result.error);
+      assert(result.seconds < 10, "timeout cleanup must be bounded");
+      assert(result.stderr.includes("timeout after 4000ms"));
+      assert(
+        result.stderr.includes("grandchild reaped"),
+        "fixture parent must reap its child",
+      );
+      assert(absent(parent), "direct child must be reaped by runner");
+      assert(absent(grandchild), "grandchild must be stopped and reaped");
+      const summary = JSON.parse(
+        await Deno.readTextFile(`${output}/tree.result.json`),
+      );
+      assert(summary.timedOut && summary.code === 124);
+      assert(summary.name === "fixture/tree" && summary.log === "tree");
+    } finally {
+      if (parent !== undefined) emergencyKill(parent);
+      await running;
+      await Deno.remove(output, { recursive: true });
+    }
+  },
+});
+
+Deno.test({
+  name:
+    "execute escalates SIGTERM to SIGKILL and reaps a stubborn direct child",
+  ignore: Deno.build.os !== "linux",
+  async fn() {
+    const output = await Deno.makeTempDir();
+    try {
+      const result = await execute(fixtureTask("stubborn", output), {
+        output,
+        label: "stubborn",
+        timeoutMs: 2_000,
+        killGraceMs: 100,
+      });
+      assert(result.stdout.includes("stubborn ready"));
+      assert(result.stderr.includes("ignored SIGTERM"));
+      assert(
+        result.timedOut && result.code === 124 && result.signal === "SIGKILL",
+        JSON.stringify(result),
+      );
+      assert(!result.error, result.error);
+      assert(result.seconds < 8);
+      assert(absent(Number(await Deno.readTextFile(`${output}/parent.pid`))));
+    } finally {
+      await Deno.remove(output, { recursive: true });
+    }
+  },
+});
+
+Deno.test({
+  name: "execute records spawn failure instead of an empty passing suite",
+  ignore: Deno.build.os !== "linux",
+  async fn() {
+    const output = await Deno.makeTempDir();
+    try {
+      const result = await execute({
+        name: "missing",
+        command: `${output}/does-not-exist`,
+        args: [],
+        cwd: output,
+      }, { output, label: "missing", timeoutMs: 2_000, killGraceMs: 10 });
+      assert(result.code !== 0 && !result.timedOut);
+      assert(result.error?.includes("ENOENT"), JSON.stringify(result));
+      assert(
+        (await Deno.readTextFile(`${output}/missing.stderr`)).includes(
+          "ENOENT",
+        ),
+      );
+      const summary = JSON.parse(
+        await Deno.readTextFile(`${output}/missing.result.json`),
+      );
+      assert(summary.code !== 0 && summary.error.includes("ENOENT"));
+    } finally {
+      await Deno.remove(output, { recursive: true });
+    }
+  },
+});
+
+Deno.test({
+  name:
+    "runtime listing persists complete name inventory for large and small binaries",
+  ignore: Deno.build.os !== "linux",
+  async fn() {
+    const output = await Deno.makeTempDir();
+    try {
+      const discovery = await discoverRustCommands(
+        [
+          fixtureTask("list-large", output),
+          fixtureTask("list-small", output),
+        ],
+        2,
+        { output },
+      );
+      assert(discovery.inventory.complete);
+      assert(
+        discovery.inventory.rustBinaries === 2 &&
+          discovery.inventory.selectedCases === 1137,
+      );
+      assert(discovery.tasks.length === 33);
+      const persisted = JSON.parse(
+        await Deno.readTextFile(`${output}/inventory.json`),
+      ) as typeof discovery.inventory;
+      assert(persisted.complete && persisted.selectedCases === 1137);
+      const large = persisted.binaries[0];
+      assert(large.shards.length === 32 && large.selectedCases === 1135);
+      persisted.binaries.forEach((binary, binaryIndex) =>
+        binary.shards.forEach((shard, shardIndex) => {
+          assert(shard.task.inventoryRef?.binaryIndex === binaryIndex);
+          assert(shard.task.inventoryRef?.shardIndex === shardIndex);
+        })
+      );
+      const assigned = large.shards.flatMap((shard) => shard.selectedNames);
+      assert(new Set(assigned).size === 1135 && assigned.length === 1135);
+      assert([...assigned].sort().join() === listedNames(1135).sort().join());
+      assert(
+        persisted.binaries[1].shards[0].task.args.join() ===
+          fixtureTask("list-small", output).args.join(),
+      );
+      assert(
+        persisted.binaries.every((binary) =>
+          binary.listing.code === 0 && binary.listing.seconds > 0
+        ),
+      );
+    } finally {
+      await Deno.remove(output, { recursive: true });
+    }
+  },
+});
+
+for (const mode of ["list-nonzero", "list-unknown"]) {
+  Deno.test({
+    name:
+      `discovery ${mode} fails with saved evidence rather than running a partial selection`,
+    ignore: Deno.build.os !== "linux",
+    async fn() {
+      const output = await Deno.makeTempDir();
+      try {
+        const discovery = await discoverRustCommands(
+          [
+            fixtureTask("list-small", output),
+            fixtureTask(mode, output),
+          ],
+          2,
+          { output },
+        );
+        assert(!discovery.inventory.complete);
+        assert(discovery.inventory.selectedCases === null);
+        assert(discovery.tasks.length === 0);
+        const persisted = JSON.parse(
+          await Deno.readTextFile(`${output}/inventory.json`),
+        ) as typeof discovery.inventory;
+        assert(
+          !persisted.complete && persisted.binaries[1].shards.length === 0,
+        );
+        assert(
+          persisted.binaries[1].error?.includes(
+            mode === "list-nonzero" ? "code 7" : "unknown Rust listing format",
+          ),
+        );
+        const listing = JSON.parse(
+          await Deno.readTextFile(`${output}/list-001.result.json`),
+        );
+        assert(listing.code === (mode === "list-nonzero" ? 7 : 0));
+        assert(
+          persisted.discoveredCases === 2,
+          "partial discovery must not be presented as complete selection",
+        );
+      } finally {
+        await Deno.remove(output, { recursive: true });
+      }
+    },
+  });
+}
+
+async function textIfPresent(path: string): Promise<string> {
+  try {
+    return await Deno.readTextFile(path);
+  } catch (error) {
+    if (error instanceof Deno.errors.NotFound) return "";
+    throw error;
+  }
+}
+
+for (const signal of ["SIGINT", "SIGTERM"] as const) {
+  Deno.test({
+    name:
+      `parent ${signal} cleans up active groups, reaps children and does not spawn queued commands`,
+    ignore: Deno.build.os !== "linux",
+    async fn() {
+      const output = await Deno.makeTempDir();
+      const parent = new Deno.Command(Deno.execPath(), {
+        args: fixtureArgs("runner-parent", output),
+        stdout: "piped",
+        stderr: "piped",
+      }).spawn();
+      const done = parent.output();
+      const pids: number[] = [];
+      try {
+        await waitUntil(
+          async () =>
+            (await textIfPresent(`${output}/tree/tree.stdout`)).includes(
+              "grandchild ready",
+            ) &&
+            (await textIfPresent(`${output}/stubborn/stubborn.stdout`))
+              .includes("stubborn ready"),
+          "parent runner fixtures did not become ready",
+        );
+        for (
+          const file of [
+            "tree/parent.pid",
+            "tree/grandchild.pid",
+            "stubborn/parent.pid",
+          ]
+        ) {
+          pids.push(Number(await Deno.readTextFile(`${output}/${file}`)));
+        }
+        process.kill(parent.pid, signal);
+        const status = await done;
+        assert(
+          status.code === (signal === "SIGINT" ? 130 : 143),
+          new TextDecoder().decode(status.stderr),
+        );
+        assert(absent(parent.pid));
+        assert(
+          pids.every(absent),
+          "active children and grandchild must be reaped",
+        );
+        const results = JSON.parse(
+          await Deno.readTextFile(`${output}/cancel-summary.json`),
+        ) as ReturnType<typeof resultSummary>[];
+        assert(results.length === 3);
+        assert(
+          results.every((result) =>
+            result.cancelled && result.cancellationReason === signal &&
+            !result.timedOut && !result.error
+          ),
+        );
+        assert(
+          results[1].signal === "SIGKILL",
+          "stubborn child requires escalation on parent cancellation too",
+        );
+        assert(
+          await textIfPresent(`${output}/queued/parent.pid`) === "",
+          "cancelled queue must not spawn",
+        );
+        assert(
+          (await Deno.readTextFile(`${output}/tree/tree.stderr`)).includes(
+            "grandchild reaped",
+          ),
+        );
+      } finally {
+        if (!absent(parent.pid)) process.kill(parent.pid, "SIGTERM");
+        await done;
+        for (const pid of pids) emergencyKill(pid);
+        await Deno.remove(output, { recursive: true });
+      }
+    },
+  });
+}
+
+// Real subprocess fixtures live here rather than introducing another executable.
+// A watchdog is independent of execute, so a broken runner cannot leak fixtures
+// forever. The tree's parent cooperatively reaps its child on TERM; KILL behavior
+// is exercised separately by the stubborn fixture.
+if (import.meta.main) {
+  const [mode, output] = Deno.args;
+  setTimeout(() => Deno.exit(99), 20_000);
+  if (mode.startsWith("list-")) {
+    assert(
+      Deno.args.slice(2).join() === "--list,--format,terse",
+      "discovery must use libtest terse listing",
+    );
+    if (mode === "list-unknown") {
+      console.log("unexpected harness output");
+    } else {
+      console.log(
+        listedNames(mode === "list-large" ? 1135 : 2).map((name) =>
+          `${name}: test`
+        ).join("\n"),
+      );
+    }
+    Deno.exit(mode === "list-nonzero" ? 7 : 0);
+  } else if (mode === "runner-parent") {
+    const cancellation = installCancellationHandlers();
+    try {
+      for (const dir of ["tree", "stubborn", "queued"]) {
+        await Deno.mkdir(`${output}/${dir}`);
+      }
+      const results = await mapConcurrent(
+        ["tree", "stubborn", "queued"],
+        2,
+        async (name) => {
+          const dir = `${output}/${name}`;
+          return resultSummary(
+            await execute(
+              fixtureTask(name === "queued" ? "success" : name, dir),
+              {
+                output: dir,
+                label: name,
+                timeoutMs: 15_000,
+                killGraceMs: 200,
+                signal: cancellation.signal,
+              },
+            ),
+          );
+        },
+      );
+      await Deno.writeTextFile(
+        `${output}/cancel-summary.json`,
+        JSON.stringify(results),
+      );
+      Deno.exitCode = results[0].code;
+    } finally {
+      cancellation.dispose();
+    }
+    Deno.exit(Deno.exitCode);
+  } else if (mode === "grandchild") {
+    Deno.addSignalListener("SIGTERM", () => Deno.exit(0));
+    Deno.writeTextFileSync(`${output}/grandchild.pid`, String(Deno.pid));
+    console.log("grandchild ready");
+  } else {
+    Deno.writeTextFileSync(`${output}/parent.pid`, String(Deno.pid));
+    console.log("fixture stdout 日本語");
+    console.error("fixture stderr");
+    if (mode === "success" || mode === "nonzero") {
+      Deno.exit(mode === "success" ? 0 : 7);
+    } else if (mode === "stubborn") {
+      Deno.addSignalListener("SIGTERM", () => console.error("ignored SIGTERM"));
+      console.log("stubborn ready");
+    } else if (mode === "tree") {
+      // The grandchild inherits the new group from execute's direct child.
+      // The parent intentionally does NOT forward TERM: only group signaling
+      // can stop the grandchild before its independent watchdog.
+      Deno.addSignalListener("SIGTERM", () => {});
+      const child = spawn(Deno.execPath(), fixtureArgs("grandchild", output), {
+        stdio: "inherit",
+      });
+      child.on("close", () => {
+        console.error("grandchild reaped");
+        Deno.exit(0);
+      });
+    } else {
+      throw new Error(`unknown fixture mode: ${mode}`);
+    }
+  }
+}

tools/test.ts

@@ -0,0 +1,719 @@
+// Bound top-level suites/shards, not total processes; large shards use two test threads.
+// Cargo selects targets/features. Retain all listed names; never skip cached successes.
+import { spawn } from "node:child_process";
+import process from "node:process";
+import { availableParallelism } from "node:os";
+import { fileURLToPath } from "node:url";
+
+// Safety limits, NOT the separate 60s performance goal.
+export const SUITE_TIMEOUT_MS = 180_000;
+export const BUILD_TIMEOUT_MS = 1_800_000;
+export const MAX_JOBS = 64;
+
+export function defaultJobs(cpus = availableParallelism()): number {
+  // Leave headroom for internal runners; higher top-level counts can be chosen
+  // explicitly, but are not assumed to improve throughput or timing stability.
+  return Math.min(32, Math.max(1, Math.ceil(cpus * 4 / 3)));
+}
+
+export function parseJobs(args: string[]): number {
+  if (!args.length) return defaultJobs();
+  if (args.length !== 2 || args[0] !== "--jobs") {
+    throw new Error("usage: deno task test [--jobs N]");
+  }
+  const jobs = Number(args[1]);
+  if (!Number.isInteger(jobs) || jobs < 1 || jobs > MAX_JOBS) {
+    throw new Error(`jobs must be 1..${MAX_JOBS}`);
+  }
+  return jobs;
+}
+
+/** libtest's terse --list has one `name: test`/`name: benchmark` per line.
+ * An empty listing is valid for a binary with no tests. Never silently ignore
+ * unrecognized lines or deduplicate names: either would hide selection errors.
+ * The listing does not identify ignored tests; retain them and libtest's defaults.
+ */
+export function parseRustListing(stdout: string): string[] {
+  const names: string[] = [];
+  const seen = new Set<string>();
+  const lines = stdout.replaceAll("\r\n", "\n").split("\n");
+  if (lines.at(-1) === "") lines.pop();
+  for (const line of lines) {
+    // Reject control bytes rather than treating terminal decoration as test names.
+    // deno-lint-ignore no-control-regex
+    const match = /^([^\x00-\x1f\x7f]+): (test|benchmark)$/.exec(line);
+    if (!match) {
+      throw new Error(`unknown Rust listing format: ${JSON.stringify(line)}`);
+    }
+    const name = match[1];
+    if (seen.has(name)) throw new Error(`duplicate Rust listing name: ${name}`);
+    seen.add(name);
+    names.push(name);
+  }
+  return names;
+}
+
+export interface RustShard {
+  task: TestCommand;
+  selectedNames: string[];
+}
+
+export function shardRustCommand(
+  task: TestCommand,
+  names: string[],
+): RustShard[] {
+  if (new Set(names).size !== names.length) {
+    throw new Error("cannot shard duplicate test names");
+  }
+  if (names.length <= 72) return [{ task, selectedNames: [...names] }];
+  const count = Math.min(32, Math.ceil(names.length / 36));
+  const shards: string[][] = Array.from({ length: count }, () => []);
+  names.forEach((name, index) => shards[index % count].push(name));
+  return shards.map((selectedNames, index) => ({
+    task: {
+      ...task,
+      name: `${task.name}/shard-${
+        String(index + 1).padStart(2, "0")
+      }-of-${count}`,
+      args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames],
+    },
+    selectedNames,
+  }));
+}
+
+export async function discoverRustCommands(
+  commands: TestCommand[],
+  jobs: number,
+  options: { output: string; signal?: AbortSignal },
+) {
+  const binaries = await mapConcurrent(commands, jobs, async (task, index) => {
+    const listing = await execute({
+      ...task,
+      name: `${task.name}/list`,
+      args: [...task.args, "--list", "--format", "terse"],
+    }, {
+      ...options,
+      label: `list-${String(index).padStart(3, "0")}`,
+      timeoutMs: SUITE_TIMEOUT_MS,
+    });
+    let shards: RustShard[] = [];
+    let error: string | undefined;
+    try {
+      if (listing.code !== 0) {
+        throw new Error(
+          `listing failed with code ${listing.code}; see ${listing.log}.stderr`,
+        );
+      }
+      shards = shardRustCommand(task, parseRustListing(listing.stdout)).map((
+        shard,
+        shardIndex,
+      ) => ({
+        ...shard,
+        task: {
+          ...shard.task,
+          inventoryRef: { binaryIndex: index, shardIndex },
+        },
+      }));
+    } catch (cause) {
+      error = String(cause);
+    }
+    return {
+      name: task.name,
+      binary: task.command,
+      listing: resultSummary(listing),
+      error,
+      selectedCases: error
+        ? null
+        : shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0),
+      shards,
+    };
+  });
+  const complete = binaries.every((binary) => !binary.error);
+  const discoveredCases = binaries.reduce(
+    (sum, binary) => sum + (binary.selectedCases ?? 0),
+    0,
+  );
+  const inventory = {
+    complete,
+    rustBinaries: commands.length,
+    // Includes ignored names, not a claim that ignored bodies were executed.
+    selectedCases: complete ? discoveredCases : null,
+    discoveredCases,
+    binaries,
+  };
+  await Deno.writeTextFile(
+    `${options.output}/inventory.json`,
+    JSON.stringify(inventory, null, 2),
+  );
+  return {
+    inventory,
+    tasks: complete
+      ? binaries.flatMap((binary) => binary.shards.map((shard) => shard.task))
+      : [],
+  };
+}
+
+function cancellationExitCode(reason: unknown): number {
+  return reason === "SIGTERM" ? 143 : 130;
+}
+
+/** Keep the parent alive until active groups have been stopped and summaries saved. */
+export function installCancellationHandlers() {
+  const controller = new AbortController();
+  const interrupt = () => controller.abort("SIGINT");
+  const terminate = () => controller.abort("SIGTERM");
+  Deno.addSignalListener("SIGINT", interrupt);
+  Deno.addSignalListener("SIGTERM", terminate);
+  return {
+    signal: controller.signal,
+    dispose() {
+      Deno.removeSignalListener("SIGINT", interrupt);
+      Deno.removeSignalListener("SIGTERM", terminate);
+    },
+  };
+}
+
+export interface ExecutionResult {
+  name: string;
+  code: number;
+  signal: string | null;
+  timedOut: boolean;
+  cancelled: boolean;
+  cancellationReason?: string;
+  error?: string;
+  seconds: number;
+  stdout: string;
+  stderr: string;
+  log: string;
+  timeoutMs: number;
+  inventoryRef?: TestCommand["inventoryRef"];
+}
+
+/** Linux only: a new process group contains ordinary inherited descendants.
+ * TERM then KILL targets the whole group, even if its leader already exited.
+ * Node's close event reaps the direct child and drains its pipes. Descendants are
+ * reaped by their parents/the OS; deliberately escaping via setsid is not covered.
+ */
+export async function execute(
+  task: TestCommand,
+  options: {
+    output: string;
+    label: string;
+    timeoutMs: number;
+    signal?: AbortSignal;
+    killGraceMs?: number;
+    closeGraceMs?: number;
+  },
+): Promise<ExecutionResult> {
+  const {
+    output,
+    label,
+    timeoutMs,
+    signal: abortSignal,
+    killGraceMs = 1_000,
+    closeGraceMs = 5_000,
+  } = options;
+  if (Deno.build.os !== "linux") {
+    throw new Error(
+      "test runner process-group cleanup is supported only on Linux",
+    );
+  }
+  for (const ms of [timeoutMs, killGraceMs, closeGraceMs]) {
+    if (!Number.isSafeInteger(ms) || ms < 1 || ms > 2_147_483_647) {
+      throw new Error(
+        "process deadlines must be positive timer-safe milliseconds",
+      );
+    }
+  }
+  const start = performance.now();
+  const stdoutFile = Deno.openSync(`${output}/${label}.stdout`, {
+    write: true,
+    create: true,
+    truncate: true,
+  });
+  let stderrFile: Deno.FsFile;
+  try {
+    stderrFile = Deno.openSync(`${output}/${label}.stderr`, {
+      write: true,
+      create: true,
+      truncate: true,
+    });
+  } catch (error) {
+    stdoutFile.close();
+    throw error;
+  }
+  const chunks: { stdout: Uint8Array[]; stderr: Uint8Array[] } = {
+    stdout: [],
+    stderr: [],
+  };
+  const errors: string[] = [];
+  let timedOut = false;
+  let cancelled = false;
+  let deadline: ReturnType<typeof setTimeout> | undefined;
+  let code = 1;
+  let signal: string | null = null;
+  let stop: (() => void) | undefined;
+  function save(stream: "stdout" | "stderr", data: Uint8Array) {
+    chunks[stream].push(data);
+    try {
+      const file = stream === "stdout" ? stdoutFile : stderrFile;
+      let offset = 0;
+      while (offset < data.length) {
+        const written = file.writeSync(data.subarray(offset));
+        if (!written) throw new Error("log write made no progress");
+        offset += written;
+      }
+    } catch (error) {
+      errors.push(`saving ${stream}: ${error}`);
+      stop?.();
+    }
+  }
+  const diagnostic = (text: string) =>
+    save("stderr", new TextEncoder().encode(`\n[runner] ${text}\n`));
+  const cancel = () => {
+    cancelled = true;
+    clearTimeout(deadline);
+    diagnostic(`cancelled by ${String(abortSignal?.reason)}`);
+    stop?.();
+  };
+  try {
+    if (abortSignal?.aborted) {
+      cancel(); // Queued commands are recorded as cancelled, never spawned.
+    } else {
+      abortSignal?.addEventListener("abort", cancel, { once: true });
+      const child = spawn(task.command, task.args, {
+        cwd: task.cwd,
+        env: { ...process.env, ...task.env },
+        detached: true,
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+      let closed = false;
+      const close = new Promise<void>((resolve) => {
+        child.once("close", (exitCode, exitSignal) => {
+          closed = true;
+          code = exitCode ?? 1;
+          signal = exitSignal;
+          resolve();
+        });
+      });
+      child.on("error", (error) => {
+        errors.push(String(error));
+        stop?.();
+      });
+      child.stdout!.on("data", (data: Uint8Array) => save("stdout", data));
+      child.stderr!.on("data", (data: Uint8Array) => save("stderr", data));
+      for (const pipe of [child.stdout!, child.stderr!]) {
+        pipe.on("error", (error) => {
+          errors.push(`reading process output: ${error}`);
+          stop?.();
+        });
+      }
+      const killGroup = (sig: "SIGTERM" | "SIGKILL") => {
+        if (child.pid === undefined) return;
+        try {
+          process.kill(-child.pid, sig);
+        } catch (error) {
+          if ((error as NodeJS.ErrnoException).code !== "ESRCH") {
+            errors.push(`process-group ${sig}: ${error}`);
+          }
+        }
+      };
+      let stopping: Promise<void> | undefined;
+      let resolveStopped: () => void;
+      const stopped = new Promise<void>((resolve) => resolveStopped = resolve);
+      stop = () => {
+        if (stopping) return;
+        stopping = (async () => {
+          killGroup("SIGTERM");
+          // Do not cancel escalation on leader close: descendants may still live,
+          // including ones that closed stdout/stderr or ignore SIGTERM.
+          await new Promise((resolve) => setTimeout(resolve, killGraceMs));
+          killGroup("SIGKILL");
+          let drainTimer: ReturnType<typeof setTimeout> | undefined;
+          await Promise.race([
+            close,
+            new Promise<void>((resolve) => {
+              drainTimer = setTimeout(resolve, closeGraceMs);
+            }),
+          ]);
+          clearTimeout(drainTimer);
+          if (!closed) {
+            errors.push(
+              "cleanup deadline exceeded: child/pipes did not close after SIGKILL",
+            );
+            child.stdout!.destroy();
+            child.stderr!.destroy();
+            child.unref();
+          }
+          resolveStopped();
+        })();
+      };
+      deadline = setTimeout(() => {
+        timedOut = true;
+        diagnostic(`timeout after ${timeoutMs}ms; terminating process group`);
+        stop!();
+      }, timeoutMs);
+      try {
+        await Promise.race([close, stopped]);
+        clearTimeout(deadline);
+        if (stopping) await stopping;
+      } finally {
+        clearTimeout(deadline);
+      }
+    }
+  } catch (error) {
+    errors.push(String(error));
+  } finally {
+    abortSignal?.removeEventListener("abort", cancel);
+    clearTimeout(deadline);
+    stop = undefined;
+    if (errors.length) diagnostic(errors.join("\n"));
+    stdoutFile.close();
+    stderrFile.close();
+  }
+  function text(parts: Uint8Array[]) {
+    const decoder = new TextDecoder();
+    return parts.map((part) => decoder.decode(part, { stream: true })).join(
+      "",
+    ) +
+      decoder.decode();
+  }
+  const result: ExecutionResult = {
+    name: task.name,
+    code: cancelled
+      ? cancellationExitCode(abortSignal?.reason)
+      : timedOut
+      ? 124
+      : errors.length
+      ? 1
+      : code,
+    signal,
+    timedOut,
+    cancelled,
+    ...(cancelled ? { cancellationReason: String(abortSignal?.reason) } : {}),
+    ...(errors.length ? { error: errors.join("\n") } : {}),
+    seconds: (performance.now() - start) / 1000,
+    stdout: text(chunks.stdout),
+    stderr: text(chunks.stderr),
+    log: label,
+    timeoutMs,
+    ...(task.inventoryRef ? { inventoryRef: task.inventoryRef } : {}),
+  };
+  // A per-command record also survives early metadata/build failures.
+  await Deno.writeTextFile(
+    `${output}/${label}.result.json`,
+    JSON.stringify(resultSummary(result), null, 2),
+  );
+  return result;
+}
+
+export function resultSummary(
+  { stdout: _stdout, stderr: _stderr, ...summary }: ExecutionResult,
+) {
+  return summary;
+}
+
+export interface CargoArtifact {
+  reason?: string;
+  package_id?: string;
+  executable?: string | null;
+  profile?: { test?: boolean };
+  target?: { name: string };
+}
+
+export interface TestCommand {
+  name: string;
+  command: string;
+  args: string[];
+  cwd: string;
+  env?: Record<string, string>;
+  // Exact link from each execution result into inventory.json, even when two
+  // Cargo targets happen to share the same package/target display name.
+  inventoryRef?: { binaryIndex: number; shardIndex: number };
+}
+
+export function rustTestCommands(
+  artifacts: CargoArtifact[],
+  packages: { id: string; name: string; manifest_path: string }[],
+): TestCommand[] {
+  const commands = new Map<string, TestCommand>();
+  for (const artifact of artifacts) {
+    if (
+      artifact.reason !== "compiler-artifact" || !artifact.profile?.test ||
+      !artifact.executable
+    ) continue;
+    const pkg = packages.find((pkg) => pkg.id === artifact.package_id);
+    if (!pkg || !artifact.target) {
+      throw new Error("test artifact has no Cargo package/target");
+    }
+    const cwd = pkg.manifest_path.replace(/[/\\][^/\\]+$/, "");
+    commands.set(artifact.executable, {
+      name: `${pkg.name}/${artifact.target.name}`,
+      command: artifact.executable,
+      args: [],
+      cwd,
+      env: { CARGO_MANIFEST_DIR: cwd },
+    });
+  }
+  if (!commands.size) throw new Error("Cargo produced no test executables");
+  return [...commands.values()];
+}
+
+export async function mapConcurrent<T, R>(
+  items: T[],
+  jobs: number,
+  run: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  if (!Number.isInteger(jobs) || jobs < 1) {
+    throw new Error("jobs must be a positive integer");
+  }
+  const results: R[] = new Array(items.length);
+  let next = 0;
+  // Drain every active worker even if one command fails to launch/write logs;
+  // callers must not remove signal handlers while detached groups are active.
+  const workers = await Promise.allSettled(
+    Array.from({ length: Math.min(jobs, items.length) }, async () => {
+      for (;;) {
+        const index = next++;
+        if (index >= items.length) return;
+        results[index] = await run(items[index], index);
+      }
+    }),
+  );
+  const failure = workers.find((worker) => worker.status === "rejected");
+  if (failure?.status === "rejected") throw failure.reason;
+  return results;
+}
+
+function jsonLines(text: string): CargoArtifact[] {
+  return text.split("\n").filter((line) => line.startsWith("{")).map((line) =>
+    JSON.parse(line)
+  );
+}
+
+async function main() {
+  const root = fileURLToPath(new URL("../", import.meta.url)).replace(
+    /[/\\]$/,
+    "",
+  );
+  const jobs = parseJobs(Deno.args);
+  const output = `${root}/target/normal-tests/${
+    new Date().toISOString().replaceAll(":", "-")
+  }`;
+  await Deno.mkdir(output, { recursive: true });
+  const cancellation = installCancellationHandlers();
+  const run = (
+    task: TestCommand,
+    label: string,
+    timeoutMs = SUITE_TIMEOUT_MS,
+  ) => execute(task, { output, label, timeoutMs, signal: cancellation.signal });
+  const summary: Record<string, unknown> = { phase: "metadata", failed: false };
+  let executionStart: number | undefined;
+  const writeSummary = () =>
+    Deno.writeTextFile(
+      `${output}/summary.json`,
+      JSON.stringify(
+        {
+          jobs,
+          concurrency:
+            "top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism",
+          platform: "linux",
+          suiteTimeoutMs: SUITE_TIMEOUT_MS,
+          buildTimeoutMs: BUILD_TIMEOUT_MS,
+          ...summary,
+        },
+        null,
+        2,
+      ),
+    );
+  console.log(
+    `Logs: ${output}; suite timeout: ${
+      SUITE_TIMEOUT_MS / 1000
+    }s; build timeout: ${
+      BUILD_TIMEOUT_MS / 1000
+    }s (not the 60s performance goal).`,
+  );
+  try {
+    const metadata = await run({
+      name: "metadata",
+      command: "cargo",
+      args: ["metadata", "--no-deps", "--format-version", "1"],
+      cwd: root,
+    }, "metadata");
+    summary.preparation = [resultSummary(metadata)];
+    if (metadata.code) {
+      Deno.exitCode = metadata.code;
+      throw new Error(`metadata failed: ${metadata.stderr}`);
+    }
+    summary.phase = "build";
+    console.log(
+      "Building all normal Rust test targets (separate from execution timing)…",
+    );
+    const build = await run(
+      {
+        name: "build",
+        command: "cargo",
+        args: [
+          "test",
+          "--workspace",
+          "--exclude",
+          "yoi-e2e",
+          "--no-run",
+          "--message-format=json",
+        ],
+        cwd: root,
+      },
+      "build",
+      BUILD_TIMEOUT_MS,
+    );
+    console.log(`Build: ${build.seconds.toFixed(2)}s`);
+    summary.buildSeconds = build.seconds;
+    summary.preparation = [resultSummary(metadata), resultSummary(build)];
+    if (build.code) {
+      Deno.exitCode = build.code;
+      throw new Error(`build failed: ${build.stderr}\n${build.stdout}`);
+    }
+    // Everything after compilation, including --list discovery, is execution time.
+    executionStart = performance.now();
+    summary.phase = "discovery";
+    const rust = rustTestCommands(
+      jsonLines(build.stdout),
+      JSON.parse(metadata.stdout).packages,
+    );
+    // Cargo reports dependencies first; running that order leaves the largest
+    // application suites until the end. Use executable size only as a scheduling
+    // hint. Every discovered target still runs, with no cached success/skip state.
+    const sizes = new Map(
+      await Promise.all(
+        rust.map(async (task) =>
+          [task.command, (await Deno.stat(task.command)).size] as const
+        ),
+      ),
+    );
+    rust.sort((a, b) =>
+      sizes.get(b.command)! - sizes.get(a.command)! ||
+      a.name.localeCompare(b.name)
+    );
+    summary.rustBinaries = rust.length;
+    console.log(`Listing all ${rust.length} Rust binaries before sharding…`);
+    const discoveryStart = performance.now();
+    const discovery = await discoverRustCommands(rust, jobs, {
+      output,
+      signal: cancellation.signal,
+    });
+    summary.discoverySeconds = (performance.now() - discoveryStart) / 1000;
+    summary.inventory = "inventory.json";
+    summary.discoveryComplete = discovery.inventory.complete;
+    summary.selectedCases = discovery.inventory.selectedCases;
+    summary.discovery = discovery.inventory.binaries.map((
+      { name, listing, error },
+    ) => ({ name, listing, error }));
+    if (!discovery.inventory.complete) {
+      throw new Error(
+        "Rust discovery failed; see summary.json, inventory.json and list-*.stderr",
+      );
+    }
+    summary.phase = "execution";
+    const tasks: TestCommand[] = [
+      {
+        name: "rust/doctests",
+        command: "cargo",
+        args: [
+          "test",
+          "--workspace",
+          "--exclude",
+          "yoi-e2e",
+          "--doc",
+          "--no-fail-fast",
+        ],
+        cwd: root,
+      },
+      {
+        name: "web/unit",
+        command: "deno",
+        args: ["task", "test"],
+        cwd: `${root}/web/workspace`,
+      },
+      {
+        name: "web/component",
+        command: "deno",
+        args: ["task", "test:component"],
+        cwd: `${root}/web/workspace`,
+      },
+      {
+        name: "web-ux/unit",
+        command: "deno",
+        args: ["task", "test"],
+        cwd: `${root}/tools/web-ux`,
+      },
+      {
+        name: "tools/unit",
+        command: "deno",
+        args: [
+          "test",
+          "--allow-read",
+          "--allow-write",
+          "--allow-run",
+          "--allow-env",
+          "tools/test.test.ts",
+          "tools/check-state-contract-names.test.ts",
+        ],
+        cwd: root,
+      },
+      ...discovery.tasks,
+    ];
+    console.log(
+      `Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
+    );
+    const results: ReturnType<typeof resultSummary>[] = [];
+    summary.results = results;
+    await mapConcurrent(tasks, jobs, async (task, index) => {
+      const label = `${String(index).padStart(3, "0")}-${
+        task.name.replaceAll(/[^\w.-]/g, "-")
+      }`;
+      const result = await run(task, label);
+      console.log(
+        `${result.code ? "FAIL" : "PASS"} ${task.name} (${
+          result.seconds.toFixed(2)
+        }s)`,
+      );
+      if (result.code) {
+        console.error(result.stdout.slice(-8000), result.stderr.slice(-8000));
+      }
+      results[index] = resultSummary(result);
+    });
+    const seconds = (performance.now() - executionStart) / 1000;
+    const failed = results.filter((result) => result.code !== 0);
+    summary.failed = failed.length > 0;
+    console.log(
+      `Execution: ${seconds.toFixed(2)}s; ${
+        results.length - failed.length
+      } suites passed, ${failed.length} failed. Build: ${
+        build.seconds.toFixed(2)
+      }s.`,
+    );
+    if (failed.length) Deno.exitCode = 1;
+  } catch (error) {
+    summary.failed = true;
+    summary.error = String(error);
+    console.error(error);
+    if (!Deno.exitCode) Deno.exitCode = 1;
+  } finally {
+    if (executionStart !== undefined) {
+      summary.executionSeconds = (performance.now() - executionStart) / 1000;
+    }
+    try {
+      if (cancellation.signal.aborted) {
+        summary.cancelled = true;
+        summary.cancellationReason = String(cancellation.signal.reason);
+        summary.failed = true;
+        Deno.exitCode = cancellationExitCode(cancellation.signal.reason);
+      }
+      await writeSummary();
+    } finally {
+      cancellation.dispose();
+    }
+  }
+}
+
+if (import.meta.main) await main();