diff --git a/deno.json b/deno.json index 502070192..7b9ab8d69 100644 --- a/deno.json +++ b/deno.json @@ -1,4 +1,8 @@ { + "tasks": { + "test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts", + "test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts" + }, "fmt": { "proseWrap": "never" } diff --git a/tools/test.test.ts b/tools/test.test.ts new file mode 100644 index 000000000..aa8c69d12 --- /dev/null +++ b/tools/test.test.ts @@ -0,0 +1,701 @@ +import { + defaultJobs, + discoverRustCommands, + execute, + installCancellationHandlers, + mapConcurrent, + parseJobs, + parseRustListing, + resultSummary, + rustTestCommands, + shardRustCommand, +} from "./test.ts"; +import { spawn } from "node:child_process"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +function assert( + condition: unknown, + message = "assertion failed", +): asserts condition { + if (!condition) throw new Error(message); +} + +Deno.test("Cargo artifact selection retains every test target and deduplicates executable reports", () => { + const test = (path: string, target: string) => ({ + reason: "compiler-artifact", + package_id: "pkg", + executable: path, + target: { name: target }, + profile: { test: true }, + }); + const commands = rustTestCommands([ + test("/repo/target/lib-test", "lib"), + test("/repo/target/integration", "integration"), + test("/repo/target/integration", "integration"), + { ...test("/repo/target/app", "app"), profile: { test: false } }, + { reason: "build-finished" }, + ], [{ + id: "pkg", + name: "example", + manifest_path: "/repo/crates/example/Cargo.toml", + }]); + assert(commands.length === 2); + assert(commands[0].cwd === "/repo/crates/example"); + assert(commands[0].env?.CARGO_MANIFEST_DIR === commands[0].cwd); + assert(commands[1].name === "example/integration"); + assert( + commands.every((command) => command.args.length === 0), + "no filtering or ignore flags", + ); +}); + +Deno.test("missing Cargo package or executables fails instead of claiming an empty suite passed", () => { + for ( + const artifacts of [[], [{ + reason: "compiler-artifact", + package_id: "missing", + executable: "/test", + profile: { test: true }, + target: { name: "test" }, + }]] + ) { + let failed = false; + try { + rustTestCommands(artifacts, []); + } catch { + failed = true; + } + assert(failed); + } +}); + +Deno.test("bounded scheduling runs every command once and retains failing results", async () => { + let active = 0, maximum = 0; + const seen: number[] = []; + const results = await mapConcurrent([0, 1, 2, 3, 4], 2, async (item) => { + active++; + maximum = Math.max(active, maximum); + await Promise.resolve(); + seen.push(item); + active--; + return item === 2 ? 1 : 0; + }); + assert(maximum === 2); + assert(seen.sort().join() === "0,1,2,3,4"); + assert(results.join() === "0,0,1,0,0"); + for (const jobs of [0, -1, 1.5]) { + let failed = false; + try { + await mapConcurrent([], jobs, () => Promise.resolve(0)); + } catch { + failed = true; + } + assert(failed); + } +}); + +function assertThrows(fn: () => unknown) { + let failed = false; + try { + fn(); + } catch { + failed = true; + } + assert(failed, "expected failure"); +} + +Deno.test("CPU-derived jobs accommodates concurrent suites within the CLI bound", () => { + assert(defaultJobs(24) === 32); + assert(defaultJobs(128) === 32); + assert(defaultJobs(6) === 8); + assert(defaultJobs(1) === 2); + assert(parseJobs([]) === defaultJobs()); + assert(parseJobs(["--jobs", "64"]) === 64); + assert(parseJobs(["--jobs", "1"]) === 1); + for ( + const args of [ + ["--jobs", "65"], + ["--jobs", "0"], + ["--jobs", "1.5"], + ["--jobs", "NaN"], + ["--jobs"], + ["--other", "4"], + ] + ) { + assertThrows(() => parseJobs(args)); + } +}); + +Deno.test("Rust terse listing accepts every name and fails closed on unknown or duplicate lines", () => { + assert(parseRustListing("").length === 0); + assert( + parseRustListing("unit::a: test\r\nunit::bench: benchmark\r\n").join() === + "unit::a,unit::bench", + ); + for ( + const text of [ + "unit::a\n", + "unit::a: ignored\n", + "unit::a: test\n1 test, 0 benchmarks\n", + "unit::a: test\nunit::a: test\n", + "\n", + "unit::a: test\n\n", + "\x1b[31munit::a: test\n", + ] + ) { + assertThrows(() => parseRustListing(text)); + } +}); + +function listedNames(count: number) { + return Array.from({ length: count }, (_, i) => `unit::case_${i}`); +} + +Deno.test("round-robin shards retain every listed name exactly once at thresholds and the 32-shard cap", () => { + const task = { + name: "pkg/binary", + command: "/test-binary", + args: [], + cwd: "/pkg", + env: { CARGO_MANIFEST_DIR: "/pkg" }, + }; + for (const count of [0, 1, 72, 73, 108, 1135, 1152, 1153, 2000]) { + const names = parseRustListing( + listedNames(count).map((name) => `${name}: test\n`).join(""), + ); + const shards = shardRustCommand(task, names); + const assigned = shards.flatMap((shard) => shard.selectedNames); + assert(assigned.length === count && new Set(assigned).size === count); + assert( + [...assigned].sort().join("\n") === [...names].sort().join("\n"), + "no names may be excluded", + ); + if (count <= 72) { + assert( + shards.length === 1 && shards[0].task === task, + "small binaries must be unchanged", + ); + } else { + const expected = Math.min(32, Math.ceil(count / 36)); + assert(shards.length === expected); + const lengths = shards.map((shard) => shard.selectedNames.length); + assert(Math.max(...lengths) - Math.min(...lengths) <= 1); + shards.forEach((shard, index) => { + assert( + shard.selectedNames.join() === + names.filter((_, i) => i % expected === index).join(), + "must distribute round-robin", + ); + assert( + shard.task.args.slice(0, 3).join() === "--exact,--test-threads,2", + ); + assert(shard.task.args.slice(3).join() === shard.selectedNames.join()); + assert( + !shard.task.args.includes("--ignored") && + !shard.task.args.includes("--include-ignored"), + ); + assert(shard.task.cwd === task.cwd && shard.task.env === task.env); + }); + } + } + assertThrows(() => shardRustCommand(task, ["same", "same"])); +}); + +Deno.test("scheduler drains active workers before propagating a callback error", async () => { + let drained = false; + let failed = false; + try { + await mapConcurrent([0, 1], 2, async (item) => { + if (!item) throw new Error("callback failed"); + await new Promise((resolve) => setTimeout(resolve, 10)); + drained = true; + }); + } catch { + failed = true; + } + assert(failed && drained); +}); + +const fixturePath = fileURLToPath(import.meta.url); +function fixtureArgs(mode: string, output: string) { + return [ + "run", + "--allow-read", + "--allow-write", + "--allow-run", + "--allow-env", + fixturePath, + mode, + output, + ]; +} +function fixtureTask(mode: string, output: string) { + return { + name: `fixture/${mode}`, + command: Deno.execPath(), + args: fixtureArgs(mode, output), + cwd: output, + }; +} +async function waitUntil( + check: () => boolean | Promise, + message: string, +) { + const end = performance.now() + 10_000; + while (performance.now() < end) { + if (await check()) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error(message); +} +function absent(pid: number) { + try { + process.kill(pid, 0); + return false; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return true; + throw error; + } +} +function emergencyKill(pid: number) { + try { + process.kill(-pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } +} + +for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) { + Deno.test({ + name: `execute streams real ${mode} child output and records its exit`, + ignore: Deno.build.os !== "linux", + async fn() { + const output = await Deno.makeTempDir(); + try { + const result = await execute({ + ...fixtureTask(mode, output), + inventoryRef: { binaryIndex: 3, shardIndex: 1 }, + }, { + output, + label: mode, + timeoutMs: 10_000, + }); + assert(result.code === code, JSON.stringify(result)); + assert(!result.timedOut); + assert(!result.error, result.error); + assert(result.stdout.includes("fixture stdout 日本語")); + assert(result.stderr.includes("fixture stderr")); + assert( + await Deno.readTextFile(`${output}/${mode}.stdout`) === result.stdout, + ); + assert( + await Deno.readTextFile(`${output}/${mode}.stderr`) === result.stderr, + ); + const summary = JSON.parse( + await Deno.readTextFile(`${output}/${mode}.result.json`), + ); + assert(summary.code === code && summary.timedOut === false); + assert(summary.log === mode && summary.timeoutMs === 10_000); + assert( + summary.inventoryRef.binaryIndex === 3 && + summary.inventoryRef.shardIndex === 1, + "results must link unambiguously to inventory entries", + ); + assert( + !("stdout" in summary), + "summary references logs, not full output", + ); + assert( + absent(Number(await Deno.readTextFile(`${output}/parent.pid`))), + "direct child must be reaped", + ); + } finally { + await Deno.remove(output, { recursive: true }); + } + }, + }); +} + +Deno.test({ + name: + "execute preserves live stdout/stderr and times out a group containing a grandchild", + ignore: Deno.build.os !== "linux", + async fn() { + const output = await Deno.makeTempDir(); + let parent: number | undefined; + let finished = false; + const running = execute(fixtureTask("tree", output), { + output, + label: "tree", + timeoutMs: 4_000, + killGraceMs: 500, + }).finally(() => finished = true); + try { + await waitUntil(async () => { + const text = await Deno.readTextFile(`${output}/tree.stdout`); + return text.includes("grandchild ready"); + }, "grandchild did not become ready"); + parent = Number(await Deno.readTextFile(`${output}/parent.pid`)); + const grandchild = Number( + await Deno.readTextFile(`${output}/grandchild.pid`), + ); + assert(!finished, "logs must be visible before execute resolves"); + assert( + (await Deno.readTextFile(`${output}/tree.stderr`)).includes( + "fixture stderr", + ), + ); + assert( + !absent(parent) && !absent(grandchild), + "both processes must actually be running", + ); + const result = await running; + assert(result.timedOut && result.code === 124, JSON.stringify(result)); + assert(!result.error, result.error); + assert(result.seconds < 10, "timeout cleanup must be bounded"); + assert(result.stderr.includes("timeout after 4000ms")); + assert( + result.stderr.includes("grandchild reaped"), + "fixture parent must reap its child", + ); + assert(absent(parent), "direct child must be reaped by runner"); + assert(absent(grandchild), "grandchild must be stopped and reaped"); + const summary = JSON.parse( + await Deno.readTextFile(`${output}/tree.result.json`), + ); + assert(summary.timedOut && summary.code === 124); + assert(summary.name === "fixture/tree" && summary.log === "tree"); + } finally { + if (parent !== undefined) emergencyKill(parent); + await running; + await Deno.remove(output, { recursive: true }); + } + }, +}); + +Deno.test({ + name: + "execute escalates SIGTERM to SIGKILL and reaps a stubborn direct child", + ignore: Deno.build.os !== "linux", + async fn() { + const output = await Deno.makeTempDir(); + try { + const result = await execute(fixtureTask("stubborn", output), { + output, + label: "stubborn", + timeoutMs: 2_000, + killGraceMs: 100, + }); + assert(result.stdout.includes("stubborn ready")); + assert(result.stderr.includes("ignored SIGTERM")); + assert( + result.timedOut && result.code === 124 && result.signal === "SIGKILL", + JSON.stringify(result), + ); + assert(!result.error, result.error); + assert(result.seconds < 8); + assert(absent(Number(await Deno.readTextFile(`${output}/parent.pid`)))); + } finally { + await Deno.remove(output, { recursive: true }); + } + }, +}); + +Deno.test({ + name: "execute records spawn failure instead of an empty passing suite", + ignore: Deno.build.os !== "linux", + async fn() { + const output = await Deno.makeTempDir(); + try { + const result = await execute({ + name: "missing", + command: `${output}/does-not-exist`, + args: [], + cwd: output, + }, { output, label: "missing", timeoutMs: 2_000, killGraceMs: 10 }); + assert(result.code !== 0 && !result.timedOut); + assert(result.error?.includes("ENOENT"), JSON.stringify(result)); + assert( + (await Deno.readTextFile(`${output}/missing.stderr`)).includes( + "ENOENT", + ), + ); + const summary = JSON.parse( + await Deno.readTextFile(`${output}/missing.result.json`), + ); + assert(summary.code !== 0 && summary.error.includes("ENOENT")); + } finally { + await Deno.remove(output, { recursive: true }); + } + }, +}); + +Deno.test({ + name: + "runtime listing persists complete name inventory for large and small binaries", + ignore: Deno.build.os !== "linux", + async fn() { + const output = await Deno.makeTempDir(); + try { + const discovery = await discoverRustCommands( + [ + fixtureTask("list-large", output), + fixtureTask("list-small", output), + ], + 2, + { output }, + ); + assert(discovery.inventory.complete); + assert( + discovery.inventory.rustBinaries === 2 && + discovery.inventory.selectedCases === 1137, + ); + assert(discovery.tasks.length === 33); + const persisted = JSON.parse( + await Deno.readTextFile(`${output}/inventory.json`), + ) as typeof discovery.inventory; + assert(persisted.complete && persisted.selectedCases === 1137); + const large = persisted.binaries[0]; + assert(large.shards.length === 32 && large.selectedCases === 1135); + persisted.binaries.forEach((binary, binaryIndex) => + binary.shards.forEach((shard, shardIndex) => { + assert(shard.task.inventoryRef?.binaryIndex === binaryIndex); + assert(shard.task.inventoryRef?.shardIndex === shardIndex); + }) + ); + const assigned = large.shards.flatMap((shard) => shard.selectedNames); + assert(new Set(assigned).size === 1135 && assigned.length === 1135); + assert([...assigned].sort().join() === listedNames(1135).sort().join()); + assert( + persisted.binaries[1].shards[0].task.args.join() === + fixtureTask("list-small", output).args.join(), + ); + assert( + persisted.binaries.every((binary) => + binary.listing.code === 0 && binary.listing.seconds > 0 + ), + ); + } finally { + await Deno.remove(output, { recursive: true }); + } + }, +}); + +for (const mode of ["list-nonzero", "list-unknown"]) { + Deno.test({ + name: + `discovery ${mode} fails with saved evidence rather than running a partial selection`, + ignore: Deno.build.os !== "linux", + async fn() { + const output = await Deno.makeTempDir(); + try { + const discovery = await discoverRustCommands( + [ + fixtureTask("list-small", output), + fixtureTask(mode, output), + ], + 2, + { output }, + ); + assert(!discovery.inventory.complete); + assert(discovery.inventory.selectedCases === null); + assert(discovery.tasks.length === 0); + const persisted = JSON.parse( + await Deno.readTextFile(`${output}/inventory.json`), + ) as typeof discovery.inventory; + assert( + !persisted.complete && persisted.binaries[1].shards.length === 0, + ); + assert( + persisted.binaries[1].error?.includes( + mode === "list-nonzero" ? "code 7" : "unknown Rust listing format", + ), + ); + const listing = JSON.parse( + await Deno.readTextFile(`${output}/list-001.result.json`), + ); + assert(listing.code === (mode === "list-nonzero" ? 7 : 0)); + assert( + persisted.discoveredCases === 2, + "partial discovery must not be presented as complete selection", + ); + } finally { + await Deno.remove(output, { recursive: true }); + } + }, + }); +} + +async function textIfPresent(path: string): Promise { + try { + return await Deno.readTextFile(path); + } catch (error) { + if (error instanceof Deno.errors.NotFound) return ""; + throw error; + } +} + +for (const signal of ["SIGINT", "SIGTERM"] as const) { + Deno.test({ + name: + `parent ${signal} cleans up active groups, reaps children and does not spawn queued commands`, + ignore: Deno.build.os !== "linux", + async fn() { + const output = await Deno.makeTempDir(); + const parent = new Deno.Command(Deno.execPath(), { + args: fixtureArgs("runner-parent", output), + stdout: "piped", + stderr: "piped", + }).spawn(); + const done = parent.output(); + const pids: number[] = []; + try { + await waitUntil( + async () => + (await textIfPresent(`${output}/tree/tree.stdout`)).includes( + "grandchild ready", + ) && + (await textIfPresent(`${output}/stubborn/stubborn.stdout`)) + .includes("stubborn ready"), + "parent runner fixtures did not become ready", + ); + for ( + const file of [ + "tree/parent.pid", + "tree/grandchild.pid", + "stubborn/parent.pid", + ] + ) { + pids.push(Number(await Deno.readTextFile(`${output}/${file}`))); + } + process.kill(parent.pid, signal); + const status = await done; + assert( + status.code === (signal === "SIGINT" ? 130 : 143), + new TextDecoder().decode(status.stderr), + ); + assert(absent(parent.pid)); + assert( + pids.every(absent), + "active children and grandchild must be reaped", + ); + const results = JSON.parse( + await Deno.readTextFile(`${output}/cancel-summary.json`), + ) as ReturnType[]; + assert(results.length === 3); + assert( + results.every((result) => + result.cancelled && result.cancellationReason === signal && + !result.timedOut && !result.error + ), + ); + assert( + results[1].signal === "SIGKILL", + "stubborn child requires escalation on parent cancellation too", + ); + assert( + await textIfPresent(`${output}/queued/parent.pid`) === "", + "cancelled queue must not spawn", + ); + assert( + (await Deno.readTextFile(`${output}/tree/tree.stderr`)).includes( + "grandchild reaped", + ), + ); + } finally { + if (!absent(parent.pid)) process.kill(parent.pid, "SIGTERM"); + await done; + for (const pid of pids) emergencyKill(pid); + await Deno.remove(output, { recursive: true }); + } + }, + }); +} + +// Real subprocess fixtures live here rather than introducing another executable. +// A watchdog is independent of execute, so a broken runner cannot leak fixtures +// forever. The tree's parent cooperatively reaps its child on TERM; KILL behavior +// is exercised separately by the stubborn fixture. +if (import.meta.main) { + const [mode, output] = Deno.args; + setTimeout(() => Deno.exit(99), 20_000); + if (mode.startsWith("list-")) { + assert( + Deno.args.slice(2).join() === "--list,--format,terse", + "discovery must use libtest terse listing", + ); + if (mode === "list-unknown") { + console.log("unexpected harness output"); + } else { + console.log( + listedNames(mode === "list-large" ? 1135 : 2).map((name) => + `${name}: test` + ).join("\n"), + ); + } + Deno.exit(mode === "list-nonzero" ? 7 : 0); + } else if (mode === "runner-parent") { + const cancellation = installCancellationHandlers(); + try { + for (const dir of ["tree", "stubborn", "queued"]) { + await Deno.mkdir(`${output}/${dir}`); + } + const results = await mapConcurrent( + ["tree", "stubborn", "queued"], + 2, + async (name) => { + const dir = `${output}/${name}`; + return resultSummary( + await execute( + fixtureTask(name === "queued" ? "success" : name, dir), + { + output: dir, + label: name, + timeoutMs: 15_000, + killGraceMs: 200, + signal: cancellation.signal, + }, + ), + ); + }, + ); + await Deno.writeTextFile( + `${output}/cancel-summary.json`, + JSON.stringify(results), + ); + Deno.exitCode = results[0].code; + } finally { + cancellation.dispose(); + } + Deno.exit(Deno.exitCode); + } else if (mode === "grandchild") { + Deno.addSignalListener("SIGTERM", () => Deno.exit(0)); + Deno.writeTextFileSync(`${output}/grandchild.pid`, String(Deno.pid)); + console.log("grandchild ready"); + } else { + Deno.writeTextFileSync(`${output}/parent.pid`, String(Deno.pid)); + console.log("fixture stdout 日本語"); + console.error("fixture stderr"); + if (mode === "success" || mode === "nonzero") { + Deno.exit(mode === "success" ? 0 : 7); + } else if (mode === "stubborn") { + Deno.addSignalListener("SIGTERM", () => console.error("ignored SIGTERM")); + console.log("stubborn ready"); + } else if (mode === "tree") { + // The grandchild inherits the new group from execute's direct child. + // The parent intentionally does NOT forward TERM: only group signaling + // can stop the grandchild before its independent watchdog. + Deno.addSignalListener("SIGTERM", () => {}); + const child = spawn(Deno.execPath(), fixtureArgs("grandchild", output), { + stdio: "inherit", + }); + child.on("close", () => { + console.error("grandchild reaped"); + Deno.exit(0); + }); + } else { + throw new Error(`unknown fixture mode: ${mode}`); + } + } +} diff --git a/tools/test.ts b/tools/test.ts new file mode 100644 index 000000000..14cb347d4 --- /dev/null +++ b/tools/test.ts @@ -0,0 +1,719 @@ +// Bound top-level suites/shards, not total processes; large shards use two test threads. +// Cargo selects targets/features. Retain all listed names; never skip cached successes. +import { spawn } from "node:child_process"; +import process from "node:process"; +import { availableParallelism } from "node:os"; +import { fileURLToPath } from "node:url"; + +// Safety limits, NOT the separate 60s performance goal. +export const SUITE_TIMEOUT_MS = 180_000; +export const BUILD_TIMEOUT_MS = 1_800_000; +export const MAX_JOBS = 64; + +export function defaultJobs(cpus = availableParallelism()): number { + // Leave headroom for internal runners; higher top-level counts can be chosen + // explicitly, but are not assumed to improve throughput or timing stability. + return Math.min(32, Math.max(1, Math.ceil(cpus * 4 / 3))); +} + +export function parseJobs(args: string[]): number { + if (!args.length) return defaultJobs(); + if (args.length !== 2 || args[0] !== "--jobs") { + throw new Error("usage: deno task test [--jobs N]"); + } + const jobs = Number(args[1]); + if (!Number.isInteger(jobs) || jobs < 1 || jobs > MAX_JOBS) { + throw new Error(`jobs must be 1..${MAX_JOBS}`); + } + return jobs; +} + +/** libtest's terse --list has one `name: test`/`name: benchmark` per line. + * An empty listing is valid for a binary with no tests. Never silently ignore + * unrecognized lines or deduplicate names: either would hide selection errors. + * The listing does not identify ignored tests; retain them and libtest's defaults. + */ +export function parseRustListing(stdout: string): string[] { + const names: string[] = []; + const seen = new Set(); + const lines = stdout.replaceAll("\r\n", "\n").split("\n"); + if (lines.at(-1) === "") lines.pop(); + for (const line of lines) { + // Reject control bytes rather than treating terminal decoration as test names. + // deno-lint-ignore no-control-regex + const match = /^([^\x00-\x1f\x7f]+): (test|benchmark)$/.exec(line); + if (!match) { + throw new Error(`unknown Rust listing format: ${JSON.stringify(line)}`); + } + const name = match[1]; + if (seen.has(name)) throw new Error(`duplicate Rust listing name: ${name}`); + seen.add(name); + names.push(name); + } + return names; +} + +export interface RustShard { + task: TestCommand; + selectedNames: string[]; +} + +export function shardRustCommand( + task: TestCommand, + names: string[], +): RustShard[] { + if (new Set(names).size !== names.length) { + throw new Error("cannot shard duplicate test names"); + } + if (names.length <= 72) return [{ task, selectedNames: [...names] }]; + const count = Math.min(32, Math.ceil(names.length / 36)); + const shards: string[][] = Array.from({ length: count }, () => []); + names.forEach((name, index) => shards[index % count].push(name)); + return shards.map((selectedNames, index) => ({ + task: { + ...task, + name: `${task.name}/shard-${ + String(index + 1).padStart(2, "0") + }-of-${count}`, + args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames], + }, + selectedNames, + })); +} + +export async function discoverRustCommands( + commands: TestCommand[], + jobs: number, + options: { output: string; signal?: AbortSignal }, +) { + const binaries = await mapConcurrent(commands, jobs, async (task, index) => { + const listing = await execute({ + ...task, + name: `${task.name}/list`, + args: [...task.args, "--list", "--format", "terse"], + }, { + ...options, + label: `list-${String(index).padStart(3, "0")}`, + timeoutMs: SUITE_TIMEOUT_MS, + }); + let shards: RustShard[] = []; + let error: string | undefined; + try { + if (listing.code !== 0) { + throw new Error( + `listing failed with code ${listing.code}; see ${listing.log}.stderr`, + ); + } + shards = shardRustCommand(task, parseRustListing(listing.stdout)).map(( + shard, + shardIndex, + ) => ({ + ...shard, + task: { + ...shard.task, + inventoryRef: { binaryIndex: index, shardIndex }, + }, + })); + } catch (cause) { + error = String(cause); + } + return { + name: task.name, + binary: task.command, + listing: resultSummary(listing), + error, + selectedCases: error + ? null + : shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0), + shards, + }; + }); + const complete = binaries.every((binary) => !binary.error); + const discoveredCases = binaries.reduce( + (sum, binary) => sum + (binary.selectedCases ?? 0), + 0, + ); + const inventory = { + complete, + rustBinaries: commands.length, + // Includes ignored names, not a claim that ignored bodies were executed. + selectedCases: complete ? discoveredCases : null, + discoveredCases, + binaries, + }; + await Deno.writeTextFile( + `${options.output}/inventory.json`, + JSON.stringify(inventory, null, 2), + ); + return { + inventory, + tasks: complete + ? binaries.flatMap((binary) => binary.shards.map((shard) => shard.task)) + : [], + }; +} + +function cancellationExitCode(reason: unknown): number { + return reason === "SIGTERM" ? 143 : 130; +} + +/** Keep the parent alive until active groups have been stopped and summaries saved. */ +export function installCancellationHandlers() { + const controller = new AbortController(); + const interrupt = () => controller.abort("SIGINT"); + const terminate = () => controller.abort("SIGTERM"); + Deno.addSignalListener("SIGINT", interrupt); + Deno.addSignalListener("SIGTERM", terminate); + return { + signal: controller.signal, + dispose() { + Deno.removeSignalListener("SIGINT", interrupt); + Deno.removeSignalListener("SIGTERM", terminate); + }, + }; +} + +export interface ExecutionResult { + name: string; + code: number; + signal: string | null; + timedOut: boolean; + cancelled: boolean; + cancellationReason?: string; + error?: string; + seconds: number; + stdout: string; + stderr: string; + log: string; + timeoutMs: number; + inventoryRef?: TestCommand["inventoryRef"]; +} + +/** Linux only: a new process group contains ordinary inherited descendants. + * TERM then KILL targets the whole group, even if its leader already exited. + * Node's close event reaps the direct child and drains its pipes. Descendants are + * reaped by their parents/the OS; deliberately escaping via setsid is not covered. + */ +export async function execute( + task: TestCommand, + options: { + output: string; + label: string; + timeoutMs: number; + signal?: AbortSignal; + killGraceMs?: number; + closeGraceMs?: number; + }, +): Promise { + const { + output, + label, + timeoutMs, + signal: abortSignal, + killGraceMs = 1_000, + closeGraceMs = 5_000, + } = options; + if (Deno.build.os !== "linux") { + throw new Error( + "test runner process-group cleanup is supported only on Linux", + ); + } + for (const ms of [timeoutMs, killGraceMs, closeGraceMs]) { + if (!Number.isSafeInteger(ms) || ms < 1 || ms > 2_147_483_647) { + throw new Error( + "process deadlines must be positive timer-safe milliseconds", + ); + } + } + const start = performance.now(); + const stdoutFile = Deno.openSync(`${output}/${label}.stdout`, { + write: true, + create: true, + truncate: true, + }); + let stderrFile: Deno.FsFile; + try { + stderrFile = Deno.openSync(`${output}/${label}.stderr`, { + write: true, + create: true, + truncate: true, + }); + } catch (error) { + stdoutFile.close(); + throw error; + } + const chunks: { stdout: Uint8Array[]; stderr: Uint8Array[] } = { + stdout: [], + stderr: [], + }; + const errors: string[] = []; + let timedOut = false; + let cancelled = false; + let deadline: ReturnType | undefined; + let code = 1; + let signal: string | null = null; + let stop: (() => void) | undefined; + function save(stream: "stdout" | "stderr", data: Uint8Array) { + chunks[stream].push(data); + try { + const file = stream === "stdout" ? stdoutFile : stderrFile; + let offset = 0; + while (offset < data.length) { + const written = file.writeSync(data.subarray(offset)); + if (!written) throw new Error("log write made no progress"); + offset += written; + } + } catch (error) { + errors.push(`saving ${stream}: ${error}`); + stop?.(); + } + } + const diagnostic = (text: string) => + save("stderr", new TextEncoder().encode(`\n[runner] ${text}\n`)); + const cancel = () => { + cancelled = true; + clearTimeout(deadline); + diagnostic(`cancelled by ${String(abortSignal?.reason)}`); + stop?.(); + }; + try { + if (abortSignal?.aborted) { + cancel(); // Queued commands are recorded as cancelled, never spawned. + } else { + abortSignal?.addEventListener("abort", cancel, { once: true }); + const child = spawn(task.command, task.args, { + cwd: task.cwd, + env: { ...process.env, ...task.env }, + detached: true, + stdio: ["ignore", "pipe", "pipe"], + }); + let closed = false; + const close = new Promise((resolve) => { + child.once("close", (exitCode, exitSignal) => { + closed = true; + code = exitCode ?? 1; + signal = exitSignal; + resolve(); + }); + }); + child.on("error", (error) => { + errors.push(String(error)); + stop?.(); + }); + child.stdout!.on("data", (data: Uint8Array) => save("stdout", data)); + child.stderr!.on("data", (data: Uint8Array) => save("stderr", data)); + for (const pipe of [child.stdout!, child.stderr!]) { + pipe.on("error", (error) => { + errors.push(`reading process output: ${error}`); + stop?.(); + }); + } + const killGroup = (sig: "SIGTERM" | "SIGKILL") => { + if (child.pid === undefined) return; + try { + process.kill(-child.pid, sig); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") { + errors.push(`process-group ${sig}: ${error}`); + } + } + }; + let stopping: Promise | undefined; + let resolveStopped: () => void; + const stopped = new Promise((resolve) => resolveStopped = resolve); + stop = () => { + if (stopping) return; + stopping = (async () => { + killGroup("SIGTERM"); + // Do not cancel escalation on leader close: descendants may still live, + // including ones that closed stdout/stderr or ignore SIGTERM. + await new Promise((resolve) => setTimeout(resolve, killGraceMs)); + killGroup("SIGKILL"); + let drainTimer: ReturnType | undefined; + await Promise.race([ + close, + new Promise((resolve) => { + drainTimer = setTimeout(resolve, closeGraceMs); + }), + ]); + clearTimeout(drainTimer); + if (!closed) { + errors.push( + "cleanup deadline exceeded: child/pipes did not close after SIGKILL", + ); + child.stdout!.destroy(); + child.stderr!.destroy(); + child.unref(); + } + resolveStopped(); + })(); + }; + deadline = setTimeout(() => { + timedOut = true; + diagnostic(`timeout after ${timeoutMs}ms; terminating process group`); + stop!(); + }, timeoutMs); + try { + await Promise.race([close, stopped]); + clearTimeout(deadline); + if (stopping) await stopping; + } finally { + clearTimeout(deadline); + } + } + } catch (error) { + errors.push(String(error)); + } finally { + abortSignal?.removeEventListener("abort", cancel); + clearTimeout(deadline); + stop = undefined; + if (errors.length) diagnostic(errors.join("\n")); + stdoutFile.close(); + stderrFile.close(); + } + function text(parts: Uint8Array[]) { + const decoder = new TextDecoder(); + return parts.map((part) => decoder.decode(part, { stream: true })).join( + "", + ) + + decoder.decode(); + } + const result: ExecutionResult = { + name: task.name, + code: cancelled + ? cancellationExitCode(abortSignal?.reason) + : timedOut + ? 124 + : errors.length + ? 1 + : code, + signal, + timedOut, + cancelled, + ...(cancelled ? { cancellationReason: String(abortSignal?.reason) } : {}), + ...(errors.length ? { error: errors.join("\n") } : {}), + seconds: (performance.now() - start) / 1000, + stdout: text(chunks.stdout), + stderr: text(chunks.stderr), + log: label, + timeoutMs, + ...(task.inventoryRef ? { inventoryRef: task.inventoryRef } : {}), + }; + // A per-command record also survives early metadata/build failures. + await Deno.writeTextFile( + `${output}/${label}.result.json`, + JSON.stringify(resultSummary(result), null, 2), + ); + return result; +} + +export function resultSummary( + { stdout: _stdout, stderr: _stderr, ...summary }: ExecutionResult, +) { + return summary; +} + +export interface CargoArtifact { + reason?: string; + package_id?: string; + executable?: string | null; + profile?: { test?: boolean }; + target?: { name: string }; +} + +export interface TestCommand { + name: string; + command: string; + args: string[]; + cwd: string; + env?: Record; + // Exact link from each execution result into inventory.json, even when two + // Cargo targets happen to share the same package/target display name. + inventoryRef?: { binaryIndex: number; shardIndex: number }; +} + +export function rustTestCommands( + artifacts: CargoArtifact[], + packages: { id: string; name: string; manifest_path: string }[], +): TestCommand[] { + const commands = new Map(); + for (const artifact of artifacts) { + if ( + artifact.reason !== "compiler-artifact" || !artifact.profile?.test || + !artifact.executable + ) continue; + const pkg = packages.find((pkg) => pkg.id === artifact.package_id); + if (!pkg || !artifact.target) { + throw new Error("test artifact has no Cargo package/target"); + } + const cwd = pkg.manifest_path.replace(/[/\\][^/\\]+$/, ""); + commands.set(artifact.executable, { + name: `${pkg.name}/${artifact.target.name}`, + command: artifact.executable, + args: [], + cwd, + env: { CARGO_MANIFEST_DIR: cwd }, + }); + } + if (!commands.size) throw new Error("Cargo produced no test executables"); + return [...commands.values()]; +} + +export async function mapConcurrent( + items: T[], + jobs: number, + run: (item: T, index: number) => Promise, +): Promise { + if (!Number.isInteger(jobs) || jobs < 1) { + throw new Error("jobs must be a positive integer"); + } + const results: R[] = new Array(items.length); + let next = 0; + // Drain every active worker even if one command fails to launch/write logs; + // callers must not remove signal handlers while detached groups are active. + const workers = await Promise.allSettled( + Array.from({ length: Math.min(jobs, items.length) }, async () => { + for (;;) { + const index = next++; + if (index >= items.length) return; + results[index] = await run(items[index], index); + } + }), + ); + const failure = workers.find((worker) => worker.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; + return results; +} + +function jsonLines(text: string): CargoArtifact[] { + return text.split("\n").filter((line) => line.startsWith("{")).map((line) => + JSON.parse(line) + ); +} + +async function main() { + const root = fileURLToPath(new URL("../", import.meta.url)).replace( + /[/\\]$/, + "", + ); + const jobs = parseJobs(Deno.args); + const output = `${root}/target/normal-tests/${ + new Date().toISOString().replaceAll(":", "-") + }`; + await Deno.mkdir(output, { recursive: true }); + const cancellation = installCancellationHandlers(); + const run = ( + task: TestCommand, + label: string, + timeoutMs = SUITE_TIMEOUT_MS, + ) => execute(task, { output, label, timeoutMs, signal: cancellation.signal }); + const summary: Record = { phase: "metadata", failed: false }; + let executionStart: number | undefined; + const writeSummary = () => + Deno.writeTextFile( + `${output}/summary.json`, + JSON.stringify( + { + jobs, + concurrency: + "top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism", + platform: "linux", + suiteTimeoutMs: SUITE_TIMEOUT_MS, + buildTimeoutMs: BUILD_TIMEOUT_MS, + ...summary, + }, + null, + 2, + ), + ); + console.log( + `Logs: ${output}; suite timeout: ${ + SUITE_TIMEOUT_MS / 1000 + }s; build timeout: ${ + BUILD_TIMEOUT_MS / 1000 + }s (not the 60s performance goal).`, + ); + try { + const metadata = await run({ + name: "metadata", + command: "cargo", + args: ["metadata", "--no-deps", "--format-version", "1"], + cwd: root, + }, "metadata"); + summary.preparation = [resultSummary(metadata)]; + if (metadata.code) { + Deno.exitCode = metadata.code; + throw new Error(`metadata failed: ${metadata.stderr}`); + } + summary.phase = "build"; + console.log( + "Building all normal Rust test targets (separate from execution timing)…", + ); + const build = await run( + { + name: "build", + command: "cargo", + args: [ + "test", + "--workspace", + "--exclude", + "yoi-e2e", + "--no-run", + "--message-format=json", + ], + cwd: root, + }, + "build", + BUILD_TIMEOUT_MS, + ); + console.log(`Build: ${build.seconds.toFixed(2)}s`); + summary.buildSeconds = build.seconds; + summary.preparation = [resultSummary(metadata), resultSummary(build)]; + if (build.code) { + Deno.exitCode = build.code; + throw new Error(`build failed: ${build.stderr}\n${build.stdout}`); + } + // Everything after compilation, including --list discovery, is execution time. + executionStart = performance.now(); + summary.phase = "discovery"; + const rust = rustTestCommands( + jsonLines(build.stdout), + JSON.parse(metadata.stdout).packages, + ); + // Cargo reports dependencies first; running that order leaves the largest + // application suites until the end. Use executable size only as a scheduling + // hint. Every discovered target still runs, with no cached success/skip state. + const sizes = new Map( + await Promise.all( + rust.map(async (task) => + [task.command, (await Deno.stat(task.command)).size] as const + ), + ), + ); + rust.sort((a, b) => + sizes.get(b.command)! - sizes.get(a.command)! || + a.name.localeCompare(b.name) + ); + summary.rustBinaries = rust.length; + console.log(`Listing all ${rust.length} Rust binaries before sharding…`); + const discoveryStart = performance.now(); + const discovery = await discoverRustCommands(rust, jobs, { + output, + signal: cancellation.signal, + }); + summary.discoverySeconds = (performance.now() - discoveryStart) / 1000; + summary.inventory = "inventory.json"; + summary.discoveryComplete = discovery.inventory.complete; + summary.selectedCases = discovery.inventory.selectedCases; + summary.discovery = discovery.inventory.binaries.map(( + { name, listing, error }, + ) => ({ name, listing, error })); + if (!discovery.inventory.complete) { + throw new Error( + "Rust discovery failed; see summary.json, inventory.json and list-*.stderr", + ); + } + summary.phase = "execution"; + const tasks: TestCommand[] = [ + { + name: "rust/doctests", + command: "cargo", + args: [ + "test", + "--workspace", + "--exclude", + "yoi-e2e", + "--doc", + "--no-fail-fast", + ], + cwd: root, + }, + { + name: "web/unit", + command: "deno", + args: ["task", "test"], + cwd: `${root}/web/workspace`, + }, + { + name: "web/component", + command: "deno", + args: ["task", "test:component"], + cwd: `${root}/web/workspace`, + }, + { + name: "web-ux/unit", + command: "deno", + args: ["task", "test"], + cwd: `${root}/tools/web-ux`, + }, + { + name: "tools/unit", + command: "deno", + args: [ + "test", + "--allow-read", + "--allow-write", + "--allow-run", + "--allow-env", + "tools/test.test.ts", + "tools/check-state-contract-names.test.ts", + ], + cwd: root, + }, + ...discovery.tasks, + ]; + console.log( + `Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`, + ); + const results: ReturnType[] = []; + summary.results = results; + await mapConcurrent(tasks, jobs, async (task, index) => { + const label = `${String(index).padStart(3, "0")}-${ + task.name.replaceAll(/[^\w.-]/g, "-") + }`; + const result = await run(task, label); + console.log( + `${result.code ? "FAIL" : "PASS"} ${task.name} (${ + result.seconds.toFixed(2) + }s)`, + ); + if (result.code) { + console.error(result.stdout.slice(-8000), result.stderr.slice(-8000)); + } + results[index] = resultSummary(result); + }); + const seconds = (performance.now() - executionStart) / 1000; + const failed = results.filter((result) => result.code !== 0); + summary.failed = failed.length > 0; + console.log( + `Execution: ${seconds.toFixed(2)}s; ${ + results.length - failed.length + } suites passed, ${failed.length} failed. Build: ${ + build.seconds.toFixed(2) + }s.`, + ); + if (failed.length) Deno.exitCode = 1; + } catch (error) { + summary.failed = true; + summary.error = String(error); + console.error(error); + if (!Deno.exitCode) Deno.exitCode = 1; + } finally { + if (executionStart !== undefined) { + summary.executionSeconds = (performance.now() - executionStart) / 1000; + } + try { + if (cancellation.signal.aborted) { + summary.cancelled = true; + summary.cancellationReason = String(cancellation.signal.reason); + summary.failed = true; + Deno.exitCode = cancellationExitCode(cancellation.signal.reason); + } + await writeSummary(); + } finally { + cancellation.dispose(); + } + } +} + +if (import.meta.main) await main();