feat(worker): separate restored candidates, repair, and activation
This commit is contained in:
+367
-2
@@ -6,6 +6,7 @@ import {
|
||||
mapConcurrent,
|
||||
parseJobs,
|
||||
parseRustListing,
|
||||
prepareTestIsolation,
|
||||
resultSummary,
|
||||
rustTestCommands,
|
||||
shardRustCommand,
|
||||
@@ -202,6 +203,39 @@ Deno.test("round-robin shards retain every listed name exactly once at threshold
|
||||
assertThrows(() => shardRustCommand(task, ["same", "same"]));
|
||||
});
|
||||
|
||||
Deno.test("global runtime-path packages serialize every normal target without losing names", () => {
|
||||
for (const pkg of ["worker", "worker-runtime", "yoi-workspace-server"]) {
|
||||
for (const target of ["lib", "controller_restore_test"]) {
|
||||
const task = {
|
||||
name: `${pkg}/${target}`,
|
||||
command: "/test-binary",
|
||||
args: [],
|
||||
cwd: "/pkg",
|
||||
};
|
||||
for (const count of [0, 1, 59, 72, 73, 1135]) {
|
||||
const names = listedNames(count);
|
||||
const shards = shardRustCommand(task, names);
|
||||
assert(
|
||||
shards.flatMap((shard) => shard.selectedNames).sort().join() ===
|
||||
[...names].sort().join(),
|
||||
);
|
||||
for (const shard of shards) {
|
||||
const args = shard.task.args;
|
||||
assert(args[args.indexOf("--test-threads") + 1] === "1");
|
||||
if (count > 72) {
|
||||
assert(args[0] === "--exact");
|
||||
assert(args.slice(3).join() === shard.selectedNames.join());
|
||||
} else {
|
||||
assert(args.join() === "--test-threads,1");
|
||||
}
|
||||
assert(!args.includes("--ignored") && !args.includes("--skip"));
|
||||
}
|
||||
}
|
||||
assert(task.args.length === 0, "do not mutate discovered commands");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("scheduler drains active workers before propagating a callback error", async () => {
|
||||
let drained = false;
|
||||
let failed = false;
|
||||
@@ -266,6 +300,276 @@ function emergencyKill(pid: number) {
|
||||
}
|
||||
}
|
||||
|
||||
async function assertRejects(
|
||||
run: () => Promise<unknown>,
|
||||
message: string,
|
||||
) {
|
||||
try {
|
||||
await run();
|
||||
} catch (error) {
|
||||
assert(String(error).includes(message), String(error));
|
||||
return;
|
||||
}
|
||||
throw new Error(`expected rejection containing ${message}`);
|
||||
}
|
||||
|
||||
Deno.test({
|
||||
name:
|
||||
"isolated real children use bounded disjoint tables and sanitized Git/compiler environments",
|
||||
ignore: Deno.build.os !== "linux",
|
||||
async fn() {
|
||||
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||
const environmentSnapshot = () =>
|
||||
JSON.stringify(
|
||||
Object.entries(process.env).sort(([a], [b]) => a.localeCompare(b)),
|
||||
);
|
||||
const before = environmentSnapshot();
|
||||
try {
|
||||
// An ancestor repository must not make an uninitialized tempfile look like
|
||||
// a repository. No developer repository or configuration is borrowed.
|
||||
for (const dir of [".git/objects", ".git/refs"]) {
|
||||
await Deno.mkdir(`${checkout}/${dir}`, { recursive: true });
|
||||
}
|
||||
await Deno.writeTextFile(
|
||||
`${checkout}/.git/HEAD`,
|
||||
"ref: refs/heads/main\n",
|
||||
);
|
||||
const [first, second] = await Promise.all([
|
||||
prepareTestIsolation(checkout, "same/output/id"),
|
||||
prepareTestIsolation(checkout, "same/output/id"),
|
||||
]);
|
||||
assert(
|
||||
first.root !== second.root,
|
||||
"concurrent runner ids must not collide",
|
||||
);
|
||||
const task = {
|
||||
...fixtureTask("isolation-env", checkout),
|
||||
env: {
|
||||
CARGO_MANIFEST_DIR: `${checkout}/crates/example`,
|
||||
CARGO_HOME: `${checkout}/compiler-cache`,
|
||||
RUSTUP_HOME: `${checkout}/toolchain-cache`,
|
||||
YOI_HOME: "/live/yoi",
|
||||
YOI_CONFIG_DIR: "/live/config",
|
||||
YOI_DATA_DIR: "/live/data",
|
||||
YOI_RUNTIME_DIR: "/live/run",
|
||||
HOME: "/live/home",
|
||||
TMPDIR: "/deleted/compiler-tmp",
|
||||
GIT_DIR: "/live/git",
|
||||
GIT_WORK_TREE: "/live/tree",
|
||||
GIT_CONFIG: "/live/gitconfig",
|
||||
GIT_CONFIG_GLOBAL: "/live/gitconfig",
|
||||
GIT_CONFIG_SYSTEM: "/live/system-gitconfig",
|
||||
GIT_TEMPLATE_DIR: "/live/template",
|
||||
GIT_CONFIG_PARAMETERS: "'core.hooksPath'='/live/hooks'",
|
||||
GIT_CONFIG_COUNT: "1",
|
||||
GIT_CONFIG_KEY_0: "core.hooksPath",
|
||||
GIT_CONFIG_VALUE_0: "/live/hooks",
|
||||
HTTP_PROXY: "http://live.invalid",
|
||||
https_proxy: "http://live.invalid",
|
||||
ALL_PROXY: "http://live.invalid",
|
||||
NO_PROXY: "live.invalid",
|
||||
RUSTC_WRAPPER: "/live/sccache",
|
||||
RUSTC_WORKSPACE_WRAPPER: "/live/wrapper",
|
||||
SCCACHE_DISABLE: "0",
|
||||
},
|
||||
};
|
||||
const taskBefore = JSON.stringify(task);
|
||||
const commands = await Promise.all([
|
||||
first.wrapCommand(task, "pkg/a"),
|
||||
first.wrapCommand(task, "pkg?a"), // same sanitized label
|
||||
first.wrapCommand(task, "pkg/a"), // same exact label
|
||||
second.wrapCommand(task, "pkg/a"),
|
||||
]);
|
||||
assert(JSON.stringify(task) === taskBefore);
|
||||
assert(
|
||||
environmentSnapshot() === before,
|
||||
"host environment changed",
|
||||
);
|
||||
const results = await Promise.all(
|
||||
commands.map((command, index) =>
|
||||
execute(command, {
|
||||
output: checkout,
|
||||
label: `environment-${index}`,
|
||||
timeoutMs: 10_000,
|
||||
})
|
||||
),
|
||||
);
|
||||
const tables = new Set<string>();
|
||||
for (const [index, result] of results.entries()) {
|
||||
assert(result.code === 0 && !result.error, JSON.stringify(result));
|
||||
const observed = JSON.parse(result.stdout);
|
||||
const env = observed.env as Record<string, string>;
|
||||
assert(observed.cwd === checkout && commands[index].cwd === task.cwd);
|
||||
assert(commands[index].args === task.args);
|
||||
assert(env.CARGO_MANIFEST_DIR === task.env.CARGO_MANIFEST_DIR);
|
||||
assert(env.CARGO_HOME === task.env.CARGO_HOME);
|
||||
assert(env.RUSTUP_HOME === task.env.RUSTUP_HOME);
|
||||
const namespace = env.YOI_RUNTIME_DIR.replace(/\/run$/, "");
|
||||
const prefix = `${checkout}/target/restore-test-environment/`;
|
||||
assert(namespace.startsWith(prefix));
|
||||
const suffix = namespace.slice(prefix.length);
|
||||
assert(
|
||||
suffix.split("/").every((part) => /^[a-zA-Z0-9_-]+$/.test(part)),
|
||||
);
|
||||
assert(env.YOI_CONFIG_DIR === `${namespace}/config`);
|
||||
assert(env.YOI_DATA_DIR === `${namespace}/data`);
|
||||
assert(
|
||||
env.YOI_HOME === env.YOI_DATA_DIR,
|
||||
"secret data must equal data",
|
||||
);
|
||||
assert(env.HOME === `${namespace}/home`);
|
||||
for (
|
||||
const dir of [
|
||||
namespace,
|
||||
env.YOI_CONFIG_DIR,
|
||||
env.YOI_DATA_DIR,
|
||||
env.YOI_RUNTIME_DIR,
|
||||
env.HOME,
|
||||
]
|
||||
) {
|
||||
assert((await Deno.lstat(dir)).isDirectory);
|
||||
assert(await Deno.realPath(dir) === dir);
|
||||
}
|
||||
for (const key of ["TMPDIR", "TMP", "TEMP"]) {
|
||||
assert(env[key] === `${checkout}/target/t`);
|
||||
}
|
||||
for (
|
||||
const key of [
|
||||
"GIT_DIR",
|
||||
"GIT_WORK_TREE",
|
||||
"GIT_CONFIG",
|
||||
"GIT_CONFIG_PARAMETERS",
|
||||
"GIT_CONFIG_COUNT",
|
||||
"GIT_CONFIG_KEY_0",
|
||||
"GIT_CONFIG_VALUE_0",
|
||||
"HTTP_PROXY",
|
||||
"https_proxy",
|
||||
"ALL_PROXY",
|
||||
"NO_PROXY",
|
||||
]
|
||||
) {
|
||||
assert(!(key in env), `${key} leaked to child`);
|
||||
}
|
||||
assert(env.GIT_CONFIG_GLOBAL === "/dev/null");
|
||||
assert(env.GIT_CONFIG_SYSTEM === "/dev/null");
|
||||
assert(env.GIT_CONFIG_NOSYSTEM === "1");
|
||||
assert(env.GIT_CEILING_DIRECTORIES === checkout);
|
||||
assert(env.GIT_TEMPLATE_DIR === `${namespace}/git-template`);
|
||||
assert(observed.globalConfig === "");
|
||||
assert(observed.ancestorDiscovered === false);
|
||||
assert(observed.hooks.length === 0 && observed.info.length === 0);
|
||||
assert(observed.author.includes("normal-tests@example.invalid"));
|
||||
assert(env.GIT_AUTHOR_NAME === env.GIT_COMMITTER_NAME);
|
||||
assert(env.GIT_AUTHOR_EMAIL === env.GIT_COMMITTER_EMAIL);
|
||||
assert(env.RUSTC_WRAPPER === "" && env.RUSTC_WORKSPACE_WRAPPER === "");
|
||||
assert(env.SCCACHE_DISABLE === "1");
|
||||
const table = `${env.YOI_RUNTIME_DIR}/workers.json`;
|
||||
tables.add(table);
|
||||
assert(await Deno.readTextFile(table) === String(observed.pid));
|
||||
}
|
||||
assert(
|
||||
tables.size === commands.length,
|
||||
"independent commands shared a table",
|
||||
);
|
||||
assert(environmentSnapshot() === before);
|
||||
} finally {
|
||||
await Deno.remove(checkout, { recursive: true });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
Deno.test("isolation refuses lexical aliases and symlinked owned paths before external writes", async () => {
|
||||
for (
|
||||
const unsafe of ["target", "target/restore-test-environment", "target/t"]
|
||||
) {
|
||||
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||
const outside = await Deno.realPath(await Deno.makeTempDir());
|
||||
try {
|
||||
await Deno.writeTextFile(`${outside}/sentinel`, "untouched");
|
||||
if (unsafe !== "target") await Deno.mkdir(`${checkout}/target`);
|
||||
await Deno.symlink(outside, `${checkout}/${unsafe}`);
|
||||
await assertRejects(
|
||||
() => prepareTestIsolation(checkout, "run"),
|
||||
"symlink",
|
||||
);
|
||||
assert(
|
||||
Array.from(Deno.readDirSync(outside)).map((entry) => entry.name)
|
||||
.join() ===
|
||||
"sentinel",
|
||||
);
|
||||
assert(await Deno.readTextFile(`${outside}/sentinel`) === "untouched");
|
||||
} finally {
|
||||
await Deno.remove(checkout, { recursive: true });
|
||||
await Deno.remove(outside, { recursive: true });
|
||||
}
|
||||
}
|
||||
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||
try {
|
||||
await Deno.symlink(checkout, `${checkout}/alias`);
|
||||
await assertRejects(
|
||||
() => prepareTestIsolation(`${checkout}/alias`, "run"),
|
||||
"symlink",
|
||||
);
|
||||
for (const suffix of ["/../escape", "/./child", "//child"]) {
|
||||
await assertRejects(
|
||||
() => prepareTestIsolation(`${checkout}${suffix}`, "run"),
|
||||
"normal-only",
|
||||
);
|
||||
}
|
||||
assert(
|
||||
Array.from(Deno.readDirSync(checkout)).map((entry) => entry.name)
|
||||
.join() ===
|
||||
"alias",
|
||||
);
|
||||
} finally {
|
||||
await Deno.remove(checkout, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("command wrapping rechecks owned ancestors and refuses symlinked namespace children", async () => {
|
||||
for (
|
||||
const unsafe of [
|
||||
"ancestor",
|
||||
"tmp",
|
||||
"config",
|
||||
"data",
|
||||
"run",
|
||||
"home",
|
||||
"git-template",
|
||||
"git-template/info",
|
||||
"git-template/hooks",
|
||||
]
|
||||
) {
|
||||
const checkout = await Deno.realPath(await Deno.makeTempDir());
|
||||
const outside = await Deno.realPath(await Deno.makeTempDir());
|
||||
try {
|
||||
const isolation = await prepareTestIsolation(checkout, "run");
|
||||
let path: string;
|
||||
if (unsafe === "ancestor") {
|
||||
path = isolation.root;
|
||||
await Deno.remove(path);
|
||||
} else if (unsafe === "tmp") {
|
||||
path = `${checkout}/target/t`;
|
||||
await Deno.remove(path);
|
||||
} else {
|
||||
const namespace = `${isolation.root}/unsafe-0`;
|
||||
await Deno.mkdir(namespace);
|
||||
if (unsafe.includes("/")) await Deno.mkdir(`${namespace}/git-template`);
|
||||
path = `${namespace}/${unsafe}`;
|
||||
}
|
||||
await Deno.symlink(outside, path);
|
||||
await assertRejects(
|
||||
() => isolation.wrapCommand(fixtureTask("success", checkout), "unsafe"),
|
||||
"symlink",
|
||||
);
|
||||
assert(Array.from(Deno.readDirSync(outside)).length === 0);
|
||||
} finally {
|
||||
await Deno.remove(checkout, { recursive: true });
|
||||
await Deno.remove(outside, { recursive: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) {
|
||||
Deno.test({
|
||||
name: `execute streams real ${mode} child output and records its exit`,
|
||||
@@ -436,15 +740,28 @@ Deno.test({
|
||||
"runtime listing persists complete name inventory for large and small binaries",
|
||||
ignore: Deno.build.os !== "linux",
|
||||
async fn() {
|
||||
const output = await Deno.makeTempDir();
|
||||
const output = await Deno.realPath(await Deno.makeTempDir());
|
||||
try {
|
||||
const isolation = await prepareTestIsolation(output, "discovery");
|
||||
const wrappedTables = new Set<string>();
|
||||
const discovery = await discoverRustCommands(
|
||||
[
|
||||
fixtureTask("list-large", output),
|
||||
fixtureTask("list-small", output),
|
||||
],
|
||||
2,
|
||||
{ output },
|
||||
{
|
||||
output,
|
||||
run: async (task, label) => {
|
||||
const wrapped = await isolation.wrapCommand(task, label);
|
||||
wrappedTables.add(`${wrapped.env!.YOI_RUNTIME_DIR}/workers.json`);
|
||||
return await execute(wrapped, { output, label, timeoutMs: 10_000 });
|
||||
},
|
||||
},
|
||||
);
|
||||
assert(
|
||||
wrappedTables.size === 2,
|
||||
"all discovery launches must use the callback",
|
||||
);
|
||||
assert(discovery.inventory.complete);
|
||||
assert(
|
||||
@@ -635,6 +952,54 @@ if (import.meta.main) {
|
||||
);
|
||||
}
|
||||
Deno.exit(mode === "list-nonzero" ? 7 : 0);
|
||||
} else if (mode === "isolation-env") {
|
||||
const env = Deno.env.toObject();
|
||||
const scratch = await Deno.makeTempDir({ dir: env.TMPDIR });
|
||||
const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes);
|
||||
async function git(args: string[]) {
|
||||
return await new Deno.Command("git", {
|
||||
args,
|
||||
cwd: scratch,
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
}).output();
|
||||
}
|
||||
try {
|
||||
const ownedRepo = await git([
|
||||
"-C",
|
||||
output,
|
||||
"rev-parse",
|
||||
"--show-toplevel",
|
||||
]);
|
||||
assert(
|
||||
ownedRepo.success && text(ownedRepo.stdout).trim() === output,
|
||||
"owned ancestor repository fixture must be discoverable at its root",
|
||||
);
|
||||
const ancestor = await git(["rev-parse", "--show-toplevel"]);
|
||||
const init = await git(["init", "--quiet"]);
|
||||
assert(init.success, text(init.stderr));
|
||||
const global = await git(["config", "--global", "--list"]);
|
||||
assert(global.success, text(global.stderr));
|
||||
const author = await git(["var", "GIT_AUTHOR_IDENT"]);
|
||||
assert(author.success, text(author.stderr));
|
||||
await Deno.writeTextFile(
|
||||
`${env.YOI_RUNTIME_DIR}/workers.json`,
|
||||
String(Deno.pid),
|
||||
);
|
||||
console.log(JSON.stringify({
|
||||
env,
|
||||
cwd: Deno.cwd(),
|
||||
pid: Deno.pid,
|
||||
ancestorDiscovered: ancestor.success,
|
||||
globalConfig: text(global.stdout),
|
||||
author: text(author.stdout),
|
||||
hooks: Array.from(Deno.readDirSync(`${scratch}/.git/hooks`)),
|
||||
info: Array.from(Deno.readDirSync(`${scratch}/.git/info`)),
|
||||
}));
|
||||
} finally {
|
||||
await Deno.remove(scratch, { recursive: true });
|
||||
}
|
||||
Deno.exit(0);
|
||||
} else if (mode === "runner-parent") {
|
||||
const cancellation = installCancellationHandlers();
|
||||
try {
|
||||
|
||||
+197
-19
@@ -1,9 +1,10 @@
|
||||
// Bound top-level suites/shards, not total processes; large shards use two test threads.
|
||||
// Bound top-level suites/shards, not total processes; large shards normally use two threads.
|
||||
// Cargo selects targets/features. Retain all listed names; never skip cached successes.
|
||||
import { spawn } from "node:child_process";
|
||||
import process from "node:process";
|
||||
import { availableParallelism } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { posix } from "node:path";
|
||||
|
||||
// Safety limits, NOT the separate 60s performance goal.
|
||||
export const SUITE_TIMEOUT_MS = 180_000;
|
||||
@@ -65,7 +66,19 @@ export function shardRustCommand(
|
||||
if (new Set(names).size !== names.length) {
|
||||
throw new Error("cannot shard duplicate test names");
|
||||
}
|
||||
if (names.length <= 72) return [{ task, selectedNames: [...names] }];
|
||||
// RuntimeDirSandbox changes process-global paths. Do not overlap its writes
|
||||
// with restore guards in the same process, including small integration targets.
|
||||
const serial = ["worker", "worker-runtime", "yoi-workspace-server"].includes(
|
||||
task.name.split("/")[0],
|
||||
);
|
||||
if (names.length <= 72) {
|
||||
return [{
|
||||
task: serial
|
||||
? { ...task, args: [...task.args, "--test-threads", "1"] }
|
||||
: task,
|
||||
selectedNames: [...names],
|
||||
}];
|
||||
}
|
||||
const count = Math.min(32, Math.ceil(names.length / 36));
|
||||
const shards: string[][] = Array.from({ length: count }, () => []);
|
||||
names.forEach((name, index) => shards[index % count].push(name));
|
||||
@@ -75,7 +88,13 @@ export function shardRustCommand(
|
||||
name: `${task.name}/shard-${
|
||||
String(index + 1).padStart(2, "0")
|
||||
}-of-${count}`,
|
||||
args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames],
|
||||
args: [
|
||||
...task.args,
|
||||
"--exact",
|
||||
"--test-threads",
|
||||
serial ? "1" : "2",
|
||||
...selectedNames,
|
||||
],
|
||||
},
|
||||
selectedNames,
|
||||
}));
|
||||
@@ -84,18 +103,27 @@ export function shardRustCommand(
|
||||
export async function discoverRustCommands(
|
||||
commands: TestCommand[],
|
||||
jobs: number,
|
||||
options: { output: string; signal?: AbortSignal },
|
||||
options: {
|
||||
output: string;
|
||||
signal?: AbortSignal;
|
||||
run?: (task: TestCommand, label: string) => Promise<ExecutionResult>;
|
||||
},
|
||||
) {
|
||||
const binaries = await mapConcurrent(commands, jobs, async (task, index) => {
|
||||
const listing = await execute({
|
||||
const listingTask = {
|
||||
...task,
|
||||
name: `${task.name}/list`,
|
||||
args: [...task.args, "--list", "--format", "terse"],
|
||||
}, {
|
||||
...options,
|
||||
label: `list-${String(index).padStart(3, "0")}`,
|
||||
timeoutMs: SUITE_TIMEOUT_MS,
|
||||
});
|
||||
};
|
||||
const label = `list-${String(index).padStart(3, "0")}`;
|
||||
const listing = await (options.run
|
||||
? options.run(listingTask, label)
|
||||
: execute(listingTask, {
|
||||
output: options.output,
|
||||
signal: options.signal,
|
||||
label,
|
||||
timeoutMs: SUITE_TIMEOUT_MS,
|
||||
}));
|
||||
let shards: RustShard[] = [];
|
||||
let error: string | undefined;
|
||||
try {
|
||||
@@ -124,7 +152,8 @@ export async function discoverRustCommands(
|
||||
error,
|
||||
selectedCases: error
|
||||
? null
|
||||
: shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0),
|
||||
: shards.reduce((sum, shard) =>
|
||||
sum + shard.selectedNames.length, 0),
|
||||
shards,
|
||||
};
|
||||
});
|
||||
@@ -426,12 +455,149 @@ export interface TestCommand {
|
||||
command: string;
|
||||
args: string[];
|
||||
cwd: string;
|
||||
env?: Record<string, string>;
|
||||
env?: Record<string, string | undefined>;
|
||||
// Exact link from each execution result into inventory.json, even when two
|
||||
// Cargo targets happen to share the same package/target display name.
|
||||
inventoryRef?: { binaryIndex: number; shardIndex: number };
|
||||
}
|
||||
|
||||
// Walk with lstat before creating any child: recursive mkdir/realPath alone
|
||||
// would follow an existing symlink first. Reject aliases rather than silently
|
||||
// canonicalizing a caller's checkout. As with the Rust guard, this assumes the
|
||||
// owned checkout is not concurrently replaced by an adversarial filesystem actor.
|
||||
async function ownedTestDirectory(checkout: string, relative: string) {
|
||||
if (
|
||||
!posix.isAbsolute(checkout) || checkout === "/" ||
|
||||
posix.normalize(checkout) !== checkout
|
||||
) throw new Error("test checkout must be an absolute normal-only path");
|
||||
|
||||
async function checkDirectory(path: string, owned: boolean) {
|
||||
const stat = await Deno.lstat(path);
|
||||
if (!stat.isDirectory || stat.isSymlink) {
|
||||
throw new Error(
|
||||
`test isolation refuses non-directory or symlink: ${path}`,
|
||||
);
|
||||
}
|
||||
if (owned && process.getuid && stat.uid !== process.getuid()) {
|
||||
throw new Error(
|
||||
`test isolation directory is not owned by this user: ${path}`,
|
||||
);
|
||||
}
|
||||
if (await Deno.realPath(path) !== path) {
|
||||
throw new Error(
|
||||
`test isolation refuses non-canonical directory: ${path}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
let checkoutPath = "";
|
||||
for (const part of checkout.slice(1).split("/")) {
|
||||
checkoutPath += `/${part}`;
|
||||
await checkDirectory(checkoutPath, false);
|
||||
}
|
||||
let path = checkout;
|
||||
for (const part of relative.split("/")) {
|
||||
if (!part || part === "." || part === "..") {
|
||||
throw new Error("invalid owned test directory component");
|
||||
}
|
||||
path += `/${part}`;
|
||||
try {
|
||||
await Deno.mkdir(path, { mode: 0o700 });
|
||||
} catch (error) {
|
||||
if (!(error instanceof Deno.errors.AlreadyExists)) throw error;
|
||||
}
|
||||
await checkDirectory(path, true);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Prepare child-only isolation for the maintained normal-test entrypoint.
|
||||
* Each wrap gets a fresh allocation table, even for repeated/sanitized labels or
|
||||
* concurrent runners using the same output id. Resources stay under target for
|
||||
* diagnosis; callers with temporary checkouts own their recursive cleanup.
|
||||
*/
|
||||
export async function prepareTestIsolation(checkout: string, runId: string) {
|
||||
const ownedDirectory = (relative: string) =>
|
||||
ownedTestDirectory(checkout, relative);
|
||||
const prefix = "target/restore-test-environment";
|
||||
await ownedDirectory(prefix);
|
||||
// Keep tempfile Unix-socket references short. tempfile owns unique resources
|
||||
// here; neither runners nor suites remove/replace this shared directory.
|
||||
const tmp = await ownedDirectory("target/t");
|
||||
const digest = new Uint8Array(
|
||||
await crypto.subtle.digest("SHA-256", new TextEncoder().encode(runId)),
|
||||
);
|
||||
const id = Array.from(
|
||||
digest.slice(0, 6),
|
||||
(byte) => byte.toString(16).padStart(2, "0"),
|
||||
).join("");
|
||||
const relativeRoot = `${prefix}/runner-${id}-${crypto.randomUUID()}`;
|
||||
const root = await ownedDirectory(relativeRoot);
|
||||
let next = 0;
|
||||
return {
|
||||
root,
|
||||
async wrapCommand(task: TestCommand, label: string): Promise<TestCommand> {
|
||||
const safeLabel = label.replaceAll(/[^a-zA-Z0-9_-]/g, "-").slice(0, 40) ||
|
||||
"command";
|
||||
const relative = `${relativeRoot}/${safeLabel}-${next++}`;
|
||||
// Recheck shared ancestors as well as the command namespace on every wrap.
|
||||
await ownedDirectory("target/t");
|
||||
const namespace = await ownedDirectory(relative);
|
||||
for (
|
||||
const dir of [
|
||||
"config",
|
||||
"data",
|
||||
"run",
|
||||
"home",
|
||||
"git-template/info",
|
||||
"git-template/hooks",
|
||||
]
|
||||
) {
|
||||
await ownedDirectory(`${relative}/${dir}`);
|
||||
}
|
||||
const inherited = { ...process.env, ...task.env };
|
||||
const env: NonNullable<TestCommand["env"]> = { ...task.env };
|
||||
// Undefined shadows inherited entries in execute's merge; node spawn omits
|
||||
// them. Strip indexed Git config and less common Git path overrides too.
|
||||
for (const key of Object.keys(inherited)) {
|
||||
if (/^GIT_/i.test(key) || /(^|_)proxy$/i.test(key)) {
|
||||
env[key] = undefined;
|
||||
}
|
||||
}
|
||||
Object.assign(env, {
|
||||
HOME: `${namespace}/home`,
|
||||
YOI_HOME: `${namespace}/data`,
|
||||
YOI_CONFIG_DIR: `${namespace}/config`,
|
||||
YOI_DATA_DIR: `${namespace}/data`,
|
||||
YOI_RUNTIME_DIR: `${namespace}/run`,
|
||||
TMPDIR: tmp,
|
||||
TMP: tmp,
|
||||
TEMP: tmp,
|
||||
GIT_TEMPLATE_DIR: `${namespace}/git-template`,
|
||||
GIT_CONFIG_GLOBAL: "/dev/null",
|
||||
GIT_CONFIG_SYSTEM: "/dev/null",
|
||||
GIT_CONFIG_NOSYSTEM: "1",
|
||||
GIT_ATTR_NOSYSTEM: "1",
|
||||
GIT_CEILING_DIRECTORIES: checkout,
|
||||
GIT_AUTHOR_NAME: "Normal test runner",
|
||||
GIT_AUTHOR_EMAIL: "normal-tests@example.invalid",
|
||||
GIT_COMMITTER_NAME: "Normal test runner",
|
||||
GIT_COMMITTER_EMAIL: "normal-tests@example.invalid",
|
||||
RUSTC_WRAPPER: "",
|
||||
RUSTC_WORKSPACE_WRAPPER: "",
|
||||
SCCACHE_DISABLE: "1",
|
||||
});
|
||||
// HOME is isolated, but toolchain/cache discovery must keep its ordinary
|
||||
// host locations (these are compiler settings, not live product state).
|
||||
for (const key of ["CARGO_HOME", "RUSTUP_HOME"] as const) {
|
||||
const fallback = key === "CARGO_HOME" ? ".cargo" : ".rustup";
|
||||
env[key] = inherited[key] ||
|
||||
(process.env.HOME ? `${process.env.HOME}/${fallback}` : undefined);
|
||||
}
|
||||
return { ...task, env };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function rustTestCommands(
|
||||
artifacts: CargoArtifact[],
|
||||
packages: { id: string; name: string; manifest_path: string }[],
|
||||
@@ -497,16 +663,26 @@ async function main() {
|
||||
"",
|
||||
);
|
||||
const jobs = parseJobs(Deno.args);
|
||||
const output = `${root}/target/normal-tests/${
|
||||
const outputId = `${
|
||||
new Date().toISOString().replaceAll(":", "-")
|
||||
}`;
|
||||
await Deno.mkdir(output, { recursive: true });
|
||||
}-${crypto.randomUUID()}`;
|
||||
const isolation = await prepareTestIsolation(root, outputId);
|
||||
const output = await ownedTestDirectory(
|
||||
root,
|
||||
`target/normal-tests/${outputId}`,
|
||||
);
|
||||
const cancellation = installCancellationHandlers();
|
||||
const run = (
|
||||
const run = async (
|
||||
task: TestCommand,
|
||||
label: string,
|
||||
timeoutMs = SUITE_TIMEOUT_MS,
|
||||
) => execute(task, { output, label, timeoutMs, signal: cancellation.signal });
|
||||
) =>
|
||||
execute(await isolation.wrapCommand(task, label), {
|
||||
output,
|
||||
label,
|
||||
timeoutMs,
|
||||
signal: cancellation.signal,
|
||||
});
|
||||
const summary: Record<string, unknown> = { phase: "metadata", failed: false };
|
||||
let executionStart: number | undefined;
|
||||
const writeSummary = () =>
|
||||
@@ -516,7 +692,7 @@ async function main() {
|
||||
{
|
||||
jobs,
|
||||
concurrency:
|
||||
"top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism",
|
||||
"top-level suites/shards; worker, worker-runtime and yoi-workspace-server use 1 test thread; other large Rust shards use 2, other suites retain internal parallelism",
|
||||
platform: "linux",
|
||||
suiteTimeoutMs: SUITE_TIMEOUT_MS,
|
||||
buildTimeoutMs: BUILD_TIMEOUT_MS,
|
||||
@@ -600,6 +776,7 @@ async function main() {
|
||||
const discovery = await discoverRustCommands(rust, jobs, {
|
||||
output,
|
||||
signal: cancellation.signal,
|
||||
run,
|
||||
});
|
||||
summary.discoverySeconds = (performance.now() - discoveryStart) / 1000;
|
||||
summary.inventory = "inventory.json";
|
||||
@@ -655,6 +832,7 @@ async function main() {
|
||||
"--allow-write",
|
||||
"--allow-run",
|
||||
"--allow-env",
|
||||
"--allow-sys=uid",
|
||||
"tools/test.test.ts",
|
||||
"tools/check-state-contract-names.test.ts",
|
||||
],
|
||||
@@ -663,7 +841,7 @@ async function main() {
|
||||
...discovery.tasks,
|
||||
];
|
||||
console.log(
|
||||
`Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
|
||||
`Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (worker/worker-runtime/yoi-workspace-server: 1 test thread; other large Rust shards: 2; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
|
||||
);
|
||||
const results: ReturnType<typeof resultSummary>[] = [];
|
||||
summary.results = results;
|
||||
|
||||
Reference in New Issue
Block a user