From 9dccf99c50d331cf65f503a0db96b81faea195e0 Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 11 Aug 2026 04:03:54 +0900 Subject: [PATCH 01/12] ticket: own typed schema migrations --- crates/ticket/src/lib.rs | 194 +--- crates/ticket/src/sqlite_schema.rs | 1077 ++++++++++++++++++++++ crates/workspace-server/src/authority.rs | 5 +- crates/workspace-server/src/server.rs | 9 +- crates/workspace-server/src/store.rs | 40 + crates/yoi/src/ticket_cli.rs | 4 +- 6 files changed, 1171 insertions(+), 158 deletions(-) create mode 100644 crates/ticket/src/sqlite_schema.rs diff --git a/crates/ticket/src/lib.rs b/crates/ticket/src/lib.rs index e613a841..29f6f244 100644 --- a/crates/ticket/src/lib.rs +++ b/crates/ticket/src/lib.rs @@ -20,8 +20,13 @@ use serde_yaml::{Mapping as YamlMapping, Value as YamlValue}; use thiserror::Error; pub mod config; +mod sqlite_schema; pub mod tool; +pub use sqlite_schema::{ + LATEST_SQLITE_TICKET_SCHEMA_VERSION, migrate_sqlite_ticket_schema, verify_sqlite_ticket_schema, +}; + const REQUIRED_FIELDS: [&str; 4] = ["title", "state", "created_at", "updated_at"]; const MAX_STATE_CHANGE_REASON_BYTES: usize = 1024; const MAX_INTAKE_SUMMARY_BODY_BYTES: usize = 16 * 1024; @@ -2301,7 +2306,7 @@ impl fmt::Debug for SqliteTicketBackend { } impl SqliteTicketBackend { - pub fn new(db_path: impl Into, workspace_id: impl Into) -> Self { + fn configured(db_path: impl Into, workspace_id: impl Into) -> Self { Self { db_path: db_path.into(), workspace_id: workspace_id.into(), @@ -2311,6 +2316,27 @@ impl SqliteTicketBackend { } } + /// Opens a standalone Ticket backend, applying all Ticket-owned migrations once. + pub fn open(db_path: impl Into, workspace_id: impl Into) -> Result { + let backend = Self::configured(db_path, workspace_id); + let connection = backend.connect()?; + migrate_sqlite_ticket_schema(&connection)?; + Ok(backend) + } + + /// Connects to a database whose Ticket schema was composed by its startup owner. + /// + /// This performs verification only and never creates or alters schema objects. + pub fn open_verified( + db_path: impl Into, + workspace_id: impl Into, + ) -> Result { + let backend = Self::configured(db_path, workspace_id); + let connection = backend.connect()?; + verify_sqlite_ticket_schema(&connection)?; + Ok(backend) + } + pub fn with_event_attributes(mut self, attributes: BTreeMap) -> Self { self.event_attributes = attributes; self @@ -2338,7 +2364,6 @@ impl SqliteTicketBackend { pub fn import_from_local_backend(&self, local: &LocalTicketBackend) -> Result<()> { let conn = self.open_connection()?; - self.ensure_schema(&conn)?; conn.execute_batch("BEGIN IMMEDIATE").map_err(sqlite_err)?; let result = (|| { for summary in local.list(TicketListQuery::all())? { @@ -2351,7 +2376,7 @@ impl SqliteTicketBackend { finish_sqlite_transaction(&conn, result) } - fn open_connection(&self) -> Result { + fn connect(&self) -> Result { if let Some(parent) = self.db_path.parent() { fs::create_dir_all(parent).map_err(|error| io_err(parent, error))?; } @@ -2361,115 +2386,20 @@ impl SqliteTicketBackend { Ok(conn) } - fn ensure_schema(&self, conn: &Connection) -> Result<()> { - conn.execute_batch(r#" -CREATE TABLE IF NOT EXISTS typed_tickets ( - workspace_id TEXT NOT NULL, - ticket_id TEXT NOT NULL, - slug TEXT NOT NULL, - title TEXT NOT NULL, - status TEXT NOT NULL, - kind TEXT NOT NULL, - priority TEXT NOT NULL, - body TEXT NOT NULL, - created_at TEXT, - updated_at TEXT, - assignee TEXT, - readiness TEXT, - workflow_state TEXT NOT NULL, - workflow_state_explicit INTEGER NOT NULL, - queued_by TEXT, - queued_at TEXT, - resolution TEXT, - repository_id TEXT, - ref_selector TEXT, - PRIMARY KEY (workspace_id, ticket_id) -); -CREATE TABLE IF NOT EXISTS typed_ticket_labels ( - workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, ordinal INTEGER NOT NULL, label TEXT NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, ordinal), - FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS typed_ticket_risk_flags ( - workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, ordinal INTEGER NOT NULL, risk_flag TEXT NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, ordinal), - FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS typed_ticket_raw_frontmatter ( - workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, key), - FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS typed_ticket_events ( - workspace_id TEXT NOT NULL, - ticket_id TEXT NOT NULL, - event_index INTEGER NOT NULL, - kind TEXT NOT NULL, - author TEXT, - at TEXT, - status TEXT, - from_state TEXT, - to_state TEXT, - reason TEXT, - state_field TEXT, - heading TEXT, - body TEXT NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, event_index), - FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS typed_ticket_event_references ( - workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, event_index INTEGER NOT NULL, ordinal INTEGER NOT NULL, kind TEXT NOT NULL, target TEXT NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, event_index, ordinal), - FOREIGN KEY (workspace_id, ticket_id, event_index) REFERENCES typed_ticket_events(workspace_id, ticket_id, event_index) ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS typed_ticket_event_attributes ( - workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, event_index INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, event_index, key), - FOREIGN KEY (workspace_id, ticket_id, event_index) REFERENCES typed_ticket_events(workspace_id, ticket_id, event_index) ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS typed_ticket_relations ( - workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, kind TEXT NOT NULL, target TEXT NOT NULL, note TEXT, author TEXT NOT NULL, at TEXT NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, kind, target), - FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS typed_ticket_orchestration_plans ( - workspace_id TEXT NOT NULL, - ticket_id TEXT NOT NULL, - record_id TEXT NOT NULL, - kind TEXT NOT NULL, - related_ticket TEXT, - note TEXT, - accepted_summary TEXT, - accepted_branch TEXT, - accepted_worktree TEXT, - accepted_role_plan TEXT, - author TEXT NOT NULL, - at TEXT NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, record_id), - FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS typed_ticket_artifacts ( - workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, relative_path TEXT NOT NULL, content BLOB NOT NULL, - PRIMARY KEY (workspace_id, ticket_id, relative_path), - FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE -); -"#) - .map_err(sqlite_err)?; - ensure_sqlite_ticket_column(conn, "repository_id", "TEXT")?; - ensure_sqlite_ticket_column(conn, "ref_selector", "TEXT")?; - Ok(()) + fn open_connection(&self) -> Result { + let connection = self.connect()?; + verify_sqlite_ticket_schema(&connection)?; + Ok(connection) } fn with_write(&self, op: impl FnOnce(&Connection) -> Result) -> Result { let conn = self.open_connection()?; - self.ensure_schema(&conn)?; conn.execute_batch("BEGIN IMMEDIATE").map_err(sqlite_err)?; finish_sqlite_transaction(&conn, op(&conn)) } fn with_read(&self, op: impl FnOnce(&Connection) -> Result) -> Result { let conn = self.open_connection()?; - self.ensure_schema(&conn)?; op(&conn) } @@ -2915,30 +2845,6 @@ CREATE TABLE IF NOT EXISTS typed_ticket_artifacts ( } } -fn ensure_sqlite_ticket_column( - conn: &rusqlite::Connection, - name: &str, - sql_type: &str, -) -> Result<()> { - let mut statement = conn - .prepare("PRAGMA table_info(typed_tickets)") - .map_err(sqlite_err)?; - let columns = statement - .query_map([], |row| row.get::<_, String>(1)) - .map_err(sqlite_err)?; - for column in columns { - if column.map_err(sqlite_err)? == name { - return Ok(()); - } - } - conn.execute( - format!("ALTER TABLE typed_tickets ADD COLUMN {name} {sql_type}").as_str(), - [], - ) - .map_err(sqlite_err)?; - Ok(()) -} - fn finish_sqlite_transaction(conn: &Connection, result: Result) -> Result { match result { Ok(output) => { @@ -6406,28 +6312,11 @@ state: planning #[test] fn sqlite_backend_persists_and_edits_ticket_target() { let tmp = TempDir::new().unwrap(); - let backend = SqliteTicketBackend::new(tmp.path().join("workspace.db"), "workspace-test"); + let backend = + SqliteTicketBackend::open(tmp.path().join("workspace.db"), "workspace-test").unwrap(); assert_ticket_target_edit_semantics(&backend); } - #[test] - fn sqlite_ticket_target_columns_are_added_to_existing_table() { - let tmp = TempDir::new().unwrap(); - let conn = rusqlite::Connection::open(tmp.path().join("workspace.db")).unwrap(); - conn.execute_batch("CREATE TABLE typed_tickets (ticket_id TEXT PRIMARY KEY);") - .unwrap(); - ensure_sqlite_ticket_column(&conn, "repository_id", "TEXT").unwrap(); - ensure_sqlite_ticket_column(&conn, "ref_selector", "TEXT").unwrap(); - let mut statement = conn.prepare("PRAGMA table_info(typed_tickets)").unwrap(); - let columns = statement - .query_map([], |row| row.get::<_, String>(1)) - .unwrap() - .collect::, _>>() - .unwrap(); - assert!(columns.iter().any(|column| column == "repository_id")); - assert!(columns.iter().any(|column| column == "ref_selector")); - } - #[test] fn local_backend_edit_item_supports_partial_body_replacement() { let tmp = TempDir::new().unwrap(); @@ -6438,7 +6327,8 @@ state: planning #[test] fn sqlite_backend_edit_item_supports_partial_body_replacement() { let tmp = TempDir::new().unwrap(); - let backend = SqliteTicketBackend::new(tmp.path().join("workspace.db"), "workspace-test"); + let backend = + SqliteTicketBackend::open(tmp.path().join("workspace.db"), "workspace-test").unwrap(); assert_partial_body_replacement_semantics(&backend); } @@ -6446,7 +6336,7 @@ state: planning fn sqlite_mutation_hook_failure_rolls_back_ticket_event() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join("workspace.db"); - let backend = SqliteTicketBackend::new(&db_path, "workspace-test"); + let backend = SqliteTicketBackend::open(&db_path, "workspace-test").unwrap(); let created = backend.create(NewTicket::new("Atomic mutation")).unwrap(); let before = backend .show(TicketIdOrSlug::Id(created.id.clone())) @@ -6480,7 +6370,8 @@ state: planning #[test] fn sqlite_backend_persists_core_ticket_operations() { let tmp = TempDir::new().unwrap(); - let backend = SqliteTicketBackend::new(tmp.path().join("workspace.db"), "workspace-test"); + let backend = + SqliteTicketBackend::open(tmp.path().join("workspace.db"), "workspace-test").unwrap(); let created = backend.create(NewTicket::new("SQLite Ticket")).unwrap(); backend .add_event( @@ -6501,7 +6392,9 @@ state: planning ) .unwrap(); - let reopened = SqliteTicketBackend::new(tmp.path().join("workspace.db"), "workspace-test"); + let reopened = + SqliteTicketBackend::open_verified(tmp.path().join("workspace.db"), "workspace-test") + .unwrap(); let list = reopened.list(TicketListQuery::all()).unwrap(); assert_eq!(list.len(), 1); assert_eq!(list[0].id, created.id); @@ -6531,7 +6424,8 @@ state: planning let tmp = TempDir::new().unwrap(); let local = backend(&tmp); let created = local.create(NewTicket::new("Legacy Ticket")).unwrap(); - let db = SqliteTicketBackend::new(tmp.path().join("workspace.db"), "workspace-test"); + let db = + SqliteTicketBackend::open(tmp.path().join("workspace.db"), "workspace-test").unwrap(); db.import_from_local_backend(&local).unwrap(); let ticket = db.show(TicketIdOrSlug::Id(created.id.clone())).unwrap(); diff --git a/crates/ticket/src/sqlite_schema.rs b/crates/ticket/src/sqlite_schema.rs new file mode 100644 index 00000000..87536232 --- /dev/null +++ b/crates/ticket/src/sqlite_schema.rs @@ -0,0 +1,1077 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::{Result, TicketError, sqlite_err}; + +const MIGRATION_TABLE: &str = "ticket_schema_migrations"; +const MAX_SCHEMA_DIAGNOSTICS: usize = 32; +pub const LATEST_SQLITE_TICKET_SCHEMA_VERSION: i64 = 2; + +#[derive(Clone, Copy)] +struct Migration { + version: i64, + name: &'static str, + apply: fn(&Connection) -> Result<()>, +} + +const MIGRATIONS: &[Migration] = &[ + Migration { + version: 1, + name: "create_typed_ticket_tables", + apply: create_typed_ticket_tables, + }, + Migration { + version: 2, + name: "add_ticket_repository_target", + apply: add_ticket_repository_target, + }, +]; + +#[derive(Clone, Copy)] +struct ExpectedColumn { + name: &'static str, + data_type: &'static str, + not_null: bool, + primary_key_position: i64, +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct ExpectedForeignKey { + from: &'static str, + target_table: &'static str, + to: &'static str, + on_delete: &'static str, +} + +const OWNED_TABLES: &[&str] = &[ + "typed_tickets", + "typed_ticket_labels", + "typed_ticket_risk_flags", + "typed_ticket_raw_frontmatter", + "typed_ticket_events", + "typed_ticket_event_references", + "typed_ticket_event_attributes", + "typed_ticket_relations", + "typed_ticket_orchestration_plans", + "typed_ticket_artifacts", +]; + +const MIGRATION_COLUMNS: &[ExpectedColumn] = &[ + column("version", "INTEGER", false, 1), + column("name", "TEXT", true, 0), + column("applied_at", "TEXT", true, 0), +]; + +const TICKET_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("slug", "TEXT", true, 0), + column("title", "TEXT", true, 0), + column("status", "TEXT", true, 0), + column("kind", "TEXT", true, 0), + column("priority", "TEXT", true, 0), + column("body", "TEXT", true, 0), + column("created_at", "TEXT", false, 0), + column("updated_at", "TEXT", false, 0), + column("assignee", "TEXT", false, 0), + column("readiness", "TEXT", false, 0), + column("workflow_state", "TEXT", true, 0), + column("workflow_state_explicit", "INTEGER", true, 0), + column("queued_by", "TEXT", false, 0), + column("queued_at", "TEXT", false, 0), + column("resolution", "TEXT", false, 0), + column("repository_id", "TEXT", false, 0), + column("ref_selector", "TEXT", false, 0), +]; + +const LABEL_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("ordinal", "INTEGER", true, 3), + column("label", "TEXT", true, 0), +]; + +const RISK_FLAG_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("ordinal", "INTEGER", true, 3), + column("risk_flag", "TEXT", true, 0), +]; + +const RAW_FRONTMATTER_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("key", "TEXT", true, 3), + column("value", "TEXT", true, 0), +]; + +const EVENT_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("event_index", "INTEGER", true, 3), + column("kind", "TEXT", true, 0), + column("author", "TEXT", false, 0), + column("at", "TEXT", false, 0), + column("status", "TEXT", false, 0), + column("from_state", "TEXT", false, 0), + column("to_state", "TEXT", false, 0), + column("reason", "TEXT", false, 0), + column("state_field", "TEXT", false, 0), + column("heading", "TEXT", false, 0), + column("body", "TEXT", true, 0), +]; + +const EVENT_REFERENCE_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("event_index", "INTEGER", true, 3), + column("ordinal", "INTEGER", true, 4), + column("kind", "TEXT", true, 0), + column("target", "TEXT", true, 0), +]; + +const EVENT_ATTRIBUTE_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("event_index", "INTEGER", true, 3), + column("key", "TEXT", true, 4), + column("value", "TEXT", true, 0), +]; + +const RELATION_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("kind", "TEXT", true, 3), + column("target", "TEXT", true, 4), + column("note", "TEXT", false, 0), + column("author", "TEXT", true, 0), + column("at", "TEXT", true, 0), +]; + +const ORCHESTRATION_PLAN_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("record_id", "TEXT", true, 3), + column("kind", "TEXT", true, 0), + column("related_ticket", "TEXT", false, 0), + column("note", "TEXT", false, 0), + column("accepted_summary", "TEXT", false, 0), + column("accepted_branch", "TEXT", false, 0), + column("accepted_worktree", "TEXT", false, 0), + column("accepted_role_plan", "TEXT", false, 0), + column("author", "TEXT", true, 0), + column("at", "TEXT", true, 0), +]; + +const ARTIFACT_COLUMNS: &[ExpectedColumn] = &[ + column("workspace_id", "TEXT", true, 1), + column("ticket_id", "TEXT", true, 2), + column("relative_path", "TEXT", true, 3), + column("content", "BLOB", true, 0), +]; + +const TICKET_FOREIGN_KEYS: &[ExpectedForeignKey] = &[]; +const CHILD_FOREIGN_KEYS: &[ExpectedForeignKey] = &[ + ExpectedForeignKey { + from: "workspace_id", + target_table: "typed_tickets", + to: "workspace_id", + on_delete: "CASCADE", + }, + ExpectedForeignKey { + from: "ticket_id", + target_table: "typed_tickets", + to: "ticket_id", + on_delete: "CASCADE", + }, +]; +const EVENT_CHILD_FOREIGN_KEYS: &[ExpectedForeignKey] = &[ + ExpectedForeignKey { + from: "workspace_id", + target_table: "typed_ticket_events", + to: "workspace_id", + on_delete: "CASCADE", + }, + ExpectedForeignKey { + from: "ticket_id", + target_table: "typed_ticket_events", + to: "ticket_id", + on_delete: "CASCADE", + }, + ExpectedForeignKey { + from: "event_index", + target_table: "typed_ticket_events", + to: "event_index", + on_delete: "CASCADE", + }, +]; + +const fn column( + name: &'static str, + data_type: &'static str, + not_null: bool, + primary_key_position: i64, +) -> ExpectedColumn { + ExpectedColumn { + name, + data_type, + not_null, + primary_key_position, + } +} + +/// Applies the Ticket crate's SQLite migrations and verifies the resulting schema. +/// +/// This is a startup/standalone-open operation. Normal Ticket request handling must +/// use [`verify_sqlite_ticket_schema`] instead, so request paths never acquire DDL +/// authority. +pub fn migrate_sqlite_ticket_schema(connection: &Connection) -> Result<()> { + connection + .busy_timeout(Duration::from_secs(5)) + .map_err(sqlite_err)?; + connection + .execute_batch("BEGIN IMMEDIATE") + .map_err(sqlite_err)?; + + let result = (|| { + connection + .execute_batch( + "CREATE TABLE IF NOT EXISTS ticket_schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL + );", + ) + .map_err(sqlite_err)?; + verify_table(connection, MIGRATION_TABLE, MIGRATION_COLUMNS, &[], false)?; + + let applied = load_applied_migrations(connection)?; + validate_applied_migrations(&applied)?; + + for migration in MIGRATIONS { + if applied.contains_key(&migration.version) { + continue; + } + (migration.apply)(connection)?; + connection + .execute( + "INSERT INTO ticket_schema_migrations (version, name, applied_at) + VALUES (?1, ?2, ?3)", + params![ + migration.version, + migration.name, + chrono::Utc::now().to_rfc3339() + ], + ) + .map_err(sqlite_err)?; + } + + verify_sqlite_ticket_schema(connection) + })(); + + match result { + Ok(()) => connection.execute_batch("COMMIT").map_err(sqlite_err), + Err(error) => { + let _ = connection.execute_batch("ROLLBACK"); + Err(error) + } + } +} + +/// Verifies the current Ticket-owned SQLite schema without executing DDL. +pub fn verify_sqlite_ticket_schema(connection: &Connection) -> Result<()> { + let mut diagnostics = Vec::new(); + + collect_table_diagnostics( + connection, + MIGRATION_TABLE, + MIGRATION_COLUMNS, + &[], + &mut diagnostics, + ); + + match load_applied_migrations(connection) { + Ok(applied) => { + if let Err(error) = validate_applied_migrations(&applied) { + push_diagnostic(&mut diagnostics, error.to_string()); + } else if applied.len() != MIGRATIONS.len() { + push_diagnostic( + &mut diagnostics, + format!( + "Ticket schema is not current: found {} migration(s), expected {}", + applied.len(), + MIGRATIONS.len() + ), + ); + } + } + Err(error) => push_diagnostic(&mut diagnostics, error.to_string()), + } + + for (table, columns, foreign_keys) in [ + ("typed_tickets", TICKET_COLUMNS, TICKET_FOREIGN_KEYS), + ("typed_ticket_labels", LABEL_COLUMNS, CHILD_FOREIGN_KEYS), + ( + "typed_ticket_risk_flags", + RISK_FLAG_COLUMNS, + CHILD_FOREIGN_KEYS, + ), + ( + "typed_ticket_raw_frontmatter", + RAW_FRONTMATTER_COLUMNS, + CHILD_FOREIGN_KEYS, + ), + ("typed_ticket_events", EVENT_COLUMNS, CHILD_FOREIGN_KEYS), + ( + "typed_ticket_event_references", + EVENT_REFERENCE_COLUMNS, + EVENT_CHILD_FOREIGN_KEYS, + ), + ( + "typed_ticket_event_attributes", + EVENT_ATTRIBUTE_COLUMNS, + EVENT_CHILD_FOREIGN_KEYS, + ), + ( + "typed_ticket_relations", + RELATION_COLUMNS, + CHILD_FOREIGN_KEYS, + ), + ( + "typed_ticket_orchestration_plans", + ORCHESTRATION_PLAN_COLUMNS, + CHILD_FOREIGN_KEYS, + ), + ( + "typed_ticket_artifacts", + ARTIFACT_COLUMNS, + CHILD_FOREIGN_KEYS, + ), + ] { + collect_table_diagnostics(connection, table, columns, foreign_keys, &mut diagnostics); + } + + for table in OWNED_TABLES { + collect_foreign_key_check_diagnostics(connection, table, &mut diagnostics); + } + + if diagnostics.is_empty() { + Ok(()) + } else { + let was_truncated = diagnostics.len() > MAX_SCHEMA_DIAGNOSTICS; + diagnostics.truncate(MAX_SCHEMA_DIAGNOSTICS); + let mut message = format!( + "Ticket SQLite schema verification failed: {}", + diagnostics.join("; ") + ); + if was_truncated { + message.push_str("; additional diagnostics omitted"); + } + Err(TicketError::Sqlite(message)) + } +} + +fn create_typed_ticket_tables(connection: &Connection) -> Result<()> { + connection + .execute_batch( + r#" +CREATE TABLE IF NOT EXISTS typed_tickets ( + workspace_id TEXT NOT NULL, + ticket_id TEXT NOT NULL, + slug TEXT NOT NULL, + title TEXT NOT NULL, + status TEXT NOT NULL, + kind TEXT NOT NULL, + priority TEXT NOT NULL, + body TEXT NOT NULL, + created_at TEXT, + updated_at TEXT, + assignee TEXT, + readiness TEXT, + workflow_state TEXT NOT NULL, + workflow_state_explicit INTEGER NOT NULL, + queued_by TEXT, + queued_at TEXT, + resolution TEXT, + PRIMARY KEY (workspace_id, ticket_id) +); +CREATE TABLE IF NOT EXISTS typed_ticket_labels ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, ordinal INTEGER NOT NULL, label TEXT NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, ordinal), + FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS typed_ticket_risk_flags ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, ordinal INTEGER NOT NULL, risk_flag TEXT NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, ordinal), + FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS typed_ticket_raw_frontmatter ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, key), + FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS typed_ticket_events ( + workspace_id TEXT NOT NULL, + ticket_id TEXT NOT NULL, + event_index INTEGER NOT NULL, + kind TEXT NOT NULL, + author TEXT, + at TEXT, + status TEXT, + from_state TEXT, + to_state TEXT, + reason TEXT, + state_field TEXT, + heading TEXT, + body TEXT NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, event_index), + FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS typed_ticket_event_references ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, event_index INTEGER NOT NULL, ordinal INTEGER NOT NULL, kind TEXT NOT NULL, target TEXT NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, event_index, ordinal), + FOREIGN KEY (workspace_id, ticket_id, event_index) REFERENCES typed_ticket_events(workspace_id, ticket_id, event_index) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS typed_ticket_event_attributes ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, event_index INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, event_index, key), + FOREIGN KEY (workspace_id, ticket_id, event_index) REFERENCES typed_ticket_events(workspace_id, ticket_id, event_index) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS typed_ticket_relations ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, kind TEXT NOT NULL, target TEXT NOT NULL, note TEXT, author TEXT NOT NULL, at TEXT NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, kind, target), + FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS typed_ticket_orchestration_plans ( + workspace_id TEXT NOT NULL, + ticket_id TEXT NOT NULL, + record_id TEXT NOT NULL, + kind TEXT NOT NULL, + related_ticket TEXT, + note TEXT, + accepted_summary TEXT, + accepted_branch TEXT, + accepted_worktree TEXT, + accepted_role_plan TEXT, + author TEXT NOT NULL, + at TEXT NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, record_id), + FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS typed_ticket_artifacts ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, relative_path TEXT NOT NULL, content BLOB NOT NULL, + PRIMARY KEY (workspace_id, ticket_id, relative_path), + FOREIGN KEY (workspace_id, ticket_id) REFERENCES typed_tickets(workspace_id, ticket_id) ON DELETE CASCADE +); +"#, + ) + .map_err(sqlite_err) +} + +fn add_ticket_repository_target(connection: &Connection) -> Result<()> { + add_column_if_missing(connection, "typed_tickets", "repository_id", "TEXT")?; + add_column_if_missing(connection, "typed_tickets", "ref_selector", "TEXT") +} + +fn add_column_if_missing( + connection: &Connection, + table: &str, + column: &str, + declaration: &str, +) -> Result<()> { + let columns = load_columns(connection, table)?; + if columns.iter().any(|found| found.name == column) { + return Ok(()); + } + connection + .execute_batch(&format!( + "ALTER TABLE {table} ADD COLUMN {column} {declaration}" + )) + .map_err(sqlite_err) +} + +fn load_applied_migrations(connection: &Connection) -> Result> { + let mut statement = connection + .prepare("SELECT version, name FROM ticket_schema_migrations ORDER BY version") + .map_err(sqlite_err)?; + let rows = statement + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + }) + .map_err(sqlite_err)?; + let mut applied = BTreeMap::new(); + for row in rows { + let (version, name) = row.map_err(sqlite_err)?; + if applied.insert(version, name).is_some() { + return Err(TicketError::Sqlite(format!( + "duplicate Ticket schema migration version {version}" + ))); + } + } + Ok(applied) +} + +fn validate_applied_migrations(applied: &BTreeMap) -> Result<()> { + for (&version, name) in applied { + let Some(expected) = MIGRATIONS + .iter() + .find(|migration| migration.version == version) + else { + return Err(TicketError::Sqlite(format!( + "unsupported Ticket schema migration version {version}; latest supported version is {LATEST_SQLITE_TICKET_SCHEMA_VERSION}" + ))); + }; + if name != expected.name { + return Err(TicketError::Sqlite(format!( + "Ticket schema migration {version} is named {name:?}, expected {:?}", + expected.name + ))); + } + } + for migration in MIGRATIONS { + if applied.keys().any(|version| *version > migration.version) + && !applied.contains_key(&migration.version) + { + return Err(TicketError::Sqlite(format!( + "Ticket schema migration history has a gap at version {}", + migration.version + ))); + } + } + Ok(()) +} + +#[derive(Debug)] +struct ActualColumn { + name: String, + data_type: String, + not_null: bool, + primary_key_position: i64, +} + +fn load_columns(connection: &Connection, table: &str) -> Result> { + let exists = connection + .query_row( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1", + params![table], + |_| Ok(()), + ) + .optional() + .map_err(sqlite_err)? + .is_some(); + if !exists { + return Err(TicketError::Sqlite(format!( + "required Ticket schema table {table:?} is missing" + ))); + } + + let mut statement = connection + .prepare(&format!("PRAGMA table_info({table})")) + .map_err(sqlite_err)?; + let rows = statement + .query_map([], |row| { + Ok(ActualColumn { + name: row.get(1)?, + data_type: row.get::<_, String>(2)?.to_ascii_uppercase(), + not_null: row.get::<_, i64>(3)? != 0, + primary_key_position: row.get(5)?, + }) + }) + .map_err(sqlite_err)?; + rows.collect::, _>>() + .map_err(sqlite_err) +} + +fn verify_table( + connection: &Connection, + table: &str, + expected_columns: &[ExpectedColumn], + expected_foreign_keys: &[ExpectedForeignKey], + check_foreign_keys: bool, +) -> Result<()> { + let mut diagnostics = Vec::new(); + collect_column_diagnostics(connection, table, expected_columns, &mut diagnostics); + if check_foreign_keys { + collect_foreign_key_diagnostics(connection, table, expected_foreign_keys, &mut diagnostics); + } + if diagnostics.is_empty() { + Ok(()) + } else { + Err(TicketError::Sqlite(diagnostics.join("; "))) + } +} + +fn collect_table_diagnostics( + connection: &Connection, + table: &str, + expected_columns: &[ExpectedColumn], + expected_foreign_keys: &[ExpectedForeignKey], + diagnostics: &mut Vec, +) { + collect_column_diagnostics(connection, table, expected_columns, diagnostics); + if diagnostics.len() < MAX_SCHEMA_DIAGNOSTICS { + collect_foreign_key_diagnostics(connection, table, expected_foreign_keys, diagnostics); + } +} + +fn collect_column_diagnostics( + connection: &Connection, + table: &str, + expected: &[ExpectedColumn], + diagnostics: &mut Vec, +) { + let actual = match load_columns(connection, table) { + Ok(actual) => actual, + Err(error) => { + push_diagnostic(diagnostics, error.to_string()); + return; + } + }; + + let expected_names = expected + .iter() + .map(|column| column.name) + .collect::>(); + let actual_names = actual + .iter() + .map(|column| column.name.as_str()) + .collect::>(); + for missing in expected_names.difference(&actual_names) { + push_diagnostic( + diagnostics, + format!("table {table:?} is missing column {missing:?}"), + ); + } + for unexpected in actual_names.difference(&expected_names) { + push_diagnostic( + diagnostics, + format!("table {table:?} has unexpected column {unexpected:?}"), + ); + } + + for expected in expected { + let Some(actual) = actual.iter().find(|column| column.name == expected.name) else { + continue; + }; + if actual.data_type != expected.data_type { + push_diagnostic( + diagnostics, + format!( + "table {table:?} column {:?} has type {:?}, expected {:?}", + expected.name, actual.data_type, expected.data_type + ), + ); + } + if actual.not_null != expected.not_null { + push_diagnostic( + diagnostics, + format!( + "table {table:?} column {:?} NOT NULL is {}, expected {}", + expected.name, actual.not_null, expected.not_null + ), + ); + } + if actual.primary_key_position != expected.primary_key_position { + push_diagnostic( + diagnostics, + format!( + "table {table:?} column {:?} primary-key position is {}, expected {}", + expected.name, actual.primary_key_position, expected.primary_key_position + ), + ); + } + } +} + +fn collect_foreign_key_diagnostics( + connection: &Connection, + table: &str, + expected: &[ExpectedForeignKey], + diagnostics: &mut Vec, +) { + let mut statement = match connection.prepare(&format!("PRAGMA foreign_key_list({table})")) { + Ok(statement) => statement, + Err(error) => { + push_diagnostic(diagnostics, sqlite_err(error).to_string()); + return; + } + }; + let rows = match statement.query_map([], |row| { + Ok(( + row.get::<_, String>(3)?, + row.get::<_, String>(2)?, + row.get::<_, String>(4)?, + row.get::<_, String>(6)?.to_ascii_uppercase(), + )) + }) { + Ok(rows) => rows, + Err(error) => { + push_diagnostic(diagnostics, sqlite_err(error).to_string()); + return; + } + }; + let mut actual = BTreeSet::new(); + for row in rows { + match row { + Ok(foreign_key) => { + actual.insert(foreign_key); + } + Err(error) => push_diagnostic(diagnostics, sqlite_err(error).to_string()), + } + } + let expected = expected + .iter() + .map(|foreign_key| { + ( + foreign_key.from.to_string(), + foreign_key.target_table.to_string(), + foreign_key.to.to_string(), + foreign_key.on_delete.to_string(), + ) + }) + .collect::>(); + for missing in expected.difference(&actual) { + push_diagnostic( + diagnostics, + format!("table {table:?} is missing foreign key {missing:?}"), + ); + } + for unexpected in actual.difference(&expected) { + push_diagnostic( + diagnostics, + format!("table {table:?} has unexpected foreign key {unexpected:?}"), + ); + } +} + +fn collect_foreign_key_check_diagnostics( + connection: &Connection, + table: &str, + diagnostics: &mut Vec, +) { + let mut statement = match connection.prepare(&format!("PRAGMA foreign_key_check({table})")) { + Ok(statement) => statement, + Err(error) => { + push_diagnostic(diagnostics, sqlite_err(error).to_string()); + return; + } + }; + let rows = match statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + )) + }) { + Ok(rows) => rows, + Err(error) => { + push_diagnostic(diagnostics, sqlite_err(error).to_string()); + return; + } + }; + for row in rows { + match row { + Ok((table, row_id, parent, foreign_key_id)) => push_diagnostic( + diagnostics, + format!( + "foreign-key violation in table {table:?} row {row_id:?} referencing {parent:?} (foreign key {foreign_key_id})" + ), + ), + Err(error) => push_diagnostic(diagnostics, sqlite_err(error).to_string()), + } + } +} + +fn push_diagnostic(diagnostics: &mut Vec, diagnostic: String) { + if diagnostics.len() <= MAX_SCHEMA_DIAGNOSTICS { + diagnostics.push(diagnostic); + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Barrier}; + use std::thread; + + use rusqlite::Connection; + use tempfile::tempdir; + + use super::*; + use crate::SqliteTicketBackend; + + #[test] + fn migrates_fresh_database_to_current_ticket_schema() { + let connection = Connection::open_in_memory().unwrap(); + migrate_sqlite_ticket_schema(&connection).unwrap(); + verify_sqlite_ticket_schema(&connection).unwrap(); + + let versions = load_applied_migrations(&connection).unwrap(); + assert_eq!(versions.len(), 2); + assert_eq!( + versions.get(&LATEST_SQLITE_TICKET_SCHEMA_VERSION), + Some(&"add_ticket_repository_target".to_string()) + ); + } + + #[test] + fn adopts_existing_current_schema_without_losing_data() { + let connection = Connection::open_in_memory().unwrap(); + create_typed_ticket_tables(&connection).unwrap(); + add_ticket_repository_target(&connection).unwrap(); + connection + .execute( + "INSERT INTO typed_tickets ( + workspace_id, ticket_id, slug, title, status, kind, priority, body, + workflow_state, workflow_state_explicit, repository_id, ref_selector + ) VALUES ('workspace-1', 'ticket-1', 'ticket-1', 'kept', 'open', + 'task', 'medium', 'body', 'ready', 1, 'main', 'develop')", + [], + ) + .unwrap(); + connection + .execute_batch( + "INSERT INTO typed_ticket_events ( + workspace_id, ticket_id, event_index, kind, author, at, heading, body + ) VALUES ( + 'workspace-1', 'ticket-1', 0, 'comment', 'hare', + '2026-08-10T00:00:00Z', 'Evidence', 'event kept' + ); + INSERT INTO typed_ticket_event_references ( + workspace_id, ticket_id, event_index, ordinal, kind, target + ) VALUES ('workspace-1', 'ticket-1', 0, 0, 'commit', 'abc123'); + INSERT INTO typed_ticket_relations ( + workspace_id, ticket_id, kind, target, note, author, at + ) VALUES ( + 'workspace-1', 'ticket-1', 'related', 'ticket-2', 'relation kept', + 'hare', '2026-08-10T00:00:00Z' + ); + INSERT INTO typed_ticket_orchestration_plans ( + workspace_id, ticket_id, record_id, kind, note, author, at + ) VALUES ( + 'workspace-1', 'ticket-1', 'plan-1', 'waiting_capacity_note', + 'plan kept', 'hare', '2026-08-10T00:00:00Z' + ); + INSERT INTO typed_ticket_artifacts ( + workspace_id, ticket_id, relative_path, content + ) VALUES ('workspace-1', 'ticket-1', 'evidence.txt', X'6b657074');", + ) + .unwrap(); + + migrate_sqlite_ticket_schema(&connection).unwrap(); + + let row = connection + .query_row( + "SELECT title, repository_id, ref_selector FROM typed_tickets", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .unwrap(); + assert_eq!(row, ("kept".into(), "main".into(), "develop".into())); + let preserved = connection + .query_row( + "SELECT + (SELECT COUNT(*) FROM typed_ticket_events), + (SELECT COUNT(*) FROM typed_ticket_event_references), + (SELECT COUNT(*) FROM typed_ticket_relations), + (SELECT COUNT(*) FROM typed_ticket_orchestration_plans), + (SELECT COUNT(*) FROM typed_ticket_artifacts)", + [], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + )) + }, + ) + .unwrap(); + assert_eq!(preserved, (1, 1, 1, 1, 1)); + } + + #[test] + fn upgrades_legacy_schema_without_repository_target_columns() { + let connection = Connection::open_in_memory().unwrap(); + create_typed_ticket_tables(&connection).unwrap(); + connection + .execute( + "INSERT INTO typed_tickets ( + workspace_id, ticket_id, slug, title, status, kind, priority, body, + workflow_state, workflow_state_explicit + ) VALUES ('workspace-1', 'ticket-1', 'ticket-1', 'legacy', 'open', + 'task', 'medium', 'body', 'ready', 1)", + [], + ) + .unwrap(); + connection + .execute_batch( + "CREATE TABLE ticket_schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL + ); + INSERT INTO ticket_schema_migrations (version, name, applied_at) + VALUES (1, 'create_typed_ticket_tables', '2026-08-10T00:00:00Z');", + ) + .unwrap(); + + migrate_sqlite_ticket_schema(&connection).unwrap(); + verify_sqlite_ticket_schema(&connection).unwrap(); + + let columns = load_columns(&connection, "typed_tickets").unwrap(); + assert!(columns.iter().any(|column| column.name == "repository_id")); + assert!(columns.iter().any(|column| column.name == "ref_selector")); + let title = connection + .query_row("SELECT title FROM typed_tickets", [], |row| { + row.get::<_, String>(0) + }) + .unwrap(); + assert_eq!(title, "legacy"); + } + + #[test] + fn rejects_unknown_future_migration_history_without_changing_schema() { + let connection = Connection::open_in_memory().unwrap(); + migrate_sqlite_ticket_schema(&connection).unwrap(); + connection + .execute( + "INSERT INTO ticket_schema_migrations (version, name, applied_at) + VALUES (99, 'future', '2026-08-10T00:00:00Z')", + [], + ) + .unwrap(); + + let error = migrate_sqlite_ticket_schema(&connection).unwrap_err(); + assert!( + error + .to_string() + .contains("unsupported Ticket schema migration version 99") + ); + assert_eq!(load_applied_migrations(&connection).unwrap().len(), 3); + } + + #[test] + fn verified_backend_open_fails_on_drift_without_repairing_request_schema() { + let directory = tempdir().unwrap(); + let database = directory.path().join("tickets.db"); + SqliteTicketBackend::open(&database, "workspace-1").unwrap(); + let connection = Connection::open(&database).unwrap(); + connection + .execute_batch("DROP TABLE typed_ticket_artifacts") + .unwrap(); + + let error = match SqliteTicketBackend::open_verified(&database, "workspace-1") { + Ok(_) => panic!("drifted schema unexpectedly passed request-path verification"), + Err(error) => error, + }; + assert!(error.to_string().contains("typed_ticket_artifacts")); + let still_missing = connection + .query_row( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'typed_ticket_artifacts'", + [], + |_| Ok(()), + ) + .optional() + .unwrap() + .is_none(); + assert!(still_missing); + } + + #[test] + fn verification_does_not_claim_unrelated_foreign_key_authority() { + let connection = Connection::open_in_memory().unwrap(); + migrate_sqlite_ticket_schema(&connection).unwrap(); + connection + .pragma_update(None, "foreign_keys", "OFF") + .unwrap(); + connection + .execute_batch( + "CREATE TABLE unrelated_parent (id TEXT PRIMARY KEY); + CREATE TABLE unrelated_child ( + id TEXT PRIMARY KEY, + parent_id TEXT NOT NULL REFERENCES unrelated_parent(id) + ); + INSERT INTO unrelated_child (id, parent_id) VALUES ('child', 'missing');", + ) + .unwrap(); + + verify_sqlite_ticket_schema(&connection).unwrap(); + } + + #[test] + fn migration_rejects_constraint_drift_and_rolls_back_version_adoption() { + let connection = Connection::open_in_memory().unwrap(); + connection + .execute_batch( + "CREATE TABLE typed_tickets ( + workspace_id TEXT NOT NULL, + ticket_id TEXT NOT NULL, + slug TEXT NOT NULL, + title TEXT NOT NULL, + status TEXT NOT NULL, + kind TEXT NOT NULL, + priority TEXT NOT NULL, + body TEXT NOT NULL, + created_at TEXT, + updated_at TEXT, + assignee TEXT, + readiness TEXT, + workflow_state TEXT NOT NULL, + workflow_state_explicit INTEGER NOT NULL, + queued_by TEXT, + queued_at TEXT, + resolution TEXT, + PRIMARY KEY (ticket_id, workspace_id) + );", + ) + .unwrap(); + + let error = migrate_sqlite_ticket_schema(&connection).unwrap_err(); + assert!(error.to_string().contains("primary-key position")); + let migration_table_exists = connection + .query_row( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'ticket_schema_migrations'", + [], + |_| Ok(()), + ) + .optional() + .unwrap() + .is_some(); + assert!(!migration_table_exists); + } + + #[test] + fn concurrent_migrators_converge_on_one_version_history() { + let directory = tempdir().unwrap(); + let database = directory.path().join("tickets.db"); + let barrier = Arc::new(Barrier::new(3)); + let mut joins = Vec::new(); + for _ in 0..2 { + let database = database.clone(); + let barrier = barrier.clone(); + joins.push(thread::spawn(move || { + let connection = Connection::open(database).unwrap(); + barrier.wait(); + migrate_sqlite_ticket_schema(&connection) + })); + } + barrier.wait(); + for join in joins { + join.join().unwrap().unwrap(); + } + + let connection = Connection::open(database).unwrap(); + verify_sqlite_ticket_schema(&connection).unwrap(); + assert_eq!(load_applied_migrations(&connection).unwrap().len(), 2); + } +} diff --git a/crates/workspace-server/src/authority.rs b/crates/workspace-server/src/authority.rs index 528b955e..e982bb67 100644 --- a/crates/workspace-server/src/authority.rs +++ b/crates/workspace-server/src/authority.rs @@ -131,7 +131,7 @@ impl SqliteWorkspaceAuthority { Ok(Self { workspace_id: workspace_id.clone(), store: SqliteWorkspaceStore::open(&database_path)?, - ticket_backend: SqliteTicketBackend::new(database_path, workspace_id), + ticket_backend: SqliteTicketBackend::open_verified(database_path, workspace_id)?, }) } @@ -725,7 +725,8 @@ mod tests { let dir = tempfile::tempdir().unwrap(); write_ticket(dir.path(), "00000000001J2", "Read bridge", "ready"); let db_path = dir.path().join("workspace.db"); - SqliteTicketBackend::new(&db_path, "workspace-test") + SqliteTicketBackend::open(&db_path, "workspace-test") + .unwrap() .import_from_local_backend(&ticket::LocalTicketBackend::new( dir.path().join(".yoi/tickets"), )) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 6652821e..fdc60a38 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -2383,10 +2383,10 @@ struct BrowserCloseTicketRequest { fn browser_ticket_backend(api: &WorkspaceApi) -> Result { let config = ticket::config::TicketConfig::load_workspace(&api.config.workspace_root) .map_err(|error| Error::Config(format!("load Ticket workspace settings: {error}")))?; - Ok(SqliteTicketBackend::new( + Ok(SqliteTicketBackend::open_verified( api.config.database_path.clone(), api.config.workspace_id.clone(), - ) + )? .with_record_language(config.ticket_record_language())) } @@ -2553,10 +2553,11 @@ async fn execute_worker_ticket_rest_operation( validate_workspace_scope(api, workspace_id)?; let config = ticket::config::TicketConfig::load_workspace(&api.config.workspace_root) .map_err(|error| Error::Config(format!("load Ticket workspace settings: {error}")))?; - let mut backend = SqliteTicketBackend::new( + let mut backend = SqliteTicketBackend::open_verified( api.config.database_path.clone(), api.config.workspace_id.clone(), ) + .map_err(Error::from)? .with_record_language(config.ticket_record_language()); let operation_kind = ticket_mutation_operation_kind(&operation); let is_mutation = operation_kind != "read"; @@ -15225,7 +15226,7 @@ mod tests { ) { use ticket::TicketBackend as _; - let backend = ticket::SqliteTicketBackend::new(database_path, workspace_id); + let backend = ticket::SqliteTicketBackend::open(database_path, workspace_id).unwrap(); let mut input = ticket::NewTicket::new(title); input.workflow_state = Some(state); backend.create(input).unwrap(); diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index c180414f..0b3617a6 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -754,6 +754,7 @@ impl SqliteWorkspaceStore { pub fn from_connection(conn: Connection) -> Result { configure_sqlite(&conn)?; apply_migrations(&conn)?; + ticket::migrate_sqlite_ticket_schema(&conn)?; Ok(Self { conn: Arc::new(Mutex::new(conn)), }) @@ -4526,6 +4527,45 @@ mod tests { use super::*; use std::collections::BTreeSet; + #[test] + fn startup_composes_ticket_migrations_when_control_plane_is_current() { + let conn = Connection::open_in_memory().unwrap(); + configure_sqlite(&conn).unwrap(); + apply_migrations(&conn).unwrap(); + assert!(!table_exists(&conn, "ticket_schema_migrations").unwrap()); + + let store = SqliteWorkspaceStore::from_connection(conn).unwrap(); + store + .with_conn(|conn| { + ticket::verify_sqlite_ticket_schema(conn)?; + let latest = conn.query_row( + "SELECT MAX(version) FROM ticket_schema_migrations", + [], + |row| row.get::<_, i64>(0), + )?; + assert_eq!(latest, ticket::LATEST_SQLITE_TICKET_SCHEMA_VERSION); + Ok(()) + }) + .unwrap(); + } + + #[test] + fn startup_fails_closed_when_current_ticket_schema_has_drifted() { + let conn = Connection::open_in_memory().unwrap(); + configure_sqlite(&conn).unwrap(); + apply_migrations(&conn).unwrap(); + ticket::migrate_sqlite_ticket_schema(&conn).unwrap(); + conn.execute_batch("DROP TABLE typed_ticket_artifacts") + .unwrap(); + + let result = SqliteWorkspaceStore::from_connection(conn); + let error = match result { + Ok(_) => panic!("schema drift unexpectedly passed startup verification"), + Err(error) => error, + }; + assert!(error.to_string().contains("typed_ticket_artifacts")); + } + #[test] fn removes_unused_control_plane_ticket_tables() { let conn = Connection::open_in_memory().unwrap(); diff --git a/crates/yoi/src/ticket_cli.rs b/crates/yoi/src/ticket_cli.rs index 487f9e4f..c51ca8a8 100644 --- a/crates/yoi/src/ticket_cli.rs +++ b/crates/yoi/src/ticket_cli.rs @@ -370,7 +370,7 @@ fn backend_for_workspace(workspace: &Path) -> Result, Tic let workspace_id = workspace_id_for_workspace(workspace)?; let db_path = server_database_path(workspace)?; Ok(Box::new( - SqliteTicketBackend::new(db_path, workspace_id) + SqliteTicketBackend::open(db_path, workspace_id)? .with_record_language(config.ticket_record_language()), )) } @@ -381,7 +381,7 @@ fn import_local(workspace: &Path) -> Result { .with_record_language(config.ticket_record_language()); let workspace_id = workspace_id_for_workspace(workspace)?; let db_path = server_database_path(workspace)?; - let sqlite = SqliteTicketBackend::new(db_path.clone(), workspace_id) + let sqlite = SqliteTicketBackend::open(db_path.clone(), workspace_id)? .with_record_language(config.ticket_record_language()); sqlite.import_from_local_backend(&local)?; Ok(success(format!( From 7f0d02531290defcc08159513df6863fc0ab6f28 Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 11 Aug 2026 05:41:48 +0900 Subject: [PATCH 02/12] server: scope repository identity by workspace --- crates/workspace-server/src/server.rs | 252 ++++++++++-- crates/workspace-server/src/store.rs | 526 +++++++++++++++++++++++++- 2 files changed, 743 insertions(+), 35 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index fdc60a38..6529ad6d 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -396,6 +396,7 @@ impl WorkspaceApi { runtime_id: &str, mut request: WorkerSpawnRequest, ) -> ApiResult { + self.validate_worker_spawn_repository_scope(&request)?; let workspace_api = self.workspace_api_ref(runtime_id); request.resolved_workspace_api = Some(workspace_api.clone()); let attachment_reservation = @@ -573,6 +574,64 @@ impl WorkspaceApi { fn repository_reader(&self) -> RepositoryRegistryReader { RepositoryRegistryReader::new(self.config.repositories.clone()) } + + fn require_workspace_repository(&self, repository_id: &str) -> ApiResult { + self.store + .get_repository(&self.config.workspace_id, repository_id)? + .ok_or_else(|| ApiError::from(Error::UnknownRepository(repository_id.to_string()))) + } + + fn require_configured_workspace_repository( + &self, + repository_id: &str, + ) -> ApiResult { + self.require_workspace_repository(repository_id)?; + self.config + .repositories + .iter() + .find(|repository| repository.id == repository_id) + .cloned() + .ok_or_else(|| ApiError::from(Error::UnknownRepository(repository_id.to_string()))) + } + + fn validate_worker_spawn_repository_scope( + &self, + request: &WorkerSpawnRequest, + ) -> ApiResult<()> { + let selected_repository_id = + if let Some(working_directory) = request.resolved_working_directory_request.as_ref() { + let repository_id = working_directory.repository.id.as_str(); + self.require_workspace_repository(repository_id)?; + Some(repository_id.to_string()) + } else if let Some(claim) = request.resolved_working_directory.as_ref() { + let workdir = self + .store + .get_workdir_registry(&self.config.workspace_id, &claim.working_directory_id)? + .ok_or_else(|| { + ApiError::from(Error::Config(format!( + "unknown working directory `{}` in this Workspace", + claim.working_directory_id + ))) + })?; + self.require_workspace_repository(&workdir.repository_id)?; + Some(workdir.repository_id) + } else { + None + }; + + if let WorkerSpawnIntent::TicketRole { ticket_id, .. } = &request.intent { + let ticket = self.authority.ticket(ticket_id)?; + if let Some(repository_id) = ticket.repository_id.as_deref() { + self.require_workspace_repository(repository_id)?; + if selected_repository_id.as_deref() != Some(repository_id) { + return Err(ApiError::from(Error::Config(format!( + "Ticket `{ticket_id}` targets repository `{repository_id}`, but the Worker launch does not resolve that repository in this Workspace" + )))); + } + } + } + Ok(()) + } } fn import_configured_repositories( @@ -2401,11 +2460,10 @@ async fn scoped_edit_ticket_item( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; if let Some(TicketTargetEdit::Set { repository_id, .. }) = request.target.as_ref() { - if !api + if api .store - .list_repositories(&api.config.workspace_id)? - .iter() - .any(|repository| repository.repository_id == *repository_id) + .get_repository(&api.config.workspace_id, repository_id)? + .is_none() { return Err(settings_bad_request( "unknown_ticket_repository", @@ -2563,6 +2621,7 @@ async fn execute_worker_ticket_rest_operation( let is_mutation = operation_kind != "read"; let target = ticket_mutation_target(&operation).cloned(); let source = authenticate_worker_mutation_source(api, workspace_id, &headers)?; + validate_ticket_repository_operation(api, &operation)?; let before = target.as_ref().and_then(|id| backend.show(id.clone()).ok()); let previous_state = before .as_ref() @@ -3106,6 +3165,24 @@ fn ticket_mutation_target(operation: &TicketBackendOperation) -> Option<&TicketI } } +fn validate_ticket_repository_operation( + api: &WorkspaceApi, + operation: &TicketBackendOperation, +) -> ApiResult<()> { + let repository_id = match operation { + TicketBackendOperation::Create { input } => input.repository_id.as_deref(), + TicketBackendOperation::EditItem { edit, .. } => match edit.target.as_ref() { + Some(TicketTargetEdit::Set { repository_id, .. }) => Some(repository_id.as_str()), + _ => None, + }, + _ => None, + }; + if let Some(repository_id) = repository_id { + api.require_workspace_repository(repository_id)?; + } + Ok(()) +} + fn bind_worker_ticket_operation_source( source: &WorkerMutationSource, operation: &mut TicketBackendOperation, @@ -6819,19 +6896,22 @@ fn working_directory_request_from_repository( } fn configured_working_directory_request( - config: &ServerConfig, + api: &WorkspaceApi, request: &WorkerSpawnWorkingDirectoryRequest, ) -> Result { - let repository = config + if api + .store + .get_repository(&api.config.workspace_id, &request.repository_id)? + .is_none() + { + return Err(Error::UnknownRepository(request.repository_id.clone())); + } + let repository = api + .config .repositories .iter() .find(|repository| repository.id == request.repository_id) - .ok_or_else(|| { - Error::Config(format!( - "unknown repository id `{}` for Worker working directory", - request.repository_id - )) - })?; + .ok_or_else(|| Error::UnknownRepository(request.repository_id.clone()))?; Ok(working_directory_request_from_repository( repository, request.selector.as_deref(), @@ -7635,9 +7715,7 @@ async fn create_runtime_worker( request.resolved_working_directory_request = request .working_directory_request .as_ref() - .map(|working_directory| { - configured_working_directory_request(&api.config, working_directory) - }) + .map(|working_directory| configured_working_directory_request(&api, working_directory)) .transpose()?; let prepared_workdir_id = if let Some(working_directory_request) = request.resolved_working_directory_request.as_mut() @@ -9568,12 +9646,7 @@ fn working_directory_request_for_browser( api: &WorkspaceApi, request: BrowserWorkingDirectoryCreateRequest, ) -> ApiResult { - let repository = api - .config - .repositories - .iter() - .find(|repository| repository.id == request.repository_id) - .ok_or_else(|| Error::UnknownRepository(request.repository_id.clone()))?; + let repository = api.require_configured_workspace_repository(&request.repository_id)?; let selector = request .selector .or_else(|| repository.default_selector.clone()) @@ -10336,6 +10409,111 @@ mod tests { ); } + #[tokio::test] + async fn repository_bound_ticket_flow_and_workdir_launches_fail_closed_across_workspaces() { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let other_workspace = WorkspaceRecord { + workspace_id: "other-workspace".to_string(), + owner_account_id: None, + display_name: "Other Workspace".to_string(), + state: "active".to_string(), + created_at: "1".to_string(), + updated_at: "1".to_string(), + }; + api.store.upsert_workspace(&other_workspace).await.unwrap(); + api.store + .upsert_repository(&RepositoryRecord { + workspace_id: other_workspace.workspace_id.clone(), + repository_id: "foreign".to_string(), + name: "Foreign".to_string(), + kind: "git".to_string(), + provider: Some("git".to_string()), + uri: dir.path().join("foreign").display().to_string(), + default_ref: Some("HEAD".to_string()), + auth_ref_kind: None, + auth_ref_key: None, + created_at: "1".to_string(), + updated_at: "1".to_string(), + }) + .unwrap(); + + let mut create_input = ticket::NewTicket::new("Foreign repository target"); + create_input.repository_id = Some("foreign".to_string()); + assert!( + validate_ticket_repository_operation( + &api, + &TicketBackendOperation::Create { + input: create_input.clone(), + }, + ) + .is_err() + ); + + let ticket = browser_ticket_backend(&api) + .unwrap() + .create(create_input) + .unwrap(); + let flow_ticket_launch = WorkerSpawnRequest { + requested_worker_name: Some("cross-workspace-ticket".to_string()), + intent: WorkerSpawnIntent::TicketRole { + ticket_id: ticket.id, + role: TicketWorkerRole::Coder, + }, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 2, + }, + profile: ProfileSelector::Builtin("builtin:coder".to_string()), + ticket_assignment: None, + initial_submit: vec![ + Segment::Flow { + selector: "builtin:coder-review".to_string(), + }, + Segment::text("Implement the Ticket"), + ], + working_directory_request: None, + resolved_working_directory_request: None, + resolved_working_directory: None, + resolved_config_bundle: None, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + }; + assert!( + api.validate_worker_spawn_repository_scope(&flow_ticket_launch) + .is_err() + ); + + let mut foreign_repository = api.config.repositories[0].clone(); + foreign_repository.id = "foreign".to_string(); + let workdir_flow_launch = WorkerSpawnRequest { + requested_worker_name: Some("cross-workspace-workdir".to_string()), + intent: WorkerSpawnIntent::WorkspaceCoding, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 1, + }, + profile: ProfileSelector::Builtin("builtin:coder".to_string()), + ticket_assignment: None, + initial_submit: vec![Segment::Flow { + selector: "builtin:coder-review".to_string(), + }], + working_directory_request: None, + resolved_working_directory_request: Some(working_directory_request_from_repository( + &foreign_repository, + Some("HEAD"), + )), + resolved_working_directory: None, + resolved_config_bundle: None, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + }; + assert!( + api.validate_worker_spawn_repository_scope(&workdir_flow_launch) + .is_err() + ); + } + #[test] fn worker_ticket_assignment_projects_coder_intent_and_run_acceptance() { let initial_submit = vec![ @@ -10678,6 +10856,7 @@ mod tests { async fn workspace_workdir_summaries_include_runtime_observed_rows() { let dir = tempfile::tempdir().unwrap(); let api = test_api(dir.path()).await; + seed_test_repository(&api, "repo"); api.store .upsert_workdir_registry(&WorkdirRegistryRecord { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -12563,7 +12742,34 @@ mod tests { runtime_worker_id.to_string() } + fn seed_test_repository(api: &WorkspaceApi, repository_id: &str) { + if api + .store + .get_repository(&api.config.workspace_id, repository_id) + .unwrap() + .is_some() + { + return; + } + api.store + .upsert_repository(&RepositoryRecord { + workspace_id: api.config.workspace_id.clone(), + repository_id: repository_id.to_string(), + name: repository_id.to_string(), + kind: "git".to_string(), + provider: Some("git".to_string()), + uri: api.config.workspace_root.display().to_string(), + default_ref: Some("HEAD".to_string()), + auth_ref_kind: None, + auth_ref_key: None, + created_at: "1".to_string(), + updated_at: "1".to_string(), + }) + .unwrap(); + } + fn seed_cleanup_workdir(api: &WorkspaceApi, workdir_id: &str, status: &str, cleanliness: &str) { + seed_test_repository(api, "repo-test"); let now = now_registry_timestamp(); api.store .upsert_workdir_registry(&WorkdirRegistryRecord { @@ -14449,9 +14655,7 @@ mod tests { "/api/runtimes/embedded-worker-runtime/workers", json!({ "intent": { - "kind": "ticket_role", - "ticket_id": "00001KVZSGT0Q", - "role": "coder" + "kind": "workspace_coding" }, "requested_worker_name": "api-friendly-name", "acceptance": { diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 0b3617a6..e98b77b2 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -151,6 +151,11 @@ const MIGRATIONS: &[Migration] = &[ name: "remove Backend-owned Flow runtime authority", apply: remove_backend_flow_runtime_authority, }, + Migration { + version: 27, + name: "scope Repository identity and references by Workspace", + apply: scope_repository_identity_by_workspace, + }, ]; struct Migration { @@ -452,6 +457,11 @@ pub trait ControlPlaneStore: Send + Sync { async fn get_workspace(&self, workspace_id: &str) -> Result>; fn list_workspaces(&self) -> Result>; fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>; + fn get_repository( + &self, + workspace_id: &str, + repository_id: &str, + ) -> Result>; fn list_repositories(&self, workspace_id: &str) -> Result>; fn put_flow_source_for_kind( @@ -755,6 +765,7 @@ impl SqliteWorkspaceStore { configure_sqlite(&conn)?; apply_migrations(&conn)?; ticket::migrate_sqlite_ticket_schema(&conn)?; + validate_workspace_repository_references(&conn)?; Ok(Self { conn: Arc::new(Mutex::new(conn)), }) @@ -888,8 +899,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { workspace_id, repository_id, name, kind, provider, uri, default_ref, auth_ref_kind, auth_ref_key, created_at, updated_at ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) - ON CONFLICT(repository_id) DO UPDATE SET - workspace_id = excluded.workspace_id, + ON CONFLICT(workspace_id, repository_id) DO UPDATE SET name = excluded.name, kind = excluded.kind, provider = excluded.provider, @@ -916,6 +926,25 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + fn get_repository( + &self, + workspace_id: &str, + repository_id: &str, + ) -> Result> { + self.with_conn(|conn| { + conn.query_row( + r#"SELECT workspace_id, repository_id, name, kind, provider, uri, default_ref, + auth_ref_kind, auth_ref_key, created_at, updated_at + FROM repositories + WHERE workspace_id = ?1 AND repository_id = ?2"#, + params![workspace_id, repository_id], + read_repository_record, + ) + .optional() + .map_err(Error::from) + }) + } + fn list_repositories(&self, workspace_id: &str) -> Result> { self.with_conn(|conn| { let mut stmt = conn.prepare( @@ -4020,6 +4049,205 @@ DROP TABLE IF EXISTS flow_instances; Ok(()) } +fn scope_repository_identity_by_workspace(conn: &Connection) -> Result<()> { + validate_workspace_repository_references(conn)?; + conn.execute_batch( + r#" +CREATE TABLE repositories_v27 ( + workspace_id TEXT NOT NULL, + repository_id TEXT NOT NULL, + name TEXT NOT NULL, + kind TEXT NOT NULL, + provider TEXT, + uri TEXT NOT NULL, + default_ref TEXT, + auth_ref_kind TEXT, + auth_ref_key TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, repository_id), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE +); +INSERT INTO repositories_v27 ( + workspace_id, repository_id, name, kind, provider, uri, default_ref, + auth_ref_kind, auth_ref_key, created_at, updated_at +) +SELECT workspace_id, repository_id, name, kind, provider, uri, default_ref, + auth_ref_kind, auth_ref_key, created_at, updated_at +FROM repositories; + +CREATE TABLE artifacts_v27 ( + workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + artifact_id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + uri TEXT NOT NULL, + media_type TEXT, + sha256 TEXT, + size_bytes INTEGER, + summary TEXT, + created_at TEXT NOT NULL, + created_by_kind TEXT NOT NULL, + created_by_key TEXT NOT NULL, + created_by_display TEXT NOT NULL, + created_by_source_kind TEXT, + created_by_source_key TEXT, + ticket_id TEXT, + objective_id TEXT, + event_id TEXT, + worker_ref_kind TEXT, + worker_ref_key TEXT, + worker_display TEXT, + repository_id TEXT, + source_kind TEXT, + source_revision TEXT, + FOREIGN KEY (workspace_id, repository_id) + REFERENCES repositories_v27(workspace_id, repository_id) +); +INSERT INTO artifacts_v27 ( + workspace_id, artifact_id, kind, uri, media_type, sha256, size_bytes, summary, + created_at, created_by_kind, created_by_key, created_by_display, + created_by_source_kind, created_by_source_key, ticket_id, objective_id, event_id, + worker_ref_kind, worker_ref_key, worker_display, repository_id, source_kind, + source_revision +) +SELECT workspace_id, artifact_id, kind, uri, media_type, sha256, size_bytes, summary, + created_at, created_by_kind, created_by_key, created_by_display, + created_by_source_kind, created_by_source_key, ticket_id, objective_id, event_id, + worker_ref_kind, worker_ref_key, worker_display, repository_id, source_kind, + source_revision +FROM artifacts; + +CREATE TABLE workdir_registry_v27 ( + workspace_id TEXT NOT NULL, + workdir_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + repository_id TEXT NOT NULL, + creation_selector TEXT, + creation_ref TEXT, + materialization_status TEXT NOT NULL CHECK (materialization_status IN ('pending', 'present', 'not_found', 'corrupted', 'unknown', 'failed')), + cleanliness TEXT NOT NULL CHECK (cleanliness IN ('clean', 'dirty', 'unknown')), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + current_selector TEXT, + current_ref TEXT, + PRIMARY KEY (workspace_id, workdir_id), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + FOREIGN KEY (workspace_id, repository_id) + REFERENCES repositories_v27(workspace_id, repository_id) +); +INSERT INTO workdir_registry_v27 ( + workspace_id, workdir_id, runtime_id, repository_id, + creation_selector, creation_ref, current_selector, current_ref, + materialization_status, cleanliness, created_at, updated_at +) +SELECT workspace_id, workdir_id, runtime_id, repository_id, + creation_selector, creation_ref, current_selector, current_ref, + materialization_status, cleanliness, created_at, updated_at +FROM workdir_registry; + +CREATE TABLE worker_workdir_links_v27 ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + runtime_worker_id INTEGER NOT NULL, + workdir_id TEXT NOT NULL, + role TEXT NOT NULL, + linked_at TEXT NOT NULL, + unlinked_at TEXT, + PRIMARY KEY (workspace_id, runtime_id, runtime_worker_id, workdir_id, role), + FOREIGN KEY (workspace_id, runtime_id, runtime_worker_id) + REFERENCES worker_registry(workspace_id, runtime_id, runtime_worker_id) ON DELETE CASCADE, + FOREIGN KEY (workspace_id, workdir_id) + REFERENCES workdir_registry_v27(workspace_id, workdir_id) ON DELETE CASCADE +); +INSERT INTO worker_workdir_links_v27 ( + workspace_id, runtime_id, runtime_worker_id, workdir_id, role, linked_at, unlinked_at +) +SELECT workspace_id, runtime_id, runtime_worker_id, workdir_id, role, linked_at, unlinked_at +FROM worker_workdir_links; + +CREATE TABLE worker_workdir_attachment_reservations_v27 ( + workspace_id TEXT NOT NULL, + workdir_id TEXT NOT NULL, + reservation_id TEXT NOT NULL, + reserved_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, workdir_id), + FOREIGN KEY (workspace_id, workdir_id) + REFERENCES workdir_registry_v27(workspace_id, workdir_id) ON DELETE CASCADE +); +INSERT INTO worker_workdir_attachment_reservations_v27 ( + workspace_id, workdir_id, reservation_id, reserved_at +) +SELECT workspace_id, workdir_id, reservation_id, reserved_at +FROM worker_workdir_attachment_reservations; + +DROP TABLE worker_workdir_links; +DROP TABLE worker_workdir_attachment_reservations; +DROP TABLE workdir_registry; +DROP TABLE artifacts; +DROP TABLE repositories; +ALTER TABLE repositories_v27 RENAME TO repositories; +ALTER TABLE artifacts_v27 RENAME TO artifacts; +ALTER TABLE workdir_registry_v27 RENAME TO workdir_registry; +ALTER TABLE worker_workdir_links_v27 RENAME TO worker_workdir_links; +ALTER TABLE worker_workdir_attachment_reservations_v27 + RENAME TO worker_workdir_attachment_reservations; + +CREATE INDEX idx_workdir_registry_workspace_updated + ON workdir_registry(workspace_id, updated_at DESC); +CREATE INDEX idx_worker_workdir_links_worker + ON worker_workdir_links(workspace_id, runtime_id, runtime_worker_id, linked_at DESC); +CREATE UNIQUE INDEX ux_worker_workdir_links_active_worker + ON worker_workdir_links(workspace_id, runtime_id, runtime_worker_id) + WHERE unlinked_at IS NULL; +CREATE UNIQUE INDEX ux_worker_workdir_links_active_workdir + ON worker_workdir_links(workspace_id, workdir_id) + WHERE unlinked_at IS NULL; +CREATE UNIQUE INDEX ux_worker_workdir_attachment_reservation_id + ON worker_workdir_attachment_reservations(workspace_id, reservation_id); +"#, + )?; + Ok(()) +} + +fn validate_workspace_repository_references(conn: &Connection) -> Result<()> { + for (table, repository_nullable) in [ + ("workdir_registry", false), + ("artifacts", true), + // `typed_tickets` is owned and migrated by the Ticket component. The control-plane + // migration may reject an already-invalid integrated reference, but must not rebuild + // that component table or claim its schema authority. + ("typed_tickets", true), + ] { + if !table_exists(conn, table)? || !column_exists(conn, table, "repository_id")? { + continue; + } + let null_filter = if repository_nullable { + "child.repository_id IS NOT NULL AND" + } else { + "" + }; + let sql = format!( + "SELECT child.workspace_id, child.repository_id FROM {table} AS child \ + WHERE {null_filter} NOT EXISTS (\ + SELECT 1 FROM repositories AS repository \ + WHERE repository.workspace_id = child.workspace_id \ + AND repository.repository_id = child.repository_id\ + ) LIMIT 1" + ); + let invalid = conn + .query_row(&sql, [], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .optional()?; + if let Some((workspace_id, repository_id)) = invalid { + return Err(Error::Store(format!( + "invalid Workspace-owned repository reference: {table} contains repository `{repository_id}` outside Workspace `{workspace_id}`" + ))); + } + } + Ok(()) +} + fn current_schema_version(conn: &Connection) -> Result { conn.query_row( "SELECT COALESCE(MAX(version), 0) FROM __yoi_schema_migrations", @@ -4618,7 +4846,7 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 26); + assert_eq!(current_schema_version(&conn).unwrap(), 27); assert!(table_exists(&conn, "worker_workdir_attachment_reservations").unwrap()); } @@ -4651,7 +4879,7 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 26); + assert_eq!(current_schema_version(&conn).unwrap(), 27); assert!(table_exists(&conn, "flow_sources").unwrap()); assert!(table_exists(&conn, "flow_source_revisions").unwrap()); assert!(!table_exists(&conn, "flow_instances").unwrap()); @@ -4659,13 +4887,246 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY); assert!(!table_exists(&conn, "flow_events").unwrap()); } + #[test] + fn schema_v27_upgrades_repository_identity_without_losing_workspace_owned_references() { + let conn = Connection::open_in_memory().unwrap(); + configure_sqlite(&conn).unwrap(); + for migration in MIGRATIONS + .iter() + .filter(|migration| migration.version <= 26) + { + let tx = conn.unchecked_transaction().unwrap(); + (migration.apply)(&tx).unwrap(); + tx.execute( + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", + params![migration.version, migration.name], + ) + .unwrap(); + tx.commit().unwrap(); + } + conn.execute_batch( + r#" +INSERT INTO workspaces ( + workspace_id, display_name, state, created_at, updated_at +) VALUES + ('workspace-a', 'Workspace A', 'active', '1', '1'), + ('workspace-b', 'Workspace B', 'active', '1', '1'); +INSERT INTO repositories ( + repository_id, workspace_id, name, kind, uri, created_at, updated_at +) VALUES ('main', 'workspace-a', 'Main', 'git', '/repo-a', '1', '1'); +INSERT INTO artifacts ( + workspace_id, artifact_id, kind, uri, created_at, + created_by_kind, created_by_key, created_by_display, repository_id +) VALUES ( + 'workspace-a', 'artifact-1', 'report', 'artifact://1', '1', + 'worker', 'worker-1', 'Worker 1', 'main' +); +INSERT INTO worker_registry ( + workspace_id, runtime_id, runtime_worker_id, display_name, + retention_state, created_at, updated_at +) VALUES ('workspace-a', 'runtime-a', 1, 'Worker 1', 'normal', '1', '1'); +INSERT INTO workdir_registry ( + workspace_id, workdir_id, runtime_id, repository_id, + creation_selector, creation_ref, materialization_status, + cleanliness, created_at, updated_at, current_selector, current_ref +) VALUES ( + 'workspace-a', 'workdir-1', 'runtime-a', 'main', + 'develop', 'abc', 'present', 'clean', '1', '1', 'develop', 'abc' +); +INSERT INTO worker_workdir_links ( + workspace_id, runtime_id, runtime_worker_id, workdir_id, + role, linked_at, unlinked_at +) VALUES ('workspace-a', 'runtime-a', 1, 'workdir-1', 'attachment', '1', NULL); +INSERT INTO worker_workdir_attachment_reservations ( + workspace_id, workdir_id, reservation_id, reserved_at +) VALUES ('workspace-a', 'workdir-1', 'reservation-1', '1'); +"#, + ) + .unwrap(); + + apply_migrations(&conn).unwrap(); + + assert_eq!(current_schema_version(&conn).unwrap(), 27); + let repositories_sql: String = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'repositories'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(repositories_sql.contains("PRIMARY KEY (workspace_id, repository_id)")); + let preserved: (i64, i64, i64) = ( + conn.query_row("SELECT COUNT(*) FROM artifacts", [], |row| row.get(0)) + .unwrap(), + conn.query_row("SELECT COUNT(*) FROM worker_workdir_links", [], |row| { + row.get(0) + }) + .unwrap(), + conn.query_row( + "SELECT COUNT(*) FROM worker_workdir_attachment_reservations", + [], + |row| row.get(0), + ) + .unwrap(), + ); + assert_eq!(preserved, (1, 1, 1)); + let foreign_key_violations: i64 = conn + .query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(foreign_key_violations, 0); + conn.execute( + r#"INSERT INTO repositories ( + workspace_id, repository_id, name, kind, uri, created_at, updated_at + ) VALUES ('workspace-b', 'main', 'Other Main', 'git', '/repo-b', '2', '2')"#, + [], + ) + .unwrap(); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM repositories WHERE repository_id = 'main'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 2 + ); + assert!( + conn.execute( + r#"INSERT INTO workdir_registry ( + workspace_id, workdir_id, runtime_id, repository_id, + materialization_status, cleanliness, created_at, updated_at + ) VALUES ('workspace-b', 'invalid', 'runtime-b', 'missing', + 'present', 'clean', '2', '2')"#, + [], + ) + .is_err() + ); + assert!( + conn.execute( + r#"INSERT INTO artifacts ( + workspace_id, artifact_id, kind, uri, created_at, + created_by_kind, created_by_key, created_by_display, repository_id + ) VALUES ('workspace-b', 'invalid-artifact', 'report', 'artifact://invalid', '2', + 'worker', 'worker-2', 'Worker 2', 'missing')"#, + [], + ) + .is_err() + ); + } + + #[test] + fn schema_v27_rejects_cross_workspace_legacy_repository_references() { + let conn = Connection::open_in_memory().unwrap(); + configure_sqlite(&conn).unwrap(); + for migration in MIGRATIONS + .iter() + .filter(|migration| migration.version <= 26) + { + let tx = conn.unchecked_transaction().unwrap(); + (migration.apply)(&tx).unwrap(); + tx.execute( + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", + params![migration.version, migration.name], + ) + .unwrap(); + tx.commit().unwrap(); + } + conn.execute_batch( + r#" +INSERT INTO workspaces ( + workspace_id, display_name, state, created_at, updated_at +) VALUES + ('workspace-a', 'Workspace A', 'active', '1', '1'), + ('workspace-b', 'Workspace B', 'active', '1', '1'); +INSERT INTO repositories ( + repository_id, workspace_id, name, kind, uri, created_at, updated_at +) VALUES ('main', 'workspace-a', 'Main', 'git', '/repo-a', '1', '1'); +INSERT INTO workdir_registry ( + workspace_id, workdir_id, runtime_id, repository_id, + materialization_status, cleanliness, created_at, updated_at +) VALUES ('workspace-b', 'foreign-workdir', 'runtime-b', 'main', + 'present', 'clean', '1', '1'); +"#, + ) + .unwrap(); + + let error = apply_migrations(&conn).unwrap_err(); + + assert!(error.to_string().contains("workdir_registry")); + assert!(error.to_string().contains("workspace-b")); + assert_eq!(current_schema_version(&conn).unwrap(), 26); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM workdir_registry", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); + } + + #[tokio::test] + async fn startup_rejects_cross_workspace_ticket_repository_reference_without_claiming_ticket_schema() + { + let dir = tempfile::tempdir().unwrap(); + let database_path = dir.path().join("workspace.sqlite"); + let store = SqliteWorkspaceStore::open(&database_path).unwrap(); + for workspace_id in ["workspace-a", "workspace-b"] { + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: workspace_id.to_string(), + owner_account_id: None, + display_name: workspace_id.to_string(), + state: "active".to_string(), + created_at: "1".to_string(), + updated_at: "1".to_string(), + }) + .await + .unwrap(); + } + store + .upsert_repository(&RepositoryRecord { + workspace_id: "workspace-a".to_string(), + repository_id: "main".to_string(), + name: "Main".to_string(), + kind: "git".to_string(), + provider: Some("git".to_string()), + uri: "/repo-a".to_string(), + default_ref: Some("HEAD".to_string()), + auth_ref_kind: None, + auth_ref_key: None, + created_at: "1".to_string(), + updated_at: "1".to_string(), + }) + .unwrap(); + drop(store); + + let backend = ticket::SqliteTicketBackend::open_verified( + database_path.clone(), + "workspace-b".to_string(), + ) + .unwrap(); + let mut input = ticket::NewTicket::new("Foreign repository"); + input.repository_id = Some("main".to_string()); + ticket::TicketBackend::create(&backend, input).unwrap(); + drop(backend); + + let error = match SqliteWorkspaceStore::open(&database_path) { + Ok(_) => panic!("cross-Workspace Ticket repository reference must fail closed"), + Err(error) => error, + }; + assert!(error.to_string().contains("typed_tickets")); + assert!(error.to_string().contains("workspace-b")); + } + #[tokio::test] async fn migrates_sqlite_and_preserves_workspace_record() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("control-plane.sqlite"); let store = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 26); + assert_eq!(store.schema_version().await.unwrap(), 27); assert!( !store .with_conn(|conn| table_exists(conn, "worker_workspace_credentials")) @@ -4682,7 +5143,7 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY); store.upsert_workspace(&record).await.unwrap(); let reopened = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(reopened.schema_version().await.unwrap(), 26); + assert_eq!(reopened.schema_version().await.unwrap(), 27); assert_eq!( reopened.get_workspace("local-dev").await.unwrap(), Some(record) @@ -5229,7 +5690,7 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY); .unwrap(); let store = SqliteWorkspaceStore::from_connection(conn).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 26); + assert_eq!(store.schema_version().await.unwrap(), 27); store .with_conn(|conn| { @@ -5418,7 +5879,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn repository_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 26); + assert_eq!(store.schema_version().await.unwrap(), 27); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -5443,20 +5904,48 @@ CREATE TABLE ticket_assignment_operations ( updated_at: "2".to_string(), }; store.upsert_repository(&repository).unwrap(); + assert_eq!( + store.get_repository("local-dev", "main").unwrap(), + Some(repository.clone()) + ); assert_eq!( store.list_repositories("local-dev").unwrap(), - vec![repository] + vec![repository.clone()] ); assert_eq!( store.list_repositories("other-workspace").unwrap(), Vec::new() ); + + let other_workspace = WorkspaceRecord { + workspace_id: "other-workspace".to_string(), + owner_account_id: None, + display_name: "Other Workspace".to_string(), + state: "active".to_string(), + created_at: "3".to_string(), + updated_at: "3".to_string(), + }; + store.upsert_workspace(&other_workspace).await.unwrap(); + let mut other_repository = repository.clone(); + other_repository.workspace_id = other_workspace.workspace_id.clone(); + other_repository.name = "Other Yoi".to_string(); + other_repository.uri = "/other/yoi".to_string(); + store.upsert_repository(&other_repository).unwrap(); + + assert_eq!( + store.get_repository("local-dev", "main").unwrap(), + Some(repository) + ); + assert_eq!( + store.get_repository("other-workspace", "main").unwrap(), + Some(other_repository) + ); } #[tokio::test] async fn memory_authority_records_round_trip_and_close_staging() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 26); + assert_eq!(store.schema_version().await.unwrap(), 27); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -5542,6 +6031,21 @@ CREATE TABLE ticket_assignment_operations ( updated_at: "1".to_string(), }; store.upsert_workspace(&workspace).await.unwrap(); + store + .upsert_repository(&RepositoryRecord { + workspace_id: workspace.workspace_id.clone(), + repository_id: "repo".to_string(), + name: "Repository".to_string(), + kind: "git".to_string(), + provider: Some("git".to_string()), + uri: ".".to_string(), + default_ref: Some("HEAD".to_string()), + auth_ref_kind: None, + auth_ref_key: None, + created_at: "1".to_string(), + updated_at: "1".to_string(), + }) + .unwrap(); let worker = WorkerRegistryRecord { workspace_id: "local-dev".to_string(), @@ -5704,7 +6208,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn account_and_login_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 26); + assert_eq!(store.schema_version().await.unwrap(), 27); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { account_id: "acct-user-alice".to_string(), From 1814c3570185ef17b8c8176751f64d45693fd3f0 Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 11 Aug 2026 19:57:03 +0900 Subject: [PATCH 03/12] feat: add merge request review authority --- .yoi/workflow/ticket-intake-workflow.md | 2 +- Cargo.lock | 12 + Cargo.toml | 3 + crates/merge-request/Cargo.toml | 14 + crates/merge-request/src/lib.rs | 1312 +++++++++++++++++ crates/merge-request/tests/store.rs | 404 +++++ crates/ticket/src/lib.rs | 165 +-- crates/ticket/src/sqlite_schema.rs | 67 +- crates/ticket/src/tool.rs | 95 +- crates/tui/src/dashboard/tests.rs | 8 +- crates/worker/src/feature/builtin.rs | 1 + .../src/feature/builtin/merge_request.rs | 299 ++++ crates/worker/src/feature/builtin/ticket.rs | 57 +- crates/worker/src/internal_worker.rs | 4 + crates/worker/src/spawn/tool.rs | 171 ++- crates/worker/src/worker.rs | 118 ++ crates/workspace-server/Cargo.toml | 1 + crates/workspace-server/src/lib.rs | 10 + crates/workspace-server/src/server.rs | 559 ++++++- crates/workspace-server/src/store.rs | 1 + crates/yoi/src/ticket_cli.rs | 138 +- docs/development/work-items.md | 12 +- resources/flows/coder-review.dcdl | 20 +- resources/profiles/reviewer.dcdl | 2 +- resources/prompts/role/coder.md | 8 +- resources/prompts/role/reviewer.md | 8 +- .../console/worker-console.ui.test.ts | 4 +- .../tickets/[ticketId]/+page.svelte | 80 +- .../[workspaceId]/tickets/[ticketId]/+page.ts | 49 +- 29 files changed, 3069 insertions(+), 555 deletions(-) create mode 100644 crates/merge-request/Cargo.toml create mode 100644 crates/merge-request/src/lib.rs create mode 100644 crates/merge-request/tests/store.rs create mode 100644 crates/worker/src/feature/builtin/merge_request.rs diff --git a/.yoi/workflow/ticket-intake-workflow.md b/.yoi/workflow/ticket-intake-workflow.md index e195c70b..f2f044e5 100644 --- a/.yoi/workflow/ticket-intake-workflow.md +++ b/.yoi/workflow/ticket-intake-workflow.md @@ -76,7 +76,7 @@ Intake は以下を行う。 - `TicketComment`: 既存 Ticket refinement / decision / plan の記録。 - `TicketDoctor`: 必要に応じた整合性確認。 -Intake は `TicketReview`, `TicketWorkflowState`, `TicketClose` を通常使わない。review / state transition / close は Orchestrator または reviewer / maintainer workflow の責務である。 +Intake は `MergeRequest*`, `TicketWorkflowState`, `TicketClose` を通常使わない。review authority は assigned Coder が起動した read-only direct-child Reviewer の immutable Merge Request attempt に属し、completion / merge / close は各guarded workflowの責務である。 Ticket tools が利用できない環境では、勝手に file write で代替しない。ユーザーまたは Orchestrator に「Ticket tools がないため materialize できない」と報告し、必要なら `yoi ticket` を使える人間/親 workflow に戻す。 diff --git a/Cargo.lock b/Cargo.lock index 14427abf..5b925ccb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2481,6 +2481,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "merge-request" +version = "0.1.0" +dependencies = [ + "rusqlite", + "serde", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", +] + [[package]] name = "mime" version = "0.3.17" @@ -6164,6 +6175,7 @@ dependencies = [ "futures", "manifest", "memory", + "merge-request", "project-record", "protocol", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index b7cec2ce..e8f55dcd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "crates/tui", "crates/memory", "crates/ticket", + "crates/merge-request", "crates/project-record", "crates/workspace-server", "tests/e2e", @@ -50,6 +51,7 @@ default-members = [ "crates/tui", "crates/memory", "crates/ticket", + "crates/merge-request", "crates/project-record", "crates/workspace-server", ] @@ -67,6 +69,7 @@ manifest = { path = "crates/manifest" } mcp = { path = "crates/mcp" } lint-common = { path = "crates/lint-common" } memory = { path = "crates/memory" } +merge-request = { path = "crates/merge-request" } ticket = { path = "crates/ticket" } project-record = { path = "crates/project-record" } worker = { path = "crates/worker" } diff --git a/crates/merge-request/Cargo.toml b/crates/merge-request/Cargo.toml new file mode 100644 index 00000000..d5865a9c --- /dev/null +++ b/crates/merge-request/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "merge-request" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +rusqlite.workspace = true +serde = { workspace = true, features = ["derive"] } +sha2.workspace = true +thiserror.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/merge-request/src/lib.rs b/crates/merge-request/src/lib.rs new file mode 100644 index 00000000..22844355 --- /dev/null +++ b/crates/merge-request/src/lib.rs @@ -0,0 +1,1312 @@ +//! Workspace-scoped Merge Request authority. +//! +//! Merge Requests deliberately do not reuse Ticket thread review events. A review is +//! evidence for one immutable revision and can only be committed with a one-shot +//! capability registered from an actual Runtime-owned direct-child reviewer session. + +use rusqlite::{Connection, OptionalExtension, params}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use thiserror::Error; + +const SCHEMA_VERSION: i64 = 7; +const REVIEWER_PROFILE: &str = "builtin:reviewer"; +const MAX_SUMMARY_BYTES: usize = 16 * 1024; +const MAX_REVIEW_BODY_BYTES: usize = 64 * 1024; +const MAX_CHANGED_PATHS: usize = 1_000; +const MAX_FINDINGS: usize = 1_000; +const MAX_FIELD_BYTES: usize = 4 * 1024; + +pub type Result = std::result::Result; + +#[derive(Debug, Error)] +pub enum MergeRequestError { + #[error("merge request database error: {0}")] + Database(String), + #[error("{0} must not be empty")] + Empty(&'static str), + #[error("{field} exceeds its bounded limit of {max} bytes/items")] + TooLarge { field: &'static str, max: usize }, + #[error("merge request not found for ticket {0}")] + NotFound(String), + #[error("merge request already exists for ticket {0}")] + AlreadyExists(String), + #[error("immutable revision {0} already exists with different content")] + RevisionConflict(String), + #[error("stale merge request revision: expected {expected}, current {current}")] + StaleRevision { expected: String, current: String }, + #[error("current Ticket assignment does not match the authenticated Coder")] + AssignmentMismatch, + #[error("reviewer must be an actual direct-child with effective profile builtin:reviewer")] + InvalidReviewer, + #[error("review attempt is invalid, revoked, already used, or belongs to another revision")] + InvalidReviewAttempt, + #[error("review result cannot be supplied by the assigned Coder itself")] + SelfApproval, + #[error("merge request current revision is not approved")] + NotApproved, + #[error("merge request is {0}, expected open")] + NotOpen(String), + #[error("completion operation id was reused with different input")] + OperationConflict, + #[error("Ticket must be inprogress before Merge Request completion (current: {0})")] + TicketStateConflict(String), + #[error("only an authenticated user with explicit confirmation may merge")] + MergeConfirmationRequired, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MergeRequestState { + Draft, + Open, + Closed, + Merged, +} + +impl MergeRequestState { + fn as_str(self) -> &'static str { + match self { + Self::Draft => "draft", + Self::Open => "open", + Self::Closed => "closed", + Self::Merged => "merged", + } + } + + fn parse(value: &str) -> Self { + match value { + "draft" => Self::Draft, + "closed" => Self::Closed, + "merged" => Self::Merged, + _ => Self::Open, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReviewDecision { + Approve, + RequestChanges, +} + +impl ReviewDecision { + fn as_str(self) -> &'static str { + match self { + Self::Approve => "approve", + Self::RequestChanges => "request_changes", + } + } + + fn parse(value: &str) -> Self { + match value { + "approve" => Self::Approve, + _ => Self::RequestChanges, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReviewStatus { + Pending, + Approved, + ChangesRequested, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MergeRequestRevision { + pub revision_id: String, + pub ordinal: u64, + pub base_commit: String, + pub head_commit: String, + pub head_tree: String, + pub diff_digest: String, + pub changed_paths: Vec, + pub summary: String, + pub assignment_id: String, + pub created_at: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReviewFinding { + pub severity: String, + pub code: Option, + pub path: Option, + pub line: Option, + pub body: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MergeRequestReview { + pub attempt_id: String, + pub revision_id: String, + pub decision: ReviewDecision, + pub body: String, + pub findings: Vec, + pub parent_assignment_id: String, + pub parent_runtime_id: String, + pub parent_worker_id: String, + pub reviewer_child_session_id: String, + pub reviewer_effective_profile: String, + pub submitted_at: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MergeRequest { + pub merge_request_id: String, + pub workspace_id: String, + pub ticket_id: String, + pub repository_id: String, + pub state: MergeRequestState, + pub lifecycle_generation: u64, + pub current_revision: MergeRequestRevision, + pub review_status: ReviewStatus, + pub current_review: Option, + pub created_at: String, + pub updated_at: String, + pub merged_by_account_id: Option, + pub merged_at: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OpenMergeRequest { + pub merge_request_id: String, + pub ticket_id: String, + pub repository_id: String, + pub revision: MergeRequestRevision, + pub authenticated_runtime_id: String, + pub authenticated_worker_id: String, + pub now: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AddRevision { + pub ticket_id: String, + pub expected_current_revision_id: String, + pub revision: MergeRequestRevision, + pub authenticated_runtime_id: String, + pub authenticated_worker_id: String, + pub now: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RegisterReviewerChildSession { + pub parent_runtime_id: String, + pub parent_worker_id: String, + pub child_session_id: String, + pub now: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RegisterReviewAttempt { + pub attempt_id: String, + pub ticket_id: String, + pub revision_id: String, + pub parent_assignment_id: String, + pub parent_runtime_id: String, + pub parent_worker_id: String, + pub child_session_id: String, + /// A secret generated by the trusted spawn layer and injected only into the child client. + pub capability_token: String, + pub now: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SubmitReview { + pub ticket_id: String, + pub revision_id: String, + pub capability_token: String, + pub decision: ReviewDecision, + pub body: String, + pub findings: Vec, + pub now: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompleteMergeRequest { + pub operation_id: String, + pub ticket_id: String, + pub expected_revision_id: String, + pub assignment_id: String, + pub authenticated_runtime_id: String, + pub authenticated_worker_id: String, + pub now: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CompletionOutcome { + pub operation_id: String, + pub ticket_id: String, + pub revision_id: String, + pub ticket_state: String, + pub replayed: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MergeRequestReadiness { + pub ticket_id: String, + pub merge_request_id: String, + pub revision_id: String, + pub ready: bool, + pub review_status: ReviewStatus, + pub blockers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MergeConfirmation { + pub ticket_id: String, + pub expected_revision_id: String, + pub authenticated_account_id: String, + pub actor_kind: String, + pub explicit_confirmation: bool, + pub now: String, +} + +#[derive(Clone, Debug)] +pub struct SqliteMergeRequestStore { + db_path: PathBuf, + workspace_id: String, +} + +impl SqliteMergeRequestStore { + pub fn open(db_path: impl Into, workspace_id: impl Into) -> Result { + let store = Self { + db_path: db_path.into(), + workspace_id: workspace_id.into(), + }; + let conn = store.connect()?; + migrate(&conn)?; + Ok(store) + } + + pub fn open_verified( + db_path: impl Into, + workspace_id: impl Into, + ) -> Result { + let store = Self { + db_path: db_path.into(), + workspace_id: workspace_id.into(), + }; + verify(&store.connect()?)?; + Ok(store) + } + + pub fn db_path(&self) -> &Path { + &self.db_path + } + + pub fn workspace_id(&self) -> &str { + &self.workspace_id + } + + fn connect(&self) -> Result { + let conn = Connection::open(&self.db_path).map_err(db)?; + conn.busy_timeout(Duration::from_secs(5)).map_err(db)?; + conn.pragma_update(None, "foreign_keys", "ON").map_err(db)?; + Ok(conn) + } + + fn write(&self, op: impl FnOnce(&Connection) -> Result) -> Result { + let conn = self.connect()?; + verify(&conn)?; + conn.execute_batch("BEGIN IMMEDIATE").map_err(db)?; + match op(&conn) { + Ok(value) => { + conn.execute_batch("COMMIT").map_err(db)?; + Ok(value) + } + Err(error) => { + let _ = conn.execute_batch("ROLLBACK"); + Err(error) + } + } + } + + pub fn show_for_ticket(&self, ticket_id: &str) -> Result> { + nonempty("ticket_id", ticket_id)?; + let conn = self.connect()?; + verify(&conn)?; + load_merge_request(&conn, &self.workspace_id, ticket_id) + } + + pub fn readiness_for_ticket(&self, ticket_id: &str) -> Result { + let mr = self + .show_for_ticket(ticket_id)? + .ok_or_else(|| MergeRequestError::NotFound(ticket_id.to_string()))?; + let mut blockers = Vec::new(); + if mr.state != MergeRequestState::Open { + blockers.push(format!("merge request is {}", mr.state.as_str())); + } + match mr.review_status { + ReviewStatus::Pending => blockers.push("current revision has no review result".into()), + ReviewStatus::ChangesRequested => { + blockers.push("current revision has request_changes".into()) + } + ReviewStatus::Approved => {} + } + Ok(MergeRequestReadiness { + ticket_id: ticket_id.to_string(), + merge_request_id: mr.merge_request_id, + revision_id: mr.current_revision.revision_id, + ready: blockers.is_empty(), + review_status: mr.review_status, + blockers, + }) + } + + pub fn open_merge_request(&self, input: OpenMergeRequest) -> Result { + validate_revision(&input.revision)?; + for (name, value) in [ + ("merge_request_id", input.merge_request_id.as_str()), + ("ticket_id", input.ticket_id.as_str()), + ("repository_id", input.repository_id.as_str()), + ("runtime_id", input.authenticated_runtime_id.as_str()), + ("worker_id", input.authenticated_worker_id.as_str()), + ] { + nonempty(name, value)?; + } + self.write(|conn| { + validate_current_assignment( + conn, + &self.workspace_id, + &input.ticket_id, + &input.revision.assignment_id, + &input.authenticated_runtime_id, + &input.authenticated_worker_id, + )?; + conn.execute( + "INSERT INTO merge_requests (workspace_id, merge_request_id, repository_id, state, lifecycle_generation, current_revision_id, created_at, updated_at) VALUES (?1, ?2, ?3, 'open', 1, ?4, ?5, ?5)", + params![self.workspace_id, input.merge_request_id, input.repository_id, input.revision.revision_id, input.now], + ).map_err(db)?; + conn.execute( + "INSERT INTO merge_request_ticket_relations (workspace_id,merge_request_id,ticket_id,relation_kind,created_at) VALUES (?1,?2,?3,'implements',?4)", + params![self.workspace_id,input.merge_request_id,input.ticket_id,input.now], + ).map_err(db)?; + insert_revision(conn, &self.workspace_id, &input.merge_request_id, &input.revision)?; + load_merge_request(conn, &self.workspace_id, &input.ticket_id)?.ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone())) + }) + } + + pub fn add_revision(&self, input: AddRevision) -> Result { + validate_revision(&input.revision)?; + self.write(|conn| { + let current = load_merge_request(conn, &self.workspace_id, &input.ticket_id)? + .ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone()))?; + ensure_open(¤t)?; + if current.current_revision.revision_id != input.expected_current_revision_id { + return Err(MergeRequestError::StaleRevision { + expected: input.expected_current_revision_id.clone(), + current: current.current_revision.revision_id, + }); + } + validate_current_assignment( + conn, + &self.workspace_id, + &input.ticket_id, + &input.revision.assignment_id, + &input.authenticated_runtime_id, + &input.authenticated_worker_id, + )?; + if input.revision.ordinal != current.current_revision.ordinal + 1 { + return Err(MergeRequestError::RevisionConflict(input.revision.revision_id.clone())); + } + let existing: Option<(String, String, String, String)> = conn.query_row( + "SELECT base_commit, head_commit, head_tree, diff_digest FROM merge_request_revisions WHERE workspace_id=?1 AND merge_request_id=?2 AND revision_id=?3", + params![self.workspace_id, current.merge_request_id, input.revision.revision_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ).optional().map_err(db)?; + if let Some(existing) = existing { + if existing == (input.revision.base_commit.clone(), input.revision.head_commit.clone(), input.revision.head_tree.clone(), input.revision.diff_digest.clone()) { + return Ok(current); + } + return Err(MergeRequestError::RevisionConflict(input.revision.revision_id.clone())); + } + insert_revision(conn, &self.workspace_id, ¤t.merge_request_id, &input.revision)?; + conn.execute( + "UPDATE merge_requests SET current_revision_id=?3, updated_at=?4 WHERE workspace_id=?1 AND merge_request_id=?2 AND current_revision_id=?5", + params![self.workspace_id, current.merge_request_id, input.revision.revision_id, input.now, input.expected_current_revision_id], + ).map_err(db)?; + load_merge_request(conn, &self.workspace_id, &input.ticket_id)?.ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone())) + }) + } + + pub fn register_reviewer_child_session( + &self, + input: RegisterReviewerChildSession, + ) -> Result<()> { + nonempty("runtime_id", &input.parent_runtime_id)?; + nonempty("worker_id", &input.parent_worker_id)?; + nonempty("child_session_id", &input.child_session_id)?; + self.write(|conn| { + conn.execute( + "INSERT INTO merge_request_reviewer_child_sessions (workspace_id,child_session_id,parent_runtime_id,parent_worker_id,effective_profile,registered_at) VALUES (?1,?2,?3,?4,'builtin:reviewer',?5)", + params![self.workspace_id,input.child_session_id,input.parent_runtime_id,input.parent_worker_id,input.now], + ).map_err(|_| MergeRequestError::InvalidReviewer)?; + Ok(()) + }) + } + + pub fn register_review_attempt(&self, input: RegisterReviewAttempt) -> Result<()> { + for (name, value) in [ + ("attempt_id", input.attempt_id.as_str()), + ("capability_token", input.capability_token.as_str()), + ("child_session_id", input.child_session_id.as_str()), + ] { + nonempty(name, value)?; + } + if input.child_session_id == input.parent_worker_id { + return Err(MergeRequestError::SelfApproval); + } + self.write(|conn| { + let mr = load_merge_request(conn, &self.workspace_id, &input.ticket_id)? + .ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone()))?; + ensure_open(&mr)?; + if mr.current_revision.revision_id != input.revision_id { + return Err(MergeRequestError::StaleRevision { expected: input.revision_id.clone(), current: mr.current_revision.revision_id }); + } + validate_current_assignment(conn, &self.workspace_id, &input.ticket_id, &input.parent_assignment_id, &input.parent_runtime_id, &input.parent_worker_id)?; + let effective_profile: Option = conn.query_row( + "SELECT effective_profile FROM merge_request_reviewer_child_sessions WHERE workspace_id=?1 AND child_session_id=?2 AND parent_runtime_id=?3 AND parent_worker_id=?4", + params![self.workspace_id,input.child_session_id,input.parent_runtime_id,input.parent_worker_id], + |row| row.get(0), + ).optional().map_err(db)?; + if effective_profile.as_deref() != Some(REVIEWER_PROFILE) { + return Err(MergeRequestError::InvalidReviewer); + } + conn.execute( + "INSERT INTO merge_request_review_attempts (workspace_id, attempt_id, merge_request_id, ticket_id, revision_id, lifecycle_generation, parent_assignment_id, parent_runtime_id, parent_worker_id, child_session_id, child_effective_profile, capability_token_sha256, status, created_at) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,'open',?13)", + params![self.workspace_id, input.attempt_id, mr.merge_request_id, input.ticket_id, input.revision_id, mr.lifecycle_generation as i64, input.parent_assignment_id, input.parent_runtime_id, input.parent_worker_id, input.child_session_id, REVIEWER_PROFILE, token_hash(&input.capability_token), input.now], + ).map_err(|_| MergeRequestError::InvalidReviewAttempt)?; + Ok(()) + }) + } + + pub fn revoke_review_attempt( + &self, + attempt_id: &str, + child_session_id: &str, + now: &str, + ) -> Result { + self.write(|conn| { + let changed = conn.execute( + "UPDATE merge_request_review_attempts SET status='revoked', consumed_at=?4 WHERE workspace_id=?1 AND attempt_id=?2 AND child_session_id=?3 AND status='open'", + params![self.workspace_id, attempt_id, child_session_id, now], + ).map_err(db)?; + Ok(changed == 1) + }) + } + + pub fn submit_review(&self, input: SubmitReview) -> Result { + nonempty("capability_token", &input.capability_token)?; + validate_review_input(&input)?; + self.write(|conn| { + let token = token_hash(&input.capability_token); + let attempt: Option<(String,String,String,String,String,String,String,String,i64)> = conn.query_row( + "SELECT attempt_id, merge_request_id, parent_assignment_id, parent_runtime_id, parent_worker_id, child_session_id, child_effective_profile, status, lifecycle_generation FROM merge_request_review_attempts WHERE workspace_id=?1 AND ticket_id=?2 AND revision_id=?3 AND capability_token_sha256=?4", + params![self.workspace_id, input.ticket_id, input.revision_id, token], + |row| Ok((row.get(0)?,row.get(1)?,row.get(2)?,row.get(3)?,row.get(4)?,row.get(5)?,row.get(6)?,row.get(7)?,row.get(8)?)), + ).optional().map_err(db)?; + let Some((attempt_id, mr_id, assignment_id, runtime_id, worker_id, child_session_id, effective_profile, status, lifecycle_generation)) = attempt else { + return Err(MergeRequestError::InvalidReviewAttempt); + }; + if status != "open" || effective_profile != REVIEWER_PROFILE || child_session_id == worker_id { + return Err(MergeRequestError::InvalidReviewAttempt); + } + let mr = load_merge_request(conn, &self.workspace_id, &input.ticket_id)? + .ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone()))?; + if lifecycle_generation != mr.lifecycle_generation as i64 { + return Err(MergeRequestError::InvalidReviewAttempt); + } + ensure_open(&mr)?; + if mr.current_revision.revision_id != input.revision_id { + return Err(MergeRequestError::StaleRevision { expected: input.revision_id.clone(), current: mr.current_revision.revision_id }); + } + validate_current_assignment(conn, &self.workspace_id, &input.ticket_id, &assignment_id, &runtime_id, &worker_id)?; + conn.execute( + "INSERT INTO merge_request_reviews (workspace_id, attempt_id, merge_request_id, revision_id, decision, body, submitted_at) VALUES (?1,?2,?3,?4,?5,?6,?7)", + params![self.workspace_id, attempt_id, mr_id, input.revision_id, input.decision.as_str(), input.body, input.now], + ).map_err(|_| MergeRequestError::InvalidReviewAttempt)?; + for (ordinal, finding) in input.findings.iter().enumerate() { + nonempty("finding.body", &finding.body)?; + conn.execute( + "INSERT INTO merge_request_review_findings (workspace_id, attempt_id, ordinal, severity, code, path, line, body) VALUES (?1,?2,?3,?4,?5,?6,?7,?8)", + params![self.workspace_id, attempt_id, ordinal as i64, finding.severity, finding.code, finding.path, finding.line.map(|v| v as i64), finding.body], + ).map_err(db)?; + } + conn.execute( + "UPDATE merge_request_review_attempts SET status='submitted', consumed_at=?3 WHERE workspace_id=?1 AND attempt_id=?2 AND status='open'", + params![self.workspace_id, attempt_id, input.now], + ).map_err(db)?; + load_review(conn, &self.workspace_id, &attempt_id)?.ok_or(MergeRequestError::InvalidReviewAttempt) + }) + } + + pub fn complete(&self, input: CompleteMergeRequest) -> Result { + for (name, value) in [ + ("operation_id", input.operation_id.as_str()), + ("ticket_id", input.ticket_id.as_str()), + ("revision_id", input.expected_revision_id.as_str()), + ] { + nonempty(name, value)?; + } + let fingerprint = completion_fingerprint(&input); + self.write(|conn| { + if let Some((stored, status, state)) = conn.query_row( + "SELECT fingerprint, status, result_ticket_state FROM merge_request_completion_operations WHERE workspace_id=?1 AND operation_id=?2", + params![self.workspace_id, input.operation_id], + |row| Ok((row.get::<_,String>(0)?, row.get::<_,String>(1)?, row.get::<_,Option>(2)?)), + ).optional().map_err(db)? { + if stored != fingerprint { return Err(MergeRequestError::OperationConflict); } + if status == "completed" { + return Ok(CompletionOutcome { operation_id: input.operation_id.clone(), ticket_id: input.ticket_id.clone(), revision_id: input.expected_revision_id.clone(), ticket_state: state.unwrap_or_else(|| "done".into()), replayed: true }); + } + } else { + conn.execute( + "INSERT INTO merge_request_completion_operations (workspace_id, operation_id, ticket_id, revision_id, assignment_id, fingerprint, status, created_at, updated_at) VALUES (?1,?2,?3,?4,?5,?6,'pending',?7,?7)", + params![self.workspace_id, input.operation_id, input.ticket_id, input.expected_revision_id, input.assignment_id, fingerprint, input.now], + ).map_err(db)?; + } + let mr = load_merge_request(conn, &self.workspace_id, &input.ticket_id)? + .ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone()))?; + ensure_open(&mr)?; + if mr.current_revision.revision_id != input.expected_revision_id { + return Err(MergeRequestError::StaleRevision { expected: input.expected_revision_id.clone(), current: mr.current_revision.revision_id }); + } + validate_current_assignment(conn, &self.workspace_id, &input.ticket_id, &input.assignment_id, &input.authenticated_runtime_id, &input.authenticated_worker_id)?; + if mr.review_status != ReviewStatus::Approved { return Err(MergeRequestError::NotApproved); } + let current_state: String = conn.query_row( + "SELECT workflow_state FROM typed_tickets WHERE workspace_id=?1 AND ticket_id=?2", + params![self.workspace_id, input.ticket_id], |row| row.get(0), + ).optional().map_err(db)?.ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone()))?; + if current_state != "inprogress" { + return Err(MergeRequestError::TicketStateConflict(current_state)); + } + let changed = conn.execute( + "UPDATE typed_tickets SET workflow_state='done', workflow_state_explicit=1, updated_at=?3 WHERE workspace_id=?1 AND ticket_id=?2 AND workflow_state='inprogress'", + params![self.workspace_id, input.ticket_id, input.now], + ).map_err(db)?; + if changed != 1 { return Err(MergeRequestError::TicketStateConflict("concurrent_change".into())); } + append_completion_event(conn, &self.workspace_id, &input)?; + conn.execute( + "UPDATE merge_request_completion_operations SET status='completed', result_ticket_state='done', updated_at=?3 WHERE workspace_id=?1 AND operation_id=?2 AND status='pending'", + params![self.workspace_id, input.operation_id, input.now], + ).map_err(db)?; + Ok(CompletionOutcome { operation_id: input.operation_id, ticket_id: input.ticket_id, revision_id: input.expected_revision_id, ticket_state: "done".into(), replayed: false }) + }) + } + + pub fn close( + &self, + ticket_id: &str, + expected_revision_id: &str, + now: &str, + ) -> Result { + self.transition_open(ticket_id, expected_revision_id, "closed", now) + } + + pub fn reopen( + &self, + ticket_id: &str, + expected_revision_id: &str, + now: &str, + ) -> Result { + self.write(|conn| { + let mr = load_merge_request(conn, &self.workspace_id, ticket_id)?.ok_or_else(|| MergeRequestError::NotFound(ticket_id.into()))?; + if mr.state != MergeRequestState::Closed { return Err(MergeRequestError::NotOpen(mr.state.as_str().into())); } + if mr.current_revision.revision_id != expected_revision_id { return Err(MergeRequestError::StaleRevision { expected: expected_revision_id.into(), current: mr.current_revision.revision_id }); } + conn.execute("UPDATE merge_requests SET state='open', lifecycle_generation=lifecycle_generation+1, updated_at=?3 WHERE workspace_id=?1 AND merge_request_id=?2 AND state='closed'", params![self.workspace_id, mr.merge_request_id, now]).map_err(db)?; + load_merge_request(conn, &self.workspace_id, ticket_id)?.ok_or_else(|| MergeRequestError::NotFound(ticket_id.into())) + }) + } + + pub fn confirm_merge(&self, input: MergeConfirmation) -> Result { + if !input.explicit_confirmation + || input.actor_kind != "user" + || input.authenticated_account_id.trim().is_empty() + { + return Err(MergeRequestError::MergeConfirmationRequired); + } + self.write(|conn| { + let mr = load_merge_request(conn, &self.workspace_id, &input.ticket_id)?.ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone()))?; + ensure_open(&mr)?; + if mr.current_revision.revision_id != input.expected_revision_id { return Err(MergeRequestError::StaleRevision { expected: input.expected_revision_id.clone(), current: mr.current_revision.revision_id }); } + if mr.review_status != ReviewStatus::Approved { return Err(MergeRequestError::NotApproved); } + let ticket_state: String = conn.query_row("SELECT workflow_state FROM typed_tickets WHERE workspace_id=?1 AND ticket_id=?2", params![self.workspace_id, input.ticket_id], |row| row.get(0)).map_err(db)?; + if ticket_state != "done" { return Err(MergeRequestError::TicketStateConflict(ticket_state)); } + conn.execute("UPDATE merge_requests SET state='merged', merged_by_account_id=?3, merged_at=?4, updated_at=?4 WHERE workspace_id=?1 AND merge_request_id=?2 AND state='open'", params![self.workspace_id, mr.merge_request_id, input.authenticated_account_id, input.now]).map_err(db)?; + load_merge_request(conn, &self.workspace_id, &input.ticket_id)?.ok_or_else(|| MergeRequestError::NotFound(input.ticket_id.clone())) + }) + } + + fn transition_open( + &self, + ticket_id: &str, + expected_revision_id: &str, + state: &str, + now: &str, + ) -> Result { + self.write(|conn| { + let mr = load_merge_request(conn, &self.workspace_id, ticket_id)?.ok_or_else(|| MergeRequestError::NotFound(ticket_id.into()))?; + ensure_open(&mr)?; + if mr.current_revision.revision_id != expected_revision_id { return Err(MergeRequestError::StaleRevision { expected: expected_revision_id.into(), current: mr.current_revision.revision_id }); } + conn.execute("UPDATE merge_requests SET state=?3, updated_at=?4 WHERE workspace_id=?1 AND merge_request_id=?2 AND state='open'", params![self.workspace_id, mr.merge_request_id, state, now]).map_err(db)?; + load_merge_request(conn, &self.workspace_id, ticket_id)?.ok_or_else(|| MergeRequestError::NotFound(ticket_id.into())) + }) + } +} + +pub fn migrate(conn: &Connection) -> Result<()> { + conn.pragma_update(None, "foreign_keys", "ON").map_err(db)?; + conn.execute_batch("CREATE TABLE IF NOT EXISTS merge_request_schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);").map_err(db)?; + let version: i64 = conn + .query_row( + "SELECT COALESCE(MAX(version),0) FROM merge_request_schema_migrations", + [], + |row| row.get(0), + ) + .map_err(db)?; + archive_incompatible_legacy_tables(conn, version)?; + conn.execute_batch(SCHEMA_V1).map_err(db)?; + if version < 1 { + conn.execute( + "INSERT INTO merge_request_schema_migrations(version) VALUES (1)", + [], + ) + .map_err(db)?; + } + if version < SCHEMA_VERSION { + // Version 6 is the fresh bounded-context authority marker. Versions 1..=5 + // were emitted by the retired implementation; their relational evidence is + // preserved and revalidated by the current typed store rather than rewritten. + if column_exists(conn, "merge_request_schema_migrations", "name")? { + conn.execute( + "INSERT OR IGNORE INTO merge_request_schema_migrations(version,name) VALUES (?1,'fresh_bounded_context_authority')", + params![SCHEMA_VERSION], + ).map_err(db)?; + } else { + conn.execute( + "INSERT OR IGNORE INTO merge_request_schema_migrations(version) VALUES (?1)", + params![SCHEMA_VERSION], + ) + .map_err(db)?; + } + } + verify(conn) +} + +pub fn verify(conn: &Connection) -> Result<()> { + let version: i64 = conn + .query_row( + "SELECT COALESCE(MAX(version),0) FROM merge_request_schema_migrations", + [], + |row| row.get(0), + ) + .map_err(db)?; + if !(1..=SCHEMA_VERSION).contains(&version) { + return Err(MergeRequestError::Database(format!( + "unsupported merge request schema version {version}, expected at most {SCHEMA_VERSION}" + ))); + } + for table in [ + "merge_requests", + "merge_request_ticket_relations", + "merge_request_revisions", + "merge_request_revision_paths", + "merge_request_reviewer_child_sessions", + "merge_request_review_attempts", + "merge_request_reviews", + "merge_request_review_findings", + "merge_request_completion_operations", + ] { + let present: Option = conn + .query_row( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1", + params![table], + |row| row.get(0), + ) + .optional() + .map_err(db)?; + if present.is_none() { + return Err(MergeRequestError::Database(format!( + "missing table {table}" + ))); + } + } + for (table, required) in [ + ( + "merge_requests", + &[ + "workspace_id", + "merge_request_id", + "repository_id", + "state", + "lifecycle_generation", + "current_revision_id", + ] as &[_], + ), + ( + "merge_request_ticket_relations", + &[ + "workspace_id", + "merge_request_id", + "ticket_id", + "relation_kind", + ] as &[_], + ), + ( + "merge_request_revisions", + &[ + "workspace_id", + "merge_request_id", + "revision_id", + "ordinal", + "base_commit", + "head_commit", + "head_tree", + "diff_digest", + "assignment_id", + ] as &[_], + ), + ( + "merge_request_reviewer_child_sessions", + &[ + "workspace_id", + "child_session_id", + "parent_runtime_id", + "parent_worker_id", + "effective_profile", + ] as &[_], + ), + ( + "merge_request_review_attempts", + &[ + "workspace_id", + "attempt_id", + "merge_request_id", + "ticket_id", + "revision_id", + "lifecycle_generation", + "parent_assignment_id", + "parent_runtime_id", + "parent_worker_id", + "child_session_id", + "child_effective_profile", + "capability_token_sha256", + "status", + ] as &[_], + ), + ( + "merge_request_reviews", + &[ + "workspace_id", + "attempt_id", + "merge_request_id", + "revision_id", + "decision", + "body", + ] as &[_], + ), + ( + "merge_request_completion_operations", + &[ + "workspace_id", + "operation_id", + "ticket_id", + "revision_id", + "assignment_id", + "fingerprint", + "status", + "result_ticket_state", + ] as &[_], + ), + ] { + let mut statement = conn + .prepare(&format!("PRAGMA table_info({table})")) + .map_err(db)?; + let columns = statement + .query_map([], |row| row.get::<_, String>(1)) + .map_err(db)? + .collect::, _>>() + .map_err(db)?; + for column in required { + if !columns.iter().any(|actual| actual == column) { + return Err(MergeRequestError::Database(format!( + "schema drift: table {table} is missing required column {column}" + ))); + } + } + } + Ok(()) +} + +const SCHEMA_V1: &str = r#" +CREATE TABLE IF NOT EXISTS merge_requests ( + workspace_id TEXT NOT NULL, merge_request_id TEXT NOT NULL, + repository_id TEXT NOT NULL, state TEXT NOT NULL CHECK(state IN ('draft','open','closed','merged')), + lifecycle_generation INTEGER NOT NULL, current_revision_id TEXT NOT NULL, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL, merged_by_account_id TEXT, merged_at TEXT, + PRIMARY KEY(workspace_id,merge_request_id), + FOREIGN KEY(workspace_id,repository_id) REFERENCES repositories(workspace_id,repository_id) +); +CREATE TABLE IF NOT EXISTS merge_request_ticket_relations ( + workspace_id TEXT NOT NULL, merge_request_id TEXT NOT NULL, ticket_id TEXT NOT NULL, + relation_kind TEXT NOT NULL CHECK(relation_kind='implements'), created_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,merge_request_id,ticket_id), + FOREIGN KEY(workspace_id,merge_request_id) REFERENCES merge_requests(workspace_id,merge_request_id) ON DELETE CASCADE, + FOREIGN KEY(workspace_id,ticket_id) REFERENCES typed_tickets(workspace_id,ticket_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS merge_request_revisions ( + workspace_id TEXT NOT NULL, merge_request_id TEXT NOT NULL, revision_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, base_commit TEXT NOT NULL, head_commit TEXT NOT NULL, head_tree TEXT NOT NULL, diff_digest TEXT NOT NULL, + summary TEXT NOT NULL, assignment_id TEXT NOT NULL, created_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,merge_request_id,revision_id), UNIQUE(workspace_id,merge_request_id,ordinal), + FOREIGN KEY(workspace_id,merge_request_id) REFERENCES merge_requests(workspace_id,merge_request_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS merge_request_revision_paths ( + workspace_id TEXT NOT NULL, merge_request_id TEXT NOT NULL, revision_id TEXT NOT NULL, ordinal INTEGER NOT NULL, path TEXT NOT NULL, + PRIMARY KEY(workspace_id,merge_request_id,revision_id,ordinal), + FOREIGN KEY(workspace_id,merge_request_id,revision_id) REFERENCES merge_request_revisions(workspace_id,merge_request_id,revision_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS merge_request_reviewer_child_sessions ( + workspace_id TEXT NOT NULL, child_session_id TEXT NOT NULL, parent_runtime_id TEXT NOT NULL, + parent_worker_id TEXT NOT NULL, effective_profile TEXT NOT NULL CHECK(effective_profile='builtin:reviewer'), registered_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,child_session_id) +); +CREATE TABLE IF NOT EXISTS merge_request_review_attempts ( + workspace_id TEXT NOT NULL, attempt_id TEXT NOT NULL, merge_request_id TEXT NOT NULL, ticket_id TEXT NOT NULL, + revision_id TEXT NOT NULL, lifecycle_generation INTEGER NOT NULL, + parent_assignment_id TEXT NOT NULL, parent_runtime_id TEXT NOT NULL, parent_worker_id TEXT NOT NULL, + child_session_id TEXT NOT NULL, child_effective_profile TEXT NOT NULL CHECK(child_effective_profile='builtin:reviewer'), + capability_token_sha256 TEXT NOT NULL, status TEXT NOT NULL CHECK(status IN ('open','submitted','revoked')), + created_at TEXT NOT NULL, consumed_at TEXT, + PRIMARY KEY(workspace_id,attempt_id), UNIQUE(workspace_id,capability_token_sha256), UNIQUE(workspace_id,child_session_id), + FOREIGN KEY(workspace_id,merge_request_id,revision_id) REFERENCES merge_request_revisions(workspace_id,merge_request_id,revision_id), + FOREIGN KEY(workspace_id,ticket_id,parent_assignment_id) REFERENCES ticket_worker_assignments(workspace_id,ticket_id,assignment_id) +); +CREATE TABLE IF NOT EXISTS merge_request_reviews ( + workspace_id TEXT NOT NULL, attempt_id TEXT NOT NULL, merge_request_id TEXT NOT NULL, revision_id TEXT NOT NULL, + decision TEXT NOT NULL CHECK(decision IN ('approve','request_changes')), body TEXT NOT NULL, submitted_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,attempt_id), + FOREIGN KEY(workspace_id,attempt_id) REFERENCES merge_request_review_attempts(workspace_id,attempt_id), + FOREIGN KEY(workspace_id,merge_request_id,revision_id) REFERENCES merge_request_revisions(workspace_id,merge_request_id,revision_id) +); +CREATE TABLE IF NOT EXISTS merge_request_review_findings ( + workspace_id TEXT NOT NULL, attempt_id TEXT NOT NULL, ordinal INTEGER NOT NULL, severity TEXT NOT NULL, + code TEXT, path TEXT, line INTEGER, body TEXT NOT NULL, PRIMARY KEY(workspace_id,attempt_id,ordinal), + FOREIGN KEY(workspace_id,attempt_id) REFERENCES merge_request_reviews(workspace_id,attempt_id) ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS merge_request_completion_operations ( + workspace_id TEXT NOT NULL, operation_id TEXT NOT NULL, ticket_id TEXT NOT NULL, revision_id TEXT NOT NULL, + assignment_id TEXT NOT NULL, fingerprint TEXT NOT NULL, status TEXT NOT NULL CHECK(status IN ('pending','completed')), + result_ticket_state TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,operation_id), + FOREIGN KEY(workspace_id,ticket_id) REFERENCES typed_tickets(workspace_id,ticket_id) +); +"#; + +fn archive_incompatible_legacy_tables(conn: &Connection, version: i64) -> Result<()> { + if version == 0 || !table_exists(conn, "merge_requests")? { + return Ok(()); + } + let incompatible = column_exists(conn, "merge_requests", "ticket_id")? + || !table_has_columns( + conn, + "merge_requests", + &[ + "workspace_id", + "merge_request_id", + "repository_id", + "state", + "lifecycle_generation", + "current_revision_id", + ], + )? + || !table_has_columns( + conn, + "merge_request_ticket_relations", + &[ + "workspace_id", + "merge_request_id", + "ticket_id", + "relation_kind", + ], + )? + || !table_has_columns( + conn, + "merge_request_revisions", + &[ + "workspace_id", + "merge_request_id", + "revision_id", + "ordinal", + "base_commit", + "head_commit", + "head_tree", + "diff_digest", + "assignment_id", + ], + )?; + if !incompatible { + return Ok(()); + } + let tables = [ + "merge_request_review_findings", + "merge_request_reviews", + "merge_request_review_attempts", + "merge_request_reviewer_child_sessions", + "merge_request_completion_operations", + "merge_request_revision_paths", + "merge_request_ticket_relations", + "merge_request_revisions", + "merge_requests", + ]; + conn.pragma_update(None, "foreign_keys", "OFF") + .map_err(db)?; + for table in tables { + if !table_exists(conn, table)? { + continue; + } + let archive = format!("legacy_v6_{table}"); + if table_exists(conn, &archive)? { + conn.pragma_update(None, "foreign_keys", "ON").map_err(db)?; + return Err(MergeRequestError::Database(format!( + "legacy archive table {archive} already exists" + ))); + } + conn.execute_batch(&format!("ALTER TABLE {table} RENAME TO {archive};")) + .map_err(db)?; + } + conn.pragma_update(None, "foreign_keys", "ON").map_err(db)?; + Ok(()) +} + +fn table_has_columns(conn: &Connection, table: &str, required: &[&str]) -> Result { + if !table_exists(conn, table)? { + return Ok(false); + } + for column in required { + if !column_exists(conn, table, column)? { + return Ok(false); + } + } + Ok(true) +} + +fn table_exists(conn: &Connection, table: &str) -> Result { + let present: Option = conn + .query_row( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1", + params![table], + |row| row.get(0), + ) + .optional() + .map_err(db)?; + Ok(present.is_some()) +} + +fn column_exists(conn: &Connection, table: &str, column: &str) -> Result { + let mut statement = conn + .prepare(&format!("PRAGMA table_info({table})")) + .map_err(db)?; + let names = statement + .query_map([], |row| row.get::<_, String>(1)) + .map_err(db)? + .collect::, _>>() + .map_err(db)?; + Ok(names.iter().any(|name| name == column)) +} + +fn load_merge_request( + conn: &Connection, + workspace_id: &str, + ticket_id: &str, +) -> Result> { + let row: Option<(String,String,String,String,i64,String,String,String,Option,Option)> = conn.query_row( + "SELECT mr.merge_request_id,rel.ticket_id,mr.repository_id,mr.state,mr.lifecycle_generation,mr.current_revision_id,mr.created_at,mr.updated_at,mr.merged_by_account_id,mr.merged_at FROM merge_requests mr JOIN merge_request_ticket_relations rel ON rel.workspace_id=mr.workspace_id AND rel.merge_request_id=mr.merge_request_id WHERE mr.workspace_id=?1 AND rel.ticket_id=?2 AND rel.relation_kind='implements' ORDER BY mr.updated_at DESC,mr.merge_request_id DESC LIMIT 1", + params![workspace_id,ticket_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?,r.get(8)?,r.get(9)?)), + ).optional().map_err(db)?; + let Some(( + mr_id, + ticket_id, + repository_id, + state, + generation, + revision_id, + created_at, + updated_at, + merged_by_account_id, + merged_at, + )) = row + else { + return Ok(None); + }; + let revision = load_revision(conn, workspace_id, &mr_id, &revision_id)?; + let current_review = load_latest_review(conn, workspace_id, &mr_id, &revision_id, generation)?; + let review_status = match current_review.as_ref().map(|review| review.decision) { + Some(ReviewDecision::Approve) => ReviewStatus::Approved, + Some(ReviewDecision::RequestChanges) => ReviewStatus::ChangesRequested, + None => ReviewStatus::Pending, + }; + Ok(Some(MergeRequest { + merge_request_id: mr_id, + workspace_id: workspace_id.into(), + ticket_id, + repository_id, + state: MergeRequestState::parse(&state), + lifecycle_generation: generation as u64, + current_revision: revision, + review_status, + current_review, + created_at, + updated_at, + merged_by_account_id, + merged_at, + })) +} + +fn load_revision( + conn: &Connection, + workspace_id: &str, + mr_id: &str, + revision_id: &str, +) -> Result { + let mut revision: MergeRequestRevision = conn.query_row( + "SELECT revision_id,ordinal,base_commit,head_commit,head_tree,diff_digest,summary,assignment_id,created_at FROM merge_request_revisions WHERE workspace_id=?1 AND merge_request_id=?2 AND revision_id=?3", + params![workspace_id,mr_id,revision_id], |r| Ok(MergeRequestRevision { revision_id:r.get(0)?, ordinal:r.get::<_,i64>(1)? as u64, base_commit:r.get(2)?, head_commit:r.get(3)?, head_tree:r.get(4)?, diff_digest:r.get(5)?, changed_paths:Vec::new(), summary:r.get(6)?, assignment_id:r.get(7)?, created_at:r.get(8)? }), + ).map_err(db)?; + let mut statement = conn.prepare("SELECT path FROM merge_request_revision_paths WHERE workspace_id=?1 AND merge_request_id=?2 AND revision_id=?3 ORDER BY ordinal").map_err(db)?; + revision.changed_paths = statement + .query_map(params![workspace_id, mr_id, revision_id], |r| r.get(0)) + .map_err(db)? + .collect::, _>>() + .map_err(db)?; + Ok(revision) +} + +fn insert_revision( + conn: &Connection, + workspace_id: &str, + mr_id: &str, + revision: &MergeRequestRevision, +) -> Result<()> { + conn.execute("INSERT INTO merge_request_revisions (workspace_id,merge_request_id,revision_id,ordinal,base_commit,head_commit,head_tree,diff_digest,summary,assignment_id,created_at) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)", params![workspace_id,mr_id,revision.revision_id,revision.ordinal as i64,revision.base_commit,revision.head_commit,revision.head_tree,revision.diff_digest,revision.summary,revision.assignment_id,revision.created_at]).map_err(db)?; + for (ordinal, path) in revision.changed_paths.iter().enumerate() { + conn.execute("INSERT INTO merge_request_revision_paths (workspace_id,merge_request_id,revision_id,ordinal,path) VALUES (?1,?2,?3,?4,?5)", params![workspace_id,mr_id,revision.revision_id,ordinal as i64,path]).map_err(db)?; + } + Ok(()) +} + +fn load_latest_review( + conn: &Connection, + workspace_id: &str, + mr_id: &str, + revision_id: &str, + generation: i64, +) -> Result> { + let attempt: Option = conn.query_row("SELECT r.attempt_id FROM merge_request_reviews r JOIN merge_request_review_attempts a ON a.workspace_id=r.workspace_id AND a.attempt_id=r.attempt_id WHERE r.workspace_id=?1 AND r.merge_request_id=?2 AND r.revision_id=?3 AND a.lifecycle_generation=?4 ORDER BY r.submitted_at DESC, r.attempt_id DESC LIMIT 1", params![workspace_id,mr_id,revision_id,generation], |r| r.get(0)).optional().map_err(db)?; + match attempt { + Some(id) => load_review(conn, workspace_id, &id), + None => Ok(None), + } +} + +fn load_review( + conn: &Connection, + workspace_id: &str, + attempt_id: &str, +) -> Result> { + let row: Option<(String,String,String,String,String,String,String,String,String)> = conn.query_row( + "SELECT r.revision_id,r.decision,r.body,a.parent_assignment_id,a.parent_runtime_id,a.parent_worker_id,a.child_session_id,a.child_effective_profile,r.submitted_at FROM merge_request_reviews r JOIN merge_request_review_attempts a ON a.workspace_id=r.workspace_id AND a.attempt_id=r.attempt_id WHERE r.workspace_id=?1 AND r.attempt_id=?2", + params![workspace_id,attempt_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?,r.get(8)?)), + ).optional().map_err(db)?; + let Some(( + revision_id, + decision, + body, + assignment, + runtime, + worker, + child, + profile, + submitted_at, + )) = row + else { + return Ok(None); + }; + let mut stmt=conn.prepare("SELECT severity,code,path,line,body FROM merge_request_review_findings WHERE workspace_id=?1 AND attempt_id=?2 ORDER BY ordinal").map_err(db)?; + let findings = stmt + .query_map(params![workspace_id, attempt_id], |r| { + Ok(ReviewFinding { + severity: r.get(0)?, + code: r.get(1)?, + path: r.get(2)?, + line: r.get::<_, Option>(3)?.map(|v| v as u64), + body: r.get(4)?, + }) + }) + .map_err(db)? + .collect::, _>>() + .map_err(db)?; + Ok(Some(MergeRequestReview { + attempt_id: attempt_id.into(), + revision_id, + decision: ReviewDecision::parse(&decision), + body, + findings, + parent_assignment_id: assignment, + parent_runtime_id: runtime, + parent_worker_id: worker, + reviewer_child_session_id: child, + reviewer_effective_profile: profile, + submitted_at, + })) +} + +fn validate_current_assignment( + conn: &Connection, + workspace_id: &str, + ticket_id: &str, + assignment_id: &str, + runtime_id: &str, + worker_id: &str, +) -> Result<()> { + let valid: Option = conn.query_row("SELECT 1 FROM ticket_current_worker_assignments WHERE workspace_id=?1 AND ticket_id=?2 AND assignment_id=?3 AND runtime_id=?4 AND worker_id=?5", params![workspace_id,ticket_id,assignment_id,runtime_id,worker_id], |r| r.get(0)).optional().map_err(db)?; + if valid.is_none() { + return Err(MergeRequestError::AssignmentMismatch); + } + Ok(()) +} + +fn append_completion_event( + conn: &Connection, + workspace_id: &str, + input: &CompleteMergeRequest, +) -> Result<()> { + let index:i64=conn.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![workspace_id,input.ticket_id],|r|r.get(0)).map_err(db)?; + conn.execute("INSERT INTO typed_ticket_events (workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body) VALUES (?1,?2,?3,'state_changed',?4,?5,'inprogress','done','Merge Request completed',?6)",params![workspace_id,input.ticket_id,index,format!("worker:{}:{}",input.authenticated_runtime_id,input.authenticated_worker_id),input.now,format!("Approved immutable revision `{}` completed implementation.",input.expected_revision_id)]).map_err(db)?; + for (key, value) in [ + ("assignment_id", input.assignment_id.as_str()), + ( + "merge_request_revision_id", + input.expected_revision_id.as_str(), + ), + ("operation_id", input.operation_id.as_str()), + ("runtime_id", input.authenticated_runtime_id.as_str()), + ("worker_id", input.authenticated_worker_id.as_str()), + ] { + conn.execute("INSERT INTO typed_ticket_event_attributes (workspace_id,ticket_id,event_index,key,value) VALUES (?1,?2,?3,?4,?5)",params![workspace_id,input.ticket_id,index,key,value]).map_err(db)?; + } + Ok(()) +} + +fn validate_revision(revision: &MergeRequestRevision) -> Result<()> { + for (name, value) in [ + ("revision_id", revision.revision_id.as_str()), + ("base_commit", revision.base_commit.as_str()), + ("head_commit", revision.head_commit.as_str()), + ("head_tree", revision.head_tree.as_str()), + ("diff_digest", revision.diff_digest.as_str()), + ("assignment_id", revision.assignment_id.as_str()), + ] { + nonempty(name, value)?; + } + if revision.ordinal == 0 { + return Err(MergeRequestError::Empty("revision.ordinal")); + } + if revision.summary.len() > MAX_SUMMARY_BYTES { + return Err(MergeRequestError::TooLarge { + field: "revision.summary", + max: MAX_SUMMARY_BYTES, + }); + } + if revision.changed_paths.len() > MAX_CHANGED_PATHS { + return Err(MergeRequestError::TooLarge { + field: "revision.changed_paths", + max: MAX_CHANGED_PATHS, + }); + } + for path in &revision.changed_paths { + nonempty("changed_path", path)?; + if path.len() > MAX_FIELD_BYTES { + return Err(MergeRequestError::TooLarge { + field: "changed_path", + max: MAX_FIELD_BYTES, + }); + } + if Path::new(path).is_absolute() || path.split('/').any(|p| p == "..") { + return Err(MergeRequestError::Empty("changed_path")); + } + } + Ok(()) +} + +fn validate_review_input(input: &SubmitReview) -> Result<()> { + if input.body.len() > MAX_REVIEW_BODY_BYTES { + return Err(MergeRequestError::TooLarge { + field: "review.body", + max: MAX_REVIEW_BODY_BYTES, + }); + } + if input.findings.len() > MAX_FINDINGS { + return Err(MergeRequestError::TooLarge { + field: "review.findings", + max: MAX_FINDINGS, + }); + } + for finding in &input.findings { + nonempty("finding.severity", &finding.severity)?; + nonempty("finding.body", &finding.body)?; + for (field, value) in [ + ("finding.severity", Some(finding.severity.as_str())), + ("finding.code", finding.code.as_deref()), + ("finding.path", finding.path.as_deref()), + ("finding.body", Some(finding.body.as_str())), + ] { + if value.is_some_and(|value| value.len() > MAX_FIELD_BYTES) { + return Err(MergeRequestError::TooLarge { + field, + max: MAX_FIELD_BYTES, + }); + } + } + } + Ok(()) +} + +fn ensure_open(mr: &MergeRequest) -> Result<()> { + if mr.state != MergeRequestState::Open { + Err(MergeRequestError::NotOpen(mr.state.as_str().into())) + } else { + Ok(()) + } +} +fn nonempty(name: &'static str, value: &str) -> Result<()> { + if value.trim().is_empty() { + Err(MergeRequestError::Empty(name)) + } else { + Ok(()) + } +} +fn token_hash(token: &str) -> String { + Sha256::digest(token.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} +fn completion_fingerprint(input: &CompleteMergeRequest) -> String { + token_hash(&format!( + "{}\0{}\0{}\0{}\0{}", + input.ticket_id, + input.expected_revision_id, + input.assignment_id, + input.authenticated_runtime_id, + input.authenticated_worker_id + )) +} +fn db(error: rusqlite::Error) -> MergeRequestError { + MergeRequestError::Database(error.to_string()) +} diff --git a/crates/merge-request/tests/store.rs b/crates/merge-request/tests/store.rs new file mode 100644 index 00000000..8763799c --- /dev/null +++ b/crates/merge-request/tests/store.rs @@ -0,0 +1,404 @@ +use merge_request::*; +use rusqlite::{Connection, params}; +use tempfile::TempDir; + +fn setup() -> (TempDir, SqliteMergeRequestStore) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("server.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch(r#" + PRAGMA foreign_keys=ON; + CREATE TABLE repositories(workspace_id TEXT NOT NULL,repository_id TEXT NOT NULL,PRIMARY KEY(workspace_id,repository_id)); + CREATE TABLE typed_tickets(workspace_id TEXT NOT NULL,ticket_id TEXT NOT NULL,workflow_state TEXT NOT NULL,workflow_state_explicit INTEGER NOT NULL DEFAULT 1,updated_at TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id)); + CREATE TABLE typed_ticket_events(workspace_id TEXT NOT NULL,ticket_id TEXT NOT NULL,event_index INTEGER NOT NULL,kind TEXT NOT NULL,author TEXT,at TEXT,status TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index)); + CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT NOT NULL,ticket_id TEXT NOT NULL,event_index INTEGER NOT NULL,key TEXT NOT NULL,value TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id,event_index,key)); + CREATE TABLE ticket_worker_assignments(workspace_id TEXT NOT NULL,ticket_id TEXT NOT NULL,assignment_id TEXT NOT NULL,runtime_id TEXT NOT NULL,worker_id TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id,assignment_id)); + CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT NOT NULL,ticket_id TEXT NOT NULL,assignment_id TEXT NOT NULL,runtime_id TEXT NOT NULL,worker_id TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id)); + "#).unwrap(); + for ws in ["ws-a", "ws-b"] { + conn.execute("INSERT INTO repositories VALUES(?1,'repo')", params![ws]) + .unwrap(); + conn.execute( + "INSERT INTO typed_tickets VALUES(?1,'T1','inprogress',1,'t0')", + params![ws], + ) + .unwrap(); + conn.execute( + "INSERT INTO ticket_worker_assignments VALUES(?1,'T1','A1','R1','W1')", + params![ws], + ) + .unwrap(); + conn.execute( + "INSERT INTO ticket_current_worker_assignments VALUES(?1,'T1','A1','R1','W1')", + params![ws], + ) + .unwrap(); + } + drop(conn); + let store = SqliteMergeRequestStore::open(&path, "ws-a").unwrap(); + (dir, store) +} +fn revision(id: &str, ordinal: u64, head: &str) -> MergeRequestRevision { + MergeRequestRevision { + revision_id: id.into(), + ordinal, + base_commit: "base".into(), + head_commit: head.into(), + head_tree: format!("tree-{head}"), + diff_digest: format!("sha256:diff-{head}"), + changed_paths: vec!["src/lib.rs".into()], + summary: format!("revision {id}"), + assignment_id: "A1".into(), + created_at: format!("t{ordinal}"), + } +} +fn open(store: &SqliteMergeRequestStore) { + store + .open_merge_request(OpenMergeRequest { + merge_request_id: "MR1".into(), + ticket_id: "T1".into(), + repository_id: "repo".into(), + revision: revision("V1", 1, "h1"), + authenticated_runtime_id: "R1".into(), + authenticated_worker_id: "W1".into(), + now: "t1".into(), + }) + .unwrap(); +} +fn attempt(store: &SqliteMergeRequestStore, id: &str, revision: &str, token: &str, child: &str) { + store + .register_reviewer_child_session(RegisterReviewerChildSession { + parent_runtime_id: "R1".into(), + parent_worker_id: "W1".into(), + child_session_id: child.into(), + now: "t".into(), + }) + .unwrap(); + store + .register_review_attempt(RegisterReviewAttempt { + attempt_id: id.into(), + ticket_id: "T1".into(), + revision_id: revision.into(), + parent_assignment_id: "A1".into(), + parent_runtime_id: "R1".into(), + parent_worker_id: "W1".into(), + child_session_id: child.into(), + capability_token: token.into(), + now: "t".into(), + }) + .unwrap(); +} +fn review( + store: &SqliteMergeRequestStore, + revision: &str, + token: &str, + decision: ReviewDecision, +) -> Result { + store.submit_review(SubmitReview { + ticket_id: "T1".into(), + revision_id: revision.into(), + capability_token: token.into(), + decision, + body: "evidence".into(), + findings: vec![], + now: "tr".into(), + }) +} + +#[test] +fn storage_allows_multiple_merge_requests_for_one_ticket() { + let (_dir, store) = setup(); + open(&store); + store + .open_merge_request(OpenMergeRequest { + merge_request_id: "MR2".into(), + ticket_id: "T1".into(), + repository_id: "repo".into(), + revision: revision("V2", 1, "h2"), + authenticated_runtime_id: "R1".into(), + authenticated_worker_id: "W1".into(), + now: "t2".into(), + }) + .unwrap(); + let conn = Connection::open(store.db_path()).unwrap(); + let count:i64=conn.query_row("SELECT COUNT(*) FROM merge_request_ticket_relations WHERE workspace_id='ws-a' AND ticket_id='T1'",[],|row|row.get(0)).unwrap(); + assert_eq!(count, 2); + assert_eq!( + store + .show_for_ticket("T1") + .unwrap() + .unwrap() + .merge_request_id, + "MR2" + ); +} + +#[test] +fn bounded_context_rejects_oversized_revision_evidence() { + let (_dir, store) = setup(); + let mut oversized = revision("V1", 1, "h1"); + oversized.changed_paths = (0..=1_000).map(|i| format!("src/{i}.rs")).collect(); + let result = store.open_merge_request(OpenMergeRequest { + merge_request_id: "MR1".into(), + ticket_id: "T1".into(), + repository_id: "repo".into(), + revision: oversized, + authenticated_runtime_id: "R1".into(), + authenticated_worker_id: "W1".into(), + now: "t".into(), + }); + assert!(matches!( + result, + Err(MergeRequestError::TooLarge { + field: "revision.changed_paths", + .. + }) + )); +} + +#[test] +fn rejected_v6_schema_missing_diff_digest_is_archived_before_fresh_v7() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("legacy.db"); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE merge_request_schema_migrations(version INTEGER PRIMARY KEY,name TEXT NOT NULL,applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);\ + INSERT INTO merge_request_schema_migrations(version,name) VALUES(6,'rejected_merge_request_v6');\ + CREATE TABLE repositories(workspace_id TEXT NOT NULL,repository_id TEXT NOT NULL,PRIMARY KEY(workspace_id,repository_id));\ + CREATE TABLE typed_tickets(workspace_id TEXT NOT NULL,ticket_id TEXT NOT NULL,workflow_state TEXT NOT NULL,workflow_state_explicit INTEGER NOT NULL DEFAULT 1,updated_at TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));\ + CREATE TABLE ticket_worker_assignments(workspace_id TEXT NOT NULL,ticket_id TEXT NOT NULL,assignment_id TEXT NOT NULL,runtime_id TEXT NOT NULL,worker_id TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id,assignment_id));\ + CREATE TABLE merge_requests(workspace_id TEXT NOT NULL,merge_request_id TEXT NOT NULL,repository_id TEXT NOT NULL,state TEXT NOT NULL,lifecycle_generation INTEGER NOT NULL,current_revision_id TEXT NOT NULL,created_at TEXT NOT NULL,updated_at TEXT NOT NULL,PRIMARY KEY(workspace_id,merge_request_id));\ + CREATE TABLE merge_request_ticket_relations(workspace_id TEXT NOT NULL,merge_request_id TEXT NOT NULL,ticket_id TEXT NOT NULL,relation_kind TEXT NOT NULL,created_at TEXT NOT NULL,PRIMARY KEY(workspace_id,merge_request_id,ticket_id));\ + CREATE TABLE merge_request_revisions(workspace_id TEXT NOT NULL,merge_request_id TEXT NOT NULL,revision_id TEXT NOT NULL,ordinal INTEGER NOT NULL,base_commit TEXT NOT NULL,head_commit TEXT NOT NULL,head_tree TEXT NOT NULL,assignment_id TEXT NOT NULL,created_at TEXT NOT NULL,PRIMARY KEY(workspace_id,merge_request_id,revision_id));", + ).unwrap(); + drop(conn); + let store = SqliteMergeRequestStore::open(&path, "ws-a").unwrap(); + assert!(store.show_for_ticket("missing").unwrap().is_none()); + let conn = Connection::open(&path).unwrap(); + let archived: i64 = conn.query_row("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='legacy_v6_merge_requests'",[],|row|row.get(0)).unwrap(); + assert_eq!(archived, 1); + for table in [ + "merge_request_review_attempts", + "merge_request_completion_operations", + ] { + let present: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1", + params![table], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(present, 1); + } +} + +#[test] +fn request_changes_new_revision_resets_and_exact_completion_replay_converges() { + let (_dir, store) = setup(); + open(&store); + attempt(&store, "AT1", "V1", "tok1", "child1"); + review(&store, "V1", "tok1", ReviewDecision::RequestChanges).unwrap(); + assert_eq!( + store.show_for_ticket("T1").unwrap().unwrap().review_status, + ReviewStatus::ChangesRequested + ); + store + .add_revision(AddRevision { + ticket_id: "T1".into(), + expected_current_revision_id: "V1".into(), + revision: revision("V2", 2, "h2"), + authenticated_runtime_id: "R1".into(), + authenticated_worker_id: "W1".into(), + now: "t2".into(), + }) + .unwrap(); + assert_eq!( + store.show_for_ticket("T1").unwrap().unwrap().review_status, + ReviewStatus::Pending + ); + assert!(review(&store, "V1", "tok1", ReviewDecision::Approve).is_err()); + attempt(&store, "AT2", "V2", "tok2", "child2"); + review(&store, "V2", "tok2", ReviewDecision::Approve).unwrap(); + let input = CompleteMergeRequest { + operation_id: "OP1".into(), + ticket_id: "T1".into(), + expected_revision_id: "V2".into(), + assignment_id: "A1".into(), + authenticated_runtime_id: "R1".into(), + authenticated_worker_id: "W1".into(), + now: "tc".into(), + }; + let first = store.complete(input.clone()).unwrap(); + assert!(!first.replayed); + let replay = store.complete(input).unwrap(); + assert!(replay.replayed); + assert!(matches!( + store.confirm_merge(MergeConfirmation { + ticket_id: "T1".into(), + expected_revision_id: "V2".into(), + authenticated_account_id: "runtime".into(), + actor_kind: "worker".into(), + explicit_confirmation: true, + now: "tm".into() + }), + Err(MergeRequestError::MergeConfirmationRequired) + )); + let merged = store + .confirm_merge(MergeConfirmation { + ticket_id: "T1".into(), + expected_revision_id: "V2".into(), + authenticated_account_id: "account-1".into(), + actor_kind: "user".into(), + explicit_confirmation: true, + now: "tm".into(), + }) + .unwrap(); + assert_eq!(merged.state, MergeRequestState::Merged); + let conn = Connection::open(store.db_path()).unwrap(); + assert_eq!( + conn.query_row( + "SELECT workflow_state FROM typed_tickets WHERE workspace_id='ws-a' AND ticket_id='T1'", + [], + |r| r.get::<_, String>(0) + ) + .unwrap(), + "done" + ); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM typed_ticket_events WHERE workspace_id='ws-a' AND ticket_id='T1'", + [], + |r| r.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); +} + +#[test] +fn spoof_self_approval_replay_and_cross_workspace_are_rejected() { + let (_dir, store) = setup(); + open(&store); + let mut bad = RegisterReviewAttempt { + attempt_id: "bad".into(), + ticket_id: "T1".into(), + revision_id: "V1".into(), + parent_assignment_id: "A1".into(), + parent_runtime_id: "R1".into(), + parent_worker_id: "W1".into(), + child_session_id: "W1".into(), + capability_token: "bad".into(), + now: "t".into(), + }; + assert!(matches!( + store.register_review_attempt(bad.clone()), + Err(MergeRequestError::SelfApproval) + )); + bad.child_session_id = "child".into(); + assert!(matches!( + store.register_review_attempt(bad), + Err(MergeRequestError::InvalidReviewer) + )); + attempt(&store, "AT", "V1", "secret", "child"); + assert!(review(&store, "V1", "spoof", ReviewDecision::Approve).is_err()); + review(&store, "V1", "secret", ReviewDecision::Approve).unwrap(); + assert!(review(&store, "V1", "secret", ReviewDecision::Approve).is_err()); + let other = SqliteMergeRequestStore::open_verified(store.db_path(), "ws-b").unwrap(); + assert!(other.show_for_ticket("T1").unwrap().is_none()); +} + +#[test] +fn reopen_resets_approval_and_merge_requires_authenticated_explicit_user() { + let (_dir, store) = setup(); + open(&store); + attempt(&store, "AT", "V1", "token", "child"); + review(&store, "V1", "token", ReviewDecision::Approve).unwrap(); + store.close("T1", "V1", "tc").unwrap(); + let reopened = store.reopen("T1", "V1", "tr").unwrap(); + assert_eq!(reopened.review_status, ReviewStatus::Pending); + let denied = store.confirm_merge(MergeConfirmation { + ticket_id: "T1".into(), + expected_revision_id: "V1".into(), + authenticated_account_id: "user".into(), + actor_kind: "user".into(), + explicit_confirmation: false, + now: "tm".into(), + }); + assert!(matches!( + denied, + Err(MergeRequestError::MergeConfirmationRequired) + )); +} + +#[test] +fn concurrent_exact_completion_replays_commit_one_ticket_side_effect() { + let (_dir, store) = setup(); + open(&store); + attempt(&store, "AT", "V1", "token", "child"); + review(&store, "V1", "token", ReviewDecision::Approve).unwrap(); + let input = CompleteMergeRequest { + operation_id: "OP-concurrent".into(), + ticket_id: "T1".into(), + expected_revision_id: "V1".into(), + assignment_id: "A1".into(), + authenticated_runtime_id: "R1".into(), + authenticated_worker_id: "W1".into(), + now: "t".into(), + }; + let left_store = store.clone(); + let left_input = input.clone(); + let left = std::thread::spawn(move || left_store.complete(left_input)); + let right_store = store.clone(); + let right = std::thread::spawn(move || right_store.complete(input)); + let outcomes = [ + left.join().unwrap().unwrap(), + right.join().unwrap().unwrap(), + ]; + assert_eq!( + outcomes.iter().filter(|outcome| !outcome.replayed).count(), + 1 + ); + assert_eq!( + outcomes.iter().filter(|outcome| outcome.replayed).count(), + 1 + ); + let conn = Connection::open(store.db_path()).unwrap(); + let events: i64 = conn + .query_row( + "SELECT COUNT(*) FROM typed_ticket_events WHERE workspace_id='ws-a' AND ticket_id='T1'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(events, 1); +} + +#[test] +fn operation_key_mismatch_and_assignment_takeover_are_fenced() { + let (_dir, store) = setup(); + open(&store); + attempt(&store, "AT", "V1", "token", "child"); + review(&store, "V1", "token", ReviewDecision::Approve).unwrap(); + let mut input = CompleteMergeRequest { + operation_id: "OP".into(), + ticket_id: "T1".into(), + expected_revision_id: "V1".into(), + assignment_id: "A1".into(), + authenticated_runtime_id: "R1".into(), + authenticated_worker_id: "W1".into(), + now: "t".into(), + }; + let conn = Connection::open(store.db_path()).unwrap(); + conn.execute("UPDATE ticket_current_worker_assignments SET assignment_id='A2',runtime_id='R2',worker_id='W2' WHERE workspace_id='ws-a' AND ticket_id='T1'",[]).unwrap(); + assert!(matches!( + store.complete(input.clone()), + Err(MergeRequestError::AssignmentMismatch) + )); + conn.execute("UPDATE ticket_current_worker_assignments SET assignment_id='A1',runtime_id='R1',worker_id='W1' WHERE workspace_id='ws-a' AND ticket_id='T1'",[]).unwrap(); + store.complete(input.clone()).unwrap(); + input.expected_revision_id = "other".into(); + assert!(matches!( + store.complete(input), + Err(MergeRequestError::OperationConflict) + )); +} diff --git a/crates/ticket/src/lib.rs b/crates/ticket/src/lib.rs index 29f6f244..cde48ef7 100644 --- a/crates/ticket/src/lib.rs +++ b/crates/ticket/src/lib.rs @@ -295,7 +295,6 @@ pub enum TicketEventKind { Plan, Decision, ImplementationReport, - Review, StateChanged, IntakeSummary, StatusChanged, @@ -311,7 +310,6 @@ impl TicketEventKind { Self::Plan => "plan", Self::Decision => "decision", Self::ImplementationReport => "implementation_report", - Self::Review => "review", Self::StateChanged => "state_changed", Self::IntakeSummary => "intake_summary", Self::StatusChanged => "status_changed", @@ -327,7 +325,6 @@ impl TicketEventKind { Self::Plan => "Plan".to_string(), Self::Decision => "Decision".to_string(), Self::ImplementationReport => "Implementation report".to_string(), - Self::Review => "Review".to_string(), Self::StateChanged => "State changed".to_string(), Self::IntakeSummary => "Intake summary".to_string(), Self::StatusChanged => "Status changed".to_string(), @@ -345,7 +342,7 @@ impl From<&str> for TicketEventKind { "plan" => Self::Plan, "decision" => Self::Decision, "implementation_report" => Self::ImplementationReport, - "review" => Self::Review, + "review" => Self::Comment, "state_changed" => Self::StateChanged, "intake_summary" => Self::IntakeSummary, "status_changed" => Self::StatusChanged, @@ -355,42 +352,6 @@ impl From<&str> for TicketEventKind { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TicketReviewResult { - Approve, - RequestChanges, - Other(String), -} - -impl TicketReviewResult { - pub fn as_str(&self) -> &str { - match self { - Self::Approve => "approve", - Self::RequestChanges => "request_changes", - Self::Other(value) => value.as_str(), - } - } - - fn heading(&self) -> String { - match self { - Self::Approve => "Review: approve".to_string(), - Self::RequestChanges => "Review: request changes".to_string(), - Self::Other(value) => format!("Review: {value}"), - } - } -} - -impl From<&str> for TicketReviewResult { - fn from(value: &str) -> Self { - match value { - "approve" => Self::Approve, - "request_changes" => Self::RequestChanges, - other => Self::Other(other.to_string()), - } - } -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TicketReference { pub kind: String, @@ -461,31 +422,6 @@ impl TicketIntakeSummary { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TicketReview { - pub result: TicketReviewResult, - pub author: Option, - pub body: MarkdownText, -} - -impl TicketReview { - pub fn approve(body: impl Into) -> Self { - Self { - result: TicketReviewResult::Approve, - author: None, - body: body.into(), - } - } - - pub fn request_changes(body: impl Into) -> Self { - Self { - result: TicketReviewResult::RequestChanges, - author: None, - body: body.into(), - } - } -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct NewTicket { pub title: String, @@ -1578,7 +1514,6 @@ pub trait TicketBackend { change: TicketStateChange, ) -> Result<()>; fn queue_ready(&self, id: TicketIdOrSlug, queued_by: &str) -> Result<()>; - fn review(&self, id: TicketIdOrSlug, review: TicketReview) -> Result<()>; fn close(&self, id: TicketIdOrSlug, resolution: MarkdownText) -> Result<()>; fn add_ticket_relation( &self, @@ -1656,10 +1591,6 @@ pub enum TicketBackendOperation { id: TicketIdOrSlug, queued_by: String, }, - Review { - id: TicketIdOrSlug, - review: TicketReview, - }, Close { id: TicketIdOrSlug, resolution: MarkdownText, @@ -1763,10 +1694,6 @@ where backend.queue_ready(id, &queued_by)?; TicketBackendOperationResult::Unit } - TicketBackendOperation::Review { id, review } => { - backend.review(id, review)?; - TicketBackendOperationResult::Unit - } TicketBackendOperation::Close { id, resolution } => { backend.close(id, resolution)?; TicketBackendOperationResult::Unit @@ -3201,34 +3128,6 @@ impl TicketBackend for SqliteTicketBackend { }) } - fn review(&self, id: TicketIdOrSlug, review: TicketReview) -> Result<()> { - self.with_write(|conn| { - let ticket_id = self.resolve_ticket_id(conn, id)?; - let at = now_utc(); - let mut attributes = BTreeMap::new(); - attributes.insert("result".to_string(), review.result.as_str().to_string()); - self.insert_event( - conn, - &ticket_id, - &TicketEvent { - kind: TicketEventKind::Review, - author: Some(review.author.unwrap_or_else(default_author)), - at: Some(at.clone()), - status: Some(review.result.as_str().to_string()), - from: None, - to: None, - reason: None, - state_field: None, - heading: Some(review.result.heading()), - body: review.body, - references: Vec::new(), - attributes, - }, - )?; - self.touch_ticket(conn, &ticket_id, &at) - }) - } - fn close(&self, id: TicketIdOrSlug, resolution: MarkdownText) -> Result<()> { self.with_write(|conn| { let ticket_id = self.resolve_ticket_id(conn, id)?; @@ -3804,21 +3703,6 @@ impl TicketBackend for LocalTicketBackend { ) } - fn review(&self, id: TicketIdOrSlug, review: TicketReview) -> Result<()> { - let _lock = self.acquire_lock()?; - let dir = self.find_ticket_dir(&id)?; - let author = review.author.unwrap_or_else(default_author); - self.append_thread_event( - &dir, - "review", - &review.result.heading(), - &author, - Some(review.result.as_str()), - &[], - &review.body, - ) - } - fn close(&self, id: TicketIdOrSlug, resolution: MarkdownText) -> Result<()> { let _lock = self.acquire_lock()?; self.ensure_backend_dirs()?; @@ -5337,7 +5221,8 @@ fn parse_thread(path: &Path) -> Result> { .strip_prefix("")) { - let attrs = parse_event_comment(comment); + let mut attrs = parse_event_comment(comment); + let legacy_review = attrs.get("event").is_some_and(|value| value == "review"); let kind = attrs .get("event") .map(|value| TicketEventKind::from(value.as_str())) @@ -5369,11 +5254,22 @@ fn parse_thread(path: &Path) -> Result> { while body.ends_with('\n') { body.pop(); } + if legacy_review { + heading = Some("Legacy review (non-authoritative)".to_string()); + attrs.remove("status"); + attrs.remove("result"); + attrs.insert("event".to_string(), "comment".to_string()); + attrs.insert("legacy_event_kind".to_string(), "review".to_string()); + } events.push(TicketEvent { kind, author: attrs.get("author").cloned(), at: attrs.get("at").cloned(), - status: attrs.get("status").cloned(), + status: if legacy_review { + None + } else { + attrs.get("status").cloned() + }, from: attrs.get("from").cloned(), to: attrs.get("to").cloned(), reason: attrs.get("reason").cloned(), @@ -6379,12 +6275,6 @@ state: planning NewTicketEvent::new(TicketEventKind::Comment, "Imported into SQLite."), ) .unwrap(); - backend - .review( - TicketIdOrSlug::Id(created.id.clone()), - TicketReview::approve("Looks good."), - ) - .unwrap(); backend .close( TicketIdOrSlug::Id(created.id.clone()), @@ -6404,13 +6294,6 @@ state: planning assert!(ticket.events.iter().any(|event| { event.kind == TicketEventKind::Comment && event.body.0.contains("Imported into SQLite") })); - assert!( - ticket - .events - .iter() - .any(|event| event.kind == TicketEventKind::Review - && event.body.0.contains("Looks good")) - ); assert!( ticket .resolution @@ -6524,7 +6407,7 @@ state: planning } #[test] - fn add_event_review_status_and_close_preserve_local_layout() { + fn add_event_status_and_close_preserve_local_layout() { let tmp = TempDir::new().unwrap(); let backend = backend(&tmp); let ticket = backend.create(NewTicket::new("Flow Ticket")).unwrap(); @@ -6534,12 +6417,6 @@ state: planning NewTicketEvent::new(TicketEventKind::Plan, "Implementation plan."), ) .unwrap(); - backend - .review( - TicketIdOrSlug::Id(ticket.id.clone()), - TicketReview::approve("Looks good."), - ) - .unwrap(); let mut summary = TicketIntakeSummary::new("Ready for queue."); summary.author = Some("test".to_string()); let mut change = TicketStateChange::new( @@ -6563,8 +6440,6 @@ state: planning let closed_dir = tmp.path().join("tickets").join(&ticket.id); assert!(closed_dir.join("resolution.md").exists()); let thread = fs::read_to_string(closed_dir.join("thread.md")).unwrap(); - assert!(thread.contains("author".into()); - assert!(matches!( - backend.review(TicketIdOrSlug::Id(ticket.id.clone()), review), - Err(TicketError::Conflict(_)) - )); - assert_eq!(fs::read_to_string(&thread_path).unwrap(), original); - let invalid_kind = NewTicketEvent::new( TicketEventKind::Other("bad\nevent".into()), "Invalid event kind.", diff --git a/crates/ticket/src/sqlite_schema.rs b/crates/ticket/src/sqlite_schema.rs index 87536232..4f876294 100644 --- a/crates/ticket/src/sqlite_schema.rs +++ b/crates/ticket/src/sqlite_schema.rs @@ -7,7 +7,7 @@ use crate::{Result, TicketError, sqlite_err}; const MIGRATION_TABLE: &str = "ticket_schema_migrations"; const MAX_SCHEMA_DIAGNOSTICS: usize = 32; -pub const LATEST_SQLITE_TICKET_SCHEMA_VERSION: i64 = 2; +pub const LATEST_SQLITE_TICKET_SCHEMA_VERSION: i64 = 3; #[derive(Clone, Copy)] struct Migration { @@ -27,6 +27,11 @@ const MIGRATIONS: &[Migration] = &[ name: "add_ticket_repository_target", apply: add_ticket_repository_target, }, + Migration { + version: 3, + name: "convert_legacy_reviews_to_comments", + apply: retire_legacy_ticket_review_events, + }, ]; #[derive(Clone, Copy)] @@ -475,6 +480,33 @@ fn add_ticket_repository_target(connection: &Connection) -> Result<()> { add_column_if_missing(connection, "typed_tickets", "ref_selector", "TEXT") } +fn retire_legacy_ticket_review_events(connection: &Connection) -> Result<()> { + // Historical prose remains visible for audit, but it is explicitly converted to a + // non-authoritative comment. Approval authority now lives only in Merge Requests. + connection + .execute_batch( + r#" + INSERT OR REPLACE INTO typed_ticket_event_attributes + (workspace_id, ticket_id, event_index, key, value) + SELECT workspace_id, ticket_id, event_index, 'legacy_event_kind', 'review' + FROM typed_ticket_events WHERE kind = 'review'; + UPDATE typed_ticket_events + SET kind = 'comment', status = NULL, heading = 'Legacy review (non-authoritative)' + WHERE kind = 'review'; + DELETE FROM typed_ticket_event_attributes + WHERE key IN ('result', 'review_result', 'status') + AND EXISTS ( + SELECT 1 FROM typed_ticket_events event + WHERE event.workspace_id = typed_ticket_event_attributes.workspace_id + AND event.ticket_id = typed_ticket_event_attributes.ticket_id + AND event.event_index = typed_ticket_event_attributes.event_index + AND event.heading = 'Legacy review (non-authoritative)' + ); + "#, + ) + .map_err(sqlite_err) +} + fn add_column_if_missing( connection: &Connection, table: &str, @@ -809,10 +841,10 @@ mod tests { verify_sqlite_ticket_schema(&connection).unwrap(); let versions = load_applied_migrations(&connection).unwrap(); - assert_eq!(versions.len(), 2); + assert_eq!(versions.len(), 3); assert_eq!( versions.get(&LATEST_SQLITE_TICKET_SCHEMA_VERSION), - Some(&"add_ticket_repository_target".to_string()) + Some(&"convert_legacy_reviews_to_comments".to_string()) ); } @@ -957,7 +989,7 @@ mod tests { .to_string() .contains("unsupported Ticket schema migration version 99") ); - assert_eq!(load_applied_migrations(&connection).unwrap().len(), 3); + assert_eq!(load_applied_migrations(&connection).unwrap().len(), 4); } #[test] @@ -1050,6 +1082,31 @@ mod tests { assert!(!migration_table_exists); } + #[test] + fn legacy_review_upgrade_preserves_prose_as_non_authoritative_comment() { + let connection = Connection::open_in_memory().unwrap(); + migrate_sqlite_ticket_schema(&connection).unwrap(); + connection.execute("INSERT INTO typed_tickets (workspace_id,ticket_id,slug,title,status,kind,priority,body,workflow_state,workflow_state_explicit) VALUES ('workspace-1','ticket-1','ticket-1','title','open','task','medium','body','inprogress',1)",[]).unwrap(); + connection.execute("INSERT INTO typed_ticket_events (workspace_id,ticket_id,event_index,kind,author,at,status,heading,body) VALUES ('workspace-1','ticket-1',0,'review','reviewer','2026-08-11T00:00:00Z','approve','Review','legacy evidence')",[]).unwrap(); + connection.execute("INSERT INTO typed_ticket_event_attributes (workspace_id,ticket_id,event_index,key,value) VALUES ('workspace-1','ticket-1',0,'result','approve')",[]).unwrap(); + connection + .execute("DELETE FROM ticket_schema_migrations WHERE version=3", []) + .unwrap(); + migrate_sqlite_ticket_schema(&connection).unwrap(); + let (kind,status,heading,body):(String,Option,Option,Option)=connection.query_row("SELECT kind,status,heading,body FROM typed_ticket_events WHERE workspace_id='workspace-1' AND ticket_id='ticket-1' AND event_index=0",[],|row|Ok((row.get(0)?,row.get(1)?,row.get(2)?,row.get(3)?))).unwrap(); + assert_eq!(kind, "comment"); + assert_eq!(status, None); + assert_eq!( + heading.as_deref(), + Some("Legacy review (non-authoritative)") + ); + assert_eq!(body.as_deref(), Some("legacy evidence")); + let attributes:i64=connection.query_row("SELECT COUNT(*) FROM typed_ticket_event_attributes WHERE workspace_id='workspace-1' AND ticket_id='ticket-1'",[],|row|row.get(0)).unwrap(); + assert_eq!(attributes, 1); + let legacy:String=connection.query_row("SELECT value FROM typed_ticket_event_attributes WHERE workspace_id='workspace-1' AND ticket_id='ticket-1' AND key='legacy_event_kind'",[],|row|row.get(0)).unwrap(); + assert_eq!(legacy, "review"); + } + #[test] fn concurrent_migrators_converge_on_one_version_history() { let directory = tempdir().unwrap(); @@ -1072,6 +1129,6 @@ mod tests { let connection = Connection::open(database).unwrap(); verify_sqlite_ticket_schema(&connection).unwrap(); - assert_eq!(load_applied_migrations(&connection).unwrap().len(), 2); + assert_eq!(load_applied_migrations(&connection).unwrap().len(), 3); } } diff --git a/crates/ticket/src/tool.rs b/crates/ticket/src/tool.rs index 90e6938a..f0819548 100644 --- a/crates/ticket/src/tool.rs +++ b/crates/ticket/src/tool.rs @@ -17,8 +17,7 @@ use crate::{ Result as TicketResult, Ticket, TicketBackend, TicketBodyReplacement, TicketDoctorDiagnostic, TicketDoctorReport, TicketDoctorSeverity, TicketError, TicketEventKind, TicketIdOrSlug, TicketIntakeSummary, TicketListState, TicketRef, TicketRelation, TicketRelationKind, - TicketRelationView, TicketReview, TicketReviewResult, TicketStateChange, TicketSummary, - TicketWorkflowState, default_author, + TicketRelationView, TicketStateChange, TicketSummary, TicketWorkflowState, default_author, }; const DEFAULT_LIST_LIMIT: usize = 50; @@ -34,7 +33,7 @@ const MAX_BODY_MAX_BYTES: usize = 64 * 1024; const DEFAULT_DIAGNOSTIC_LIMIT: usize = 100; const MAX_DIAGNOSTIC_LIMIT: usize = 500; -pub const TICKET_BASE_TOOL_NAMES: [&str; 15] = [ +pub const TICKET_BASE_TOOL_NAMES: [&str; 14] = [ "TicketCreate", "TicketEditItem", "TicketList", @@ -43,7 +42,6 @@ pub const TICKET_BASE_TOOL_NAMES: [&str; 15] = [ "TicketPlan", "TicketDecision", "TicketImplementationReport", - "TicketReview", "TicketIntakeReady", "TicketQueue", "TicketWorkflowState", @@ -69,7 +67,7 @@ pub const TICKET_ORCHESTRATION_TOOL_NAMES: [&str; 4] = [ pub const TICKET_ORCHESTRATION_READ_ONLY_TOOL_NAMES: [&str; 2] = ["TicketRelationQuery", "TicketOrchestrationPlanQuery"]; -pub const TICKET_TOOL_NAMES: [&str; 19] = [ +pub const TICKET_TOOL_NAMES: [&str; 18] = [ "TicketCreate", "TicketEditItem", "TicketList", @@ -78,7 +76,6 @@ pub const TICKET_TOOL_NAMES: [&str; 19] = [ "TicketPlan", "TicketDecision", "TicketImplementationReport", - "TicketReview", "TicketIntakeReady", "TicketQueue", "TicketWorkflowState", @@ -100,14 +97,13 @@ pub const TICKET_READ_ONLY_TOOL_NAMES: [&str; 6] = [ "TicketOrchestrationPlanQuery", ]; -pub const TICKET_MUTATING_TOOL_NAMES: [&str; 13] = [ +pub const TICKET_MUTATING_TOOL_NAMES: [&str; 12] = [ "TicketCreate", "TicketEditItem", "TicketComment", "TicketPlan", "TicketDecision", "TicketImplementationReport", - "TicketReview", "TicketIntakeReady", "TicketQueue", "TicketWorkflowState", @@ -134,8 +130,6 @@ const PLAN_DESCRIPTION: &str = "Append a typed Ticket plan event. `body` is Mark const DECISION_DESCRIPTION: &str = "Append a typed Ticket decision event. `body` is Markdown."; const IMPLEMENTATION_REPORT_DESCRIPTION: &str = "Append a typed Ticket implementation_report event. `body` is Markdown."; -const REVIEW_DESCRIPTION: &str = "Append a Ticket review event. `result` must be `approve` or \ -`request_changes`; `body` is Markdown. Writes stay inside the configured Ticket backend root."; const INTAKE_READY_DESCRIPTION: &str = "Mark an existing Ticket planning lane ready through the typed \ Ticket backend. The tool appends a bounded `intake_summary`, appends a typed `state_changed` event \ for `state`, and transitions state to `ready`."; @@ -175,7 +169,6 @@ fn base_tool_description(name: &str) -> &'static str { "TicketPlan" => PLAN_DESCRIPTION, "TicketDecision" => DECISION_DESCRIPTION, "TicketImplementationReport" => IMPLEMENTATION_REPORT_DESCRIPTION, - "TicketReview" => REVIEW_DESCRIPTION, "TicketIntakeReady" => INTAKE_READY_DESCRIPTION, "TicketQueue" => QUEUE_DESCRIPTION, "TicketWorkflowState" => WORKFLOW_STATE_DESCRIPTION, @@ -319,10 +312,6 @@ impl TicketBackend for TicketToolBackend { self.backend.queue_ready(id, queued_by) } - fn review(&self, id: TicketIdOrSlug, review: TicketReview) -> TicketResult<()> { - self.backend.review(id, review) - } - fn close(&self, id: TicketIdOrSlug, resolution: MarkdownText) -> TicketResult<()> { self.backend.close(id, resolution) } @@ -554,23 +543,6 @@ struct TicketThreadEventParams { body: String, } -#[derive(Debug, Deserialize, schemars::JsonSchema)] -#[serde(rename_all = "snake_case")] -enum TicketReviewResultParam { - Approve, - RequestChanges, -} - -#[derive(Debug, Deserialize, schemars::JsonSchema)] -struct TicketReviewParams { - /// Ticket id. - ticket: String, - /// Review result: `approve` or `request_changes`. - result: TicketReviewResultParam, - /// Markdown review body. - body: String, -} - #[derive(Debug, Deserialize, schemars::JsonSchema)] struct TicketIntakeReadyParams { /// Ticket id. @@ -839,11 +811,6 @@ struct TicketImplementationReportTool { backend: TicketToolBackend, } -#[derive(Clone)] -struct TicketReviewTool { - backend: TicketToolBackend, -} - #[derive(Clone)] struct TicketIntakeReadyTool { backend: TicketToolBackend, @@ -1117,34 +1084,6 @@ impl_ticket_thread_event_tool!( TicketEventKind::ImplementationReport ); -#[async_trait] -impl Tool for TicketReviewTool { - async fn execute( - &self, - input_json: &str, - _ctx: llm_engine::tool::ToolExecutionContext, - ) -> Result { - let params: TicketReviewParams = parse_input("TicketReview", input_json)?; - let result = match params.result { - TicketReviewResultParam::Approve => TicketReviewResult::Approve, - TicketReviewResultParam::RequestChanges => TicketReviewResult::RequestChanges, - }; - let result_str = result.as_str().to_string(); - let review = TicketReview { - result, - author: None, - body: MarkdownText::new(params.body), - }; - self.backend - .review(TicketIdOrSlug::Query(params.ticket.clone()), review) - .map_err(|error| backend_error("TicketReview", error))?; - Ok(json_output( - format!("Appended {result_str} review to ticket {}", params.ticket), - json!({ "ticket": params.ticket, "review": result_str, "ok": true }), - )) - } -} - #[async_trait] impl Tool for TicketIntakeReadyTool { async fn execute( @@ -1731,7 +1670,6 @@ fn input_schema(name: &str) -> Value { "TicketComment" | "TicketPlan" | "TicketDecision" | "TicketImplementationReport" => { serde_json::to_value(schemars::schema_for!(TicketThreadEventParams)) } - "TicketReview" => serde_json::to_value(schemars::schema_for!(TicketReviewParams)), "TicketIntakeReady" => serde_json::to_value(schemars::schema_for!(TicketIntakeReadyParams)), "TicketQueue" => serde_json::to_value(schemars::schema_for!(TicketQueueParams)), "TicketWorkflowState" => { @@ -1777,7 +1715,6 @@ impl_from_backend!(TicketCommentTool); impl_from_backend!(TicketPlanTool); impl_from_backend!(TicketDecisionTool); impl_from_backend!(TicketImplementationReportTool); -impl_from_backend!(TicketReviewTool); impl_from_backend!(TicketIntakeReadyTool); impl_from_backend!(TicketQueueTool); impl_from_backend!(TicketWorkflowStateTool); @@ -1804,7 +1741,6 @@ pub fn ticket_tools(backend: impl Into) -> Vec("TicketReview", backend.clone()), tool_definition::("TicketIntakeReady", backend.clone()), tool_definition::("TicketQueue", backend.clone()), tool_definition::("TicketWorkflowState", backend.clone()), @@ -1880,7 +1816,6 @@ mod tests { "TicketPlan", "TicketDecision", "TicketImplementationReport", - "TicketReview", "TicketIntakeReady", "TicketQueue", "TicketWorkflowState", @@ -2373,12 +2308,11 @@ mod tests { } #[tokio::test] - async fn ticket_tools_comment_review_state_and_close_are_doctor_clean() { + async fn ticket_tools_report_state_and_close_are_doctor_clean() { let temp = TempDir::new().unwrap(); let backend = backend(&temp); let created = backend.create(NewTicket::new("Flow Tool")).unwrap(); let report = tool_by_name(backend.clone(), "TicketImplementationReport"); - let review = tool_by_name(backend.clone(), "TicketReview"); let close = tool_by_name(backend.clone(), "TicketClose"); let doctor = tool_by_name(backend.clone(), "TicketDoctor"); @@ -2393,18 +2327,6 @@ mod tests { ) .await .unwrap(); - review - .execute( - &json!({ - "ticket": created.id.clone(), - "result": "approve", - "body": "Looks good." - }) - .to_string(), - Default::default(), - ) - .await - .unwrap(); close .execute( &json!({ "ticket": created.id, "resolution": "Done via TicketClose.\n" }) @@ -2427,12 +2349,6 @@ mod tests { .iter() .any(|event| event.kind == TicketEventKind::ImplementationReport) ); - assert!( - closed - .events - .iter() - .any(|event| event.kind == TicketEventKind::Review) - ); assert!( closed .events @@ -2852,7 +2768,6 @@ mod tests { "TicketPlan", "TicketDecision", "TicketImplementationReport", - "TicketReview", "TicketIntakeReady", "TicketQueue", "TicketRelationRecord", diff --git a/crates/tui/src/dashboard/tests.rs b/crates/tui/src/dashboard/tests.rs index b0e21082..72ab7eb2 100644 --- a/crates/tui/src/dashboard/tests.rs +++ b/crates/tui/src/dashboard/tests.rs @@ -795,13 +795,7 @@ async fn ticket_review_action_does_not_silently_approve() { .unwrap_err(); assert!(error.to_string().contains("current action is Queue")); - let ticket = backend.show(TicketIdOrSlug::Id(ticket_id)).unwrap(); - assert!( - !ticket - .events - .iter() - .any(|event| event.kind == TicketEventKind::Review) - ); + let _ticket = backend.show(TicketIdOrSlug::Id(ticket_id)).unwrap(); } #[test] diff --git a/crates/worker/src/feature/builtin.rs b/crates/worker/src/feature/builtin.rs index 28c248fe..77c9732b 100644 --- a/crates/worker/src/feature/builtin.rs +++ b/crates/worker/src/feature/builtin.rs @@ -9,6 +9,7 @@ pub mod manage_workdir; pub mod manage_worker; pub mod memory; pub mod memory_extract; +pub mod merge_request; pub mod objective; pub mod session_explore; pub mod task; diff --git a/crates/worker/src/feature/builtin/merge_request.rs b/crates/worker/src/feature/builtin/merge_request.rs new file mode 100644 index 00000000..104c4db5 --- /dev/null +++ b/crates/worker/src/feature/builtin/merge_request.rs @@ -0,0 +1,299 @@ +use crate::feature::ToolDefinition; +use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod}; +use async_trait::async_trait; +use llm_engine::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::json; +use std::sync::Arc; + +pub const MERGE_REQUEST_COMMON_TOOL_NAMES: &[&str] = &[ + "MergeRequestShow", + "MergeRequestReadinessCheck", + "MergeRequestOpen", + "MergeRequestAddRevision", + "MergeRequestComplete", +]; +pub const MERGE_REQUEST_REVIEW_TOOL_NAME: &str = "MergeRequestReviewSubmit"; +#[derive(Clone, Copy)] +enum Kind { + Show, + Readiness, + Open, + AddRevision, + Complete, + Review, +} +#[derive(Clone)] +struct MergeRequestTool { + client: Arc, + kind: Kind, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct ShowInput { + ticket: String, +} +#[derive(Debug, Deserialize, JsonSchema)] +struct OpenInput { + ticket: String, + repository_id: String, + revision_id: String, + base_commit: String, + head_commit: String, + head_tree: String, + diff_digest: String, + #[serde(default)] + changed_paths: Vec, + #[serde(default)] + summary: String, +} +#[derive(Debug, Deserialize, JsonSchema)] +struct AddRevisionInput { + ticket: String, + expected_current_revision_id: String, + revision_id: String, + base_commit: String, + head_commit: String, + head_tree: String, + diff_digest: String, + #[serde(default)] + changed_paths: Vec, + #[serde(default)] + summary: String, +} +#[derive(Debug, Deserialize, JsonSchema)] +struct CompleteInput { + ticket: String, + operation_id: String, + expected_revision_id: String, +} +#[derive(Debug, Deserialize, JsonSchema)] +struct ReviewInput { + decision: ReviewDecisionInput, + #[serde(default)] + body: String, + #[serde(default)] + findings: Vec, +} +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum ReviewDecisionInput { + Approve, + RequestChanges, +} +#[derive(Debug, Deserialize, JsonSchema)] +struct ReviewFindingInput { + severity: String, + #[serde(default)] + code: Option, + #[serde(default)] + path: Option, + #[serde(default)] + line: Option, + body: String, +} + +impl Kind { + fn name(self) -> &'static str { + match self { + Self::Show => "MergeRequestShow", + Self::Readiness => "MergeRequestReadinessCheck", + Self::Open => "MergeRequestOpen", + Self::AddRevision => "MergeRequestAddRevision", + Self::Complete => "MergeRequestComplete", + Self::Review => "MergeRequestReviewSubmit", + } + } + fn description(self) -> &'static str { + description(self.name()).unwrap_or("Merge Request operation.") + } + fn schema(self) -> serde_json::Value { + match self { + Self::Show | Self::Readiness => json!(schemars::schema_for!(ShowInput)), + Self::Open => json!(schemars::schema_for!(OpenInput)), + Self::AddRevision => json!(schemars::schema_for!(AddRevisionInput)), + Self::Complete => json!(schemars::schema_for!(CompleteInput)), + Self::Review => json!(schemars::schema_for!(ReviewInput)), + } + } +} + +#[async_trait] +impl Tool for MergeRequestTool { + async fn execute( + &self, + input: &str, + _context: ToolExecutionContext, + ) -> Result { + let workspace_id = self.client.workspace_id().ok_or_else(|| { + ToolError::ExecutionFailed("Merge Request tools require Workspace identity".into()) + })?; + let (method, path, body) = match self.kind { + Kind::Show => { + let v: ShowInput = parse(input)?; + nonempty(&v.ticket)?; + ( + WorkspaceRequestMethod::Get, + format!("/api/w/{workspace_id}/tickets/{}/merge-request", v.ticket), + None, + ) + } + Kind::Readiness => { + let v: ShowInput = parse(input)?; + nonempty(&v.ticket)?; + ( + WorkspaceRequestMethod::Get, + format!( + "/api/w/{workspace_id}/tickets/{}/merge-request/readiness", + v.ticket + ), + None, + ) + } + Kind::Open => { + let v: OpenInput = parse(input)?; + nonempty(&v.ticket)?; + ( + WorkspaceRequestMethod::Post, + format!("/api/w/{workspace_id}/tickets/{}/merge-request", v.ticket), + Some( + json!({"repository_id":v.repository_id,"revision_id":v.revision_id,"base_commit":v.base_commit,"head_commit":v.head_commit,"head_tree":v.head_tree,"diff_digest":v.diff_digest,"changed_paths":v.changed_paths,"summary":v.summary}), + ), + ) + } + Kind::AddRevision => { + let v: AddRevisionInput = parse(input)?; + nonempty(&v.ticket)?; + ( + WorkspaceRequestMethod::Post, + format!( + "/api/w/{workspace_id}/tickets/{}/merge-request/revisions", + v.ticket + ), + Some( + json!({"expected_current_revision_id":v.expected_current_revision_id,"revision_id":v.revision_id,"base_commit":v.base_commit,"head_commit":v.head_commit,"head_tree":v.head_tree,"diff_digest":v.diff_digest,"changed_paths":v.changed_paths,"summary":v.summary}), + ), + ) + } + Kind::Complete => { + let v: CompleteInput = parse(input)?; + nonempty(&v.ticket)?; + ( + WorkspaceRequestMethod::Post, + format!( + "/api/w/{workspace_id}/tickets/{}/merge-request/complete", + v.ticket + ), + Some( + json!({"operation_id":v.operation_id,"expected_revision_id":v.expected_revision_id}), + ), + ) + } + Kind::Review => { + let v: ReviewInput = parse(input)?; + let context = self.client.reviewer_attempt_context().ok_or_else(|| { + ToolError::ExecutionFailed( + "MergeRequestReviewSubmit is available only to an attested Reviewer child" + .into(), + ) + })?; + ( + WorkspaceRequestMethod::Post, + format!( + "/api/w/{workspace_id}/tickets/{}/merge-request/reviews", + context.ticket_id + ), + Some( + json!({"decision":match v.decision{ReviewDecisionInput::Approve=>"approve",ReviewDecisionInput::RequestChanges=>"request_changes"},"body":v.body,"findings":v.findings.into_iter().map(|f|json!({"severity":f.severity,"code":f.code,"path":f.path,"line":f.line,"body":f.body})).collect::>() }), + ), + ) + } + }; + let request = match body { + Some(body) => WorkspaceRequest::json(method, path, body.to_string()), + None => WorkspaceRequest::get(path), + }; + let response = self + .client + .execute(request) + .map_err(|e| ToolError::ExecutionFailed(e.to_string()))?; + if !response.is_success() { + return Err(ToolError::ExecutionFailed(format!( + "Merge Request API returned HTTP {}: {}", + response.status, response.body + ))); + } + Ok(ToolOutput { + summary: self.kind.name().to_string(), + content: Some(response.body), + attachments: Vec::new(), + }) + } +} +fn parse(value: &str) -> Result { + serde_json::from_str(value).map_err(|e| ToolError::InvalidArgument(e.to_string())) +} +fn nonempty(value: &str) -> Result<(), ToolError> { + if value.trim().is_empty() { + Err(ToolError::InvalidArgument( + "ticket must not be empty".into(), + )) + } else { + Ok(()) + } +} +fn definition(client: Arc, kind: Kind) -> ToolDefinition { + Arc::new(move || { + let meta = ToolMeta::new(kind.name()) + .description(kind.description()) + .input_schema(kind.schema()); + let tool: Arc = Arc::new(MergeRequestTool { + client: client.clone(), + kind, + }); + (meta, tool) + }) +} +pub fn common_tools(client: Arc) -> Vec { + vec![ + definition(client.clone(), Kind::Show), + definition(client.clone(), Kind::Readiness), + definition(client.clone(), Kind::Open), + definition(client.clone(), Kind::AddRevision), + definition(client, Kind::Complete), + ] +} +pub fn reviewer_tools(client: Arc) -> Vec { + if client.reviewer_attempt_context().is_some() { + vec![ + definition(client.clone(), Kind::Show), + definition(client, Kind::Review), + ] + } else { + Vec::new() + } +} +pub fn description(name: &str) -> Option<&'static str> { + match name { + "MergeRequestShow" => Some( + "Read the authoritative Merge Request, immutable current revision, and structured review status.", + ), + "MergeRequestReadinessCheck" => { + Some("Check derived merge readiness for the current immutable revision.") + } + "MergeRequestOpen" => { + Some("Open an immutable Merge Request revision for the current assigned Coder.") + } + "MergeRequestAddRevision" => { + Some("Append an immutable revision; prior approval cannot carry to the new revision.") + } + "MergeRequestComplete" => { + Some("CAS-complete an approved revision with operation-id replay and crash fencing.") + } + "MergeRequestReviewSubmit" => Some( + "Submit the attested direct-child Reviewer result bound to its immutable revision.", + ), + _ => None, + } +} diff --git a/crates/worker/src/feature/builtin/ticket.rs b/crates/worker/src/feature/builtin/ticket.rs index d017850b..32d3eb30 100644 --- a/crates/worker/src/feature/builtin/ticket.rs +++ b/crates/worker/src/feature/builtin/ticket.rs @@ -14,12 +14,13 @@ use ticket::{ NewTicketRelation, OrchestrationPlanKind, OrchestrationPlanRecord, Result as TicketResult, Ticket, TicketBackend, TicketBackendOperation, TicketBackendOperationResult, TicketDoctorReport, TicketError, TicketIdOrSlug, TicketIntakeSummary, TicketListQuery, - TicketRef, TicketRelation, TicketRelationKind, TicketRelationView, TicketReview, - TicketStateChange, TicketSummary, + TicketRef, TicketRelation, TicketRelationKind, TicketRelationView, TicketStateChange, + TicketSummary, config::{DEFAULT_TICKET_BACKEND_RELATIVE_PATH, TicketConfig}, tool::{TICKET_TOOL_NAMES, TicketToolBackend, ticket_tool_description, ticket_tools}, }; +use super::merge_request; use crate::feature::{ FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError, FeatureInstructionContribution, FeatureInstructionDeclaration, FeatureInstructionId, @@ -100,7 +101,7 @@ impl TicketFeatureAccess { pub const fn review() -> Self { Self { authoring: false, - thread: true, + thread: false, intake: false, orchestration_control: false, } @@ -141,7 +142,7 @@ const AUTHORING_TOOL_NAMES: &[&str] = &[ "TicketRelationRecord", ]; -const THREAD_TOOL_NAMES: &[&str] = &["TicketComment", "TicketReview"]; +const THREAD_TOOL_NAMES: &[&str] = &["TicketComment"]; const INTAKE_TOOL_NAMES: &[&str] = &["TicketIntakeReady"]; @@ -152,7 +153,6 @@ const WORKSPACE_AUTHORING_TOOL_NAMES: &[&str] = &[ "TicketList", "TicketShow", "TicketComment", - "TicketReview", "TicketQueue", "TicketClose", "TicketDependencyCheck", @@ -167,7 +167,6 @@ const ORCHESTRATION_CONTROL_TOOL_NAMES: &[&str] = &[ "TicketList", "TicketShow", "TicketComment", - "TicketReview", "TicketWorkflowState", "TicketClose", "TicketDependencyCheck", @@ -340,6 +339,22 @@ impl FeatureModule for TicketFeature { ticket_tool_description(name, self.record_language.as_deref()), )); } + if let TicketFeatureBackend::WorkspaceClient(client) = &self.backend { + let names: Vec<&str> = if client.reviewer_attempt_context().is_some() { + vec![ + "MergeRequestShow", + merge_request::MERGE_REQUEST_REVIEW_TOOL_NAME, + ] + } else { + merge_request::MERGE_REQUEST_COMMON_TOOL_NAMES.to_vec() + }; + for name in names { + descriptor = descriptor.with_tool(ToolDeclaration::new( + name, + merge_request::description(name).unwrap_or("Merge Request operation."), + )); + } + } descriptor } @@ -373,6 +388,17 @@ impl FeatureModule for TicketFeature { } tools.register(ToolContribution::new(name, definition))?; } + if let TicketFeatureBackend::WorkspaceClient(client) = &self.backend { + let definitions = if client.reviewer_attempt_context().is_some() { + merge_request::reviewer_tools(client.clone()) + } else { + merge_request::common_tools(client.clone()) + }; + for definition in definitions { + let (meta, _) = definition(); + tools.register(ToolContribution::new(meta.name.clone(), definition))?; + } + } Ok(()) } } @@ -611,14 +637,6 @@ impl WorkspaceHttpTicketBackend { format!("{base}/{}/workflow/queue", Self::ticket_path(&id)), None, ), - TicketBackendOperation::Review { id, review } => Self::request_unit( - client, - WorkspaceRequestMethod::Post, - format!("{base}/{}/workflow/review", Self::ticket_path(&id)), - Some(serde_json::to_value(review).map_err(|error| { - TicketError::Conflict(format!("serialize Ticket review: {error}")) - })?), - ), TicketBackendOperation::Close { id, resolution } => Self::request_unit( client, WorkspaceRequestMethod::Post, @@ -844,15 +862,6 @@ impl TicketBackend for WorkspaceHttpTicketBackend { } } - fn review(&self, id: TicketIdOrSlug, review: TicketReview) -> TicketResult<()> { - match self.invoke(TicketBackendOperation::Review { id, review })? { - TicketBackendOperationResult::Unit => Ok(()), - other => Err(TicketError::Conflict(format!( - "unexpected ticket backend response: {other:?}" - ))), - } - } - fn close(&self, id: TicketIdOrSlug, resolution: MarkdownText) -> TicketResult<()> { match self.invoke(TicketBackendOperation::Close { id, resolution })? { TicketBackendOperationResult::Unit => Ok(()), @@ -1075,7 +1084,6 @@ mod tests { .map(|tool| tool.name.as_str()) .collect::>(); assert!(work_report_tools.contains(&"TicketComment")); - assert!(work_report_tools.contains(&"TicketReview")); assert!(!work_report_tools.contains(&"TicketWorkflowState")); let review = ticket_tools_feature_with_access(temp.path(), TicketFeatureAccess::review()); @@ -1085,7 +1093,6 @@ mod tests { .iter() .map(|tool| tool.name.as_str()) .collect::>(); - assert!(review_tools.contains(&"TicketReview")); assert!(!review_tools.contains(&"TicketWorkflowState")); } diff --git a/crates/worker/src/internal_worker.rs b/crates/worker/src/internal_worker.rs index 7f7811db..3e637b47 100644 --- a/crates/worker/src/internal_worker.rs +++ b/crates/worker/src/internal_worker.rs @@ -259,6 +259,10 @@ pub(crate) struct InternalWorkerSessionHandle { } impl InternalWorkerSessionHandle { + pub(crate) fn session_id_string(&self) -> String { + self.session_id.to_string() + } + pub(crate) fn status(&self) -> InternalWorkerSessionStatus { InternalWorkerSessionStatus::decode(self.status.load(std::sync::atomic::Ordering::Acquire)) } diff --git a/crates/worker/src/spawn/tool.rs b/crates/worker/src/spawn/tool.rs index 9ead8643..21fbc182 100644 --- a/crates/worker/src/spawn/tool.rs +++ b/crates/worker/src/spawn/tool.rs @@ -27,7 +27,10 @@ use crate::internal_worker::{ }; use crate::prompt::catalog::PromptCatalog; use crate::spawn::registry::SpawnedWorkerRegistry; -use crate::worker::{Worker, WorkerFilesystemAuthority}; +use crate::worker::{ + ReviewerAttemptContext, ReviewerChildWorkspaceClient, Worker, WorkerFilesystemAuthority, + WorkspaceRequest, WorkspaceRequestMethod, +}; use protocol::Method; #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -55,6 +58,16 @@ struct SubWorkerSpawnInput { /// spawner's explicit delegation authority; direct tool scope alone is not /// sufficient. Omit `recursive` for normal workspace/worktree delegation; it defaults to true. scope: Vec, + /// Binds an actual read-only builtin Reviewer child to an immutable Merge Request revision. + /// Review attempt identity and capability material are generated by the trusted spawn layer. + #[serde(default)] + review: Option, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct ReviewerHandoffInput { + ticket_id: String, + revision_id: String, } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -320,6 +333,32 @@ impl SubWorkerSpawnTool { } } +fn validate_reviewer_handoff(input: &SubWorkerSpawnInput) -> Result<(), ToolError> { + let Some(review) = &input.review else { + return Ok(()); + }; + if review.ticket_id.trim().is_empty() || review.revision_id.trim().is_empty() { + return Err(ToolError::InvalidArgument( + "reviewer handoff requires non-empty ticket_id and revision_id".to_string(), + )); + } + if input.profile.as_deref() != Some("builtin:reviewer") { + return Err(ToolError::InvalidArgument( + "reviewer handoff requires the explicit effective profile builtin:reviewer".to_string(), + )); + } + if input + .scope + .iter() + .any(|rule| matches!(rule.permission, PermissionInput::Write)) + { + return Err(ToolError::InvalidArgument( + "Merge Request Reviewer SubWorkers must have read-only delegated scope".to_string(), + )); + } + Ok(()) +} + #[async_trait] impl Tool for SubWorkerSpawnTool { async fn execute( @@ -340,6 +379,7 @@ impl Tool for SubWorkerSpawnTool { input.name ))); } + validate_reviewer_handoff(&input)?; let name_reservation = self .registry .reserve_internal_name(input.name.clone()) @@ -378,6 +418,48 @@ impl Tool for SubWorkerSpawnTool { .map_err(|error| { ToolError::ExecutionFailed(format!("resolve child manifest: {error}")) })?; + let reviewer_attempt = input.review.as_ref().map(|review| { + ( + review.ticket_id.clone(), + review.revision_id.clone(), + uuid::Uuid::now_v7().to_string(), + format!( + "{}{}", + uuid::Uuid::now_v7().simple(), + uuid::Uuid::now_v7().simple() + ), + ) + }); + let child_workspace_context = + if let Some((ticket_id, revision_id, _, capability_token)) = &reviewer_attempt { + let workspace_id = + self.workspace_context + .workspace_id() + .cloned() + .ok_or_else(|| { + ToolError::InvalidArgument( + "reviewer handoff requires Workspace identity".to_string(), + ) + })?; + let parent_client = self.workspace_context.client_handle(); + if !parent_client.is_available() { + return Err(ToolError::InvalidArgument( + "reviewer handoff requires Workspace API authority".to_string(), + )); + } + let child_client: Arc = + Arc::new(ReviewerChildWorkspaceClient::new( + parent_client.clone(), + ReviewerAttemptContext { + ticket_id: ticket_id.clone(), + revision_id: revision_id.clone(), + }, + capability_token.clone(), + )); + crate::worker::WorkerWorkspaceContext::with_client(Some(workspace_id), child_client) + } else { + self.workspace_context.clone() + }; let store = EphemeralSessionStore::default(); let filesystem_authority = WorkerFilesystemAuthority::local(self.workspace_root.clone(), child_cwd.clone()); @@ -385,7 +467,7 @@ impl Tool for SubWorkerSpawnTool { child_manifest, store.clone(), self.prompt_loader.clone(), - self.workspace_context.clone(), + child_workspace_context, filesystem_authority, self.internal_client_override .as_ref() @@ -465,6 +547,66 @@ impl Tool for SubWorkerSpawnTool { } }; + if let Some((ticket_id, revision_id, attempt_id, capability_token)) = &reviewer_attempt { + let workspace_id = self.workspace_context.workspace_id().ok_or_else(|| { + ToolError::ExecutionFailed("reviewer attempt lost Workspace identity".to_string()) + })?; + let child_session_id = session.session_id_string(); + let child_registration = WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + format!( + "/api/w/{}/internal/reviewer-child-sessions", + workspace_id.as_str() + ), + serde_json::json!({"child_session_id": child_session_id}).to_string(), + ); + let child_response = self + .workspace_context + .client() + .execute(child_registration) + .map_err(|error| { + ToolError::ExecutionFailed(format!( + "register Runtime-owned Reviewer child session: {error}" + )) + })?; + if !child_response.is_success() { + let _ = session.stop().await; + return Err(ToolError::ExecutionFailed(format!( + "register Runtime-owned Reviewer child session failed with status {}: {}", + child_response.status, child_response.body + ))); + } + let body = serde_json::json!({ + "attempt_id": attempt_id, + "revision_id": revision_id, + "child_session_id": child_session_id, + "capability_token": capability_token, + }); + let request = WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + format!( + "/api/w/{}/tickets/{}/merge-request/review-attempts", + workspace_id.as_str(), + ticket_id + ), + body.to_string(), + ); + let response = self + .workspace_context + .client() + .execute(request) + .map_err(|error| { + ToolError::ExecutionFailed(format!("register reviewer attempt: {error}")) + })?; + if !response.is_success() { + let _ = session.stop().await; + return Err(ToolError::ExecutionFailed(format!( + "register reviewer attempt failed with status {}: {}", + response.status, response.body + ))); + } + } + let record = crate::spawn::registry::InternalSpawnedWorkerRecord::new( input.name.clone(), scope_allow, @@ -899,6 +1041,31 @@ mod tests { WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceResponse, }; + #[test] + fn reviewer_handoff_requires_explicit_builtin_profile_and_read_only_scope() { + let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({ + "name":"reviewer","task":"review","profile":"builtin:reviewer", + "scope":[{"target":"/tmp/work","permission":"read"}], + "review":{"ticket_id":"T1","revision_id":"V1"} + })) + .unwrap(); + assert!(validate_reviewer_handoff(&valid).is_ok()); + let wrong_profile: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({ + "name":"reviewer","task":"review","profile":"builtin:coder", + "scope":[{"target":"/tmp/work","permission":"read"}], + "review":{"ticket_id":"T1","revision_id":"V1"} + })) + .unwrap(); + assert!(validate_reviewer_handoff(&wrong_profile).is_err()); + let writable: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({ + "name":"reviewer","task":"review","profile":"builtin:reviewer", + "scope":[{"target":"/tmp/work","permission":"write"}], + "review":{"ticket_id":"T1","revision_id":"V1"} + })) + .unwrap(); + assert!(validate_reviewer_handoff(&writable).is_err()); + } + fn abs_rule(path: &Path, permission: Permission) -> ScopeRule { ScopeRule { target: path.to_path_buf(), diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 48ec8b53..26d857f7 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -223,6 +223,94 @@ pub trait WorkspaceClient: std::fmt::Debug + Send + Sync { fn is_available(&self) -> bool; fn execute(&self, request: WorkspaceRequest) -> Result; + + /// Trusted review-attempt context is injected by the Internal SubWorker spawn layer. + /// It is never accepted from a model-visible tool argument. + fn reviewer_attempt_context(&self) -> Option<&ReviewerAttemptContext> { + None + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReviewerAttemptContext { + pub ticket_id: String, + pub revision_id: String, +} + +#[derive(Debug)] +pub struct ReviewerChildWorkspaceClient { + inner: Arc, + context: ReviewerAttemptContext, + capability_token: String, +} + +impl ReviewerChildWorkspaceClient { + pub fn new( + inner: Arc, + context: ReviewerAttemptContext, + capability_token: String, + ) -> Self { + Self { + inner, + context, + capability_token, + } + } +} + +impl WorkspaceClient for ReviewerChildWorkspaceClient { + fn workspace_id(&self) -> Option<&str> { + self.inner.workspace_id() + } + fn kind(&self) -> &str { + "runtime-reviewer-child" + } + fn is_available(&self) -> bool { + self.inner.is_available() + } + fn reviewer_attempt_context(&self) -> Option<&ReviewerAttemptContext> { + Some(&self.context) + } + + fn execute( + &self, + mut request: WorkspaceRequest, + ) -> Result { + let expected_path = format!( + "/api/w/{}/tickets/{}/merge-request/reviews", + self.workspace_id().unwrap_or_default(), + self.context.ticket_id + ); + if request.method == WorkspaceRequestMethod::Post && request.path == expected_path { + let body = request.body.take().ok_or_else(|| { + WorkspaceClientError::Request("review submission requires a JSON body".to_string()) + })?; + let mut value: serde_json::Value = serde_json::from_str(&body) + .map_err(|error| WorkspaceClientError::Request(error.to_string()))?; + let object = value.as_object_mut().ok_or_else(|| { + WorkspaceClientError::Request( + "review submission body must be an object".to_string(), + ) + })?; + object.insert( + "revision_id".to_string(), + serde_json::Value::String(self.context.revision_id.clone()), + ); + object.insert( + "capability_token".to_string(), + serde_json::Value::String(self.capability_token.clone()), + ); + request.body = Some( + serde_json::to_string(&value) + .map_err(|error| WorkspaceClientError::Request(error.to_string()))?, + ); + } else if request.method != WorkspaceRequestMethod::Get { + return Err(WorkspaceClientError::Unavailable( + "Reviewer child Workspace authority is read-only except for its one attested Merge Request review submission".to_string(), + )); + } + self.inner.execute(request) + } } /// HTTP forwarding client created by Runtime for one concrete Worker execution. @@ -365,6 +453,36 @@ impl WorkspaceClient for MarkerWorkspaceClient { } } +#[cfg(test)] +mod reviewer_client_tests { + use super::*; + + #[test] + fn reviewer_child_client_denies_non_review_workspace_mutations() { + let inner: Arc = Arc::new(MarkerWorkspaceClient { + workspace_id: Some("ws".to_string()), + kind: "marker".to_string(), + available: true, + reason: "forwarded".to_string(), + }); + let client = ReviewerChildWorkspaceClient::new( + inner, + ReviewerAttemptContext { + ticket_id: "T1".into(), + revision_id: "V1".into(), + }, + "secret".into(), + ); + let request = WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + "/api/w/ws/tickets/T1/comments", + "{}".to_string(), + ); + let error = client.execute(request).unwrap_err(); + assert!(error.to_string().contains("read-only")); + } +} + pub fn unavailable_workspace_client( workspace_id: Option<&WorkspaceId>, reason: impl Into, diff --git a/crates/workspace-server/Cargo.toml b/crates/workspace-server/Cargo.toml index eda17ca8..4f7f7e4f 100644 --- a/crates/workspace-server/Cargo.toml +++ b/crates/workspace-server/Cargo.toml @@ -32,6 +32,7 @@ sha2.workspace = true thiserror.workspace = true ticket.workspace = true memory.workspace = true +merge-request.workspace = true tokio = { workspace = true, features = ["fs", "macros", "net", "rt-multi-thread", "sync", "time"] } tokio-tungstenite.workspace = true worker.workspace = true diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 2e2d67ca..32708b85 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -55,6 +55,8 @@ pub enum Error { Sqlite(#[from] rusqlite::Error), #[error("ticket error: {0}")] Ticket(#[from] ticket::TicketError), + #[error("merge request error: {0}")] + MergeRequest(#[from] merge_request::MergeRequestError), #[error("yaml error: {0}")] Yaml(#[from] serde_yaml::Error), #[error("invalid input: {0}")] @@ -88,6 +90,14 @@ pub enum Error { }, #[error("unknown local repository `{0}`")] UnknownRepository(String), + #[error( + "merge confirmation requires an authenticated Browser session; API tokens and Worker actors are not accepted" + )] + BrowserMergeConfirmationRequired, + #[error( + "Merge Request reopen requires an authenticated Browser session and explicit confirmation" + )] + BrowserReopenConfirmationRequired, #[error("workspace id does not match this Workspace backend")] WorkspaceIdMismatch, #[error("Ticket assignment conflict: {0}")] diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 6529ad6d..9dfe907a 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -24,8 +24,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use ticket::{ MarkdownText, NewTicketEvent, TicketBackend, TicketBodyReplacement, TicketEventKind, - TicketIdOrSlug, TicketItemEdit, TicketReview, TicketReviewResult, TicketStateChange, - TicketTargetEdit, TicketWorkflowState, + TicketIdOrSlug, TicketItemEdit, TicketStateChange, TicketTargetEdit, TicketWorkflowState, }; use ticket::{ SqliteTicketBackend, TicketBackendOperation, TicketBackendOperationResult, @@ -861,8 +860,40 @@ pub fn build_router(api: WorkspaceApi) -> Router { post(scoped_queue_ticket_record), ) .route( - "/api/w/{workspace_id}/tickets/{id}/workflow/review", - post(scoped_review_ticket_record), + "/api/w/{workspace_id}/tickets/{id}/merge-request", + get(scoped_show_merge_request).post(scoped_open_merge_request), + ) + .route( + "/api/w/{workspace_id}/tickets/{id}/merge-request/readiness", + get(scoped_merge_request_readiness), + ) + .route( + "/api/w/{workspace_id}/tickets/{id}/merge-request/revisions", + post(scoped_add_merge_request_revision), + ) + .route( + "/api/w/{workspace_id}/internal/reviewer-child-sessions", + post(scoped_register_reviewer_child_session), + ) + .route( + "/api/w/{workspace_id}/tickets/{id}/merge-request/review-attempts", + post(scoped_register_merge_request_review_attempt), + ) + .route( + "/api/w/{workspace_id}/tickets/{id}/merge-request/reviews", + post(scoped_submit_merge_request_review), + ) + .route( + "/api/w/{workspace_id}/tickets/{id}/merge-request/complete", + post(scoped_complete_merge_request), + ) + .route( + "/api/w/{workspace_id}/tickets/{id}/merge-request/reopen", + post(scoped_reopen_merge_request), + ) + .route( + "/api/w/{workspace_id}/tickets/{id}/merge-request/merge", + post(scoped_confirm_merge_request), ) .route( "/api/w/{workspace_id}/tickets/{id}/workflow/close", @@ -902,10 +933,6 @@ pub fn build_router(api: WorkspaceApi) -> Router { "/api/w/{workspace_id}/tickets/{id}/events", post(scoped_append_ticket_event), ) - .route( - "/api/w/{workspace_id}/tickets/{id}/reviews", - post(scoped_review_ticket), - ) .route( "/api/w/{workspace_id}/tickets/{id}/queue", post(scoped_queue_ticket), @@ -2419,14 +2446,6 @@ struct BrowserAppendTicketEventRequest { author: Option, } -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct BrowserReviewTicketRequest { - result: TicketReviewResult, - body: String, - author: Option, -} - #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct BrowserQueueTicketRequest { @@ -2505,6 +2524,11 @@ async fn scoped_transition_ticket_state( Json(request): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; + if request.state == TicketWorkflowState::Done { + return Err(Error::TicketAssignmentConflict( + "done is guarded by MergeRequestComplete with an approved immutable revision and operation_id".to_string(), + ).into()); + } let current = api.authority.ticket(&path.id)?; let mut change = TicketStateChange::new( current.state, @@ -2535,25 +2559,6 @@ async fn scoped_append_ticket_event( browser_ticket_detail(&api, &path.id) } -async fn scoped_review_ticket( - State(api): State, - AxumPath(path): AxumPath, - Json(request): Json, -) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - browser_ticket_backend(&api)? - .review( - TicketIdOrSlug::Id(path.id.clone()), - TicketReview { - result: request.result, - body: MarkdownText::new(request.body), - author: request.author, - }, - ) - .map_err(Error::from)?; - browser_ticket_detail(&api, &path.id) -} - async fn scoped_queue_ticket( State(api): State, AxumPath(path): AxumPath, @@ -2602,6 +2607,16 @@ async fn scoped_close_ticket( browser_ticket_detail(&api, &path.id) } +fn reject_unguarded_ticket_completion(operation: &TicketBackendOperation) -> Result<()> { + if matches!(operation, TicketBackendOperation::SetWorkflowState { change, .. } if change.to == "done") + { + return Err(Error::TicketAssignmentConflict( + "done is guarded by MergeRequestComplete with an approved immutable revision and operation_id".to_string(), + )); + } + Ok(()) +} + async fn execute_worker_ticket_rest_operation( api: &WorkspaceApi, workspace_id: &str, @@ -2621,6 +2636,7 @@ async fn execute_worker_ticket_rest_operation( let is_mutation = operation_kind != "read"; let target = ticket_mutation_target(&operation).cloned(); let source = authenticate_worker_mutation_source(api, workspace_id, &headers)?; + reject_unguarded_ticket_completion(&operation)?; validate_ticket_repository_operation(api, &operation)?; let before = target.as_ref().and_then(|id| backend.show(id.clone()).ok()); let previous_state = before @@ -2969,23 +2985,393 @@ async fn scoped_queue_ticket_record( ticket_rest_unit(result) } -async fn scoped_review_ticket_record( - State(api): State, - AxumPath((workspace_id, id)): AxumPath<(String, String)>, - headers: HeaderMap, - Json(review): Json, -) -> ApiResult { - let result = execute_worker_ticket_rest_operation( - &api, - &workspace_id, - headers, - TicketBackendOperation::Review { - id: TicketIdOrSlug::Query(id), - review, - }, +#[derive(Debug, serde::Deserialize)] +struct OpenMergeRequestRequest { + repository_id: String, + revision_id: String, + base_commit: String, + head_commit: String, + head_tree: String, + diff_digest: String, + #[serde(default)] + changed_paths: Vec, + #[serde(default)] + summary: String, +} + +#[derive(Debug, serde::Deserialize)] +struct AddMergeRequestRevisionRequest { + expected_current_revision_id: String, + revision_id: String, + base_commit: String, + head_commit: String, + head_tree: String, + diff_digest: String, + #[serde(default)] + changed_paths: Vec, + #[serde(default)] + summary: String, +} + +#[derive(Debug, serde::Deserialize)] +struct RegisterReviewerChildSessionRequest { + child_session_id: String, +} + +#[derive(Debug, serde::Deserialize)] +struct RegisterMergeRequestReviewAttemptRequest { + attempt_id: String, + revision_id: String, + child_session_id: String, + capability_token: String, +} + +#[derive(Debug, serde::Deserialize)] +struct SubmitMergeRequestReviewRequest { + revision_id: String, + capability_token: String, + decision: merge_request::ReviewDecision, + #[serde(default)] + body: String, + #[serde(default)] + findings: Vec, +} + +#[derive(Debug, serde::Deserialize)] +struct CompleteMergeRequestRequest { + operation_id: String, + expected_revision_id: String, +} + +#[derive(Debug, serde::Deserialize)] +struct RevisionTransitionRequest { + expected_revision_id: String, + explicit_confirmation: bool, +} + +#[derive(Debug, serde::Deserialize)] +struct ConfirmMergeRequestRequest { + expected_revision_id: String, + explicit_confirmation: bool, +} + +fn parse_workspace_id(value: &str) -> ApiResult { + if value.trim().is_empty() { + return Err(Error::InvalidInput("workspace_id must not be empty".to_string()).into()); + } + Ok(value.to_string()) +} + +fn require_workspace_access(workspace_id: &str, api: &WorkspaceApi) -> ApiResult<()> { + if workspace_id != api.workspace_id() { + return Err(Error::WorkspaceIdMismatch.into()); + } + Ok(()) +} + +fn merge_request_store( + api: &WorkspaceApi, + workspace_id: &str, +) -> ApiResult { + require_workspace_access(workspace_id, api)?; + merge_request::SqliteMergeRequestStore::open_verified( + api.config.database_path.clone(), + workspace_id, ) - .await?; - ticket_rest_unit(result) + .map_err(Error::from) + .map_err(Into::into) +} + +async fn scoped_show_merge_request( + State(api): State, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, +) -> ApiResult> { + let workspace_id = parse_workspace_id(&workspace_id)?; + let store = merge_request_store(&api, &workspace_id)?; + let value = store + .show_for_ticket(&ticket_id)? + .ok_or_else(|| Error::from(merge_request::MergeRequestError::NotFound(ticket_id)))?; + Ok(Json(value)) +} + +async fn scoped_merge_request_readiness( + State(api): State, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, +) -> ApiResult> { + let workspace_id = parse_workspace_id(&workspace_id)?; + Ok(Json( + merge_request_store(&api, &workspace_id)?.readiness_for_ticket(&ticket_id)?, + )) +} + +async fn scoped_open_merge_request( + State(api): State, + headers: HeaderMap, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, + Json(input): Json, +) -> ApiResult> { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let assignment = api + .store + .get_current_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no current assigned Coder".to_string()) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not the current Ticket assignee".to_string(), + ) + .into()); + } + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + let revision = merge_request::MergeRequestRevision { + revision_id: input.revision_id, + ordinal: 1, + base_commit: input.base_commit, + head_commit: input.head_commit, + head_tree: input.head_tree, + diff_digest: input.diff_digest, + changed_paths: input.changed_paths, + summary: input.summary, + assignment_id: assignment.assignment_id.clone(), + created_at: now.clone(), + }; + let mr = merge_request_store(&api, &workspace_id)?.open_merge_request( + merge_request::OpenMergeRequest { + merge_request_id: format!("mr_{}", Uuid::now_v7().simple()), + ticket_id, + repository_id: input.repository_id, + revision, + authenticated_runtime_id: source.runtime_id, + authenticated_worker_id: source.worker_id, + now, + }, + )?; + Ok(Json(mr)) +} + +async fn scoped_add_merge_request_revision( + State(api): State, + headers: HeaderMap, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, + Json(input): Json, +) -> ApiResult> { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let assignment = api + .store + .get_current_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no current assigned Coder".into()) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not the current Ticket assignee".into(), + ) + .into()); + } + let current = merge_request_store(&api, &workspace_id)? + .show_for_ticket(&ticket_id)? + .ok_or_else(|| { + Error::from(merge_request::MergeRequestError::NotFound( + ticket_id.clone(), + )) + })?; + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + let mr = + merge_request_store(&api, &workspace_id)?.add_revision(merge_request::AddRevision { + ticket_id, + expected_current_revision_id: input.expected_current_revision_id, + revision: merge_request::MergeRequestRevision { + revision_id: input.revision_id, + ordinal: current.current_revision.ordinal + 1, + base_commit: input.base_commit, + head_commit: input.head_commit, + head_tree: input.head_tree, + diff_digest: input.diff_digest, + changed_paths: input.changed_paths, + summary: input.summary, + assignment_id: assignment.assignment_id, + created_at: now.clone(), + }, + authenticated_runtime_id: source.runtime_id, + authenticated_worker_id: source.worker_id, + now, + })?; + Ok(Json(mr)) +} + +async fn scoped_register_reviewer_child_session( + State(api): State, + headers: HeaderMap, + AxumPath(workspace_id): AxumPath, + Json(input): Json, +) -> ApiResult { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + merge_request_store(&api, &workspace_id)?.register_reviewer_child_session( + merge_request::RegisterReviewerChildSession { + parent_runtime_id: source.runtime_id, + parent_worker_id: source.worker_id, + child_session_id: input.child_session_id, + now: Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + }, + )?; + Ok(StatusCode::NO_CONTENT) +} + +async fn scoped_register_merge_request_review_attempt( + State(api): State, + headers: HeaderMap, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, + Json(input): Json, +) -> ApiResult { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let assignment = api + .store + .get_current_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no current assigned Coder".into()) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not the current Ticket assignee".into(), + ) + .into()); + } + merge_request_store(&api, &workspace_id)?.register_review_attempt( + merge_request::RegisterReviewAttempt { + attempt_id: input.attempt_id, + ticket_id, + revision_id: input.revision_id, + parent_assignment_id: assignment.assignment_id, + parent_runtime_id: source.runtime_id, + parent_worker_id: source.worker_id, + child_session_id: input.child_session_id, + capability_token: input.capability_token, + now: Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + }, + )?; + Ok(StatusCode::NO_CONTENT) +} + +async fn scoped_submit_merge_request_review( + State(api): State, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, + Json(input): Json, +) -> ApiResult> { + let workspace_id = parse_workspace_id(&workspace_id)?; + let review = + merge_request_store(&api, &workspace_id)?.submit_review(merge_request::SubmitReview { + ticket_id, + revision_id: input.revision_id, + capability_token: input.capability_token, + decision: input.decision, + body: input.body, + findings: input.findings, + now: Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + })?; + Ok(Json(review)) +} + +async fn scoped_complete_merge_request( + State(api): State, + headers: HeaderMap, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, + Json(input): Json, +) -> ApiResult> { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + let source = authenticate_worker_mutation_source(&api, &workspace_id, &headers)?; + let assignment = api + .store + .get_current_ticket_worker_assignment(&workspace_id, &ticket_id)? + .ok_or_else(|| { + Error::TicketAssignmentConflict("Ticket has no current assigned Coder".into()) + })?; + if assignment.worker.runtime_id != source.runtime_id + || assignment.worker.worker_id != source.worker_id + { + return Err(Error::TicketAssignmentConflict( + "authenticated Worker is not the current Ticket assignee".into(), + ) + .into()); + } + let outcome = merge_request_store(&api, &workspace_id)?.complete( + merge_request::CompleteMergeRequest { + operation_id: input.operation_id, + ticket_id, + expected_revision_id: input.expected_revision_id, + assignment_id: assignment.assignment_id, + authenticated_runtime_id: source.runtime_id, + authenticated_worker_id: source.worker_id, + now: Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + }, + )?; + Ok(Json(outcome)) +} + +async fn scoped_reopen_merge_request( + State(api): State, + headers: HeaderMap, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, + Json(input): Json, +) -> ApiResult> { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + reject_non_browser_merge_auth(&headers) + .map_err(|_| Error::BrowserReopenConfirmationRequired)?; + let _actor = require_actor(&api, &headers).await?; + if !input.explicit_confirmation { + return Err(Error::BrowserReopenConfirmationRequired.into()); + } + Ok(Json(merge_request_store(&api, &workspace_id)?.reopen( + &ticket_id, + &input.expected_revision_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + )?)) +} + +fn reject_non_browser_merge_auth(headers: &HeaderMap) -> Result<()> { + if headers.contains_key("authorization") { + return Err(Error::BrowserMergeConfirmationRequired); + } + Ok(()) +} + +async fn scoped_confirm_merge_request( + State(api): State, + headers: HeaderMap, + AxumPath((workspace_id, ticket_id)): AxumPath<(String, String)>, + Json(input): Json, +) -> ApiResult> { + let workspace_id = parse_workspace_id(&workspace_id)?; + require_workspace_access(&workspace_id, &api)?; + reject_non_browser_merge_auth(&headers)?; + let actor = require_actor(&api, &headers).await?; + let mr = merge_request_store(&api, &workspace_id)?.confirm_merge( + merge_request::MergeConfirmation { + ticket_id, + expected_revision_id: input.expected_revision_id, + authenticated_account_id: actor.account_id, + actor_kind: "user".to_string(), + explicit_confirmation: input.explicit_confirmation, + now: Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + }, + )?; + Ok(Json(mr)) } async fn scoped_close_ticket_record( @@ -3157,7 +3543,6 @@ fn ticket_mutation_target(operation: &TicketBackendOperation) -> Option<&TicketI | TicketBackendOperation::SetWorkflowState { id, .. } | TicketBackendOperation::MarkIntakeReady { id, .. } | TicketBackendOperation::QueueReady { id, .. } - | TicketBackendOperation::Review { id, .. } | TicketBackendOperation::Close { id, .. } | TicketBackendOperation::AddTicketRelation { id, .. } | TicketBackendOperation::AddOrchestrationPlanRecord { id, .. } => Some(id), @@ -3203,7 +3588,6 @@ fn bind_worker_ticket_operation_source( change.author = Some(author); } TicketBackendOperation::QueueReady { queued_by, .. } => *queued_by = author, - TicketBackendOperation::Review { review, .. } => review.author = Some(author), TicketBackendOperation::AddTicketRelation { relation, .. } => { relation.author = Some(author) } @@ -3225,7 +3609,6 @@ fn ticket_mutation_operation_kind(operation: &TicketBackendOperation) -> &'stati TicketBackendOperation::SetWorkflowState { .. } => "set_workflow_state", TicketBackendOperation::MarkIntakeReady { .. } => "mark_intake_ready", TicketBackendOperation::QueueReady { .. } => "queue_ready", - TicketBackendOperation::Review { .. } => "review", TicketBackendOperation::Close { .. } => "close", TicketBackendOperation::AddTicketRelation { .. } => "add_relation", TicketBackendOperation::AddOrchestrationPlanRecord { .. } => "add_plan_record", @@ -7299,10 +7682,7 @@ struct RuntimeConfigBundleAvailabilityQuery { digest: String, } -fn reject_workdir_for_embedded_runtime( - runtime_id: &str, - has_workdir: bool, -) -> std::result::Result<(), ApiError> { +fn reject_workdir_for_embedded_runtime(runtime_id: &str, has_workdir: bool) -> ApiResult<()> { if runtime_id != EMBEDDED_WORKER_RUNTIME_ID || !has_workdir { return Ok(()); } @@ -7320,9 +7700,7 @@ fn reject_workdir_for_embedded_runtime( )) } -fn reject_no_workdir_for_non_embedded_runtime( - runtime_id: &str, -) -> std::result::Result<(), ApiError> { +fn reject_no_workdir_for_non_embedded_runtime(runtime_id: &str) -> ApiResult<()> { if runtime_id == EMBEDDED_WORKER_RUNTIME_ID { return Ok(()); } @@ -9974,6 +10352,12 @@ struct ApiErrorLog { diagnostics: Vec, } +impl From for ApiError { + fn from(error: merge_request::MergeRequestError) -> Self { + Error::MergeRequest(error).into() + } +} + impl From for ApiError { fn from(error: Error) -> Self { let diagnostics = match &error { @@ -10013,6 +10397,9 @@ impl ApiError { impl IntoResponse for ApiError { fn into_response(self) -> Response { let status = match &self.error { + Error::BrowserMergeConfirmationRequired | Error::BrowserReopenConfirmationRequired => { + StatusCode::FORBIDDEN + } Error::TicketAssignmentConflict(_) | Error::WorkdirAttachmentConflict(_) => { StatusCode::CONFLICT } @@ -10020,7 +10407,14 @@ impl IntoResponse for ApiError { Error::InvalidRuntimeIdentifier { .. } | Error::ReservedWorkerName(_) => { StatusCode::BAD_REQUEST } - Error::Ticket(ticket::TicketError::NotFound(_)) => StatusCode::NOT_FOUND, + Error::Ticket(ticket::TicketError::NotFound(_)) + | Error::MergeRequest(merge_request::MergeRequestError::NotFound(_)) => { + StatusCode::NOT_FOUND + } + Error::MergeRequest(merge_request::MergeRequestError::Empty(_)) => { + StatusCode::BAD_REQUEST + } + Error::MergeRequest(_) => StatusCode::CONFLICT, Error::Ticket( ticket::TicketError::Ambiguous { .. } | ticket::TicketError::Locked { .. } @@ -10161,6 +10555,32 @@ mod tests { ObjectiveTicketLinkRecord, SqliteWorkspaceStore, WorkspaceRecord, }; + #[test] + fn merge_confirmation_rejects_api_token_actor_before_session_resolution() { + let mut headers = HeaderMap::new(); + headers.insert("authorization", "Bearer api-token".parse().unwrap()); + assert!(matches!( + reject_non_browser_merge_auth(&headers), + Err(Error::BrowserMergeConfirmationRequired) + )); + assert!(reject_non_browser_merge_auth(&HeaderMap::new()).is_ok()); + } + + #[test] + fn flow_or_generic_worker_state_change_is_not_ticket_completion_authority() { + let operation = TicketBackendOperation::SetWorkflowState { + id: TicketIdOrSlug::Query("T1".to_string()), + change: TicketStateChange::new( + "inprogress", + "done", + "flow reached terminal state", + "terminal flow state", + ), + }; + let error = reject_unguarded_ticket_completion(&operation).unwrap_err(); + assert!(error.to_string().contains("MergeRequestComplete")); + } + #[test] fn failed_api_log_is_structured_and_omits_query_values() { let uri = "/api/w/workspace/tickets?access_token=secret" @@ -12596,21 +13016,6 @@ mod tests { .unwrap(); assert_eq!(queued.state, "queued"); assert_eq!(queued.queued_by.as_deref(), Some("browser-user")); - let Json(reviewed) = scoped_review_ticket( - State(api.clone()), - AxumPath(path()), - Json(BrowserReviewTicketRequest { - result: TicketReviewResult::Approve, - body: "API review".to_string(), - author: Some("reviewer".to_string()), - }), - ) - .await - .unwrap(); - assert!(reviewed.events.iter().any(|event| { - event.kind == "review" && event.body.as_deref() == Some("API review") - })); - let Json(closed) = scoped_close_ticket( State(api), AxumPath(path()), diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index e98b77b2..92cdfd03 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -765,6 +765,7 @@ impl SqliteWorkspaceStore { configure_sqlite(&conn)?; apply_migrations(&conn)?; ticket::migrate_sqlite_ticket_schema(&conn)?; + merge_request::migrate(&conn).map_err(|error| Error::Store(error.to_string()))?; validate_workspace_repository_references(&conn)?; Ok(Self { conn: Arc::new(Mutex::new(conn)), diff --git a/crates/yoi/src/ticket_cli.rs b/crates/yoi/src/ticket_cli.rs index c51ca8a8..cad60e55 100644 --- a/crates/yoi/src/ticket_cli.rs +++ b/crates/yoi/src/ticket_cli.rs @@ -12,8 +12,8 @@ use ticket::config::{ use ticket::{ LocalTicketBackend, MarkdownText, NewTicket, NewTicketEvent, NewTicketRelation, SqliteTicketBackend, TicketBackend, TicketDoctorSeverity, TicketEventKind, TicketIdOrSlug, - TicketIntakeSummary, TicketListQuery, TicketListState, TicketRelationKind, TicketReview, - TicketReviewResult, TicketSummary, TicketWorkflowState, + TicketIntakeSummary, TicketListQuery, TicketListState, TicketRelationKind, TicketSummary, + TicketWorkflowState, }; const DEFAULT_LIST_LIMIT: usize = 50; @@ -35,7 +35,6 @@ pub enum TicketCommand { List(ListOptions), Show { query: String }, Comment(CommentOptions), - Review(ReviewOptions), State(StateOptions), Close(CloseOptions), Relation(RelationOptions), @@ -67,13 +66,6 @@ pub struct CommentOptions { pub body: BodySource, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ReviewOptions { - pub query: String, - pub result: TicketReviewResult, - pub body: BodySource, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StateTarget { Planning, @@ -194,7 +186,6 @@ pub fn parse_ticket_args(args: &[String]) -> Result { query: parse_one_positional("show", &args[1..])?, }, "comment" => TicketCommand::Comment(parse_comment(&args[1..])?), - "review" => TicketCommand::Review(parse_review(&args[1..])?), "state" => TicketCommand::State(parse_state(&args[1..])?), "close" => TicketCommand::Close(parse_close(&args[1..])?), "relation" => TicketCommand::Relation(parse_relation(&args[1..])?), @@ -249,7 +240,6 @@ fn run_command( TicketCommand::List(options) => list(backend.as_ref(), options), TicketCommand::Show { query } => show(backend.as_ref(), query), TicketCommand::Comment(options) => comment(backend.as_ref(), options), - TicketCommand::Review(options) => review(backend.as_ref(), options), TicketCommand::State(options) => state(backend.as_ref(), options), TicketCommand::Close(options) => close(backend.as_ref(), options), TicketCommand::Relation(options) => relation(backend.as_ref(), options), @@ -633,23 +623,6 @@ fn comment( Ok(success(format!("appended\t{}\t{}\n", options.query, role))) } -fn review( - backend: &dyn TicketBackend, - options: ReviewOptions, -) -> Result { - let result = options.result.as_str().to_string(); - let review = TicketReview { - result: options.result, - author: Some(default_author()), - body: MarkdownText::new(read_body_source(&options.body)?), - }; - backend.review(TicketIdOrSlug::Query(options.query.clone()), review)?; - Ok(success(format!( - "reviewed\t{}\t{}\n", - options.query, result - ))) -} - fn state( backend: &dyn TicketBackend, options: StateOptions, @@ -660,7 +633,11 @@ fn state( StateTarget::Ready => TicketWorkflowState::Ready, StateTarget::Queued => TicketWorkflowState::Queued, StateTarget::InProgress => TicketWorkflowState::InProgress, - StateTarget::Done => TicketWorkflowState::Done, + StateTarget::Done => { + return Err(TicketCliError::new( + "done is guarded by MergeRequestComplete with an approved immutable revision and operation_id", + )); + } StateTarget::Closed => { return Err(TicketCliError::new( "yoi ticket state closed cannot write resolution.md; use `yoi ticket close --resolution ` instead", @@ -965,64 +942,6 @@ fn parse_comment(args: &[String]) -> Result { }) } -fn parse_review(args: &[String]) -> Result { - if args.is_empty() || args[0].starts_with('-') { - return Err(TicketCliError::new("review requires ")); - } - let query = args[0].clone(); - let mut approve = false; - let mut request_changes = false; - let mut file = None; - let mut message = None; - let mut i = 1; - while i < args.len() { - match args[i].as_str() { - "--approve" => { - approve = true; - i += 1; - } - "--request-changes" => { - request_changes = true; - i += 1; - } - _ => match option_with_value(args, &mut i)? { - Some(("--file", value)) => file = Some(PathBuf::from(value)), - Some(("--message", value)) => message = Some(value), - Some((name, _)) => { - return Err(TicketCliError::new(format!( - "unknown review argument: {name}" - ))); - } - None => { - return Err(TicketCliError::new(format!( - "unknown review argument: {}", - args[i] - ))); - } - }, - } - } - let result = match (approve, request_changes) { - (true, false) => TicketReviewResult::Approve, - (false, true) => TicketReviewResult::RequestChanges, - (false, false) => { - return Err(TicketCliError::new( - "review requires exactly one of --approve or --request-changes", - )); - } - (true, true) => { - return Err(TicketCliError::new( - "review accepts exactly one of --approve or --request-changes", - )); - } - }; - Ok(ReviewOptions { - query, - result, - body: exactly_one_body("review", file, message)?, - }) -} - fn parse_state(args: &[String]) -> Result { if args.len() != 2 { return Err(TicketCliError::new( @@ -1244,7 +1163,7 @@ fn default_author() -> String { } fn help_text() -> &'static str { - "yoi ticket\n\nUsage:\n yoi ticket init\n yoi ticket import-local\n yoi ticket create --title \n yoi ticket list [--state active|all|planning|ready|queued|inprogress|done|closed[,..]] [--limit <n>]\n yoi ticket show <id>\n yoi ticket comment <id> [--role comment|plan|decision|implementation_report] (--file <path>|--message <text>)\n yoi ticket review <id> (--approve|--request-changes) (--file <path>|--message <text>)\n yoi ticket state <id> <planning|ready|queued|inprogress|done|closed>\n yoi ticket close <id> (--resolution <text>|--file <path>)\n yoi ticket relation add --ticket <id> --kind <depends_on|blocks|related|supersedes|duplicate_of> --target <id> [--note <text>]\n yoi ticket relation list [--ticket <id>] [--kind <kind>]\n yoi ticket doctor\n\nOptions:\n -h, --help Print help\n\nBackend:\n Tickets are stored in the workspace SQLite DB under the Yoi data directory.\n `yoi ticket import-local` imports the legacy .yoi/tickets backend root configured in .yoi/workspace.toml.\n `yoi ticket init` writes explicit fixed role profiles and optional [ticket].language into .yoi/workspace.toml, but does not create .yoi/tickets.\n" + "yoi ticket\n\nUsage:\n yoi ticket init\n yoi ticket import-local\n yoi ticket create --title <title>\n yoi ticket list [--state active|all|planning|ready|queued|inprogress|done|closed[,..]] [--limit <n>]\n yoi ticket show <id>\n yoi ticket comment <id> [--role comment|plan|decision|implementation_report] (--file <path>|--message <text>)\n yoi ticket state <id> <planning|ready|queued|inprogress|closed>\n yoi ticket close <id> (--resolution <text>|--file <path>)\n yoi ticket relation add --ticket <id> --kind <depends_on|blocks|related|supersedes|duplicate_of> --target <id> [--note <text>]\n yoi ticket relation list [--ticket <id>] [--kind <kind>]\n yoi ticket doctor\n\nOptions:\n -h, --help Print help\n\nBackend:\n Tickets are stored in the workspace SQLite DB under the Yoi data directory.\n `yoi ticket import-local` imports the legacy .yoi/tickets backend root configured in .yoi/workspace.toml.\n `yoi ticket init` writes explicit fixed role profiles and optional [ticket].language into .yoi/workspace.toml, but does not create .yoi/tickets.\n" } #[cfg(test)] @@ -1375,7 +1294,7 @@ mod tests { } #[test] - fn ticket_cli_create_list_show_comment_review_state_close_and_doctor() { + fn ticket_cli_create_list_show_comment_state_close_and_doctor() { let temp = TempDir::new().unwrap(); let created = run(&temp, &["create", "--title", "CLI Created"]); @@ -1416,22 +1335,6 @@ mod tests { .contains(&format!("appended\t{}\timplementation_report", ticket_id)) ); - let reviewed = run( - &temp, - &[ - "review", - &ticket_id, - "--approve", - "--message", - "Looks good.", - ], - ); - assert!( - reviewed - .stdout - .contains(&format!("reviewed\t{}\tapprove", ticket_id)) - ); - let ready = run(&temp, &["state", &ticket_id, "ready"]); assert_eq!(ready.stdout, format!("state\t{}\tready\n", ticket_id)); let ready_listed = run(&temp, &["list", "--state", "ready"]); @@ -1450,10 +1353,10 @@ mod tests { let inprogress_listed = run(&temp, &["list", "--state", "inprogress"]); assert!(inprogress_listed.stdout.contains(&ticket_id)); - let done = run(&temp, &["state", &ticket_id, "done"]); - assert_eq!(done.stdout, format!("state\t{}\tdone\n", ticket_id)); - let done_listed = run(&temp, &["list", "--state", "done"]); - assert!(done_listed.stdout.contains(&ticket_id)); + let done_error = parse_ticket_args(&args(&["state", &ticket_id, "done"])) + .and_then(|cli| run_in_workspace(cli, temp.path())) + .unwrap_err(); + assert!(done_error.to_string().contains("MergeRequestComplete")); let closed = run( &temp, @@ -1469,7 +1372,6 @@ mod tests { assert!(final_show.stdout.contains("State: closed")); assert!(final_show.stdout.contains("Done via yoi ticket.")); assert!(final_show.stdout.contains("implementation_report")); - assert!(final_show.stdout.contains("review")); } #[test] @@ -1594,20 +1496,6 @@ mod tests { assert!(err.to_string().contains("exactly one")); } - #[test] - fn ticket_cli_rejects_ambiguous_review_result() { - let err = parse_ticket_args(&args(&[ - "review", - "ticket", - "--approve", - "--request-changes", - "--message", - "body", - ])) - .unwrap_err(); - assert!(err.to_string().contains("exactly one")); - } - #[test] fn ticket_cli_state_closed_requires_close_command() { let temp = TempDir::new().unwrap(); diff --git a/docs/development/work-items.md b/docs/development/work-items.md index 940ac58f..fdd78af5 100644 --- a/docs/development/work-items.md +++ b/docs/development/work-items.md @@ -24,7 +24,7 @@ Use the highest-level interface that matches the work: - Use `yoi panel` for the Ticket/Intake/Orchestrator workspace Dashboard and role-launch actions. - Use `yoi objective ...` for lightweight medium-term Objective records and their non-blocking canonical Ticket links. -- Inside Workers, use typed Ticket tools to create, inspect, comment, review, and close Tickets. +- Inside Workers, use typed Ticket tools for Ticket records and typed Merge Request tools for immutable implementation/review/completion evidence. - For multi-step work, follow the typed Ticket role surfaces and recorded Ticket lifecycle gates. Maintainers can inspect the local `.yoi/tickets/` files directly when debugging storage, but normal user instructions should go through `yoi panel`, Ticket tools, or `yoi ticket ...`. @@ -37,8 +37,8 @@ Workers with the Ticket built-in feature can use typed Ticket tools: - `TicketList` — lightweight bounded overview for selecting ids; it returns short summaries only and must not be used as body/thread/artifact authority. - `TicketShow` — detailed authority for a single Ticket, including body/thread/artifact metadata/resolution context subject to its own bounds. - `TicketComment` -- `TicketReview` -- `TicketWorkflowState` +- `MergeRequestShow`, `MergeRequestOpen`, `MergeRequestAddRevision`, `MergeRequestComplete` +- `MergeRequestReviewSubmit` — available only inside the attested direct-child Reviewer attempt; attempt/revision capability material is not model input. - `TicketClose` - `TicketRelationRecord` - `TicketRelationQuery` @@ -52,7 +52,7 @@ Use them when a Worker needs to materialize or update project records: - Intake creates a new Ticket after user agreement. - Orchestrator records routing decisions and intent packets. -- Reviewer records approve/request-changes review results. +- Reviewer commits an approve/request-changes result against one immutable Merge Request revision. - Maintainer closes a Ticket with a resolution when merge/validation/cleanup evidence is complete. Do not bypass Ticket lifecycle gates just because Ticket tools are available. Ticket mutation is a project-record operation and should remain auditable. @@ -241,9 +241,9 @@ Implementation normally happens in a child git worktree created by the Orchestra ### 5. Review -Reviewer Workers should be sibling Workers, not children of coder Workers. They should read the Ticket, intent packet, diff, implementation report, and validation evidence. +The assigned Coder launches the Reviewer as an actual direct-child `builtin:reviewer` SubWorker with read-only scope and a structured handoff bound to the current immutable Merge Request revision. Server authority revalidates the parent assignment, Runtime-owned child session, effective profile, one-shot review attempt, and revision; prose output is not approval. -Review results should be recorded with the `TicketReview` tool. Maintainers working directly with the local backend can use the `yoi ticket` CLI documented later. +The Reviewer records the structured result with `MergeRequestReviewSubmit`. Request changes requires a new immutable revision and a fresh child attempt. `MergeRequestComplete` performs guarded Ticket completion with operation-id dedupe/CAS semantics; Flow transitions are not completion authority. Blockers must be fixed or explicitly escalated before merge-ready submission. diff --git a/resources/flows/coder-review.dcdl b/resources/flows/coder-review.dcdl index ec4abdcd..c39db17c 100644 --- a/resources/flows/coder-review.dcdl +++ b/resources/flows/coder-review.dcdl @@ -5,7 +5,7 @@ states = { implement = { - instructions = "Implement the requested Ticket scope, run the narrow and dependent validation required by the changed contracts, and record the concrete repository/test evidence. When the implementation is ready for independent review, request a Flow transition."; + instructions = "Open or update the Ticket Merge Request with immutable repository revision evidence, run the narrow and dependent validation required by the changed contracts, and record concrete evidence. When the current MR revision is ready for independent review, request a Flow transition. A Flow transition is never Ticket completion authority."; transitions = { review = { target = "review"; @@ -15,11 +15,11 @@ }; review = { - instructions = "Spawn one independent Reviewer SubWorker with bounded Ticket, repository, diff, and validation context. Read its committed review through worker observation. Do not review your own implementation or treat a prose status as approval. After the Reviewer returns a typed approval or concrete requested changes, request a Flow transition."; + instructions = "Spawn one actual direct-child SubWorker with profile builtin:reviewer, read-only scope, and a structured review handoff bound to the current immutable Merge Request revision. The child must commit MergeRequestReviewSubmit; prose output and Worker observation are not approval authority. After the structured current-revision result exists, request a Flow transition."; transitions = { approved = { - target = "done"; - condition = "The latest independent Reviewer attempt for the current implementation completed and approved it, with no later unresolved request_changes finding."; + target = "complete"; + condition = "The authoritative Merge Request current revision has a structured approve result from its registered direct-child builtin:reviewer attempt, with no later unresolved request_changes finding. The Flow transition itself does not complete the Ticket."; }; changes_requested = { target = "fix"; @@ -38,8 +38,18 @@ }; }; + complete = { + instructions = "Call MergeRequestComplete with a fresh operation_id and the approved current revision. The Server must revalidate current assignment, immutable revision, registered Reviewer attempt, and Ticket inprogress CAS. Only after the authoritative operation returns Ticket state done, request a Flow transition."; + transitions = { + completed = { + target = "done"; + condition = "MergeRequestComplete durably returned done for this exact operation_id and current approved revision. A Flow state or prose report alone is never sufficient."; + }; + }; + }; + done = { - instructions = "The Coder implementation and independent review loop is complete."; + instructions = "The guarded Merge Request completion operation committed Ticket state done. Flow terminal state only reflects that durable authority."; terminal = true; }; }; diff --git a/resources/profiles/reviewer.dcdl b/resources/profiles/reviewer.dcdl index d3938983..e86a7406 100644 --- a/resources/profiles/reviewer.dcdl +++ b/resources/profiles/reviewer.dcdl @@ -9,6 +9,6 @@ import "./base.dcdl" // { web = { enabled = true; }; sub_worker = { enabled = false; }; worker = { enabled = false; }; - ticket = { enabled = true; thread = true; }; + ticket = { enabled = true; thread = false; }; }; } diff --git a/resources/prompts/role/coder.md b/resources/prompts/role/coder.md index df051409..3cd33586 100644 --- a/resources/prompts/role/coder.md +++ b/resources/prompts/role/coder.md @@ -1,7 +1,7 @@ -You are the Ticket Coder role. +You are the assigned Coder. Implement the requested scope in the provided Workdir and keep durable evidence on the Ticket and its Merge Request. -Keep role behavior here and treat the first committed user message as concrete Ticket/action context only. Implement only within the delegated worktree/branch and authority scope. Treat the Ticket, intent packet, binding decisions/invariants, implementation latitude, validation expectations, and report expectations as the contract. +Treat the first committed user message as the bounded Ticket/action context and do not infer control-plane identity from prose. -Choose local implementation tactics within that contract. Escalate to the Orchestrator instead of expanding scope when design, permission, dependency, prompt-boundary, or Ticket-boundary questions appear. Do not merge, push, close Tickets, delete worktrees, or create generated memory/local/runtime/log/lock/cache/socket/secret-like `.yoi` state. +Before review, open or append an immutable Merge Request revision containing the exact base/head/tree and changed-path evidence. Spawn the Reviewer only as your actual direct-child `builtin:reviewer` SubWorker, delegate read-only scope, and include the structured `review` handoff with the Ticket id and current MR revision id. Reviewer prose is not approval: the child must commit `MergeRequestReviewSubmit` through its injected attempt authority. -Keep the repository operational throughout the work unless the Ticket explicitly permits a bounded incomplete state. Report the implementation and proportionate validation through the available typed Ticket tools; do not edit Ticket storage directly. +A request-changes result requires a new immutable revision and a fresh Reviewer child attempt. Flow terminal state is not Ticket completion authority. Complete only through `MergeRequestComplete` with a unique operation id and the currently approved revision; the Server revalidates assignment and fences Ticket state side effects. diff --git a/resources/prompts/role/reviewer.md b/resources/prompts/role/reviewer.md index c23429e7..67334119 100644 --- a/resources/prompts/role/reviewer.md +++ b/resources/prompts/role/reviewer.md @@ -1,7 +1,7 @@ -You are the Ticket Reviewer role. +You are the Ticket Reviewer role running as an actual Runtime-owned direct child of the assigned Coder. -Keep role behavior here and treat the first committed user message as concrete Ticket/action context only. Review the implementation against the Ticket intent, binding decisions/invariants, acceptance criteria, and project design boundaries. Prefer read-only inspection and focused validation; do not merge, close, clean up worktrees, or take over implementation unless explicitly asked. +Keep role behavior here and treat the first committed user message as bounded Ticket/Merge Request context only. Review the immutable current Merge Request revision against Ticket intent, binding decisions/invariants, acceptance criteria, and project design boundaries. Use read-only inspection and focused validation; do not merge, close, mutate the Workdir, or take over implementation. -Report clear approve/request-changes evidence with risks, validation performed, and any unresolved requirement or design-boundary concern. When a workflow is invoked, follow that workflow as the procedural authority for reviewer handoff and report shape. +Your prose response is not review authority. Before finishing, call `MergeRequestReviewSubmit` exactly once with `approve` or `request_changes`, a bounded evidence summary, and concrete structured findings. Attempt identity and revision identity are injected by your child Workspace client and are not model inputs. If the authoritative revision changed, submission must fail rather than approving stale work. -Review more than the diff: verify the implementation satisfies the Ticket intent and acceptance criteria, remains coherent with the codebase design, and does not introduce unnecessary behavior or compatibility. +Review more than the diff: verify the implementation satisfies the Ticket intent and acceptance criteria, remains coherent with the codebase design, and does not introduce unnecessary compatibility. diff --git a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts index 9108d2be..0550e7f9 100644 --- a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts +++ b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts @@ -224,7 +224,9 @@ Deno.test("workspace Tickets surface provides Kanban and lifecycle controls", as ticketDetailLoad.includes("/repositories") && ticketDetailPage.includes('mutate("state", "/state"') && ticketDetailPage.includes('mutate("queue", "/queue"') && - ticketDetailPage.includes('mutate("review", "/review"') && + ticketDetailPage.includes("/merge-request/merge") && + ticketDetailPage.includes("explicit_confirmation: true") && + !ticketDetailPage.includes('mutate("review", "/review"') && ticketDetailPage.includes('mutate("close", "/close"') && ticketDetailPage.includes("ticketWorkerLaunchHref") && ticketDetailPage.includes("ticket.relations.outgoing"), diff --git a/web/workspace/src/routes/w/[workspaceId]/tickets/[ticketId]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/tickets/[ticketId]/+page.svelte index ed096582..0f808103 100644 --- a/web/workspace/src/routes/w/[workspaceId]/tickets/[ticketId]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/tickets/[ticketId]/+page.svelte @@ -17,6 +17,16 @@ TicketDetail, } from "$lib/workspace/sidebar/types"; + type MergeRequestDetail = { + state: "draft" | "open" | "closed" | "merged"; + review_status: "pending" | "approved" | "changes_requested"; + current_revision: { revision_id: string; head_commit: string; head_tree: string; diff_digest: string; changed_paths: string[]; summary: string }; + current_review?: { decision: string; body: string; reviewer_effective_profile: string } | null; + merged_at?: string | null; + }; + + const MUTABLE_TICKET_STATES = TICKET_STATES.filter((state) => state !== "done"); + const { data } = $props<{ data: { workspaceId: string; @@ -24,6 +34,7 @@ ticket: ApiResult<TicketDetail>; repositories: ApiResult<RepositoryListResponse>; orchestrator: ApiResult<WorkspaceOrchestratorStatus>; + mergeRequest: ApiResult<MergeRequestDetail | null>; }; }>(); @@ -34,6 +45,7 @@ const orchestratorOnline = initialData.orchestrator.data?.online ?? false; let ticket = $state<TicketDetail>(loadedTicket); + let mergeRequest = $state<MergeRequestDetail | null>(initialData.mergeRequest.data ?? null); let editing = $state(false); let editTitle = $state(loadedTicket.title); let editBody = $state(loadedTicket.body); @@ -43,8 +55,7 @@ let transitionReason = $state(""); let threadRole = $state("comment"); let threadBody = $state(""); - let reviewResult = $state("approve"); - let reviewBody = $state(""); + let confirmMerge = $state(false); let resolution = $state(""); let busy = $state<string | null>(null); let errorMessage = $state<string | null>(null); @@ -134,15 +145,27 @@ ) threadBody = ""; } - async function review(event: SubmitEvent) { - event.preventDefault(); - if (!reviewBody.trim()) return; - if ( - await mutate("review", "/review", { - result: reviewResult, - body: reviewBody.trim(), - }) - ) reviewBody = ""; + async function mergeConfirmedRevision() { + if (!mergeRequest || !confirmMerge || busy) return; + busy = "merge"; + errorMessage = null; + try { + mergeRequest = await workspaceApiJsonWithBody<MergeRequestDetail>( + `${ticketPath}/merge-request/merge`, + { + method: "POST", + body: JSON.stringify({ + expected_revision_id: mergeRequest.current_revision.revision_id, + explicit_confirmation: true, + }), + }, + ); + confirmMerge = false; + } catch (error) { + errorMessage = error instanceof Error ? error.message : String(error); + } finally { + busy = null; + } } async function closeTicket(event: SubmitEvent) { @@ -277,7 +300,6 @@ <p>The Orchestrator is online. Start a role-specific Worker with the Ticket target below.</p> <div class="ticket-role-actions"> <a class="workspace-primary-button" href={ticketWorkerLaunchHref(data.workspaceId, ticket, "coder")}>Coder</a> - <a class="workspace-secondary-button" href={ticketWorkerLaunchHref(data.workspaceId, ticket, "reviewer")}>Reviewer</a> </div> {:else} <p class="workspace-callout">Start the Workspace Orchestrator from the Ticket panel before launching Ticket Workers.</p> @@ -311,7 +333,7 @@ <form class="ticket-control-form" onsubmit={transition}> <label>State <select bind:value={nextState}> - {#each TICKET_STATES as state}<option value={state}>{state}</option>{/each} + {#each MUTABLE_TICKET_STATES as state}<option value={state}>{state}</option>{/each} </select> </label> <label>Reason<input bind:value={transitionReason} placeholder="Optional decision context" /></label> @@ -340,17 +362,27 @@ </form> </details> - <details class="ticket-control-card"> - <summary>Record review</summary> - <form class="ticket-control-form" onsubmit={review}> - <label>Result<select bind:value={reviewResult}> - <option value="approve">Approve</option> - <option value="request_changes">Request changes</option> - </select></label> - <label>Review body<textarea bind:value={reviewBody} rows="5" required></textarea></label> - <button class="workspace-secondary-button" type="submit" disabled={busy === "review" || !reviewBody.trim()}>Record review</button> - </form> - </details> + <section class="ticket-control-card"> + <header><h2>Merge Request</h2></header> + {#if data.mergeRequest.error} + <p class="workspace-callout is-error">{data.mergeRequest.error}</p> + {:else if mergeRequest} + <p><strong>{mergeRequest.state}</strong> · {mergeRequest.review_status}</p> + <p><code>{mergeRequest.current_revision.revision_id}</code></p> + <p>Head <code>{mergeRequest.current_revision.head_commit}</code></p> + {#if mergeRequest.current_revision.summary}<p>{mergeRequest.current_revision.summary}</p>{/if} + {#if mergeRequest.current_review} + <p><strong>{mergeRequest.current_review.decision}</strong> by {mergeRequest.current_review.reviewer_effective_profile}</p> + {#if mergeRequest.current_review.body}<RichMarkdown text={mergeRequest.current_review.body} />{/if} + {/if} + {#if mergeRequest.state === "open" && mergeRequest.review_status === "approved"} + <label><input type="checkbox" bind:checked={confirmMerge} /> Explicitly confirm merge of this revision</label> + <button class="workspace-primary-button" type="button" disabled={!confirmMerge || busy !== null} onclick={mergeConfirmedRevision}>Confirm merge</button> + {/if} + {:else} + <p class="workspace-empty-copy">The assigned Coder has not opened a Merge Request.</p> + {/if} + </section> {#if ticket.state !== "closed"} <details class="ticket-control-card ticket-close-card"> diff --git a/web/workspace/src/routes/w/[workspaceId]/tickets/[ticketId]/+page.ts b/web/workspace/src/routes/w/[workspaceId]/tickets/[ticketId]/+page.ts index 14fa8c45..a5dc7973 100644 --- a/web/workspace/src/routes/w/[workspaceId]/tickets/[ticketId]/+page.ts +++ b/web/workspace/src/routes/w/[workspaceId]/tickets/[ticketId]/+page.ts @@ -1,35 +1,26 @@ import { loadJson, workspaceApiPath } from "$lib/workspace/api/http"; import type { WorkspaceOrchestratorStatus } from "$lib/workspace/tickets/ticket-panel"; -import type { - RepositoryListResponse, - TicketDetail, -} from "$lib/workspace/sidebar/types"; +import type { RepositoryListResponse, TicketDetail } from "$lib/workspace/sidebar/types"; import type { PageLoad } from "./$types"; -export const load = (async ({ fetch, params }) => { - const [ticket, repositories, orchestrator] = await Promise.all([ - loadJson<TicketDetail>( - fetch, - workspaceApiPath( - params.workspaceId, - `/tickets/${encodeURIComponent(params.ticketId)}`, - ), - ), - loadJson<RepositoryListResponse>( - fetch, - workspaceApiPath(params.workspaceId, "/repositories"), - ), - loadJson<WorkspaceOrchestratorStatus>( - fetch, - workspaceApiPath(params.workspaceId, "/orchestrator"), - ), - ]); +async function loadOptionalJson<T>(fetcher: typeof fetch, path: string): Promise<{ data: T | null; error: string | null }> { + try { + const response = await fetcher(path); + if (response.status === 404) return { data: null, error: null }; + if (!response.ok) return { data: null, error: await response.text() || `HTTP ${response.status}` }; + return { data: await response.json() as T, error: null }; + } catch (error) { + return { data: null, error: error instanceof Error ? error.message : String(error) }; + } +} - return { - workspaceId: params.workspaceId, - ticketId: params.ticketId, - ticket, - repositories, - orchestrator, - }; +export const load = (async ({ fetch, params }) => { + const ticketPath = workspaceApiPath(params.workspaceId, `/tickets/${encodeURIComponent(params.ticketId)}`); + const [ticket, repositories, orchestrator, mergeRequest] = await Promise.all([ + loadJson<TicketDetail>(fetch, ticketPath), + loadJson<RepositoryListResponse>(fetch, workspaceApiPath(params.workspaceId, "/repositories")), + loadJson<WorkspaceOrchestratorStatus>(fetch, workspaceApiPath(params.workspaceId, "/orchestrator")), + loadOptionalJson<Record<string, unknown>>(fetch, `${ticketPath}/merge-request`), + ]); + return { workspaceId: params.workspaceId, ticketId: params.ticketId, ticket, repositories, orchestrator, mergeRequest }; }) satisfies PageLoad; From 34f8949e85dc9cca59f26c9e524d8007a7097907 Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Tue, 11 Aug 2026 22:38:29 +0900 Subject: [PATCH 04/12] runtime: canonicalize Worker aggregates --- crates/session-store/src/lib.rs | 8 +- crates/session-store/src/worker_metadata.rs | 130 ++ .../session-store/src/worker_session_store.rs | 422 +++++++ crates/tui/src/ui.rs | 2 +- crates/worker-runtime/src/execution.rs | 4 + crates/worker-runtime/src/fs_store.rs | 1063 ++++++++++++++++- crates/worker-runtime/src/main.rs | 16 +- crates/worker-runtime/src/runtime.rs | 117 +- crates/worker-runtime/src/worker_backend.rs | 244 ++-- crates/worker/src/controller.rs | 33 +- crates/worker/src/runtime/dir.rs | 49 +- .../runtime/worker_allocation/lifecycle.rs | 8 +- crates/worker/src/worker.rs | 4 +- crates/worker/tests/compact_events_test.rs | 10 +- crates/workspace-server/src/server.rs | 18 + docs/design/worker-session-state.md | 86 +- 16 files changed, 2013 insertions(+), 201 deletions(-) create mode 100644 crates/session-store/src/worker_session_store.rs diff --git a/crates/session-store/src/lib.rs b/crates/session-store/src/lib.rs index 3a48ccf6..17035316 100644 --- a/crates/session-store/src/lib.rs +++ b/crates/session-store/src/lib.rs @@ -38,6 +38,7 @@ pub mod segment_log; pub mod store; pub mod system_item; pub mod worker_metadata; +pub mod worker_session_store; pub use event_trace::{TraceEntry, TracePayload}; pub use fs_store::FsStore; @@ -54,10 +55,11 @@ pub use segment_log::{LogEntry, RestoredState, SegmentOrigin, SessionExtension, pub use store::{Store, StoreError}; pub use system_item::{SystemItem, SystemReminder, SystemReminderSource, render_worker_event}; pub use worker_metadata::{ - CombinedStore, FsWorkerStore, WorkerActiveSegmentRef, WorkerMetadata, WorkerMetadataStore, - WorkerPeer, WorkerReclaimedChild, WorkerSpawnedChild, WorkerSpawnedScopeRule, WorkerStoreError, - validate_worker_name, + CombinedStore, FsWorkerStore, WorkerActiveSegmentRef, WorkerAggregateStore, WorkerMetadata, + WorkerMetadataStore, WorkerPeer, WorkerReclaimedChild, WorkerSpawnedChild, + WorkerSpawnedScopeRule, WorkerStoreError, validate_worker_name, }; +pub use worker_session_store::WorkerSessionStore; /// Session identifier — the fork-tree root. UUID v7 (time-ordered). /// diff --git a/crates/session-store/src/worker_metadata.rs b/crates/session-store/src/worker_metadata.rs index ff689e85..77b82f7e 100644 --- a/crates/session-store/src/worker_metadata.rs +++ b/crates/session-store/src/worker_metadata.rs @@ -319,6 +319,115 @@ pub trait WorkerMetadataStore: Send + Sync { } } +/// Metadata store for one canonical Worker aggregate. +/// +/// The Worker key is fixed by Runtime authority and metadata lives directly at +/// `<worker-aggregate>/metadata.json`. This store cannot address another +/// Worker's global metadata root. +#[derive(Clone)] +pub struct WorkerAggregateStore { + root: PathBuf, + worker_name: String, +} + +impl WorkerAggregateStore { + pub fn new( + root: impl Into<PathBuf>, + worker_name: impl Into<String>, + ) -> Result<Self, WorkerStoreError> { + let root = root.into(); + let worker_name = worker_name.into(); + validate_worker_name(&worker_name)?; + fs::create_dir_all(&root)?; + Ok(Self { root, worker_name }) + } + + fn validate_name(&self, worker_name: &str) -> Result<(), WorkerStoreError> { + validate_worker_name(worker_name)?; + if worker_name == self.worker_name { + Ok(()) + } else { + Err(WorkerStoreError::InvalidWorkerName(format!( + "aggregate owns `{}`; requested `{worker_name}`", + self.worker_name + ))) + } + } + + fn metadata_path(&self) -> PathBuf { + self.root.join("metadata.json") + } +} + +impl WorkerMetadataStore for WorkerAggregateStore { + fn write(&self, metadata: &WorkerMetadata) -> Result<(), WorkerStoreError> { + self.validate_name(&metadata.worker_name)?; + let mut content = serde_json::to_vec_pretty(metadata)?; + content.push(b'\n'); + let path = self.metadata_path(); + let temp = self.root.join(format!( + ".metadata.json.tmp-{}-{}", + std::process::id(), + uuid::Uuid::now_v7() + )); + let result = (|| -> Result<(), WorkerStoreError> { + use std::io::Write; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp)?; + file.write_all(&content)?; + file.sync_all()?; + drop(file); + fs::rename(&temp, &path)?; + std::fs::File::open(&self.root)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(temp); + } + result + } + + fn read_by_name(&self, worker_name: &str) -> Result<Option<WorkerMetadata>, WorkerStoreError> { + self.validate_name(worker_name)?; + let content = match fs::read(self.metadata_path()) { + Ok(content) => content, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + let metadata: WorkerMetadata = serde_json::from_slice(&content)?; + if metadata.worker_name != self.worker_name { + return Err(WorkerStoreError::InvalidWorkerName(format!( + "aggregate identity mismatch: expected `{}`, found `{}`", + self.worker_name, metadata.worker_name + ))); + } + Ok(Some(metadata)) + } + + fn list_names(&self) -> Result<Vec<String>, WorkerStoreError> { + Ok(if self.metadata_path().is_file() { + vec![self.worker_name.clone()] + } else { + Vec::new() + }) + } + + fn root_dir(&self) -> Option<PathBuf> { + Some(self.root.clone()) + } + + fn delete_by_name(&self, worker_name: &str) -> Result<(), WorkerStoreError> { + self.validate_name(worker_name)?; + match fs::remove_file(self.metadata_path()) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + } +} + /// Filesystem-backed Worker metadata store. #[derive(Clone)] pub struct FsWorkerStore { @@ -555,6 +664,27 @@ mod tests { assert_eq!(restored, metadata); } + #[test] + fn worker_aggregate_store_writes_one_fixed_metadata_identity() { + let tmp = tempfile::tempdir().unwrap(); + let store = WorkerAggregateStore::new(tmp.path(), "worker-runtime-7").unwrap(); + let metadata = WorkerMetadata::new("worker-runtime-7", None); + store.write(&metadata).unwrap(); + + assert!(tmp.path().join("metadata.json").is_file()); + assert_eq!(store.list_names().unwrap(), vec!["worker-runtime-7"]); + assert_eq!( + store + .read_by_name("worker-runtime-7") + .unwrap() + .unwrap() + .worker_name, + "worker-runtime-7" + ); + let error = store.read_by_name("worker-runtime-8").unwrap_err(); + assert!(error.to_string().contains("aggregate owns")); + } + #[test] fn fs_store_writes_under_worker_state_root_only() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/crates/session-store/src/worker_session_store.rs b/crates/session-store/src/worker_session_store.rs new file mode 100644 index 00000000..902b5dd8 --- /dev/null +++ b/crates/session-store/src/worker_session_store.rs @@ -0,0 +1,422 @@ +//! Filesystem store for the canonical `1 Worker = 1 Session` aggregate. +//! +//! Layout under one Worker aggregate: +//! - `session/session.json` — immutable Session identity +//! - `session/segments/<segment_id>.jsonl` +//! - `session/segments/<segment_id>.trace.jsonl` +//! +//! Unlike [`crate::FsStore`], this store cannot enumerate or switch between +//! arbitrary Sessions. The first segment materializes the sole Session identity; +//! every later operation must use that same ID. + +use crate::event_trace::TraceEntry; +use crate::segment_log::LogEntry; +use crate::store::{Store, StoreError}; +use crate::{SegmentId, SessionId}; +use serde::{Deserialize, Serialize}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; + +const SESSION_SCHEMA_VERSION: u32 = 1; +const SESSION_FILE: &str = "session.json"; +const SEGMENTS_DIR: &str = "segments"; + +#[derive(Clone)] +pub struct WorkerSessionStore { + root: PathBuf, + session_id: Arc<Mutex<Option<SessionId>>>, + append_lock: Arc<Mutex<()>>, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SessionManifest { + schema_version: u32, + session_id: SessionId, +} + +impl WorkerSessionStore { + /// Open the Session store rooted at `<worker-aggregate>/session`. + pub fn new(root: impl Into<PathBuf>) -> Result<Self, StoreError> { + let root = root.into(); + fs::create_dir_all(root.join(SEGMENTS_DIR))?; + let session_id = match fs::read(root.join(SESSION_FILE)) { + Ok(bytes) => { + let manifest: SessionManifest = serde_json::from_slice(&bytes)?; + if manifest.schema_version != SESSION_SCHEMA_VERSION { + return Err(StoreError::Corrupt { + line: 0, + message: format!( + "unsupported Worker Session schema version {}, expected {}", + manifest.schema_version, SESSION_SCHEMA_VERSION + ), + }); + } + Some(manifest.session_id) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + Ok(Self { + root, + session_id: Arc::new(Mutex::new(session_id)), + append_lock: Arc::new(Mutex::new(())), + }) + } + + pub fn root_dir(&self) -> &Path { + &self.root + } + + pub fn session_id(&self) -> Result<Option<SessionId>, StoreError> { + self.session_id + .lock() + .map(|session_id| *session_id) + .map_err(|_| std::io::Error::other("Worker Session identity lock was poisoned").into()) + } + + pub fn session_modified_at(&self) -> Result<Option<SystemTime>, StoreError> { + let metadata = match fs::metadata(&self.root) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + let mut latest = Some(metadata.modified()?); + for entry in fs::read_dir(self.root.join(SEGMENTS_DIR))? { + let modified = entry?.metadata()?.modified()?; + if latest.map(|current| modified > current).unwrap_or(true) { + latest = Some(modified); + } + } + Ok(latest) + } + + fn ensure_session(&self, requested: SessionId, materialize: bool) -> Result<(), StoreError> { + let mut session_id = self + .session_id + .lock() + .map_err(|_| std::io::Error::other("Worker Session identity lock was poisoned"))?; + match *session_id { + Some(existing) if existing == requested => Ok(()), + Some(existing) => Err(StoreError::Corrupt { + line: 0, + message: format!( + "Worker aggregate owns Session {existing}; cannot attach or switch to Session {requested}" + ), + }), + None if !materialize => Err(StoreError::Corrupt { + line: 0, + message: format!( + "Worker aggregate has no materialized Session; requested Session {requested}" + ), + }), + None => { + let manifest = SessionManifest { + schema_version: SESSION_SCHEMA_VERSION, + session_id: requested, + }; + atomic_write_json(&self.root.join(SESSION_FILE), &manifest)?; + *session_id = Some(requested); + Ok(()) + } + } + } + + fn log_path(&self, segment_id: SegmentId) -> PathBuf { + self.root + .join(SEGMENTS_DIR) + .join(format!("{segment_id}.jsonl")) + } + + fn trace_path(&self, segment_id: SegmentId) -> PathBuf { + self.root + .join(SEGMENTS_DIR) + .join(format!("{segment_id}.trace.jsonl")) + } + + fn append_line(&self, path: &Path, line: &str) -> Result<(), StoreError> { + let _guard = self + .append_lock + .lock() + .map_err(|_| std::io::Error::other("Worker Session append lock was poisoned"))?; + let mut file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .append(true) + .open(path)?; + let committed_len = truncate_uncommitted_tail(&mut file)?; + let mut record = Vec::with_capacity(line.len() + 1); + record.extend_from_slice(line.as_bytes()); + record.push(b'\n'); + if let Err(write_error) = file.write_all(&record) { + return match file.set_len(committed_len) { + Ok(()) => Err(write_error.into()), + Err(rollback_error) => Err(std::io::Error::new( + rollback_error.kind(), + format!( + "session append failed ({write_error}) and rollback failed: {rollback_error}" + ), + ) + .into()), + }; + } + Ok(()) + } +} + +impl Store for WorkerSessionStore { + fn append( + &self, + session_id: SessionId, + segment_id: SegmentId, + entry: &LogEntry, + ) -> Result<(), StoreError> { + self.ensure_session(session_id, true)?; + self.append_line(&self.log_path(segment_id), &serde_json::to_string(entry)?) + } + + fn read_all( + &self, + session_id: SessionId, + segment_id: SegmentId, + ) -> Result<Vec<LogEntry>, StoreError> { + self.ensure_session(session_id, false)?; + let path = self.log_path(segment_id); + if !path.exists() { + return Err(StoreError::NotFound(segment_id)); + } + parse_jsonl(&fs::read(path)?) + } + + fn list_sessions(&self) -> Result<Vec<SessionId>, StoreError> { + Ok(self.session_id()?.into_iter().collect()) + } + + fn list_segments(&self, session_id: SessionId) -> Result<Vec<SegmentId>, StoreError> { + self.ensure_session(session_id, false)?; + let mut segments: Vec<SegmentId> = Vec::new(); + for entry in fs::read_dir(self.root.join(SEGMENTS_DIR))? { + let path = entry?.path(); + let name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""); + if name.ends_with(".jsonl") + && !name.ends_with(".trace.jsonl") + && let Ok(segment_id) = name.trim_end_matches(".jsonl").parse() + { + segments.push(segment_id); + } + } + segments.sort_by(|left, right| right.cmp(left)); + Ok(segments) + } + + fn lookup_session_of(&self, segment_id: SegmentId) -> Result<Option<SessionId>, StoreError> { + let session_id = self.session_id()?; + Ok(session_id.filter(|_| self.log_path(segment_id).exists())) + } + + fn create_segment( + &self, + session_id: SessionId, + segment_id: SegmentId, + entries: &[LogEntry], + ) -> Result<(), StoreError> { + self.ensure_session(session_id, true)?; + let mut content = Vec::new(); + for entry in entries { + serde_json::to_writer(&mut content, entry)?; + content.push(b'\n'); + } + atomic_write_bytes(&self.log_path(segment_id), &content)?; + Ok(()) + } + + fn exists(&self, session_id: SessionId, segment_id: SegmentId) -> Result<bool, StoreError> { + self.ensure_session(session_id, false)?; + Ok(self.log_path(segment_id).exists()) + } + + fn read_entry_count( + &self, + session_id: SessionId, + segment_id: SegmentId, + ) -> Result<usize, StoreError> { + self.ensure_session(session_id, false)?; + let path = self.log_path(segment_id); + if !path.exists() { + return Err(StoreError::NotFound(segment_id)); + } + let content = fs::read(path)?; + let complete = complete_jsonl_prefix(&content); + let complete = std::str::from_utf8(complete).map_err(|error| StoreError::Corrupt { + line: complete[..error.valid_up_to()] + .iter() + .filter(|byte| **byte == b'\n') + .count() + + 1, + message: error.to_string(), + })?; + Ok(complete + .lines() + .filter(|line| !line.trim().is_empty()) + .count()) + } + + fn append_trace( + &self, + session_id: SessionId, + segment_id: SegmentId, + entry: &TraceEntry, + ) -> Result<(), StoreError> { + self.ensure_session(session_id, true)?; + self.append_line(&self.trace_path(segment_id), &serde_json::to_string(entry)?) + } +} + +fn atomic_write_json<T: Serialize>(path: &Path, value: &T) -> Result<(), StoreError> { + let mut bytes = serde_json::to_vec_pretty(value)?; + bytes.push(b'\n'); + atomic_write_bytes(path, &bytes) +} + +fn atomic_write_bytes(path: &Path, bytes: &[u8]) -> Result<(), StoreError> { + let parent = path + .parent() + .ok_or_else(|| std::io::Error::other("Worker Session path has no parent"))?; + fs::create_dir_all(parent)?; + let tmp = path.with_file_name(format!( + ".{}.tmp-{}-{}", + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("session"), + std::process::id(), + uuid::Uuid::now_v7() + )); + let result = (|| -> Result<(), StoreError> { + let mut file = OpenOptions::new().create_new(true).write(true).open(&tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + fs::rename(&tmp, path)?; + File::open(parent)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&tmp); + } + result +} + +fn complete_jsonl_prefix(content: &[u8]) -> &[u8] { + if content.last() == Some(&b'\n') { + return content; + } + content + .iter() + .rposition(|byte| *byte == b'\n') + .map(|index| &content[..=index]) + .unwrap_or(&[]) +} + +fn parse_jsonl<T: serde::de::DeserializeOwned>(content: &[u8]) -> Result<Vec<T>, StoreError> { + let complete = complete_jsonl_prefix(content); + let content = std::str::from_utf8(complete).map_err(|error| StoreError::Corrupt { + line: complete[..error.valid_up_to()] + .iter() + .filter(|byte| **byte == b'\n') + .count() + + 1, + message: error.to_string(), + })?; + content + .lines() + .enumerate() + .filter(|(_, line)| !line.trim().is_empty()) + .map(|(index, line)| { + serde_json::from_str(line).map_err(|error| StoreError::Corrupt { + line: index + 1, + message: error.to_string(), + }) + }) + .collect() +} + +fn truncate_uncommitted_tail(file: &mut File) -> std::io::Result<u64> { + const SCAN_BYTES: usize = 8 * 1024; + let len = file.metadata()?.len(); + if len == 0 { + return Ok(0); + } + file.seek(SeekFrom::End(-1))?; + let mut last = [0_u8; 1]; + file.read_exact(&mut last)?; + if last[0] == b'\n' { + return Ok(len); + } + let mut end = len; + let mut buffer = [0_u8; SCAN_BYTES]; + while end > 0 { + let start = end.saturating_sub(SCAN_BYTES as u64); + let chunk_len = (end - start) as usize; + file.seek(SeekFrom::Start(start))?; + file.read_exact(&mut buffer[..chunk_len])?; + if let Some(index) = buffer[..chunk_len].iter().rposition(|byte| *byte == b'\n') { + let committed_len = start + index as u64 + 1; + file.set_len(committed_len)?; + return Ok(committed_len); + } + end = start; + } + file.set_len(0)?; + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Store, new_segment_id, new_session_id}; + + #[test] + fn canonical_layout_and_single_session_invariant() { + let root = tempfile::tempdir().unwrap(); + let store = WorkerSessionStore::new(root.path().join("session")).unwrap(); + let session_id = new_session_id(); + let segment_id = new_segment_id(); + store.create_segment(session_id, segment_id, &[]).unwrap(); + + assert!(root.path().join("session/session.json").is_file()); + assert!( + root.path() + .join(format!("session/segments/{segment_id}.jsonl")) + .is_file() + ); + assert_eq!(store.list_sessions().unwrap(), vec![session_id]); + + let other = new_session_id(); + let error = store + .create_segment(other, new_segment_id(), &[]) + .unwrap_err(); + assert!(error.to_string().contains("cannot attach or switch")); + assert_eq!(store.list_sessions().unwrap(), vec![session_id]); + } + + #[test] + fn reopen_preserves_session_and_segment_ids() { + let root = tempfile::tempdir().unwrap(); + let session_id = new_session_id(); + let segment_id = new_segment_id(); + WorkerSessionStore::new(root.path()) + .unwrap() + .create_segment(session_id, segment_id, &[]) + .unwrap(); + + let reopened = WorkerSessionStore::new(root.path()).unwrap(); + assert_eq!(reopened.session_id().unwrap(), Some(session_id)); + assert!(reopened.exists(session_id, segment_id).unwrap()); + } +} diff --git a/crates/tui/src/ui.rs b/crates/tui/src/ui.rs index 873a95d5..c5b005fa 100644 --- a/crates/tui/src/ui.rs +++ b/crates/tui/src/ui.rs @@ -1524,7 +1524,7 @@ fn render_compact(lines: &mut Vec<Line<'static>>, evt: &CompactEvent, width: u16 .collect::<String>(); let elapsed = elapsed_suffix(*elapsed_secs); ( - format!("[compact] done (new session {short}){elapsed}"), + format!("[compact] done (new segment {short}){elapsed}"), MessageKind::NoticeWarn, ) } diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index c4a618cf..5f43b43a 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -245,6 +245,8 @@ impl fmt::Debug for WorkerExecutionContext { #[derive(Clone, Debug)] pub struct WorkerExecutionSpawnRequest { pub worker_ref: WorkerRef, + /// Monotonic execution generation reserved durably before launch. + pub run_generation: u64, pub request: crate::catalog::CreateWorkerRequest, pub context: WorkerExecutionContext, pub working_directory: Option<WorkingDirectoryBinding>, @@ -255,6 +257,8 @@ pub struct WorkerExecutionSpawnRequest { #[derive(Clone, Debug)] pub struct WorkerExecutionRestoreRequest { pub worker_ref: WorkerRef, + /// Monotonic execution generation reserved durably before restore. + pub run_generation: u64, pub request: crate::catalog::CreateWorkerRequest, pub context: WorkerExecutionContext, pub previous_working_directory: Option<WorkingDirectoryStatus>, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index 608436c4..fe8a1b9e 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -5,7 +5,8 @@ use crate::error::RuntimeError; use crate::identity::{WorkerId, WorkerRef}; use crate::management::{RuntimeBackendKind, RuntimeStatus}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use session_store::{FsStore, Store, TraceEntry, WorkerMetadata}; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::{self, File, OpenOptions}; use std::io::{BufReader, Write}; use std::path::{Path, PathBuf}; @@ -55,6 +56,26 @@ impl FsRuntimeStore { &self.root } + /// Migrate legacy global Worker metadata/Session sources into canonical + /// Runtime-owned Worker aggregates. Legacy roots are read-only and remain + /// in place; only the canonical aggregate is used after this completes. + pub fn migrate_legacy_worker_aggregates( + runtime_root: impl Into<PathBuf>, + legacy_session_root: impl AsRef<Path>, + legacy_worker_metadata_root: impl AsRef<Path>, + ) -> Result<(), RuntimeError> { + let opened = Self::open_or_create(runtime_root.into())?; + let Some(persisted) = opened.state else { + return Ok(()); + }; + migrate_worker_aggregates( + &opened.store.root, + legacy_session_root.as_ref(), + legacy_worker_metadata_root.as_ref(), + persisted.workers.values(), + ) + } + pub(crate) fn open_or_create(root: PathBuf) -> Result<OpenedFsRuntimeStore, RuntimeError> { let existed = root.exists(); if existed && !root.is_dir() { @@ -259,6 +280,8 @@ pub(crate) struct PersistedWorkerRecord { pub(crate) worker_ref: WorkerRef, pub(crate) worker_id: WorkerId, pub(crate) request: CreateWorkerRequest, + /// Last generation durably reserved for this Worker's execution. + pub(crate) run_generation: u64, pub(crate) workspace_id: Option<String>, pub(crate) working_directory: Option<WorkingDirectoryStatus>, } @@ -353,6 +376,8 @@ struct WorkerSnapshot { worker_ref: WorkerRef, worker_id: WorkerId, request: CreateWorkerRequest, + #[serde(default)] + run_generation: u64, #[serde(default, skip_serializing_if = "Option::is_none")] workspace_id: Option<String>, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -376,6 +401,7 @@ impl WorkerSnapshot { worker_ref: worker.worker_ref.clone(), worker_id: worker.worker_id.clone(), request: worker.request.clone(), + run_generation: worker.run_generation, workspace_id: worker.workspace_id.clone(), working_directory: worker.working_directory.clone(), legacy_execution: None, @@ -417,6 +443,7 @@ impl WorkerSnapshot { worker_ref: self.worker_ref, worker_id: self.worker_id, request: self.request, + run_generation: self.run_generation, workspace_id, working_directory: self.working_directory.or_else(|| { self.legacy_execution @@ -498,6 +525,675 @@ fn rename_if_exists(src: &Path, dst: &Path, operation: &'static str) -> Result<( }) } +#[derive(Debug, Serialize, Deserialize)] +struct WorkerAggregateMigrationManifest { + schema_version: u32, + complete: bool, + workers: BTreeMap<String, WorkerAggregateMigrationCheckpoint>, + #[serde(default)] + diagnostics: Vec<WorkerAggregateMigrationDiagnostic>, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +struct WorkerAggregateMigrationDiagnostic { + kind: String, + source: String, + message: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct WorkerAggregateMigrationCheckpoint { + worker_name: String, + session_id: Option<String>, + state: String, +} + +#[derive(Serialize, Deserialize)] +struct CanonicalSessionManifest { + schema_version: u32, + session_id: session_store::SessionId, +} + +fn migrate_worker_aggregates<'a>( + runtime_root: &Path, + legacy_session_root: &Path, + legacy_worker_metadata_root: &Path, + workers: impl Iterator<Item = &'a PersistedWorkerRecord>, +) -> Result<(), RuntimeError> { + if !legacy_session_root.exists() && !legacy_worker_metadata_root.exists() { + return Ok(()); + } + let operation = "migrate legacy Worker aggregates"; + fs::create_dir_all(runtime_root).map_err(|source| RuntimeError::StoreIo { + operation, + path: runtime_root.to_path_buf(), + source, + })?; + let lock_path = runtime_root.join(".worker-aggregate-v1.lock"); + let lock = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(&lock_path) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: lock_path.clone(), + source, + })?; + lock.lock().map_err(|source| RuntimeError::StoreIo { + operation, + path: lock_path, + source, + })?; + + let migration_dir = runtime_root.join("migrations"); + fs::create_dir_all(&migration_dir).map_err(|source| RuntimeError::StoreIo { + operation, + path: migration_dir.clone(), + source, + })?; + let manifest_path = migration_dir.join("worker-aggregate-v1.json"); + let mut manifest = if manifest_path.is_file() { + let manifest: WorkerAggregateMigrationManifest = read_json(&manifest_path, operation)?; + if manifest.schema_version != 1 { + return Err(RuntimeError::StoreCorrupt { + operation, + path: manifest_path, + message: format!( + "unsupported Worker aggregate migration schema {}", + manifest.schema_version + ), + }); + } + manifest + } else { + WorkerAggregateMigrationManifest { + schema_version: 1, + complete: false, + workers: BTreeMap::new(), + diagnostics: Vec::new(), + } + }; + + let workers = workers.cloned().collect::<Vec<_>>(); + let catalog_worker_ids = workers + .iter() + .map(|worker| worker.worker_ref.worker_id.to_string()) + .collect::<BTreeSet<_>>(); + manifest.complete = false; + manifest.diagnostics = collect_legacy_orphan_diagnostics( + legacy_session_root, + legacy_worker_metadata_root, + &workers, + operation, + )?; + let shared_sessions = shared_legacy_session_references(legacy_worker_metadata_root, operation)?; + if !shared_sessions.is_empty() { + for (session_id, worker_ids) in &shared_sessions { + manifest.diagnostics.push(WorkerAggregateMigrationDiagnostic { + kind: "shared_session_reference".to_string(), + source: legacy_session_root.join(session_id).display().to_string(), + message: format!( + "legacy Session {session_id} is referenced by metadata sources {} and was not assigned", + worker_ids.join(", ") + ), + }); + for source_name in worker_ids { + let Some(worker_id) = source_name.strip_prefix("worker-runtime-") else { + continue; + }; + if !catalog_worker_ids.contains(worker_id) { + continue; + } + manifest.workers.insert( + worker_id.to_string(), + WorkerAggregateMigrationCheckpoint { + worker_name: source_name.clone(), + session_id: Some(session_id.clone()), + state: "shared_session_collision".to_string(), + }, + ); + } + } + atomic_write_json(&manifest_path, &manifest, operation)?; + return Err(RuntimeError::StoreCorrupt { + operation, + path: manifest_path, + message: "legacy Session ownership is ambiguous; see migration diagnostics".to_string(), + }); + } + atomic_write_json(&manifest_path, &manifest, operation)?; + + for worker in workers { + let worker_id = worker.worker_ref.worker_id.to_string(); + let worker_name = format!("worker-runtime-{}", worker.worker_ref.worker_id); + let legacy_metadata = legacy_worker_metadata_root + .join(&worker_name) + .join("metadata.json"); + let target_root = runtime_root.join(WORKERS_DIR).join(&worker_id); + let target_metadata = target_root.join("metadata.json"); + + if !legacy_metadata.is_file() { + manifest.workers.insert( + worker_id, + WorkerAggregateMigrationCheckpoint { + worker_name, + session_id: None, + state: "no_legacy_metadata".to_string(), + }, + ); + atomic_write_json(&manifest_path, &manifest, operation)?; + continue; + } + + let legacy_metadata_bytes = + fs::read(&legacy_metadata).map_err(|source| RuntimeError::StoreIo { + operation, + path: legacy_metadata.clone(), + source, + })?; + let metadata: WorkerMetadata = + serde_json::from_slice(&legacy_metadata_bytes).map_err(|source| { + RuntimeError::StoreCorrupt { + operation, + path: legacy_metadata.clone(), + message: source.to_string(), + } + })?; + if metadata.worker_name != worker_name { + return Err(RuntimeError::StoreCorrupt { + operation, + path: legacy_metadata, + message: format!( + "legacy Worker metadata identity mismatch: expected `{worker_name}`, found `{}`", + metadata.worker_name + ), + }); + } + + let session_id = metadata.active.as_ref().map(|active| active.session_id); + if let Some(session_id) = session_id { + let legacy_session = legacy_session_root.join(session_id.to_string()); + if !legacy_session.is_dir() { + return Err(RuntimeError::StoreCorrupt { + operation, + path: legacy_session, + message: format!( + "Worker `{worker_name}` references missing legacy Session {session_id}" + ), + }); + } + validate_legacy_session(legacy_session_root, session_id, operation)?; + migrate_one_session( + &legacy_session, + &target_root.join("session"), + session_id, + operation, + )?; + } + + copy_atomic_or_validate( + &legacy_metadata_bytes, + &target_metadata, + operation, + "Worker metadata collision", + )?; + manifest.workers.insert( + worker_id, + WorkerAggregateMigrationCheckpoint { + worker_name, + session_id: session_id.map(|id| id.to_string()), + state: "migrated".to_string(), + }, + ); + atomic_write_json(&manifest_path, &manifest, operation)?; + } + + manifest.complete = true; + atomic_write_json(&manifest_path, &manifest, operation) +} + +fn shared_legacy_session_references( + legacy_worker_metadata_root: &Path, + operation: &'static str, +) -> Result<BTreeMap<String, Vec<String>>, RuntimeError> { + let mut owners = BTreeMap::<String, Vec<String>>::new(); + if !legacy_worker_metadata_root.is_dir() { + return Ok(owners); + } + for entry in sorted_directory_entries(legacy_worker_metadata_root, operation)? { + let metadata_path = entry.path().join("metadata.json"); + if !entry.path().is_dir() || !metadata_path.is_file() { + continue; + } + let source_name = entry.file_name().to_string_lossy().into_owned(); + let metadata: WorkerMetadata = read_json(&metadata_path, operation)?; + if let Some(active) = metadata.active { + owners + .entry(active.session_id.to_string()) + .or_default() + .push(source_name); + } + } + owners.retain(|_, source_names| source_names.len() > 1); + Ok(owners) +} + +fn collect_legacy_orphan_diagnostics( + legacy_session_root: &Path, + legacy_worker_metadata_root: &Path, + workers: &[PersistedWorkerRecord], + operation: &'static str, +) -> Result<Vec<WorkerAggregateMigrationDiagnostic>, RuntimeError> { + let expected_worker_names = workers + .iter() + .map(|worker| format!("worker-runtime-{}", worker.worker_ref.worker_id)) + .collect::<BTreeSet<_>>(); + let mut referenced_sessions = BTreeSet::new(); + for worker_name in &expected_worker_names { + let metadata_path = legacy_worker_metadata_root + .join(worker_name) + .join("metadata.json"); + if metadata_path.is_file() { + let metadata: WorkerMetadata = read_json(&metadata_path, operation)?; + if let Some(active) = metadata.active { + referenced_sessions.insert(active.session_id.to_string()); + } + } + } + + let mut diagnostics = Vec::new(); + if legacy_worker_metadata_root.is_dir() { + for entry in sorted_directory_entries(legacy_worker_metadata_root, operation)? { + let name = entry.file_name().to_string_lossy().into_owned(); + if entry.path().is_dir() + && entry.path().join("metadata.json").is_file() + && !expected_worker_names.contains(&name) + { + diagnostics.push(WorkerAggregateMigrationDiagnostic { + kind: "orphan_worker_metadata".to_string(), + source: entry.path().display().to_string(), + message: + "legacy Worker metadata has no Runtime catalog Worker and was left in place" + .to_string(), + }); + } + } + } + if legacy_session_root.is_dir() { + for entry in sorted_directory_entries(legacy_session_root, operation)? { + let name = entry.file_name().to_string_lossy().into_owned(); + if entry.path().is_dir() && !referenced_sessions.contains(&name) { + diagnostics.push(WorkerAggregateMigrationDiagnostic { + kind: "orphan_session".to_string(), + source: entry.path().display().to_string(), + message: "legacy Session has no catalog-backed Worker reference and was left in place" + .to_string(), + }); + } + } + } + Ok(diagnostics) +} + +fn validate_legacy_session( + legacy_session_root: &Path, + session_id: session_store::SessionId, + operation: &'static str, +) -> Result<(), RuntimeError> { + let legacy_session = legacy_session_root.join(session_id.to_string()); + for entry in sorted_directory_entries(&legacy_session, operation)? { + let path = entry.path(); + let file_type = entry.file_type().map_err(|source| RuntimeError::StoreIo { + operation, + path: path.clone(), + source, + })?; + if !file_type.is_file() { + return Err(RuntimeError::StoreCorrupt { + operation, + path, + message: "legacy Session contains a non-file entry".to_string(), + }); + } + let name = entry.file_name().to_string_lossy().into_owned(); + if let Some(segment) = name.strip_suffix(".trace.jsonl") { + segment + .parse::<session_store::SegmentId>() + .map_err(|error| RuntimeError::StoreCorrupt { + operation, + path: path.clone(), + message: format!("invalid trace segment filename: {error}"), + })?; + validate_trace_jsonl(&path, operation)?; + } else if let Some(segment) = name.strip_suffix(".jsonl") { + segment + .parse::<session_store::SegmentId>() + .map_err(|error| RuntimeError::StoreCorrupt { + operation, + path, + message: format!("invalid segment filename: {error}"), + })?; + } else { + return Err(RuntimeError::StoreCorrupt { + operation, + path, + message: "legacy Session contains an unknown file".to_string(), + }); + } + } + let store = FsStore::new(legacy_session_root).map_err(|error| RuntimeError::StoreCorrupt { + operation, + path: legacy_session_root.to_path_buf(), + message: error.to_string(), + })?; + for segment_id in + store + .list_segments(session_id) + .map_err(|error| RuntimeError::StoreCorrupt { + operation, + path: legacy_session_root.join(session_id.to_string()), + message: error.to_string(), + })? + { + store + .read_all(session_id, segment_id) + .map_err(|error| RuntimeError::StoreCorrupt { + operation, + path: legacy_session_root + .join(session_id.to_string()) + .join(format!("{segment_id}.jsonl")), + message: error.to_string(), + })?; + } + Ok(()) +} + +fn validate_trace_jsonl(path: &Path, operation: &'static str) -> Result<(), RuntimeError> { + let bytes = fs::read(path).map_err(|source| RuntimeError::StoreIo { + operation, + path: path.to_path_buf(), + source, + })?; + let complete = if bytes.last() == Some(&b'\n') { + bytes.as_slice() + } else { + bytes + .iter() + .rposition(|byte| *byte == b'\n') + .map(|index| &bytes[..=index]) + .unwrap_or(&[]) + }; + let text = std::str::from_utf8(complete).map_err(|error| RuntimeError::StoreCorrupt { + operation, + path: path.to_path_buf(), + message: error.to_string(), + })?; + for (index, line) in text.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + serde_json::from_str::<TraceEntry>(line).map_err(|error| RuntimeError::StoreCorrupt { + operation, + path: path.to_path_buf(), + message: format!("invalid trace JSONL at line {}: {error}", index + 1), + })?; + } + Ok(()) +} + +fn migrate_one_session( + legacy_session: &Path, + target_session: &Path, + session_id: session_store::SessionId, + operation: &'static str, +) -> Result<(), RuntimeError> { + if target_session.exists() { + validate_existing_canonical_session(legacy_session, target_session, session_id, operation)?; + return Ok(()); + } + let target_root = target_session + .parent() + .ok_or_else(|| RuntimeError::StoreCorrupt { + operation, + path: target_session.to_path_buf(), + message: "canonical Session path has no Worker aggregate parent".to_string(), + })?; + fs::create_dir_all(target_root).map_err(|source| RuntimeError::StoreIo { + operation, + path: target_root.to_path_buf(), + source, + })?; + let staging = target_root.join(format!( + ".session.migrating-{}-{}", + std::process::id(), + NEXT_TMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + let result = (|| { + fs::create_dir(&staging).map_err(|source| RuntimeError::StoreIo { + operation, + path: staging.clone(), + source, + })?; + fs::create_dir(staging.join("segments")).map_err(|source| RuntimeError::StoreIo { + operation, + path: staging.join("segments"), + source, + })?; + let session_manifest = CanonicalSessionManifest { + schema_version: 1, + session_id, + }; + atomic_write_json(&staging.join("session.json"), &session_manifest, operation)?; + for entry in sorted_directory_entries(legacy_session, operation)? { + let file_type = entry.file_type().map_err(|source| RuntimeError::StoreIo { + operation, + path: entry.path(), + source, + })?; + if !file_type.is_file() { + return Err(RuntimeError::StoreCorrupt { + operation, + path: entry.path(), + message: "legacy Session contains a non-file entry".to_string(), + }); + } + let bytes = fs::read(entry.path()).map_err(|source| RuntimeError::StoreIo { + operation, + path: entry.path(), + source, + })?; + copy_atomic_or_validate( + &bytes, + &staging.join("segments").join(entry.file_name()), + operation, + "staged Session file collision", + )?; + } + sync_directory(&staging.join("segments"), operation)?; + sync_directory(&staging, operation)?; + fs::rename(&staging, target_session).map_err(|source| RuntimeError::StoreIo { + operation, + path: target_session.to_path_buf(), + source, + })?; + sync_directory(target_root, operation) + })(); + if result.is_err() { + let _ = fs::remove_dir_all(&staging); + } + result +} + +fn validate_existing_canonical_session( + legacy_session: &Path, + target_session: &Path, + session_id: session_store::SessionId, + operation: &'static str, +) -> Result<(), RuntimeError> { + if !target_session.is_dir() { + return Err(RuntimeError::StoreCorrupt { + operation, + path: target_session.to_path_buf(), + message: "canonical Session path collision is not a directory".to_string(), + }); + } + let manifest_path = target_session.join("session.json"); + let manifest: CanonicalSessionManifest = read_json(&manifest_path, operation)?; + if manifest.schema_version != 1 || manifest.session_id != session_id { + return Err(RuntimeError::StoreCorrupt { + operation, + path: manifest_path, + message: format!( + "canonical Session collision: expected Session {session_id}, found {}", + manifest.session_id + ), + }); + } + let source_entries = sorted_directory_entries(legacy_session, operation)?; + let target_segments = target_session.join("segments"); + let target_entries = sorted_directory_entries(&target_segments, operation)?; + let source_names = source_entries + .iter() + .map(|entry| entry.file_name()) + .collect::<BTreeSet<_>>(); + let target_names = target_entries + .iter() + .map(|entry| entry.file_name()) + .collect::<BTreeSet<_>>(); + if source_names != target_names { + return Err(RuntimeError::StoreCorrupt { + operation, + path: target_segments, + message: format!( + "canonical Session file set collision: legacy={source_names:?}, canonical={target_names:?}" + ), + }); + } + for entry in source_entries { + if !entry.path().is_file() { + return Err(RuntimeError::StoreCorrupt { + operation, + path: entry.path(), + message: "legacy Session contains a non-file entry".to_string(), + }); + } + let source = fs::read(entry.path()).map_err(|source| RuntimeError::StoreIo { + operation, + path: entry.path(), + source, + })?; + let target = target_session.join("segments").join(entry.file_name()); + if !target.is_file() { + return Err(RuntimeError::StoreCorrupt { + operation, + path: target, + message: "canonical Session entry is not a file".to_string(), + }); + } + let existing = fs::read(&target).map_err(|source| RuntimeError::StoreIo { + operation, + path: target.clone(), + source, + })?; + if source != existing { + return Err(RuntimeError::StoreCorrupt { + operation, + path: target, + message: "canonical Session file collision differs from legacy source".to_string(), + }); + } + } + Ok(()) +} + +fn sorted_directory_entries( + path: &Path, + operation: &'static str, +) -> Result<Vec<fs::DirEntry>, RuntimeError> { + let mut entries = fs::read_dir(path) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: path.to_path_buf(), + source, + })? + .collect::<Result<Vec<_>, _>>() + .map_err(|source| RuntimeError::StoreIo { + operation, + path: path.to_path_buf(), + source, + })?; + entries.sort_by_key(|entry| entry.file_name()); + Ok(entries) +} + +fn copy_atomic_or_validate( + bytes: &[u8], + target: &Path, + operation: &'static str, + collision: &'static str, +) -> Result<(), RuntimeError> { + if target.exists() { + let existing = fs::read(target).map_err(|source| RuntimeError::StoreIo { + operation, + path: target.to_path_buf(), + source, + })?; + if existing == bytes { + return Ok(()); + } + return Err(RuntimeError::StoreCorrupt { + operation, + path: target.to_path_buf(), + message: collision.to_string(), + }); + } + let parent = target.parent().ok_or_else(|| RuntimeError::StoreCorrupt { + operation, + path: target.to_path_buf(), + message: "migration target has no parent".to_string(), + })?; + fs::create_dir_all(parent).map_err(|source| RuntimeError::StoreIo { + operation, + path: parent.to_path_buf(), + source, + })?; + let temp = tmp_path_for(target); + let result = (|| { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: temp.clone(), + source, + })?; + file.write_all(bytes) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: temp.clone(), + source, + })?; + file.sync_all().map_err(|source| RuntimeError::StoreIo { + operation, + path: temp.clone(), + source, + })?; + drop(file); + fs::rename(&temp, target).map_err(|source| RuntimeError::StoreIo { + operation, + path: target.to_path_buf(), + source, + })?; + sync_directory(parent, operation) + })(); + if result.is_err() { + let _ = fs::remove_file(&temp); + } + result +} + fn read_json<T>(path: &Path, operation: &'static str) -> Result<T, RuntimeError> where T: for<'de> Deserialize<'de>, @@ -600,3 +1296,368 @@ fn sync_directory(path: &Path, operation: &'static str) -> Result<(), RuntimeErr source, }) } + +#[cfg(test)] +mod worker_aggregate_migration_tests { + use super::*; + use crate::catalog::{ + ConfigBundleRef, ProfileSelector, ProfileSourceArchiveHttpRef, ProfileSourceArchiveSource, + }; + use crate::profile_archive::{ProfileSourceArchiveRef, ProfileSourceGraphSummary}; + use session_store::{ + FsWorkerStore, WorkerActiveSegmentRef, WorkerMetadataStore, new_segment_id, new_session_id, + }; + use std::sync::{Arc, Barrier}; + + fn request() -> CreateWorkerRequest { + CreateWorkerRequest { + idempotency_key: None, + idempotency_fingerprint: None, + profile: ProfileSelector::Builtin("builtin:coder".to_string()), + display_name: None, + profile_source: ProfileSourceArchiveSource::Http { + location: ProfileSourceArchiveHttpRef { + url: "http://127.0.0.1/profile-source.tar".to_string(), + etag: None, + archive: ProfileSourceArchiveRef { + id: "test-profile-source".to_string(), + digest: "test-digest".to_string(), + size_bytes: 0, + source_graph: ProfileSourceGraphSummary { + source_count: 0, + total_source_bytes: 0, + entrypoints: BTreeMap::new(), + import_count: 0, + }, + }, + }, + }, + config_bundle: Some(ConfigBundleRef { + id: "bundle".to_string(), + digest: "digest".to_string(), + }), + initial_input: None, + working_directory_request: None, + working_directory: None, + worker_observation_enabled: false, + worker_observation_grants: Vec::new(), + workspace_api: None, + } + } + + fn initialize_runtime(root: &Path, worker_ids: &[u64]) { + let opened = FsRuntimeStore::open_or_create(root.to_path_buf()).unwrap(); + for id in worker_ids { + let worker_id = WorkerId::new(*id); + opened + .store + .write_worker_snapshot(&PersistedWorkerRecord { + worker_ref: WorkerRef::new(worker_id), + worker_id, + request: request(), + run_generation: 0, + workspace_id: None, + working_directory: None, + }) + .unwrap(); + } + let mut state = opened.state.unwrap_or(PersistedRuntimeState { + display_name: None, + status: RuntimeStatus::Running, + next_worker_sequence: 1, + next_diagnostic_id: 1, + workers: BTreeMap::new(), + workspace_owners: BTreeMap::new(), + config_bundles: BTreeMap::new(), + diagnostics: Vec::new(), + }); + state.next_worker_sequence = worker_ids.iter().copied().max().unwrap_or(0) + 1; + opened.store.write_runtime_snapshot(&state).unwrap(); + } + + fn create_legacy_worker( + session_root: &Path, + metadata_root: &Path, + worker_id: u64, + ) -> (session_store::SessionId, session_store::SegmentId) { + let session_id = new_session_id(); + let segment_id = new_segment_id(); + let sessions = FsStore::new(session_root).unwrap(); + sessions + .create_segment(session_id, segment_id, &[]) + .unwrap(); + let metadata = FsWorkerStore::new(metadata_root).unwrap(); + metadata + .write(&WorkerMetadata::new( + format!("worker-runtime-{worker_id}"), + Some(WorkerActiveSegmentRef { + session_id, + segment_id: Some(segment_id), + }), + )) + .unwrap(); + (session_id, segment_id) + } + + #[test] + fn migration_preserves_legacy_ids_and_is_idempotent_for_mixed_layout() { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("runtime"); + let sessions = temp.path().join("legacy-sessions"); + let metadata = temp.path().join("legacy-metadata"); + initialize_runtime(&runtime, &[1, 2]); + let (session_id, segment_id) = create_legacy_worker(&sessions, &metadata, 1); + + // Worker 2 is already canonical and has no legacy source (mixed old/new). + let aggregate2 = runtime.join("workers/2"); + fs::write(aggregate2.join("metadata.json"), b"canonical-new-worker\n").unwrap(); + + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata).unwrap(); + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata).unwrap(); + + let aggregate1 = runtime.join("workers/1"); + let manifest: CanonicalSessionManifest = + read_json(&aggregate1.join("session/session.json"), "test").unwrap(); + assert_eq!(manifest.session_id, session_id); + assert!( + aggregate1 + .join(format!("session/segments/{segment_id}.jsonl")) + .is_file() + ); + assert!(aggregate1.join("metadata.json").is_file()); + assert_eq!( + fs::read(aggregate2.join("metadata.json")).unwrap(), + b"canonical-new-worker\n" + ); + assert!(sessions.join(session_id.to_string()).is_dir()); + assert!(metadata.join("worker-runtime-1/metadata.json").is_file()); + let checkpoint: WorkerAggregateMigrationManifest = + read_json(&runtime.join("migrations/worker-aggregate-v1.json"), "test").unwrap(); + assert!(checkpoint.complete); + assert_eq!(checkpoint.workers["1"].state, "migrated"); + assert_eq!(checkpoint.workers["2"].state, "no_legacy_metadata"); + } + + #[test] + fn partial_rerun_finishes_after_session_atomic_rename_before_metadata_checkpoint() { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("runtime"); + let sessions = temp.path().join("legacy-sessions"); + let metadata = temp.path().join("legacy-metadata"); + initialize_runtime(&runtime, &[1]); + let (session_id, _) = create_legacy_worker(&sessions, &metadata, 1); + let source = sessions.join(session_id.to_string()); + let target = runtime.join("workers/1/session"); + migrate_one_session(&source, &target, session_id, "test partial migration").unwrap(); + assert!(!runtime.join("workers/1/metadata.json").exists()); + + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata).unwrap(); + assert!(runtime.join("workers/1/metadata.json").is_file()); + assert!(target.join("session.json").is_file()); + } + + #[test] + fn collision_and_corruption_fail_without_overwriting_either_copy() { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("runtime"); + let sessions = temp.path().join("legacy-sessions"); + let metadata = temp.path().join("legacy-metadata"); + initialize_runtime(&runtime, &[1]); + let (session_id, segment_id) = create_legacy_worker(&sessions, &metadata, 1); + let source_path = sessions + .join(session_id.to_string()) + .join(format!("{segment_id}.jsonl")); + fs::write(&source_path, b"not-json\n").unwrap(); + let corrupt = + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata) + .unwrap_err(); + assert!( + corrupt + .to_string() + .contains("migrate legacy Worker aggregates") + ); + assert!(!runtime.join("workers/1/session").exists()); + assert_eq!(fs::read(&source_path).unwrap(), b"not-json\n"); + + fs::write(&source_path, b"").unwrap(); + fs::write(runtime.join("workers/1/metadata.json"), b"collision\n").unwrap(); + let collision = + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata) + .unwrap_err(); + assert!(collision.to_string().contains("collision")); + assert_eq!( + fs::read(runtime.join("workers/1/metadata.json")).unwrap(), + b"collision\n" + ); + assert!(metadata.join("worker-runtime-1/metadata.json").is_file()); + } + + #[test] + fn shared_session_reference_fails_closed_and_records_manifest_diagnostic() { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("runtime"); + let sessions = temp.path().join("legacy-sessions"); + let metadata = temp.path().join("legacy-metadata"); + initialize_runtime(&runtime, &[1, 2]); + let (session_id, segment_id) = create_legacy_worker(&sessions, &metadata, 1); + FsWorkerStore::new(&metadata) + .unwrap() + .write(&WorkerMetadata::new( + "worker-runtime-2", + Some(WorkerActiveSegmentRef { + session_id, + segment_id: Some(segment_id), + }), + )) + .unwrap(); + + let error = + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata) + .unwrap_err(); + assert!(error.to_string().contains("ownership is ambiguous")); + assert!(!runtime.join("workers/1/session").exists()); + assert!(!runtime.join("workers/2/session").exists()); + let manifest: WorkerAggregateMigrationManifest = + read_json(&runtime.join("migrations/worker-aggregate-v1.json"), "test").unwrap(); + assert_eq!(manifest.workers["1"].state, "shared_session_collision"); + assert_eq!(manifest.workers["2"].state, "shared_session_collision"); + assert!( + manifest + .diagnostics + .iter() + .any(|diagnostic| diagnostic.kind == "shared_session_reference") + ); + } + + #[test] + fn orphan_metadata_sharing_catalog_session_prevents_assignment() { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("runtime"); + let sessions = temp.path().join("legacy-sessions"); + let metadata = temp.path().join("legacy-metadata"); + initialize_runtime(&runtime, &[1]); + let (session_id, segment_id) = create_legacy_worker(&sessions, &metadata, 1); + FsWorkerStore::new(&metadata) + .unwrap() + .write(&WorkerMetadata::new( + "worker-runtime-999", + Some(WorkerActiveSegmentRef { + session_id, + segment_id: Some(segment_id), + }), + )) + .unwrap(); + + let error = + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata) + .unwrap_err(); + assert!(error.to_string().contains("ownership is ambiguous")); + assert!(!runtime.join("workers/1/session").exists()); + assert!(metadata.join("worker-runtime-999/metadata.json").is_file()); + let manifest: WorkerAggregateMigrationManifest = + read_json(&runtime.join("migrations/worker-aggregate-v1.json"), "test").unwrap(); + assert_eq!(manifest.workers["1"].state, "shared_session_collision"); + assert!(!manifest.workers.contains_key("999")); + assert!(manifest.diagnostics.iter().any(|diagnostic| { + diagnostic.kind == "orphan_worker_metadata" + && diagnostic.source.contains("worker-runtime-999") + })); + } + + #[test] + fn orphan_sources_are_preserved_and_reported_without_becoming_authority() { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("runtime"); + let sessions = temp.path().join("legacy-sessions"); + let metadata = temp.path().join("legacy-metadata"); + initialize_runtime(&runtime, &[1]); + create_legacy_worker(&sessions, &metadata, 1); + let orphan_session = new_session_id(); + FsStore::new(&sessions) + .unwrap() + .create_segment(orphan_session, new_segment_id(), &[]) + .unwrap(); + FsWorkerStore::new(&metadata) + .unwrap() + .write(&WorkerMetadata::new("worker-runtime-999", None)) + .unwrap(); + + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata).unwrap(); + let manifest: WorkerAggregateMigrationManifest = + read_json(&runtime.join("migrations/worker-aggregate-v1.json"), "test").unwrap(); + assert!(manifest.complete); + assert!(manifest.diagnostics.iter().any(|diagnostic| { + diagnostic.kind == "orphan_worker_metadata" + && diagnostic.source.contains("worker-runtime-999") + })); + assert!(manifest.diagnostics.iter().any(|diagnostic| { + diagnostic.kind == "orphan_session" + && diagnostic.source.contains(&orphan_session.to_string()) + })); + assert!(metadata.join("worker-runtime-999/metadata.json").is_file()); + assert!(sessions.join(orphan_session.to_string()).is_dir()); + assert!(!runtime.join("workers/999").exists()); + } + + #[test] + fn trace_corruption_and_extra_canonical_history_are_rejected() { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("runtime"); + let sessions = temp.path().join("legacy-sessions"); + let metadata = temp.path().join("legacy-metadata"); + initialize_runtime(&runtime, &[1]); + let (session_id, segment_id) = create_legacy_worker(&sessions, &metadata, 1); + let trace = sessions + .join(session_id.to_string()) + .join(format!("{segment_id}.trace.jsonl")); + fs::write(&trace, b"not-json\n").unwrap(); + let corrupt = + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata) + .unwrap_err(); + assert!(corrupt.to_string().contains("invalid trace JSONL")); + assert!(!runtime.join("workers/1/session").exists()); + + fs::remove_file(trace).unwrap(); + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata).unwrap(); + let extra = runtime.join("workers/1/session/segments/extra.jsonl"); + fs::write(&extra, b"").unwrap(); + let collision = + FsRuntimeStore::migrate_legacy_worker_aggregates(&runtime, &sessions, &metadata) + .unwrap_err(); + assert!(collision.to_string().contains("file set collision")); + assert!(extra.is_file()); + } + + #[test] + fn concurrent_startup_serializes_one_idempotent_migration() { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("runtime"); + let sessions = temp.path().join("legacy-sessions"); + let metadata = temp.path().join("legacy-metadata"); + initialize_runtime(&runtime, &[1]); + let (session_id, _) = create_legacy_worker(&sessions, &metadata, 1); + let barrier = Arc::new(Barrier::new(3)); + let mut threads = Vec::new(); + for _ in 0..2 { + let runtime = runtime.clone(); + let sessions = sessions.clone(); + let metadata = metadata.clone(); + let barrier = barrier.clone(); + threads.push(std::thread::spawn(move || { + barrier.wait(); + FsRuntimeStore::migrate_legacy_worker_aggregates(runtime, sessions, metadata) + })); + } + barrier.wait(); + for thread in threads { + thread.join().unwrap().unwrap(); + } + let manifest: WorkerAggregateMigrationManifest = + read_json(&runtime.join("migrations/worker-aggregate-v1.json"), "test").unwrap(); + assert!(manifest.complete); + assert_eq!( + manifest.workers["1"].session_id.as_deref(), + Some(session_id.to_string().as_str()) + ); + } +} diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index 1ecc3695..448ffa42 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -18,7 +18,7 @@ use worker_runtime::auth::{ RuntimeHttpAuthConfig, RuntimeIdentityMaterial, TrustedServerKey, decode_public_key, }; use worker_runtime::error::RuntimeError; -use worker_runtime::fs_store::FsRuntimeStoreOptions; +use worker_runtime::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions}; use worker_runtime::http_server::{ RuntimeHttpServerConfig, RuntimeHttpServerError, RuntimeHttpStoreSelection, }; @@ -80,9 +80,13 @@ fn run() -> Result<(), ProcessError> { fn build_runtime(config: &ProcessConfig) -> Result<Runtime, ProcessError> { let fs_paths = config.resolved_fs_paths(); + let runtime_store_dir = match &config.http.store { + RuntimeHttpStoreSelection::Memory => fs_paths.runtime_dir.clone(), + RuntimeHttpStoreSelection::Fs { root } => root.clone(), + _ => fs_paths.runtime_dir.clone(), + }; let mut factory = ProfileRuntimeWorkerFactory::new(fs_paths.worker_dir.join("worker-root")) - .with_store_dir(fs_paths.worker_dir.join("sessions")) - .with_worker_metadata_dir(fs_paths.worker_dir.join("metadata")); + .with_runtime_store_dir(runtime_store_dir); if let Some(endpoint) = config.backend_resource_endpoint.clone() { factory = factory.with_resource_client(Arc::new( worker_runtime::resource::HttpBackendResourceClient::new( @@ -105,6 +109,12 @@ fn build_runtime(config: &ProcessConfig) -> Result<Runtime, ProcessError> { .map_err(ProcessError::Runtime) } RuntimeHttpStoreSelection::Fs { root } => { + FsRuntimeStore::migrate_legacy_worker_aggregates( + root, + fs_paths.worker_dir.join("sessions"), + fs_paths.worker_dir.join("metadata"), + ) + .map_err(ProcessError::Runtime)?; let mut options = FsRuntimeStoreOptions::new(root.clone()); options.display_name = config.http.display_name.clone(); Runtime::with_fs_store_and_execution_backend(options, backend) diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index e4c8b3fa..336bf86d 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -532,12 +532,16 @@ impl Runtime { status: WorkerStatus::Stopped, workspace_id: scope.map(|scope| scope.workspace_id.clone()), request: request.clone(), + run_generation: 1, working_directory: None, execution_handle: None, }; state.workers.insert(worker_id, record); + state.persist_runtime_snapshot()?; + state.persist_worker(&worker_ref.worker_id)?; let spawn_request = WorkerExecutionSpawnRequest { worker_ref: worker_ref.clone(), + run_generation: 1, request, context: self.execution_context(worker_ref.clone()), working_directory: None, @@ -859,35 +863,46 @@ impl Runtime { /// present this is idempotent; otherwise the configured backend is tried. pub fn restore_worker(&self, worker_ref: &WorkerRef) -> Result<WorkerDetail, RuntimeError> { let (backend, request) = { - let state = self.lock()?; + let mut state = self.lock()?; state.ensure_running()?; - let worker = state.worker(worker_ref)?; - if worker.execution_handle.is_some() { - return Ok(worker.detail()); - } - if worker.status == WorkerStatus::Cancelled { - return Err(RuntimeError::InvalidRequest(format!( - "worker {} is cancelled", - worker_ref.worker_id - ))); - } + let (worker_request, previous_working_directory, config_bundle, run_generation) = { + let worker = state.worker(worker_ref)?; + if worker.execution_handle.is_some() { + return Ok(worker.detail()); + } + if worker.status == WorkerStatus::Cancelled { + return Err(RuntimeError::InvalidRequest(format!( + "worker {} is cancelled", + worker_ref.worker_id + ))); + } + let config_bundle = worker + .request + .config_bundle + .as_ref() + .and_then(|bundle_ref| state.config_bundles.get(&bundle_ref.id)) + .cloned(); + ( + worker.request.clone(), + worker.working_directory.clone(), + config_bundle, + worker.run_generation.saturating_add(1).max(1), + ) + }; let backend = state.execution_backend.clone().ok_or_else(|| { RuntimeError::WorkerExecutionUnavailable { worker_id: worker_ref.worker_id.clone(), message: "runtime has no execution backend".to_string(), } })?; - let config_bundle = worker - .request - .config_bundle - .as_ref() - .and_then(|bundle_ref| state.config_bundles.get(&bundle_ref.id)) - .cloned(); + state.worker_mut(worker_ref)?.run_generation = run_generation; + state.persist_worker(&worker_ref.worker_id)?; let request = WorkerExecutionRestoreRequest { worker_ref: worker_ref.clone(), - request: worker.request.clone(), + run_generation, + request: worker_request, context: self.execution_context(worker_ref.clone()), - previous_working_directory: worker.working_directory.clone(), + previous_working_directory, working_directory: None, config_bundle, }; @@ -1514,34 +1529,64 @@ impl Runtime { struct RestoreCandidate { worker_ref: WorkerRef, request: CreateWorkerRequest, + run_generation: u64, previous_working_directory: Option<CatalogWorkingDirectoryStatus>, config_bundle: Option<ConfigBundle>, } let candidates = { - let state = self.lock()?; + let mut state = self.lock()?; if state.execution_backend.is_none() { return Ok(()); } - state + let worker_ids = state .workers .values() .filter(|worker| worker.execution_handle.is_none()) - .map(|worker| { + .map(|worker| worker.worker_id) + .collect::<Vec<_>>(); + let mut candidates = Vec::with_capacity(worker_ids.len()); + for worker_id in worker_ids { + let ( + worker_ref, + request, + previous_working_directory, + config_bundle, + run_generation, + ) = { + let worker = state + .workers + .get(&worker_id) + .expect("collected Worker exists"); let config_bundle = worker .request .config_bundle .as_ref() .and_then(|bundle_ref| state.config_bundles.get(&bundle_ref.id)) .cloned(); - RestoreCandidate { - worker_ref: worker.worker_ref.clone(), - request: worker.request.clone(), - previous_working_directory: worker.working_directory.clone(), + ( + worker.worker_ref.clone(), + worker.request.clone(), + worker.working_directory.clone(), config_bundle, - } - }) - .collect::<Vec<_>>() + worker.run_generation.saturating_add(1).max(1), + ) + }; + state + .workers + .get_mut(&worker_id) + .expect("collected Worker exists") + .run_generation = run_generation; + state.persist_worker(&worker_id)?; + candidates.push(RestoreCandidate { + worker_ref, + request, + run_generation, + previous_working_directory, + config_bundle, + }); + } + candidates }; for candidate in candidates { @@ -1554,6 +1599,7 @@ impl Runtime { }; let request = WorkerExecutionRestoreRequest { worker_ref: candidate.worker_ref.clone(), + run_generation: candidate.run_generation, request: candidate.request, context: self.execution_context(candidate.worker_ref.clone()), previous_working_directory: candidate.previous_working_directory, @@ -1723,6 +1769,7 @@ impl RuntimeState { status: WorkerStatus::Stopped, workspace_id: worker.workspace_id, request: worker.request, + run_generation: worker.run_generation, working_directory: worker.working_directory, execution_handle: None, }, @@ -2281,6 +2328,7 @@ struct WorkerRecord { status: WorkerStatus, workspace_id: Option<String>, request: CreateWorkerRequest, + run_generation: u64, working_directory: Option<CatalogWorkingDirectoryStatus>, execution_handle: Option<WorkerExecutionHandle>, } @@ -2324,6 +2372,7 @@ impl WorkerRecord { worker_ref: self.worker_ref.clone(), worker_id: self.worker_id.clone(), request: self.request.clone(), + run_generation: self.run_generation, workspace_id: self.workspace_id.clone(), working_directory: self.working_directory.clone(), } @@ -2627,6 +2676,7 @@ mod tests { dispatch_result: Mutex<Option<WorkerExecutionResult>>, restore_result: Mutex<Option<WorkerExecutionSpawnResult>>, restore_count: Mutex<u64>, + run_generations: Mutex<Vec<u64>>, contexts: Mutex<BTreeMap<WorkerId, WorkerExecutionContext>>, dispatched_inputs: Mutex<Vec<WorkerInput>>, preserve_commit_ack_submission_id: AtomicBool, @@ -2670,6 +2720,10 @@ mod tests { } fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { + self.run_generations + .lock() + .unwrap() + .push(request.run_generation); self.contexts .lock() .unwrap() @@ -2689,6 +2743,10 @@ mod tests { request: WorkerExecutionRestoreRequest, ) -> WorkerExecutionSpawnResult { *self.restore_count.lock().unwrap() += 1; + self.run_generations + .lock() + .unwrap() + .push(request.run_generation); if let Some(result) = self.restore_result.lock().unwrap().clone() { return result; } @@ -3551,6 +3609,7 @@ mod tests { .unwrap(); assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]); assert_eq!( runtime.worker_detail(&detail.worker_ref).unwrap().status, WorkerStatus::Idle diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 0e58922c..2825c162 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; use std::future::Future; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, mpsc}; use std::time::Duration; @@ -30,9 +30,12 @@ use crate::working_directory::{ WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer, }; use async_trait::async_trait; -use manifest::paths; use protocol::{Event, Method, Segment, WorkerStatus}; -use session_store::{CombinedStore, FsStore, FsWorkerStore, LogEntry, collect_state}; +use session_store::{ + CombinedStore, LogEntry, WorkerAggregateStore, WorkerSessionStore, collect_state, +}; +#[cfg(test)] +use session_store::{FsStore, FsWorkerStore}; use tokio::runtime::Runtime; #[cfg(feature = "ws-server")] use tokio::sync::broadcast; @@ -57,7 +60,6 @@ const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10); // Keep this below the adapter task timeout so a failed acknowledgement task // returns a typed execution error instead of leaving the outer waiter to time out. const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9); -static NEXT_RUNTIME_ARTIFACT_ROOT: AtomicU64 = AtomicU64::new(1); fn user_input_has_submission(entry: &LogEntry, submission_id: &str) -> bool { let LogEntry::UserInput { extensions, .. } = entry else { @@ -69,43 +71,9 @@ fn user_input_has_submission(entry: &LogEntry, submission_id: &str) -> bool { }) } -#[derive(Clone)] -enum RuntimeArtifactRoot { - Owned(Arc<OwnedRuntimeArtifactRoot>), - External(PathBuf), -} - -impl RuntimeArtifactRoot { - fn owned() -> Self { - let sequence = NEXT_RUNTIME_ARTIFACT_ROOT.fetch_add(1, Ordering::Relaxed); - Self::Owned(Arc::new(OwnedRuntimeArtifactRoot { - path: std::env::temp_dir().join(format!( - "yoi-worker-runtime-artifacts-{}-{sequence}", - std::process::id() - )), - })) - } - - fn path(&self) -> &std::path::Path { - match self { - Self::Owned(root) => &root.path, - Self::External(path) => path, - } - } -} - -struct OwnedRuntimeArtifactRoot { - path: PathBuf, -} - -impl Drop for OwnedRuntimeArtifactRoot { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.path); - } -} - pub struct RuntimeWorkerController { pub handle: WorkerHandle, + pub shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>, pub workspace_client: Arc<dyn WorkspaceClient>, } @@ -244,9 +212,7 @@ impl WorkerObservationProvider for RuntimeGrantedWorkerObservationProvider { pub struct ProfileRuntimeWorkerFactory { observation_hub: Arc<RuntimeWorkerObservationHub>, profile_base_dir: PathBuf, - store_dir: Option<PathBuf>, - worker_metadata_dir: Option<PathBuf>, - runtime_base_dir: RuntimeArtifactRoot, + worker_aggregate_root: Option<PathBuf>, resource_client: Option<Arc<dyn BackendResourceClient>>, profile_archive_cache: Arc<ProfileSourceArchiveCache>, } @@ -257,26 +223,14 @@ impl ProfileRuntimeWorkerFactory { Self { observation_hub: Arc::new(RuntimeWorkerObservationHub::default()), profile_base_dir, - store_dir: None, - worker_metadata_dir: None, - runtime_base_dir: RuntimeArtifactRoot::owned(), + worker_aggregate_root: None, resource_client: None, profile_archive_cache: Arc::new(ProfileSourceArchiveCache::default()), } } - pub fn with_store_dir(mut self, store_dir: impl Into<PathBuf>) -> Self { - self.store_dir = Some(store_dir.into()); - self - } - - pub fn with_worker_metadata_dir(mut self, worker_metadata_dir: impl Into<PathBuf>) -> Self { - self.worker_metadata_dir = Some(worker_metadata_dir.into()); - self - } - - pub fn with_runtime_base_dir(mut self, runtime_base_dir: impl Into<PathBuf>) -> Self { - self.runtime_base_dir = RuntimeArtifactRoot::External(runtime_base_dir.into()); + pub fn with_runtime_store_dir(mut self, runtime_store_dir: impl Into<PathBuf>) -> Self { + self.worker_aggregate_root = Some(runtime_store_dir.into().join("workers")); self } @@ -285,28 +239,16 @@ impl ProfileRuntimeWorkerFactory { self } - fn store_dir(&self) -> Result<PathBuf, String> { - self.store_dir - .clone() - .or_else(paths::sessions_dir) + fn worker_aggregate_dir(&self, worker_ref: &WorkerRef) -> Result<PathBuf, String> { + self.worker_aggregate_root + .as_ref() + .map(|root| root.join(worker_ref.worker_id.to_string())) .ok_or_else(|| { - "could not resolve sessions directory (set YOI_DATA_DIR, YOI_HOME, XDG_DATA_HOME, or HOME)" + "Runtime Worker aggregate root is not configured; global Session/metadata roots are migration-only" .to_string() }) } - fn worker_metadata_dir(&self, store_dir: &std::path::Path) -> PathBuf { - self.worker_metadata_dir - .clone() - .or_else(|| paths::data_dir().map(|data_dir| data_dir.join("workers"))) - .or_else(|| store_dir.parent().map(|parent| parent.join("workers"))) - .unwrap_or_else(|| PathBuf::from("workers")) - } - - fn runtime_base_dir(&self) -> Result<PathBuf, String> { - Ok(self.runtime_base_dir.path().to_path_buf()) - } - fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String { format!("worker-runtime-{}", worker_ref.worker_id) } @@ -575,20 +517,23 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { }; let flow_transition_enabled = manifest.feature.flow.enabled; - let store_dir = self.store_dir()?; - let session_store = FsStore::new(&store_dir).map_err(|err| { + let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; + let session_dir = worker_aggregate_dir.join("session"); + let session_store = WorkerSessionStore::new(&session_dir).map_err(|err| { format!( - "failed to initialize session store at {}: {err}", - store_dir.display() - ) - })?; - let worker_metadata_dir = self.worker_metadata_dir(&store_dir); - let worker_metadata_store = FsWorkerStore::new(&worker_metadata_dir).map_err(|err| { - format!( - "failed to initialize worker metadata store at {}: {err}", - worker_metadata_dir.display() + "failed to initialize canonical Worker Session store at {}: {err}", + session_dir.display() ) })?; + let worker_metadata_store = + WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()).map_err( + |err| { + format!( + "failed to initialize canonical Worker metadata store at {}: {err}", + worker_aggregate_dir.display() + ) + }, + )?; let store = CombinedStore::new(session_store, worker_metadata_store); let mut worker = Worker::from_manifest_with_context( @@ -642,10 +587,17 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } let workspace_client = worker.workspace_client_handle(); - let runtime_base = self.runtime_base_dir()?; - let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base) + let run_dir = worker_aggregate_dir + .join("runs") + .join(request.run_generation.to_string()); + let (handle, shutdown_rx) = WorkerController::spawn_runtime_managed_run(worker, &run_dir) .await - .map_err(|err| format!("failed to spawn Worker controller: {err}"))?; + .map_err(|err| { + format!( + "failed to spawn Worker controller in {}: {err}", + run_dir.display() + ) + })?; if flow_transition_enabled { handle.shared_state.enable_flow_transition(); } @@ -656,6 +608,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { ); Ok(RuntimeWorkerController { handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), workspace_client, }) } @@ -692,26 +645,29 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { let workspace_context = workspace_backend_ref.worker_context(&request.worker_ref); let (manifest, loader) = Self::restore_fallback_manifest(&worker_name)?; - let store_dir = self.store_dir()?; - let session_store = FsStore::new(&store_dir).map_err(|err| { + let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; + let session_dir = worker_aggregate_dir.join("session"); + let session_store = WorkerSessionStore::new(&session_dir).map_err(|err| { format!( - "failed to initialize session store at {}: {err}", - store_dir.display() - ) - })?; - let worker_metadata_dir = self.worker_metadata_dir(&store_dir); - let worker_metadata_store = FsWorkerStore::new(&worker_metadata_dir).map_err(|err| { - format!( - "failed to initialize worker metadata store at {}: {err}", - worker_metadata_dir.display() + "failed to initialize canonical Worker Session store at {}: {err}", + session_dir.display() ) })?; + let worker_metadata_store = + WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()).map_err( + |err| { + format!( + "failed to initialize canonical Worker metadata store at {}: {err}", + worker_aggregate_dir.display() + ) + }, + )?; let store = CombinedStore::new(session_store, worker_metadata_store); let mut worker = match Worker::restore_from_worker_metadata_with_context( &worker_name, manifest.clone(), - store, + store.clone(), loader.clone(), workspace_context.clone(), filesystem_authority.clone(), @@ -722,20 +678,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { Err(WorkerError::WorkerMetadataPending { .. }) if request.request.initial_input.is_none() => { - let session_store = FsStore::new(&store_dir).map_err(|err| { - format!( - "failed to initialize session store at {}: {err}", - store_dir.display() - ) - })?; - let worker_metadata_store = - FsWorkerStore::new(&worker_metadata_dir).map_err(|err| { - format!( - "failed to initialize worker metadata store at {}: {err}", - worker_metadata_dir.display() - ) - })?; - let store = CombinedStore::new(session_store, worker_metadata_store); Worker::restore_pending_from_worker_metadata_with_context( &worker_name, manifest.clone(), @@ -792,10 +734,17 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } let workspace_client = worker.workspace_client_handle(); - let runtime_base = self.runtime_base_dir()?; - let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base) + let run_dir = worker_aggregate_dir + .join("runs") + .join(request.run_generation.to_string()); + let (handle, shutdown_rx) = WorkerController::spawn_runtime_managed_run(worker, &run_dir) .await - .map_err(|err| format!("failed to spawn restored Worker controller: {err}"))?; + .map_err(|err| { + format!( + "failed to spawn restored Worker controller in {}: {err}", + run_dir.display() + ) + })?; if flow_transition_enabled { handle.shared_state.enable_flow_transition(); } @@ -806,6 +755,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { ); Ok(RuntimeWorkerController { handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), workspace_client, }) } @@ -813,6 +763,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { struct RuntimeWorkerExecution { handle: WorkerHandle, + shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>, busy: Arc<AtomicBool>, workspace_client: Option<Arc<dyn WorkspaceClient>>, } @@ -828,7 +779,10 @@ pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> { impl WorkerRuntimeExecutionBackend<ProfileRuntimeWorkerFactory> { pub fn from_workspace(workspace_root: impl Into<PathBuf>) -> Result<Self, String> { - Self::new(ProfileRuntimeWorkerFactory::new(workspace_root)) + let workspace_root = workspace_root.into(); + let factory = ProfileRuntimeWorkerFactory::new(&workspace_root) + .with_runtime_store_dir(workspace_root.join(".yoi/runtime-store")); + Self::new(factory) } } @@ -1094,6 +1048,7 @@ where worker_ref: crate::identity::WorkerRef, bridge_context: crate::execution::WorkerExecutionContext, handle: WorkerHandle, + shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>, working_directory: Option<WorkingDirectoryBinding>, workspace_client: Option<Arc<dyn WorkspaceClient>>, ) -> WorkerExecutionSpawnResult { @@ -1157,6 +1112,7 @@ where worker_ref.clone(), RuntimeWorkerExecution { handle, + shutdown, busy, workspace_client, }, @@ -1393,6 +1349,7 @@ where worker_ref, bridge_context, controller.handle, + controller.shutdown, working_directory, Some(controller.workspace_client), ) @@ -1489,6 +1446,7 @@ where worker_ref, bridge_context, controller.handle, + controller.shutdown, working_directory, Some(controller.workspace_client), ) @@ -1698,12 +1656,28 @@ where "execution handle does not reference a live Worker", ); }; - self.send_method( + let shutdown = execution.shutdown.clone(); + let result = self.send_method( WorkerExecutionOperation::Stop, execution.handle, Method::Shutdown, WorkerExecutionRunState::Stopped, - ) + ); + if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { + return result; + } + match self.run_on_adapter_runtime(async move { + let receiver = shutdown.lock().await.take(); + if let Some(receiver) = receiver { + receiver + .await + .map_err(|_| "Worker shutdown completion channel closed".to_string())?; + } + Ok(()) + }) { + Ok(()) => result, + Err(message) => WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message), + } } fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { @@ -1924,12 +1898,13 @@ mod tests { ) .await .map_err(|err| err.to_string())?; - let (handle, _shutdown_rx) = + let (handle, shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &self.runtime_base) .await .map_err(|err| err.to_string())?; Ok(RuntimeWorkerController { handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), workspace_client, }) } @@ -1939,6 +1914,7 @@ mod tests { ) -> Result<RuntimeWorkerController, String> { let request = WorkerExecutionSpawnRequest { worker_ref: request.worker_ref, + run_generation: request.run_generation, request: request.request, context: request.context, working_directory: request.working_directory, @@ -2210,6 +2186,7 @@ mod tests { let worker_ref = crate::identity::WorkerRef::new(crate::identity::WorkerId::new(1)); let request = WorkerExecutionSpawnRequest { worker_ref: worker_ref.clone(), + run_generation: 1, request: create_request("1"), context: test_execution_context(worker_ref), working_directory: None, @@ -2263,9 +2240,9 @@ mod tests { #[tokio::test] async fn restore_pending_worker_uses_saved_manifest_snapshot() { let root = tempfile::tempdir().unwrap(); - let store_dir = root.path().join("sessions"); - let worker_metadata_dir = root.path().join("workers"); + let runtime_store_dir = root.path().join("runtime"); let worker_ref = WorkerRef::new(crate::identity::WorkerId::new(1)); + let worker_aggregate_dir = runtime_store_dir.join("workers/1"); let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref); let session_id = session_store::new_session_id(); let manifest = manifest::WorkerManifest::from_toml(&format!( @@ -2294,7 +2271,7 @@ mod tests { root.path().display(), )) .unwrap(); - FsWorkerStore::new(&worker_metadata_dir) + WorkerAggregateStore::new(&worker_aggregate_dir, &worker_name) .unwrap() .set_active( &worker_name, @@ -2312,10 +2289,10 @@ mod tests { runtime_id: Some("runtime-restore".to_string()), }); let controller = ProfileRuntimeWorkerFactory::new(root.path()) - .with_store_dir(&store_dir) - .with_worker_metadata_dir(&worker_metadata_dir) + .with_runtime_store_dir(&runtime_store_dir) .restore_controller(WorkerExecutionRestoreRequest { worker_ref: worker_ref.clone(), + run_generation: 1, request, context: test_execution_context(worker_ref), previous_working_directory: None, @@ -2325,8 +2302,23 @@ mod tests { .await .expect("pending restore should use the saved manifest snapshot"); assert!(controller.handle.shared_state.flow_transition_enabled()); + let run_dir = runtime_store_dir.join("workers/1/runs/1"); + assert!(run_dir.join("worker.sock").exists()); + assert!(run_dir.join("worker.out.log").is_file()); + assert!(run_dir.join("worker.err.log").is_file()); + assert!(run_dir.join("artifacts").is_dir()); + assert!(run_dir.join("spawned").is_dir()); + let shutdown = controller.shutdown.clone(); controller.handle.send(Method::Shutdown).await.unwrap(); + if let Some(receiver) = shutdown.lock().await.take() { + receiver.await.unwrap(); + } + assert!( + run_dir.is_dir(), + "run evidence remains until a separate retention policy disposes it" + ); + assert!(!run_dir.join("worker.sock").exists()); } #[test] diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 6dfdd42c..f72568ac 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -211,7 +211,7 @@ impl WorkerController { C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { - Self::spawn_inner(worker, runtime_base, false).await + Self::spawn_inner(worker, runtime_base, false, None).await } /// Spawn a Worker owned by `worker-runtime`. @@ -227,20 +227,37 @@ impl WorkerController { C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { - Self::spawn_inner(worker, runtime_base, true).await + Self::spawn_inner(worker, runtime_base, true, None).await + } + + /// Spawn into an exact persistent `runs/<generation>` directory. + pub async fn spawn_runtime_managed_run<C, St>( + worker: Worker<C, St>, + run_dir: &Path, + ) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error> + where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + let parent = run_dir + .parent() + .ok_or_else(|| std::io::Error::other("run path has no parent"))?; + Self::spawn_inner(worker, parent, true, Some(run_dir)).await } async fn spawn_inner<C, St>( worker: Worker<C, St>, runtime_base: &Path, runtime_managed: bool, + runtime_run: Option<&Path>, ) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error> where C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { let session = worker.workdir_session().cloned(); - let result = Self::spawn_initialized(worker, runtime_base, runtime_managed).await; + let result = + Self::spawn_initialized(worker, runtime_base, runtime_managed, runtime_run).await; if result.is_err() && let Some(session) = session && let Err(error) = session.close().await @@ -254,6 +271,7 @@ impl WorkerController { mut worker: Worker<C, St>, runtime_base: &Path, runtime_managed: bool, + runtime_run: Option<&Path>, ) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error> where C: LlmClient + Clone + 'static, @@ -273,7 +291,9 @@ impl WorkerController { // the spawn-tool factories need its socket path, and before the // initial status/history writes consume the greeting we build // after registration is complete. - let runtime_dir = Arc::new(if runtime_managed { + let runtime_dir = Arc::new(if let Some(run_dir) = runtime_run { + RuntimeDir::create_worker_run(run_dir).await? + } else if runtime_managed { RuntimeDir::create_transient(runtime_base, &worker.manifest().worker.name).await? } else { RuntimeDir::create(runtime_base, &worker.manifest().worker.name).await? @@ -1298,6 +1318,11 @@ async fn controller_loop<C, St>( } } + drop(_socket_server); + if let Err(error) = runtime_dir.close_socket().await { + tracing::warn!(%error, "Worker runtime socket cleanup failed"); + } + // Background memory jobs own extract/consolidate workers after a // turn completes. Join them before closing the Workdir session so no // Worker-owned task can outlive its operation attachment. diff --git a/crates/worker/src/runtime/dir.rs b/crates/worker/src/runtime/dir.rs index ed41d0b4..ad462f39 100644 --- a/crates/worker/src/runtime/dir.rs +++ b/crates/worker/src/runtime/dir.rs @@ -42,6 +42,8 @@ pub struct SpawnedWorkerRecord { pub struct RuntimeDir { path: PathBuf, write_legacy_snapshots: bool, + preserve_on_drop: bool, + socket_file_name: &'static str, } impl RuntimeDir { @@ -56,6 +58,8 @@ impl RuntimeDir { Ok(Self { path, write_legacy_snapshots: true, + preserve_on_drop: false, + socket_file_name: "sock", }) } @@ -69,6 +73,36 @@ impl RuntimeDir { Ok(Self { path, write_legacy_snapshots: false, + preserve_on_drop: false, + socket_file_name: "sock", + }) + } + + /// Create an exact, persistent generation-scoped Worker run directory. + /// Existing directories are rejected so stale artifacts cannot be reused. + pub async fn create_worker_run(path: &Path) -> Result<Self, io::Error> { + let parent = path + .parent() + .ok_or_else(|| io::Error::other("run path has no parent"))?; + fs::create_dir_all(parent).await?; + fs::create_dir(path).await?; + fs::create_dir(path.join("artifacts")).await?; + fs::create_dir(path.join("spawned")).await?; + for log in ["worker.out.log", "worker.err.log"] { + let file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path.join(log)) + .await?; + file.sync_all().await?; + } + std::fs::File::open(path)?.sync_all()?; + std::fs::File::open(parent)?.sync_all()?; + Ok(Self { + path: path.to_path_buf(), + write_legacy_snapshots: false, + preserve_on_drop: true, + socket_file_name: "worker.sock", }) } @@ -116,13 +150,24 @@ impl RuntimeDir { /// that only know the worker name (e.g. the TUI's attach flow) /// predict the same path via [`manifest::paths::worker_socket_path`]. pub fn socket_path(&self) -> PathBuf { - self.path.join("sock") + self.path.join(self.socket_file_name) + } + + pub async fn close_socket(&self) -> Result<(), io::Error> { + match fs::remove_file(self.socket_path()).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } } } impl Drop for RuntimeDir { fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.path); + let _ = std::fs::remove_file(self.socket_path()); + if !self.preserve_on_drop { + let _ = std::fs::remove_dir_all(&self.path); + } } } diff --git a/crates/worker/src/runtime/worker_allocation/lifecycle.rs b/crates/worker/src/runtime/worker_allocation/lifecycle.rs index e86f4b83..480d895a 100644 --- a/crates/worker/src/runtime/worker_allocation/lifecycle.rs +++ b/crates/worker/src/runtime/worker_allocation/lifecycle.rs @@ -120,15 +120,15 @@ pub fn adopt_allocation( /// The Worker's in-memory `segment_id` can change underneath the /// allocation in two normal places: /// -/// - `Worker::compact` mints a fresh session and swaps it in. -/// - `session_store::ensure_head_or_fork` auto-forks when another +/// - `Worker::compact` mints a fresh Segment in the same Session. +/// - `session_store::ensure_head_or_fork` auto-forks within that Session when another /// writer has advanced the store head behind our back. /// /// Both paths must call this so subsequent [`lookup_segment`] queries -/// find the live session id, not the old one. Without this update a +/// find the live Segment id, not the old one. Without this update a /// concurrent `restore_from_manifest(new_id)` would see "no live /// writer" and proceed to register a competing allocation on the -/// session this Worker just moved into. +/// Segment lineage this Worker just moved into. /// /// The lock is opened once and the allocation is rewritten inside the /// guard, so the segment_id collision check is atomic with the diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 26d857f7..d9fc0f7f 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -3067,13 +3067,13 @@ impl<C: LlmClient, St: Store> Worker<C, St> { /// Compact the current session by summarising history via a /// disposable Engine, then replacing history with - /// `[summary, ...recent_turns]` and creating a new session. + /// `[summary, ...recent_turns]` in a new Segment of the same Session. /// /// The summary Engine uses: /// - `compaction.model` from the manifest if configured, or /// - a clone of the main LlmClient via `clone_boxed()`. /// - /// Returns the new session ID. + /// Returns the new Segment ID. The Worker keeps its Session ID. pub async fn compact(&mut self, retained_tokens: u64) -> Result<SegmentId, WorkerError> { use crate::compact::worker::{ CompactWorkerContext, CompactWorkerInterceptor, add_reference_tool, diff --git a/crates/worker/tests/compact_events_test.rs b/crates/worker/tests/compact_events_test.rs index 76282bf2..cfbd2ba0 100644 --- a/crates/worker/tests/compact_events_test.rs +++ b/crates/worker/tests/compact_events_test.rs @@ -421,7 +421,11 @@ async fn pre_run_compact_success_broadcasts_start_and_done() { // Drain run events so only compact events remain in `rx`. let _ = drain(&mut rx); + let session_before = worker.session_id(); + let segment_before = worker.segment_id(); worker.try_pre_run_compact().await; + assert_eq!(worker.session_id(), session_before); + assert_ne!(worker.segment_id(), segment_before); let events = drain(&mut rx); let kinds: Vec<&str> = events @@ -442,7 +446,7 @@ async fn pre_run_compact_success_broadcasts_start_and_done() { "unexpected CompactFailed in {kinds:?}" ); - // CompactDone carries the new session id. + // CompactDone carries the new Segment ID; the Session ID is unchanged. let new_id_in_event = events.iter().find_map(|e| match e { Event::CompactDone { new_segment_id } => Some(*new_segment_id), _ => None, @@ -583,11 +587,11 @@ async fn compact_resets_extract_pointer_so_extract_can_fire_again() { ); // Compact runs. Without the fix the in-memory pointer would still - // reference the old session's history_len. + // reference the old Segment's history_len. worker.try_pre_run_compact().await; assert!( worker.extract_pointer().is_none(), - "extract_pointer must be reset to None after compact (matches cold-restore on the new session)" + "extract_pointer must be reset to None after compact (matches cold-restore on the new Segment)" ); } diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 9dfe907a..95ce0bf4 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -269,6 +269,7 @@ impl WorkspaceApi { let resource_broker = BackendResourceBroker::default(); let execution_backend = WorkerRuntimeExecutionBackend::new( ProfileRuntimeWorkerFactory::new(config.workspace_root.clone()) + .with_runtime_store_dir(config.embedded_runtime_store_root.clone()) .with_resource_client(Arc::new(resource_broker.clone())), ) .map_err(|err| { @@ -318,6 +319,23 @@ impl WorkspaceApi { .await?; import_configured_repositories(store.as_ref(), &config)?; config.repositories = load_configured_repositories_from_store(store.as_ref(), &config)?; + let default_embedded_root = + ServerConfig::default_embedded_runtime_store_root(&config.workspace_id); + if config.embedded_runtime_store_root == default_embedded_root + && let (Some(legacy_sessions), Some(data_dir)) = + (manifest::paths::sessions_dir(), manifest::paths::data_dir()) + { + worker_runtime::fs_store::FsRuntimeStore::migrate_legacy_worker_aggregates( + &config.embedded_runtime_store_root, + legacy_sessions, + data_dir.join("workers"), + ) + .map_err(|error| { + crate::Error::Store(format!( + "failed to migrate embedded Runtime Worker aggregates: {error}" + )) + })?; + } let embedded_runtime = EmbeddedWorkerRuntime::new_fs_store_with_execution_backend( config.workspace_id.clone(), config.embedded_runtime_store_root.clone(), diff --git a/docs/design/worker-session-state.md b/docs/design/worker-session-state.md index b92e0811..1a605ff6 100644 --- a/docs/design/worker-session-state.md +++ b/docs/design/worker-session-state.md @@ -1,45 +1,85 @@ -# Worker, session, and state authority +# Worker aggregate, Session, and run authority -Yoi separates replayable history from current Worker identity because they answer different questions. +A Runtime-managed Worker is the canonical durable aggregate. One Worker owns exactly one Session for its lifetime; compaction and forks create Segments inside that Session rather than replacing the Session identity. -A session log answers: "what happened and what can be replayed?" Worker metadata answers: "what does this Worker name currently refer to?" Live sockets and registries answer only: "what seems reachable right now?" +This identity rule separates durable conversation history from execution attempts: -## Session logs +- **Worker ID** identifies the Runtime catalog aggregate and remains stable across stop/restore. +- **Session ID** identifies that Worker's sole replayable history and remains stable across compaction. +- **Segment ID** identifies a branch or compacted history projection inside the Session. +- **run generation** identifies one process/controller execution attempt and increases before every spawn or restore. -Session JSONL is the durable replay record. It contains committed user inputs, assistant items, tool results, system/runtime events that must explain later behavior, segment boundaries, and persisted effective snapshots needed to understand a run. +## Canonical Runtime layout -The session log should be append-oriented and schema drift should be compile-visible. Compatibility shims that silently reinterpret old plural/current entries make future readers less safe. +Filesystem Runtime stores materialize one aggregate under `workers/<worker_id>/`: -Session logs do not own current Worker-name state. A historical session can be replayable without being the active session for a Worker name. +```text +workers/<worker_id>/ + worker.json + metadata.json + session/ + session.json + segments/ + <segment_id>.jsonl + <segment_id>.trace.jsonl + runs/ + <generation>/ + worker.sock + worker.out.log + worker.err.log + artifacts/ + spawned/ +``` -## Worker metadata +`worker.json` is Runtime catalog authority: Workspace attribution, create/restore request, execution binding, and the last durably reserved run generation. `metadata.json` is the current Worker projection used to restore active/pending Segment pointers, resolved manifest state, delegation metadata, and child/peer visibility. It is not a second transcript. -Worker metadata is the current-state layer keyed by Worker name. It records active/pending session pointers, resolved manifest snapshots, current delegation metadata, spawned-child visibility, and restoration information. +`session/session.json` fixes the single Session ID for the aggregate. The Worker-specific Session store rejects attempts to address another Session ID. Session JSONL is the append-oriented replay authority for committed user inputs, assistant items, tool results, system/runtime events, Segment lineage, and effective snapshots required to explain later behavior. -This avoids reconstructing current Worker state by scanning every session log. It also gives `--worker <name>`, TUI resume, `ListWorkers`, and `RestoreWorker` a single current authority. +The normal execution path resolves all three stores from the trusted `WorkerRef`. It does not fall back to process-global Session or Worker-metadata roots. Those roots are legacy migration inputs only. -Worker metadata should stay thin. It is not a second transcript, and it should not duplicate model conversation content. +## Segment lifecycle -## Live runtime hints +A new Worker materializes its Session when the initial Segment is created. The Session ID then remains stable. -Sockets, process registries, and runtime files are liveness hints. They are useful for attach, status probing, and fast discovery, but they are not final proof that work completed or that a Worker's state changed durably. +Compaction writes a new Segment in the same Session with `compacted_from` lineage. Forking likewise writes a sibling Segment in the same Session with `forked_from` lineage. Allocation, UI, and event surfaces therefore report the new **Segment ID**, never a "new Session". A different Session requires a different Worker aggregate. -A reachable pending Worker should be visible even if durable logs have not materialized yet. Missing restore labels should degrade labels and diagnostics, not hide a live attachable Worker. +This keeps existing Worker IDs, Session IDs, Segment references, observation entry references, and UI routes meaningful across compaction and restore. -## Spawned children and delegation +## Run lifecycle -Parent-visible children are sourced from Worker metadata, not from a transient runtime mirror. Restoring a parent should reconstruct reachable children where possible and keep stopped-but-restorable children visible when metadata supports it. +Run generations are monotonic per Worker. Runtime durably reserves and persists the next generation before invoking the execution backend: -Delegated write scope is a capability loan. Stopping, shutting down, or pruning a child must reclaim the parent's effective write permissions while preserving explicit base denies. +- initial spawn reserves generation `1`; +- explicit restore reserves the next generation; +- startup restoration after a process crash reserves the next generation before reconnecting providers or observation state. -## Peer Workers +A generation directory is created with `create_new` semantics. An existing directory is a collision and is never reused as a new execution. This makes a crash between reservation and controller startup recoverable: startup consumes another generation instead of treating stale socket/log state as live authority. -Peer visibility is also Worker metadata, but it is distinct from spawned-child delegation. A TUI user can run `:peer <worker-name>` while attached to an idle Worker to register reciprocal peer metadata with another existing Worker. This is a metadata-level registration, not live target-controller consent. +Stopping a Worker waits for controller shutdown completion and removes `worker.sock`. The generation directory and diagnostic files remain evidence. How old generations are retained, archived, or purged is a separate policy; aggregate creation and migration do not invent that disposition. -A peer relationship only makes the Workers mutually visible through `ListWorkers` with visibility source `peer`. It does not grant filesystem scope, create a child output cursor, make either Worker the other's parent, or imply child completion notifications. Peer messages use `SendToPeerWorker`, which delivers a labeled notification into the target Worker's normal durable notification/history path. `SendToPeerWorker` requires the peer to be live and fails clearly for non-live peers rather than auto-restoring them. +Live sockets and provider sessions are execution hints, not durable identity. Restore reconstructs Workspace client attribution, observation registration, and Workdir/provider bindings from Runtime/Backend authority while keeping the Worker and Session identities unchanged. -## Notifications are not authority +## Legacy migration -Worker completion notifications are UX hints. Before treating delegated work as complete, inspect queryable evidence: child output, session/log state, worktree status, diffs, and validation output. +Startup migration recognizes the versioned `worker-aggregate-v1` format and treats legacy process-global metadata/Session directories as read-only sources. -This is why orchestration code should expose state-aware operations such as `ListWorkers` and `RestoreWorker`, rather than letting a background alert decide workflow state by itself. +The migration is serialized by an OS file lock and writes a versioned manifest with per-Worker checkpoints and diagnostics. For each unambiguous catalog Worker it: + +1. validates Worker metadata identity and the referenced Session; +2. parses Segment and trace JSONL, tolerating only the existing crash-truncated final-line rule; +3. stages `session.json` plus all Segment files under the target Worker aggregate; +4. fsyncs files and directories, atomically renames the staged Session directory, and fsyncs its parent; +5. atomically writes and fsyncs `metadata.json`; +6. atomically updates the migration checkpoint. + +Reruns validate exact Session identity, the complete source/target filename set, and file bytes before accepting an existing target. Mixed old/new stores and a crash between Session rename, metadata copy, and checkpoint update therefore converge without replacing divergent data. + +Migration fails closed on target collisions, corrupt complete JSONL records, or ambiguous ownership. If multiple legacy metadata sources reference one Session—including metadata with no catalog Worker—the Session is not assigned to either aggregate. The manifest records the shared reference. Legacy metadata or Session directories with no catalog-backed owner are also recorded as orphans and left untouched; they do not become normal authority and are not silently deleted. + +Legacy sources remain available for audit and recovery after a successful copy. Archive/retention disposition and Orchestrator-driven Worker removal are intentionally outside this migration contract. + +## Child, peer, and notification state + +Parent-visible children and peer registrations are current Worker metadata, distinct from Session history and run liveness. Restoring a parent reconstructs reachable children where possible and retains stopped-but-restorable visibility when metadata supports it. Delegated write scope is a capability loan; stopping or pruning a child must reclaim the parent's effective permissions. + +Peer registration does not grant filesystem authority, imply parent ownership, or make notifications completion proof. Notifications remain UX hints committed through the normal Worker history path. Completion decisions must reread durable Ticket, repository, review, and test evidence rather than relying on a socket event or final assistant message. From da8313fa1a6a14c34e009fc8501ba067d657cd41 Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Tue, 11 Aug 2026 23:57:37 +0900 Subject: [PATCH 05/12] runtime: add revisioned worker retention authority --- crates/worker-runtime/src/lib.rs | 2 + crates/worker-runtime/src/retention.rs | 1184 +++++++++++++++++++++ crates/worker-runtime/src/runtime.rs | 135 +++ crates/workspace-server/src/hosts.rs | 3 + crates/workspace-server/src/lib.rs | 1 + crates/workspace-server/src/retention.rs | 1194 ++++++++++++++++++++++ crates/workspace-server/src/store.rs | 92 +- 7 files changed, 2599 insertions(+), 12 deletions(-) create mode 100644 crates/worker-runtime/src/retention.rs create mode 100644 crates/workspace-server/src/retention.rs diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index 43d226bf..001a102a 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -22,6 +22,8 @@ pub mod management; pub mod observation; pub mod profile_archive; pub mod resource; +#[cfg(feature = "fs-store")] +pub mod retention; mod runtime; pub mod worker_backend; pub mod working_directory; diff --git a/crates/worker-runtime/src/retention.rs b/crates/worker-runtime/src/retention.rs new file mode 100644 index 00000000..e77a8372 --- /dev/null +++ b/crates/worker-runtime/src/retention.rs @@ -0,0 +1,1184 @@ +//! Runtime-owned execution of Backend-resolved Worker retention dispositions. +//! +//! This boundary deliberately accepts only stable ids and resolved dispositions. +//! Host paths and provider handles never cross it. + +use crate::error::RuntimeError; +use crate::identity::WorkerId; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +const ARCHIVE_SCHEMA_VERSION: u32 = 1; +const OPERATION_SCHEMA_VERSION: u32 = 1; +const RETENTION_LOCK: &str = ".worker-retention.lock"; +static NEXT_TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(1); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SessionDisposition { + Archive, + Purge, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DiagnosticsDisposition { + Purge, + /// The Backend catalog owns the expiry. Runtime keeps only bounded stdout/stderr + /// evidence and never mixes it into the Session archive. + Retain, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerRetentionInventory { + pub workspace_id: String, + pub runtime_id: String, + pub worker_id: WorkerId, + pub run_generation: u64, + pub session_id: Option<String>, + pub segment_ids: Vec<String>, + pub session_bytes: u64, + pub diagnostics_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerRetentionExecutionRequest { + pub operation_id: String, + pub input_fingerprint: String, + pub archive_id: Option<String>, + pub workspace_id: String, + pub source_runtime_id: String, + pub worker_id: WorkerId, + pub expected_run_generation: u64, + pub source_created_at: String, + pub removed_at: String, + pub effective_profile: Option<String>, + pub retention_class: Option<String>, + pub policy_id: String, + pub policy_revision: u64, + pub session_disposition: SessionDisposition, + pub diagnostics_disposition: DiagnosticsDisposition, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerSessionArchiveManifest { + pub schema_version: u32, + pub archive_id: String, + pub workspace_id: String, + pub source_runtime_id: String, + pub source_worker_id: WorkerId, + pub source_session_id: String, + pub segment_ids: Vec<String>, + pub source_created_at: String, + pub removed_at: String, + pub archived_at_unix_seconds: u64, + pub effective_profile: Option<String>, + pub retention_class: Option<String>, + pub content_checksum_sha256: String, + pub content_bytes: u64, + pub content_file_count: u64, + pub policy_id: String, + pub policy_revision: u64, + pub operation_id: String, + pub input_fingerprint: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerRetentionExecutionResult { + pub operation_id: String, + pub input_fingerprint: String, + pub worker_id: WorkerId, + pub session_disposition: SessionDisposition, + pub diagnostics_disposition: DiagnosticsDisposition, + pub archive: Option<WorkerSessionArchiveManifest>, + pub source_removed: bool, + pub diagnostics_retained: bool, +} + +pub(crate) trait WorkerRetentionProvider: Send + Sync { + fn inventory( + &self, + workspace_id: &str, + runtime_id: &str, + worker_id: WorkerId, + run_generation: u64, + ) -> Result<WorkerRetentionInventory, RuntimeError>; + + fn execute( + &self, + request: &WorkerRetentionExecutionRequest, + ) -> Result<WorkerRetentionExecutionResult, RuntimeError>; + + fn completed( + &self, + operation_id: &str, + input_fingerprint: &str, + ) -> Result<Option<WorkerRetentionExecutionResult>, RuntimeError>; +} + +/// Filesystem provider for the canonical Runtime Worker aggregate. +#[derive(Clone, Debug)] +pub(crate) struct FsWorkerRetentionProvider { + runtime_root: PathBuf, +} + +impl FsWorkerRetentionProvider { + pub(crate) fn new(runtime_root: impl Into<PathBuf>) -> Self { + Self { + runtime_root: runtime_root.into(), + } + } + + pub(crate) fn recover_after_source_removal( + &self, + request: &WorkerRetentionExecutionRequest, + ) -> Result<WorkerRetentionExecutionResult, RuntimeError> { + if self.worker_dir(request.worker_id).exists() { + return Err(RuntimeError::WorkerNotFound { + worker_id: request.worker_id, + }); + } + self.execute(request) + } + + fn worker_dir(&self, worker_id: WorkerId) -> PathBuf { + self.runtime_root + .join("workers") + .join(worker_id.to_string()) + } + + fn operation_path(&self, operation_id: &str) -> Result<PathBuf, RuntimeError> { + validate_id("operation_id", operation_id)?; + Ok(self + .runtime_root + .join("retention") + .join("operations") + .join(format!("{operation_id}.json"))) + } + + fn archive_dir(&self, archive_id: &str) -> Result<PathBuf, RuntimeError> { + validate_id("archive_id", archive_id)?; + Ok(self + .runtime_root + .join("archives") + .join("workers") + .join(archive_id)) + } + + fn diagnostics_dir(&self, operation_id: &str) -> Result<PathBuf, RuntimeError> { + validate_id("operation_id", operation_id)?; + Ok(self + .runtime_root + .join("archives") + .join("diagnostics") + .join(operation_id)) + } +} + +impl WorkerRetentionProvider for FsWorkerRetentionProvider { + fn inventory( + &self, + workspace_id: &str, + runtime_id: &str, + worker_id: WorkerId, + run_generation: u64, + ) -> Result<WorkerRetentionInventory, RuntimeError> { + let worker_dir = self.worker_dir(worker_id); + if !worker_dir.is_dir() { + return Err(RuntimeError::WorkerNotFound { worker_id }); + } + let session_dir = worker_dir.join("session"); + let (session_id, segment_ids, session_bytes) = if session_dir.is_dir() { + let manifest: CanonicalSessionManifest = read_json( + &session_dir.join("session.json"), + "inventory Worker retention", + )?; + let files = collect_files(&session_dir, "inventory Worker retention")?; + let mut segment_ids = BTreeSet::new(); + let mut bytes = 0_u64; + for (relative, path) in files { + bytes = bytes.saturating_add(file_len(&path, "inventory Worker retention")?); + if let Some(name) = relative.file_name().and_then(|value| value.to_str()) { + let segment = name + .strip_suffix(".trace.jsonl") + .or_else(|| name.strip_suffix(".jsonl")); + if let Some(segment) = segment { + segment_ids.insert(segment.to_string()); + } + } + } + ( + Some(manifest.session_id), + segment_ids.into_iter().collect(), + bytes, + ) + } else { + (None, Vec::new(), 0) + }; + let diagnostics_bytes = diagnostics_files(&worker_dir, "inventory Worker retention")? + .into_iter() + .try_fold(0_u64, |total, path| { + file_len(&path, "inventory Worker retention").map(|size| total.saturating_add(size)) + })?; + Ok(WorkerRetentionInventory { + workspace_id: workspace_id.to_string(), + runtime_id: runtime_id.to_string(), + worker_id, + run_generation, + session_id, + segment_ids, + session_bytes, + diagnostics_bytes, + }) + } + + fn execute( + &self, + request: &WorkerRetentionExecutionRequest, + ) -> Result<WorkerRetentionExecutionResult, RuntimeError> { + validate_request(request)?; + fs::create_dir_all(&self.runtime_root).map_err(|source| RuntimeError::StoreIo { + operation: "prepare Worker retention", + path: self.runtime_root.clone(), + source, + })?; + let lock_path = self.runtime_root.join(RETENTION_LOCK); + let lock = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(&lock_path) + .map_err(|source| RuntimeError::StoreIo { + operation: "lock Worker retention", + path: lock_path.clone(), + source, + })?; + lock.lock().map_err(|source| RuntimeError::StoreIo { + operation: "lock Worker retention", + path: lock_path, + source, + })?; + + let pending = read_operation_receipt( + &self.operation_path(&request.operation_id)?, + &request.input_fingerprint, + )?; + if let Some(receipt) = &pending { + if receipt.result.source_removed { + return Ok(receipt.result.clone()); + } + } + + let worker_dir = self.worker_dir(request.worker_id); + if !worker_dir.is_dir() { + if let Some(mut receipt) = pending { + // A prior attempt durably committed all disposition evidence and + // removed the aggregate, then stopped before finalizing its receipt. + receipt.result.source_removed = true; + atomic_write_json( + &self.operation_path(&request.operation_id)?, + &receipt, + "recover Worker retention receipt", + )?; + return Ok(receipt.result); + } + return Err(RuntimeError::WorkerNotFound { + worker_id: request.worker_id, + }); + } + let snapshot: WorkerGenerationSnapshot = + read_json(&worker_dir.join("worker.json"), "execute Worker retention")?; + if snapshot.run_generation != request.expected_run_generation { + return Err(RuntimeError::InvalidRequest(format!( + "Worker retention plan expected generation {}, current generation is {}", + request.expected_run_generation, snapshot.run_generation + ))); + } + + let archive = match request.session_disposition { + SessionDisposition::Archive => { + Some(commit_session_archive(self, request, &worker_dir)?) + } + SessionDisposition::Purge => None, + }; + let diagnostics_retained = match request.diagnostics_disposition { + DiagnosticsDisposition::Purge => false, + DiagnosticsDisposition::Retain => { + commit_diagnostics_archive(self, request, &worker_dir)?; + true + } + }; + + let mut result = WorkerRetentionExecutionResult { + operation_id: request.operation_id.clone(), + input_fingerprint: request.input_fingerprint.clone(), + worker_id: request.worker_id, + session_disposition: request.session_disposition, + diagnostics_disposition: request.diagnostics_disposition, + archive, + source_removed: false, + diagnostics_retained, + }; + let mut receipt = RetentionOperationReceipt { + schema_version: OPERATION_SCHEMA_VERSION, + result: result.clone(), + }; + // Pending receipt makes the delete/final-receipt crash window + // recoverable without treating an uncommitted archive as completion. + atomic_write_json( + &self.operation_path(&request.operation_id)?, + &receipt, + "commit pending Worker retention receipt", + )?; + + fs::remove_dir_all(&worker_dir).map_err(|source| RuntimeError::StoreIo { + operation: "remove retained Worker aggregate", + path: worker_dir.clone(), + source, + })?; + sync_directory( + worker_dir.parent().unwrap_or(&self.runtime_root), + "remove retained Worker aggregate", + )?; + + result.source_removed = true; + receipt.result = result.clone(); + atomic_write_json( + &self.operation_path(&request.operation_id)?, + &receipt, + "commit Worker retention receipt", + )?; + Ok(result) + } + + fn completed( + &self, + operation_id: &str, + input_fingerprint: &str, + ) -> Result<Option<WorkerRetentionExecutionResult>, RuntimeError> { + let path = self.operation_path(operation_id)?; + let Some(receipt) = read_operation_receipt(&path, input_fingerprint)? else { + return Ok(None); + }; + Ok(receipt.result.source_removed.then_some(receipt.result)) + } +} + +#[derive(Deserialize)] +struct WorkerGenerationSnapshot { + #[serde(default)] + run_generation: u64, +} + +#[derive(Deserialize)] +struct CanonicalSessionManifest { + session_id: String, +} + +#[derive(Serialize, Deserialize)] +struct RetentionOperationReceipt { + schema_version: u32, + result: WorkerRetentionExecutionResult, +} + +fn read_operation_receipt( + path: &Path, + input_fingerprint: &str, +) -> Result<Option<RetentionOperationReceipt>, RuntimeError> { + if !path.is_file() { + return Ok(None); + } + let receipt: RetentionOperationReceipt = read_json(path, "read Worker retention receipt")?; + if receipt.schema_version != OPERATION_SCHEMA_VERSION { + return Err(RuntimeError::StoreCorrupt { + operation: "read Worker retention receipt", + path: path.to_path_buf(), + message: format!( + "unsupported operation receipt schema {}", + receipt.schema_version + ), + }); + } + if receipt.result.input_fingerprint != input_fingerprint { + return Err(RuntimeError::InvalidRequest(format!( + "retention operation {} was already used with different input", + receipt.result.operation_id + ))); + } + Ok(Some(receipt)) +} + +#[derive(Serialize, Deserialize)] +struct DiagnosticsArchiveManifest { + schema_version: u32, + operation_id: String, + workspace_id: String, + source_runtime_id: String, + source_worker_id: WorkerId, + input_fingerprint: String, + content_checksum_sha256: String, + content_bytes: u64, + content_file_count: u64, +} + +fn validate_request(request: &WorkerRetentionExecutionRequest) -> Result<(), RuntimeError> { + validate_id("operation_id", &request.operation_id)?; + validate_id("workspace_id", &request.workspace_id)?; + validate_id("source_runtime_id", &request.source_runtime_id)?; + validate_id("policy_id", &request.policy_id)?; + if request.input_fingerprint.trim().is_empty() { + return Err(RuntimeError::InvalidRequest( + "retention input fingerprint must not be empty".to_string(), + )); + } + match (request.session_disposition, request.archive_id.as_deref()) { + (SessionDisposition::Archive, Some(id)) => validate_id("archive_id", id), + (SessionDisposition::Archive, None) => Err(RuntimeError::InvalidRequest( + "archive disposition requires archive_id".to_string(), + )), + (SessionDisposition::Purge, None) => Ok(()), + (SessionDisposition::Purge, Some(_)) => Err(RuntimeError::InvalidRequest( + "purge disposition must not include archive_id".to_string(), + )), + } +} + +fn validate_id(kind: &str, value: &str) -> Result<(), RuntimeError> { + if value.is_empty() + || value.len() > 160 + || value == "." + || value == ".." + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':')) + { + return Err(RuntimeError::InvalidRequest(format!( + "invalid retention {kind}" + ))); + } + Ok(()) +} + +fn commit_session_archive( + provider: &FsWorkerRetentionProvider, + request: &WorkerRetentionExecutionRequest, + worker_dir: &Path, +) -> Result<WorkerSessionArchiveManifest, RuntimeError> { + let archive_id = request.archive_id.as_deref().ok_or_else(|| { + RuntimeError::InvalidRequest("archive disposition requires archive_id".to_string()) + })?; + let session_dir = worker_dir.join("session"); + if !session_dir.is_dir() { + return Err(RuntimeError::StoreMissing { + operation: "archive Worker Session", + path: session_dir, + }); + } + let session: CanonicalSessionManifest = + read_json(&session_dir.join("session.json"), "archive Worker Session")?; + let source_files = collect_files(&session_dir, "archive Worker Session")?; + let (checksum, bytes, count) = checksum_files(&source_files, "archive Worker Session")?; + let mut segment_ids = BTreeSet::new(); + for (relative, _) in &source_files { + if let Some(name) = relative.file_name().and_then(|name| name.to_str()) { + if let Some(segment) = name + .strip_suffix(".trace.jsonl") + .or_else(|| name.strip_suffix(".jsonl")) + { + segment_ids.insert(segment.to_string()); + } + } + } + let manifest = WorkerSessionArchiveManifest { + schema_version: ARCHIVE_SCHEMA_VERSION, + archive_id: archive_id.to_string(), + workspace_id: request.workspace_id.clone(), + source_runtime_id: request.source_runtime_id.clone(), + source_worker_id: request.worker_id, + source_session_id: session.session_id, + segment_ids: segment_ids.into_iter().collect(), + source_created_at: request.source_created_at.clone(), + removed_at: request.removed_at.clone(), + archived_at_unix_seconds: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + effective_profile: request.effective_profile.clone(), + retention_class: request.retention_class.clone(), + content_checksum_sha256: checksum, + content_bytes: bytes, + content_file_count: count, + policy_id: request.policy_id.clone(), + policy_revision: request.policy_revision, + operation_id: request.operation_id.clone(), + input_fingerprint: request.input_fingerprint.clone(), + }; + let archive_dir = provider.archive_dir(archive_id)?; + if archive_dir.exists() { + return validate_existing_archive(&archive_dir, &manifest); + } + let parent = archive_dir + .parent() + .ok_or_else(|| RuntimeError::StoreCorrupt { + operation: "archive Worker Session", + path: archive_dir.clone(), + message: "archive target has no parent".to_string(), + })?; + fs::create_dir_all(parent).map_err(|source| RuntimeError::StoreIo { + operation: "archive Worker Session", + path: parent.to_path_buf(), + source, + })?; + let staging = temporary_path(&archive_dir); + let result = (|| { + fs::create_dir(&staging).map_err(|source| RuntimeError::StoreIo { + operation: "archive Worker Session", + path: staging.clone(), + source, + })?; + let target_session = staging.join("session"); + fs::create_dir(&target_session).map_err(|source| RuntimeError::StoreIo { + operation: "archive Worker Session", + path: target_session.clone(), + source, + })?; + copy_files(&source_files, &target_session, "archive Worker Session")?; + atomic_write_json( + &staging.join("manifest.json"), + &manifest, + "archive Worker Session", + )?; + sync_tree(&staging, "archive Worker Session")?; + fs::rename(&staging, &archive_dir).map_err(|source| RuntimeError::StoreIo { + operation: "archive Worker Session", + path: archive_dir.clone(), + source, + })?; + sync_directory(parent, "archive Worker Session")?; + validate_existing_archive(&archive_dir, &manifest) + })(); + if result.is_err() { + let _ = fs::remove_dir_all(&staging); + } + result +} + +fn validate_existing_archive( + archive_dir: &Path, + expected: &WorkerSessionArchiveManifest, +) -> Result<WorkerSessionArchiveManifest, RuntimeError> { + let existing: WorkerSessionArchiveManifest = read_json( + &archive_dir.join("manifest.json"), + "verify Worker Session archive", + )?; + let mut comparable_expected = expected.clone(); + comparable_expected.archived_at_unix_seconds = existing.archived_at_unix_seconds; + if existing != comparable_expected { + return Err(RuntimeError::StoreCorrupt { + operation: "verify Worker Session archive", + path: archive_dir.join("manifest.json"), + message: "archive id collision or manifest mismatch".to_string(), + }); + } + let files = collect_files( + &archive_dir.join("session"), + "verify Worker Session archive", + )?; + let (checksum, bytes, count) = checksum_files(&files, "verify Worker Session archive")?; + if checksum != existing.content_checksum_sha256 + || bytes != existing.content_bytes + || count != existing.content_file_count + { + return Err(RuntimeError::StoreCorrupt { + operation: "verify Worker Session archive", + path: archive_dir.to_path_buf(), + message: "archive checksum or content summary mismatch".to_string(), + }); + } + Ok(existing) +} + +fn commit_diagnostics_archive( + provider: &FsWorkerRetentionProvider, + request: &WorkerRetentionExecutionRequest, + worker_dir: &Path, +) -> Result<(), RuntimeError> { + let files = diagnostics_files(worker_dir, "archive Worker diagnostics")?; + let target = provider.diagnostics_dir(&request.operation_id)?; + if target.exists() { + let manifest: DiagnosticsArchiveManifest = read_json( + &target.join("manifest.json"), + "verify Worker diagnostics archive", + )?; + if manifest.input_fingerprint != request.input_fingerprint { + return Err(RuntimeError::InvalidRequest(format!( + "diagnostics archive operation {} was reused with different input", + request.operation_id + ))); + } + return Ok(()); + } + let parent = target.parent().ok_or_else(|| RuntimeError::StoreCorrupt { + operation: "archive Worker diagnostics", + path: target.clone(), + message: "diagnostics archive target has no parent".to_string(), + })?; + fs::create_dir_all(parent).map_err(|source| RuntimeError::StoreIo { + operation: "archive Worker diagnostics", + path: parent.to_path_buf(), + source, + })?; + let staging = temporary_path(&target); + let result = (|| { + fs::create_dir(&staging).map_err(|source| RuntimeError::StoreIo { + operation: "archive Worker diagnostics", + path: staging.clone(), + source, + })?; + let source_files = files + .iter() + .map(|path| { + let relative = + path.strip_prefix(worker_dir) + .map_err(|_| RuntimeError::StoreCorrupt { + operation: "archive Worker diagnostics", + path: path.clone(), + message: "diagnostics path escaped Worker aggregate".to_string(), + })?; + Ok((relative.to_path_buf(), path.clone())) + }) + .collect::<Result<Vec<_>, RuntimeError>>()?; + copy_files(&source_files, &staging, "archive Worker diagnostics")?; + let (checksum, bytes, count) = checksum_files(&source_files, "archive Worker diagnostics")?; + let manifest = DiagnosticsArchiveManifest { + schema_version: ARCHIVE_SCHEMA_VERSION, + operation_id: request.operation_id.clone(), + workspace_id: request.workspace_id.clone(), + source_runtime_id: request.source_runtime_id.clone(), + source_worker_id: request.worker_id, + input_fingerprint: request.input_fingerprint.clone(), + content_checksum_sha256: checksum, + content_bytes: bytes, + content_file_count: count, + }; + atomic_write_json( + &staging.join("manifest.json"), + &manifest, + "archive Worker diagnostics", + )?; + sync_tree(&staging, "archive Worker diagnostics")?; + fs::rename(&staging, &target).map_err(|source| RuntimeError::StoreIo { + operation: "archive Worker diagnostics", + path: target.clone(), + source, + })?; + sync_directory(parent, "archive Worker diagnostics") + })(); + if result.is_err() { + let _ = fs::remove_dir_all(&staging); + } + result +} + +fn diagnostics_files( + worker_dir: &Path, + operation: &'static str, +) -> Result<Vec<PathBuf>, RuntimeError> { + let runs = worker_dir.join("runs"); + if !runs.is_dir() { + return Ok(Vec::new()); + } + let mut files = Vec::new(); + for (_, path) in collect_files(&runs, operation)? { + let name = path.file_name().and_then(|name| name.to_str()); + if matches!(name, Some("worker.out.log" | "worker.err.log")) { + files.push(path); + } + } + files.sort(); + Ok(files) +} + +fn collect_files( + root: &Path, + operation: &'static str, +) -> Result<Vec<(PathBuf, PathBuf)>, RuntimeError> { + fn visit( + root: &Path, + current: &Path, + operation: &'static str, + files: &mut Vec<(PathBuf, PathBuf)>, + ) -> Result<(), RuntimeError> { + let mut entries = fs::read_dir(current) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: current.to_path_buf(), + source, + })? + .collect::<Result<Vec<_>, _>>() + .map_err(|source| RuntimeError::StoreIo { + operation, + path: current.to_path_buf(), + source, + })?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let file_type = entry.file_type().map_err(|source| RuntimeError::StoreIo { + operation, + path: path.clone(), + source, + })?; + if file_type.is_symlink() { + return Err(RuntimeError::StoreCorrupt { + operation, + path, + message: "symlinks are not allowed in retained Worker evidence".to_string(), + }); + } + if file_type.is_dir() { + visit(root, &path, operation, files)?; + } else if file_type.is_file() { + let relative = path + .strip_prefix(root) + .map_err(|_| RuntimeError::StoreCorrupt { + operation, + path: path.clone(), + message: "retention source escaped its aggregate root".to_string(), + })? + .to_path_buf(); + files.push((relative, path)); + } else { + return Err(RuntimeError::StoreCorrupt { + operation, + path, + message: "unsupported retained Worker evidence entry".to_string(), + }); + } + } + Ok(()) + } + if !root.is_dir() { + return Ok(Vec::new()); + } + let mut files = Vec::new(); + visit(root, root, operation, &mut files)?; + files.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(files) +} + +fn checksum_files( + files: &[(PathBuf, PathBuf)], + operation: &'static str, +) -> Result<(String, u64, u64), RuntimeError> { + let mut hasher = Sha256::new(); + let mut total = 0_u64; + for (relative, path) in files { + let relative = relative.to_string_lossy(); + hasher.update((relative.len() as u64).to_be_bytes()); + hasher.update(relative.as_bytes()); + let mut file = File::open(path).map_err(|source| RuntimeError::StoreIo { + operation, + path: path.clone(), + source, + })?; + let length = file_len(path, operation)?; + hasher.update(length.to_be_bytes()); + let mut buffer = [0_u8; 8192]; + loop { + let read = file + .read(&mut buffer) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: path.clone(), + source, + })?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + total = total.saturating_add(length); + } + let digest = hasher.finalize(); + let checksum = digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(); + Ok((checksum, total, files.len() as u64)) +} + +fn copy_files( + files: &[(PathBuf, PathBuf)], + target_root: &Path, + operation: &'static str, +) -> Result<(), RuntimeError> { + for (relative, source_path) in files { + let target = target_root.join(relative); + let parent = target.parent().ok_or_else(|| RuntimeError::StoreCorrupt { + operation, + path: target.clone(), + message: "retention copy target has no parent".to_string(), + })?; + fs::create_dir_all(parent).map_err(|source| RuntimeError::StoreIo { + operation, + path: parent.to_path_buf(), + source, + })?; + let bytes = fs::read(source_path).map_err(|source| RuntimeError::StoreIo { + operation, + path: source_path.clone(), + source, + })?; + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&target) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: target.clone(), + source, + })?; + file.write_all(&bytes) + .and_then(|_| file.sync_all()) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: target, + source, + })?; + } + Ok(()) +} + +fn atomic_write_json<T: Serialize>( + path: &Path, + value: &T, + operation: &'static str, +) -> Result<(), RuntimeError> { + let parent = path.parent().ok_or_else(|| RuntimeError::StoreCorrupt { + operation, + path: path.to_path_buf(), + message: "retention record has no parent".to_string(), + })?; + fs::create_dir_all(parent).map_err(|source| RuntimeError::StoreIo { + operation, + path: parent.to_path_buf(), + source, + })?; + let temporary = temporary_path(path); + let result = (|| { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: temporary.clone(), + source, + })?; + serde_json::to_writer_pretty(&mut file, value).map_err(|source| { + RuntimeError::StoreCorrupt { + operation, + path: temporary.clone(), + message: source.to_string(), + } + })?; + file.write_all(b"\n") + .and_then(|_| file.sync_all()) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: temporary.clone(), + source, + })?; + drop(file); + fs::rename(&temporary, path).map_err(|source| RuntimeError::StoreIo { + operation, + path: path.to_path_buf(), + source, + })?; + sync_directory(parent, operation) + })(); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} + +fn read_json<T: for<'de> Deserialize<'de>>( + path: &Path, + operation: &'static str, +) -> Result<T, RuntimeError> { + let file = File::open(path).map_err(|source| match source.kind() { + std::io::ErrorKind::NotFound => RuntimeError::StoreMissing { + operation, + path: path.to_path_buf(), + }, + _ => RuntimeError::StoreIo { + operation, + path: path.to_path_buf(), + source, + }, + })?; + serde_json::from_reader(file).map_err(|source| RuntimeError::StoreCorrupt { + operation, + path: path.to_path_buf(), + message: source.to_string(), + }) +} + +fn sync_tree(path: &Path, operation: &'static str) -> Result<(), RuntimeError> { + let mut directories = vec![path.to_path_buf()]; + let mut index = 0; + while index < directories.len() { + let current = directories[index].clone(); + index += 1; + for entry in fs::read_dir(¤t).map_err(|source| RuntimeError::StoreIo { + operation, + path: current.clone(), + source, + })? { + let entry = entry.map_err(|source| RuntimeError::StoreIo { + operation, + path: current.clone(), + source, + })?; + if entry.path().is_dir() { + directories.push(entry.path()); + } + } + } + for directory in directories.into_iter().rev() { + sync_directory(&directory, operation)?; + } + Ok(()) +} + +fn sync_directory(path: &Path, operation: &'static str) -> Result<(), RuntimeError> { + File::open(path) + .and_then(|file| file.sync_all()) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: path.to_path_buf(), + source, + }) +} + +fn file_len(path: &Path, operation: &'static str) -> Result<u64, RuntimeError> { + fs::metadata(path) + .map(|metadata| metadata.len()) + .map_err(|source| RuntimeError::StoreIo { + operation, + path: path.to_path_buf(), + source, + }) +} + +fn temporary_path(path: &Path) -> PathBuf { + let sequence = NEXT_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("retention"); + path.with_file_name(format!(".{name}.tmp-{}-{sequence}", std::process::id())) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Barrier}; + + fn write_json(path: &Path, value: &impl Serialize) { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, serde_json::to_vec_pretty(value).unwrap()).unwrap(); + } + + fn source(root: &Path, worker_id: WorkerId, generation: u64) { + let worker = root.join("workers").join(worker_id.to_string()); + write_json( + &worker.join("worker.json"), + &serde_json::json!({"run_generation": generation}), + ); + write_json( + &worker.join("session/session.json"), + &serde_json::json!({"schema_version": 1, "session_id": "session-a"}), + ); + fs::create_dir_all(worker.join("session/segments")).unwrap(); + fs::write(worker.join("session/segments/segment-a.jsonl"), b"one\n").unwrap(); + fs::create_dir_all(worker.join(format!("runs/{generation}"))).unwrap(); + fs::write( + worker.join(format!("runs/{generation}/worker.out.log")), + b"diagnostic\n", + ) + .unwrap(); + fs::write( + worker.join(format!("runs/{generation}/worker.sock")), + b"not retained", + ) + .unwrap(); + } + + fn request( + worker_id: WorkerId, + generation: u64, + disposition: SessionDisposition, + ) -> WorkerRetentionExecutionRequest { + WorkerRetentionExecutionRequest { + operation_id: "operation-a".to_string(), + input_fingerprint: "fingerprint-a".to_string(), + archive_id: (disposition == SessionDisposition::Archive) + .then(|| "archive-a".to_string()), + workspace_id: "workspace-a".to_string(), + source_runtime_id: "runtime-a".to_string(), + worker_id, + expected_run_generation: generation, + source_created_at: "2026-01-01T00:00:00Z".to_string(), + removed_at: "2026-01-02T00:00:00Z".to_string(), + effective_profile: Some("builtin:coder".to_string()), + retention_class: None, + policy_id: "policy-a".to_string(), + policy_revision: 3, + session_disposition: disposition, + diagnostics_disposition: DiagnosticsDisposition::Purge, + } + } + + #[test] + fn archive_is_verified_before_source_removal_and_retry_converges() { + let temp = tempfile::tempdir().unwrap(); + let worker_id = WorkerId::new(7); + source(temp.path(), worker_id, 4); + let provider = FsWorkerRetentionProvider::new(temp.path()); + let request = request(worker_id, 4, SessionDisposition::Archive); + + let first = provider.execute(&request).unwrap(); + assert!(first.source_removed); + let archive = first.archive.as_ref().unwrap(); + assert_eq!(archive.source_session_id, "session-a"); + assert_eq!(archive.segment_ids, vec!["segment-a"]); + assert!(!temp.path().join("workers/7").exists()); + assert!( + temp.path() + .join("archives/workers/archive-a/session/segments/segment-a.jsonl") + .is_file() + ); + assert!(!temp.path().join("archives/workers/archive-a/runs").exists()); + + let retry = provider.execute(&request).unwrap(); + assert_eq!(retry, first); + assert_eq!( + fs::read_dir(temp.path().join("archives/workers")) + .unwrap() + .count(), + 1 + ); + } + + #[test] + fn archive_failure_keeps_live_source_for_retry() { + let temp = tempfile::tempdir().unwrap(); + let worker_id = WorkerId::new(8); + source(temp.path(), worker_id, 2); + let collision = temp.path().join("archives/workers/archive-a"); + fs::create_dir_all(&collision).unwrap(); + fs::write(collision.join("manifest.json"), b"not-json").unwrap(); + let provider = FsWorkerRetentionProvider::new(temp.path()); + + assert!( + provider + .execute(&request(worker_id, 2, SessionDisposition::Archive)) + .is_err() + ); + assert!(temp.path().join("workers/8/session").is_dir()); + assert!( + !temp + .path() + .join("retention/operations/operation-a.json") + .exists() + ); + } + + #[test] + fn purge_removes_aggregate_and_rejects_stale_generation() { + let temp = tempfile::tempdir().unwrap(); + let provider = FsWorkerRetentionProvider::new(temp.path()); + let worker_id = WorkerId::new(9); + source(temp.path(), worker_id, 5); + let stale = request(worker_id, 4, SessionDisposition::Purge); + assert!(provider.execute(&stale).is_err()); + assert!(temp.path().join("workers/9/session").is_dir()); + + let mut current = request(worker_id, 5, SessionDisposition::Purge); + current.operation_id = "operation-current".to_string(); + current.input_fingerprint = "fingerprint-current".to_string(); + let result = provider.execute(¤t).unwrap(); + assert!(result.archive.is_none()); + assert!(!temp.path().join("workers/9").exists()); + assert!( + temp.path() + .join("retention/operations/operation-current.json") + .is_file() + ); + } + + #[test] + fn pending_receipt_recovers_delete_to_receipt_crash_window() { + let temp = tempfile::tempdir().unwrap(); + let worker_id = WorkerId::new(11); + source(temp.path(), worker_id, 1); + let provider = FsWorkerRetentionProvider::new(temp.path()); + let request = request(worker_id, 1, SessionDisposition::Archive); + let completed = provider.execute(&request).unwrap(); + let receipt_path = temp.path().join("retention/operations/operation-a.json"); + let mut receipt: RetentionOperationReceipt = + serde_json::from_slice(&fs::read(&receipt_path).unwrap()).unwrap(); + receipt.result.source_removed = false; + fs::write(&receipt_path, serde_json::to_vec_pretty(&receipt).unwrap()).unwrap(); + + let recovered = provider.execute(&request).unwrap(); + assert_eq!(recovered, completed); + assert!( + provider + .completed("operation-a", "fingerprint-a") + .unwrap() + .is_some() + ); + } + + #[test] + fn concurrent_retry_produces_one_archive() { + let temp = tempfile::tempdir().unwrap(); + let worker_id = WorkerId::new(10); + source(temp.path(), worker_id, 1); + let provider = Arc::new(FsWorkerRetentionProvider::new(temp.path())); + let request = Arc::new(request(worker_id, 1, SessionDisposition::Archive)); + let barrier = Arc::new(Barrier::new(3)); + let handles = (0..2) + .map(|_| { + let provider = provider.clone(); + let request = request.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + provider.execute(&request) + }) + }) + .collect::<Vec<_>>(); + barrier.wait(); + let results = handles + .into_iter() + .map(|handle| handle.join().unwrap().unwrap()) + .collect::<Vec<_>>(); + assert_eq!(results[0], results[1]); + assert_eq!( + fs::read_dir(temp.path().join("archives/workers")) + .unwrap() + .count(), + 1 + ); + } +} diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 336bf86d..e93a421d 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -26,6 +26,11 @@ use crate::management::{ }; #[cfg(feature = "ws-server")] use crate::observation::{WorkerObservationCursor, WorkerObservationEvent}; +#[cfg(feature = "fs-store")] +use crate::retention::{ + FsWorkerRetentionProvider, WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, + WorkerRetentionInventory, WorkerRetentionProvider, +}; use protocol::subscription::{ EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot, SubscriptionValidationError, SubscriptionWorkdirId, SubscriptionWorker, SubscriptionWorkerId, @@ -1648,6 +1653,118 @@ impl Runtime { Ok(()) } + /// Bind the Backend registry identity once. Retention evidence fails closed + /// until the Runtime host supplies this trusted configuration. + pub fn bind_runtime_identity(&self, runtime_id: &str) -> Result<(), RuntimeError> { + if runtime_id.trim().is_empty() || runtime_id.len() > 160 { + return Err(RuntimeError::InvalidRequest( + "Runtime identity must be non-empty and bounded".to_string(), + )); + } + let mut state = self.lock()?; + match state.runtime_identity.as_deref() { + Some(current) if current == runtime_id => Ok(()), + Some(_) => Err(RuntimeError::InvalidRequest( + "Runtime identity is already bound".to_string(), + )), + None => { + state.runtime_identity = Some(runtime_id.to_string()); + Ok(()) + } + } + } + + /// Read canonical aggregate facts needed by a Backend removal plan. + #[cfg(feature = "fs-store")] + pub fn worker_retention_inventory( + &self, + workspace_id: &str, + worker_ref: &WorkerRef, + ) -> Result<WorkerRetentionInventory, RuntimeError> { + let state = self.lock()?; + let runtime_id = state.runtime_identity.as_deref().ok_or_else(|| { + RuntimeError::InvalidRequest( + "Runtime identity is not bound for Worker retention".to_string(), + ) + })?; + let worker = state.worker(worker_ref)?; + if worker.workspace_id.as_deref() != Some(workspace_id) { + return Err(RuntimeError::WorkerNotFound { + worker_id: worker_ref.worker_id, + }); + } + let store = state.fs_store().ok_or_else(|| { + RuntimeError::InvalidRequest( + "Worker retention archive authority requires an fs-backed Runtime".to_string(), + ) + })?; + FsWorkerRetentionProvider::new(store.runtime_dir()).inventory( + workspace_id, + runtime_id, + worker.worker_id, + worker.run_generation, + ) + } + + /// Execute a Backend-resolved retention plan. Only stopped Workers are + /// eligible. Provider receipt lookup happens before live lookup so exact + /// retries converge after aggregate removal. + #[cfg(feature = "fs-store")] + pub fn execute_worker_retention( + &self, + request: &WorkerRetentionExecutionRequest, + ) -> Result<WorkerRetentionExecutionResult, RuntimeError> { + let mut state = self.lock()?; + let runtime_id = state.runtime_identity.clone().ok_or_else(|| { + RuntimeError::InvalidRequest( + "Runtime identity is not bound for Worker retention".to_string(), + ) + })?; + if request.source_runtime_id != runtime_id { + return Err(RuntimeError::InvalidRequest( + "Worker retention Runtime identity mismatch".to_string(), + )); + } + let store = state.fs_store().ok_or_else(|| { + RuntimeError::InvalidRequest( + "Worker retention execution requires an fs-backed Runtime".to_string(), + ) + })?; + let provider = FsWorkerRetentionProvider::new(store.runtime_dir()); + if let Some(completed) = + provider.completed(&request.operation_id, &request.input_fingerprint)? + { + state.workers.remove(&request.worker_id); + state.persist_runtime_snapshot()?; + return Ok(completed); + } + let Some(worker) = state.workers.get(&request.worker_id) else { + // Recover a pending receipt after a crash between aggregate removal + // and final receipt/Runtime catalog commit. + return provider.recover_after_source_removal(request); + }; + if worker.workspace_id.as_deref() != Some(request.workspace_id.as_str()) { + return Err(RuntimeError::WorkerNotFound { + worker_id: request.worker_id, + }); + } + if worker.status != WorkerStatus::Stopped { + return Err(RuntimeError::InvalidRequest( + "Worker retention requires a stopped Worker".to_string(), + )); + } + if worker.run_generation != request.expected_run_generation { + return Err(RuntimeError::InvalidRequest(format!( + "Worker retention plan expected generation {}, current generation is {}", + request.expected_run_generation, worker.run_generation + ))); + } + let result = provider.execute(request)?; + state.workers.remove(&request.worker_id); + state.persist_runtime_snapshot()?; + Ok(result) + } + fn lock(&self) -> Result<MutexGuard<'_, RuntimeState>, RuntimeError> { self.inner.lock().map_err(|_| RuntimeError::StatePoisoned) } @@ -1673,6 +1790,9 @@ struct SubscriptionSink { struct RuntimeState { display_name: Option<String>, backend: RuntimeBackendKind, + /// Backend-bound stable identity used for cross-boundary retention evidence. + /// It is configured once by the Runtime host and never model input. + runtime_identity: Option<String>, #[cfg_attr(not(feature = "fs-store"), allow(dead_code))] persistence: RuntimePersistence, status: RuntimeStatus, @@ -1701,6 +1821,7 @@ impl RuntimeState { Self { display_name, backend: RuntimeBackendKind::Memory, + runtime_identity: None, persistence: RuntimePersistence::Memory, status: RuntimeStatus::Running, execution_backend: None, @@ -1729,6 +1850,7 @@ impl RuntimeState { Self { display_name, backend: RuntimeBackendKind::FsStore, + runtime_identity: None, persistence: RuntimePersistence::Fs(store), status: RuntimeStatus::Running, execution_backend: None, @@ -1779,6 +1901,7 @@ impl RuntimeState { Ok(Self { display_name: persisted.display_name, backend: RuntimeBackendKind::FsStore, + runtime_identity: None, persistence: RuntimePersistence::Fs(store), status: persisted.status, execution_backend: None, @@ -2550,6 +2673,18 @@ mod tests { use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; + #[test] + fn runtime_identity_binding_is_immutable_and_host_owned() { + let runtime = Runtime::new_memory(); + runtime.bind_runtime_identity("runtime-a").unwrap(); + runtime.bind_runtime_identity("runtime-a").unwrap(); + assert!(runtime.bind_runtime_identity("runtime-b").is_err()); + assert_eq!( + runtime.lock().unwrap().runtime_identity.as_deref(), + Some("runtime-a") + ); + } + #[test] fn typed_segments_allow_empty_flat_content() { let input = WorkerInput { diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index b2046416..46aecd87 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -1497,6 +1497,9 @@ impl EmbeddedWorkerRuntime { pub fn from_runtime(workspace_id: impl AsRef<str>, runtime: worker_runtime::Runtime) -> Self { let workspace_id = workspace_id.as_ref().to_string(); + runtime + .bind_runtime_identity(EMBEDDED_RUNTIME_ID) + .expect("fresh embedded Runtime must accept its Backend-owned identity"); Self { runtime_id: EMBEDDED_RUNTIME_ID.to_string(), host_id: host_id_for_embedded_workspace(&workspace_id), diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 32708b85..544f38fb 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -19,6 +19,7 @@ pub mod records; pub use records::ticket_api_typescript; pub mod repositories; pub mod resource_broker; +pub mod retention; pub mod runtime_subscription; pub mod server; pub mod skills; diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs new file mode 100644 index 00000000..b3b21ff6 --- /dev/null +++ b/crates/workspace-server/src/retention.rs @@ -0,0 +1,1194 @@ +//! Workspace DB authority for deterministic Worker retention planning. +//! Runtime receives only resolved dispositions and stable ids; policy authority +//! never comes from prompts, profiles, or model input. + +use crate::{Error as StoreError, store::SqliteWorkspaceStore}; +use chrono::Utc; +use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use worker_runtime::identity::RuntimeWorkerRef; +use worker_runtime::retention::{ + DiagnosticsDisposition, SessionDisposition, WorkerRetentionExecutionRequest, + WorkerRetentionExecutionResult, WorkerRetentionInventory, +}; + +pub const CONSERVATIVE_POLICY_ID: &str = "workspace-default-conservative"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MetadataDisposition { + Tombstone, + Purge, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ArchiveRetention { + Forever, + ForSeconds { seconds: u64 }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerRetentionPolicy { + pub workspace_id: String, + pub policy_id: String, + pub revision: u64, + pub session_disposition: SessionDisposition, + pub metadata_disposition: MetadataDisposition, + pub archive_retention: ArchiveRetention, + pub diagnostics_disposition: DiagnosticsDisposition, + pub diagnostics_retention_seconds: Option<u64>, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerRetentionPolicyUpdate { + pub policy_id: String, + pub session_disposition: SessionDisposition, + pub metadata_disposition: MetadataDisposition, + pub archive_retention: ArchiveRetention, + pub diagnostics_disposition: DiagnosticsDisposition, + pub diagnostics_retention_seconds: Option<u64>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerRemovalPlanRequest { + pub workspace_id: String, + pub worker: RuntimeWorkerRef, + pub expected_worker_revision: String, + pub reason: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum WorkerRemovalBlocker { + Hold, + CurrentAssignment { + assignment_id: String, + ticket_id: String, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkerRemovalPlanState { + Planned, + Blocked, + Executing, + Failed, + Stale, + Succeeded, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerRemovalPlan { + pub plan_id: String, + pub operation_id: String, + pub input_fingerprint: String, + pub workspace_id: String, + pub worker: RuntimeWorkerRef, + pub worker_revision: String, + pub run_generation: u64, + pub policy_id: String, + pub policy_revision: u64, + pub session_disposition: SessionDisposition, + pub metadata_disposition: MetadataDisposition, + pub archive_retention: ArchiveRetention, + pub diagnostics_disposition: DiagnosticsDisposition, + pub diagnostics_retention_seconds: Option<u64>, + pub archive_id: Option<String>, + pub blockers: Vec<WorkerRemovalBlocker>, + pub state: WorkerRemovalPlanState, + pub reason: String, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PreparedWorkerRemoval { + pub plan: WorkerRemovalPlan, + pub runtime_request: WorkerRetentionExecutionRequest, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerTombstone { + pub workspace_id: String, + pub worker: RuntimeWorkerRef, + pub display_name: String, + pub profile: Option<String>, + pub created_at: String, + pub removed_at: String, + pub archive_id: Option<String>, + pub policy_id: String, + pub policy_revision: u64, + pub operation_id: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerOrphanDiagnostic { + pub diagnostic_id: String, + pub workspace_id: String, + pub runtime_id: String, + pub worker_id: String, + pub category: String, + pub detail: String, + pub observed_at: String, +} + +#[derive(thiserror::Error, Debug)] +pub enum WorkerRetentionError { + #[error(transparent)] + Store(#[from] StoreError), + #[error("Worker retention policy is not configured for Workspace {workspace_id}")] + PolicyMissing { workspace_id: String }, + #[error("Worker retention policy revision conflict: expected {expected}, current {actual}")] + PolicyRevisionConflict { expected: u64, actual: u64 }, + #[error("Worker was not found in the requested Workspace")] + WorkerNotFound, + #[error("Worker belongs to a different Workspace")] + CrossWorkspace, + #[error("Worker revision changed: expected {expected}, current {actual}")] + WorkerRevisionConflict { expected: String, actual: String }, + #[error("Worker removal is blocked: {0:?}")] + Blocked(Vec<WorkerRemovalBlocker>), + #[error("Worker removal plan {plan_id} is stale: {reason}")] + StalePlan { plan_id: String, reason: String }, + #[error("Worker removal operation {operation_id} was reused with different input")] + OperationFingerprintConflict { operation_id: String }, + #[error("invalid Worker retention input: {0}")] + Invalid(String), +} + +pub(crate) fn create_worker_retention_tables(conn: &Connection) -> crate::Result<()> { + conn.execute_batch(r#" + CREATE TABLE workspace_worker_retention_policy_revisions ( + workspace_id TEXT NOT NULL, policy_id TEXT NOT NULL, revision INTEGER NOT NULL CHECK(revision>0), + session_disposition TEXT NOT NULL CHECK(session_disposition IN ('archive','purge')), + metadata_disposition TEXT NOT NULL CHECK(metadata_disposition IN ('tombstone','purge')), + archive_retention_kind TEXT NOT NULL CHECK(archive_retention_kind IN ('forever','for_seconds')), + archive_retention_seconds INTEGER, + diagnostics_disposition TEXT NOT NULL CHECK(diagnostics_disposition IN ('purge','retain')), + diagnostics_retention_seconds INTEGER, created_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,policy_id,revision), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE); + CREATE TABLE workspace_worker_retention_policies ( + workspace_id TEXT PRIMARY KEY, policy_id TEXT NOT NULL, revision INTEGER NOT NULL, updated_at TEXT NOT NULL, + FOREIGN KEY(workspace_id,policy_id,revision) REFERENCES workspace_worker_retention_policy_revisions(workspace_id,policy_id,revision), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE); + CREATE TABLE worker_removal_operations ( + operation_id TEXT PRIMARY KEY, plan_id TEXT NOT NULL UNIQUE, input_fingerprint TEXT NOT NULL, + workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, + worker_revision TEXT NOT NULL, run_generation INTEGER NOT NULL CHECK(run_generation>=0), + policy_id TEXT NOT NULL, policy_revision INTEGER NOT NULL, + session_disposition TEXT NOT NULL, metadata_disposition TEXT NOT NULL, + archive_retention_kind TEXT NOT NULL, archive_retention_seconds INTEGER, + diagnostics_disposition TEXT NOT NULL, + diagnostics_retention_seconds INTEGER, archive_id TEXT UNIQUE, blockers_json TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('planned','blocked','executing','failed','stale','succeeded')), + reason TEXT NOT NULL, failure_category TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE); + CREATE INDEX worker_removal_operations_worker_idx ON worker_removal_operations(workspace_id,runtime_id,worker_id,created_at); + CREATE TABLE worker_session_archives ( + archive_id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, + session_id TEXT NOT NULL, checksum_sha256 TEXT NOT NULL, content_bytes INTEGER NOT NULL, + policy_id TEXT NOT NULL, policy_revision INTEGER NOT NULL, operation_id TEXT NOT NULL UNIQUE, + committed_at TEXT NOT NULL, expires_at TEXT, + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + FOREIGN KEY(operation_id) REFERENCES worker_removal_operations(operation_id)); + CREATE TABLE worker_diagnostics_archives ( + operation_id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, policy_id TEXT NOT NULL, policy_revision INTEGER NOT NULL, + committed_at TEXT NOT NULL, expires_at TEXT NOT NULL, + FOREIGN KEY(operation_id) REFERENCES worker_removal_operations(operation_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE); + CREATE TABLE worker_tombstones ( + workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, + display_name TEXT NOT NULL, profile TEXT, worker_created_at TEXT NOT NULL, removed_at TEXT NOT NULL, + archive_id TEXT, policy_id TEXT NOT NULL, policy_revision INTEGER NOT NULL, operation_id TEXT NOT NULL UNIQUE, + PRIMARY KEY(workspace_id,runtime_id,worker_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + FOREIGN KEY(archive_id) REFERENCES worker_session_archives(archive_id), + FOREIGN KEY(operation_id) REFERENCES worker_removal_operations(operation_id)); + CREATE TABLE worker_orphan_diagnostics ( + diagnostic_id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, + category TEXT NOT NULL, detail TEXT NOT NULL, observed_at TEXT NOT NULL, + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE); + CREATE TABLE worker_retention_audit_events ( + event_id TEXT PRIMARY KEY, operation_id TEXT NOT NULL, workspace_id TEXT NOT NULL, + event_kind TEXT NOT NULL, detail TEXT NOT NULL, created_at TEXT NOT NULL, + FOREIGN KEY(operation_id) REFERENCES worker_removal_operations(operation_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE); + CREATE TRIGGER seed_worker_retention_policy_after_workspace_insert AFTER INSERT ON workspaces BEGIN + INSERT INTO workspace_worker_retention_policy_revisions + (workspace_id,policy_id,revision,session_disposition,metadata_disposition,archive_retention_kind,archive_retention_seconds,diagnostics_disposition,diagnostics_retention_seconds,created_at) + VALUES(NEW.workspace_id,'workspace-default-conservative',1,'archive','tombstone','forever',NULL,'purge',NULL,NEW.created_at); + INSERT INTO workspace_worker_retention_policies(workspace_id,policy_id,revision,updated_at) + VALUES(NEW.workspace_id,'workspace-default-conservative',1,NEW.created_at); + END; + "#)?; + let now = Utc::now().to_rfc3339(); + conn.execute("INSERT OR IGNORE INTO workspace_worker_retention_policy_revisions + (workspace_id,policy_id,revision,session_disposition,metadata_disposition,archive_retention_kind,archive_retention_seconds,diagnostics_disposition,diagnostics_retention_seconds,created_at) + SELECT workspace_id,?1,1,'archive','tombstone','forever',NULL,'purge',NULL,?2 FROM workspaces", params![CONSERVATIVE_POLICY_ID,now])?; + conn.execute("INSERT OR IGNORE INTO workspace_worker_retention_policies(workspace_id,policy_id,revision,updated_at) + SELECT workspace_id,?1,1,?2 FROM workspaces", params![CONSERVATIVE_POLICY_ID,now])?; + Ok(()) +} + +impl SqliteWorkspaceStore { + pub fn worker_retention_policy( + &self, + workspace_id: &str, + ) -> crate::Result<Option<WorkerRetentionPolicy>> { + self.with_conn(|conn| load_policy(conn, workspace_id)) + } + + pub fn update_worker_retention_policy( + &self, + workspace_id: &str, + expected: u64, + update: &WorkerRetentionPolicyUpdate, + ) -> Result<WorkerRetentionPolicy, WorkerRetentionError> { + validate_policy(update)?; + self.with_conn_mut(|conn| { + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current=load_policy(&tx,workspace_id)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?; + if current.revision!=expected { return Err(StoreError::InvalidInput(format!("policy-conflict:{expected}:{}",current.revision))); } + let revision=current.revision+1; let now=Utc::now().to_rfc3339(); + tx.execute("INSERT INTO workspace_worker_retention_policy_revisions + (workspace_id,policy_id,revision,session_disposition,metadata_disposition,archive_retention_kind,archive_retention_seconds,diagnostics_disposition,diagnostics_retention_seconds,created_at) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",params![workspace_id,update.policy_id,revision,sess(update.session_disposition),meta(update.metadata_disposition),archive_kind(update.archive_retention),archive_seconds(update.archive_retention),diag(update.diagnostics_disposition),update.diagnostics_retention_seconds,now])?; + let changed=tx.execute("UPDATE workspace_worker_retention_policies SET policy_id=?1,revision=?2,updated_at=?3 WHERE workspace_id=?4 AND revision=?5", + params![update.policy_id,revision,now,workspace_id,expected])?; + if changed!=1 { return Err(StoreError::InvalidInput(format!("policy-conflict:{expected}:{revision}"))); } + tx.commit()?; load_policy(conn,workspace_id)?.ok_or_else(|| StoreError::InvalidInput("updated policy missing".into())) + }).map_err(map_error) + } + + pub fn plan_worker_removal( + &self, + req: &WorkerRemovalPlanRequest, + inv: &WorkerRetentionInventory, + ) -> Result<WorkerRemovalPlan, WorkerRetentionError> { + validate_plan(req, inv)?; + let now = Utc::now().to_rfc3339(); + self.with_conn_mut(|conn| { + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let policy=load_policy(&tx,&req.workspace_id)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?; + let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? { + Some(v)=>v, + None=>{ + let other:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM worker_registry WHERE runtime_id=?1 AND runtime_worker_id=?2 AND workspace_id!=?3)",params![req.worker.runtime_id,req.worker.worker_id,req.workspace_id],|r|r.get(0))?; + return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()})); + } + }; + if worker.updated_at!=req.expected_worker_revision { return Err(StoreError::InvalidInput(format!("worker-conflict:{}:{}",req.expected_worker_revision,worker.updated_at))); } + let mut blockers=Vec::new(); + if worker.retention_state=="pinned" { blockers.push(WorkerRemovalBlocker::Hold); } + if let Some((assignment_id,ticket_id))=tx.query_row("SELECT a.assignment_id,a.ticket_id FROM ticket_current_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()? { + blockers.push(WorkerRemovalBlocker::CurrentAssignment{assignment_id,ticket_id}); + } + let fp=fingerprint(req,inv,&policy,&blockers)?; + let plan_id=stable("wrp",&fp); let operation_id=stable("wro",&fp); + let archive_id=(policy.session_disposition==SessionDisposition::Archive).then(||stable("wra",&fp)); + let state=if blockers.is_empty(){WorkerRemovalPlanState::Planned}else{WorkerRemovalPlanState::Blocked}; + tx.execute("INSERT OR IGNORE INTO worker_removal_operations(operation_id,plan_id,input_fingerprint,workspace_id,runtime_id,worker_id,worker_revision,run_generation,policy_id,policy_revision,session_disposition,metadata_disposition,archive_retention_kind,archive_retention_seconds,diagnostics_disposition,diagnostics_retention_seconds,archive_id,blockers_json,state,reason,created_at,updated_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20,?21,?21)",params![operation_id,plan_id,fp,req.workspace_id,req.worker.runtime_id,req.worker.worker_id,req.expected_worker_revision,inv.run_generation,policy.policy_id,policy.revision,sess(policy.session_disposition),meta(policy.metadata_disposition),archive_kind(policy.archive_retention),archive_seconds(policy.archive_retention),diag(policy.diagnostics_disposition),policy.diagnostics_retention_seconds,archive_id,serde_json::to_string(&blockers).map_err(|e|StoreError::InvalidInput(e.to_string()))?,state_s(state),req.reason,now])?; + let plan=load_plan(&tx,&plan_id)?.ok_or_else(||StoreError::InvalidInput("plan missing".into()))?; + if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));} + tx.commit()?; Ok(plan) + }).map_err(map_error) + } + + pub fn begin_worker_removal( + &self, + workspace_id: &str, + plan_id: &str, + fp: &str, + ) -> Result<WorkerRemovalPlan, WorkerRetentionError> { + self.with_conn_mut(|conn|{ + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let mut plan=load_plan(&tx,plan_id)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:plan missing")))?; + if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));} + if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));} + if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);} + if plan.state==WorkerRemovalPlanState::Blocked { + return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap()))); + } + if !matches!(plan.state, WorkerRemovalPlanState::Planned | WorkerRemovalPlanState::Failed | WorkerRemovalPlanState::Executing) { + return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state)))); + } + if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));} + let policy=load_policy(&tx,workspace_id)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?; + if policy.policy_id!=plan.policy_id||policy.revision!=plan.policy_revision{ + mark_stale(&tx,&plan,"policy revision changed")?;tx.commit()?; + return Err(stale_error(&plan,"policy revision changed")); + } + let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:Worker missing")))?; + if worker.updated_at!=plan.worker_revision{ + mark_stale(&tx,&plan,"Worker revision changed")?;tx.commit()?; + return Err(stale_error(&plan,"Worker revision changed")); + } + if worker.retention_state=="pinned"{ + mark_stale(&tx,&plan,"hold added")?;tx.commit()?; + return Err(stale_error(&plan,"hold added")); + } + let assigned:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_current_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id],|r|r.get(0))?; + if assigned{ + mark_stale(&tx,&plan,"current assignment added")?;tx.commit()?; + return Err(stale_error(&plan,"current assignment added")); + } + let now=Utc::now().to_rfc3339(); + tx.execute("UPDATE worker_removal_operations SET state='executing',failure_category=NULL,updated_at=?1 WHERE operation_id=?2",params![now,plan.operation_id])?; + plan.state=WorkerRemovalPlanState::Executing;plan.updated_at=now;tx.commit()?;Ok(plan) + }).map_err(map_error) + } + + /// Revalidates Backend authority and derives the complete Runtime request + /// from the immutable plan. Callers cannot substitute generation or + /// dispositions without causing a fingerprint/manifest mismatch. + pub fn prepare_worker_removal_execution( + &self, + workspace_id: &str, + plan_id: &str, + input_fingerprint: &str, + ) -> Result<PreparedWorkerRemoval, WorkerRetentionError> { + let plan = self.begin_worker_removal(workspace_id, plan_id, input_fingerprint)?; + let worker = self + .with_conn(|conn| load_worker(conn, workspace_id, &plan.worker))? + .ok_or_else(|| WorkerRetentionError::StalePlan { + plan_id: plan.plan_id.clone(), + reason: "Worker disappeared after execution fence".to_string(), + })?; + let worker_number = plan.worker.worker_id.parse::<u64>().map_err(|_| { + WorkerRetentionError::Invalid( + "Runtime Worker id is not a canonical unsigned integer".to_string(), + ) + })?; + let removed_at = Utc::now().to_rfc3339(); + Ok(PreparedWorkerRemoval { + runtime_request: WorkerRetentionExecutionRequest { + operation_id: plan.operation_id.clone(), + input_fingerprint: plan.input_fingerprint.clone(), + archive_id: plan.archive_id.clone(), + workspace_id: plan.workspace_id.clone(), + source_runtime_id: plan.worker.runtime_id.clone(), + worker_id: worker_runtime::identity::WorkerId::new(worker_number), + expected_run_generation: plan.run_generation, + source_created_at: worker.created_at, + removed_at, + effective_profile: worker.profile, + retention_class: None, + policy_id: plan.policy_id.clone(), + policy_revision: plan.policy_revision, + session_disposition: plan.session_disposition, + diagnostics_disposition: plan.diagnostics_disposition, + }, + plan, + }) + } + + pub fn fail_worker_removal( + &self, + workspace_id: &str, + operation_id: &str, + fp: &str, + category: &str, + ) -> Result<(), WorkerRetentionError> { + bounded("failure category", category, 160)?; + self.with_conn_mut(|conn|{ + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let changed=tx.execute("UPDATE worker_removal_operations SET state='failed',failure_category=?1,updated_at=?2 WHERE workspace_id=?3 AND operation_id=?4 AND input_fingerprint=?5 AND state IN ('planned','executing','failed')",params![category,Utc::now().to_rfc3339(),workspace_id,operation_id,fp])?; + if changed!=1{return Err(StoreError::InvalidInput("active operation mismatch".into()));} + tx.commit()?;Ok(()) + })?; + Ok(()) + } + + pub fn commit_worker_removal( + &self, + workspace_id: &str, + operation_id: &str, + fp: &str, + result: &WorkerRetentionExecutionResult, + ) -> Result<WorkerRemovalPlan, WorkerRetentionError> { + self.with_conn_mut(|conn|{ + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let mut plan=load_plan_op(&tx,operation_id)?.ok_or_else(||StoreError::InvalidInput("operation missing".into()))?; + if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));} + if plan.input_fingerprint!=fp||result.input_fingerprint!=fp||result.operation_id!=operation_id{return Err(StoreError::InvalidInput(format!("fingerprint:{operation_id}")));} + if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);} + if plan.state != WorkerRemovalPlanState::Executing { + return Err(StoreError::InvalidInput(format!("stale:{}:plan state {} is not committable", plan.plan_id, state_s(plan.state)))); + } + if result.worker_id.to_string() != plan.worker.worker_id + || result.session_disposition != plan.session_disposition + || result.diagnostics_disposition != plan.diagnostics_disposition + { + return Err(StoreError::InvalidInput("Runtime retention result does not match removal plan".into())); + } + if !result.source_removed{return Err(StoreError::InvalidInput("Runtime source was not removed".into()));} + let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput("Worker missing before commit".into()))?; + if worker.updated_at!=plan.worker_revision{return Err(StoreError::InvalidInput(format!("stale:{}:Worker revision changed",plan.plan_id)));} + if worker.retention_state=="pinned" { return Err(StoreError::InvalidInput(format!("stale:{}:hold added",plan.plan_id))); } + let assigned:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_current_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id],|row|row.get(0))?; + if assigned { return Err(StoreError::InvalidInput(format!("stale:{}:current assignment added",plan.plan_id))); } + let now=Utc::now().to_rfc3339(); + if let Some(a)=&result.archive{ + if plan.archive_id.as_deref()!=Some(&a.archive_id)||a.workspace_id!=workspace_id||a.source_runtime_id!=plan.worker.runtime_id||a.source_worker_id.to_string()!=plan.worker.worker_id||a.policy_id!=plan.policy_id||a.policy_revision!=plan.policy_revision{return Err(StoreError::InvalidInput("archive manifest mismatch".into()));} + let expires_at=match plan.archive_retention { ArchiveRetention::Forever=>None, ArchiveRetention::ForSeconds{seconds}=>{let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("archive retention deadline overflow".into()))?;Some((Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339())} }; + tx.execute("INSERT OR IGNORE INTO worker_session_archives(archive_id,workspace_id,runtime_id,worker_id,session_id,checksum_sha256,content_bytes,policy_id,policy_revision,operation_id,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",params![a.archive_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,a.source_session_id,a.content_checksum_sha256,a.content_bytes,plan.policy_id,plan.policy_revision,operation_id,now,expires_at])?; + }else if plan.session_disposition==SessionDisposition::Archive{return Err(StoreError::InvalidInput("archive manifest missing".into()));} + match plan.diagnostics_disposition { + DiagnosticsDisposition::Purge if result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime retained diagnostics for purge disposition".into())), + DiagnosticsDisposition::Retain if !result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime did not retain diagnostics".into())), + DiagnosticsDisposition::Retain => { + let seconds=plan.diagnostics_retention_seconds.ok_or_else(||StoreError::InvalidInput("diagnostics retention deadline missing".into()))?; + let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("diagnostics retention deadline overflow".into()))?; + let expires_at=(Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339(); + tx.execute("INSERT OR IGNORE INTO worker_diagnostics_archives(operation_id,workspace_id,runtime_id,worker_id,policy_id,policy_revision,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)",params![operation_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.policy_id,plan.policy_revision,now,expires_at])?; + } + DiagnosticsDisposition::Purge => {} + } + if plan.metadata_disposition==MetadataDisposition::Tombstone{ + tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_revision,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,plan.archive_id,plan.policy_id,plan.policy_revision,operation_id])?; + } + let deleted=tx.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND runtime_worker_id=?3 AND updated_at=?4",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.worker_revision])?; + if deleted!=1{return Err(StoreError::InvalidInput(format!("stale:{}:removal fence changed",plan.plan_id)));} + tx.execute("UPDATE worker_removal_operations SET state='succeeded',failure_category=NULL,updated_at=?1 WHERE operation_id=?2",params![now,operation_id])?; + tx.execute("INSERT OR IGNORE INTO worker_retention_audit_events(event_id,operation_id,workspace_id,event_kind,detail,created_at) VALUES(?1,?2,?3,'worker_removed',?4,?5)",params![stable("wre",operation_id),operation_id,workspace_id,format!("runtime_id={} worker_id={} session={} metadata={} diagnostics={}",plan.worker.runtime_id,plan.worker.worker_id,sess(plan.session_disposition),meta(plan.metadata_disposition),diag(plan.diagnostics_disposition)),now])?; + tx.commit()?;plan.state=WorkerRemovalPlanState::Succeeded;plan.updated_at=now;Ok(plan) + }).map_err(map_error) + } + + pub fn record_worker_orphan_diagnostic( + &self, + d: &WorkerOrphanDiagnostic, + ) -> Result<(), WorkerRetentionError> { + bounded("orphan category", &d.category, 160)?; + bounded("orphan detail", &d.detail, 2000)?; + self.with_conn(|conn|{conn.execute("INSERT OR IGNORE INTO worker_orphan_diagnostics(diagnostic_id,workspace_id,runtime_id,worker_id,category,detail,observed_at) VALUES(?1,?2,?3,?4,?5,?6,?7)",params![d.diagnostic_id,d.workspace_id,d.runtime_id,d.worker_id,d.category,d.detail,d.observed_at])?;Ok(())})?; + Ok(()) + } + + pub fn worker_tombstone( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + ) -> crate::Result<Option<WorkerTombstone>> { + self.with_conn(|conn|conn.query_row("SELECT display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_revision,operation_id FROM worker_tombstones WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",params![workspace_id,worker.runtime_id,worker.worker_id],|r|Ok(WorkerTombstone{workspace_id:workspace_id.into(),worker:worker.clone(),display_name:r.get(0)?,profile:r.get(1)?,created_at:r.get(2)?,removed_at:r.get(3)?,archive_id:r.get(4)?,policy_id:r.get(5)?,policy_revision:r.get::<_,i64>(6)? as u64,operation_id:r.get(7)?})).optional().map_err(StoreError::from)) + } +} + +#[derive(Clone)] +struct WorkerRow { + display_name: String, + profile: Option<String>, + retention_state: String, + created_at: String, + updated_at: String, +} +fn load_worker(c: &Connection, w: &str, r: &RuntimeWorkerRef) -> crate::Result<Option<WorkerRow>> { + c.query_row("SELECT display_name,profile,retention_state,created_at,updated_at FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND runtime_worker_id=?3",params![w,r.runtime_id,r.worker_id],|x|Ok(WorkerRow{display_name:x.get(0)?,profile:x.get(1)?,retention_state:x.get(2)?,created_at:x.get(3)?,updated_at:x.get(4)?})).optional().map_err(StoreError::from) +} +fn load_policy(c: &Connection, w: &str) -> crate::Result<Option<WorkerRetentionPolicy>> { + c.query_row( + "SELECT p.policy_id,p.revision,r.session_disposition,r.metadata_disposition, + r.archive_retention_kind,r.archive_retention_seconds, + r.diagnostics_disposition,r.diagnostics_retention_seconds,r.created_at,p.updated_at + FROM workspace_worker_retention_policies p + JOIN workspace_worker_retention_policy_revisions r + ON r.workspace_id=p.workspace_id AND r.policy_id=p.policy_id AND r.revision=p.revision + WHERE p.workspace_id=?1", + params![w], + |row| { + let session: String = row.get(2)?; + let metadata: String = row.get(3)?; + let archive_kind: String = row.get(4)?; + let archive_seconds: Option<i64> = row.get(5)?; + let diagnostics: String = row.get(6)?; + Ok(WorkerRetentionPolicy { + workspace_id: w.into(), + policy_id: row.get(0)?, + revision: row.get::<_, i64>(1)? as u64, + session_disposition: parse_s(&session)?, + metadata_disposition: parse_m(&metadata)?, + archive_retention: parse_archive(&archive_kind, archive_seconds)?, + diagnostics_disposition: parse_d(&diagnostics)?, + diagnostics_retention_seconds: row.get::<_, Option<i64>>(7)?.map(|v| v as u64), + created_at: row.get(8)?, + updated_at: row.get(9)?, + }) + }, + ) + .optional() + .map_err(StoreError::from) +} +fn load_plan(c: &Connection, id: &str) -> crate::Result<Option<WorkerRemovalPlan>> { + load_plan_q(c, "plan_id", id) +} +fn load_plan_op(c: &Connection, id: &str) -> crate::Result<Option<WorkerRemovalPlan>> { + load_plan_q(c, "operation_id", id) +} +fn load_plan_q(c: &Connection, key: &str, id: &str) -> crate::Result<Option<WorkerRemovalPlan>> { + let query = format!( + "SELECT plan_id,operation_id,input_fingerprint,workspace_id,runtime_id,worker_id, + worker_revision,run_generation,policy_id,policy_revision,session_disposition, + metadata_disposition,archive_retention_kind,archive_retention_seconds, + diagnostics_disposition,diagnostics_retention_seconds,archive_id,blockers_json, + state,reason,created_at,updated_at + FROM worker_removal_operations WHERE {key}=?1" + ); + c.query_row(&query, params![id], |row| { + let session: String = row.get(10)?; + let metadata: String = row.get(11)?; + let archive_kind: String = row.get(12)?; + let archive_seconds: Option<i64> = row.get(13)?; + let diagnostics: String = row.get(14)?; + let blockers: String = row.get(17)?; + let state: String = row.get(18)?; + Ok(WorkerRemovalPlan { + plan_id: row.get(0)?, + operation_id: row.get(1)?, + input_fingerprint: row.get(2)?, + workspace_id: row.get(3)?, + worker: RuntimeWorkerRef { + runtime_id: row.get(4)?, + worker_id: row.get(5)?, + }, + worker_revision: row.get(6)?, + run_generation: row.get::<_, i64>(7)? as u64, + policy_id: row.get(8)?, + policy_revision: row.get::<_, i64>(9)? as u64, + session_disposition: parse_s(&session)?, + metadata_disposition: parse_m(&metadata)?, + archive_retention: parse_archive(&archive_kind, archive_seconds)?, + diagnostics_disposition: parse_d(&diagnostics)?, + diagnostics_retention_seconds: row.get::<_, Option<i64>>(15)?.map(|v| v as u64), + archive_id: row.get(16)?, + blockers: serde_json::from_str(&blockers).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 17, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + state: parse_state(&state)?, + reason: row.get(19)?, + created_at: row.get(20)?, + updated_at: row.get(21)?, + }) + }) + .optional() + .map_err(StoreError::from) +} +fn mark_stale( + tx: &rusqlite::Transaction<'_>, + plan: &WorkerRemovalPlan, + reason: &str, +) -> crate::Result<()> { + tx.execute("UPDATE worker_removal_operations SET state='stale',failure_category=?1,updated_at=?2 WHERE operation_id=?3",params![reason,Utc::now().to_rfc3339(),plan.operation_id])?; + Ok(()) +} +fn stale_error(plan: &WorkerRemovalPlan, reason: &str) -> StoreError { + StoreError::InvalidInput(format!("stale:{}:{reason}", plan.plan_id)) +} +fn fingerprint( + r: &WorkerRemovalPlanRequest, + i: &WorkerRetentionInventory, + p: &WorkerRetentionPolicy, + b: &[WorkerRemovalBlocker], +) -> crate::Result<String> { + serde_json::to_vec(&serde_json::json!([ + r.workspace_id, + r.worker.runtime_id, + r.worker.worker_id, + r.expected_worker_revision, + i.run_generation, + i.session_id, + i.segment_ids, + p.policy_id, + p.revision, + p.session_disposition, + p.metadata_disposition, + p.archive_retention, + p.diagnostics_disposition, + p.diagnostics_retention_seconds, + b, + r.reason + ])) + .map(|v| hash(&v)) + .map_err(|e| StoreError::InvalidInput(e.to_string())) +} +fn hash(b: &[u8]) -> String { + Sha256::digest(b) + .iter() + .map(|v| format!("{v:02x}")) + .collect() +} +fn stable(p: &str, v: &str) -> String { + format!("{p}_{}", &hash(v.as_bytes())[..32]) +} +fn validate_plan( + r: &WorkerRemovalPlanRequest, + i: &WorkerRetentionInventory, +) -> Result<(), WorkerRetentionError> { + bounded("workspace", &r.workspace_id, 160)?; + bounded("revision", &r.expected_worker_revision, 256)?; + bounded("reason", &r.reason, 2000)?; + if i.workspace_id != r.workspace_id + || i.runtime_id != r.worker.runtime_id + || i.worker_id.to_string() != r.worker.worker_id + { + return Err(WorkerRetentionError::CrossWorkspace); + } + Ok(()) +} +fn validate_policy(u: &WorkerRetentionPolicyUpdate) -> Result<(), WorkerRetentionError> { + bounded("policy id", &u.policy_id, 160)?; + if matches!( + u.archive_retention, + ArchiveRetention::ForSeconds { seconds: 0 } + ) { + return Err(WorkerRetentionError::Invalid( + "archive retention seconds must be positive".to_string(), + )); + } + match (u.diagnostics_disposition, u.diagnostics_retention_seconds) { + (DiagnosticsDisposition::Purge, None) | (DiagnosticsDisposition::Retain, Some(1..)) => { + Ok(()) + } + _ => Err(WorkerRetentionError::Invalid( + "diagnostics retention/disposition mismatch".into(), + )), + } +} +fn bounded(k: &str, v: &str, n: usize) -> Result<(), WorkerRetentionError> { + if v.trim().is_empty() || v.len() > n { + Err(WorkerRetentionError::Invalid(format!( + "{k} must be non-empty and at most {n} bytes" + ))) + } else { + Ok(()) + } +} +fn map_error(e: StoreError) -> WorkerRetentionError { + let StoreError::InvalidInput(m) = &e else { + return WorkerRetentionError::Store(e); + }; + if let Some(x) = m.strip_prefix("policy-missing:") { + return WorkerRetentionError::PolicyMissing { + workspace_id: x.into(), + }; + } + if let Some(x) = m.strip_prefix("policy-conflict:") { + let mut s = x.split(':'); + return WorkerRetentionError::PolicyRevisionConflict { + expected: s.next().and_then(|v| v.parse().ok()).unwrap_or(0), + actual: s.next().and_then(|v| v.parse().ok()).unwrap_or(0), + }; + } + if m == "cross-workspace" { + return WorkerRetentionError::CrossWorkspace; + } + if m == "worker-missing" { + return WorkerRetentionError::WorkerNotFound; + } + if let Some(x) = m.strip_prefix("worker-conflict:") { + let mut s = x.splitn(2, ':'); + return WorkerRetentionError::WorkerRevisionConflict { + expected: s.next().unwrap_or_default().into(), + actual: s.next().unwrap_or_default().into(), + }; + } + if let Some(x) = m.strip_prefix("fingerprint:") { + return WorkerRetentionError::OperationFingerprintConflict { + operation_id: x.into(), + }; + } + if let Some(x) = m.strip_prefix("blocked:") { + return WorkerRetentionError::Blocked(serde_json::from_str(x).unwrap_or_default()); + } + if let Some(x) = m.strip_prefix("stale:") { + let mut s = x.splitn(2, ':'); + return WorkerRetentionError::StalePlan { + plan_id: s.next().unwrap_or_default().into(), + reason: s.next().unwrap_or_default().into(), + }; + } + WorkerRetentionError::Store(e) +} +fn archive_kind(value: ArchiveRetention) -> &'static str { + match value { + ArchiveRetention::Forever => "forever", + ArchiveRetention::ForSeconds { .. } => "for_seconds", + } +} +fn archive_seconds(value: ArchiveRetention) -> Option<u64> { + match value { + ArchiveRetention::Forever => None, + ArchiveRetention::ForSeconds { seconds } => Some(seconds), + } +} +fn parse_archive(kind: &str, seconds: Option<i64>) -> rusqlite::Result<ArchiveRetention> { + match (kind, seconds) { + ("forever", None) => Ok(ArchiveRetention::Forever), + ("for_seconds", Some(seconds)) if seconds > 0 => Ok(ArchiveRetention::ForSeconds { + seconds: seconds as u64, + }), + _ => Err(bad("archive retention", kind)), + } +} +fn sess(v: SessionDisposition) -> &'static str { + match v { + SessionDisposition::Archive => "archive", + SessionDisposition::Purge => "purge", + } +} +fn meta(v: MetadataDisposition) -> &'static str { + match v { + MetadataDisposition::Tombstone => "tombstone", + MetadataDisposition::Purge => "purge", + } +} +fn diag(v: DiagnosticsDisposition) -> &'static str { + match v { + DiagnosticsDisposition::Purge => "purge", + DiagnosticsDisposition::Retain => "retain", + } +} +fn state_s(v: WorkerRemovalPlanState) -> &'static str { + match v { + WorkerRemovalPlanState::Planned => "planned", + WorkerRemovalPlanState::Blocked => "blocked", + WorkerRemovalPlanState::Executing => "executing", + WorkerRemovalPlanState::Failed => "failed", + WorkerRemovalPlanState::Stale => "stale", + WorkerRemovalPlanState::Succeeded => "succeeded", + } +} +fn bad(k: &str, v: &str) -> rusqlite::Error { + rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Text, + format!("invalid {k}: {v}").into(), + ) +} +fn parse_s(v: &str) -> rusqlite::Result<SessionDisposition> { + match v { + "archive" => Ok(SessionDisposition::Archive), + "purge" => Ok(SessionDisposition::Purge), + _ => Err(bad("session", v)), + } +} +fn parse_m(v: &str) -> rusqlite::Result<MetadataDisposition> { + match v { + "tombstone" => Ok(MetadataDisposition::Tombstone), + "purge" => Ok(MetadataDisposition::Purge), + _ => Err(bad("metadata", v)), + } +} +fn parse_d(v: &str) -> rusqlite::Result<DiagnosticsDisposition> { + match v { + "purge" => Ok(DiagnosticsDisposition::Purge), + "retain" => Ok(DiagnosticsDisposition::Retain), + _ => Err(bad("diagnostics", v)), + } +} +fn parse_state(v: &str) -> rusqlite::Result<WorkerRemovalPlanState> { + match v { + "planned" => Ok(WorkerRemovalPlanState::Planned), + "blocked" => Ok(WorkerRemovalPlanState::Blocked), + "executing" => Ok(WorkerRemovalPlanState::Executing), + "failed" => Ok(WorkerRemovalPlanState::Failed), + "stale" => Ok(WorkerRemovalPlanState::Stale), + "succeeded" => Ok(WorkerRemovalPlanState::Succeeded), + _ => Err(bad("state", v)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::{ControlPlaneStore, TicketWorkerAssignmentRecord, WorkerRegistryRecord}; + use worker_runtime::identity::WorkerId; + fn setup() -> SqliteWorkspaceStore { + let s = SqliteWorkspaceStore::in_memory().unwrap(); + s.with_conn(|c|{c.execute("INSERT INTO workspaces(workspace_id,display_name,state,created_at,updated_at)VALUES('w','W','active','t','t')",[])?;c.execute("INSERT INTO worker_registry(workspace_id,runtime_id,runtime_worker_id,display_name,profile,retention_state,created_at,updated_at)VALUES('w','r',1,'one','builtin:coder','normal','created','rev1')",[])?;Ok(())}).unwrap(); + s + } + fn inv() -> WorkerRetentionInventory { + WorkerRetentionInventory { + workspace_id: "w".into(), + runtime_id: "r".into(), + worker_id: WorkerId::new(1), + run_generation: 2, + session_id: Some("s".into()), + segment_ids: vec!["a".into()], + session_bytes: 1, + diagnostics_bytes: 0, + } + } + fn req() -> WorkerRemovalPlanRequest { + WorkerRemovalPlanRequest { + workspace_id: "w".into(), + worker: RuntimeWorkerRef { + runtime_id: "r".into(), + worker_id: "1".into(), + }, + expected_worker_revision: "rev1".into(), + reason: "cleanup".into(), + } + } + #[test] + fn conservative_seed_and_policy_conflict() { + let s = setup(); + let p = s.worker_retention_policy("w").unwrap().unwrap(); + assert_eq!(p.session_disposition, SessionDisposition::Archive); + assert_eq!(p.archive_retention, ArchiveRetention::Forever); + let u = WorkerRetentionPolicyUpdate { + policy_id: "p".into(), + session_disposition: SessionDisposition::Archive, + metadata_disposition: MetadataDisposition::Purge, + archive_retention: ArchiveRetention::ForSeconds { seconds: 3_600 }, + diagnostics_disposition: DiagnosticsDisposition::Purge, + diagnostics_retention_seconds: None, + }; + let updated = s.update_worker_retention_policy("w", 1, &u).unwrap(); + assert_eq!(updated.revision, 2); + assert_eq!( + updated.archive_retention, + ArchiveRetention::ForSeconds { seconds: 3_600 } + ); + assert!(matches!( + s.update_worker_retention_policy("w", 1, &u), + Err(WorkerRetentionError::PolicyRevisionConflict { .. }) + )); + } + #[test] + fn deterministic_plan_hold_and_cross_workspace() { + let s = setup(); + let a = s.plan_worker_removal(&req(), &inv()).unwrap(); + let b = s.plan_worker_removal(&req(), &inv()).unwrap(); + assert_eq!(a.plan_id, b.plan_id); + s.with_conn(|c| { + c.execute( + "UPDATE worker_registry SET retention_state='pinned' WHERE workspace_id='w'", + [], + )?; + Ok(()) + }) + .unwrap(); + let mut q = req(); + q.expected_worker_revision = "rev1".into(); + let p = s.plan_worker_removal(&q, &inv()).unwrap(); + assert_eq!(p.blockers, vec![WorkerRemovalBlocker::Hold]); + assert!(matches!( + s.begin_worker_removal("w", &p.plan_id, &p.input_fingerprint), + Err(WorkerRetentionError::Blocked(_)) + )); + let mut i = inv(); + i.workspace_id = "other".into(); + assert!(matches!( + s.plan_worker_removal(&req(), &i), + Err(WorkerRetentionError::CrossWorkspace) + )); + i.workspace_id = "w".into(); + i.runtime_id = "other-runtime".into(); + assert!(matches!( + s.plan_worker_removal(&req(), &i), + Err(WorkerRetentionError::CrossWorkspace) + )); + } + #[test] + fn stale_policy_and_failed_retry_restore_fence() { + let s = setup(); + let p = s.plan_worker_removal(&req(), &inv()).unwrap(); + let u = WorkerRetentionPolicyUpdate { + policy_id: "new".into(), + session_disposition: SessionDisposition::Purge, + metadata_disposition: MetadataDisposition::Purge, + archive_retention: ArchiveRetention::Forever, + diagnostics_disposition: DiagnosticsDisposition::Purge, + diagnostics_retention_seconds: None, + }; + s.update_worker_retention_policy("w", 1, &u).unwrap(); + assert!(matches!( + s.begin_worker_removal("w", &p.plan_id, &p.input_fingerprint), + Err(WorkerRetentionError::StalePlan { .. }) + )); + let state: String = s + .with_conn(|conn| { + conn.query_row( + "SELECT state FROM worker_removal_operations WHERE plan_id=?1", + params![p.plan_id], + |row| row.get(0), + ) + .map_err(StoreError::from) + }) + .unwrap(); + assert_eq!(state, "stale"); + assert!(matches!( + s.begin_worker_removal("w", &p.plan_id, &p.input_fingerprint), + Err(WorkerRetentionError::StalePlan { .. }) + )); + } + #[test] + fn prepared_execution_is_derived_from_pinned_plan_generation() { + let s = setup(); + let plan = s.plan_worker_removal(&req(), &inv()).unwrap(); + let prepared = s + .prepare_worker_removal_execution("w", &plan.plan_id, &plan.input_fingerprint) + .unwrap(); + assert_eq!(prepared.runtime_request.expected_run_generation, 2); + assert_eq!( + prepared.runtime_request.session_disposition, + SessionDisposition::Archive + ); + assert_eq!(prepared.runtime_request.policy_revision, 1); + assert_eq!(prepared.runtime_request.worker_id, WorkerId::new(1)); + } + + #[test] + fn purge_tombstone_commit_is_idempotent() { + let s = setup(); + s.with_conn(|conn| { + conn.execute("INSERT INTO ticket_worker_assignments(workspace_id,ticket_id,assignment_id,runtime_id,worker_id,assigned_by,assigned_at) VALUES('w','ticket-old','assignment-old','r','1','test','t')", [])?; + Ok(()) + }).unwrap(); + let p = s.plan_worker_removal(&req(), &inv()).unwrap(); + s.begin_worker_removal("w", &p.plan_id, &p.input_fingerprint) + .unwrap(); + let result = WorkerRetentionExecutionResult { + operation_id: p.operation_id.clone(), + input_fingerprint: p.input_fingerprint.clone(), + worker_id: WorkerId::new(1), + session_disposition: p.session_disposition, + diagnostics_disposition: p.diagnostics_disposition, + archive: Some(worker_runtime::retention::WorkerSessionArchiveManifest { + schema_version: 1, + archive_id: p.archive_id.clone().unwrap(), + workspace_id: "w".into(), + source_runtime_id: "r".into(), + source_worker_id: WorkerId::new(1), + source_session_id: "s".into(), + segment_ids: vec!["a".into()], + source_created_at: "created".into(), + removed_at: "removed".into(), + archived_at_unix_seconds: 1, + effective_profile: None, + retention_class: None, + content_checksum_sha256: "sum".into(), + content_bytes: 1, + content_file_count: 1, + policy_id: p.policy_id.clone(), + policy_revision: p.policy_revision, + operation_id: p.operation_id.clone(), + input_fingerprint: p.input_fingerprint.clone(), + }), + source_removed: true, + diagnostics_retained: false, + }; + assert_eq!( + s.commit_worker_removal("w", &p.operation_id, &p.input_fingerprint, &result) + .unwrap() + .state, + WorkerRemovalPlanState::Succeeded + ); + assert!(s.worker_tombstone("w", &p.worker).unwrap().is_some()); + assert_eq!( + s.commit_worker_removal("w", &p.operation_id, &p.input_fingerprint, &result) + .unwrap() + .state, + WorkerRemovalPlanState::Succeeded + ); + let historical: i64 = s.with_conn(|conn| conn.query_row( + "SELECT COUNT(*) FROM ticket_worker_assignments WHERE workspace_id='w' AND assignment_id='assignment-old'", + [], + |row| row.get(0), + ).map_err(StoreError::from)).unwrap(); + assert_eq!(historical, 1); + } + #[test] + fn assignment_and_orphan_are_authoritative() { + let s = setup(); + s.with_conn(|c|{c.execute("INSERT INTO ticket_worker_assignments(workspace_id,ticket_id,assignment_id,runtime_id,worker_id,assigned_by,assigned_at)VALUES('w','ticket','assignment','r','1','test','t')",[])?;c.execute("INSERT INTO ticket_current_worker_assignments(workspace_id,ticket_id,assignment_id,runtime_id,worker_id,updated_at)VALUES('w','ticket','assignment','r','1','t')",[])?;Ok(())}).unwrap(); + let p = s.plan_worker_removal(&req(), &inv()).unwrap(); + assert!( + matches!(&p.blockers[..],[WorkerRemovalBlocker::CurrentAssignment{assignment_id,ticket_id}] if assignment_id=="assignment"&&ticket_id=="ticket") + ); + let d = WorkerOrphanDiagnostic { + diagnostic_id: "orphan".into(), + workspace_id: "w".into(), + runtime_id: "r".into(), + worker_id: "missing".into(), + category: "runtime_without_catalog".into(), + detail: "bounded diagnostic".into(), + observed_at: "t".into(), + }; + s.record_worker_orphan_diagnostic(&d).unwrap(); + let n: i64 = s + .with_conn(|c| { + c.query_row( + "SELECT COUNT(*) FROM worker_orphan_diagnostics WHERE diagnostic_id='orphan'", + [], + |r| r.get(0), + ) + .map_err(StoreError::from) + }) + .unwrap(); + assert_eq!(n, 1); + } + #[test] + fn concurrent_plan_converges_and_purge_omits_tombstone() { + let s = std::sync::Arc::new(setup()); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(3)); + let handles = (0..2) + .map(|_| { + let s = s.clone(); + let b = barrier.clone(); + std::thread::spawn(move || { + b.wait(); + s.plan_worker_removal(&req(), &inv()).unwrap() + }) + }) + .collect::<Vec<_>>(); + barrier.wait(); + let plans = handles + .into_iter() + .map(|h| h.join().unwrap()) + .collect::<Vec<_>>(); + assert_eq!(plans[0].plan_id, plans[1].plan_id); + let s = setup(); + let u = WorkerRetentionPolicyUpdate { + policy_id: "purge".into(), + session_disposition: SessionDisposition::Purge, + metadata_disposition: MetadataDisposition::Purge, + archive_retention: ArchiveRetention::Forever, + diagnostics_disposition: DiagnosticsDisposition::Purge, + diagnostics_retention_seconds: None, + }; + s.update_worker_retention_policy("w", 1, &u).unwrap(); + let p = s.plan_worker_removal(&req(), &inv()).unwrap(); + s.begin_worker_removal("w", &p.plan_id, &p.input_fingerprint) + .unwrap(); + let r = WorkerRetentionExecutionResult { + operation_id: p.operation_id.clone(), + input_fingerprint: p.input_fingerprint.clone(), + worker_id: WorkerId::new(1), + session_disposition: SessionDisposition::Purge, + diagnostics_disposition: DiagnosticsDisposition::Purge, + archive: None, + source_removed: true, + diagnostics_retained: false, + }; + s.commit_worker_removal("w", &p.operation_id, &p.input_fingerprint, &r) + .unwrap(); + assert!(s.worker_tombstone("w", &p.worker).unwrap().is_none()); + } + #[test] + fn commit_requires_executing_state_and_exact_runtime_result() { + let store = setup(); + let plan = store.plan_worker_removal(&req(), &inv()).unwrap(); + let mut result = WorkerRetentionExecutionResult { + operation_id: plan.operation_id.clone(), + input_fingerprint: plan.input_fingerprint.clone(), + worker_id: WorkerId::new(1), + session_disposition: plan.session_disposition, + diagnostics_disposition: plan.diagnostics_disposition, + archive: None, + source_removed: true, + diagnostics_retained: false, + }; + assert!(matches!( + store.commit_worker_removal("w", &plan.operation_id, &plan.input_fingerprint, &result), + Err(WorkerRetentionError::StalePlan { .. }) + )); + store + .begin_worker_removal("w", &plan.plan_id, &plan.input_fingerprint) + .unwrap(); + result.worker_id = WorkerId::new(2); + assert!( + store + .commit_worker_removal("w", &plan.operation_id, &plan.input_fingerprint, &result) + .is_err() + ); + let count: i64 = store.with_conn(|conn| conn.query_row( + "SELECT COUNT(*) FROM worker_registry WHERE workspace_id='w' AND runtime_id='r' AND runtime_worker_id=1", + [], + |row| row.get(0), + ).map_err(StoreError::from)).unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn execution_fence_blocks_stale_upsert_and_new_assignment() { + let store = setup(); + let plan = store.plan_worker_removal(&req(), &inv()).unwrap(); + store + .begin_worker_removal("w", &plan.plan_id, &plan.input_fingerprint) + .unwrap(); + let stale = WorkerRegistryRecord { + workspace_id: "w".into(), + worker: RuntimeWorkerRef { + runtime_id: "r".into(), + worker_id: "1".into(), + }, + display_name: "stale".into(), + profile: None, + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "created".into(), + updated_at: "rev2".into(), + }; + store.upsert_worker_registry(&stale).unwrap(); + let revision: String = store.with_conn(|conn| conn.query_row( + "SELECT updated_at FROM worker_registry WHERE workspace_id='w' AND runtime_id='r' AND runtime_worker_id=1", + [], + |row| row.get(0), + ).map_err(StoreError::from)).unwrap(); + assert_eq!(revision, "rev1"); + + let assignment = TicketWorkerAssignmentRecord { + workspace_id: "w".into(), + ticket_id: "new-ticket".into(), + assignment_id: "new-assignment".into(), + worker: RuntimeWorkerRef { + runtime_id: "r".into(), + worker_id: "1".into(), + }, + assigned_by: "test".into(), + assigned_at: "t".into(), + }; + assert!( + store + .set_current_ticket_worker_assignment( + &assignment, + None, + "event", + "assignment-operation", + false, + ) + .is_err() + ); + } + + #[test] + fn old_schema_upgrade_seeds_existing_workspace() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("server.db"); + { + let s = SqliteWorkspaceStore::open(&path).unwrap(); + s.with_conn(|c|{c.execute("INSERT INTO workspaces(workspace_id,display_name,state,created_at,updated_at)VALUES('legacy','Legacy','active','old','old')",[])?;c.execute_batch("DROP TRIGGER seed_worker_retention_policy_after_workspace_insert;DROP TABLE worker_retention_audit_events;DROP TABLE worker_tombstones;DROP TABLE worker_session_archives;DROP TABLE worker_diagnostics_archives;DROP TABLE worker_orphan_diagnostics;DROP TABLE worker_removal_operations;DROP TABLE workspace_worker_retention_policies;DROP TABLE workspace_worker_retention_policy_revisions;DELETE FROM __yoi_schema_migrations WHERE version=28;")?;Ok(())}).unwrap(); + } + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + let p = reopened.worker_retention_policy("legacy").unwrap().unwrap(); + assert_eq!(p.policy_id, CONSERVATIVE_POLICY_ID); + assert_eq!(p.session_disposition, SessionDisposition::Archive); + assert_eq!(p.metadata_disposition, MetadataDisposition::Tombstone); + } +} diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 92cdfd03..8b3786e8 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -156,6 +156,11 @@ const MIGRATIONS: &[Migration] = &[ name: "scope Repository identity and references by Workspace", apply: scope_repository_identity_by_workspace, }, + Migration { + version: 28, + name: "create Worker retention authority", + apply: crate::retention::create_worker_retention_tables, + }, ]; struct Migration { @@ -772,7 +777,7 @@ impl SqliteWorkspaceStore { }) } - fn with_conn<T>(&self, f: impl FnOnce(&Connection) -> Result<T>) -> Result<T> { + pub(crate) fn with_conn<T>(&self, f: impl FnOnce(&Connection) -> Result<T>) -> Result<T> { let conn = self .conn .lock() @@ -780,6 +785,17 @@ impl SqliteWorkspaceStore { f(&conn) } + pub(crate) fn with_conn_mut<T>( + &self, + f: impl FnOnce(&mut Connection) -> Result<T>, + ) -> Result<T> { + let mut conn = self + .conn + .lock() + .map_err(|_| Error::Store("sqlite connection lock poisoned".to_string()))?; + f(&mut conn) + } + pub fn upsert_trusted_runtime(&self, record: &TrustedRuntimeRecord) -> Result<()> { self.with_conn(|conn| { conn.execute( @@ -1919,6 +1935,23 @@ impl ControlPlaneStore for SqliteWorkspaceStore { fn upsert_worker_registry(&self, record: &WorkerRegistryRecord) -> Result<()> { self.with_conn(|conn| { + let removal_blocks_upsert: bool = conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM worker_removal_operations + WHERE workspace_id = ?1 AND runtime_id = ?2 + AND CAST(worker_id AS INTEGER) = ?3 + AND state IN ('executing', 'failed', 'succeeded') + )", + params![ + record.workspace_id, + record.worker.runtime_id, + record.worker.worker_id + ], + |row| row.get(0), + )?; + if removal_blocks_upsert { + return Ok(()); + } conn.execute( r#"INSERT INTO worker_registry ( workspace_id, runtime_id, runtime_worker_id, display_name, profile, @@ -1937,7 +1970,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore { session_ref = excluded.session_ref, summary_ref = excluded.summary_ref, diagnostics_ref = excluded.diagnostics_ref, - updated_at = excluded.updated_at"#, + updated_at = excluded.updated_at + WHERE NOT EXISTS ( + SELECT 1 FROM worker_removal_operations retention + WHERE retention.workspace_id = excluded.workspace_id + AND retention.runtime_id = excluded.runtime_id + AND CAST(retention.worker_id AS INTEGER) = excluded.runtime_worker_id + AND retention.state IN ('executing', 'failed', 'succeeded') + )"#, params![ record.workspace_id, record.worker.runtime_id, @@ -2006,7 +2046,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let changed = conn.execute( r#"UPDATE worker_registry SET retention_state = ?4, updated_at = ?5 - WHERE workspace_id = ?1 AND runtime_id = ?2 AND runtime_worker_id = ?3"#, + WHERE workspace_id = ?1 AND runtime_id = ?2 AND runtime_worker_id = ?3 + AND NOT EXISTS ( + SELECT 1 FROM worker_removal_operations retention + WHERE retention.workspace_id = ?1 AND retention.runtime_id = ?2 + AND CAST(retention.worker_id AS INTEGER) = ?3 + AND retention.state IN ('executing', 'failed') + )"#, params![ workspace_id, worker.runtime_id, @@ -2192,6 +2238,28 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result<TicketWorkerAssignmentUpdate> { self.with_conn(|conn| { let tx = conn.unchecked_transaction()?; + let removal_blocks_assignment: bool = tx.query_row( + "SELECT EXISTS( + SELECT 1 FROM worker_removal_operations + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + AND state IN ('executing', 'failed', 'succeeded') + UNION ALL + SELECT 1 FROM worker_tombstones + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + )", + params![ + record.workspace_id, + record.worker.runtime_id, + record.worker.worker_id + ], + |row| row.get(0), + )?; + if removal_blocks_assignment { + return Err(Error::TicketAssignmentConflict(format!( + "Worker {}/{} is being retained or has been removed", + record.worker.runtime_id, record.worker.worker_id + ))); + } let mut reserved_operation = false; if let Some(existing) = read_assignment_operation(&tx, &record.workspace_id, operation_id)? @@ -4847,7 +4915,7 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 27); + assert_eq!(current_schema_version(&conn).unwrap(), 28); assert!(table_exists(&conn, "worker_workdir_attachment_reservations").unwrap()); } @@ -4880,7 +4948,7 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 27); + assert_eq!(current_schema_version(&conn).unwrap(), 28); assert!(table_exists(&conn, "flow_sources").unwrap()); assert!(table_exists(&conn, "flow_source_revisions").unwrap()); assert!(!table_exists(&conn, "flow_instances").unwrap()); @@ -4947,7 +5015,7 @@ INSERT INTO worker_workdir_attachment_reservations ( apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 27); + assert_eq!(current_schema_version(&conn).unwrap(), 28); let repositories_sql: String = conn .query_row( "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'repositories'", @@ -5127,7 +5195,7 @@ INSERT INTO workdir_registry ( let db = dir.path().join("control-plane.sqlite"); let store = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 27); + assert_eq!(store.schema_version().await.unwrap(), 28); assert!( !store .with_conn(|conn| table_exists(conn, "worker_workspace_credentials")) @@ -5144,7 +5212,7 @@ INSERT INTO workdir_registry ( store.upsert_workspace(&record).await.unwrap(); let reopened = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(reopened.schema_version().await.unwrap(), 27); + assert_eq!(reopened.schema_version().await.unwrap(), 28); assert_eq!( reopened.get_workspace("local-dev").await.unwrap(), Some(record) @@ -5691,7 +5759,7 @@ INSERT INTO workdir_registry ( .unwrap(); let store = SqliteWorkspaceStore::from_connection(conn).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 27); + assert_eq!(store.schema_version().await.unwrap(), 28); store .with_conn(|conn| { @@ -5880,7 +5948,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn repository_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 27); + assert_eq!(store.schema_version().await.unwrap(), 28); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -5946,7 +6014,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn memory_authority_records_round_trip_and_close_staging() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 27); + assert_eq!(store.schema_version().await.unwrap(), 28); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -6209,7 +6277,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn account_and_login_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 27); + assert_eq!(store.schema_version().await.unwrap(), 28); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { account_id: "acct-user-alice".to_string(), From 5e5ce73fd0e4c738552be83963a6e43071398815 Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Wed, 12 Aug 2026 00:46:39 +0900 Subject: [PATCH 06/12] runtime: harden retention reconciliation and retry --- crates/worker-runtime/src/retention.rs | 450 ++++++++++++++++++++--- crates/worker-runtime/src/runtime.rs | 29 +- crates/workspace-server/src/retention.rs | 217 +++++++++-- 3 files changed, 619 insertions(+), 77 deletions(-) diff --git a/crates/worker-runtime/src/retention.rs b/crates/worker-runtime/src/retention.rs index e77a8372..699f660d 100644 --- a/crates/worker-runtime/src/retention.rs +++ b/crates/worker-runtime/src/retention.rs @@ -46,6 +46,69 @@ pub struct WorkerRetentionInventory { pub diagnostics_bytes: u64, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RuntimeWorkerAggregateDiagnostic { + worker_id: String, + category: String, + detail: String, +} + +impl RuntimeWorkerAggregateDiagnostic { + pub fn worker_id(&self) -> &str { + &self.worker_id + } + + pub fn category(&self) -> &str { + &self.category + } + + pub fn detail(&self) -> &str { + &self.detail + } +} + +/// Opaque host-derived inventory snapshot. Callers can inspect but cannot +/// construct or alter the Runtime/Workspace scope used by reconciliation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct WorkerRetentionInventorySnapshot { + workspace_id: String, + runtime_id: String, + workers: Vec<WorkerRetentionInventory>, + diagnostics: Vec<RuntimeWorkerAggregateDiagnostic>, +} + +impl WorkerRetentionInventorySnapshot { + pub(crate) fn new( + workspace_id: String, + runtime_id: String, + workers: Vec<WorkerRetentionInventory>, + diagnostics: Vec<RuntimeWorkerAggregateDiagnostic>, + ) -> Self { + Self { + workspace_id, + runtime_id, + workers, + diagnostics, + } + } + + pub fn workspace_id(&self) -> &str { + &self.workspace_id + } + + pub fn runtime_id(&self) -> &str { + &self.runtime_id + } + + pub fn workers(&self) -> &[WorkerRetentionInventory] { + &self.workers + } + + pub fn diagnostics(&self) -> &[RuntimeWorkerAggregateDiagnostic] { + &self.diagnostics + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkerRetentionExecutionRequest { pub operation_id: String, @@ -113,12 +176,6 @@ pub(crate) trait WorkerRetentionProvider: Send + Sync { &self, request: &WorkerRetentionExecutionRequest, ) -> Result<WorkerRetentionExecutionResult, RuntimeError>; - - fn completed( - &self, - operation_id: &str, - input_fingerprint: &str, - ) -> Result<Option<WorkerRetentionExecutionResult>, RuntimeError>; } /// Filesystem provider for the canonical Runtime Worker aggregate. @@ -134,6 +191,137 @@ impl FsWorkerRetentionProvider { } } + pub(crate) fn completed_for( + &self, + request: &WorkerRetentionExecutionRequest, + ) -> Result<Option<WorkerRetentionExecutionResult>, RuntimeError> { + let path = self.operation_path(&request.operation_id)?; + let Some(receipt) = read_operation_receipt(&path)? else { + return Ok(None); + }; + validate_receipt_request(&receipt, request)?; + Ok(receipt.result.source_removed.then_some(receipt.result)) + } + + pub(crate) fn snapshot( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result<WorkerRetentionInventorySnapshot, RuntimeError> { + fs::create_dir_all(&self.runtime_root).map_err(|source| RuntimeError::StoreIo { + operation: "prepare Worker retention inventory", + path: self.runtime_root.clone(), + source, + })?; + let lock_path = self.runtime_root.join(RETENTION_LOCK); + let lock = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(&lock_path) + .map_err(|source| RuntimeError::StoreIo { + operation: "lock Worker retention inventory", + path: lock_path.clone(), + source, + })?; + lock.lock().map_err(|source| RuntimeError::StoreIo { + operation: "lock Worker retention inventory", + path: lock_path, + source, + })?; + let workers_root = self.runtime_root.join("workers"); + if !workers_root.is_dir() { + return Ok(WorkerRetentionInventorySnapshot::new( + workspace_id.to_string(), + runtime_id.to_string(), + Vec::new(), + Vec::new(), + )); + } + let mut entries = fs::read_dir(&workers_root) + .map_err(|source| RuntimeError::StoreIo { + operation: "scan Worker retention inventory", + path: workers_root.clone(), + source, + })? + .collect::<Result<Vec<_>, _>>() + .map_err(|source| RuntimeError::StoreIo { + operation: "scan Worker retention inventory", + path: workers_root.clone(), + source, + })?; + entries.sort_by_key(|entry| entry.file_name()); + let mut workers = Vec::new(); + let mut diagnostics = Vec::new(); + for entry in entries { + let raw_id = entry.file_name().to_string_lossy().to_string(); + let bounded_id = bounded_diagnostic_id(&raw_id); + let file_type = match entry.file_type() { + Ok(file_type) => file_type, + Err(_) => { + diagnostics.push(runtime_aggregate_diagnostic( + &bounded_id, + "aggregate_metadata_unreadable", + )); + continue; + } + }; + if !file_type.is_dir() || file_type.is_symlink() { + diagnostics.push(runtime_aggregate_diagnostic( + &bounded_id, + "aggregate_entry_unsupported", + )); + continue; + } + let Ok(worker_number) = raw_id.parse::<u64>() else { + diagnostics.push(runtime_aggregate_diagnostic( + &bounded_id, + "aggregate_worker_id_invalid", + )); + continue; + }; + let worker_id = WorkerId::new(worker_number); + let worker_dir = self.worker_dir(worker_id); + let snapshot: WorkerGenerationSnapshot = match read_json( + &worker_dir.join("worker.json"), + "scan Worker retention inventory", + ) { + Ok(snapshot) => snapshot, + Err(_) => { + diagnostics.push(runtime_aggregate_diagnostic( + &bounded_id, + "aggregate_worker_record_corrupt", + )); + continue; + } + }; + if snapshot.workspace_id.as_deref() != Some(workspace_id) { + diagnostics.push(runtime_aggregate_diagnostic( + &bounded_id, + "aggregate_workspace_mismatch", + )); + continue; + } + match self.inventory(workspace_id, runtime_id, worker_id, snapshot.run_generation) { + Ok(item) => workers.push(item), + Err(_) => diagnostics.push(runtime_aggregate_diagnostic( + &bounded_id, + "aggregate_session_inventory_failed", + )), + } + } + workers.sort_by_key(|item| item.worker_id); + diagnostics.sort_by(|left, right| { + (&left.worker_id, &left.category).cmp(&(&right.worker_id, &right.category)) + }); + Ok(WorkerRetentionInventorySnapshot::new( + workspace_id.to_string(), + runtime_id.to_string(), + workers, + diagnostics, + )) + } + pub(crate) fn recover_after_source_removal( &self, request: &WorkerRetentionExecutionRequest, @@ -192,6 +380,19 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider { if !worker_dir.is_dir() { return Err(RuntimeError::WorkerNotFound { worker_id }); } + let worker: WorkerGenerationSnapshot = read_json( + &worker_dir.join("worker.json"), + "inventory Worker retention", + )?; + if worker.workspace_id.as_deref() != Some(workspace_id) { + return Err(RuntimeError::WorkerNotFound { worker_id }); + } + if worker.run_generation != run_generation { + return Err(RuntimeError::InvalidRequest(format!( + "Worker retention inventory expected generation {run_generation}, current generation is {}", + worker.run_generation + ))); + } let session_dir = worker_dir.join("session"); let (session_id, segment_ids, session_bytes) = if session_dir.is_dir() { let manifest: CanonicalSessionManifest = read_json( @@ -264,11 +465,9 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider { source, })?; - let pending = read_operation_receipt( - &self.operation_path(&request.operation_id)?, - &request.input_fingerprint, - )?; + let pending = read_operation_receipt(&self.operation_path(&request.operation_id)?)?; if let Some(receipt) = &pending { + validate_receipt_request(receipt, request)?; if receipt.result.source_removed { return Ok(receipt.result.clone()); } @@ -293,6 +492,11 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider { } let snapshot: WorkerGenerationSnapshot = read_json(&worker_dir.join("worker.json"), "execute Worker retention")?; + if snapshot.workspace_id.as_deref() != Some(request.workspace_id.as_str()) { + return Err(RuntimeError::WorkerNotFound { + worker_id: request.worker_id, + }); + } if snapshot.run_generation != request.expected_run_generation { return Err(RuntimeError::InvalidRequest(format!( "Worker retention plan expected generation {}, current generation is {}", @@ -326,6 +530,7 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider { }; let mut receipt = RetentionOperationReceipt { schema_version: OPERATION_SCHEMA_VERSION, + request: request.clone(), result: result.clone(), }; // Pending receipt makes the delete/final-receipt crash window @@ -355,22 +560,12 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider { )?; Ok(result) } - - fn completed( - &self, - operation_id: &str, - input_fingerprint: &str, - ) -> Result<Option<WorkerRetentionExecutionResult>, RuntimeError> { - let path = self.operation_path(operation_id)?; - let Some(receipt) = read_operation_receipt(&path, input_fingerprint)? else { - return Ok(None); - }; - Ok(receipt.result.source_removed.then_some(receipt.result)) - } } #[derive(Deserialize)] struct WorkerGenerationSnapshot { + #[serde(default)] + workspace_id: Option<String>, #[serde(default)] run_generation: u64, } @@ -383,13 +578,11 @@ struct CanonicalSessionManifest { #[derive(Serialize, Deserialize)] struct RetentionOperationReceipt { schema_version: u32, + request: WorkerRetentionExecutionRequest, result: WorkerRetentionExecutionResult, } -fn read_operation_receipt( - path: &Path, - input_fingerprint: &str, -) -> Result<Option<RetentionOperationReceipt>, RuntimeError> { +fn read_operation_receipt(path: &Path) -> Result<Option<RetentionOperationReceipt>, RuntimeError> { if !path.is_file() { return Ok(None); } @@ -404,16 +597,26 @@ fn read_operation_receipt( ), }); } - if receipt.result.input_fingerprint != input_fingerprint { - return Err(RuntimeError::InvalidRequest(format!( - "retention operation {} was already used with different input", - receipt.result.operation_id - ))); - } Ok(Some(receipt)) } -#[derive(Serialize, Deserialize)] +fn validate_receipt_request( + receipt: &RetentionOperationReceipt, + request: &WorkerRetentionExecutionRequest, +) -> Result<(), RuntimeError> { + if &receipt.request != request + || receipt.result.operation_id != request.operation_id + || receipt.result.input_fingerprint != request.input_fingerprint + { + return Err(RuntimeError::InvalidRequest(format!( + "retention operation {} was already used with different input", + request.operation_id + ))); + } + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] struct DiagnosticsArchiveManifest { schema_version: u32, operation_id: String, @@ -426,6 +629,36 @@ struct DiagnosticsArchiveManifest { content_file_count: u64, } +fn runtime_aggregate_diagnostic( + worker_id: &str, + category: &str, +) -> RuntimeWorkerAggregateDiagnostic { + RuntimeWorkerAggregateDiagnostic { + worker_id: worker_id.to_string(), + category: category.to_string(), + detail: "Canonical Runtime Worker aggregate requires diagnostic reconciliation".to_string(), + } +} + +fn bounded_diagnostic_id(value: &str) -> String { + let sanitized = value + .chars() + .take(64) + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.') { + character + } else { + '_' + } + }) + .collect::<String>(); + if sanitized.is_empty() { + "unknown".to_string() + } else { + sanitized + } +} + fn validate_request(request: &WorkerRetentionExecutionRequest) -> Result<(), RuntimeError> { validate_id("operation_id", &request.operation_id)?; validate_id("workspace_id", &request.workspace_id)?; @@ -611,16 +844,7 @@ fn commit_diagnostics_archive( let files = diagnostics_files(worker_dir, "archive Worker diagnostics")?; let target = provider.diagnostics_dir(&request.operation_id)?; if target.exists() { - let manifest: DiagnosticsArchiveManifest = read_json( - &target.join("manifest.json"), - "verify Worker diagnostics archive", - )?; - if manifest.input_fingerprint != request.input_fingerprint { - return Err(RuntimeError::InvalidRequest(format!( - "diagnostics archive operation {} was reused with different input", - request.operation_id - ))); - } + validate_existing_diagnostics_archive(&target, request)?; return Ok(()); } let parent = target.parent().ok_or_else(|| RuntimeError::StoreCorrupt { @@ -677,7 +901,8 @@ fn commit_diagnostics_archive( path: target.clone(), source, })?; - sync_directory(parent, "archive Worker diagnostics") + sync_directory(parent, "archive Worker diagnostics")?; + validate_existing_diagnostics_archive(&target, request) })(); if result.is_err() { let _ = fs::remove_dir_all(&staging); @@ -685,6 +910,42 @@ fn commit_diagnostics_archive( result } +fn validate_existing_diagnostics_archive( + target: &Path, + request: &WorkerRetentionExecutionRequest, +) -> Result<(), RuntimeError> { + let manifest: DiagnosticsArchiveManifest = read_json( + &target.join("manifest.json"), + "verify Worker diagnostics archive", + )?; + if manifest.schema_version != ARCHIVE_SCHEMA_VERSION + || manifest.operation_id != request.operation_id + || manifest.workspace_id != request.workspace_id + || manifest.source_runtime_id != request.source_runtime_id + || manifest.source_worker_id != request.worker_id + || manifest.input_fingerprint != request.input_fingerprint + { + return Err(RuntimeError::InvalidRequest(format!( + "diagnostics archive operation {} does not match the retention request", + request.operation_id + ))); + } + let mut files = collect_files(target, "verify Worker diagnostics archive")?; + files.retain(|(relative, _)| relative != Path::new("manifest.json")); + let (checksum, bytes, count) = checksum_files(&files, "verify Worker diagnostics archive")?; + if checksum != manifest.content_checksum_sha256 + || bytes != manifest.content_bytes + || count != manifest.content_file_count + { + return Err(RuntimeError::StoreCorrupt { + operation: "verify Worker diagnostics archive", + path: target.to_path_buf(), + message: "diagnostics archive checksum or content summary mismatch".to_string(), + }); + } + Ok(()) +} + fn diagnostics_files( worker_dir: &Path, operation: &'static str, @@ -1001,7 +1262,7 @@ mod tests { let worker = root.join("workers").join(worker_id.to_string()); write_json( &worker.join("worker.json"), - &serde_json::json!({"run_generation": generation}), + &serde_json::json!({"workspace_id": "workspace-a", "run_generation": generation}), ); write_json( &worker.join("session/session.json"), @@ -1102,6 +1363,36 @@ mod tests { ); } + #[test] + fn target_inventory_and_execute_reject_cross_workspace_aggregate() { + let temp = tempfile::tempdir().unwrap(); + let worker_id = WorkerId::new(16); + source(temp.path(), worker_id, 3); + let provider = FsWorkerRetentionProvider::new(temp.path()); + assert!(matches!( + provider.inventory("other-workspace", "runtime-a", worker_id, 3), + Err(RuntimeError::WorkerNotFound { .. }) + )); + let mut request = request(worker_id, 3, SessionDisposition::Purge); + request.workspace_id = "other-workspace".to_string(); + assert!(matches!( + provider.execute(&request), + Err(RuntimeError::WorkerNotFound { .. }) + )); + assert!(temp.path().join("workers/16/session").is_dir()); + assert!( + !temp + .path() + .join("retention/operations/operation-a.json") + .exists() + ); + + request.workspace_id = "workspace-a".to_string(); + provider.execute(&request).unwrap(); + request.workspace_id = "other-workspace".to_string(); + assert!(provider.execute(&request).is_err()); + } + #[test] fn purge_removes_aggregate_and_rejects_stale_generation() { let temp = tempfile::tempdir().unwrap(); @@ -1141,12 +1432,71 @@ mod tests { let recovered = provider.execute(&request).unwrap(); assert_eq!(recovered, completed); - assert!( - provider - .completed("operation-a", "fingerprint-a") - .unwrap() - .is_some() + assert!(provider.completed_for(&request).unwrap().is_some()); + } + + #[test] + fn provider_snapshot_scans_aggregate_storage_independent_of_runtime_catalog() { + let temp = tempfile::tempdir().unwrap(); + source(temp.path(), WorkerId::new(13), 2); + source(temp.path(), WorkerId::new(14), 1); + write_json( + &temp.path().join("workers/14/worker.json"), + &serde_json::json!({"workspace_id": "other-workspace", "run_generation": 1}), ); + fs::create_dir_all(temp.path().join("workers/not-a-worker")).unwrap(); + fs::write( + temp.path().join("workers/not-a-worker/worker.json"), + b"not-json", + ) + .unwrap(); + fs::create_dir_all(temp.path().join("workers/15")).unwrap(); + fs::write(temp.path().join("workers/15/worker.json"), b"not-json").unwrap(); + let provider = FsWorkerRetentionProvider::new(temp.path()); + + let snapshot = provider.snapshot("workspace-a", "runtime-a").unwrap(); + assert_eq!(snapshot.workers().len(), 1); + assert_eq!(snapshot.workers()[0].worker_id, WorkerId::new(13)); + assert!(snapshot.diagnostics().iter().any(|diagnostic| { + diagnostic.worker_id() == "14" + && diagnostic.category() == "aggregate_workspace_mismatch" + })); + assert!(snapshot.diagnostics().iter().any(|diagnostic| { + diagnostic.worker_id() == "not-a-worker" + && diagnostic.category() == "aggregate_worker_id_invalid" + })); + assert!(snapshot.diagnostics().iter().any(|diagnostic| { + diagnostic.worker_id() == "15" + && diagnostic.category() == "aggregate_worker_record_corrupt" + })); + } + + #[test] + fn diagnostics_retry_rejects_corrupt_existing_archive_before_source_delete() { + let temp = tempfile::tempdir().unwrap(); + let worker_id = WorkerId::new(12); + source(temp.path(), worker_id, 1); + let provider = FsWorkerRetentionProvider::new(temp.path()); + let mut request = request(worker_id, 1, SessionDisposition::Archive); + request.diagnostics_disposition = DiagnosticsDisposition::Retain; + provider.execute(&request).unwrap(); + + let receipt_path = temp.path().join("retention/operations/operation-a.json"); + let mut receipt: RetentionOperationReceipt = + serde_json::from_slice(&fs::read(&receipt_path).unwrap()).unwrap(); + receipt.result.source_removed = false; + fs::write(&receipt_path, serde_json::to_vec_pretty(&receipt).unwrap()).unwrap(); + source(temp.path(), worker_id, 1); + fs::write( + temp.path() + .join("archives/diagnostics/operation-a/runs/1/worker.out.log"), + b"corrupt\n", + ) + .unwrap(); + + assert!(provider.execute(&request).is_err()); + assert!(temp.path().join("workers/12/session").is_dir()); + assert!(provider.completed_for(&request).unwrap().is_none()); } #[test] diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index e93a421d..2c9a4961 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -29,7 +29,7 @@ use crate::observation::{WorkerObservationCursor, WorkerObservationEvent}; #[cfg(feature = "fs-store")] use crate::retention::{ FsWorkerRetentionProvider, WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, - WorkerRetentionInventory, WorkerRetentionProvider, + WorkerRetentionInventory, WorkerRetentionInventorySnapshot, WorkerRetentionProvider, }; use protocol::subscription::{ EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot, @@ -1706,6 +1706,29 @@ impl Runtime { ) } + /// Enumerate host-authoritative Runtime inventory for Backend orphan + /// reconciliation. Runtime identity and Workspace scope are derived here, + /// not accepted in a diagnostic payload. + #[cfg(feature = "fs-store")] + pub fn list_worker_retention_inventory( + &self, + workspace_id: &str, + ) -> Result<WorkerRetentionInventorySnapshot, RuntimeError> { + let state = self.lock()?; + let runtime_id = state.runtime_identity.as_deref().ok_or_else(|| { + RuntimeError::InvalidRequest( + "Runtime identity is not bound for Worker retention".to_string(), + ) + })?; + let store = state.fs_store().ok_or_else(|| { + RuntimeError::InvalidRequest( + "Worker retention inventory requires an fs-backed Runtime".to_string(), + ) + })?; + let provider = FsWorkerRetentionProvider::new(store.runtime_dir()); + provider.snapshot(workspace_id, runtime_id) + } + /// Execute a Backend-resolved retention plan. Only stopped Workers are /// eligible. Provider receipt lookup happens before live lookup so exact /// retries converge after aggregate removal. @@ -1731,9 +1754,7 @@ impl Runtime { ) })?; let provider = FsWorkerRetentionProvider::new(store.runtime_dir()); - if let Some(completed) = - provider.completed(&request.operation_id, &request.input_fingerprint)? - { + if let Some(completed) = provider.completed_for(request)? { state.workers.remove(&request.worker_id); state.persist_runtime_snapshot()?; return Ok(completed); diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index b3b21ff6..561d6e48 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -7,10 +7,12 @@ use chrono::Utc; use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; use worker_runtime::identity::RuntimeWorkerRef; use worker_runtime::retention::{ - DiagnosticsDisposition, SessionDisposition, WorkerRetentionExecutionRequest, - WorkerRetentionExecutionResult, WorkerRetentionInventory, + DiagnosticsDisposition, RuntimeWorkerAggregateDiagnostic, SessionDisposition, + WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory, + WorkerRetentionInventorySnapshot, }; pub const CONSERVATIVE_POLICY_ID: &str = "workspace-default-conservative"; @@ -366,7 +368,7 @@ impl SqliteWorkspaceStore { "Runtime Worker id is not a canonical unsigned integer".to_string(), ) })?; - let removed_at = Utc::now().to_rfc3339(); + let removed_at = plan.created_at.clone(); Ok(PreparedWorkerRemoval { runtime_request: WorkerRetentionExecutionRequest { operation_id: plan.operation_id.clone(), @@ -462,14 +464,109 @@ impl SqliteWorkspaceStore { }).map_err(map_error) } - pub fn record_worker_orphan_diagnostic( + /// Compare trusted Runtime inventory with the Backend worker registry and + /// persist bounded diagnostics for both orphan directions. This operation + /// is diagnostic-only: a Runtime aggregate without Backend authority is + /// never assigned an implicit purge disposition. + pub fn reconcile_worker_retention_inventory( &self, - d: &WorkerOrphanDiagnostic, - ) -> Result<(), WorkerRetentionError> { - bounded("orphan category", &d.category, 160)?; - bounded("orphan detail", &d.detail, 2000)?; - self.with_conn(|conn|{conn.execute("INSERT OR IGNORE INTO worker_orphan_diagnostics(diagnostic_id,workspace_id,runtime_id,worker_id,category,detail,observed_at) VALUES(?1,?2,?3,?4,?5,?6,?7)",params![d.diagnostic_id,d.workspace_id,d.runtime_id,d.worker_id,d.category,d.detail,d.observed_at])?;Ok(())})?; - Ok(()) + snapshot: &WorkerRetentionInventorySnapshot, + ) -> Result<Vec<WorkerOrphanDiagnostic>, WorkerRetentionError> { + self.reconcile_worker_retention_inventory_parts( + snapshot.workspace_id(), + snapshot.runtime_id(), + snapshot.workers(), + snapshot.diagnostics(), + ) + } + + fn reconcile_worker_retention_inventory_parts( + &self, + workspace_id: &str, + runtime_id: &str, + inventory: &[WorkerRetentionInventory], + runtime_diagnostics: &[RuntimeWorkerAggregateDiagnostic], + ) -> Result<Vec<WorkerOrphanDiagnostic>, WorkerRetentionError> { + bounded("Workspace id", workspace_id, 160)?; + bounded("Runtime id", runtime_id, 160)?; + let mut runtime_workers = BTreeMap::new(); + for item in inventory { + if item.workspace_id != workspace_id || item.runtime_id != runtime_id { + return Err(WorkerRetentionError::CrossWorkspace); + } + let worker_id = item.worker_id.to_string(); + if runtime_workers.insert(worker_id.clone(), item).is_some() { + return Err(WorkerRetentionError::Invalid(format!( + "duplicate Runtime inventory for Worker {worker_id}" + ))); + } + } + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let policy_configured = load_policy(&tx, workspace_id)?.is_some(); + let mut statement = tx.prepare( + "SELECT CAST(runtime_worker_id AS TEXT), retention_state + FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2", + )?; + let registry = statement + .query_map(params![workspace_id, runtime_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::<Result<BTreeMap<_, _>, _>>()?; + drop(statement); + let runtime_ids = runtime_workers.keys().cloned().collect::<BTreeSet<_>>(); + let registry_ids = registry.keys().cloned().collect::<BTreeSet<_>>(); + let observed_at = Utc::now().to_rfc3339(); + let mut diagnostics = runtime_diagnostics + .iter() + .map(|diagnostic| { + orphan_diagnostic( + workspace_id, + runtime_id, + diagnostic.worker_id(), + diagnostic.category(), + diagnostic.detail(), + &observed_at, + ) + }) + .collect::<Vec<_>>(); + for worker_id in runtime_ids.difference(®istry_ids) { + let category = if policy_configured { + "runtime_aggregate_without_backend_registry" + } else { + "runtime_aggregate_policy_missing_fail_closed" + }; + diagnostics.push(orphan_diagnostic( + workspace_id, + runtime_id, + worker_id, + category, + "Runtime canonical aggregate is absent from Backend worker_registry; removal is blocked pending reconciliation", + &observed_at, + )); + } + for worker_id in registry_ids.difference(&runtime_ids) { + let category = if registry.get(worker_id).map(String::as_str) == Some("pinned") { + "backend_registry_without_runtime_aggregate_pinned" + } else { + "backend_registry_without_runtime_aggregate" + }; + diagnostics.push(orphan_diagnostic( + workspace_id, + runtime_id, + worker_id, + category, + "Backend worker_registry record has no Runtime canonical aggregate", + &observed_at, + )); + } + for diagnostic in &diagnostics { + insert_orphan_diagnostic(&tx, diagnostic)?; + } + tx.commit()?; + Ok(diagnostics) + }) + .map_err(WorkerRetentionError::Store) } pub fn worker_tombstone( @@ -481,6 +578,49 @@ impl SqliteWorkspaceStore { } } +fn orphan_diagnostic( + workspace_id: &str, + runtime_id: &str, + worker_id: &str, + category: &str, + detail: &str, + observed_at: &str, +) -> WorkerOrphanDiagnostic { + let identity = format!("{workspace_id}\0{runtime_id}\0{worker_id}\0{category}"); + WorkerOrphanDiagnostic { + diagnostic_id: stable("wod", &identity), + workspace_id: workspace_id.to_string(), + runtime_id: runtime_id.to_string(), + worker_id: worker_id.to_string(), + category: category.to_string(), + detail: detail.to_string(), + observed_at: observed_at.to_string(), + } +} + +fn insert_orphan_diagnostic( + conn: &Connection, + diagnostic: &WorkerOrphanDiagnostic, +) -> crate::Result<()> { + conn.execute( + "INSERT INTO worker_orphan_diagnostics + (diagnostic_id,workspace_id,runtime_id,worker_id,category,detail,observed_at) + VALUES(?1,?2,?3,?4,?5,?6,?7) + ON CONFLICT(diagnostic_id) DO UPDATE SET + detail=excluded.detail, observed_at=excluded.observed_at", + params![ + diagnostic.diagnostic_id, + diagnostic.workspace_id, + diagnostic.runtime_id, + diagnostic.worker_id, + diagnostic.category, + diagnostic.detail, + diagnostic.observed_at + ], + )?; + Ok(()) +} + #[derive(Clone)] struct WorkerRow { display_name: String, @@ -948,6 +1088,10 @@ mod tests { ); assert_eq!(prepared.runtime_request.policy_revision, 1); assert_eq!(prepared.runtime_request.worker_id, WorkerId::new(1)); + let retry = s + .prepare_worker_removal_execution("w", &plan.plan_id, &plan.input_fingerprint) + .unwrap(); + assert_eq!(retry.runtime_request, prepared.runtime_request); } #[test] @@ -1018,27 +1162,54 @@ mod tests { assert!( matches!(&p.blockers[..],[WorkerRemovalBlocker::CurrentAssignment{assignment_id,ticket_id}] if assignment_id=="assignment"&&ticket_id=="ticket") ); - let d = WorkerOrphanDiagnostic { - diagnostic_id: "orphan".into(), + let runtime_only = WorkerRetentionInventory { workspace_id: "w".into(), runtime_id: "r".into(), - worker_id: "missing".into(), - category: "runtime_without_catalog".into(), - detail: "bounded diagnostic".into(), - observed_at: "t".into(), + worker_id: WorkerId::new(2), + run_generation: 1, + session_id: Some("orphan-session".into()), + segment_ids: vec![], + session_bytes: 10, + diagnostics_bytes: 0, }; - s.record_worker_orphan_diagnostic(&d).unwrap(); - let n: i64 = s - .with_conn(|c| { - c.query_row( - "SELECT COUNT(*) FROM worker_orphan_diagnostics WHERE diagnostic_id='orphan'", + let diagnostics = s + .reconcile_worker_retention_inventory_parts("w", "r", &[runtime_only], &[]) + .unwrap(); + assert_eq!(diagnostics.len(), 2); + assert!(diagnostics.iter().any(|item| { + item.worker_id == "2" && item.category == "runtime_aggregate_without_backend_registry" + })); + assert!(diagnostics.iter().any(|item| { + item.worker_id == "1" && item.category == "backend_registry_without_runtime_aggregate" + })); + let count: i64 = s + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM worker_orphan_diagnostics WHERE workspace_id='w' AND runtime_id='r'", [], - |r| r.get(0), + |row| row.get(0), ) .map_err(StoreError::from) }) .unwrap(); - assert_eq!(n, 1); + assert_eq!(count, 2); + let mut wrong_scope = inv(); + wrong_scope.workspace_id = "other".into(); + assert!(matches!( + s.reconcile_worker_retention_inventory_parts("w", "r", &[wrong_scope], &[]), + Err(WorkerRetentionError::CrossWorkspace) + )); + let unchanged: i64 = s + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM worker_orphan_diagnostics WHERE workspace_id='w' AND runtime_id='r'", + [], + |row| row.get(0), + ) + .map_err(StoreError::from) + }) + .unwrap(); + assert_eq!(unchanged, 2); } #[test] fn concurrent_plan_converges_and_purge_omits_tombstone() { From 8cc0aaf8d2c048d7cc4c0bae374e253b6549444b Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Wed, 12 Aug 2026 04:02:47 +0900 Subject: [PATCH 07/12] runtime: prove Worker mutation source authority --- crates/worker-runtime/src/auth.rs | 315 +++++++++ crates/worker-runtime/src/catalog.rs | 31 +- crates/worker-runtime/src/execution.rs | 2 + crates/worker-runtime/src/http_server.rs | 2 - crates/worker-runtime/src/lib.rs | 1 + crates/worker-runtime/src/main.rs | 3 + crates/worker-runtime/src/runtime.rs | 28 +- crates/worker-runtime/src/worker_backend.rs | 181 ++++-- crates/worker-runtime/src/worker_source.rs | 613 ++++++++++++++++++ crates/worker/src/feature/builtin/memory.rs | 4 +- .../worker/src/feature/builtin/objective.rs | 7 +- crates/worker/src/feature/builtin/ticket.rs | 18 +- crates/worker/src/lib.rs | 8 +- crates/worker/src/skill.rs | 11 +- crates/worker/src/worker.rs | 147 +---- crates/workspace-server/src/hosts.rs | 3 +- crates/workspace-server/src/lib.rs | 1 + crates/workspace-server/src/retention.rs | 2 +- crates/workspace-server/src/server.rs | 470 +++++++++++++- crates/workspace-server/src/store.rs | 132 +++- crates/workspace-server/src/worker_source.rs | 288 ++++++++ 21 files changed, 2037 insertions(+), 230 deletions(-) create mode 100644 crates/worker-runtime/src/worker_source.rs create mode 100644 crates/workspace-server/src/worker_source.rs diff --git a/crates/worker-runtime/src/auth.rs b/crates/worker-runtime/src/auth.rs index 3c90b176..15dc0fb2 100644 --- a/crates/worker-runtime/src/auth.rs +++ b/crates/worker-runtime/src/auth.rs @@ -10,6 +10,10 @@ const PUBLIC_KEY_PREFIX: &str = "yoi-ed25519-pub:v1:"; const PRIVATE_KEY_PREFIX: &str = "yoi-ed25519-pkcs8:v1:"; const TOKEN_PREFIX: &str = "yoi-cap-v1"; const SIGNING_INPUT_PREFIX: &str = "yoi-cap-v1."; +pub const WORKER_MUTATION_SOURCE_PROOF_HEADER: &str = "x-yoi-worker-mutation-proof"; +const WORKER_MUTATION_SOURCE_PROOF_PREFIX: &str = "yoi-worker-source-v1"; +const WORKER_MUTATION_SOURCE_SIGNING_INPUT_PREFIX: &str = "yoi-worker-source-v1."; +pub const WORKER_REMOVE_PERMISSION: &str = "workspace:worker-remove"; #[derive(Debug, thiserror::Error)] pub enum RuntimeAuthError { @@ -41,6 +45,16 @@ pub enum RuntimeAuthError { MissingWorkspaceScope, #[error("capability token is missing required permission `{0}`")] MissingPermission(String), + #[error("source proof workspace `{actual}` does not match `{expected}`")] + WrongWorkspace { expected: String, actual: String }, + #[error("source proof Worker `{actual}` does not match `{expected}`")] + WrongWorker { expected: String, actual: String }, + #[error("source proof actor kind is not allowed")] + WrongActorKind, + #[error("source proof operation is not allowed")] + WrongOperation, + #[error("source proof mutation target does not match the request")] + WrongMutationTarget, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -210,6 +224,201 @@ pub fn verify_capability_token( }) } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerMutationSourceClaims { + pub iss: String, + pub aud: String, + pub workspace_id: String, + pub worker_id: String, + pub actor_kind: WorkerMutationActorKind, + pub operation: WorkerMutationOperation, + pub target_runtime_id: String, + pub target_worker_id: String, + pub permission: String, + pub iat: u64, + pub exp: u64, + pub jti: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkerMutationActorKind { + Worker, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkerMutationOperation { + WorkerRemove, +} + +#[derive(Clone)] +pub struct RuntimeWorkerMutationSourceSigner { + runtime_id: String, + private_key: String, +} + +impl fmt::Debug for RuntimeWorkerMutationSourceSigner { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RuntimeWorkerMutationSourceSigner") + .field("runtime_id", &self.runtime_id) + .field("private_key", &"[redacted]") + .finish() + } +} + +impl RuntimeWorkerMutationSourceSigner { + pub fn from_identity(identity: &RuntimeIdentityMaterial) -> Self { + Self { + runtime_id: identity.identity_id.clone(), + private_key: identity.private_key.clone(), + } + } + + pub fn runtime_id(&self) -> &str { + &self.runtime_id + } + + pub fn issue_worker_remove( + &self, + audience: impl Into<String>, + workspace_id: impl Into<String>, + source_worker_id: impl Into<String>, + target_runtime_id: impl Into<String>, + target_worker_id: impl Into<String>, + ttl_seconds: u64, + ) -> Result<String, RuntimeAuthError> { + let issued_at = unix_now_seconds(); + let claims = WorkerMutationSourceClaims { + iss: self.runtime_id.clone(), + aud: audience.into(), + workspace_id: workspace_id.into(), + worker_id: source_worker_id.into(), + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id: target_runtime_id.into(), + target_worker_id: target_worker_id.into(), + permission: WORKER_REMOVE_PERMISSION.to_string(), + iat: issued_at, + exp: issued_at.saturating_add(ttl_seconds), + jti: new_token_id()?, + }; + self.sign(&claims) + } + + pub fn sign(&self, claims: &WorkerMutationSourceClaims) -> Result<String, RuntimeAuthError> { + if claims.iss != self.runtime_id { + return Err(RuntimeAuthError::UnknownIssuer(claims.iss.clone())); + } + let private = decode_private_key(&self.private_key)?; + let pair = Ed25519KeyPair::from_pkcs8(&private) + .map_err(|_| RuntimeAuthError::InvalidPrivateKey)?; + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims)?); + let signing_input = format!("{WORKER_MUTATION_SOURCE_SIGNING_INPUT_PREFIX}{payload}"); + let signature = pair.sign(signing_input.as_bytes()); + Ok(format!( + "{WORKER_MUTATION_SOURCE_PROOF_PREFIX}.{payload}.{}", + URL_SAFE_NO_PAD.encode(signature.as_ref()) + )) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkerMutationSourceExpectation<'a> { + pub runtime_id: &'a str, + pub audience: &'a str, + pub workspace_id: &'a str, + pub worker_id: Option<&'a str>, + pub actor_kind: WorkerMutationActorKind, + pub operation: WorkerMutationOperation, + pub target_runtime_id: &'a str, + pub target_worker_id: &'a str, + pub permission: &'a str, +} + +pub fn decode_worker_mutation_source_claims( + token: &str, +) -> Result<WorkerMutationSourceClaims, RuntimeAuthError> { + let (payload, _) = split_worker_mutation_source_proof(token)?; + Ok(serde_json::from_slice(&URL_SAFE_NO_PAD.decode(payload)?)?) +} + +pub fn verify_worker_mutation_source_proof( + trusted_runtime_public_key: &str, + token: &str, + expected: &WorkerMutationSourceExpectation<'_>, + now_seconds: u64, +) -> Result<WorkerMutationSourceClaims, RuntimeAuthError> { + let (payload, signature) = split_worker_mutation_source_proof(token)?; + let claims_json = URL_SAFE_NO_PAD.decode(payload)?; + let claims: WorkerMutationSourceClaims = serde_json::from_slice(&claims_json)?; + let public_key = decode_public_key(trusted_runtime_public_key)?; + let signing_input = format!("{WORKER_MUTATION_SOURCE_SIGNING_INPUT_PREFIX}{payload}"); + UnparsedPublicKey::new(&ED25519, public_key) + .verify(signing_input.as_bytes(), &signature) + .map_err(|_| RuntimeAuthError::InvalidSignature)?; + + if claims.iss != expected.runtime_id { + return Err(RuntimeAuthError::UnknownIssuer(claims.iss)); + } + if claims.aud != expected.audience { + return Err(RuntimeAuthError::WrongAudience { + expected: expected.audience.to_string(), + actual: claims.aud, + }); + } + if claims.exp <= now_seconds || claims.iat > now_seconds.saturating_add(60) { + return Err(RuntimeAuthError::Expired); + } + if claims.workspace_id != expected.workspace_id { + return Err(RuntimeAuthError::WrongWorkspace { + expected: expected.workspace_id.to_string(), + actual: claims.workspace_id, + }); + } + if let Some(worker_id) = expected.worker_id { + if claims.worker_id != worker_id { + return Err(RuntimeAuthError::WrongWorker { + expected: worker_id.to_string(), + actual: claims.worker_id, + }); + } + } + if claims.actor_kind != expected.actor_kind { + return Err(RuntimeAuthError::WrongActorKind); + } + if claims.operation != expected.operation { + return Err(RuntimeAuthError::WrongOperation); + } + if claims.target_runtime_id != expected.target_runtime_id + || claims.target_worker_id != expected.target_worker_id + { + return Err(RuntimeAuthError::WrongMutationTarget); + } + if claims.permission != expected.permission { + return Err(RuntimeAuthError::MissingPermission( + expected.permission.to_string(), + )); + } + if claims.jti.trim().is_empty() { + return Err(RuntimeAuthError::InvalidTokenFormat); + } + Ok(claims) +} + +fn split_worker_mutation_source_proof(token: &str) -> Result<(&str, Vec<u8>), RuntimeAuthError> { + let mut parts = token.split('.'); + match (parts.next(), parts.next(), parts.next(), parts.next()) { + (Some(prefix), Some(payload), Some(signature), None) + if prefix == WORKER_MUTATION_SOURCE_PROOF_PREFIX => + { + Ok((payload, URL_SAFE_NO_PAD.decode(signature)?)) + } + _ => Err(RuntimeAuthError::InvalidTokenFormat), + } +} + fn split_token(token: &str) -> Result<(&str, Vec<u8>), RuntimeAuthError> { let mut parts = token.split('.'); match (parts.next(), parts.next(), parts.next(), parts.next()) { @@ -277,6 +486,112 @@ impl fmt::Display for RuntimeAuthContext { mod tests { use super::*; + #[test] + fn worker_mutation_source_proof_binds_all_source_authority_claims() { + let runtime = RuntimeIdentityMaterial::generate("runtime-main").unwrap(); + let signer = RuntimeWorkerMutationSourceSigner::from_identity(&runtime); + let claims = WorkerMutationSourceClaims { + iss: "runtime-main".to_string(), + aud: "server-main".to_string(), + workspace_id: "workspace-a".to_string(), + worker_id: "worker-7".to_string(), + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id: "runtime-target".to_string(), + target_worker_id: "worker-target".to_string(), + permission: WORKER_REMOVE_PERMISSION.to_string(), + iat: 90, + exp: 100, + jti: "source-proof-1".to_string(), + }; + let token = signer.sign(&claims).unwrap(); + let expected = WorkerMutationSourceExpectation { + runtime_id: "runtime-main", + audience: "server-main", + workspace_id: "workspace-a", + worker_id: Some("worker-7"), + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id: "runtime-target", + target_worker_id: "worker-target", + permission: WORKER_REMOVE_PERMISSION, + }; + + assert_eq!( + verify_worker_mutation_source_proof(&runtime.public_key, &token, &expected, 99) + .unwrap(), + claims + ); + + let wrong_worker = WorkerMutationSourceExpectation { + worker_id: Some("worker-8"), + ..expected.clone() + }; + assert!(matches!( + verify_worker_mutation_source_proof(&runtime.public_key, &token, &wrong_worker, 99), + Err(RuntimeAuthError::WrongWorker { .. }) + )); + let wrong_scope = WorkerMutationSourceExpectation { + workspace_id: "workspace-b", + ..expected.clone() + }; + assert!(matches!( + verify_worker_mutation_source_proof(&runtime.public_key, &token, &wrong_scope, 99), + Err(RuntimeAuthError::WrongWorkspace { .. }) + )); + let wrong_audience = WorkerMutationSourceExpectation { + audience: "server-other", + ..expected.clone() + }; + assert!(matches!( + verify_worker_mutation_source_proof(&runtime.public_key, &token, &wrong_audience, 99), + Err(RuntimeAuthError::WrongAudience { .. }) + )); + assert!(matches!( + verify_worker_mutation_source_proof(&runtime.public_key, &token, &expected, 101), + Err(RuntimeAuthError::Expired) + )); + } + + #[test] + fn worker_mutation_source_proof_rejects_spoofed_runtime_signature() { + let trusted = RuntimeIdentityMaterial::generate("runtime-main").unwrap(); + let spoofed = RuntimeIdentityMaterial::generate("runtime-main").unwrap(); + let claims = WorkerMutationSourceClaims { + iss: "runtime-main".to_string(), + aud: "server-main".to_string(), + workspace_id: "workspace-a".to_string(), + worker_id: "worker-7".to_string(), + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id: "runtime-target".to_string(), + target_worker_id: "worker-target".to_string(), + permission: WORKER_REMOVE_PERMISSION.to_string(), + iat: 90, + exp: 100, + jti: "source-proof-2".to_string(), + }; + let token = RuntimeWorkerMutationSourceSigner::from_identity(&spoofed) + .sign(&claims) + .unwrap(); + let expected = WorkerMutationSourceExpectation { + runtime_id: "runtime-main", + audience: "server-main", + workspace_id: "workspace-a", + worker_id: Some("worker-7"), + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id: "runtime-target", + target_worker_id: "worker-target", + permission: WORKER_REMOVE_PERMISSION, + }; + + assert!(matches!( + verify_worker_mutation_source_proof(&trusted.public_key, &token, &expected, 99), + Err(RuntimeAuthError::InvalidSignature) + )); + } + #[test] fn capability_token_verifies_signature_audience_expiry_and_permission() { let server = RuntimeIdentityMaterial::generate("server-main").unwrap(); diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index 3adbfb45..e32f450d 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -179,8 +179,6 @@ pub struct WorkingDirectoryStatus { pub struct WorkspaceApiRef { pub workspace_id: String, pub base_url: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub runtime_id: Option<String>, } impl std::fmt::Debug for WorkspaceApiRef { @@ -189,7 +187,6 @@ impl std::fmt::Debug for WorkspaceApiRef { .debug_struct("WorkspaceApiRef") .field("workspace_id", &self.workspace_id) .field("base_url", &self.base_url) - .field("runtime_id", &self.runtime_id) .finish() } } @@ -292,3 +289,31 @@ pub struct WorkerLifecycleAck { pub worker_ref: WorkerRef, pub status: WorkerStatus, } + +#[cfg(test)] +mod tests { + use super::WorkspaceApiRef; + + #[test] + fn workspace_api_ref_public_schema_contains_no_source_credentials_or_claim_choices() { + let value = serde_json::to_value(WorkspaceApiRef { + workspace_id: "workspace-a".to_string(), + base_url: "https://server.invalid".to_string(), + }) + .unwrap(); + let object = value.as_object().unwrap(); + assert_eq!(object.len(), 2); + assert!(object.contains_key("workspace_id")); + assert!(object.contains_key("base_url")); + for forbidden in [ + "runtime_id", + "worker_id", + "permission", + "private_key", + "bearer_token", + "signing_handle", + ] { + assert!(!object.contains_key(forbidden), "unexpected {forbidden}"); + } + } +} diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index 5f43b43a..381e0c84 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -248,6 +248,7 @@ pub struct WorkerExecutionSpawnRequest { /// Monotonic execution generation reserved durably before launch. pub run_generation: u64, pub request: crate::catalog::CreateWorkerRequest, + pub workspace_scope: Option<crate::runtime::RuntimeWorkspaceScope>, pub context: WorkerExecutionContext, pub working_directory: Option<WorkingDirectoryBinding>, pub config_bundle: Option<ConfigBundle>, @@ -260,6 +261,7 @@ pub struct WorkerExecutionRestoreRequest { /// Monotonic execution generation reserved durably before restore. pub run_generation: u64, pub request: crate::catalog::CreateWorkerRequest, + pub workspace_scope: Option<crate::runtime::RuntimeWorkspaceScope>, pub context: WorkerExecutionContext, pub previous_working_directory: Option<WorkingDirectoryStatus>, pub working_directory: Option<WorkingDirectoryBinding>, diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index f08f245b..dce58519 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -1753,7 +1753,6 @@ mod tests { request.workspace_api = Some(WorkspaceApiRef { workspace_id: workspace_id.to_string(), base_url: format!("https://workspace.example/{workspace_id}"), - runtime_id: None, }); request } @@ -2324,7 +2323,6 @@ mod tests { workspace_api: WorkspaceApiRef { workspace_id: "local".to_string(), base_url: "http://127.0.0.1:8787".to_string(), - runtime_id: None, }, }, ) diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index 001a102a..44817026 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -26,6 +26,7 @@ pub mod resource; pub mod retention; mod runtime; pub mod worker_backend; +pub mod worker_source; pub mod working_directory; #[cfg(feature = "fs-store")] diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index 448ffa42..0e4371e2 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -87,6 +87,9 @@ fn build_runtime(config: &ProcessConfig) -> Result<Runtime, ProcessError> { }; let mut factory = ProfileRuntimeWorkerFactory::new(fs_paths.worker_dir.join("worker-root")) .with_runtime_store_dir(runtime_store_dir); + if let Some(identity) = read_runtime_auth_file(&runtime_auth_path(config))?.identity { + factory = factory.with_remote_worker_mutation_identity(identity); + } if let Some(endpoint) = config.backend_resource_endpoint.clone() { factory = factory.with_resource_client(Arc::new( worker_runtime::resource::HttpBackendResourceClient::new( diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 2c9a4961..2b70adce 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -548,6 +548,7 @@ impl Runtime { worker_ref: worker_ref.clone(), run_generation: 1, request, + workspace_scope: scope.cloned(), context: self.execution_context(worker_ref.clone()), working_directory: None, config_bundle: None, @@ -826,13 +827,10 @@ impl Runtime { if let Some(existing) = worker.request.workspace_api.as_ref() && (existing.workspace_id != workspace_api.workspace_id || existing.base_url.trim_end_matches('/') - != workspace_api.base_url.trim_end_matches('/') - || existing.runtime_id.as_ref().is_some_and(|runtime_id| { - workspace_api.runtime_id.as_ref() != Some(runtime_id) - })) + != workspace_api.base_url.trim_end_matches('/')) { return Err(RuntimeError::InvalidRequest( - "Workspace API replacement cannot change Worker Workspace identity, Runtime identity, or base URL" + "Workspace API replacement cannot change Worker Workspace identity or base URL" .to_string(), )); } @@ -902,10 +900,17 @@ impl Runtime { })?; state.worker_mut(worker_ref)?.run_generation = run_generation; state.persist_worker(&worker_ref.worker_id)?; + let workspace_scope = worker_request.workspace_api.as_ref().and_then(|api| { + state + .workspace_owners + .get(&api.workspace_id) + .map(|server_id| RuntimeWorkspaceScope::new(&api.workspace_id, server_id)) + }); let request = WorkerExecutionRestoreRequest { worker_ref: worker_ref.clone(), run_generation, request: worker_request, + workspace_scope, context: self.execution_context(worker_ref.clone()), previous_working_directory, working_directory: None, @@ -1595,9 +1600,15 @@ impl Runtime { }; for candidate in candidates { - let backend = { + let (backend, workspace_scope) = { let state = self.lock()?; - state.execution_backend.clone() + let workspace_scope = candidate.request.workspace_api.as_ref().and_then(|api| { + state + .workspace_owners + .get(&api.workspace_id) + .map(|server_id| RuntimeWorkspaceScope::new(&api.workspace_id, server_id)) + }); + (state.execution_backend.clone(), workspace_scope) }; let Some(backend) = backend else { return Ok(()); @@ -1606,6 +1617,7 @@ impl Runtime { worker_ref: candidate.worker_ref.clone(), run_generation: candidate.run_generation, request: candidate.request, + workspace_scope, context: self.execution_context(candidate.worker_ref.clone()), previous_working_directory: candidate.previous_working_directory, working_directory: None, @@ -2790,7 +2802,6 @@ mod tests { request.workspace_api = Some(WorkspaceApiRef { workspace_id: workspace_id.to_string(), base_url: format!("https://workspace.example/{workspace_id}"), - runtime_id: None, }); request } @@ -3239,7 +3250,6 @@ mod tests { let replacement = WorkspaceApiRef { workspace_id: "workspace-a".to_string(), base_url: "https://workspace.example/workspace-a/".to_string(), - runtime_id: Some("runtime-a".to_string()), }; runtime diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 2825c162..e4fe6303 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -14,6 +14,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, mpsc}; use std::time::Duration; +use crate::auth::RuntimeIdentityMaterial; use crate::catalog::{ CreateWorkerRequest, ProfileSourceArchiveHttpRef, ProfileSourceArchiveSource, WorkingDirectoryRequest, WorkingDirectoryStatus, @@ -26,6 +27,9 @@ use crate::execution::{ use crate::identity::WorkerRef; use crate::interaction::{WorkerInput, WorkerInputKind}; use crate::resource::{BackendResourceClient, ProfileSourceArchiveCache}; +use crate::worker_source::{ + EmbeddedWorkerMutationDispatcher, RuntimeOwnedWorkspaceClient, RuntimeWorkerMutationForwarder, +}; use crate::working_directory::{ WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer, }; @@ -49,10 +53,9 @@ use worker::feature::builtin::{ #[cfg(feature = "ws-server")] use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session}; use worker::{ - PromptLoader, RuntimeWorkspaceHttpClient, SegmentLogSink, - WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerController, WorkerError, - WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, WorkerWorkspaceContext, - WorkspaceClient, WorkspaceId, + PromptLoader, SegmentLogSink, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, + WorkerController, WorkerError, WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, + WorkerWorkspaceContext, WorkspaceClient, WorkspaceId, }; const DEFAULT_BACKEND_ID: &str = "worker-crate"; @@ -215,6 +218,9 @@ pub struct ProfileRuntimeWorkerFactory { worker_aggregate_root: Option<PathBuf>, resource_client: Option<Arc<dyn BackendResourceClient>>, profile_archive_cache: Arc<ProfileSourceArchiveCache>, + runtime_id: Option<String>, + worker_mutation_identity: Option<RuntimeIdentityMaterial>, + embedded_worker_mutation_dispatcher: Option<Arc<dyn EmbeddedWorkerMutationDispatcher>>, } impl ProfileRuntimeWorkerFactory { @@ -226,9 +232,38 @@ impl ProfileRuntimeWorkerFactory { worker_aggregate_root: None, resource_client: None, profile_archive_cache: Arc::new(ProfileSourceArchiveCache::default()), + runtime_id: None, + worker_mutation_identity: None, + embedded_worker_mutation_dispatcher: None, } } + pub fn with_runtime_id(mut self, runtime_id: impl Into<String>) -> Self { + self.runtime_id = Some(runtime_id.into()); + self + } + + pub fn with_remote_worker_mutation_identity( + mut self, + identity: RuntimeIdentityMaterial, + ) -> Self { + self.runtime_id = Some(identity.identity_id.clone()); + self.worker_mutation_identity = Some(identity); + self.embedded_worker_mutation_dispatcher = None; + self + } + + pub fn with_embedded_worker_mutation_dispatcher( + mut self, + runtime_id: impl Into<String>, + dispatcher: Arc<dyn EmbeddedWorkerMutationDispatcher>, + ) -> Self { + self.runtime_id = Some(runtime_id.into()); + self.worker_mutation_identity = None; + self.embedded_worker_mutation_dispatcher = Some(dispatcher); + self + } + pub fn with_runtime_store_dir(mut self, runtime_store_dir: impl Into<PathBuf>) -> Self { self.worker_aggregate_root = Some(runtime_store_dir.into().join("workers")); self @@ -348,38 +383,59 @@ enum RuntimeWorkspaceBackendRef { } impl RuntimeWorkspaceBackendRef { - fn from_worker_request(request: &CreateWorkerRequest) -> Self { - if let Some(api) = request.workspace_api.as_ref() - && let Some(runtime_id) = api - .runtime_id - .as_ref() - .filter(|runtime_id| !runtime_id.trim().is_empty()) - { + fn from_worker_request(request: &CreateWorkerRequest, runtime_id: Option<&str>) -> Self { + if let (Some(api), Some(runtime_id)) = (request.workspace_api.as_ref(), runtime_id) { return Self::Http { workspace_id: api.workspace_id.clone(), base_url: api.base_url.clone(), - runtime_id: runtime_id.clone(), + runtime_id: runtime_id.to_string(), }; } Self::None } - fn worker_context(&self, worker_ref: &WorkerRef) -> WorkerWorkspaceContext { + fn worker_context( + &self, + worker_ref: &WorkerRef, + workspace_scope: Option<&crate::runtime::RuntimeWorkspaceScope>, + mutation_identity: Option<&RuntimeIdentityMaterial>, + embedded_dispatcher: Option<&Arc<dyn EmbeddedWorkerMutationDispatcher>>, + ) -> WorkerWorkspaceContext { match self { Self::None => WorkerWorkspaceContext::no_workspace(), Self::Http { workspace_id, base_url, runtime_id, - } => WorkerWorkspaceContext::with_client( - WorkspaceId::new(workspace_id.clone()).ok(), - Arc::new(RuntimeWorkspaceHttpClient::new( + } => { + let mut client = RuntimeOwnedWorkspaceClient::new( workspace_id.clone(), base_url.clone(), runtime_id.clone(), worker_ref.worker_id.to_string(), - )), - ), + ); + if let (Some(scope), Some(identity)) = (workspace_scope, mutation_identity) { + client = client.with_worker_remove(RuntimeWorkerMutationForwarder::remote( + identity, + scope.clone(), + worker_ref.worker_id.to_string(), + base_url.clone(), + )); + } else if let (Some(scope), Some(dispatcher)) = + (workspace_scope, embedded_dispatcher) + { + client = client.with_worker_remove(RuntimeWorkerMutationForwarder::embedded( + runtime_id, + scope.clone(), + worker_ref.worker_id.to_string(), + (*dispatcher).clone(), + )); + } + WorkerWorkspaceContext::with_client( + WorkspaceId::new(workspace_id.clone()).ok(), + Arc::new(client), + ) + } } } } @@ -478,13 +534,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { ) }) .unwrap_or(WorkerFilesystemAuthority::None); - let workspace_backend_ref = - RuntimeWorkspaceBackendRef::from_worker_request(&request.request); - let observation_runtime_id = request - .request - .workspace_api - .as_ref() - .and_then(|api| api.runtime_id.clone()); + let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ); + let observation_runtime_id = self.runtime_id.clone(); let observation_workspace_id = request .request .workspace_api @@ -492,7 +546,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map(|api| api.workspace_id.clone()); let observation_grants = request.request.worker_observation_grants.clone(); let observation_enabled = request.request.worker_observation_enabled; - let workspace_context = workspace_backend_ref.worker_context(&request.worker_ref); + let workspace_context = workspace_backend_ref.worker_context( + &request.worker_ref, + request.workspace_scope.as_ref(), + self.worker_mutation_identity.as_ref(), + self.embedded_worker_mutation_dispatcher.as_ref(), + ); let selector = profile.as_ref(); let archive = self .resolve_profile_source_archive(&request.request.profile_source) @@ -628,13 +687,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { ) }) .unwrap_or(WorkerFilesystemAuthority::None); - let workspace_backend_ref = - RuntimeWorkspaceBackendRef::from_worker_request(&request.request); - let observation_runtime_id = request - .request - .workspace_api - .as_ref() - .and_then(|api| api.runtime_id.clone()); + let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ); + let observation_runtime_id = self.runtime_id.clone(); let observation_workspace_id = request .request .workspace_api @@ -642,7 +699,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map(|api| api.workspace_id.clone()); let observation_grants = request.request.worker_observation_grants.clone(); let observation_enabled = request.request.worker_observation_enabled; - let workspace_context = workspace_backend_ref.worker_context(&request.worker_ref); + let workspace_context = workspace_backend_ref.worker_context( + &request.worker_ref, + request.workspace_scope.as_ref(), + self.worker_mutation_identity.as_ref(), + self.embedded_worker_mutation_dispatcher.as_ref(), + ); let (manifest, loader) = Self::restore_fallback_manifest(&worker_name)?; let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; @@ -1756,6 +1818,36 @@ mod tests { use manifest::{Scope, WorkerManifest}; use session_store::{LogEntry, WorkerMetadataStore}; + #[test] + fn restart_restore_reconstructs_runtime_owned_worker_mutation_client() { + let identity = RuntimeIdentityMaterial::generate("runtime-source").unwrap(); + let worker_ref = WorkerRef::new(crate::identity::WorkerId::new(17)); + let backend = RuntimeWorkspaceBackendRef::Http { + workspace_id: "workspace-a".to_string(), + base_url: "https://server.invalid".to_string(), + runtime_id: "runtime-source".to_string(), + }; + let scope = crate::runtime::RuntimeWorkspaceScope::new("workspace-a", "server-main"); + + let before_restart = + backend.worker_context(&worker_ref, Some(&scope), Some(&identity), None); + let after_restore = + backend.worker_context(&worker_ref, Some(&scope), Some(&identity), None); + + assert_eq!( + before_restart.client_handle().kind(), + "runtime-owned-workspace-client" + ); + assert_eq!( + after_restore.client_handle().kind(), + "runtime-owned-workspace-client" + ); + assert_eq!( + after_restore.client_handle().workspace_id(), + Some("workspace-a") + ); + } + #[test] fn notify_run_state_allows_running_worker_inbox_delivery() { assert_eq!( @@ -1878,9 +1970,16 @@ mod tests { .as_ref() .map(|binding| binding.root().to_path_buf()) .unwrap_or_else(|| self.cwd.clone()); - let workspace_backend_ref = - RuntimeWorkspaceBackendRef::from_worker_request(&request.request); - let workspace_context = workspace_backend_ref.worker_context(&request.worker_ref); + let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + Some("runtime-test"), + ); + let workspace_context = workspace_backend_ref.worker_context( + &request.worker_ref, + request.workspace_scope.as_ref(), + None, + None, + ); let workspace_client = workspace_context.client_handle(); self.observed_workspace_clients.lock().unwrap().push(( workspace_client.kind().to_string(), @@ -1916,6 +2015,7 @@ mod tests { worker_ref: request.worker_ref, run_generation: request.run_generation, request: request.request, + workspace_scope: request.workspace_scope, context: request.context, working_directory: request.working_directory, config_bundle: request.config_bundle, @@ -2188,6 +2288,7 @@ mod tests { worker_ref: worker_ref.clone(), run_generation: 1, request: create_request("1"), + workspace_scope: None, context: test_execution_context(worker_ref), working_directory: None, config_bundle: None, @@ -2286,14 +2387,15 @@ mod tests { request.workspace_api = Some(crate::catalog::WorkspaceApiRef { workspace_id: "workspace-restore".to_string(), base_url: "http://workspace.invalid".to_string(), - runtime_id: Some("runtime-restore".to_string()), }); let controller = ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_id("runtime-restore") .with_runtime_store_dir(&runtime_store_dir) .restore_controller(WorkerExecutionRestoreRequest { worker_ref: worker_ref.clone(), run_generation: 1, request, + workspace_scope: None, context: test_execution_context(worker_ref), previous_working_directory: None, working_directory: None, @@ -2420,7 +2522,6 @@ mod tests { request.workspace_api = Some(crate::catalog::WorkspaceApiRef { workspace_id: "ws-test".to_string(), base_url: "http://127.0.0.1:3999".to_string(), - runtime_id: Some("runtime-test".to_string()), }); let detail = runtime.create_worker(request).unwrap(); @@ -2453,7 +2554,7 @@ mod tests { assert_eq!( observed_workspace_clients.lock().unwrap().as_slice(), &[( - "runtime-http-proxy".to_string(), + "runtime-owned-workspace-client".to_string(), Some("ws-test".to_string()), true, )] diff --git a/crates/worker-runtime/src/worker_source.rs b/crates/worker-runtime/src/worker_source.rs new file mode 100644 index 00000000..b9fdd0c1 --- /dev/null +++ b/crates/worker-runtime/src/worker_source.rs @@ -0,0 +1,613 @@ +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use worker::{ + WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod, + WorkspaceResponse, +}; + +use crate::auth::{ + RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner, + WORKER_REMOVE_PERMISSION, WorkerMutationActorKind, WorkerMutationOperation, + WorkerMutationSourceClaims, new_token_id, +}; +use crate::runtime::RuntimeWorkspaceScope; + +pub const DEFAULT_WORKER_MUTATION_SOURCE_TTL_SECONDS: u64 = 60; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RuntimeOwnedWorkerMutationProof { + Remote(String), + InProcess(InProcessWorkerMutationProof), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InProcessWorkerMutationProof { + claims: WorkerMutationSourceClaims, +} + +impl InProcessWorkerMutationProof { + pub fn claims(&self) -> &WorkerMutationSourceClaims { + &self.claims + } + + pub fn into_claims(self) -> WorkerMutationSourceClaims { + self.claims + } +} + +#[derive(Clone)] +pub struct RuntimeWorkerMutationSourceAuthority { + mode: RuntimeWorkerMutationSourceMode, +} + +#[derive(Clone)] +enum RuntimeWorkerMutationSourceMode { + Remote { + signer: RuntimeWorkerMutationSourceSigner, + }, + Embedded { + runtime_id: String, + audience: String, + }, +} + +impl RuntimeWorkerMutationSourceAuthority { + pub fn remote(identity: &RuntimeIdentityMaterial) -> Self { + Self { + mode: RuntimeWorkerMutationSourceMode::Remote { + signer: RuntimeWorkerMutationSourceSigner::from_identity(identity), + }, + } + } + + pub fn embedded(runtime_id: impl Into<String>, workspace_id: impl AsRef<str>) -> Self { + Self { + mode: RuntimeWorkerMutationSourceMode::Embedded { + runtime_id: runtime_id.into(), + audience: format!("embedded:{}", workspace_id.as_ref()), + }, + } + } + + pub fn issue_worker_remove( + &self, + scope: &RuntimeWorkspaceScope, + source_worker_id: &str, + target_runtime_id: &str, + target_worker_id: &str, + ) -> Result<RuntimeOwnedWorkerMutationProof, RuntimeAuthError> { + match &self.mode { + RuntimeWorkerMutationSourceMode::Remote { signer } => { + let token = signer.issue_worker_remove( + &scope.server_id, + &scope.workspace_id, + source_worker_id, + target_runtime_id, + target_worker_id, + DEFAULT_WORKER_MUTATION_SOURCE_TTL_SECONDS, + )?; + Ok(RuntimeOwnedWorkerMutationProof::Remote(token)) + } + RuntimeWorkerMutationSourceMode::Embedded { + runtime_id, + audience, + } => { + let issued_at = unix_now_seconds(); + Ok(RuntimeOwnedWorkerMutationProof::InProcess( + InProcessWorkerMutationProof { + claims: WorkerMutationSourceClaims { + iss: runtime_id.clone(), + aud: audience.clone(), + workspace_id: scope.workspace_id.clone(), + worker_id: source_worker_id.to_string(), + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id: target_runtime_id.to_string(), + target_worker_id: target_worker_id.to_string(), + permission: WORKER_REMOVE_PERMISSION.to_string(), + iat: issued_at, + exp: issued_at + .saturating_add(DEFAULT_WORKER_MUTATION_SOURCE_TTL_SECONDS), + jti: new_token_id()?, + }, + }, + )) + } + } + } +} + +pub trait EmbeddedWorkerMutationDispatcher: Send + Sync { + fn execute_worker_remove( + &self, + proof: InProcessWorkerMutationProof, + target_runtime_id: &str, + target_worker_id: &str, + ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError>; +} + +#[derive(Clone)] +enum RuntimeWorkerMutationTransport { + Remote { + base_url: String, + client: reqwest::blocking::Client, + }, + Embedded { + dispatcher: Arc<dyn EmbeddedWorkerMutationDispatcher>, + }, +} + +#[derive(Clone)] +pub struct RuntimeWorkerMutationForwarder { + authority: RuntimeWorkerMutationSourceAuthority, + scope: RuntimeWorkspaceScope, + source_worker_id: String, + transport: RuntimeWorkerMutationTransport, +} + +impl RuntimeWorkerMutationForwarder { + pub fn remote( + identity: &RuntimeIdentityMaterial, + scope: RuntimeWorkspaceScope, + source_worker_id: impl Into<String>, + base_url: impl Into<String>, + ) -> Self { + Self { + authority: RuntimeWorkerMutationSourceAuthority::remote(identity), + scope, + source_worker_id: source_worker_id.into(), + transport: RuntimeWorkerMutationTransport::Remote { + base_url: base_url.into().trim_end_matches('/').to_string(), + client: reqwest::blocking::Client::new(), + }, + } + } + + pub fn embedded( + runtime_id: impl Into<String>, + scope: RuntimeWorkspaceScope, + source_worker_id: impl Into<String>, + dispatcher: Arc<dyn EmbeddedWorkerMutationDispatcher>, + ) -> Self { + let runtime_id = runtime_id.into(); + Self { + authority: RuntimeWorkerMutationSourceAuthority::embedded( + &runtime_id, + &scope.workspace_id, + ), + scope, + source_worker_id: source_worker_id.into(), + transport: RuntimeWorkerMutationTransport::Embedded { dispatcher }, + } + } + + pub fn execute_worker_remove( + &self, + target_runtime_id: &str, + target_worker_id: &str, + ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> { + let proof = self.authority.issue_worker_remove( + &self.scope, + &self.source_worker_id, + target_runtime_id, + target_worker_id, + )?; + match (&self.transport, proof) { + ( + RuntimeWorkerMutationTransport::Remote { base_url, client }, + RuntimeOwnedWorkerMutationProof::Remote(token), + ) => { + let url = format!( + "{base_url}/api/w/{}/workers/remove", + self.scope.workspace_id + ); + let body = serde_json::json!({ + "target_runtime_id": target_runtime_id, + "target_worker_id": target_worker_id, + }); + let response = client + .post(url) + .header(crate::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER, token) + .json(&body) + .send() + .map_err(|error| { + RuntimeWorkerMutationForwardError::Transport(error.to_string()) + })?; + let status = response.status().as_u16(); + let body = response.text().map_err(|error| { + RuntimeWorkerMutationForwardError::Transport(error.to_string()) + })?; + Ok(WorkspaceResponse { status, body }) + } + ( + RuntimeWorkerMutationTransport::Embedded { dispatcher }, + RuntimeOwnedWorkerMutationProof::InProcess(claims), + ) => dispatcher.execute_worker_remove(claims, target_runtime_id, target_worker_id), + _ => Err(RuntimeWorkerMutationForwardError::AuthorityTransportMismatch), + } + } +} + +pub struct RuntimeOwnedWorkspaceClient { + workspace_id: String, + base_url: String, + runtime_id: String, + worker_id: String, + worker_remove: Option<RuntimeWorkerMutationForwarder>, +} + +impl RuntimeOwnedWorkspaceClient { + pub fn new( + workspace_id: impl Into<String>, + base_url: impl Into<String>, + runtime_id: impl Into<String>, + worker_id: impl Into<String>, + ) -> Self { + Self { + workspace_id: workspace_id.into(), + base_url: base_url.into().trim_end_matches('/').to_string(), + runtime_id: runtime_id.into(), + worker_id: worker_id.into(), + worker_remove: None, + } + } + + pub fn with_worker_remove(mut self, worker_remove: RuntimeWorkerMutationForwarder) -> Self { + self.worker_remove = Some(worker_remove); + self + } +} + +impl std::fmt::Debug for RuntimeOwnedWorkspaceClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RuntimeOwnedWorkspaceClient") + .field("workspace_id", &self.workspace_id) + .field("base_url", &self.base_url) + .field("source", &"Runtime-owned") + .field( + "worker_remove", + &self.worker_remove.as_ref().map(|_| "enabled"), + ) + .finish() + } +} + +impl WorkspaceClient for RuntimeOwnedWorkspaceClient { + fn workspace_id(&self) -> Option<&str> { + Some(&self.workspace_id) + } + + fn kind(&self) -> &str { + "runtime-owned-workspace-client" + } + + fn is_available(&self) -> bool { + true + } + + fn execute( + &self, + request: WorkspaceRequest, + ) -> Result<WorkspaceResponse, WorkspaceClientError> { + let base_url = self.base_url.clone(); + let runtime_id = self.runtime_id.clone(); + let worker_id = self.worker_id.clone(); + if tokio::runtime::Handle::try_current().is_ok() { + std::thread::spawn(move || { + execute_runtime_owned_workspace_http(&base_url, &runtime_id, &worker_id, request) + }) + .join() + .map_err(|_| { + WorkspaceClientError::Request("workspace request thread panicked".to_string()) + })? + } else { + execute_runtime_owned_workspace_http( + &self.base_url, + &self.runtime_id, + &self.worker_id, + request, + ) + } + } + + fn execute_worker_remove( + &self, + target_runtime_id: &str, + target_worker_id: &str, + ) -> Result<WorkspaceResponse, WorkspaceClientError> { + self.worker_remove + .as_ref() + .ok_or_else(|| { + WorkspaceClientError::Unavailable( + "Runtime-owned WorkerRemove forwarding is unavailable".to_string(), + ) + })? + .execute_worker_remove(target_runtime_id, target_worker_id) + .map_err(|error| WorkspaceClientError::Request(error.to_string())) + } +} + +fn execute_runtime_owned_workspace_http( + base_url: &str, + runtime_id: &str, + worker_id: &str, + request: WorkspaceRequest, +) -> Result<WorkspaceResponse, WorkspaceClientError> { + if !request.path.starts_with('/') || request.path.starts_with("//") { + return Err(WorkspaceClientError::InvalidPath(request.path)); + } + let url = format!("{base_url}{}", request.path); + let method = match request.method { + WorkspaceRequestMethod::Get => reqwest::Method::GET, + WorkspaceRequestMethod::Post => reqwest::Method::POST, + WorkspaceRequestMethod::Put => reqwest::Method::PUT, + WorkspaceRequestMethod::Patch => reqwest::Method::PATCH, + WorkspaceRequestMethod::Delete => reqwest::Method::DELETE, + }; + let client = reqwest::blocking::Client::new(); + let mut request_builder = client + .request(method, url) + .header("x-yoi-runtime-id", runtime_id) + .header("x-yoi-worker-id", worker_id); + if let Some(body) = request.body { + request_builder = request_builder + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(body); + } + let response = request_builder + .send() + .map_err(|error| WorkspaceClientError::Request(error.to_string()))?; + let status = response.status().as_u16(); + let body = response + .text() + .map_err(|error| WorkspaceClientError::Request(error.to_string()))?; + Ok(WorkspaceResponse { status, body }) +} + +#[derive(Debug, thiserror::Error)] +pub enum RuntimeWorkerMutationForwardError { + #[error(transparent)] + Auth(#[from] RuntimeAuthError), + #[error("Worker mutation forwarding transport failed: {0}")] + Transport(String), + #[error("Worker mutation source authority does not match its forwarding transport")] + AuthorityTransportMismatch, + #[error("embedded Worker mutation dispatcher failed: {0}")] + Embedded(String), +} + +fn unix_now_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::{ + WorkerMutationSourceExpectation, decode_worker_mutation_source_claims, + verify_worker_mutation_source_proof, + }; + + #[test] + fn ordinary_workspace_forwarding_stamps_legacy_source_only_inside_runtime() { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::Mutex; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let received = Arc::new(Mutex::new(String::new())); + let received_for_server = received.clone(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut bytes = [0_u8; 4096]; + let count = stream.read(&mut bytes).unwrap(); + *received_for_server.lock().unwrap() = + String::from_utf8_lossy(&bytes[..count]).into_owned(); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}") + .unwrap(); + }); + + let client = RuntimeOwnedWorkspaceClient::new( + "workspace-a", + format!("http://{address}"), + "runtime-a", + "worker-a", + ); + let response = client + .execute(WorkspaceRequest::get("/api/w/workspace-a/tickets/search")) + .unwrap(); + assert_eq!(response.status, 200); + server.join().unwrap(); + let request = received.lock().unwrap().to_ascii_lowercase(); + assert!(request.contains("x-yoi-runtime-id: runtime-a")); + assert!(request.contains("x-yoi-worker-id: worker-a")); + assert!(!request.contains("authorization:")); + } + + #[test] + fn remote_authority_stamps_and_signs_worker_remove_without_caller_claim_choices() { + let identity = RuntimeIdentityMaterial::generate("runtime-a").unwrap(); + let authority = RuntimeWorkerMutationSourceAuthority::remote(&identity); + let scope = RuntimeWorkspaceScope { + workspace_id: "workspace-a".to_string(), + server_id: "server-a".to_string(), + }; + + let RuntimeOwnedWorkerMutationProof::Remote(token) = authority + .issue_worker_remove(&scope, "worker-source", "runtime-b", "worker-target") + .unwrap() + else { + panic!("remote authority must produce a signed proof"); + }; + let claims = decode_worker_mutation_source_claims(&token).unwrap(); + let expected = WorkerMutationSourceExpectation { + runtime_id: "runtime-a", + audience: "server-a", + workspace_id: "workspace-a", + worker_id: Some("worker-source"), + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id: "runtime-b", + target_worker_id: "worker-target", + permission: WORKER_REMOVE_PERMISSION, + }; + assert_eq!( + verify_worker_mutation_source_proof( + &identity.public_key, + &token, + &expected, + claims.iat + ) + .unwrap(), + claims + ); + } + + #[test] + fn remote_forwarder_stamps_signed_proof_inside_runtime_before_http_delivery() { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::Mutex; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let received = Arc::new(Mutex::new(String::new())); + let received_for_server = received.clone(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut bytes = [0_u8; 8192]; + let count = stream.read(&mut bytes).unwrap(); + *received_for_server.lock().unwrap() = + String::from_utf8_lossy(&bytes[..count]).into_owned(); + stream + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n") + .unwrap(); + }); + + let identity = RuntimeIdentityMaterial::generate("runtime-a").unwrap(); + let scope = RuntimeWorkspaceScope::new("workspace-a", "server-a"); + let forwarder = RuntimeWorkerMutationForwarder::remote( + &identity, + scope, + "worker-source", + format!("http://{address}"), + ); + let response = forwarder + .execute_worker_remove("runtime-target", "worker-target") + .unwrap(); + assert_eq!(response.status, 204); + server.join().unwrap(); + + let request = received.lock().unwrap().clone(); + assert!(request.starts_with("POST /api/w/workspace-a/workers/remove HTTP/1.1")); + assert!(request.contains( + r#"{"target_runtime_id":"runtime-target","target_worker_id":"worker-target"}"# + )); + let token = request + .lines() + .find_map(|line| { + line.split_once(':').and_then(|(name, value)| { + name.eq_ignore_ascii_case(crate::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER) + .then(|| value.trim()) + }) + }) + .expect("proof header"); + let claims = decode_worker_mutation_source_claims(token).unwrap(); + let expected = WorkerMutationSourceExpectation { + runtime_id: "runtime-a", + audience: "server-a", + workspace_id: "workspace-a", + worker_id: Some("worker-source"), + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id: "runtime-target", + target_worker_id: "worker-target", + permission: WORKER_REMOVE_PERMISSION, + }; + verify_worker_mutation_source_proof(&identity.public_key, token, &expected, claims.iat) + .unwrap(); + } + + #[test] + fn embedded_forwarder_delivers_in_process_proof_with_the_request() { + use std::sync::Mutex; + + #[derive(Default)] + struct RecordingDispatcher { + seen: Mutex<Option<(WorkerMutationSourceClaims, String, String)>>, + } + impl EmbeddedWorkerMutationDispatcher for RecordingDispatcher { + fn execute_worker_remove( + &self, + proof: InProcessWorkerMutationProof, + target_runtime_id: &str, + target_worker_id: &str, + ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> { + *self.seen.lock().unwrap() = Some(( + proof.into_claims(), + target_runtime_id.to_string(), + target_worker_id.to_string(), + )); + Ok(WorkspaceResponse { + status: 202, + body: "accepted".to_string(), + }) + } + } + + let dispatcher = Arc::new(RecordingDispatcher::default()); + let scope = RuntimeWorkspaceScope::new("workspace-a", "server-a"); + let forwarder = RuntimeWorkerMutationForwarder::embedded( + "runtime-embedded", + scope, + "worker-source", + dispatcher.clone(), + ); + let response = forwarder + .execute_worker_remove("runtime-target", "worker-target") + .unwrap(); + assert_eq!(response.status, 202); + let (claims, target_runtime_id, target_worker_id) = + dispatcher.seen.lock().unwrap().take().unwrap(); + assert_eq!(claims.iss, "runtime-embedded"); + assert_eq!(claims.worker_id, "worker-source"); + assert_eq!(claims.target_runtime_id, "runtime-target"); + assert_eq!(claims.target_worker_id, "worker-target"); + assert_eq!(target_runtime_id, "runtime-target"); + assert_eq!(target_worker_id, "worker-target"); + } + + #[test] + fn embedded_authority_uses_the_same_claim_contract_without_a_credential() { + let authority = + RuntimeWorkerMutationSourceAuthority::embedded("runtime-embedded", "workspace-a"); + let scope = RuntimeWorkspaceScope { + workspace_id: "workspace-a".to_string(), + server_id: "server-unused-for-embedded".to_string(), + }; + + let RuntimeOwnedWorkerMutationProof::InProcess(proof) = authority + .issue_worker_remove(&scope, "worker-source", "runtime-b", "worker-target") + .unwrap() + else { + panic!("embedded authority must produce an in-process proof"); + }; + let claims = proof.claims(); + assert_eq!(claims.iss, "runtime-embedded"); + assert_eq!(claims.aud, "embedded:workspace-a"); + assert_eq!(claims.workspace_id, "workspace-a"); + assert_eq!(claims.worker_id, "worker-source"); + assert_eq!(claims.operation, WorkerMutationOperation::WorkerRemove); + assert_eq!(claims.target_runtime_id, "runtime-b"); + assert_eq!(claims.target_worker_id, "worker-target"); + assert_eq!(claims.permission, WORKER_REMOVE_PERMISSION); + assert!(!claims.jti.is_empty()); + } +} diff --git a/crates/worker/src/feature/builtin/memory.rs b/crates/worker/src/feature/builtin/memory.rs index 774dc950..b4eec17e 100644 --- a/crates/worker/src/feature/builtin/memory.rs +++ b/crates/worker/src/feature/builtin/memory.rs @@ -346,11 +346,9 @@ mod tests { use llm_engine::tool::ToolDefinition; fn test_client() -> Arc<dyn WorkspaceClient> { - Arc::new(crate::worker::RuntimeWorkspaceHttpClient::new( + Arc::new(crate::worker::TestWorkspaceHttpClient::new( "workspace", "http://backend", - "test-runtime", - "test-worker", )) } diff --git a/crates/worker/src/feature/builtin/objective.rs b/crates/worker/src/feature/builtin/objective.rs index ef2e828b..db5d86a9 100644 --- a/crates/worker/src/feature/builtin/objective.rs +++ b/crates/worker/src/feature/builtin/objective.rs @@ -628,12 +628,7 @@ mod tests { #[test] fn workspace_http_objective_tools_include_objective_crud_tools() { let names = tool_names(workspace_http_objective_tools(Arc::new( - crate::worker::RuntimeWorkspaceHttpClient::new( - "workspace", - "http://backend", - "test-runtime", - "test-worker", - ), + crate::worker::TestWorkspaceHttpClient::new("workspace", "http://backend"), ))); assert_eq!( diff --git a/crates/worker/src/feature/builtin/ticket.rs b/crates/worker/src/feature/builtin/ticket.rs index 32d3eb30..8a392b79 100644 --- a/crates/worker/src/feature/builtin/ticket.rs +++ b/crates/worker/src/feature/builtin/ticket.rs @@ -1376,12 +1376,7 @@ provider = "github" #[tokio::test(flavor = "multi_thread")] async fn workspace_http_backend_invoke_is_safe_inside_async_context() { let backend = WorkspaceHttpTicketBackend::new(Arc::new( - crate::worker::RuntimeWorkspaceHttpClient::new( - "workspace-a", - "not-a-url", - "test-runtime", - "test-worker", - ), + crate::worker::TestWorkspaceHttpClient::new("workspace-a", "not-a-url"), )); let error = backend @@ -1412,11 +1407,9 @@ provider = "github" .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n") .unwrap(); }); - let client = Arc::new(crate::worker::RuntimeWorkspaceHttpClient::new( + let client = Arc::new(crate::worker::TestWorkspaceHttpClient::new( "workspace-a", format!("http://{address}"), - "test-runtime", - "worker-a", )); let backend = WorkspaceHttpTicketBackend::new(client); @@ -1457,12 +1450,7 @@ provider = "github" }); let backend = WorkspaceHttpTicketBackend::new(Arc::new( - crate::worker::RuntimeWorkspaceHttpClient::new( - "workspace-a", - base_url, - "test-runtime", - "test-worker", - ), + crate::worker::TestWorkspaceHttpClient::new("workspace-a", base_url), )); let created = backend.create(NewTicket::new("HTTP ticket")).unwrap(); diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index 3c0d1e12..7df10a5a 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -40,9 +40,9 @@ pub use runtime::dir::RuntimeDir; pub use segment_log_sink::SegmentLogSink; pub use shared_state::WorkerSharedState; pub use worker::{ - LocalWorkingDirectory, RuntimeWorkspaceHttpClient, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, - Worker, WorkerError, WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, - WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspaceRequest, - WorkspaceRequestMethod, WorkspaceResponse, apply_worker_manifest, marker_workspace_client, + LocalWorkingDirectory, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerError, + WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceClient, + WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspaceRequest, WorkspaceRequestMethod, + WorkspaceResponse, apply_worker_manifest, marker_workspace_client, unavailable_workspace_client, }; diff --git a/crates/worker/src/skill.rs b/crates/worker/src/skill.rs index cef7a564..effd90c7 100644 --- a/crates/worker/src/skill.rs +++ b/crates/worker/src/skill.rs @@ -212,8 +212,8 @@ mod tests { break; } } - assert_eq!(runtime_header.as_deref(), Some("runtime-test")); - assert_eq!(worker_header.as_deref(), Some("test-worker")); + assert_eq!(runtime_header, None); + assert_eq!(worker_header, None); assert_eq!(authorization, None); let body = serde_json::json!({ "authority": "workspace-backend-skills-v0", @@ -236,12 +236,7 @@ mod tests { .unwrap(); }); - let client = crate::worker::RuntimeWorkspaceHttpClient::new( - "ws-1", - format!("http://{addr}"), - "runtime-test", - "test-worker", - ); + let client = crate::worker::TestWorkspaceHttpClient::new("ws-1", format!("http://{addr}")); let catalog = (&client as &dyn WorkspaceClient).list_skills().unwrap(); assert_eq!(catalog.entries[0].name, "triage-errors"); assert_eq!(catalog.entries[0].provenance.id, "workspace:triage-errors"); diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index d9fc0f7f..565c785c 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -224,6 +224,18 @@ pub trait WorkspaceClient: std::fmt::Debug + Send + Sync { fn execute(&self, request: WorkspaceRequest) -> Result<WorkspaceResponse, WorkspaceClientError>; + /// Executes the destructive WorkerRemove operation through Runtime-owned source proof. + /// Target identity is operation data; source identity and permission are never caller inputs. + fn execute_worker_remove( + &self, + _target_runtime_id: &str, + _target_worker_id: &str, + ) -> Result<WorkspaceResponse, WorkspaceClientError> { + Err(WorkspaceClientError::Unavailable( + "Runtime-owned WorkerRemove forwarding is unavailable".to_string(), + )) + } + /// Trusted review-attempt context is injected by the Internal SubWorker spawn layer. /// It is never accepted from a model-visible tool argument. fn reviewer_attempt_context(&self) -> Option<&ReviewerAttemptContext> { @@ -313,53 +325,31 @@ impl WorkspaceClient for ReviewerChildWorkspaceClient { } } -/// HTTP forwarding client created by Runtime for one concrete Worker execution. -/// -/// The upstream endpoint and source headers are private implementation details; -/// model-visible tools can only submit [`WorkspaceRequest`] values through the -/// [`WorkspaceClient`] trait. -pub struct RuntimeWorkspaceHttpClient { +#[cfg(test)] +#[derive(Clone, Debug)] +pub(crate) struct TestWorkspaceHttpClient { workspace_id: String, base_url: String, - runtime_id: String, - worker_id: String, } -impl std::fmt::Debug for RuntimeWorkspaceHttpClient { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RuntimeWorkspaceHttpClient") - .field("workspace_id", &self.workspace_id) - .field("base_url", &self.base_url) - .field("runtime_id", &self.runtime_id) - .field("worker_id", &self.worker_id) - .finish() - } -} - -impl RuntimeWorkspaceHttpClient { - pub fn new( - workspace_id: impl Into<String>, - base_url: impl Into<String>, - runtime_id: impl Into<String>, - worker_id: impl Into<String>, - ) -> Self { +#[cfg(test)] +impl TestWorkspaceHttpClient { + pub(crate) fn new(workspace_id: impl Into<String>, base_url: impl Into<String>) -> Self { Self { workspace_id: workspace_id.into(), base_url: base_url.into().trim_end_matches('/').to_string(), - runtime_id: runtime_id.into(), - worker_id: worker_id.into(), } } } -impl WorkspaceClient for RuntimeWorkspaceHttpClient { +#[cfg(test)] +impl WorkspaceClient for TestWorkspaceHttpClient { fn workspace_id(&self) -> Option<&str> { Some(&self.workspace_id) } fn kind(&self) -> &str { - "runtime-http-proxy" + "test-http" } fn is_available(&self) -> bool { @@ -371,32 +361,28 @@ impl WorkspaceClient for RuntimeWorkspaceHttpClient { request: WorkspaceRequest, ) -> Result<WorkspaceResponse, WorkspaceClientError> { let base_url = self.base_url.clone(); - let runtime_id = self.runtime_id.clone(); - let worker_id = self.worker_id.clone(); if tokio::runtime::Handle::try_current().is_ok() { - std::thread::spawn(move || { - execute_runtime_workspace_http(&base_url, &runtime_id, &worker_id, request) - }) - .join() - .map_err(|_| { - WorkspaceClientError::Request("workspace request thread panicked".to_string()) - })? + std::thread::spawn(move || execute_test_workspace_http(&base_url, request)) + .join() + .map_err(|_| { + WorkspaceClientError::Request( + "test workspace request thread panicked".to_string(), + ) + })? } else { - execute_runtime_workspace_http(&base_url, &runtime_id, &worker_id, request) + execute_test_workspace_http(&base_url, request) } } } -fn execute_runtime_workspace_http( +#[cfg(test)] +fn execute_test_workspace_http( base_url: &str, - runtime_id: &str, - worker_id: &str, request: WorkspaceRequest, ) -> Result<WorkspaceResponse, WorkspaceClientError> { if !request.path.starts_with('/') || request.path.starts_with("//") { return Err(WorkspaceClientError::InvalidPath(request.path)); } - let url = format!("{base_url}{}", request.path); let method = match request.method { WorkspaceRequestMethod::Get => reqwest::Method::GET, WorkspaceRequestMethod::Post => reqwest::Method::POST, @@ -405,16 +391,11 @@ fn execute_runtime_workspace_http( WorkspaceRequestMethod::Delete => reqwest::Method::DELETE, }; let client = reqwest::blocking::Client::new(); - let mut request_builder = client - .request(method, url) - .header("x-yoi-runtime-id", runtime_id) - .header("x-yoi-worker-id", worker_id); + let mut builder = client.request(method, format!("{base_url}{}", request.path)); if let Some(body) = request.body { - request_builder = request_builder - .header(reqwest::header::CONTENT_TYPE, "application/json") - .body(body); + builder = builder.body(body); } - let response = request_builder + let response = builder .send() .map_err(|error| WorkspaceClientError::Request(error.to_string()))?; let status = response.status().as_u16(); @@ -6959,11 +6940,9 @@ mod build_summary_prompt_tests { }); WorkerWorkspaceContext::with_client( Some(WorkspaceId::new("test-memory").unwrap()), - Arc::new(RuntimeWorkspaceHttpClient::new( + Arc::new(TestWorkspaceHttpClient::new( "test-memory", format!("http://{addr}"), - "test-runtime", - "test-worker", )), ) } @@ -7096,11 +7075,9 @@ mod build_summary_prompt_tests { store, WorkerWorkspaceContext::with_client( Some(WorkspaceId::new("ws-skill").unwrap()), - Arc::new(RuntimeWorkspaceHttpClient::new( + Arc::new(TestWorkspaceHttpClient::new( "ws-skill", format!("http://{addr}"), - "test-runtime", - "test-worker", )), ), authority, @@ -7142,58 +7119,6 @@ mod build_summary_prompt_tests { })); } - #[test] - fn runtime_workspace_client_sends_runtime_worker_identity_without_bearer() { - use std::io::{BufRead, BufReader, Write}; - use std::net::TcpListener; - - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - let mut reader = BufReader::new(stream.try_clone().unwrap()); - let mut first_line = String::new(); - reader.read_line(&mut first_line).unwrap(); - assert!(first_line.contains("/api/w/workspace-a/tickets/search")); - let mut runtime_id = String::new(); - let mut worker_id = String::new(); - let mut authorization = String::new(); - loop { - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - if let Some(value) = line.strip_prefix("x-yoi-runtime-id: ") { - runtime_id = value.trim().to_string(); - } - if let Some(value) = line.strip_prefix("x-yoi-worker-id: ") { - worker_id = value.trim().to_string(); - } - if let Some(value) = line.strip_prefix("authorization: ") { - authorization = value.trim().to_string(); - } - if line == "\r\n" || line.is_empty() { - break; - } - } - assert_eq!(runtime_id, "runtime-a"); - assert_eq!(worker_id, "worker-a"); - assert!(authorization.is_empty()); - stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}") - .unwrap(); - }); - let client = RuntimeWorkspaceHttpClient::new( - "workspace-a", - format!("http://{address}"), - "runtime-a", - "worker-a", - ); - let response = client - .execute(WorkspaceRequest::get("/api/w/workspace-a/tickets/search")) - .unwrap(); - assert_eq!(response.status, 200); - server.join().unwrap(); - } - #[tokio::test] async fn cancelled_internal_extract_does_not_commit_pointer_or_completed_audit() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 46aecd87..6d9c40d1 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -55,7 +55,7 @@ use worker_runtime::interaction::{ use worker_runtime::management::{RuntimeOptions as EmbeddedRuntimeOptions, RuntimeStatus}; use worker_runtime::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveInput}; -const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime"; +pub(crate) const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime"; const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host"; const REMOTE_HOST_KIND: &str = "remote-worker-runtime-host"; const MAX_DIAGNOSTICS: usize = 16; @@ -4039,7 +4039,6 @@ mod tests { WorkspaceApiRef { workspace_id: "workspace-test".to_string(), base_url: "http://127.0.0.1:8787".to_string(), - runtime_id: Some("runtime-test".to_string()), } } diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 544f38fb..daa9f66e 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -24,6 +24,7 @@ pub mod runtime_subscription; pub mod server; pub mod skills; pub mod store; +pub mod worker_source; mod workspace_subscription; pub use authority::{ diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index 561d6e48..f6e41e01 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -1354,7 +1354,7 @@ mod tests { let path = temp.path().join("server.db"); { let s = SqliteWorkspaceStore::open(&path).unwrap(); - s.with_conn(|c|{c.execute("INSERT INTO workspaces(workspace_id,display_name,state,created_at,updated_at)VALUES('legacy','Legacy','active','old','old')",[])?;c.execute_batch("DROP TRIGGER seed_worker_retention_policy_after_workspace_insert;DROP TABLE worker_retention_audit_events;DROP TABLE worker_tombstones;DROP TABLE worker_session_archives;DROP TABLE worker_diagnostics_archives;DROP TABLE worker_orphan_diagnostics;DROP TABLE worker_removal_operations;DROP TABLE workspace_worker_retention_policies;DROP TABLE workspace_worker_retention_policy_revisions;DELETE FROM __yoi_schema_migrations WHERE version=28;")?;Ok(())}).unwrap(); + s.with_conn(|c|{c.execute("INSERT INTO workspaces(workspace_id,display_name,state,created_at,updated_at)VALUES('legacy','Legacy','active','old','old')",[])?;c.execute_batch("DROP TRIGGER seed_worker_retention_policy_after_workspace_insert;DROP TABLE worker_retention_audit_events;DROP TABLE worker_tombstones;DROP TABLE worker_session_archives;DROP TABLE worker_diagnostics_archives;DROP TABLE worker_orphan_diagnostics;DROP TABLE worker_removal_operations;DROP TABLE workspace_worker_retention_policies;DROP TABLE workspace_worker_retention_policy_revisions;DELETE FROM __yoi_schema_migrations WHERE version>=28;")?;Ok(())}).unwrap(); } let reopened = SqliteWorkspaceStore::open(&path).unwrap(); let p = reopened.worker_retention_policy("legacy").unwrap().unwrap(); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 95ce0bf4..5f2f4d55 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -62,15 +62,15 @@ use crate::companion::{ }; use crate::config::{BackendRuntimesConfigFile, RemoteRuntimeConfigFile, resolve_remote_runtime}; use crate::hosts::{ - ConfigBundleCheckResult, ConfigBundleSyncResult, DiagnosticSeverity, EmbeddedWorkerRuntime, - HostSummary, RemoteRuntimeConfig, RemoteWorkerRuntime, RuntimeDiagnostic, RuntimeRegistry, - RuntimeRegistryError, RuntimeRegistryUnregisterResult, RuntimeSummary, TicketWorkerRole, - WorkerCapabilitySummary, WorkerCompletionsRequest, WorkerCompletionsResult, - WorkerImplementationSummary, WorkerInputKind, WorkerInputRequest, WorkerInputResult, - WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult, - WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult, - WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerTicketAssignmentRequest, - WorkerWorkspaceSummary, + ConfigBundleCheckResult, ConfigBundleSyncResult, DiagnosticSeverity, EMBEDDED_RUNTIME_ID, + EmbeddedWorkerRuntime, HostSummary, RemoteRuntimeConfig, RemoteWorkerRuntime, + RuntimeDiagnostic, RuntimeRegistry, RuntimeRegistryError, RuntimeRegistryUnregisterResult, + RuntimeSummary, TicketWorkerRole, WorkerCapabilitySummary, WorkerCompletionsRequest, + WorkerCompletionsResult, WorkerImplementationSummary, WorkerInputKind, WorkerInputRequest, + WorkerInputResult, WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState, + WorkerRestoreResult, WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, + WorkerSpawnResult, WorkerSpawnWorkingDirectoryRequest, WorkerSummary, + WorkerTicketAssignmentRequest, WorkerWorkspaceSummary, }; use crate::identity::WorkspaceIdentity; use crate::memory_backend::execute_memory_backend_operation_with_authority; @@ -250,8 +250,8 @@ const ORCHESTRATOR_ATTENTION_PROMPT: &str = include_str!(concat!( #[derive(Clone)] pub struct WorkspaceApi { - config: ServerConfig, - store: Arc<dyn ControlPlaneStore>, + pub(crate) config: ServerConfig, + pub(crate) store: Arc<dyn ControlPlaneStore>, authority: SqliteWorkspaceAuthority, runtime: Arc<RuntimeRegistry>, companion: Arc<CompanionConsole>, @@ -269,6 +269,15 @@ impl WorkspaceApi { let resource_broker = BackendResourceBroker::default(); let execution_backend = WorkerRuntimeExecutionBackend::new( ProfileRuntimeWorkerFactory::new(config.workspace_root.clone()) + .with_embedded_worker_mutation_dispatcher( + EMBEDDED_RUNTIME_ID, + Arc::new( + crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( + config.clone(), + store.clone(), + ), + ), + ) .with_runtime_store_dir(config.embedded_runtime_store_root.clone()) .with_resource_client(Arc::new(resource_broker.clone())), ) @@ -394,7 +403,7 @@ impl WorkspaceApi { &self.runtime_subscription_broker } - fn workspace_api_ref(&self, runtime_id: &str) -> WorkspaceApiRef { + fn workspace_api_ref(&self, _runtime_id: &str) -> WorkspaceApiRef { WorkspaceApiRef { workspace_id: self.config.workspace_id.clone(), base_url: self @@ -404,7 +413,6 @@ impl WorkspaceApi { .unwrap_or_else(|| "http://127.0.0.1:8787".to_string()) .trim_end_matches('/') .to_string(), - runtime_id: Some(runtime_id.to_string()), } } @@ -1118,6 +1126,10 @@ pub fn build_router(api: WorkspaceApi) -> Router { "/api/w/{workspace_id}/companion/cancel", post(scoped_post_companion_cancel), ) + .route( + "/api/w/{workspace_id}/workers/remove", + post(scoped_worker_remove_source_boundary), + ) .route( "/api/runtimes/{runtime_id}/workers", get(list_runtime_workers).post(create_runtime_worker), @@ -4811,6 +4823,71 @@ async fn scoped_workspace_protocol_ws( .into_response()) } +#[derive(Debug, Deserialize)] +struct WorkerRemoveBoundaryRequest { + target_runtime_id: String, + target_worker_id: String, +} + +async fn scoped_worker_remove_source_boundary( + State(api): State<WorkspaceApi>, + AxumPath(path): AxumPath<ScopedWorkspacePath>, + headers: HeaderMap, + Json(request): Json<WorkerRemoveBoundaryRequest>, +) -> Response { + if let Err(error) = validate_workspace_scope(&api, &path.workspace_id) { + return error.into_response(); + } + let proof = match crate::worker_source::presented_worker_remove_source(&headers, None) { + Ok(proof) => proof, + Err(error) => { + return ( + StatusCode::UNAUTHORIZED, + Json(serde_json::json!({ "error": error.to_string() })), + ) + .into_response(); + } + }; + match crate::worker_source::verify_worker_remove_source( + &api, + proof, + &request.target_runtime_id, + &request.target_worker_id, + ) + .await + { + Ok(source) => ( + StatusCode::NOT_IMPLEMENTED, + Json(serde_json::json!({ + "error": "WorkerRemove lifecycle is deferred to its consumer Ticket", + "source": { + "runtime_id": source.runtime_id, + "worker_id": source.worker_id, + "actor_kind": source.actor_kind, + "permission": source.permission, + } + })), + ) + .into_response(), + Err(error) => { + let status = match error { + crate::worker_source::WorkerMutationSourceProofError::Replay => { + StatusCode::CONFLICT + } + crate::worker_source::WorkerMutationSourceProofError::Authority(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } + _ => StatusCode::FORBIDDEN, + }; + ( + status, + Json(serde_json::json!({ "error": error.to_string() })), + ) + .into_response() + } + } +} + async fn scoped_list_workers( State(api): State<WorkspaceApi>, AxumPath(path): AxumPath<ScopedWorkspacePath>, @@ -10561,12 +10638,19 @@ mod tests { use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; use tower::ServiceExt; + use worker_runtime::auth::{ + RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner, WORKER_REMOVE_PERMISSION, + decode_worker_mutation_source_claims, + }; use worker_runtime::resource::BackendResourceClient; + use worker_runtime::worker_source::{ + RuntimeOwnedWorkerMutationProof, RuntimeWorkerMutationSourceAuthority, + }; use worker_runtime::working_directory::WorkingDirectoryMaterializer; use crate::hosts::{ - TicketWorkerRole, WorkerInputKind, WorkerOperationState, WorkerSpawnAcceptanceRequirement, - WorkerSpawnIntent, + RemoteRuntimeAuthConfig, RuntimeCapabilitySummary, TicketWorkerRole, WorkerInputKind, + WorkerOperationState, WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, }; use crate::store::{ MemoryDocumentRecord, MemoryStagingRecord, ObjectiveRecord, ObjectiveResourceRecord, @@ -10643,11 +10727,10 @@ mod tests { const TEST_REPOSITORY_ID: &str = "main"; const TEST_CREATED_AT: &str = "2026-06-23T06:43:28Z"; - fn test_worker_workspace_api(runtime_id: &str) -> WorkspaceApiRef { + fn test_worker_workspace_api(_runtime_id: &str) -> WorkspaceApiRef { WorkspaceApiRef { workspace_id: TEST_WORKSPACE_ID.to_string(), base_url: "http://127.0.0.1:8787".to_string(), - runtime_id: Some(runtime_id.to_string()), } } @@ -13141,6 +13224,359 @@ mod tests { .unwrap() } + #[tokio::test] + async fn destructive_worker_remove_rejects_browser_and_legacy_source_headers() { + let headers = HeaderMap::new(); + assert!(matches!( + crate::worker_source::presented_worker_remove_source(&headers, None), + Err(crate::worker_source::WorkerMutationSourceProofError::Missing) + )); + + let mut spoofed = HeaderMap::new(); + spoofed.insert("x-yoi-runtime-id", "runtime-spoofed".parse().unwrap()); + spoofed.insert("x-yoi-worker-id", "worker-spoofed".parse().unwrap()); + assert!(matches!( + crate::worker_source::presented_worker_remove_source(&spoofed, None), + Err(crate::worker_source::WorkerMutationSourceProofError::Missing) + )); + + let temp = tempfile::tempdir().unwrap(); + let app = build_router(test_api(temp.path()).await); + let body = r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker"}"#; + let browser = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(browser.status(), StatusCode::UNAUTHORIZED); + let legacy = app + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove")) + .header(CONTENT_TYPE, "application/json") + .header("x-yoi-runtime-id", "runtime-spoofed") + .header("x-yoi-worker-id", "worker-spoofed") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(legacy.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn embedded_worker_remove_proof_derives_source_and_rejects_replay_and_wrong_target() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + seed_worker_source_member(&api, EMBEDDED_RUNTIME_ID, "7"); + let scope = worker_runtime::RuntimeWorkspaceScope::new( + api.config.workspace_id.clone(), + "server-unused-for-embedded", + ); + let authority = RuntimeWorkerMutationSourceAuthority::embedded( + EMBEDDED_RUNTIME_ID, + &api.config.workspace_id, + ); + let RuntimeOwnedWorkerMutationProof::InProcess(proof) = authority + .issue_worker_remove(&scope, "7", "runtime-target", "target-worker") + .unwrap() + else { + panic!("embedded Runtime must produce in-process claims"); + }; + + let wrong_target = crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::InProcess(proof.clone()), + "runtime-target", + "different-worker", + ) + .await; + assert!(matches!( + wrong_target, + Err(crate::worker_source::WorkerMutationSourceProofError::Invalid) + )); + + let verified = crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::InProcess(proof.clone()), + "runtime-target", + "target-worker", + ) + .await + .unwrap(); + assert_eq!(verified.runtime_id, EMBEDDED_RUNTIME_ID); + assert_eq!(verified.worker_id, "7"); + assert_eq!(verified.permission, WORKER_REMOVE_PERMISSION); + + let replay = crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::InProcess(proof), + "runtime-target", + "target-worker", + ) + .await; + assert!(matches!( + replay, + Err(crate::worker_source::WorkerMutationSourceProofError::Replay) + )); + + let RuntimeOwnedWorkerMutationProof::InProcess(fresh_proof) = authority + .issue_worker_remove(&scope, "7", "runtime-target", "target-worker") + .unwrap() + else { + panic!("embedded Runtime must produce in-process claims"); + }; + let dispatcher = crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( + api.config.clone(), + api.store.clone(), + ); + let response = + worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher::execute_worker_remove( + &dispatcher, + fresh_proof, + "runtime-target", + "target-worker", + ) + .unwrap(); + assert_eq!(response.status, 501); + } + + #[tokio::test] + async fn remote_worker_remove_proof_requires_current_runtime_trust_scope_and_catalog_member() { + let temp = tempfile::tempdir().unwrap(); + let identity = RuntimeIdentityMaterial::generate("runtime-remote").unwrap(); + let mut config = test_server_config(temp.path()); + config.remote_runtime_sources.push(RemoteRuntimeConfig { + runtime_id: "runtime-remote".to_string(), + display_name: "Remote Runtime".to_string(), + base_url: "https://runtime.invalid".to_string(), + bearer_token: None, + auth: Some(RemoteRuntimeAuthConfig { + server_id: "server-main".to_string(), + server_private_key: identity.private_key.clone(), + }), + cached_capabilities: RuntimeCapabilitySummary { + can_list_hosts: true, + can_list_workers: true, + can_get_worker: true, + can_spawn_worker: true, + can_stop_worker: true, + has_workspace_fs: false, + has_shell: false, + has_git: false, + supports_worktrees: false, + supports_backend_internal_tools: false, + workspace_scope: TEST_WORKSPACE_ID.to_string(), + max_workers: 1, + os: "test".to_string(), + arch: "test".to_string(), + }, + cached_status: "connected".to_string(), + timeout: std::time::Duration::from_secs(1), + }); + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let trust = crate::store::TrustedRuntimeRecord { + runtime_id: "runtime-remote".to_string(), + display_name: "Remote Runtime".to_string(), + base_url: "https://runtime.invalid".to_string(), + public_key: identity.public_key.clone(), + created_at: "2026-08-11T00:00:00Z".to_string(), + updated_at: "2026-08-11T00:00:00Z".to_string(), + revoked_at: None, + }; + store.upsert_trusted_runtime(&trust).unwrap(); + let api = WorkspaceApi::new_with_execution_backend( + config, + Arc::new(store), + Arc::new(DeterministicExecutionBackend::default()), + ) + .await + .unwrap(); + seed_worker_source_member(&api, "runtime-remote", "7"); + + let signer = RuntimeWorkerMutationSourceSigner::from_identity(&identity); + let token = signer + .issue_worker_remove( + "server-main", + &api.config.workspace_id, + "7", + "runtime-target", + "target-worker", + 60, + ) + .unwrap(); + let verified = crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&token), + "runtime-target", + "target-worker", + ) + .await + .unwrap(); + assert_eq!(verified.runtime_id, "runtime-remote"); + assert_eq!(verified.worker_id, "7"); + + let wrong_scope = signer + .issue_worker_remove( + "server-wrong", + &api.config.workspace_id, + "7", + "runtime-target", + "target-worker", + 60, + ) + .unwrap(); + assert!(matches!( + crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&wrong_scope), + "runtime-target", + "target-worker", + ) + .await, + Err(crate::worker_source::WorkerMutationSourceProofError::WrongAudience) + )); + + let wrong_workspace = signer + .issue_worker_remove( + "server-main", + "workspace-other", + "7", + "runtime-target", + "target-worker", + 60, + ) + .unwrap(); + assert!(matches!( + crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&wrong_workspace), + "runtime-target", + "target-worker", + ) + .await, + Err(crate::worker_source::WorkerMutationSourceProofError::WrongWorkspace) + )); + + let missing_worker = signer + .issue_worker_remove( + "server-main", + &api.config.workspace_id, + "999", + "runtime-target", + "target-worker", + 60, + ) + .unwrap(); + assert!(matches!( + crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&missing_worker), + "runtime-target", + "target-worker", + ) + .await, + Err(crate::worker_source::WorkerMutationSourceProofError::WorkerCatalogMembership) + )); + + let mut expired_claims = decode_worker_mutation_source_claims(&token).unwrap(); + expired_claims.iat = 1; + expired_claims.exp = 2; + expired_claims.jti = "expired-proof".to_string(); + let expired = signer.sign(&expired_claims).unwrap(); + assert!(matches!( + crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&expired), + "runtime-target", + "target-worker", + ) + .await, + Err(crate::worker_source::WorkerMutationSourceProofError::Expired) + )); + + let route_token = signer + .issue_worker_remove( + "server-main", + &api.config.workspace_id, + "7", + "runtime-target", + "target-worker", + 60, + ) + .unwrap(); + let route_response = build_router(api.clone()) + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove")) + .header(CONTENT_TYPE, "application/json") + .header( + worker_runtime::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER, + route_token, + ) + .body(Body::from( + r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker"}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(route_response.status(), StatusCode::NOT_IMPLEMENTED); + + let mut revoked = trust; + revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string()); + let authority = SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap(); + authority.upsert_trusted_runtime(&revoked).unwrap(); + let revoked_token = signer + .issue_worker_remove( + "server-main", + &api.config.workspace_id, + "7", + "runtime-target", + "target-worker", + 60, + ) + .unwrap(); + assert!(matches!( + crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&revoked_token), + "runtime-target", + "target-worker", + ) + .await, + Err(crate::worker_source::WorkerMutationSourceProofError::RevokedRuntimeTrust) + )); + } + + fn seed_worker_source_member(api: &WorkspaceApi, runtime_id: &str, worker_id: &str) { + let now = now_registry_timestamp(); + api.store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: api.config.workspace_id.clone(), + worker: RuntimeWorkerRef::new(runtime_id, worker_id), + display_name: worker_id.to_string(), + profile: None, + retention_state: "normal".to_string(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: now.clone(), + updated_at: now, + }) + .unwrap(); + } + fn seed_cleanup_worker( api: &WorkspaceApi, runtime_worker_id: u64, diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 8b3786e8..f5fa9b83 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -161,6 +161,11 @@ const MIGRATIONS: &[Migration] = &[ name: "create Worker retention authority", apply: crate::retention::create_worker_retention_tables, }, + Migration { + version: 29, + name: "create Worker mutation source proof replay guard", + apply: create_worker_mutation_source_proof_replay_guard, + }, ]; struct Migration { @@ -460,6 +465,15 @@ pub trait ControlPlaneStore: Send + Sync { async fn schema_version(&self) -> Result<i64>; async fn upsert_workspace(&self, record: &WorkspaceRecord) -> Result<()>; async fn get_workspace(&self, workspace_id: &str) -> Result<Option<WorkspaceRecord>>; + async fn get_trusted_runtime(&self, runtime_id: &str) -> Result<Option<TrustedRuntimeRecord>>; + async fn consume_worker_mutation_source_jti( + &self, + runtime_id: &str, + jti: &str, + expires_at: u64, + now_seconds: u64, + consumed_at: &str, + ) -> Result<bool>; fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>>; fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>; fn get_repository( @@ -896,6 +910,44 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + async fn get_trusted_runtime(&self, runtime_id: &str) -> Result<Option<TrustedRuntimeRecord>> { + self.with_conn(|conn| { + conn.query_row( + r#"SELECT runtime_id, display_name, base_url, public_key, created_at, updated_at, revoked_at + FROM trusted_runtime_records WHERE runtime_id = ?1"#, + params![runtime_id], + read_trusted_runtime_record, + ) + .optional() + .map_err(Error::from) + }) + } + + async fn consume_worker_mutation_source_jti( + &self, + runtime_id: &str, + jti: &str, + expires_at: u64, + now_seconds: u64, + consumed_at: &str, + ) -> Result<bool> { + self.with_conn(|conn| { + let transaction = conn.unchecked_transaction()?; + transaction.execute( + "DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1", + params![now_seconds], + )?; + let inserted = transaction.execute( + r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( + runtime_id, jti, expires_at, consumed_at + ) VALUES (?1, ?2, ?3, ?4)"#, + params![runtime_id, jti, expires_at, consumed_at], + )?; + transaction.commit()?; + Ok(inserted == 1) + }) + } + fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>> { self.with_conn(|conn| { let mut stmt = conn.prepare( @@ -4326,6 +4378,23 @@ fn current_schema_version(conn: &Connection) -> Result<i64> { .map_err(Error::from) } +fn create_worker_mutation_source_proof_replay_guard(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS worker_mutation_source_proof_jtis ( + runtime_id TEXT NOT NULL, + jti TEXT NOT NULL, + expires_at INTEGER NOT NULL, + consumed_at TEXT NOT NULL, + PRIMARY KEY (runtime_id, jti) + ); + CREATE INDEX IF NOT EXISTS idx_worker_mutation_source_proof_jtis_expiry + ON worker_mutation_source_proof_jtis(expires_at); + "#, + )?; + Ok(()) +} + fn apply_migrations(conn: &Connection) -> Result<()> { let current = current_schema_version(conn)?; for migration in MIGRATIONS @@ -4915,7 +4984,7 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 28); + assert_eq!(current_schema_version(&conn).unwrap(), 29); assert!(table_exists(&conn, "worker_workdir_attachment_reservations").unwrap()); } @@ -4948,7 +5017,7 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 28); + assert_eq!(current_schema_version(&conn).unwrap(), 29); assert!(table_exists(&conn, "flow_sources").unwrap()); assert!(table_exists(&conn, "flow_source_revisions").unwrap()); assert!(!table_exists(&conn, "flow_instances").unwrap()); @@ -5015,7 +5084,7 @@ INSERT INTO worker_workdir_attachment_reservations ( apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 28); + assert_eq!(current_schema_version(&conn).unwrap(), 29); let repositories_sql: String = conn .query_row( "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'repositories'", @@ -5195,7 +5264,7 @@ INSERT INTO workdir_registry ( let db = dir.path().join("control-plane.sqlite"); let store = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 28); + assert_eq!(store.schema_version().await.unwrap(), 29); assert!( !store .with_conn(|conn| table_exists(conn, "worker_workspace_credentials")) @@ -5212,7 +5281,7 @@ INSERT INTO workdir_registry ( store.upsert_workspace(&record).await.unwrap(); let reopened = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(reopened.schema_version().await.unwrap(), 28); + assert_eq!(reopened.schema_version().await.unwrap(), 29); assert_eq!( reopened.get_workspace("local-dev").await.unwrap(), Some(record) @@ -5759,7 +5828,7 @@ INSERT INTO workdir_registry ( .unwrap(); let store = SqliteWorkspaceStore::from_connection(conn).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 28); + assert_eq!(store.schema_version().await.unwrap(), 29); store .with_conn(|conn| { @@ -5948,7 +6017,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn repository_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 28); + assert_eq!(store.schema_version().await.unwrap(), 29); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -6014,7 +6083,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn memory_authority_records_round_trip_and_close_staging() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 28); + assert_eq!(store.schema_version().await.unwrap(), 29); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -6277,7 +6346,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn account_and_login_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 28); + assert_eq!(store.schema_version().await.unwrap(), 29); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { account_id: "acct-user-alice".to_string(), @@ -6455,6 +6524,51 @@ CREATE TABLE ticket_assignment_operations ( ); } + #[tokio::test] + async fn worker_mutation_source_jti_replay_guard_survives_reopen() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("server.db"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + store + .upsert_trusted_runtime(&TrustedRuntimeRecord { + runtime_id: "runtime-a".to_string(), + display_name: "Runtime A".to_string(), + base_url: "https://runtime.invalid".to_string(), + public_key: "public-key".to_string(), + created_at: "2026-08-11T00:00:00Z".to_string(), + updated_at: "2026-08-11T00:00:00Z".to_string(), + revoked_at: None, + }) + .unwrap(); + assert!( + store + .consume_worker_mutation_source_jti( + "runtime-a", + "proof-1", + 2_000, + 1_000, + "2026-08-11T00:00:00Z", + ) + .await + .unwrap() + ); + drop(store); + + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + assert!( + !reopened + .consume_worker_mutation_source_jti( + "runtime-a", + "proof-1", + 2_000, + 1_001, + "2026-08-11T00:00:01Z", + ) + .await + .unwrap() + ); + } + fn table_names(conn: &Connection) -> BTreeSet<String> { let mut stmt = conn .prepare( diff --git a/crates/workspace-server/src/worker_source.rs b/crates/workspace-server/src/worker_source.rs new file mode 100644 index 00000000..932250ad --- /dev/null +++ b/crates/workspace-server/src/worker_source.rs @@ -0,0 +1,288 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use axum::http::HeaderMap; +use worker_runtime::auth::{ + WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims, + WorkerMutationSourceExpectation, decode_worker_mutation_source_claims, + verify_worker_mutation_source_proof, +}; +use worker_runtime::worker_source::InProcessWorkerMutationProof; + +use crate::hosts::RemoteRuntimeConfig; +use crate::server::WorkspaceApi; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PresentedWorkerMutationSourceProof<'a> { + Remote(&'a str), + InProcess(InProcessWorkerMutationProof), +} + +pub fn presented_worker_remove_source<'a>( + headers: &'a HeaderMap, + in_process: Option<InProcessWorkerMutationProof>, +) -> Result<PresentedWorkerMutationSourceProof<'a>, WorkerMutationSourceProofError> { + if let Some(claims) = in_process { + return Ok(PresentedWorkerMutationSourceProof::InProcess(claims)); + } + headers + .get(worker_runtime::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER) + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(PresentedWorkerMutationSourceProof::Remote) + .ok_or(WorkerMutationSourceProofError::Missing) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VerifiedWorkerMutationSource { + pub runtime_id: String, + pub worker_id: String, + pub actor_kind: WorkerMutationActorKind, + pub permission: String, + pub jti: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum WorkerMutationSourceProofError { + #[error("Worker mutation source proof is required")] + Missing, + #[error("Worker mutation source proof is invalid")] + Invalid, + #[error("Worker mutation source proof is not authorized for this Server")] + WrongAudience, + #[error("Worker mutation source proof is not authorized for this Workspace")] + WrongWorkspace, + #[error("Worker mutation source proof actor is not allowed")] + WrongActor, + #[error("Worker mutation source proof lacks `{0}` permission")] + MissingPermission(String), + #[error("Worker mutation source proof is expired")] + Expired, + #[error("Runtime trust is missing or revoked")] + RevokedRuntimeTrust, + #[error("Worker mutation source proof was already consumed")] + Replay, + #[error("source Worker is not a current member of this Workspace Runtime catalog")] + WorkerCatalogMembership, + #[error("source proof authority failed: {0}")] + Authority(String), +} + +pub async fn verify_worker_remove_source( + api: &WorkspaceApi, + proof: PresentedWorkerMutationSourceProof<'_>, + target_runtime_id: &str, + target_worker_id: &str, +) -> Result<VerifiedWorkerMutationSource, WorkerMutationSourceProofError> { + verify_worker_remove_source_with( + &api.config, + &api.store, + proof, + target_runtime_id, + target_worker_id, + ) + .await +} + +async fn verify_worker_remove_source_with( + config: &crate::server::ServerConfig, + store: &std::sync::Arc<dyn crate::store::ControlPlaneStore>, + proof: PresentedWorkerMutationSourceProof<'_>, + target_runtime_id: &str, + target_worker_id: &str, +) -> Result<VerifiedWorkerMutationSource, WorkerMutationSourceProofError> { + let required_permission = worker_runtime::auth::WORKER_REMOVE_PERMISSION; + let now = unix_now_seconds(); + let claims = match proof { + PresentedWorkerMutationSourceProof::Remote(token) => { + let unverified = decode_worker_mutation_source_claims(token) + .map_err(|_| WorkerMutationSourceProofError::Invalid)?; + let audience = remote_audience(config, &unverified.iss)?; + let trusted = store + .get_trusted_runtime(&unverified.iss) + .await + .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))? + .filter(|record| record.revoked_at.is_none()) + .ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?; + let expected = WorkerMutationSourceExpectation { + runtime_id: &unverified.iss, + audience, + workspace_id: &config.workspace_id, + worker_id: None, + actor_kind: WorkerMutationActorKind::Worker, + operation: WorkerMutationOperation::WorkerRemove, + target_runtime_id, + target_worker_id, + permission: required_permission, + }; + verify_worker_mutation_source_proof(&trusted.public_key, token, &expected, now) + .map_err(map_auth_error)? + } + PresentedWorkerMutationSourceProof::InProcess(proof) => { + let claims = proof.into_claims(); + if config + .remote_runtime_sources + .iter() + .any(|runtime| runtime.runtime_id == claims.iss) + { + return Err(WorkerMutationSourceProofError::Invalid); + } + validate_in_process_claims( + &claims, + &format!("embedded:{}", config.workspace_id), + &config.workspace_id, + target_runtime_id, + target_worker_id, + required_permission, + now, + )?; + claims + } + }; + + let worker = worker_runtime::identity::RuntimeWorkerRef { + runtime_id: claims.iss.clone(), + worker_id: claims.worker_id.clone(), + }; + let member = store + .get_worker_registry(&config.workspace_id, &worker) + .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + if member.is_none() { + return Err(WorkerMutationSourceProofError::WorkerCatalogMembership); + } + + let consumed_at = chrono::Utc::now().to_rfc3339(); + let consumed = store + .consume_worker_mutation_source_jti(&claims.iss, &claims.jti, claims.exp, now, &consumed_at) + .await + .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; + if !consumed { + return Err(WorkerMutationSourceProofError::Replay); + } + + Ok(VerifiedWorkerMutationSource { + runtime_id: claims.iss, + worker_id: claims.worker_id, + actor_kind: claims.actor_kind, + permission: claims.permission, + jti: claims.jti, + }) +} + +#[derive(Clone)] +pub(crate) struct EmbeddedServerWorkerMutationDispatcher { + config: crate::server::ServerConfig, + store: std::sync::Arc<dyn crate::store::ControlPlaneStore>, +} + +impl EmbeddedServerWorkerMutationDispatcher { + pub(crate) fn new( + config: crate::server::ServerConfig, + store: std::sync::Arc<dyn crate::store::ControlPlaneStore>, + ) -> Self { + Self { config, store } + } +} + +impl worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher + for EmbeddedServerWorkerMutationDispatcher +{ + fn execute_worker_remove( + &self, + proof: InProcessWorkerMutationProof, + target_runtime_id: &str, + target_worker_id: &str, + ) -> Result< + worker::WorkspaceResponse, + worker_runtime::worker_source::RuntimeWorkerMutationForwardError, + > { + futures::executor::block_on(verify_worker_remove_source_with( + &self.config, + &self.store, + PresentedWorkerMutationSourceProof::InProcess(proof), + target_runtime_id, + target_worker_id, + )) + .map_err(|error| { + worker_runtime::worker_source::RuntimeWorkerMutationForwardError::Embedded( + error.to_string(), + ) + })?; + Ok(worker::WorkspaceResponse { + status: 501, + body: "WorkerRemove lifecycle is not implemented by this operation boundary" + .to_string(), + }) + } +} + +fn remote_audience<'a>( + config: &'a crate::server::ServerConfig, + runtime_id: &str, +) -> Result<&'a str, WorkerMutationSourceProofError> { + config + .remote_runtime_sources + .iter() + .find(|runtime| runtime.runtime_id == runtime_id) + .and_then(|runtime: &RemoteRuntimeConfig| runtime.auth.as_ref()) + .map(|auth| auth.server_id.as_str()) + .ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust) +} + +fn validate_in_process_claims( + claims: &WorkerMutationSourceClaims, + audience: &str, + workspace_id: &str, + target_runtime_id: &str, + target_worker_id: &str, + permission: &str, + now: u64, +) -> Result<(), WorkerMutationSourceProofError> { + if claims.aud != audience { + return Err(WorkerMutationSourceProofError::WrongAudience); + } + if claims.workspace_id != workspace_id { + return Err(WorkerMutationSourceProofError::WrongWorkspace); + } + if claims.actor_kind != WorkerMutationActorKind::Worker { + return Err(WorkerMutationSourceProofError::WrongActor); + } + if claims.operation != WorkerMutationOperation::WorkerRemove + || claims.target_runtime_id != target_runtime_id + || claims.target_worker_id != target_worker_id + { + return Err(WorkerMutationSourceProofError::Invalid); + } + if claims.permission != permission { + return Err(WorkerMutationSourceProofError::MissingPermission( + permission.to_string(), + )); + } + if claims.exp <= now || claims.iat > now.saturating_add(60) || claims.jti.trim().is_empty() { + return Err(WorkerMutationSourceProofError::Expired); + } + Ok(()) +} + +fn map_auth_error(error: worker_runtime::auth::RuntimeAuthError) -> WorkerMutationSourceProofError { + use worker_runtime::auth::RuntimeAuthError; + match error { + RuntimeAuthError::WrongAudience { .. } => WorkerMutationSourceProofError::WrongAudience, + RuntimeAuthError::WrongWorkspace { .. } => WorkerMutationSourceProofError::WrongWorkspace, + RuntimeAuthError::WrongActorKind => WorkerMutationSourceProofError::WrongActor, + RuntimeAuthError::WrongOperation | RuntimeAuthError::WrongMutationTarget => { + WorkerMutationSourceProofError::Invalid + } + RuntimeAuthError::MissingPermission(permission) => { + WorkerMutationSourceProofError::MissingPermission(permission) + } + RuntimeAuthError::Expired => WorkerMutationSourceProofError::Expired, + _ => WorkerMutationSourceProofError::Invalid, + } +} + +fn unix_now_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} From 8ae930c5fc81acb2c60de15add07e016a1552edd Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Wed, 12 Aug 2026 17:00:32 +0900 Subject: [PATCH 08/12] worker: add guarded WorkerRemove lifecycle --- crates/worker-runtime/src/http_server.rs | 81 ++ crates/worker-runtime/src/retention.rs | 4 + crates/worker-runtime/src/worker_source.rs | 54 +- .../src/feature/builtin/manage_worker.rs | 310 ++++++-- crates/worker/src/prompt/catalog.rs | 15 + crates/worker/src/worker.rs | 2 + crates/workspace-server/src/hosts.rs | 110 ++- crates/workspace-server/src/retention.rs | 186 ++++- crates/workspace-server/src/server.rs | 725 +++++++++++++++++- crates/workspace-server/src/store.rs | 149 ++++ crates/workspace-server/src/worker_source.rs | 55 +- resources/prompts/role/orchestrator.md | 2 + 12 files changed, 1565 insertions(+), 128 deletions(-) diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index dce58519..6e71276a 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -21,6 +21,9 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck}; use crate::management::{RuntimeSummary, WorkerDeleteResult}; #[cfg(feature = "ws-server")] use crate::observation::WorkerObservationCursor; +use crate::retention::{ + WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory, +}; #[cfg(feature = "ws-server")] use crate::runtime::RuntimeSubscriptionRecvError; use crate::{Runtime, RuntimeWorkspaceScope}; @@ -217,6 +220,14 @@ fn runtime_http_router_with_optional_auth( "/v1/workers/{worker_id}", get(get_worker).delete(delete_worker), ) + .route( + "/v1/workers/{worker_id}/retention/inventory", + get(worker_retention_inventory), + ) + .route( + "/v1/workers/{worker_id}/retention/execute", + post(execute_worker_retention), + ) .route("/v1/workers/{worker_id}/input", post(send_worker_input)) .route("/v1/workers/{worker_id}/restore", post(restore_worker)) .route( @@ -1220,6 +1231,61 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event { } } +async fn worker_retention_inventory( + State(state): State<RuntimeHttpState>, + auth: Option<Extension<RuntimeAuthContext>>, + Path(worker_id): Path<String>, +) -> RestResult<WorkerRetentionInventory> { + let scope = auth_workspace_scope(&state, auth.as_ref())?.ok_or_else(|| { + RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "workspace_scope_required", + "Worker retention inventory requires workspace-scoped authorization", + ) + })?; + let worker_ref = worker_ref_for(&state.runtime, worker_id)?; + state + .runtime + .worker_retention_inventory(&scope.workspace_id, &worker_ref) + .map(Json) + .map_err(RuntimeHttpRestError::runtime) +} + +async fn execute_worker_retention( + State(state): State<RuntimeHttpState>, + auth: Option<Extension<RuntimeAuthContext>>, + Path(worker_id): Path<String>, + body: Result<Json<WorkerRetentionExecutionRequest>, JsonRejection>, +) -> RestResult<WorkerRetentionExecutionResult> { + let Json(request) = body.map_err(RuntimeHttpRestError::json_rejection)?; + if request.worker_id.to_string() != worker_id { + return Err(RuntimeHttpRestError::new( + StatusCode::BAD_REQUEST, + "worker_id_mismatch", + "Retention request worker_id does not match the route", + )); + } + let scope = auth_workspace_scope(&state, auth.as_ref())?.ok_or_else(|| { + RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "workspace_scope_required", + "Worker retention execution requires workspace-scoped authorization", + ) + })?; + if request.workspace_id != scope.workspace_id { + return Err(RuntimeHttpRestError::new( + StatusCode::NOT_FOUND, + "worker_not_found", + "Worker was not found in the authenticated Workspace", + )); + } + state + .runtime + .execute_worker_retention(&request) + .map(Json) + .map_err(RuntimeHttpRestError::runtime) +} + async fn send_worker_input( State(state): State<RuntimeHttpState>, auth: Option<Extension<RuntimeAuthContext>>, @@ -1473,6 +1539,9 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s if path.ends_with("/completions") { return Some("workers:read"); } + if path.contains("/retention/") { + return Some("workers:delete"); + } if path.starts_with("/v1/workers/") && *method == Method::DELETE { return Some("workers:delete"); } @@ -2066,6 +2135,18 @@ mod tests { } } + #[test] + fn retention_routes_require_worker_delete_permission() { + assert_eq!( + required_runtime_permission(&Method::GET, "/v1/workers/worker-1/retention/inventory",), + Some("workers:delete") + ); + assert_eq!( + required_runtime_permission(&Method::POST, "/v1/workers/worker-1/retention/execute",), + Some("workers:delete") + ); + } + #[test] fn workdir_routes_require_dedicated_operation_permission() { assert_eq!( diff --git a/crates/worker-runtime/src/retention.rs b/crates/worker-runtime/src/retention.rs index 699f660d..a81f37cd 100644 --- a/crates/worker-runtime/src/retention.rs +++ b/crates/worker-runtime/src/retention.rs @@ -117,6 +117,7 @@ pub struct WorkerRetentionExecutionRequest { pub workspace_id: String, pub source_runtime_id: String, pub worker_id: WorkerId, + pub expected_worker_revision: String, pub expected_run_generation: u64, pub source_created_at: String, pub removed_at: String, @@ -155,6 +156,7 @@ pub struct WorkerSessionArchiveManifest { pub struct WorkerRetentionExecutionResult { pub operation_id: String, pub input_fingerprint: String, + pub expected_worker_revision: String, pub worker_id: WorkerId, pub session_disposition: SessionDisposition, pub diagnostics_disposition: DiagnosticsDisposition, @@ -521,6 +523,7 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider { let mut result = WorkerRetentionExecutionResult { operation_id: request.operation_id.clone(), input_fingerprint: request.input_fingerprint.clone(), + expected_worker_revision: request.expected_worker_revision.clone(), worker_id: request.worker_id, session_disposition: request.session_disposition, diagnostics_disposition: request.diagnostics_disposition, @@ -1291,6 +1294,7 @@ mod tests { WorkerRetentionExecutionRequest { operation_id: "operation-a".to_string(), input_fingerprint: "fingerprint-a".to_string(), + expected_worker_revision: "revision-a".to_string(), archive_id: (disposition == SessionDisposition::Archive) .then(|| "archive-a".to_string()), workspace_id: "workspace-a".to_string(), diff --git a/crates/worker-runtime/src/worker_source.rs b/crates/worker-runtime/src/worker_source.rs index b9fdd0c1..f5d4728e 100644 --- a/crates/worker-runtime/src/worker_source.rs +++ b/crates/worker-runtime/src/worker_source.rs @@ -124,6 +124,8 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync { proof: InProcessWorkerMutationProof, target_runtime_id: &str, target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError>; } @@ -186,6 +188,8 @@ impl RuntimeWorkerMutationForwarder { &self, target_runtime_id: &str, target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> { let proof = self.authority.issue_worker_remove( &self.scope, @@ -205,6 +209,8 @@ impl RuntimeWorkerMutationForwarder { let body = serde_json::json!({ "target_runtime_id": target_runtime_id, "target_worker_id": target_worker_id, + "expected_worker_revision": expected_worker_revision, + "reason": reason, }); let response = client .post(url) @@ -223,7 +229,13 @@ impl RuntimeWorkerMutationForwarder { ( RuntimeWorkerMutationTransport::Embedded { dispatcher }, RuntimeOwnedWorkerMutationProof::InProcess(claims), - ) => dispatcher.execute_worker_remove(claims, target_runtime_id, target_worker_id), + ) => dispatcher.execute_worker_remove( + claims, + target_runtime_id, + target_worker_id, + expected_worker_revision, + reason, + ), _ => Err(RuntimeWorkerMutationForwardError::AuthorityTransportMismatch), } } @@ -316,6 +328,8 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient { &self, target_runtime_id: &str, target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, ) -> Result<WorkspaceResponse, WorkspaceClientError> { self.worker_remove .as_ref() @@ -324,7 +338,12 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient { "Runtime-owned WorkerRemove forwarding is unavailable".to_string(), ) })? - .execute_worker_remove(target_runtime_id, target_worker_id) + .execute_worker_remove( + target_runtime_id, + target_worker_id, + expected_worker_revision, + reason, + ) .map_err(|error| WorkspaceClientError::Request(error.to_string())) } } @@ -500,16 +519,22 @@ mod tests { format!("http://{address}"), ); let response = forwarder - .execute_worker_remove("runtime-target", "worker-target") + .execute_worker_remove( + "runtime-target", + "worker-target", + "revision-7", + "retire obsolete Worker", + ) .unwrap(); assert_eq!(response.status, 204); server.join().unwrap(); let request = received.lock().unwrap().clone(); assert!(request.starts_with("POST /api/w/workspace-a/workers/remove HTTP/1.1")); - assert!(request.contains( - r#"{"target_runtime_id":"runtime-target","target_worker_id":"worker-target"}"# - )); + assert!(request.contains("\"target_runtime_id\":\"runtime-target\"")); + assert!(request.contains("\"target_worker_id\":\"worker-target\"")); + assert!(request.contains("\"expected_worker_revision\":\"revision-7\"")); + assert!(request.contains("\"reason\":\"retire obsolete Worker\"")); let token = request .lines() .find_map(|line| { @@ -541,7 +566,7 @@ mod tests { #[derive(Default)] struct RecordingDispatcher { - seen: Mutex<Option<(WorkerMutationSourceClaims, String, String)>>, + seen: Mutex<Option<(WorkerMutationSourceClaims, String, String, String, String)>>, } impl EmbeddedWorkerMutationDispatcher for RecordingDispatcher { fn execute_worker_remove( @@ -549,11 +574,15 @@ mod tests { proof: InProcessWorkerMutationProof, target_runtime_id: &str, target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> { *self.seen.lock().unwrap() = Some(( proof.into_claims(), target_runtime_id.to_string(), target_worker_id.to_string(), + expected_worker_revision.to_string(), + reason.to_string(), )); Ok(WorkspaceResponse { status: 202, @@ -571,10 +600,15 @@ mod tests { dispatcher.clone(), ); let response = forwarder - .execute_worker_remove("runtime-target", "worker-target") + .execute_worker_remove( + "runtime-target", + "worker-target", + "revision-7", + "retire obsolete Worker", + ) .unwrap(); assert_eq!(response.status, 202); - let (claims, target_runtime_id, target_worker_id) = + let (claims, target_runtime_id, target_worker_id, expected_revision, reason) = dispatcher.seen.lock().unwrap().take().unwrap(); assert_eq!(claims.iss, "runtime-embedded"); assert_eq!(claims.worker_id, "worker-source"); @@ -582,6 +616,8 @@ mod tests { assert_eq!(claims.target_worker_id, "worker-target"); assert_eq!(target_runtime_id, "runtime-target"); assert_eq!(target_worker_id, "worker-target"); + assert_eq!(expected_revision, "revision-7"); + assert_eq!(reason, "retire obsolete Worker"); } #[test] diff --git a/crates/worker/src/feature/builtin/manage_worker.rs b/crates/worker/src/feature/builtin/manage_worker.rs index 9f50f8e3..868a33fa 100644 --- a/crates/worker/src/feature/builtin/manage_worker.rs +++ b/crates/worker/src/feature/builtin/manage_worker.rs @@ -77,6 +77,11 @@ impl FeatureModule for ManageWorkerFeature { self.client.clone(), workspace_id.clone(), ), + WorkerOperation::Remove => definition::<WorkerRemoveInput>( + operation, + self.client.clone(), + workspace_id.clone(), + ), }; context .tools() @@ -149,6 +154,15 @@ struct WorkerStopInput { reason: Option<String>, } +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct WorkerRemoveInput { + runtime_id: String, + worker_id: String, + expected_worker_revision: String, + reason: String, +} + struct WorkspaceWorkerTool { operation: WorkerOperation, client: Arc<dyn WorkspaceClient>, @@ -161,10 +175,17 @@ enum WorkerOperation { Spawn, Stop, Restore, + Remove, } impl WorkerOperation { - const ALL: [Self; 4] = [Self::List, Self::Spawn, Self::Stop, Self::Restore]; + const ALL: [Self; 5] = [ + Self::List, + Self::Spawn, + Self::Stop, + Self::Restore, + Self::Remove, + ]; fn tool_name(self) -> &'static str { match self { @@ -172,6 +193,7 @@ impl WorkerOperation { Self::Spawn => "WorkerSpawn", Self::Stop => "WorkerStop", Self::Restore => "WorkerRestore", + Self::Remove => "WorkerRemove", } } @@ -187,6 +209,9 @@ impl WorkerOperation { Self::Restore => { "Restore a stopped Backend/Runtime Worker session in the current Workspace." } + Self::Remove => { + "Remove an eligible stopped, unassigned, non-internal Worker. Supply the current Worker revision and a bounded reason; Backend validation and retention are authoritative." + } } } } @@ -198,82 +223,107 @@ impl Tool for WorkspaceWorkerTool { input_json: &str, ctx: ToolExecutionContext, ) -> Result<ToolOutput, ToolError> { - let request = match self.operation { - WorkerOperation::List => { - parse::<WorkerListInput>(input_json, "WorkerList")?; - WorkspaceRequest::get(format!("/api/w/{}/workers", self.workspace_id)) + let response = match self.operation { + WorkerOperation::Remove => { + let input = parse::<WorkerRemoveInput>(input_json, "WorkerRemove")?; + let runtime_id = authority_id(&input.runtime_id, "runtime_id")?; + let worker_id = authority_id(&input.worker_id, "worker_id")?; + let expected_worker_revision = + non_empty(input.expected_worker_revision, "expected_worker_revision")?; + let reason = non_empty(input.reason, "reason")?; + if reason.len() > 512 { + return Err(ToolError::ExecutionFailed( + "reason must contain at most 512 bytes".to_string(), + )); + } + self.client + .execute_worker_remove( + &runtime_id, + &worker_id, + &expected_worker_revision, + &reason, + ) + .map_err(|error| ToolError::ExecutionFailed(error.to_string()))? } - WorkerOperation::Spawn => { - let input = parse::<WorkerSpawnInput>(input_json, "WorkerSpawn")?; - let ticket_assignment = input - .ticket_id - .map(|ticket_id| { - let ticket_id = authority_id(&ticket_id, "ticket_id")?; - let call_id = non_empty(ctx.call_id.clone(), "tool call_id")?; - Ok::<_, ToolError>(WorkerSpawnTicketAssignmentRequest { - operation_id: format!("worker-spawn:{ticket_id}:{call_id}"), - ticket_id, - }) - }) - .transpose()?; - let request = WorkerSpawnRequest { - runtime_id: authority_id(&input.runtime_id, "runtime_id")?, - display_name: input - .display_name - .filter(|value| !value.trim().is_empty()) - .unwrap_or_else(|| "Workspace Worker".to_string()), - profile: non_empty(input.profile, "profile")?, - ticket_assignment, - initial_submit: input.initial_submit, - working_directory: WorkerWorkingDirectorySelection { - working_directory_id: authority_id( - &input.working_directory_id, - "working_directory_id", - )?, - relative_cwd: input - .relative_cwd - .map(|value| validate_relative_cwd(&value)) - .transpose()?, - }, + operation => { + let request = match operation { + WorkerOperation::List => { + parse::<WorkerListInput>(input_json, "WorkerList")?; + WorkspaceRequest::get(format!("/api/w/{}/workers", self.workspace_id)) + } + WorkerOperation::Spawn => { + let input = parse::<WorkerSpawnInput>(input_json, "WorkerSpawn")?; + let ticket_assignment = input + .ticket_id + .map(|ticket_id| { + let ticket_id = authority_id(&ticket_id, "ticket_id")?; + let call_id = non_empty(ctx.call_id.clone(), "tool call_id")?; + Ok::<_, ToolError>(WorkerSpawnTicketAssignmentRequest { + operation_id: format!("worker-spawn:{ticket_id}:{call_id}"), + ticket_id, + }) + }) + .transpose()?; + let request = WorkerSpawnRequest { + runtime_id: authority_id(&input.runtime_id, "runtime_id")?, + display_name: input + .display_name + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "Workspace Worker".to_string()), + profile: non_empty(input.profile, "profile")?, + ticket_assignment, + initial_submit: input.initial_submit, + working_directory: WorkerWorkingDirectorySelection { + working_directory_id: authority_id( + &input.working_directory_id, + "working_directory_id", + )?, + relative_cwd: input + .relative_cwd + .map(|value| validate_relative_cwd(&value)) + .transpose()?, + }, + }; + WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + format!("/api/w/{}/workers", self.workspace_id), + serde_json::to_string(&request) + .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?, + ) + } + WorkerOperation::Stop => { + let input = parse::<WorkerStopInput>(input_json, "WorkerStop")?; + let runtime_id = authority_id(&input.runtime_id, "runtime_id")?; + let worker_id = authority_id(&input.worker_id, "worker_id")?; + WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + format!( + "/api/w/{}/runtimes/{runtime_id}/workers/{worker_id}/stop", + self.workspace_id + ), + serde_json::json!({ "reason": input.reason }).to_string(), + ) + } + WorkerOperation::Restore => { + let input = parse::<WorkerTargetInput>(input_json, "WorkerRestore")?; + let runtime_id = authority_id(&input.runtime_id, "runtime_id")?; + let worker_id = authority_id(&input.worker_id, "worker_id")?; + WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + format!( + "/api/w/{}/runtimes/{runtime_id}/workers/{worker_id}/restore", + self.workspace_id + ), + "{}", + ) + } + WorkerOperation::Remove => unreachable!("handled above"), }; - WorkspaceRequest::json( - WorkspaceRequestMethod::Post, - format!("/api/w/{}/workers", self.workspace_id), - serde_json::to_string(&request) - .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?, - ) - } - WorkerOperation::Stop => { - let input = parse::<WorkerStopInput>(input_json, "WorkerStop")?; - let runtime_id = authority_id(&input.runtime_id, "runtime_id")?; - let worker_id = authority_id(&input.worker_id, "worker_id")?; - WorkspaceRequest::json( - WorkspaceRequestMethod::Post, - format!( - "/api/w/{}/runtimes/{runtime_id}/workers/{worker_id}/stop", - self.workspace_id - ), - serde_json::json!({ "reason": input.reason }).to_string(), - ) - } - WorkerOperation::Restore => { - let input = parse::<WorkerTargetInput>(input_json, "WorkerRestore")?; - let runtime_id = authority_id(&input.runtime_id, "runtime_id")?; - let worker_id = authority_id(&input.worker_id, "worker_id")?; - WorkspaceRequest::json( - WorkspaceRequestMethod::Post, - format!( - "/api/w/{}/runtimes/{runtime_id}/workers/{worker_id}/restore", - self.workspace_id - ), - "{}", - ) + self.client + .execute(request) + .map_err(|error| ToolError::ExecutionFailed(error.to_string()))? } }; - let response = self - .client - .execute(request) - .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; if !response.is_success() { return Err(ToolError::ExecutionFailed(format!( "Workspace Worker operation returned HTTP {}: {}", @@ -356,6 +406,7 @@ mod tests { #[derive(Debug, Default)] struct RecordingWorkspaceClient { requests: Mutex<Vec<WorkspaceRequest>>, + removals: Mutex<Vec<(String, String, String, String)>>, } impl WorkspaceClient for RecordingWorkspaceClient { @@ -381,6 +432,25 @@ mod tests { body: "{}".to_string(), }) } + + fn execute_worker_remove( + &self, + target_runtime_id: &str, + target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, + ) -> Result<WorkspaceResponse, WorkspaceClientError> { + self.removals.lock().unwrap().push(( + target_runtime_id.to_string(), + target_worker_id.to_string(), + expected_worker_revision.to_string(), + reason.to_string(), + )); + Ok(WorkspaceResponse { + status: 200, + body: r#"{"removed":true}"#.to_string(), + }) + } } #[tokio::test] @@ -433,7 +503,13 @@ mod tests { fn worker_tool_family_is_distinct_from_sub_worker_tools() { assert_eq!( WorkerOperation::ALL.map(WorkerOperation::tool_name), - ["WorkerList", "WorkerSpawn", "WorkerStop", "WorkerRestore"] + [ + "WorkerList", + "WorkerSpawn", + "WorkerStop", + "WorkerRestore", + "WorkerRemove", + ] ); } @@ -481,6 +557,92 @@ mod tests { assert!(value.get("initial_text").is_none()); } + #[tokio::test] + async fn worker_remove_forwards_only_target_revision_and_bounded_reason() { + let client = Arc::new(RecordingWorkspaceClient::default()); + let tool = WorkspaceWorkerTool { + operation: WorkerOperation::Remove, + client: client.clone(), + workspace_id: "workspace%2Ftest".to_string(), + }; + tool.execute( + &serde_json::json!({ + "runtime_id": "runtime-1", + "worker_id": "worker-7", + "expected_worker_revision": "2026-08-11T20:00:00Z", + "reason": " retire completed Worker " + }) + .to_string(), + ToolExecutionContext::new("call-remove", "batch-remove", 0), + ) + .await + .unwrap(); + assert_eq!( + client.removals.lock().unwrap().as_slice(), + [( + "runtime-1".to_string(), + "worker-7".to_string(), + "2026-08-11T20:00:00Z".to_string(), + "retire completed Worker".to_string(), + )] + ); + + let schema = serde_json::to_value(schemars::schema_for!(WorkerRemoveInput)) + .unwrap() + .to_string(); + for field in [ + "runtime_id", + "worker_id", + "expected_worker_revision", + "reason", + ] { + assert!(schema.contains(field)); + } + for forbidden in ["proof", "actor", "workspace_id", "policy", "plan", "stage"] { + assert!(!schema.contains(forbidden), "schema leaked {forbidden}"); + } + } + + #[tokio::test] + async fn worker_remove_rejects_empty_oversized_and_unknown_authority_input() { + let client = Arc::new(RecordingWorkspaceClient::default()); + let tool = WorkspaceWorkerTool { + operation: WorkerOperation::Remove, + client: client.clone(), + workspace_id: "workspace%2Ftest".to_string(), + }; + for reason in [" ".to_string(), "x".repeat(513)] { + let _error = tool + .execute( + &serde_json::json!({ + "runtime_id": "runtime-1", + "worker_id": "worker-7", + "expected_worker_revision": "revision-1", + "reason": reason, + }) + .to_string(), + ToolExecutionContext::new("call-invalid", "batch-remove", 0), + ) + .await + .unwrap_err(); + } + let _error = tool + .execute( + &serde_json::json!({ + "runtime_id": "runtime-1", + "worker_id": "worker-7", + "expected_worker_revision": "revision-1", + "reason": "retire", + "source_proof": "caller-controlled" + }) + .to_string(), + ToolExecutionContext::new("call-spoof", "batch-remove", 0), + ) + .await + .unwrap_err(); + assert!(client.removals.lock().unwrap().is_empty()); + } + #[test] fn worker_inputs_reject_paths_and_parent_traversal() { assert!(authority_id("https://runtime.example", "runtime_id").is_err()); diff --git a/crates/worker/src/prompt/catalog.rs b/crates/worker/src/prompt/catalog.rs index 5c6f7d5e..d6baab49 100644 --- a/crates/worker/src/prompt/catalog.rs +++ b/crates/worker/src/prompt/catalog.rs @@ -746,6 +746,21 @@ compact_system = "PREFIX\n{% include \"$yoi/internal/compact_system\" %}" assert!(rendered.contains("bypass user/Ticket authorization")); } + #[test] + fn orchestrator_role_prompt_fences_worker_remove_authority() { + let source = include_str!("../../../../resources/prompts/role/orchestrator.md"); + assert!(source.contains("Use `WorkerRemove` only for a terminal or authoritatively reassigned non-internal Coder")); + assert!(source.contains("exact current `updated_at` value")); + assert!(source.contains("must have no current Ticket assignment")); + assert!(source.contains("pending notification, Reviewer handoff, legal hold, or pin")); + assert!(source.contains("After removal, reread the Worker catalog and attachment state")); + assert!(source.contains("attachment-close, and attachment-release conflicts")); + assert!(source.contains("preserves the Workdir materialization")); + assert!(!source.contains("source proof")); + assert!(!source.contains("provider handle")); + assert!(!source.contains("retention plan")); + } + #[test] fn sub_worker_spawn_tool_description_renders_profile_block() { let cat = PromptCatalog::builtins_only().unwrap(); diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 565c785c..acbe7d17 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -230,6 +230,8 @@ pub trait WorkspaceClient: std::fmt::Debug + Send + Sync { &self, _target_runtime_id: &str, _target_worker_id: &str, + _expected_worker_revision: &str, + _reason: &str, ) -> Result<WorkspaceResponse, WorkspaceClientError> { Err(WorkspaceClientError::Unavailable( "Runtime-owned WorkerRemove forwarding is unavailable".to_string(), diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 6d9c40d1..490ca9c2 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -54,6 +54,9 @@ use worker_runtime::interaction::{ }; use worker_runtime::management::{RuntimeOptions as EmbeddedRuntimeOptions, RuntimeStatus}; use worker_runtime::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveInput}; +use worker_runtime::retention::{ + WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory, +}; pub(crate) const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime"; const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host"; @@ -856,6 +859,25 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { } } + fn worker_retention_inventory( + &self, + worker_id: &str, + ) -> Result<WorkerRetentionInventory, String> { + Err(format!( + "runtime does not implement retention inventory for '{worker_id}'" + )) + } + + fn execute_worker_retention( + &self, + request: WorkerRetentionExecutionRequest, + ) -> Result<WorkerRetentionExecutionResult, String> { + Err(format!( + "runtime does not implement retention execution for '{}'", + request.worker_id + )) + } + fn observation_source( &self, _worker_id: &str, @@ -1399,6 +1421,44 @@ impl RuntimeRegistry { Ok(runtime.delete_worker(worker_id)) } + pub fn worker_retention_inventory( + &self, + worker: &RuntimeWorkerRef, + ) -> Result<WorkerRetentionInventory, RuntimeRegistryError> { + validate_backend_identifier("runtime_id", &worker.runtime_id)?; + validate_backend_identifier("worker_id", &worker.worker_id)?; + self.runtime(&worker.runtime_id)? + .worker_retention_inventory(&worker.worker_id) + .map_err(|message| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "worker_retention_inventory_failed".to_string(), + message, + }) + } + + pub fn execute_worker_retention( + &self, + worker: &RuntimeWorkerRef, + request: WorkerRetentionExecutionRequest, + ) -> Result<WorkerRetentionExecutionResult, RuntimeRegistryError> { + validate_backend_identifier("runtime_id", &worker.runtime_id)?; + validate_backend_identifier("worker_id", &worker.worker_id)?; + if request.worker_id.to_string() != worker.worker_id { + return Err(RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "worker_id_mismatch".to_string(), + message: "retention request worker_id does not match target".to_string(), + }); + } + self.runtime(&worker.runtime_id)? + .execute_worker_retention(request) + .map_err(|message| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "worker_retention_execution_failed".to_string(), + message, + }) + } + pub fn observation_source( &self, worker: &RuntimeWorkerRef, @@ -1438,6 +1498,7 @@ impl RuntimeRegistry { #[derive(Clone)] pub struct EmbeddedWorkerRuntime { + workspace_id: String, runtime_id: String, host_id: String, runtime: worker_runtime::Runtime, @@ -1501,8 +1562,9 @@ impl EmbeddedWorkerRuntime { .bind_runtime_identity(EMBEDDED_RUNTIME_ID) .expect("fresh embedded Runtime must accept its Backend-owned identity"); Self { - runtime_id: EMBEDDED_RUNTIME_ID.to_string(), host_id: host_id_for_embedded_workspace(&workspace_id), + workspace_id, + runtime_id: EMBEDDED_RUNTIME_ID.to_string(), runtime, execution_enabled: false, resource_broker: BackendResourceBroker::default(), @@ -2106,6 +2168,30 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { } } + fn worker_retention_inventory( + &self, + worker_id: &str, + ) -> Result<WorkerRetentionInventory, String> { + let worker_ref = self + .worker_ref(worker_id) + .ok_or_else(|| format!("invalid embedded Worker id '{worker_id}'"))?; + self.runtime + .worker_retention_inventory(&self.workspace_id, &worker_ref) + .map_err(|error| error.to_string()) + } + + fn execute_worker_retention( + &self, + request: WorkerRetentionExecutionRequest, + ) -> Result<WorkerRetentionExecutionResult, String> { + if request.workspace_id != self.workspace_id { + return Err("retention request Workspace does not match embedded Runtime".to_string()); + } + self.runtime + .execute_worker_retention(&request) + .map_err(|error| error.to_string()) + } + fn observation_source( &self, worker_id: &str, @@ -3128,6 +3214,28 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { } } + fn worker_retention_inventory( + &self, + worker_id: &str, + ) -> Result<WorkerRetentionInventory, String> { + self.get_json::<WorkerRetentionInventory>(&format!( + "/v1/workers/{worker_id}/retention/inventory" + )) + .map_err(|diagnostic| diagnostic.message) + } + + fn execute_worker_retention( + &self, + request: WorkerRetentionExecutionRequest, + ) -> Result<WorkerRetentionExecutionResult, String> { + let worker_id = request.worker_id.to_string(); + self.post_json::<_, WorkerRetentionExecutionResult>( + &format!("/v1/workers/{worker_id}/retention/execute"), + &request, + ) + .map_err(|diagnostic| diagnostic.message) + } + fn observation_source( &self, worker_id: &str, diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index f6e41e01..f4e06eb6 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -377,6 +377,7 @@ impl SqliteWorkspaceStore { workspace_id: plan.workspace_id.clone(), source_runtime_id: plan.worker.runtime_id.clone(), worker_id: worker_runtime::identity::WorkerId::new(worker_number), + expected_worker_revision: plan.worker_revision.clone(), expected_run_generation: plan.run_generation, source_created_at: worker.created_at, removed_at, @@ -391,6 +392,85 @@ impl SqliteWorkspaceStore { }) } + pub fn recover_worker_removal_execution( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + expected_worker_revision: &str, + reason: &str, + ) -> Result<Option<PreparedWorkerRemoval>, WorkerRetentionError> { + bounded("workspace", workspace_id, 160)?; + bounded("revision", expected_worker_revision, 256)?; + bounded("reason", reason, 512)?; + let plan = self.with_conn(|conn| { + conn.query_row( + "SELECT plan_id FROM worker_removal_operations + WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 + AND worker_revision=?4 AND reason=?5 + AND state IN ('executing','failed','succeeded') + ORDER BY CASE state WHEN 'succeeded' THEN 0 ELSE 1 END, + created_at DESC LIMIT 1", + params![ + workspace_id, + worker.runtime_id, + worker.worker_id, + expected_worker_revision, + reason, + ], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(StoreError::from) + .and_then(|plan_id| match plan_id { + Some(plan_id) => load_plan(conn, &plan_id), + None => Ok(None), + }) + })?; + let Some(plan) = plan else { + return Ok(None); + }; + let worker_number = plan.worker.worker_id.parse::<u64>().map_err(|_| { + WorkerRetentionError::Invalid( + "Runtime Worker id is not a canonical unsigned integer".to_string(), + ) + })?; + let worker = if plan.state == WorkerRemovalPlanState::Succeeded { + None + } else { + Some( + self.with_conn(|conn| load_worker(conn, workspace_id, &plan.worker))? + .ok_or(WorkerRetentionError::WorkerNotFound)?, + ) + }; + Ok(Some(PreparedWorkerRemoval { + runtime_request: WorkerRetentionExecutionRequest { + operation_id: plan.operation_id.clone(), + input_fingerprint: plan.input_fingerprint.clone(), + archive_id: plan.archive_id.clone(), + workspace_id: plan.workspace_id.clone(), + source_runtime_id: plan.worker.runtime_id.clone(), + worker_id: worker_runtime::identity::WorkerId::new(worker_number), + expected_worker_revision: plan.worker_revision.clone(), + expected_run_generation: plan.run_generation, + source_created_at: worker + .as_ref() + .map(|worker| worker.created_at.clone()) + .unwrap_or_else(|| plan.created_at.clone()), + removed_at: plan.created_at.clone(), + effective_profile: worker + .as_ref() + .map(|worker| worker.profile.clone()) + .unwrap_or_else(|| Some("removed".to_string())), + retention_class: None, + policy_id: plan.policy_id.clone(), + policy_revision: plan.policy_revision, + session_disposition: plan.session_disposition, + diagnostics_disposition: plan.diagnostics_disposition, + }, + plan, + })) + } + pub fn fail_worker_removal( &self, workspace_id: &str, @@ -424,7 +504,8 @@ impl SqliteWorkspaceStore { if plan.state != WorkerRemovalPlanState::Executing { return Err(StoreError::InvalidInput(format!("stale:{}:plan state {} is not committable", plan.plan_id, state_s(plan.state)))); } - if result.worker_id.to_string() != plan.worker.worker_id + if result.expected_worker_revision != plan.worker_revision + || result.worker_id.to_string() != plan.worker.worker_id || result.session_disposition != plan.session_disposition || result.diagnostics_disposition != plan.diagnostics_disposition { @@ -1107,6 +1188,7 @@ mod tests { let result = WorkerRetentionExecutionResult { operation_id: p.operation_id.clone(), input_fingerprint: p.input_fingerprint.clone(), + expected_worker_revision: p.worker_revision.clone(), worker_id: WorkerId::new(1), session_disposition: p.session_disposition, diagnostics_disposition: p.diagnostics_disposition, @@ -1247,6 +1329,7 @@ mod tests { let r = WorkerRetentionExecutionResult { operation_id: p.operation_id.clone(), input_fingerprint: p.input_fingerprint.clone(), + expected_worker_revision: p.worker_revision.clone(), worker_id: WorkerId::new(1), session_disposition: SessionDisposition::Purge, diagnostics_disposition: DiagnosticsDisposition::Purge, @@ -1265,6 +1348,7 @@ mod tests { let mut result = WorkerRetentionExecutionResult { operation_id: plan.operation_id.clone(), input_fingerprint: plan.input_fingerprint.clone(), + expected_worker_revision: plan.worker_revision.clone(), worker_id: WorkerId::new(1), session_disposition: plan.session_disposition, diagnostics_disposition: plan.diagnostics_disposition, @@ -1348,6 +1432,106 @@ mod tests { ); } + #[test] + fn runtime_result_must_match_prepared_worker_revision() { + let s = setup(); + let plan = s.plan_worker_removal(&req(), &inv()).unwrap(); + let prepared = s + .prepare_worker_removal_execution("w", &plan.plan_id, &plan.input_fingerprint) + .unwrap(); + let mut runtime_result = WorkerRetentionExecutionResult { + operation_id: prepared.plan.operation_id.clone(), + input_fingerprint: prepared.plan.input_fingerprint.clone(), + expected_worker_revision: prepared.plan.worker_revision.clone(), + worker_id: WorkerId::new(1), + session_disposition: prepared.plan.session_disposition, + diagnostics_disposition: prepared.plan.diagnostics_disposition, + archive: None, + source_removed: true, + diagnostics_retained: false, + }; + runtime_result.expected_worker_revision = "stale-revision".to_string(); + let error = s + .commit_worker_removal( + "w", + &plan.operation_id, + &plan.input_fingerprint, + &runtime_result, + ) + .unwrap_err(); + assert!( + !error.to_string().is_empty(), + "mismatched Runtime revision must be rejected" + ); + } + + #[test] + fn succeeded_worker_removal_recovers_after_registry_purge() { + let s = setup(); + let request = req(); + let plan = s.plan_worker_removal(&request, &inv()).unwrap(); + let prepared = s + .prepare_worker_removal_execution("w", &plan.plan_id, &plan.input_fingerprint) + .unwrap(); + let runtime_result = WorkerRetentionExecutionResult { + operation_id: prepared.plan.operation_id.clone(), + input_fingerprint: prepared.plan.input_fingerprint.clone(), + expected_worker_revision: prepared.plan.worker_revision.clone(), + worker_id: WorkerId::new(1), + session_disposition: prepared.plan.session_disposition, + diagnostics_disposition: prepared.plan.diagnostics_disposition, + archive: Some(worker_runtime::retention::WorkerSessionArchiveManifest { + schema_version: 1, + archive_id: prepared.plan.archive_id.clone().unwrap(), + workspace_id: "w".into(), + source_runtime_id: "r".into(), + source_worker_id: WorkerId::new(1), + source_session_id: "s".into(), + segment_ids: vec!["a".into()], + source_created_at: "created".into(), + removed_at: "removed".into(), + archived_at_unix_seconds: 1, + effective_profile: None, + retention_class: None, + content_checksum_sha256: "sum".into(), + content_bytes: 1, + content_file_count: 1, + policy_id: prepared.plan.policy_id.clone(), + policy_revision: prepared.plan.policy_revision, + operation_id: prepared.plan.operation_id.clone(), + input_fingerprint: prepared.plan.input_fingerprint.clone(), + }), + source_removed: true, + diagnostics_retained: false, + }; + s.commit_worker_removal( + "w", + &plan.operation_id, + &plan.input_fingerprint, + &runtime_result, + ) + .unwrap(); + assert!( + s.get_worker_registry("w", &request.worker) + .unwrap() + .is_none() + ); + let recovered = s + .recover_worker_removal_execution( + "w", + &request.worker, + &request.expected_worker_revision, + &request.reason, + ) + .unwrap() + .unwrap(); + assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Succeeded); + assert_eq!( + recovered.runtime_request.expected_worker_revision, + request.expected_worker_revision + ); + } + #[test] fn old_schema_upgrade_seeds_existing_workspace() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 5f2f4d55..2359f1c5 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, Weak}; use axum::extract::ws::{Message as WsMessage, WebSocket, WebSocketUpgrade}; use axum::extract::{Path as AxumPath, Query, Request, State}; @@ -262,21 +262,328 @@ pub struct WorkspaceApi { resource_broker: BackendResourceBroker, workdir_sessions: Arc<Mutex<HashMap<RuntimeWorkerRef, WorkdirSessionHandle>>>, workdir_session_locks: Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<tokio::sync::Mutex<()>>>>>, + worker_remove_locks: Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<tokio::sync::Mutex<()>>>>>, +} + +#[derive(Clone)] +struct WorkspaceWorkerRemoveExecutor { + workspace_id: String, + store: Arc<dyn ControlPlaneStore>, + runtime: Weak<RuntimeRegistry>, + workdir_sessions: Arc<Mutex<HashMap<RuntimeWorkerRef, WorkdirSessionHandle>>>, + workdir_session_locks: Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<tokio::sync::Mutex<()>>>>>, + worker_remove_locks: Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<tokio::sync::Mutex<()>>>>>, +} + +impl WorkspaceWorkerRemoveExecutor { + fn new(api: &WorkspaceApi) -> Self { + Self { + workspace_id: api.config.workspace_id.clone(), + store: api.store.clone(), + runtime: Arc::downgrade(&api.runtime), + workdir_sessions: api.workdir_sessions.clone(), + workdir_session_locks: api.workdir_session_locks.clone(), + worker_remove_locks: api.worker_remove_locks.clone(), + } + } + + async fn resume_worker_retention( + &self, + runtime: &RuntimeRegistry, + target: &RuntimeWorkerRef, + prepared: crate::retention::PreparedWorkerRemoval, + ) -> std::result::Result<worker::WorkspaceResponse, String> { + let result = + match runtime.execute_worker_retention(target, prepared.runtime_request.clone()) { + Ok(result) => result, + Err(_) => { + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "runtime_retention_failed", + "Runtime retention recovery failed; removal can be retried", + )); + } + }; + match self.store.commit_worker_removal( + &self.workspace_id, + &prepared.plan.operation_id, + &prepared.plan.input_fingerprint, + &result, + ) { + Ok(_) => Ok(worker_remove_success_response(target)), + Err(error) => Ok(worker_retention_error_response(error)), + } + } + + async fn execute_async( + &self, + source: crate::worker_source::VerifiedWorkerMutationSource, + target_runtime_id: &str, + target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, + ) -> std::result::Result<worker::WorkspaceResponse, String> { + let reason = reason.trim(); + if reason.is_empty() || reason.len() > 512 { + return Ok(worker_remove_error_response( + StatusCode::BAD_REQUEST, + "invalid_reason", + "WorkerRemove reason must be between 1 and 512 bytes", + )); + } + let runtime = self.runtime.upgrade().ok_or_else(|| { + "Workspace Runtime registry is unavailable during WorkerRemove".to_string() + })?; + let source_is_current_orchestrator = + runtime.list_workers(1_000).items.into_iter().any(|worker| { + worker.worker.runtime_id == source.runtime_id + && worker.worker.worker_id == source.worker_id + && worker.singleton_key.as_deref() + == Some(crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY) + }); + if !source_is_current_orchestrator { + return Ok(worker_remove_error_response( + StatusCode::FORBIDDEN, + "orchestrator_required", + "WorkerRemove is restricted to the current Workspace Orchestrator", + )); + } + if source.runtime_id == target_runtime_id && source.worker_id == target_worker_id { + return Ok(worker_remove_error_response( + StatusCode::CONFLICT, + "self_removal_forbidden", + "The current Orchestrator cannot remove itself", + )); + } + + let target = RuntimeWorkerRef { + runtime_id: target_runtime_id.to_string(), + worker_id: target_worker_id.to_string(), + }; + let remove_lock = { + let mut locks = self + .worker_remove_locks + .lock() + .map_err(|_| "WorkerRemove lock registry was poisoned".to_string())?; + locks + .entry(target.clone()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + }; + let _remove_guard = remove_lock.lock().await; + let workdir_session_lock = { + let mut locks = self + .workdir_session_locks + .lock() + .map_err(|_| "Workdir session lock registry was poisoned".to_string())?; + locks + .entry(target.clone()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + }; + let _workdir_session_guard = workdir_session_lock.lock().await; + + let prepared = self + .store + .recover_worker_removal_execution( + &self.workspace_id, + &target, + expected_worker_revision, + reason, + ) + .map_err(|_| "Worker removal recovery authority is unavailable".to_string())?; + if let Some(prepared) = prepared { + if prepared.plan.state == crate::retention::WorkerRemovalPlanState::Succeeded { + return Ok(worker_remove_success_response(&target)); + } + return self + .resume_worker_retention(&runtime, &target, prepared) + .await; + } + + let worker = match runtime.worker(&target) { + Ok(worker) => worker, + Err(_) => { + return Ok(worker_remove_error_response( + StatusCode::NOT_FOUND, + "worker_not_found", + "Worker was not found in this Workspace", + )); + } + }; + if worker.singleton_key.is_some() { + return Ok(worker_remove_error_response( + StatusCode::CONFLICT, + "internal_worker_forbidden", + "Internal service Workers cannot be removed with WorkerRemove", + )); + } + if !worker.state.eq_ignore_ascii_case("stopped") { + return Ok(worker_remove_error_response( + StatusCode::CONFLICT, + "worker_not_stopped", + "Worker must be stopped before removal", + )); + } + + let inventory = match runtime.worker_retention_inventory(&target) { + Ok(inventory) => inventory, + Err(_) => { + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "retention_inventory_unavailable", + "Retention inventory could not be loaded; removal can be retried", + )); + } + }; + let request = crate::retention::WorkerRemovalPlanRequest { + workspace_id: self.workspace_id.clone(), + worker: target.clone(), + expected_worker_revision: expected_worker_revision.to_string(), + reason: reason.to_string(), + }; + let plan = match self.store.plan_worker_removal(&request, &inventory) { + Ok(plan) => plan, + Err(error) => return Ok(worker_retention_error_response(error)), + }; + let prepared = match self.store.prepare_worker_removal_execution( + &self.workspace_id, + &plan.plan_id, + &plan.input_fingerprint, + ) { + Ok(prepared) => prepared, + Err(error) => return Ok(worker_retention_error_response(error)), + }; + + let session = self + .workdir_sessions + .lock() + .map_err(|_| "Workdir session registry was poisoned".to_string())? + .get(&target) + .cloned(); + if let Some(session) = session { + if session.close().await.is_err() { + let _ = self.store.fail_worker_removal( + &self.workspace_id, + &plan.operation_id, + &plan.input_fingerprint, + "workdir_session_close_failed", + ); + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "attachment_close_failed", + "Worker Workdir session could not be closed; removal can be retried", + )); + } + self.workdir_sessions + .lock() + .map_err(|_| "Workdir session registry was poisoned".to_string())? + .remove(&target); + } + + if let Err(_) = self.store.detach_worker_workdir( + &self.workspace_id, + &target, + None, + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ) { + let _ = self.store.fail_worker_removal( + &self.workspace_id, + &plan.operation_id, + &plan.input_fingerprint, + "workdir_attachment_release_failed", + ); + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "attachment_release_failed", + "Worker Workdir attachment could not be released; removal can be retried", + )); + } + + let retention_result = + match runtime.execute_worker_retention(&target, prepared.runtime_request.clone()) { + Ok(result) => result, + Err(_) => { + let _ = self.store.fail_worker_removal( + &self.workspace_id, + &plan.operation_id, + &plan.input_fingerprint, + "runtime_retention_failed", + ); + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "runtime_retention_failed", + "Runtime retention execution failed; removal can be retried", + )); + } + }; + match self.store.commit_worker_removal( + &self.workspace_id, + &plan.operation_id, + &plan.input_fingerprint, + &retention_result, + ) { + Ok(_) => {} + Err(error) => { + let _ = self.store.fail_worker_removal( + &self.workspace_id, + &plan.operation_id, + &plan.input_fingerprint, + "metadata_commit_failed", + ); + return Ok(worker_retention_error_response(error)); + } + }; + Ok(worker_remove_success_response(&target)) + } +} + +impl crate::worker_source::VerifiedWorkerRemoveExecutor for WorkspaceWorkerRemoveExecutor { + fn execute( + &self, + source: crate::worker_source::VerifiedWorkerMutationSource, + target_runtime_id: &str, + target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, + ) -> std::result::Result<worker::WorkspaceResponse, String> { + let executor = self.clone(); + let target_runtime_id = target_runtime_id.to_string(); + let target_worker_id = target_worker_id.to_string(); + let expected_worker_revision = expected_worker_revision.to_string(); + let reason = reason.to_string(); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| error.to_string())? + .block_on(executor.execute_async( + source, + &target_runtime_id, + &target_worker_id, + &expected_worker_revision, + &reason, + )) + }) + .join() + .map_err(|_| "embedded WorkerRemove executor thread panicked".to_string())? + } } impl WorkspaceApi { pub async fn new(config: ServerConfig, store: Arc<dyn ControlPlaneStore>) -> Result<Self> { let resource_broker = BackendResourceBroker::default(); + let worker_remove_dispatcher = Arc::new( + crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( + config.clone(), + store.clone(), + ), + ); let execution_backend = WorkerRuntimeExecutionBackend::new( ProfileRuntimeWorkerFactory::new(config.workspace_root.clone()) .with_embedded_worker_mutation_dispatcher( EMBEDDED_RUNTIME_ID, - Arc::new( - crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( - config.clone(), - store.clone(), - ), - ), + worker_remove_dispatcher.clone(), ) .with_runtime_store_dir(config.embedded_runtime_store_root.clone()) .with_resource_client(Arc::new(resource_broker.clone())), @@ -291,6 +598,7 @@ impl WorkspaceApi { store, Arc::new(execution_backend), resource_broker, + Some(worker_remove_dispatcher), ) .await } @@ -306,6 +614,7 @@ impl WorkspaceApi { store, execution_backend, BackendResourceBroker::default(), + None, ) .await } @@ -315,6 +624,9 @@ impl WorkspaceApi { store: Arc<dyn ControlPlaneStore>, execution_backend: Arc<dyn worker_runtime::execution::WorkerExecutionBackend>, resource_broker: BackendResourceBroker, + worker_remove_dispatcher: Option< + Arc<crate::worker_source::EmbeddedServerWorkerMutationDispatcher>, + >, ) -> Result<Self> { store .upsert_workspace(&WorkspaceRecord { @@ -376,7 +688,7 @@ impl WorkspaceApi { let runtime = Arc::new(runtime); let companion = Arc::new(CompanionConsole::disabled()); let observation_proxy = BackendObservationProxy::new(config.runtime_event_sources.clone()); - Ok(Self { + let api = Self { authority: SqliteWorkspaceAuthority::new( config.database_path.clone(), config.workspace_id.clone(), @@ -392,7 +704,14 @@ impl WorkspaceApi { resource_broker, workdir_sessions: Arc::new(Mutex::new(HashMap::new())), workdir_session_locks: Arc::new(Mutex::new(HashMap::new())), - }) + worker_remove_locks: Arc::new(Mutex::new(HashMap::new())), + }; + if let Some(dispatcher) = worker_remove_dispatcher { + dispatcher + .install_executor(Arc::new(WorkspaceWorkerRemoveExecutor::new(&api))) + .map_err(|message| Error::Config(message.to_string()))?; + } + Ok(api) } pub fn workspace_id(&self) -> &str { @@ -4823,10 +5142,93 @@ async fn scoped_workspace_protocol_ws( .into_response()) } +#[derive(Debug, Serialize)] +struct WorkerRemoveSuccessResponse<'a> { + removed: bool, + runtime_id: &'a str, + worker_id: &'a str, +} + +#[derive(Debug, Serialize)] +struct WorkerRemoveErrorResponse<'a> { + code: &'a str, + message: &'a str, +} + +fn worker_remove_success_response(worker: &RuntimeWorkerRef) -> worker::WorkspaceResponse { + let body = serde_json::to_string(&WorkerRemoveSuccessResponse { + removed: true, + runtime_id: &worker.runtime_id, + worker_id: &worker.worker_id, + }) + .unwrap_or_else(|_| r#"{"removed":true}"#.to_string()); + worker::WorkspaceResponse { + status: StatusCode::OK.as_u16(), + body, + } +} + +fn worker_remove_error_response( + status: StatusCode, + code: &str, + message: &str, +) -> worker::WorkspaceResponse { + let body = + serde_json::to_string(&WorkerRemoveErrorResponse { code, message }).unwrap_or_else(|_| { + r#"{"code":"worker_remove_failed","message":"Worker removal failed"}"#.to_string() + }); + worker::WorkspaceResponse { + status: status.as_u16(), + body, + } +} + +fn worker_retention_error_response( + error: crate::retention::WorkerRetentionError, +) -> worker::WorkspaceResponse { + match error { + crate::retention::WorkerRetentionError::WorkerNotFound + | crate::retention::WorkerRetentionError::CrossWorkspace => worker_remove_error_response( + StatusCode::NOT_FOUND, + "worker_not_found", + "Worker was not found in this Workspace", + ), + crate::retention::WorkerRetentionError::WorkerRevisionConflict { .. } + | crate::retention::WorkerRetentionError::PolicyRevisionConflict { .. } + | crate::retention::WorkerRetentionError::StalePlan { .. } + | crate::retention::WorkerRetentionError::OperationFingerprintConflict { .. } => { + worker_remove_error_response( + StatusCode::CONFLICT, + "worker_revision_conflict", + "Worker removal state changed; reread the Worker and retry", + ) + } + crate::retention::WorkerRetentionError::Blocked(_) => worker_remove_error_response( + StatusCode::CONFLICT, + "worker_removal_blocked", + "Worker removal is blocked by current assignment, hold, or retention policy", + ), + crate::retention::WorkerRetentionError::Invalid(_) => worker_remove_error_response( + StatusCode::BAD_REQUEST, + "invalid_worker_remove", + "Worker removal request is invalid", + ), + crate::retention::WorkerRetentionError::PolicyMissing { .. } + | crate::retention::WorkerRetentionError::Store(_) => worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "worker_removal_authority_unavailable", + "Worker removal authority is unavailable; removal can be retried", + ), + } +} + #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] struct WorkerRemoveBoundaryRequest { target_runtime_id: String, target_worker_id: String, + expected_worker_revision: String, + reason: String, } async fn scoped_worker_remove_source_boundary( @@ -4856,19 +5258,41 @@ async fn scoped_worker_remove_source_boundary( ) .await { - Ok(source) => ( - StatusCode::NOT_IMPLEMENTED, - Json(serde_json::json!({ - "error": "WorkerRemove lifecycle is deferred to its consumer Ticket", - "source": { - "runtime_id": source.runtime_id, - "worker_id": source.worker_id, - "actor_kind": source.actor_kind, - "permission": source.permission, + Ok(source) => { + let executor = WorkspaceWorkerRemoveExecutor::new(&api); + match executor + .execute_async( + source, + &request.target_runtime_id, + &request.target_worker_id, + &request.expected_worker_revision, + &request.reason, + ) + .await + { + Ok(response) => ( + StatusCode::from_u16(response.status) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + [(CONTENT_TYPE, "application/json")], + response.body, + ) + .into_response(), + Err(_) => { + let response = worker_remove_error_response( + StatusCode::INTERNAL_SERVER_ERROR, + "worker_remove_failed", + "Worker removal failed before lifecycle execution", + ); + ( + StatusCode::from_u16(response.status) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + [(CONTENT_TYPE, "application/json")], + response.body, + ) + .into_response() } - })), - ) - .into_response(), + } + } Err(error) => { let status = match error { crate::worker_source::WorkerMutationSourceProofError::Replay => { @@ -6009,7 +6433,7 @@ async fn scoped_check_runtime_config_bundle( async fn scoped_get_runtime_worker( State(api): State<WorkspaceApi>, AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>, -) -> ApiResult<Json<WorkerSummary>> { +) -> ApiResult<Json<WorkerShowProjection>> { validate_workspace_scope(&api, &path.workspace_id)?; get_runtime_worker( State(api), @@ -7715,10 +8139,17 @@ async fn post_companion_cancel( Ok(Json(api.companion.cancel(request))) } +#[derive(Debug, Serialize)] +struct WorkerShowProjection { + #[serde(flatten)] + worker: WorkerSummary, + updated_at: String, +} + async fn get_runtime_worker( State(api): State<WorkspaceApi>, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, -) -> ApiResult<Json<WorkerSummary>> { +) -> ApiResult<Json<WorkerShowProjection>> { let worker_ref = RuntimeWorkerRef::new(runtime_id, worker_id); let worker = api .runtime @@ -7731,12 +8162,11 @@ async fn get_runtime_worker( let workdirs = api .store .list_workdir_registry(&api.config.workspace_id, 500)?; - Ok(Json(merge_worker_registry_projection( - Some(&worker), - &record, - links, - &workdirs, - ))) + let updated_at = record.updated_at.clone(); + Ok(Json(WorkerShowProjection { + worker: merge_worker_registry_projection(Some(&worker), &record, links, &workdirs), + updated_at, + })) } async fn restore_runtime_worker( @@ -13242,7 +13672,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let app = build_router(test_api(temp.path()).await); - let body = r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker"}"#; + let body = r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker","expected_worker_revision":"revision-1","reason":"retire target Worker"}"#; let browser = app .clone() .oneshot( @@ -13257,6 +13687,7 @@ mod tests { .unwrap(); assert_eq!(browser.status(), StatusCode::UNAUTHORIZED); let legacy = app + .clone() .oneshot( Request::builder() .method("POST") @@ -13270,6 +13701,20 @@ mod tests { .await .unwrap(); assert_eq!(legacy.status(), StatusCode::UNAUTHORIZED); + let body_spoof = app + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker","expected_worker_revision":"revision-1","reason":"retire target Worker","source_proof":"browser-controlled","actor":"orchestrator","policy":"purge"}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(body_spoof.status(), StatusCode::UNPROCESSABLE_ENTITY); } #[tokio::test] @@ -13338,15 +13783,220 @@ mod tests { api.config.clone(), api.store.clone(), ); - let response = + let error = worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher::execute_worker_remove( &dispatcher, fresh_proof, "runtime-target", "target-worker", + "revision-1", + "retire target Worker", + ) + .unwrap_err(); + assert!(error.to_string().contains("executor is unavailable")); + } + + #[tokio::test] + async fn worker_remove_rejects_self_running_and_stale_revision_at_caller_boundary() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + let Json(orchestrator) = scoped_start_workspace_orchestrator( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + ) + .await + .unwrap(); + let source = orchestrator.worker.unwrap().worker; + let verified_source = || crate::worker_source::VerifiedWorkerMutationSource { + runtime_id: source.runtime_id.clone(), + worker_id: source.worker_id.clone(), + actor_kind: worker_runtime::auth::WorkerMutationActorKind::Worker, + permission: worker_runtime::auth::WORKER_REMOVE_PERMISSION.to_string(), + jti: "caller-guard-proof".to_string(), + }; + let executor = WorkspaceWorkerRemoveExecutor::new(&api); + let self_response = executor + .execute_async( + verified_source(), + &source.runtime_id, + &source.worker_id, + "irrelevant", + "must reject self", + ) + .await + .unwrap(); + assert_eq!(self_response.status, StatusCode::CONFLICT.as_u16()); + assert!(self_response.body.contains("self_removal_forbidden")); + + let spawned = api + .spawn_workspace_worker( + EMBEDDED_WORKER_RUNTIME_ID, + WorkerSpawnRequest { + intent: WorkerSpawnIntent::WorkspaceCompanion, + requested_worker_name: Some("guard-target".to_string()), + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile: worker_runtime::catalog::ProfileSelector::Builtin( + "builtin:companion".to_string(), + ), + ticket_assignment: None, + initial_submit: Vec::new(), + working_directory_request: None, + resolved_working_directory_request: None, + resolved_working_directory: None, + resolved_config_bundle: Some(runtime_test_bundle()), + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + }, ) .unwrap(); - assert_eq!(response.status, 501); + let target = spawned.worker.unwrap().worker; + let running_response = executor + .execute_async( + verified_source(), + &target.runtime_id, + &target.worker_id, + "irrelevant", + "must reject a live Worker", + ) + .await + .unwrap(); + assert_eq!(running_response.status, StatusCode::CONFLICT.as_u16()); + assert!(running_response.body.contains("worker_not_stopped")); + + api.runtime + .stop_worker( + &target, + WorkerLifecycleRequest { + reason: Some("prepare stale revision guard".to_string()), + ticket_assignment: None, + }, + ) + .unwrap(); + let summary = api.runtime.worker(&target).unwrap(); + let record = sync_worker_observation(&api, &summary).unwrap(); + let stale_response = executor + .execute_async( + verified_source(), + &target.runtime_id, + &target.worker_id, + &format!("{}-stale", record.updated_at), + "must reject stale revision", + ) + .await + .unwrap(); + assert_eq!(stale_response.status, StatusCode::CONFLICT.as_u16()); + assert!(stale_response.body.contains("worker_revision_conflict")); + } + + #[tokio::test] + async fn embedded_worker_remove_executes_retention_and_returns_bounded_result() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + let Json(orchestrator) = scoped_start_workspace_orchestrator( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + ) + .await + .unwrap(); + let source = orchestrator.worker.unwrap().worker; + + let spawned = api + .spawn_workspace_worker( + EMBEDDED_WORKER_RUNTIME_ID, + WorkerSpawnRequest { + intent: WorkerSpawnIntent::WorkspaceCompanion, + requested_worker_name: Some("remove-target".to_string()), + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile: worker_runtime::catalog::ProfileSelector::Builtin( + "builtin:companion".to_string(), + ), + ticket_assignment: None, + initial_submit: Vec::new(), + working_directory_request: None, + resolved_working_directory_request: None, + resolved_working_directory: None, + resolved_config_bundle: Some(runtime_test_bundle()), + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + }, + ) + .unwrap(); + let target = spawned.worker.unwrap().worker; + let stopped = api + .runtime + .stop_worker( + &target, + WorkerLifecycleRequest { + reason: Some("prepare WorkerRemove regression".to_string()), + ticket_assignment: None, + }, + ) + .unwrap(); + assert_eq!(stopped.state, WorkerOperationState::Accepted); + let worker_root = temp + .path() + .join(".test-embedded-runtime-store/workers") + .join(&target.worker_id); + fs::create_dir_all(worker_root.join("session/segments")).unwrap(); + fs::write( + worker_root.join("session/session.json"), + serde_json::to_vec_pretty(&json!({ + "schema_version": 1, + "session_id": "worker-remove-session" + })) + .unwrap(), + ) + .unwrap(); + fs::write( + worker_root.join("session/segments/segment-a.jsonl"), + b"retained evidence\n", + ) + .unwrap(); + let summary = api.runtime.worker(&target).unwrap(); + let record = sync_worker_observation(&api, &summary).unwrap(); + + let response = WorkspaceWorkerRemoveExecutor::new(&api) + .execute_async( + crate::worker_source::VerifiedWorkerMutationSource { + runtime_id: source.runtime_id, + worker_id: source.worker_id, + actor_kind: worker_runtime::auth::WorkerMutationActorKind::Worker, + permission: worker_runtime::auth::WORKER_REMOVE_PERMISSION.to_string(), + jti: "embedded-valid-proof".to_string(), + }, + &target.runtime_id, + &target.worker_id, + &record.updated_at, + "retire completed Worker", + ) + .await + .unwrap(); + assert_eq!( + response.status, + StatusCode::OK.as_u16(), + "{}", + response.body + ); + assert!(response.body.contains("\"removed\":true")); + assert!(!response.body.contains("disposition")); + assert!(!response.body.contains("stage")); + assert!(!response.body.contains("path")); + assert!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &target) + .unwrap() + .is_none() + ); } #[tokio::test] @@ -13524,13 +14174,20 @@ mod tests { route_token, ) .body(Body::from( - r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker"}"#, + r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker","expected_worker_revision":"revision-1","reason":"retire target Worker"}"#, )) .unwrap(), ) .await .unwrap(); - assert_eq!(route_response.status(), StatusCode::NOT_IMPLEMENTED); + assert_eq!(route_response.status(), StatusCode::FORBIDDEN); + let route_body = axum::body::to_bytes(route_response.into_body(), usize::MAX) + .await + .unwrap(); + let route_body = String::from_utf8(route_body.to_vec()).unwrap(); + assert!(route_body.contains("orchestrator_required")); + assert!(!route_body.contains("source")); + assert!(!route_body.contains("proof")); let mut revoked = trust; revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string()); diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index f5fa9b83..23236931 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -474,6 +474,73 @@ pub trait ControlPlaneStore: Send + Sync { now_seconds: u64, consumed_at: &str, ) -> Result<bool>; + fn plan_worker_removal( + &self, + request: &crate::retention::WorkerRemovalPlanRequest, + inventory: &worker_runtime::retention::WorkerRetentionInventory, + ) -> std::result::Result< + crate::retention::WorkerRemovalPlan, + crate::retention::WorkerRetentionError, + > { + let _ = (request, inventory); + Err(crate::retention::WorkerRetentionError::Invalid( + "Worker retention authority is unavailable".to_string(), + )) + } + fn prepare_worker_removal_execution( + &self, + workspace_id: &str, + plan_id: &str, + input_fingerprint: &str, + ) -> std::result::Result< + crate::retention::PreparedWorkerRemoval, + crate::retention::WorkerRetentionError, + > { + let _ = (workspace_id, plan_id, input_fingerprint); + Err(crate::retention::WorkerRetentionError::Invalid( + "Worker retention authority is unavailable".to_string(), + )) + } + fn recover_worker_removal_execution( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + expected_worker_revision: &str, + reason: &str, + ) -> std::result::Result< + Option<crate::retention::PreparedWorkerRemoval>, + crate::retention::WorkerRetentionError, + > { + let _ = (workspace_id, worker, expected_worker_revision, reason); + Ok(None) + } + fn fail_worker_removal( + &self, + workspace_id: &str, + operation_id: &str, + input_fingerprint: &str, + category: &str, + ) -> std::result::Result<(), crate::retention::WorkerRetentionError> { + let _ = (workspace_id, operation_id, input_fingerprint, category); + Err(crate::retention::WorkerRetentionError::Invalid( + "Worker retention authority is unavailable".to_string(), + )) + } + fn commit_worker_removal( + &self, + workspace_id: &str, + operation_id: &str, + input_fingerprint: &str, + result: &worker_runtime::retention::WorkerRetentionExecutionResult, + ) -> std::result::Result< + crate::retention::WorkerRemovalPlan, + crate::retention::WorkerRetentionError, + > { + let _ = (workspace_id, operation_id, input_fingerprint, result); + Err(crate::retention::WorkerRetentionError::Invalid( + "Worker retention authority is unavailable".to_string(), + )) + } fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>>; fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>; fn get_repository( @@ -948,6 +1015,88 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + fn plan_worker_removal( + &self, + request: &crate::retention::WorkerRemovalPlanRequest, + inventory: &worker_runtime::retention::WorkerRetentionInventory, + ) -> std::result::Result< + crate::retention::WorkerRemovalPlan, + crate::retention::WorkerRetentionError, + > { + SqliteWorkspaceStore::plan_worker_removal(self, request, inventory) + } + + fn prepare_worker_removal_execution( + &self, + workspace_id: &str, + plan_id: &str, + input_fingerprint: &str, + ) -> std::result::Result< + crate::retention::PreparedWorkerRemoval, + crate::retention::WorkerRetentionError, + > { + SqliteWorkspaceStore::prepare_worker_removal_execution( + self, + workspace_id, + plan_id, + input_fingerprint, + ) + } + + fn recover_worker_removal_execution( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + expected_worker_revision: &str, + reason: &str, + ) -> std::result::Result< + Option<crate::retention::PreparedWorkerRemoval>, + crate::retention::WorkerRetentionError, + > { + SqliteWorkspaceStore::recover_worker_removal_execution( + self, + workspace_id, + worker, + expected_worker_revision, + reason, + ) + } + + fn fail_worker_removal( + &self, + workspace_id: &str, + operation_id: &str, + input_fingerprint: &str, + category: &str, + ) -> std::result::Result<(), crate::retention::WorkerRetentionError> { + SqliteWorkspaceStore::fail_worker_removal( + self, + workspace_id, + operation_id, + input_fingerprint, + category, + ) + } + + fn commit_worker_removal( + &self, + workspace_id: &str, + operation_id: &str, + input_fingerprint: &str, + result: &worker_runtime::retention::WorkerRetentionExecutionResult, + ) -> std::result::Result< + crate::retention::WorkerRemovalPlan, + crate::retention::WorkerRetentionError, + > { + SqliteWorkspaceStore::commit_worker_removal( + self, + workspace_id, + operation_id, + input_fingerprint, + result, + ) + } + fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>> { self.with_conn(|conn| { let mut stmt = conn.prepare( diff --git a/crates/workspace-server/src/worker_source.rs b/crates/workspace-server/src/worker_source.rs index 932250ad..4d3a98da 100644 --- a/crates/workspace-server/src/worker_source.rs +++ b/crates/workspace-server/src/worker_source.rs @@ -1,3 +1,4 @@ +use std::sync::{Arc, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; use axum::http::HeaderMap; @@ -168,18 +169,43 @@ async fn verify_worker_remove_source_with( }) } +pub(crate) trait VerifiedWorkerRemoveExecutor: Send + Sync { + fn execute( + &self, + source: VerifiedWorkerMutationSource, + target_runtime_id: &str, + target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, + ) -> Result<worker::WorkspaceResponse, String>; +} + #[derive(Clone)] pub(crate) struct EmbeddedServerWorkerMutationDispatcher { config: crate::server::ServerConfig, - store: std::sync::Arc<dyn crate::store::ControlPlaneStore>, + store: Arc<dyn crate::store::ControlPlaneStore>, + executor: Arc<OnceLock<Arc<dyn VerifiedWorkerRemoveExecutor>>>, } impl EmbeddedServerWorkerMutationDispatcher { pub(crate) fn new( config: crate::server::ServerConfig, - store: std::sync::Arc<dyn crate::store::ControlPlaneStore>, + store: Arc<dyn crate::store::ControlPlaneStore>, ) -> Self { - Self { config, store } + Self { + config, + store, + executor: Arc::new(OnceLock::new()), + } + } + + pub(crate) fn install_executor( + &self, + executor: Arc<dyn VerifiedWorkerRemoveExecutor>, + ) -> Result<(), &'static str> { + self.executor + .set(executor) + .map_err(|_| "WorkerRemove executor is already installed") } } @@ -191,11 +217,13 @@ impl worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher proof: InProcessWorkerMutationProof, target_runtime_id: &str, target_worker_id: &str, + expected_worker_revision: &str, + reason: &str, ) -> Result< worker::WorkspaceResponse, worker_runtime::worker_source::RuntimeWorkerMutationForwardError, > { - futures::executor::block_on(verify_worker_remove_source_with( + let source = futures::executor::block_on(verify_worker_remove_source_with( &self.config, &self.store, PresentedWorkerMutationSourceProof::InProcess(proof), @@ -207,11 +235,20 @@ impl worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher error.to_string(), ) })?; - Ok(worker::WorkspaceResponse { - status: 501, - body: "WorkerRemove lifecycle is not implemented by this operation boundary" - .to_string(), - }) + let executor = self.executor.get().ok_or_else(|| { + worker_runtime::worker_source::RuntimeWorkerMutationForwardError::Embedded( + "WorkerRemove executor is unavailable".to_string(), + ) + })?; + executor + .execute( + source, + target_runtime_id, + target_worker_id, + expected_worker_revision, + reason, + ) + .map_err(worker_runtime::worker_source::RuntimeWorkerMutationForwardError::Embedded) } } diff --git a/resources/prompts/role/orchestrator.md b/resources/prompts/role/orchestrator.md index a69b51be..cbb0af42 100644 --- a/resources/prompts/role/orchestrator.md +++ b/resources/prompts/role/orchestrator.md @@ -5,3 +5,5 @@ Keep durable orchestration behavior here and treat the first committed user mess Do not create or delegate an implementation worktree/branch until the Ticket records enough agreed intent, requirements, and acceptance criteria to bound the work. Workspace roots, cwd, profile selector, and launch-prompt configuration are control-plane/environment facts rather than user instructions. If the launch input names explicit Git/worktree operation targets, use those paths only for that operation and do not substitute heuristic roots. + +Use `WorkerRemove` only for a terminal or authoritatively reassigned non-internal Coder after implementation, review, fix, merge/commit, and report handoffs are complete. Do not remove a Coder merely because one turn completed or it is temporarily idle; retain it while review or request-changes work can still return. The Worker must already be stopped, must not be restoring, must have no current Ticket assignment, pending notification, Reviewer handoff, legal hold, or pin, and must not be this Orchestrator. Immediately before removal, reread authoritative Ticket state, assignment, thread/review evidence, and the target Worker with `WorkerShow`; pass the exact current `updated_at` value as `expected_worker_revision` with a concise reason. After removal, reread the Worker catalog and attachment state. Treat revision, assignment, running/restoring, retention-policy, attachment-close, and attachment-release conflicts as authoritative failures: do not guess policy or retry with stale input. `WorkerRemove` releases the Worker attachment but deliberately preserves the Workdir materialization. From f60c2d583485572697f7ec42d8cf3c8015e7c179 Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Wed, 12 Aug 2026 17:09:39 +0900 Subject: [PATCH 09/12] worker: resume failed removal operation --- crates/workspace-server/src/server.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 2359f1c5..f65acf93 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -396,6 +396,19 @@ impl WorkspaceWorkerRemoveExecutor { if prepared.plan.state == crate::retention::WorkerRemovalPlanState::Succeeded { return Ok(worker_remove_success_response(&target)); } + let prepared = + if prepared.plan.state == crate::retention::WorkerRemovalPlanState::Failed { + match self.store.prepare_worker_removal_execution( + &self.workspace_id, + &prepared.plan.plan_id, + &prepared.plan.input_fingerprint, + ) { + Ok(prepared) => prepared, + Err(error) => return Ok(worker_retention_error_response(error)), + } + } else { + prepared + }; return self .resume_worker_retention(&runtime, &target, prepared) .await; From 1e10cdecc8dcf8fde711ab280d0fce600c797b03 Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Wed, 12 Aug 2026 17:16:57 +0900 Subject: [PATCH 10/12] docs: report stale Flow verifier capture --- ...-11-flow-verifier-stale-session-capture.md | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/report/2026-08-11-flow-verifier-stale-session-capture.md diff --git a/docs/report/2026-08-11-flow-verifier-stale-session-capture.md b/docs/report/2026-08-11-flow-verifier-stale-session-capture.md new file mode 100644 index 00000000..e5a891db --- /dev/null +++ b/docs/report/2026-08-11-flow-verifier-stale-session-capture.md @@ -0,0 +1,52 @@ +# Flow verifier uses a stale committed-session capture + +Date: 2026-08-11 +Ticket: `00001KZPQW4GJ` +Flow instance: `019ff243-d4df-71f2-beb3-cbc360f58c34` + +## Symptom + +`RequestFlowTransition` repeatedly evaluated the `implement -> review` condition against a parent-session capture that ended immediately after branch creation and Ticket/plan reads. It did not observe later committed session entries containing implementation work, post-commit validation, clean-tree checks, commits, or independent Reviewer approval. + +The verifier therefore returned `indeterminate` even though the Workdir contained the implementation and the current Worker session had already recorded the required evidence. + +## Repository-visible evidence + +Named branch and commits: + +- `work/00001KZPQW4GJ-worker-remove-v3` +- `8ae930c5fc81acb2c60de15add07e016a1552edd` — `worker: add guarded WorkerRemove lifecycle` +- `f60c2d583485572697f7ec42d8cf3c8015e7c179` — `worker: resume failed removal operation` + +Post-commit validation: + +- `cargo test -p worker --lib`: 520 passed. +- `cargo test -p worker-runtime --lib`: 127 passed. +- `cargo test -p yoi-workspace-server --lib retention::tests`: 12 passed. +- `cargo test -p yoi-workspace-server --lib worker_remove`: 5 passed. +- `cargo test -p yoi-workspace-server --lib stale_policy_and_failed_retry_restore_fence`: passed. +- `cargo check -p yoi-workspace-server -p worker-runtime -p worker`: passed. +- `cargo fmt --all -- --check`: passed. +- `git diff --check`: passed. + +Independent read-only Reviewer verdict: + +> APPROVE — I found no blocker/high security or correctness issue in commits `8ae930c5` + `f60c2d58`. + +The Reviewer explicitly confirmed the constrained four-field tool input, proof-only destructive boundary, exact Runtime-result Worker revision binding, successful recovery after registry purge, and failed-operation re-entry through the authoritative prepare/executing fence. + +## Impact + +A correct, tested, independently approved implementation cannot advance from `implement` to `review` because the Flow verifier does not see newly committed Worker history. Repeating validation or review inside the same live session does not repair the verifier input. + +## Suggested fix + +Before evaluating a transition, refresh the verifier's session capture from the latest committed Worker history revision and include stable references to: + +- the current branch and commit, +- bounded validation command results, +- current clean-tree evidence, +- independent Reviewer verdicts, +- current Ticket review/evidence events. + +The refreshed evidence must be committed to Worker history before verifier context construction, following the project context-injection invariant. Do not use an unrecorded transient reminder or mutate earlier history. From 4e7eaac7d5730a72eb590f10d8ea1f1adb61c3aa Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Wed, 12 Aug 2026 17:31:29 +0900 Subject: [PATCH 11/12] worker: retry attachment cleanup stages --- crates/workspace-server/src/retention.rs | 40 ++++++++++++++++- crates/workspace-server/src/server.rs | 56 ++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index f4e06eb6..6e9e5751 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -106,12 +106,15 @@ pub struct WorkerRemovalPlan { pub reason: String, pub created_at: String, pub updated_at: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub failure_category: Option<String>, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct PreparedWorkerRemoval { pub plan: WorkerRemovalPlan, pub runtime_request: WorkerRetentionExecutionRequest, + pub prior_failure_category: Option<String>, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -369,6 +372,7 @@ impl SqliteWorkspaceStore { ) })?; let removed_at = plan.created_at.clone(); + let prior_failure_category = plan.failure_category.clone(); Ok(PreparedWorkerRemoval { runtime_request: WorkerRetentionExecutionRequest { operation_id: plan.operation_id.clone(), @@ -389,6 +393,7 @@ impl SqliteWorkspaceStore { diagnostics_disposition: plan.diagnostics_disposition, }, plan, + prior_failure_category, }) } @@ -429,6 +434,7 @@ impl SqliteWorkspaceStore { let Some(plan) = plan else { return Ok(None); }; + let prior_failure_category = plan.failure_category.clone(); let worker_number = plan.worker.worker_id.parse::<u64>().map_err(|_| { WorkerRetentionError::Invalid( "Runtime Worker id is not a canonical unsigned integer".to_string(), @@ -468,6 +474,7 @@ impl SqliteWorkspaceStore { diagnostics_disposition: plan.diagnostics_disposition, }, plan, + prior_failure_category, })) } @@ -758,7 +765,7 @@ fn load_plan_q(c: &Connection, key: &str, id: &str) -> crate::Result<Option<Work worker_revision,run_generation,policy_id,policy_revision,session_disposition, metadata_disposition,archive_retention_kind,archive_retention_seconds, diagnostics_disposition,diagnostics_retention_seconds,archive_id,blockers_json, - state,reason,created_at,updated_at + state,reason,created_at,updated_at,failure_category FROM worker_removal_operations WHERE {key}=?1" ); c.query_row(&query, params![id], |row| { @@ -799,6 +806,7 @@ fn load_plan_q(c: &Connection, key: &str, id: &str) -> crate::Result<Option<Work reason: row.get(19)?, created_at: row.get(20)?, updated_at: row.get(21)?, + failure_category: row.get(22)?, }) }) .optional() @@ -1532,6 +1540,36 @@ mod tests { ); } + #[test] + fn recovery_preserves_attachment_failure_stage_for_retry_ordering() { + let s = setup(); + let request = req(); + let plan = s.plan_worker_removal(&request, &inv()).unwrap(); + s.prepare_worker_removal_execution("w", &plan.plan_id, &plan.input_fingerprint) + .unwrap(); + s.fail_worker_removal( + "w", + &plan.operation_id, + &plan.input_fingerprint, + "workdir_attachment_release_failed", + ) + .unwrap(); + let recovered = s + .recover_worker_removal_execution( + "w", + &request.worker, + &request.expected_worker_revision, + &request.reason, + ) + .unwrap() + .unwrap(); + assert_eq!( + recovered.prior_failure_category.as_deref(), + Some("workdir_attachment_release_failed") + ); + assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Failed); + } + #[test] fn old_schema_upgrade_seeds_existing_workspace() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index f65acf93..5abd8a82 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -396,6 +396,11 @@ impl WorkspaceWorkerRemoveExecutor { if prepared.plan.state == crate::retention::WorkerRemovalPlanState::Succeeded { return Ok(worker_remove_success_response(&target)); } + let must_close_session = + prepared.prior_failure_category.as_deref() == Some("workdir_session_close_failed"); + let must_release_attachment = must_close_session + || prepared.prior_failure_category.as_deref() + == Some("workdir_attachment_release_failed"); let prepared = if prepared.plan.state == crate::retention::WorkerRemovalPlanState::Failed { match self.store.prepare_worker_removal_execution( @@ -409,6 +414,57 @@ impl WorkspaceWorkerRemoveExecutor { } else { prepared }; + if must_close_session { + let session = { + self.workdir_sessions + .lock() + .map_err(|_| "Workdir session registry was poisoned".to_string())? + .get(&target) + .cloned() + }; + if let Some(session) = session { + if session.close().await.is_err() { + let _ = self.store.fail_worker_removal( + &self.workspace_id, + &prepared.plan.operation_id, + &prepared.plan.input_fingerprint, + "workdir_session_close_failed", + ); + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "attachment_close_failed", + "Worker Workdir session could not be closed; removal can be retried", + )); + } + self.workdir_sessions + .lock() + .map_err(|_| "Workdir session registry was poisoned".to_string())? + .remove(&target); + } + } + if must_release_attachment + && self + .store + .detach_worker_workdir( + &self.workspace_id, + &target, + None, + &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + ) + .is_err() + { + let _ = self.store.fail_worker_removal( + &self.workspace_id, + &prepared.plan.operation_id, + &prepared.plan.input_fingerprint, + "workdir_attachment_release_failed", + ); + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "attachment_release_failed", + "Worker Workdir attachment could not be released; removal can be retried", + )); + } return self .resume_worker_retention(&runtime, &target, prepared) .await; From e80581d13963ef21f03561735bbb11f4f6ee8c9d Mon Sep 17 00:00:00 2001 From: Hare <kei.hiracchi.0928@gmail.com> Date: Wed, 12 Aug 2026 17:40:11 +0900 Subject: [PATCH 12/12] worker: preserve retry cleanup stage --- crates/workspace-server/src/retention.rs | 39 +++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index 6e9e5751..90b89935 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -345,7 +345,7 @@ impl SqliteWorkspaceStore { return Err(stale_error(&plan,"current assignment added")); } let now=Utc::now().to_rfc3339(); - tx.execute("UPDATE worker_removal_operations SET state='executing',failure_category=NULL,updated_at=?1 WHERE operation_id=?2",params![now,plan.operation_id])?; + tx.execute("UPDATE worker_removal_operations SET state='executing',updated_at=?1 WHERE operation_id=?2",params![now,plan.operation_id])?; plan.state=WorkerRemovalPlanState::Executing;plan.updated_at=now;tx.commit()?;Ok(plan) }).map_err(map_error) } @@ -1163,6 +1163,43 @@ mod tests { Err(WorkerRetentionError::StalePlan { .. }) )); } + #[test] + fn failed_retry_keeps_cleanup_stage_durable() { + let s = setup(); + let p = s.plan_worker_removal(&req(), &inv()).unwrap(); + s.prepare_worker_removal_execution("w", &p.plan_id, &p.input_fingerprint) + .unwrap(); + s.fail_worker_removal( + "w", + &p.operation_id, + &p.input_fingerprint, + "workdir_attachment_release_failed", + ) + .unwrap(); + let retry = s + .prepare_worker_removal_execution("w", &p.plan_id, &p.input_fingerprint) + .unwrap(); + assert_eq!(retry.plan.state, WorkerRemovalPlanState::Executing); + assert_eq!( + retry.prior_failure_category.as_deref(), + Some("workdir_attachment_release_failed") + ); + let persisted: Option<String> = s + .with_conn(|conn| { + conn.query_row( + "SELECT failure_category FROM worker_removal_operations WHERE operation_id=?1", + params![p.operation_id], + |row| row.get(0), + ) + .map_err(StoreError::from) + }) + .unwrap(); + assert_eq!( + persisted.as_deref(), + Some("workdir_attachment_release_failed") + ); + } + #[test] fn prepared_execution_is_derived_from_pinned_plan_generation() { let s = setup();