diff --git a/crates/manifest/src/config.rs b/crates/manifest/src/config.rs index 4dbba20cb..7407b8b0b 100644 --- a/crates/manifest/src/config.rs +++ b/crates/manifest/src/config.rs @@ -288,6 +288,8 @@ pub struct SubjektivFeatureConfigPartial { #[serde(default)] pub enabled: Option, #[serde(default)] + pub consolidation_tools: Option, + #[serde(default)] pub extraction: Option, } @@ -295,6 +297,7 @@ impl SubjektivFeatureConfigPartial { fn merge(self, other: Self) -> Self { Self { enabled: other.enabled.or(self.enabled), + consolidation_tools: other.consolidation_tools.or(self.consolidation_tools), extraction: merge_option( self.extraction, other.extraction, @@ -544,6 +547,7 @@ impl From for ResolvedSubjektivFeatureConfig { Self { profile: SubjektivFeatureProfileConfig { enabled: value.enabled.unwrap_or_default(), + consolidation_tools: value.consolidation_tools.unwrap_or_default(), extraction: MemoryExtractionProfileConfig { enabled: extraction.enabled.unwrap_or(true), model: extraction.model, @@ -563,6 +567,7 @@ impl From for SubjektivFeatureConfigPartial { fn from(value: ResolvedSubjektivFeatureConfig) -> Self { Self { enabled: Some(value.profile.enabled), + consolidation_tools: Some(value.profile.consolidation_tools), extraction: Some(MemoryExtractionProfileConfigPartial { enabled: Some(value.profile.extraction.enabled), model: value.profile.extraction.model, diff --git a/crates/manifest/src/lib.rs b/crates/manifest/src/lib.rs index 016b88577..03e98be88 100644 --- a/crates/manifest/src/lib.rs +++ b/crates/manifest/src/lib.rs @@ -402,6 +402,8 @@ impl ResolvedMemoryFeatureConfig { #[serde(default, deny_unknown_fields)] pub struct SubjektivFeatureProfileConfig { pub enabled: bool, + /// Explicit consolidation tool policy; never grants Job or subject authority. + pub consolidation_tools: bool, pub extraction: MemoryExtractionProfileConfig, } @@ -409,6 +411,7 @@ impl Default for SubjektivFeatureProfileConfig { fn default() -> Self { Self { enabled: false, + consolidation_tools: false, extraction: MemoryExtractionProfileConfig::default(), } } diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index 0b34c83c7..242780b6c 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -420,6 +420,18 @@ pub struct WorkerInput { pub segments: Option>, } +/// Host-authored Job/attempt identity, independent of Profile selection. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct BackendJobExecutionBinding { + pub job_id: String, + pub attempt_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_revision: Option, + #[serde(default)] + pub subjektiv_consolidation: bool, +} + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct CreateWorkerRequest { pub worker_id: WorkerId, @@ -448,6 +460,8 @@ pub struct CreateWorkerRequest { /// never activates subjektiv for an ordinary Workspace Worker. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub subjektiv_attached: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backend_job: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 8d99e78b9..0062c3bc5 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -5530,6 +5530,9 @@ pub struct SubjektivSurfacePrepareRequest {} #[serde(deny_unknown_fields)] pub struct SubjektivSurfacePrepareResponse { pub generation_id: String, + /// Existing ready surface at this exact store revision, if no rebuild is needed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_snapshot_id: Option, #[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))] pub store_revision: u64, #[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))] diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index d5d0405bc..2b00cac60 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -258,6 +258,8 @@ impl std::fmt::Debug for WorkspaceApiRef { } } +pub use worker::BackendJobExecutionBinding; + /// Canonical Runtime Worker creation request. /// /// Browser/product launch semantics are resolved by a backend before this @@ -302,6 +304,9 @@ pub struct CreateWorkerRequest { /// may enable subjektiv policy, but only this host-owned bit activates it. #[serde(default, skip_serializing_if = "is_false")] pub subjektiv_attached: bool, + /// Backend-only immutable Job/attempt capability. Never inferred from Profile policy. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backend_job: Option, } /// Last persisted Worker lifecycle status. diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 623e5b524..0326f899a 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -3129,6 +3129,7 @@ mod tests { language: "English".to_string(), }), subjektiv_attached: false, + backend_job: None, } } @@ -4685,6 +4686,7 @@ mod ws_tests { language: "English".to_string(), }), subjektiv_attached: false, + backend_job: None, } } diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 119a7e4d4..543d89244 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -898,7 +898,9 @@ impl Runtime { message: "persisted Worker restore request is unavailable".to_string(), } })?; - if existing_request.create_fingerprint != request.create_fingerprint { + if existing_request.create_fingerprint != request.create_fingerprint + || existing_request.backend_job != request.backend_job + { return Err(RuntimeError::InvalidRequest(format!( "worker {} was already created with a different fingerprint", request.worker_id @@ -1068,7 +1070,9 @@ impl Runtime { message: "persisted Worker restore request is unavailable".to_string(), } })?; - if existing_request.create_fingerprint != request.create_fingerprint { + if existing_request.create_fingerprint != request.create_fingerprint + || existing_request.backend_job != request.backend_job + { return Err(RuntimeError::InvalidRequest(format!( "worker {} was already created with a different fingerprint", request.worker_id @@ -6242,6 +6246,16 @@ fn validate_logical_workdir_attachments( } fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), RuntimeError> { + if let Some(binding) = &request.backend_job { + binding + .validate() + .map_err(|message| RuntimeError::InvalidRequest(message.into()))?; + if request.workspace_api.is_none() + || (binding.subjektiv_consolidation && !request.subjektiv_attached) + { + return Err(RuntimeError::InvalidRequest("Backend Job capability requires Workspace authority and consolidation requires trusted subject attachment".into())); + } + } if request.create_fingerprint.trim().is_empty() { return Err(RuntimeError::InvalidRequest( "create_fingerprint must not be empty".to_string(), @@ -7112,9 +7126,95 @@ mod tests { language: "English".to_string(), }), subjektiv_attached: false, + backend_job: None, } } + #[cfg(feature = "fs-store")] + #[test] + fn backend_job_binding_survives_restart_and_restore_without_profile_inference() { + let dir = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "job-runtime".into(), + display_name: None, + }; + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let mut request = task_request("bound Job"); + request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "local".into(), + base_url: "https://backend.invalid".into(), + }); + request.backend_job = Some(crate::catalog::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: Some("revision-1".into()), + subjektiv_consolidation: false, + }); + let owner = scope("local", "server-job-test"); + let created = runtime + .create_worker_scoped(&owner, request.clone()) + .unwrap(); + assert_eq!( + backend.job_bindings.lock().unwrap().as_slice(), + &[request.backend_job.clone()] + ); + let mut changed = request.clone(); + changed.backend_job.as_mut().unwrap().attempt_id = "attempt-other".into(); + assert!(runtime.create_worker_scoped(&owner, changed).is_err()); + drop(runtime); + let restored_backend = Arc::new(TestExecutionBackend::default()); + let restored = + Runtime::with_fs_store_and_execution_backend(options, restored_backend.clone()) + .unwrap(); + assert_eq!( + restored + .lock() + .unwrap() + .worker(&created.worker_ref) + .unwrap() + .request + .as_ref() + .unwrap() + .backend_job, + request.backend_job + ); + restored + .restore_worker( + &created.worker_ref, + restored.test_restore_request(&created.worker_ref), + ) + .unwrap(); + assert_eq!( + restored_backend.job_bindings.lock().unwrap().as_slice(), + &[request.backend_job] + ); + } + + #[test] + fn backend_job_binding_requires_workspace_and_consolidation_attachment() { + let mut request = task_request("Job binding validation"); + request.backend_job = Some(crate::catalog::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: None, + subjektiv_consolidation: true, + }); + assert!(validate_create_worker_request(&request).is_err()); + request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "local".into(), + base_url: "https://backend.invalid".into(), + }); + assert!(validate_create_worker_request(&request).is_err()); + request.subjektiv_attached = true; + assert!(validate_create_worker_request(&request).is_ok()); + request.backend_job.as_mut().unwrap().attempt_id.clear(); + assert!(validate_create_worker_request(&request).is_err()); + } + #[test] fn durable_worker_request_omits_repository_credentials() { let mut request = task_request("worker-secret-redaction"); @@ -7537,6 +7637,7 @@ mod tests { #[derive(Default)] struct TestExecutionBackend { spawn_result: Mutex>, + job_bindings: Mutex>>, dispatch_result: Mutex>, stop_result: Mutex>, stop_gate: Mutex>>, @@ -7653,6 +7754,10 @@ mod tests { } fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { + self.job_bindings + .lock() + .unwrap() + .push(request.request.backend_job.clone()); if let Some(result) = self.spawn_result.lock().unwrap().take() { return result; } @@ -7726,6 +7831,10 @@ mod tests { &self, request: WorkerExecutionRestoreRequest, ) -> WorkerExecutionSpawnResult { + self.job_bindings + .lock() + .unwrap() + .push(request.request.backend_job.clone()); *self.restore_count.lock().unwrap() += 1; self.restore_operation_ids .lock() diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index cb5830148..d338848a8 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -1086,10 +1086,26 @@ fn restored_workdir_router( Ok(router) } +fn validate_backend_job_profile( + manifest: &manifest::WorkerManifest, + request: &CreateWorkerRequest, +) -> Result<(), String> { + if let Some(binding) = &request.backend_job { + let policy = &manifest.feature.subjektiv.profile; + if policy.consolidation_tools != binding.subjektiv_consolidation + || (binding.subjektiv_consolidation && policy.extraction.enabled) + { + return Err("Backend Job consolidation_tools must match its trusted consolidation grant and extraction must be disabled".into()); + } + } + Ok(()) +} + fn bind_workspace_memory_settings( manifest: &mut manifest::WorkerManifest, request: &CreateWorkerRequest, ) -> Result<(), String> { + validate_backend_job_profile(manifest, request)?; let Some(snapshot) = request.memory_settings.as_ref() else { if request.workspace_api.is_some() || request.subjektiv_attached { return Err( @@ -1143,6 +1159,7 @@ fn validate_worker_memory_settings( manifest: &manifest::WorkerManifest, request: &CreateWorkerRequest, ) -> Result<(), String> { + validate_backend_job_profile(manifest, request)?; let Some(expected) = request.memory_settings.as_ref() else { if request.subjektiv_attached || request.workspace_api.is_some() { return Err( @@ -1419,6 +1436,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } })?; let worker = prepared.worker_mut(); + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(str::to_string)?; + } validate_worker_memory_settings(worker.manifest(), &request.request)?; worker .finalize_subjektiv_session_attribution( @@ -1684,6 +1706,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } Err(err) => return Err(format!("failed to restore Worker from metadata: {err}")), }; + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(str::to_string)?; + } validate_worker_memory_settings(worker.manifest(), &request.request)?; worker .finalize_subjektiv_session_attribution( @@ -7221,6 +7248,58 @@ mod tests { workspace_api: None, memory_settings: None, subjektiv_attached: false, + backend_job: None, + } + } + + #[test] + fn backend_job_profile_policy_matches_trusted_grant_without_implicit_changes() { + let mut manifest = WorkerManifest::from_toml( + r#" + [worker] + name = "job-policy-test" + pwd = "./" + [model] + scheme = "anthropic" + model_id = "chosen-model" + auth = { kind = "none" } + [engine] + instruction = "default" + [[scope.allow]] + target = "./" + permission = "read" + "#, + ) + .unwrap(); + let mut request = create_request("policy"); + for granted in [false, true] { + request.backend_job = Some(crate::catalog::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: None, + subjektiv_consolidation: granted, + }); + for tools in [false, true] { + for extraction in [false, true] { + manifest.feature.subjektiv.profile.consolidation_tools = tools; + manifest.feature.subjektiv.profile.extraction.enabled = extraction; + let expected = tools == granted && (!granted || !extraction); + let before = serde_json::to_value(&manifest).unwrap(); + assert_eq!( + validate_backend_job_profile(&manifest, &request).is_ok(), + expected + ); + assert_eq!( + bind_workspace_memory_settings(&mut manifest, &request).is_ok(), + expected + ); + assert_eq!( + validate_worker_memory_settings(&manifest, &request).is_ok(), + expected + ); + assert_eq!(serde_json::to_value(&manifest).unwrap(), before); + } + } } } diff --git a/crates/worker/src/backend_job.rs b/crates/worker/src/backend_job.rs new file mode 100644 index 000000000..e5b01b431 --- /dev/null +++ b/crates/worker/src/backend_job.rs @@ -0,0 +1,29 @@ +//! Host-authored Job authority, separate from reusable Profile policy. + +use serde::{Deserialize, Serialize}; + +/// Immutable execution capability delivered only by a trusted Backend host. +/// Names, instructions and Profiles never confer this capability. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BackendJobExecutionBinding { + pub job_id: String, + pub attempt_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_revision: Option, + #[serde(default)] + pub subjektiv_consolidation: bool, +} + +impl BackendJobExecutionBinding { + pub fn validate(&self) -> Result<(), &'static str> { + if [&self.job_id, &self.attempt_id] + .into_iter() + .chain(self.input_revision.iter()) + .any(|value| value.trim().is_empty() || value.chars().any(char::is_control)) + { + return Err("Backend Job binding contains an invalid identity or input revision"); + } + Ok(()) + } +} diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 2b47b9940..509c2a85e 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -1376,10 +1376,12 @@ where ), None => durable_parent_notifications, }; - let backend_job_profile = worker.manifest().engine.instruction == "internal.backend_job_system"; + let backend_job = worker.backend_job().cloned(); + // Bounded Jobs complete their lifecycle synchronously, never by post-run background work. + let lifecycle_enabled = worker.manifest_lifecycle_features_enabled() && backend_job.is_none(); let prompts = worker.prompts().clone(); let tracker = tools::Tracker::new(); - if !backend_job_profile { + { let paste_store = worker.store().clone(); let paste_session_id = worker.session_id(); worker @@ -1426,11 +1428,9 @@ where let worker_enabled = feature_config.worker.enabled; let sub_worker_enabled = feature_config.sub_worker.enabled; let mut feature_registry = FeatureRegistryBuilder::new(); - if !backend_job_profile { - feature_registry - .add_module(crate::feature::builtin::chat_invocation::AttachmentInvocationFeature); - } - if backend_job_profile { + feature_registry + .add_module(crate::feature::builtin::chat_invocation::AttachmentInvocationFeature); + if let Some(binding) = backend_job.as_ref() { let workspace_client = worker.workspace_client_handle(); if !workspace_client.is_available() || workspace_client.workspace_id().is_none() { return Err(std::io::Error::new( @@ -1438,11 +1438,18 @@ where "Backend Job result capability requires Backend Workspace API authority", )); } - feature_registry.add_module( + let mut result_feature = crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( - workspace_client, - ), - ); + workspace_client.clone(), + ); + if binding.subjektiv_consolidation { + let surface = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::for_job( + workspace_client, spawner_manifest.clone(), worker.llm_client_handle(), + prompts.clone(), spawner_workspace_context.clone(), + )?; + result_feature = result_feature.with_surface(surface); + } + feature_registry.add_module(result_feature); } let memory_profile = &worker.manifest().feature.memory.profile; let subjektiv_profile = &worker.manifest().feature.subjektiv.profile; @@ -1455,7 +1462,11 @@ where let mut feature_prompt_contribution = memory_install_plan.as_ref().map(|plan| { ( plan.resident_summary_source.clone(), - plan.system_prompt_override.clone(), + if backend_job.is_none() { + plan.system_prompt_override.clone() + } else { + None + }, ) }); let memory_lifecycle_config = worker.manifest().feature.memory.clone(); @@ -1469,7 +1480,9 @@ where worker.prompts().load_full(), )?; if let Some(plan) = subjektiv_consolidation_plan { - feature_prompt_contribution = Some((None, Some(plan.system_prompt_override))); + if backend_job.is_none() { + feature_prompt_contribution = Some((None, Some(plan.system_prompt_override))); + } feature_registry.add_module(plan.module); } let ordinary_subjektiv_features_enabled = @@ -1488,7 +1501,7 @@ where .to_string(); if let Some(refresh_feature) = crate::feature::builtin::memory::SubjektivResidentRestoreRefreshFeature::for_host( - worker.manifest_lifecycle_features_enabled(), + lifecycle_enabled, Arc::clone(&resident_summary_source), worker.prompts(), workspace_id, @@ -1519,7 +1532,7 @@ where } if let Some(memory_lifecycle) = crate::feature::builtin::memory_lifecycle::MemoryLifecycleFeature::from_resolved_config( - worker.manifest_lifecycle_features_enabled(), + lifecycle_enabled, memory_lifecycle_config, worker.committed_session_capture_handle(), worker.session_extension_handle(), @@ -1533,8 +1546,8 @@ where { feature_registry.add_module(memory_lifecycle); } - if let Some(surface_lifecycle) = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::from_manifest( - worker.manifest_lifecycle_features_enabled(), + if backend_job.is_none() && let Some(surface_lifecycle) = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::from_manifest( + lifecycle_enabled, worker.workspace_client_handle(), spawner_manifest.clone(), worker.llm_client_handle(), @@ -1546,7 +1559,7 @@ where if ordinary_subjektiv_features_enabled && let Some(subjektiv_lifecycle) = crate::feature::builtin::memory_lifecycle::SubjektivLifecycleFeature::from_resolved_config( - worker.manifest_lifecycle_features_enabled(), + lifecycle_enabled, worker.manifest().feature.subjektiv.clone(), worker.committed_session_capture_handle(), worker.session_extension_handle(), @@ -1721,7 +1734,7 @@ where ) })?; } - if wip_mode && !backend_job_profile { + if wip_mode { crate::checkout::mount_checkouts( &mut wip_mount_registry, worker.workdir_sessions(), diff --git a/crates/worker/src/feature/background.rs b/crates/worker/src/feature/background.rs index 976734b88..e83c8f744 100644 --- a/crates/worker/src/feature/background.rs +++ b/crates/worker/src/feature/background.rs @@ -190,7 +190,7 @@ impl BackgroundTaskCancellation { } } - fn cancel(&self) { + pub(crate) fn cancel(&self) { self.sender.send_replace(true); } } diff --git a/crates/worker/src/feature/builtin/backend_job_result.rs b/crates/worker/src/feature/builtin/backend_job_result.rs index 93808162a..bf398dcc5 100644 --- a/crates/worker/src/feature/builtin/backend_job_result.rs +++ b/crates/worker/src/feature/builtin/backend_job_result.rs @@ -4,7 +4,9 @@ //! identity come from the authenticated Workspace client, while Job, attempt, //! and input revision are resolved from the Backend-owned binding. -use std::sync::Arc; +use crate::feature::background::BackgroundTaskCancellation; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput}; use async_trait::async_trait; @@ -12,6 +14,7 @@ use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; +use crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature; use crate::feature::{ FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureModule, ToolContribution, ToolDeclaration, ToolDefinition, @@ -27,9 +30,61 @@ struct SubmitResultInput { result: Value, } +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct ConsolidationResult { + subject_id: String, + candidate_ids: Vec, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct SubmitConsolidationResultInput { + /// Closed candidate ids only. Host awaits and adds the actual surface outcome. + result: ConsolidationResult, +} + +#[derive(Default)] +struct ResultSubmissionState { + sealed: Option, + reusable_surface: Option, +} + +struct SealedResult { + original: Value, + completed: Value, +} + #[derive(Clone)] struct SubmitResultTool { client: Arc, + surface: Option, + submission: Arc>, + active: Arc>>, +} + +struct ActiveResultGuard { + id: String, + active: Arc>>, + cancellation: BackgroundTaskCancellation, +} +impl Drop for ActiveResultGuard { + fn drop(&mut self) { + self.cancellation.cancel(); + self.active + .lock() + .expect("Job result cancellation state poisoned") + .remove(&self.id); + } +} + +fn cancelled_output() -> ToolError { + ToolError::Cancelled(ToolOutput { + summary: "Job result submission cancelled; committed candidate decisions are preserved" + .into(), + content: None, + attachments: Vec::new(), + }) } #[async_trait] @@ -37,27 +92,128 @@ impl Tool for SubmitResultTool { async fn execute( &self, input_json: &str, - _context: ToolExecutionContext, + context: ToolExecutionContext, ) -> Result { let input: SubmitResultInput = serde_json::from_str(input_json) .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; + let id = context.execution_id(); + let cancellation = self + .active + .lock() + .expect("Job result cancellation state poisoned") + .entry(id.clone()) + .or_default() + .clone(); + let _active = ActiveResultGuard { + id, + active: self.active.clone(), + cancellation: cancellation.clone(), + }; + // Serialize result calls and retain the exact lifecycle outcome for ambiguous retries. + let mut submission = tokio::select! { + guard = self.submission.lock() => guard, + _ = cancellation.cancelled() => return Err(cancelled_output()), + }; + // An existing seal may already have had an effect, even if this replay + // is now definitively rejected. It must never be reopened. + let was_previously_sealed = submission.sealed.is_some(); + let result = if let Some(sealed) = submission.sealed.as_ref() { + if sealed.original != input.result { + return Err(ToolError::InvalidArgument( + "Job result is already sealed; retry the identical result".into(), + )); + } + sealed.completed.clone() + } else { + let mut result = input.result.clone(); + if let Some(surface) = &self.surface { + let checked = serde_json::from_str::(input_json) + .map_err(|error| { + ToolError::InvalidArgument(format!( + "invalid consolidation result (surface is Host-generated): {error}" + )) + })? + .result; + if checked.subject_id.trim().is_empty() + || checked.candidate_ids.iter().any(|id| id.trim().is_empty()) + { + return Err(ToolError::InvalidArgument( + "subject_id and candidate ids must not be empty".into(), + )); + } + let object = result + .as_object_mut() + .expect("typed consolidation result is an object"); + let outcome = surface + .complete_for_job(cancellation.clone(), submission.reusable_surface.as_ref()) + .await + .map_err(|error| { + if cancellation.is_cancelled() { + cancelled_output() + } else { + ToolError::ExecutionFailed(error.to_string()) + } + })?; + submission.reusable_surface = Some(outcome.clone()); + object.insert("surface".into(), outcome); + } + result + }; + if cancellation.is_cancelled() { + return Err(cancelled_output()); + } let workspace_id = self.client.workspace_id().ok_or_else(|| { ToolError::ExecutionFailed( "Backend Job result submission requires Workspace identity".to_string(), ) })?; - let response = self - .client - .execute(WorkspaceRequest::json( - WorkspaceRequestMethod::Post, - format!("/api/w/{workspace_id}/workers/self/backend-job-result"), - serde_json::to_string(&serde_json::json!({ "result": input.result })) - .map_err(|error| ToolError::Internal(error.to_string()))?, - )) - .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; + let request = WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + format!("/api/w/{workspace_id}/workers/self/backend-job-result"), + serde_json::to_string(&serde_json::json!({ "result": result })) + .map_err(|error| ToolError::Internal(error.to_string()))?, + ); + // Seal before dispatch: a transport failure may follow durable acceptance. + if submission.sealed.is_none() { + submission.sealed = Some(SealedResult { + original: input.result, + completed: result, + }); + submission.reusable_surface = None; + } + let response = match self.client.execute(request) { + Ok(response) => response, + Err(error) => { + return Err(ToolError::ExecutionFailed(format!( + "Backend Job result acceptance is unknown; retry the identical result: {error}" + ))); + } + }; + let definitive_rejection = matches!(response.status, 400 | 403 | 409 | 422); if !response.is_success() { + let sealed = submission.sealed.as_mut().expect("sealed before dispatch"); + if definitive_rejection && !was_previously_sealed { + // Only a definitively unaccepted request can be corrected. Reuse + // a ready snapshot after a new preparation confirms its revision; + // a conflict invalidates even that snapshot. Failed outcomes are + // rebuilt, since another generation can replace their marker. + submission.reusable_surface = if response.status != 409 { + sealed + .completed + .get("surface") + .filter(|surface| surface["availability"] == "ready") + .cloned() + } else { + None + }; + submission.sealed = None; + return Err(ToolError::ExecutionFailed(format!( + "Backend Job result was definitively rejected with HTTP {}; correct result fields or retry to rebuild a stale surface: {}", + response.status, response.body + ))); + } return Err(ToolError::ExecutionFailed(format!( - "Backend Job result submission was rejected with HTTP {}: {}", + "Backend Job result acceptance remains sealed; retry the identical result (HTTP {}): {}", response.status, response.body ))); } @@ -67,11 +223,30 @@ impl Tool for SubmitResultTool { attachments: Vec::new(), }) } + + async fn cancel_execution(&self, context: &ToolExecutionContext) -> Result<(), ToolError> { + self.active + .lock() + .expect("Job result cancellation state poisoned") + .entry(context.execution_id()) + .or_default() + .cancel(); + Ok(()) + } } -fn definition(client: Arc) -> ToolDefinition { - let schema = serde_json::to_value(schemars::schema_for!(SubmitResultInput)) - .expect("Backend Job result schema must serialize"); +fn definition( + client: Arc, + surface: Option, +) -> ToolDefinition { + let submission = Arc::new(tokio::sync::Mutex::new(ResultSubmissionState::default())); + let active = Arc::new(Mutex::new(HashMap::new())); + let schema = if surface.is_some() { + serde_json::to_value(schemars::schema_for!(SubmitConsolidationResultInput)) + } else { + serde_json::to_value(schemars::schema_for!(SubmitResultInput)) + } + .expect("Backend Job result schema must serialize"); Arc::new(move || { ( ToolMeta::new(TOOL_NAME) @@ -79,6 +254,9 @@ fn definition(client: Arc) -> ToolDefinition { .input_schema(schema.clone()), Arc::new(SubmitResultTool { client: client.clone(), + surface: surface.clone(), + submission: submission.clone(), + active: active.clone(), }) as Arc, ) }) @@ -87,11 +265,22 @@ fn definition(client: Arc) -> ToolDefinition { #[derive(Clone)] pub struct BackendJobResultFeature { client: Arc, + surface: Option, } impl BackendJobResultFeature { pub fn new(client: Arc) -> Self { - Self { client } + Self { + client, + surface: None, + } + } +} + +impl BackendJobResultFeature { + pub(crate) fn with_surface(mut self, surface: SubjektivSurfaceLifecycleFeature) -> Self { + self.surface = Some(surface); + self } } @@ -105,7 +294,7 @@ impl FeatureModule for BackendJobResultFeature { fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { context.tools().register(ToolContribution::new( TOOL_NAME, - definition(self.client.clone()), + definition(self.client.clone(), self.surface.clone()), )) } } @@ -172,4 +361,14 @@ mod tests { .ends_with("/workers/self/backend-job-result") ); } + #[tokio::test] + async fn cancellation_before_execution_never_submits_a_result() { + let client = Arc::new(RecordingClient::default()); + let (_, tool) = definition(client.clone(), None)(); + let context = ToolExecutionContext::new("result-call", "batch", 0); + tool.cancel_execution(&context).await.unwrap(); + let outcome = tool.execute(r#"{"result":{"valid":true}}"#, context).await; + assert!(matches!(outcome, Err(ToolError::Cancelled(_)))); + assert!(client.requests.lock().unwrap().is_empty()); + } } diff --git a/crates/worker/src/feature/builtin/memory.rs b/crates/worker/src/feature/builtin/memory.rs index 3565e8fcf..d25581cf5 100644 --- a/crates/worker/src/feature/builtin/memory.rs +++ b/crates/worker/src/feature/builtin/memory.rs @@ -1115,7 +1115,7 @@ pub(crate) fn is_builtin_subjektiv_consolidation_profile( pub(crate) fn ordinary_subjektiv_features_enabled(manifest: &manifest::WorkerManifest) -> bool { manifest.feature.subjektiv.execution_enabled() - && !is_builtin_subjektiv_consolidation_profile(manifest) + && !manifest.feature.subjektiv.profile.consolidation_tools } pub(crate) fn ordinary_subjektiv_resident_summary_source( @@ -1175,7 +1175,7 @@ impl SubjektivConsolidationFeatureInstallPlan { client: Arc, prompts: Arc, ) -> std::io::Result> { - if !is_builtin_subjektiv_consolidation_profile(manifest) { + if !manifest.feature.subjektiv.profile.consolidation_tools { return Ok(None); } let config = &manifest.feature.subjektiv; diff --git a/crates/worker/src/feature/builtin/memory_surface_lifecycle.rs b/crates/worker/src/feature/builtin/memory_surface_lifecycle.rs index 7e11ac8ee..657d9b96e 100644 --- a/crates/worker/src/feature/builtin/memory_surface_lifecycle.rs +++ b/crates/worker/src/feature/builtin/memory_surface_lifecycle.rs @@ -104,6 +104,23 @@ impl SubjektivSurfaceLifecycleFeature { if !lifecycle_enabled || !dedicated { return Ok(None); } + Self::for_job( + workspace_client, + manifest, + client, + prompts, + workspace_context, + ) + .map(Some) + } + + pub(crate) fn for_job( + workspace_client: Arc, + manifest: WorkerManifest, + client: Box, + prompts: Arc>, + workspace_context: WorkerWorkspaceContext, + ) -> std::io::Result { manifest .feature .subjektiv @@ -121,7 +138,7 @@ impl SubjektivSurfaceLifecycleFeature { "subjektiv surface generation requires Backend Workspace API authority", )); } - Ok(Some(Self { + Ok(Self { task: SubjektivSurfaceLifecycleTask { workspace_client, manifest, @@ -129,7 +146,18 @@ impl SubjektivSurfaceLifecycleFeature { prompts, workspace_context, }, - })) + }) + } + + /// The Job tool awaits this clean-context lifecycle before submitting its result. + pub(crate) async fn complete_for_job( + &self, + cancellation: BackgroundTaskCancellation, + reusable_surface: Option<&serde_json::Value>, + ) -> Result { + self.task + .generate(None, cancellation, reusable_surface) + .await } } @@ -182,80 +210,125 @@ impl FeatureBackgroundTask for SubjektivSurfaceLifecycleTask { context: BackgroundTaskContext, cancellation: BackgroundTaskCancellation, ) -> Result<(), HookError> { + self.generate(Some(&context), cancellation, None) + .await + .map(|_| ()) + } +} + +impl SubjektivSurfaceLifecycleTask { + async fn generate( + &self, + context: Option<&BackgroundTaskContext>, + cancellation: BackgroundTaskCancellation, + reusable_surface: Option<&serde_json::Value>, + ) -> Result { for attempt in 0..MAX_GENERATION_ATTEMPTS { - context.generation_fence.ensure_current()?; + if let Some(context) = context { + context.generation_fence.ensure_current()?; + } if cancellation.is_cancelled() { - return Ok(()); + return Err(HookError::new( + HookErrorCategory::Cancelled, + "surface generation cancelled", + )); } let generation = self .workspace_client .prepare_subjektiv_memory_surface() .await .map_err(surface_hook_error)?; + if context.is_none() { + if let Some(snapshot_id) = &generation.current_snapshot_id { + return Ok(serde_json::json!({ + "availability": "ready", "generation_id": generation.generation_id, + "store_revision": generation.store_revision, "snapshot_id": snapshot_id, + })); + } + } + // Preparation is Backend-fenced to the live attempt and immutable batch. + // Only a ready snapshot is immutable within its store revision. Never + // reuse a failed marker, which a later generation can replace. + if let Some(surface) = reusable_surface.filter(|surface| { + surface["availability"] == "ready" + && surface["store_revision"].as_u64() == Some(generation.store_revision) + }) { + return Ok(surface.clone()); + } if generation.materials.is_empty() && generation.active_memory_count > 0 { - self.record_failure(&generation.generation_id, "input_budget_exhausted") + return self + .record_failure(&generation, "input_budget_exhausted", context.is_none()) .await; - tracing::warn!( - active_memory_count = generation.active_memory_count, - "subject Memory surface materials could not fit the bounded input" - ); - return Ok(()); } let points = if generation.materials.is_empty() { Vec::new() } else { - match self.edit_surface(&generation, cancellation.clone()).await { - Ok(points) => points, - Err(failure) => { - self.record_failure(&generation.generation_id, failure.reason_code) - .await; + match tokio::time::timeout( + TASK_TIMEOUT, + self.edit_surface(&generation, cancellation.clone()), + ) + .await + { + Ok(Ok(points)) => points, + Ok(Err(failure)) => { + if cancellation.is_cancelled() { + return Err(HookError::new( + HookErrorCategory::Cancelled, + "surface generation cancelled", + )); + } tracing::warn!(error = %failure.error, reason_code = failure.reason_code, "subject Memory surface editor failed"); - return Ok(()); + return self + .record_failure(&generation, failure.reason_code, context.is_none()) + .await; + } + Err(_) => { + return self + .record_failure(&generation, "editor_timeout", context.is_none()) + .await; } } }; - context.generation_fence.ensure_current()?; - if cancellation.is_cancelled() { - return Ok(()); + if let Some(context) = context { + context.generation_fence.ensure_current()?; } - let publish = self + if cancellation.is_cancelled() { + return Err(HookError::new( + HookErrorCategory::Cancelled, + "surface generation cancelled", + )); + } + match self .workspace_client .publish_subjektiv_memory_surface(server_api::SubjektivSurfacePublishRequest { generation_id: generation.generation_id.clone(), points, }) - .await; - match publish { + .await + { Ok(output) => { - tracing::debug!( - snapshot_id = output.snapshot_id, - store_revision = output.built_from_store_revision, - empty = output.empty, - "published subject Memory surface" - ); - return Ok(()); + return Ok(serde_json::json!({ + "availability": "ready", "generation_id": generation.generation_id, + "store_revision": output.built_from_store_revision, "snapshot_id": output.snapshot_id, + })); } Err(WorkspaceMemoryBackendError::Http { status, .. }) if status == reqwest::StatusCode::CONFLICT && attempt + 1 < MAX_GENERATION_ATTEMPTS => { - // Confirmed Memory moved while editing. Discard the stale - // output and rebuild once from a fresh bounded generation. continue; } Err(error) => { - self.record_failure(&generation.generation_id, "publish_failed") - .await; tracing::warn!(%error, "subject Memory surface publication failed"); - return Ok(()); + return self + .record_failure(&generation, "publish_failed", context.is_none()) + .await; } } } - Ok(()) + unreachable!("bounded generation loop always returns on its last attempt") } -} -impl SubjektivSurfaceLifecycleTask { async fn edit_surface( &self, generation: &server_api::SubjektivSurfacePrepareResponse, @@ -344,17 +417,39 @@ impl SubjektivSurfaceLifecycleTask { .map_err(SurfaceEditorFailure::editor) } - async fn record_failure(&self, generation_id: &str, reason_code: &str) { - if let Err(error) = self + async fn record_failure( + &self, + generation: &server_api::SubjektivSurfacePrepareResponse, + reason_code: &str, + job_confirmation_required: bool, + ) -> Result { + let failure = self .workspace_client .fail_subjektiv_memory_surface(server_api::SubjektivSurfaceFailureRequest { - generation_id: generation_id.to_string(), + generation_id: generation.generation_id.clone(), reason_code: reason_code.to_string(), }) .await + .map_err(surface_hook_error)?; + // Success must attest the actual persisted marker, not merely receipt of + // a failure notification. A protected ready surface or moved revision + // must yield a Backend rejection/non-failed status, never a failed claim. + let confirmed_status = if job_confirmation_required { + "failed_confirmed" + } else { + "failed" + }; + if failure.status != confirmed_status || failure.store_revision != generation.store_revision { - tracing::debug!(%error, "could not record subject Memory surface failure"); + return Err(HookError::new( + HookErrorCategory::Internal, + "Backend did not confirm a failed surface for this generation's revision; no Job result was submitted", + )); } + Ok(serde_json::json!({ + "availability": "failed", "generation_id": generation.generation_id, + "store_revision": failure.store_revision, "reason_code": reason_code, + })) } } @@ -537,6 +632,7 @@ mod tests { ) -> server_api::SubjektivSurfacePrepareResponse { server_api::SubjektivSurfacePrepareResponse { generation_id: "generation-1".into(), + current_snapshot_id: None, store_revision: 1, active_memory_count: materials.len(), materials, @@ -561,6 +657,13 @@ mod tests { prepare_calls: AtomicUsize, publish_calls: AtomicUsize, failure_reasons: Mutex>, + job_results: Mutex>, + result_responses: + Mutex>>, + current_revision: AtomicUsize, + failure_status: Mutex, + failure_revision: AtomicUsize, + ready_snapshot: Mutex>, } impl SurfaceWorkspaceClient { @@ -571,6 +674,12 @@ mod tests { prepare_calls: AtomicUsize::new(0), publish_calls: AtomicUsize::new(0), failure_reasons: Mutex::new(Vec::new()), + job_results: Mutex::new(Vec::new()), + result_responses: Mutex::new(Default::default()), + current_revision: AtomicUsize::new(0), + failure_status: Mutex::new("failed".into()), + failure_revision: AtomicUsize::new(0), + ready_snapshot: Mutex::new(None), } } @@ -581,6 +690,15 @@ mod tests { } } + fn revision(&self, fallback: usize) -> u64 { + let current = self.current_revision.load(Ordering::SeqCst); + if current == 0 { + fallback as u64 + } else { + current as u64 + } + } + fn response(value: server_api::SubjektivMemoryBackendResponse) -> WorkspaceResponse { WorkspaceResponse { status: 200, @@ -606,6 +724,28 @@ mod tests { &self, request: WorkspaceRequest, ) -> Result { + if request.path.ends_with("/workers/self/backend-job-result") { + assert!( + self.publish_calls.load(Ordering::SeqCst) > 0 + || !self.failure_reasons.lock().unwrap().is_empty(), + "result must follow publication or recorded failure" + ); + self.job_results + .lock() + .unwrap() + .push(serde_json::from_str(request.body.as_deref().unwrap()).unwrap()); + return self + .result_responses + .lock() + .unwrap() + .pop_front() + .unwrap_or_else(|| { + Ok(WorkspaceResponse { + status: 200, + body: "{\"replayed\":false}".into(), + }) + }); + } let request: server_api::SubjektivMemoryBackendRequest = serde_json::from_str(request.body.as_deref().unwrap_or_default()) .map_err(|error| WorkspaceClientError::Request(error.to_string()))?; @@ -615,7 +755,8 @@ mod tests { let mut prepared = generation(self.input_token_budget, vec![material("memory-1", 32)]); prepared.generation_id = format!("generation-{call}"); - prepared.store_revision = call as u64; + prepared.store_revision = self.revision(call); + prepared.current_snapshot_id = self.ready_snapshot.lock().unwrap().clone(); Ok(Self::response( server_api::SubjektivMemoryBackendResponse::SurfacePrepared(prepared), )) @@ -633,7 +774,7 @@ mod tests { server_api::SubjektivMemoryBackendResponse::SurfacePublished( server_api::SubjektivSurfacePublishResponse { snapshot_id: format!("surface-{call}"), - built_from_store_revision: call as u64, + built_from_store_revision: self.revision(call), empty: false, }, ), @@ -644,8 +785,13 @@ mod tests { Ok(Self::response( server_api::SubjektivMemoryBackendResponse::SurfaceFailed( server_api::SubjektivSurfaceFailureResponse { - store_revision: self.prepare_calls.load(Ordering::SeqCst) as u64, - status: "failed".into(), + store_revision: if self.failure_revision.load(Ordering::SeqCst) > 0 + { + self.failure_revision.load(Ordering::SeqCst) as u64 + } else { + self.revision(self.prepare_calls.load(Ordering::SeqCst)) + }, + status: self.failure_status.lock().unwrap().clone(), }, ), )) @@ -974,4 +1120,340 @@ permission = "write" let error = bounded_editor_input("system policy", &generation).unwrap_err(); assert!(error.to_string().contains("input budget")); } + #[tokio::test] + async fn job_result_waits_for_surface_and_retries_the_exact_outcome() { + for failed in [false, true] { + let workspace = Arc::new(if failed { + SurfaceWorkspaceClient::with_input_token_budget(1) + } else { + SurfaceWorkspaceClient::new(0) + }); + *workspace.failure_status.lock().unwrap() = "failed_confirmed".into(); + let lifecycle = SubjektivSurfaceLifecycleFeature { + task: test_task(workspace.clone(), Arc::new(AtomicUsize::new(0))), + }; + let mut pending = Vec::new(); + let mut hooks = crate::hook::HookRegistryBuilder::default(); + FeatureRegistryBuilder::new() + .with_module( + crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( + workspace.clone(), + ) + .with_surface(lifecycle), + ) + .install_into_pending(&mut pending, &mut hooks); + let (_, tool) = pending.remove(0)(); + let input = r#"{"result":{"subject_id":"subject-1","candidate_ids":["candidate-1"]}}"#; + for _ in 0..2 { + tool.execute(input, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + } + let results = workspace.job_results.lock().unwrap(); + assert_eq!(results.len(), 2); + assert_eq!(results[0], results[1]); + assert_eq!( + results[0]["result"]["surface"]["availability"], + if failed { "failed" } else { "ready" } + ); + assert_eq!( + results[0]["result"]["surface"]["generation_id"], + "generation-1" + ); + assert_eq!(results[0]["result"]["surface"]["store_revision"], 1); + assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 1); + assert!( + tool.execute( + r#"{"result":{"subject_id":"subject-1","candidate_ids":[]}}"#, + agen::tool::ToolExecutionContext::direct() + ) + .await + .is_err() + ); + } + } + + #[tokio::test] + async fn job_uses_backend_confirmed_current_surface_without_regeneration() { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + *workspace.ready_snapshot.lock().unwrap() = Some("already-current".into()); + let calls = Arc::new(AtomicUsize::new(0)); + let task = test_task(workspace.clone(), calls.clone()); + let outcome = task + .generate(None, BackgroundTaskCancellation::default(), None) + .await + .unwrap(); + assert_eq!(outcome["availability"], "ready"); + assert_eq!(outcome["generation_id"], "generation-1"); + assert_eq!(outcome["snapshot_id"], "already-current"); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 0); + assert!(workspace.failure_reasons.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn job_result_rejects_model_surface_claims_before_generation() { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + let lifecycle = SubjektivSurfaceLifecycleFeature { + task: test_task(workspace.clone(), Arc::new(AtomicUsize::new(0))), + }; + let mut pending = Vec::new(); + let mut hooks = crate::hook::HookRegistryBuilder::default(); + FeatureRegistryBuilder::new() + .with_module( + crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( + workspace.clone(), + ) + .with_surface(lifecycle), + ) + .install_into_pending(&mut pending, &mut hooks); + let (_, tool) = pending.remove(0)(); + assert!(tool.execute(r#"{"result":{"subject_id":"subject-1","candidate_ids":[],"surface":{"availability":"ready"}}}"#, agen::tool::ToolExecutionContext::direct()).await.is_err()); + assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 0); + assert!(workspace.job_results.lock().unwrap().is_empty()); + } + + #[derive(Clone)] + struct PendingSurfaceEditorClient(Arc); + #[async_trait] + impl LlmClient for PendingSurfaceEditorClient { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + async fn stream( + &self, + _request: Request, + ) -> Result> + Send>>, ClientError> + { + self.0.notify_one(); + Ok(Box::pin(futures::stream::pending())) + } + } + + #[tokio::test] + async fn cancelling_job_surface_waits_for_editor_and_never_submits_result() { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + let entered = Arc::new(tokio::sync::Notify::new()); + let mut task = test_task(workspace.clone(), Arc::new(AtomicUsize::new(0))); + task.client = Box::new(PendingSurfaceEditorClient(entered.clone())); + let mut pending = Vec::new(); + let mut hooks = crate::hook::HookRegistryBuilder::default(); + FeatureRegistryBuilder::new() + .with_module( + crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( + workspace.clone(), + ) + .with_surface(SubjektivSurfaceLifecycleFeature { task }), + ) + .install_into_pending(&mut pending, &mut hooks); + let (_, tool) = pending.remove(0)(); + let running_tool = tool.clone(); + let context = agen::tool::ToolExecutionContext::new("result-call", "batch", 0); + let running_context = context.clone(); + let running = tokio::spawn(async move { + running_tool + .execute( + r#"{"result":{"subject_id":"subject-1","candidate_ids":[]}}"#, + running_context, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), entered.notified()) + .await + .unwrap(); + tool.cancel_execution(&context).await.unwrap(); + let outcome = tokio::time::timeout(Duration::from_secs(5), running) + .await + .unwrap() + .unwrap(); + assert!(matches!(outcome, Err(agen::tool::ToolError::Cancelled(_)))); + assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 0); + assert!(workspace.job_results.lock().unwrap().is_empty()); + assert!(workspace.failure_reasons.lock().unwrap().is_empty()); + } + fn job_result_tool(workspace: Arc) -> Arc { + *workspace.failure_status.lock().unwrap() = "failed_confirmed".into(); + let mut pending = Vec::new(); + let mut hooks = crate::hook::HookRegistryBuilder::default(); + FeatureRegistryBuilder::new() + .with_module( + crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( + workspace.clone(), + ) + .with_surface(SubjektivSurfaceLifecycleFeature { + task: test_task(workspace, Arc::new(AtomicUsize::new(0))), + }), + ) + .install_into_pending(&mut pending, &mut hooks); + pending.remove(0)().1 + } + + const CORRECT_RESULT: &str = + r#"{"result":{"subject_id":"subject-1","candidate_ids":["candidate-1"]}}"#; + const WRONG_RESULT: &str = r#"{"result":{"subject_id":"wrong-subject","candidate_ids":[]}}"#; + + fn rejection(status: u16) -> Result { + Ok(WorkspaceResponse { + status, + body: "explicit rejection before acceptance".into(), + }) + } + + #[tokio::test] + async fn job_result_definitive_rejection_allows_correction_and_reuses_current_ready_surface() { + for status in [400, 403, 422] { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + workspace.current_revision.store(1, Ordering::SeqCst); + workspace + .result_responses + .lock() + .unwrap() + .push_back(rejection(status)); + let tool = job_result_tool(workspace.clone()); + assert!( + tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + let results = workspace.job_results.lock().unwrap(); + assert_eq!(results.len(), 2); + assert_ne!( + results[0]["result"]["subject_id"], + results[1]["result"]["subject_id"] + ); + assert_eq!( + results[0]["result"]["surface"], + results[1]["result"]["surface"] + ); + assert_eq!( + workspace.prepare_calls.load(Ordering::SeqCst), + 2, + "reuse must verify current revision" + ); + assert_eq!( + workspace.publish_calls.load(Ordering::SeqCst), + 1, + "correction must not duplicate publication" + ); + drop(results); + assert!( + tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err(), + "accepted results stay sealed" + ); + } + } + + #[tokio::test] + async fn job_result_rejection_rebuilds_conflicting_or_stale_surfaces() { + for status in [400, 409] { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + workspace.current_revision.store(1, Ordering::SeqCst); + workspace + .result_responses + .lock() + .unwrap() + .push_back(rejection(status)); + let tool = job_result_tool(workspace.clone()); + assert!( + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + workspace.current_revision.store(2, Ordering::SeqCst); + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + let results = workspace.job_results.lock().unwrap(); + assert_eq!(results.len(), 2); + assert_eq!(results[0]["result"]["surface"]["store_revision"], 1); + assert_eq!(results[1]["result"]["surface"]["store_revision"], 2); + assert_ne!( + results[0]["result"]["surface"]["generation_id"], + results[1]["result"]["surface"]["generation_id"] + ); + assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 2); + assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 2); + } + } + + #[tokio::test] + async fn job_result_ambiguous_acceptance_never_reopens_even_after_later_rejection() { + for first in [ + Err(WorkspaceClientError::Request( + "accepted but response lost".into(), + )), + rejection(500), + ] { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + workspace + .result_responses + .lock() + .unwrap() + .extend([first, rejection(403)]); + let tool = job_result_tool(workspace.clone()); + assert!( + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + assert!( + tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + assert!( + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + assert!( + tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + let results = workspace.job_results.lock().unwrap(); + assert_eq!(results.len(), 3); + assert_eq!(results[0], results[1]); + assert_eq!(results[1], results[2]); + assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 1); + assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 1); + } + } + + #[tokio::test] + async fn job_result_never_submits_unconfirmed_failure_or_wrong_revision() { + for (status, revision) in [ + ("failed", 0), + ("ready", 0), + ("stale", 0), + ("failed_confirmed", 2), + ] { + let workspace = Arc::new(SurfaceWorkspaceClient::with_input_token_budget(1)); + workspace.current_revision.store(1, Ordering::SeqCst); + let tool = job_result_tool(workspace.clone()); + *workspace.failure_status.lock().unwrap() = status.into(); + workspace.failure_revision.store(revision, Ordering::SeqCst); + assert!( + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + assert!(workspace.job_results.lock().unwrap().is_empty()); + // Nothing was dispatched/sealed: a real confirmed outcome remains retryable. + *workspace.failure_status.lock().unwrap() = "failed_confirmed".into(); + workspace.failure_revision.store(0, Ordering::SeqCst); + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + assert_eq!(workspace.job_results.lock().unwrap().len(), 1); + } + } } diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index ae5baddae..1dbc8a526 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -1,3 +1,5 @@ +mod backend_job; +pub use backend_job::BackendJobExecutionBinding; pub mod bootstrap; pub mod checkout; pub mod compact; diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 417886ae1..e3dbde07d 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -2599,6 +2599,8 @@ struct PendingCompactionCleanup { /// `session-store` functions after each turn. pub struct Worker { manifest: WorkerManifest, + /// Host-bound once before Feature installation; never inherited by child Workers. + backend_job: std::sync::OnceLock>, /// Always `Some` outside of `run()`/`resume()`. engine: Option>, /// Sole live authority for committed model-visible history. @@ -2965,6 +2967,7 @@ impl Worker { let scope = SharedScope::new(scope); let workdir_sessions = workdir_sessions_from_authority(&filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), session: WorkerSession::new(session_id, Vec::new()), @@ -3128,6 +3131,30 @@ impl Worker { self.segment_state.session_id() } + /// Immutable host-owned Job authority, absent for ordinary and child Workers. + pub fn backend_job(&self) -> Option<&crate::BackendJobExecutionBinding> { + self.backend_job.get().and_then(Option::as_ref) + } + + /// Trusted host seam; immutable after binding and unavailable after Feature installation. + pub fn bind_backend_job( + &mut self, + binding: crate::BackendJobExecutionBinding, + ) -> Result<(), &'static str> { + binding.validate()?; + if self.interceptor_installed || self.backend_job.get().is_some() { + return Err("Backend Job capability is already sealed"); + } + if !self.workspace_client().is_available() + || self.workspace_client().workspace_id().is_none() + { + return Err("Backend Job capability requires Backend Workspace API authority"); + } + self.backend_job + .set(Some(binding)) + .map_err(|_| "Backend Job capability is already bound") + } + /// The Worker's manifest. pub fn manifest(&self) -> &WorkerManifest { &self.manifest @@ -3398,6 +3425,7 @@ impl Worker { &mut self, registry: FeatureRegistryBuilder, ) -> FeatureRegistryInstallReport { + self.backend_job.get_or_init(|| None); let worker = self.engine.as_mut().expect("worker taken during run"); let report = registry.install_into_engine(worker, &mut self.hook_builder); if report.has_errors() { @@ -7249,6 +7277,7 @@ where workdir_sessions_from_authority(&common.filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), session: WorkerSession::new(session_id, Vec::new()), @@ -7350,6 +7379,18 @@ where }); } + if self + .backend_job() + .is_some_and(|binding| binding.subjektiv_consolidation) + { + if metadata.subjektiv_session_attribution.is_some() { + return Err(WorkerError::SubjektivSessionAttribution { + state: "job_attribution_forbidden", + message: "consolidation Job execution must not carry subject-body Session attribution".into(), + }); + } + return Ok(()); + } if !subjektiv_attached { if metadata.subjektiv_session_attribution.is_some() { return Err(WorkerError::SubjektivSessionAttribution { @@ -7528,6 +7569,7 @@ where let workdir_sessions = workdir_sessions_from_authority(&common.filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(engine), session: WorkerSession::new(session_id, Vec::new()), @@ -7655,6 +7697,7 @@ where workdir_sessions_from_authority(&common.filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), session: WorkerSession::new(session_id, Vec::new()), @@ -8036,6 +8079,7 @@ where workdir_sessions_from_authority(&common.filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), session: WorkerSession::new(session_id, restored_history_entries), @@ -10485,6 +10529,56 @@ mod build_summary_prompt_tests { (worker, store, temp) } + #[tokio::test] + async fn ordinary_worker_cannot_gain_job_authority_after_feature_installation() { + let client = Arc::new(AttributionLifecycleClient::new([])); + let (mut worker, _store, _temp) = attribution_lifecycle_worker(client).await; + worker.install_features(FeatureRegistryBuilder::new()); + assert!( + worker + .bind_backend_job(crate::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: None, + subjektiv_consolidation: false + }) + .is_err() + ); + assert!(worker.backend_job().is_none()); + } + + #[tokio::test] + async fn consolidation_job_binding_is_immutable_and_has_no_subject_body_attribution() { + let client = Arc::new(AttributionLifecycleClient::new([])); + let (mut worker, store, _temp) = attribution_lifecycle_worker(client.clone()).await; + let binding = crate::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: Some("revision-1".into()), + subjektiv_consolidation: true, + }; + worker.bind_backend_job(binding.clone()).unwrap(); + assert_eq!(worker.backend_job(), Some(&binding)); + assert!(worker.bind_backend_job(binding).is_err()); + for lifecycle in [ + SubjektivSessionAttributionLifecycle::NewSession, + SubjektivSessionAttributionLifecycle::RestoredSession, + ] { + worker + .finalize_subjektiv_session_attribution(true, lifecycle) + .unwrap(); + } + assert!(client.requests.lock().unwrap().is_empty()); + assert!( + store + .read_by_name(&worker.manifest().worker.name) + .unwrap() + .unwrap() + .subjektiv_session_attribution + .is_none() + ); + } + #[tokio::test] async fn session_attribution_finalizes_once_before_runs_and_persists_subject() { let client = Arc::new(AttributionLifecycleClient::new([ diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index c1b4668e4..f79ab3fdf 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -838,36 +838,126 @@ async fn feature_flags_default_to_core_tool_surface_only() { } #[tokio::test] -async fn backend_job_profile_exposes_only_structured_result_capability() { +async fn backend_job_capability_is_explicit_and_preserves_profile_features() { + for bound in [false, true] { + let workspace = tempfile::tempdir().unwrap(); + let mut resolved = ProfileResolver::new() + .with_workspace_base(workspace.path()) + .resolve( + &ProfileSelector::source_named(ProfileRegistrySource::Builtin, "backend-job"), + ProfileResolveOptions::with_worker_name("backend-job-worker"), + ) + .unwrap(); + // A custom instruction and enabled feature must survive trusted Job binding. + if bound { + resolved.manifest.engine.instruction = "default".into(); + resolved.manifest.feature.task.enabled = true; + } + let instruction = resolved.manifest.engine.instruction.clone(); + let client = MockClient::new(simple_text_events()); + let client_for_assert = client.clone(); + let (mut worker, _pwd) = make_worker_with_manifest_and_workspace_context( + client, + resolved.manifest, + WorkerWorkspaceContext::with_client(None, Arc::new(AvailableWorkspaceClient)), + ) + .await; + if bound { + worker + .bind_backend_job(worker::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: Some("input-1".into()), + subjektiv_consolidation: false, + }) + .unwrap(); + } + assert_eq!(worker.manifest().engine.instruction, instruction); + let handle = spawn_controller(worker).await; + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Run the bounded Backend Job.", + )) + .await + .unwrap(); + wait_for_status(&handle, WorkerStatus::Idle).await; + let request = wait_for_captured_request(&client_for_assert).await; + let names = request_tool_names(&request); + assert_eq!( + names.iter().any(|name| name == "SubmitBackendJobResult"), + bound + ); + assert_eq!(names.iter().any(|name| name == "TaskCreate"), bound); + } +} + +#[tokio::test] +async fn consolidation_job_preserves_custom_instruction_and_explicit_tool_policy() { let workspace = tempfile::tempdir().unwrap(); - let resolved = ProfileResolver::new() + let mut manifest = ProfileResolver::new() .with_workspace_base(workspace.path()) .resolve( - &ProfileSelector::source_named(ProfileRegistrySource::Builtin, "backend-job"), - ProfileResolveOptions::with_worker_name("backend-job-worker"), + &ProfileSelector::source_named( + ProfileRegistrySource::Builtin, + "subjektiv-memory-consolidation", + ), + ProfileResolveOptions::with_worker_name("custom-consolidation-job"), ) + .unwrap() + .manifest; + manifest.engine.instruction = "default".into(); + manifest.feature.task.enabled = true; + manifest + .feature + .subjektiv + .bind_workspace_settings(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: "workspace-1".into(), + settings_revision: 1, + language: "English".into(), + }) .unwrap(); let client = MockClient::new(simple_text_events()); - let client_for_assert = client.clone(); - let (worker, _pwd) = make_worker_with_manifest_and_workspace_context( + let captured = client.clone(); + let (mut worker, _pwd) = make_worker_with_manifest_and_workspace_context( client, - resolved.manifest, + manifest, WorkerWorkspaceContext::with_client(None, Arc::new(AvailableWorkspaceClient)), ) .await; + worker + .engine_mut() + .set_system_prompt("chosen custom instruction"); + worker + .bind_backend_job(worker::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: None, + subjektiv_consolidation: true, + }) + .unwrap(); let handle = spawn_controller(worker).await; - handle .send(Method::submit_text( protocol::new_submission_request_id(), - "Run the bounded Backend Job.", + "Inspect the bounded batch.", )) .await .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; - - let request = wait_for_captured_request(&client_for_assert).await; - assert_eq!(request_tool_names(&request), vec!["SubmitBackendJobResult"]); + let request = wait_for_captured_request(&captured).await; + assert_eq!( + request.system_prompt.as_deref(), + Some("chosen custom instruction") + ); + let tools = request_tool_names(&request); + for tool in [ + "SubmitBackendJobResult", + "MemoryApplyCandidate", + "TaskCreate", + ] { + assert!(tools.iter().any(|name| name == tool)); + } } #[tokio::test] diff --git a/crates/workspace-server/src/backend_job.rs b/crates/workspace-server/src/backend_job.rs index 08effd749..f5b54d7a4 100644 --- a/crates/workspace-server/src/backend_job.rs +++ b/crates/workspace-server/src/backend_job.rs @@ -199,6 +199,34 @@ const fn default_max_attempts() -> u8 { DEFAULT_MAX_ATTEMPTS } +/// Explicit Backend-issued domain capabilities, never inferred from purpose or profile. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct BackendJobGrants { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subjektiv_consolidation: Option, +} + +impl BackendJobGrants { + pub fn is_empty(&self) -> bool { + self.subjektiv_consolidation.is_none() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SubjektivConsolidationGrant { + pub subject_id: String, + /// Exact immutable candidate scope. Empty is a surface-only refresh grant: + /// the subject is still locked, but no candidate access is authorized. + pub candidate_ids: Vec, +} + +/// Common runner lock identity for a subject, independent of the candidate snapshot. +pub fn subjektiv_serialization_key(subject_id: &str) -> String { + format!("subjektiv-consolidation:{subject_id}") +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct BackendJobRequest { @@ -216,6 +244,13 @@ pub struct BackendJobRequest { pub instruction: String, /// Existing configured profile selector. Job dispatch never accepts raw profile source. pub profile: String, + /// Immutable domain grants; profile resolution alone never grants access. + #[serde(default, skip_serializing_if = "BackendJobGrants::is_empty")] + pub grants: BackendJobGrants, + /// Optional common-runner single-flight resource, not a separate domain state machine. + /// Consolidation grants always derive their subject lock; an explicit key must match it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub serialization_key: Option, /// Writer provenance and optional advisory destination, not parent ownership. #[serde(default, skip_serializing_if = "Option::is_none")] pub source_worker: Option, @@ -233,10 +268,34 @@ impl BackendJobRequest { validate_bounded_text("input_ref", &self.input_ref, MAX_JOB_REFERENCE_BYTES)?; validate_bounded_text("instruction", &self.instruction, MAX_JOB_INSTRUCTION_BYTES)?; validate_identifier("profile", &self.profile, 256)?; - if self.profile != "builtin:backend-job" { - return Err(Error::InvalidInput( - "Backend Jobs must use the dedicated builtin:backend-job profile".to_string(), - )); + self.profile_selector()?; + if let Some(key) = &self.serialization_key { + validate_identifier("serialization_key", key, MAX_JOB_REFERENCE_BYTES)?; + } + if let Some(grant) = &self.grants.subjektiv_consolidation { + validate_identifier("grant subject_id", &grant.subject_id, 256)?; + if grant.candidate_ids.len() > 256 { + return Err(Error::InvalidInput( + "Backend Job consolidation grant requires 0..=256 candidate ids".into(), + )); + } + let mut seen = std::collections::HashSet::new(); + for id in &grant.candidate_ids { + validate_identifier("grant candidate_id", id, 256)?; + if !seen.insert(id) { + return Err(Error::InvalidInput( + "Backend Job consolidation grant has duplicate candidate ids".into(), + )); + } + } + if let Some(key) = &self.serialization_key + && *key != subjektiv_serialization_key(&grant.subject_id) + { + return Err(Error::InvalidInput( + "Backend Job serialization_key must match the consolidation subject lock" + .into(), + )); + } } let input = serde_json::to_vec(&self.input).map_err(|error| { Error::InvalidInput(format!("serialize Backend Job input: {error}")) @@ -249,6 +308,44 @@ impl BackendJobRequest { self.limits.validate() } + /// Validate selector syntax only. Registry existence and resolution are dispatch authority. + pub fn profile_selector(&self) -> Result { + use manifest::ProfileSelector; + validate_identifier("profile", &self.profile, 256)?; + let selector = ProfileSelector::parse_cli(&self.profile); + let valid = match &selector { + ProfileSelector::Default => true, + ProfileSelector::Path { .. } => false, + ProfileSelector::Named { source, name } => { + !name.is_empty() + && !name.starts_with('.') + && !name.contains(['/', '\\', ':']) + && name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + && ![".toml", ".json", ".dcdl", ".nix"] + .iter() + .any(|suffix| name.ends_with(suffix)) + && (source.is_some() || name != "inherit") + } + }; + if !valid { + return Err(Error::InvalidInput( + "Backend Job profile must be a manifest registry selector, not raw source or a path".into(), + )); + } + Ok(selector) + } + + /// Effective single-flight key. Typed grants cannot bypass their subject fence. + pub fn resource_key(&self) -> Option { + self.grants + .subjektiv_consolidation + .as_ref() + .map(|grant| subjektiv_serialization_key(&grant.subject_id)) + .or_else(|| self.serialization_key.clone()) + } + pub fn fingerprint(&self) -> Result { self.validate()?; let encoded = serde_json::to_vec(self).map_err(|error| { @@ -346,7 +443,12 @@ pub struct BackendJobAttemptRecord { pub struct BackendJobReservation { pub job: BackendJobRecord, pub attempt: BackendJobAttemptRecord, + /// Exact replay of the caller-selected immutable intent. pub replayed: bool, + /// A different intent already fences this resource. The returned request is + /// the original snapshot; the caller must defer its new input to a followup. + #[serde(default)] + pub resource_reused: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -486,6 +588,8 @@ mod tests { input: serde_json::json!({"title": "Check me"}), instruction: "Check this immutable Ticket snapshot.".to_string(), profile: "builtin:backend-job".to_string(), + grants: BackendJobGrants::default(), + serialization_key: None, source_worker: Some(RuntimeWorkerRef::new("runtime-a", "worker-a")), notification_target: Some(RuntimeWorkerRef::new("runtime-a", "worker-a")), limits: BackendJobLimits::default(), @@ -509,6 +613,133 @@ mod tests { } } + #[test] + fn registry_profile_syntax_accepts_configured_selectors_without_an_allowlist() { + for profile in [ + "builtin:backend-job", + "builtin:subjektiv-memory-consolidation", + "project:custom", + "user:custom", + "custom", + "default", + "builtin:not-yet-configured", + ] { + let mut request = request(); + request.profile = profile.into(); + request.validate().unwrap(); + } + for profile in [ + "", + " builtin:backend-job", + "path:custom", + "./custom.toml", + "/tmp/custom", + "project:../custom", + "project:custom.toml", + "builtin:", + "unknown:custom", + "project:a:b", + "C:\\custom", + "{\"worker\":{}}", + "inherit", + ] { + let mut request = request(); + request.profile = profile.into(); + assert!(request.validate().is_err(), "accepted {profile}"); + } + } + + #[test] + fn explicit_grants_are_immutable_bounded_and_derive_a_subject_fence() { + let mut request = request(); + let original = request.fingerprint().unwrap(); + request.grants.subjektiv_consolidation = Some(SubjektivConsolidationGrant { + subject_id: "subject-a".into(), + candidate_ids: vec!["candidate-1".into()], + }); + assert_ne!(original, request.fingerprint().unwrap()); + assert_eq!( + request.resource_key(), + Some(subjektiv_serialization_key("subject-a")) + ); + let granted = request.fingerprint().unwrap(); + request + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids + .push("candidate-2".into()); + assert_ne!(granted, request.fingerprint().unwrap()); + request.serialization_key = Some("another-subject".into()); + assert!(request.validate().is_err()); + request.serialization_key = None; + request + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids + .push("candidate-2".into()); + assert!(request.validate().is_err()); + } + + #[test] + fn empty_consolidation_grant_keeps_subject_lock_without_candidate_authority() { + let mut request = request(); + request.grants.subjektiv_consolidation = Some(SubjektivConsolidationGrant { + subject_id: "subject-a".into(), + candidate_ids: vec![], + }); + request.validate().unwrap(); + assert!( + !request.grants.is_empty(), + "empty candidate scope still carries a subject grant" + ); + assert_eq!( + request.resource_key(), + Some(subjektiv_serialization_key("subject-a")) + ); + let encoded = serde_json::to_string(&request).unwrap(); + assert!(encoded.contains("\"candidate_ids\":[]")); + let decoded: BackendJobRequest = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded, request); + let surface_only = request.fingerprint().unwrap(); + request + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids = (0..256).map(|index| format!("candidate-{index}")).collect(); + request.validate().unwrap(); + assert_ne!(surface_only, request.fingerprint().unwrap()); + request + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids + .push("candidate-256".into()); + assert!( + request.validate().is_err(), + "candidate batch remains bounded" + ); + } + + #[test] + fn legacy_request_replay_keeps_its_exact_serialization() { + let request = request(); + let original_json = serde_json::to_string(&request).unwrap(); + assert!(!original_json.contains("grants")); + assert!(!original_json.contains("serialization_key")); + let decoded: BackendJobRequest = serde_json::from_str(&original_json).unwrap(); + assert_eq!(decoded.grants, BackendJobGrants::default()); + assert_eq!( + request.fingerprint().unwrap(), + decoded.fingerprint().unwrap() + ); + } + #[test] fn tracked_runtime_ids_are_fixed_size_for_maximum_job_ids() { let job_id = "j".repeat(256); diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index c03a04f9b..a91b2b160 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -564,6 +564,8 @@ pub enum WorkerSpawnIntent { job_id: String, attempt_id: String, purpose: String, + input_revision: String, + subjektiv_consolidation: bool, }, TicketRole { ticket_id: String, @@ -5708,6 +5710,21 @@ fn runtime_create_worker_request( workspace_api: Some(workspace_api), memory_settings: request.resolved_memory_settings.clone(), subjektiv_attached: request.resolved_subjektiv_attached, + backend_job: match &request.intent { + WorkerSpawnIntent::BackendJob { + job_id, + attempt_id, + input_revision, + subjektiv_consolidation, + .. + } => Some(worker_runtime::catalog::BackendJobExecutionBinding { + job_id: job_id.clone(), + attempt_id: attempt_id.clone(), + input_revision: Some(input_revision.clone()), + subjektiv_consolidation: *subjektiv_consolidation, + }), + _ => None, + }, } } diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 78efa519b..64275a3ab 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -845,6 +845,7 @@ CREATE TABLE backend_jobs ( purpose TEXT NOT NULL, input_revision TEXT NOT NULL, input_ref TEXT NOT NULL, + resource_key TEXT, request_json TEXT NOT NULL, intent_fingerprint TEXT NOT NULL, state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown')), @@ -863,6 +864,8 @@ CREATE TABLE backend_jobs ( ); CREATE INDEX backend_jobs_active ON backend_jobs(workspace_id, state, updated_at); +CREATE INDEX backend_jobs_resource + ON backend_jobs(workspace_id, resource_key, created_at); CREATE TABLE backend_job_attempts ( workspace_id TEXT NOT NULL, job_id TEXT NOT NULL, diff --git a/crates/workspace-server/src/profile_settings.rs b/crates/workspace-server/src/profile_settings.rs index 0e350cc42..5c38a099b 100644 --- a/crates/workspace-server/src/profile_settings.rs +++ b/crates/workspace-server/src/profile_settings.rs @@ -251,6 +251,27 @@ pub fn selector_for_workspace_candidate( .then(|| worker_runtime::catalog::ProfileSelector::Named(profile.to_string())) } +/// Resolve a Host-owned launch selector against the complete existing registry. +/// Browser candidates remain a narrower UI policy; this does not widen that surface. +pub(crate) fn selector_for_registered_profile( + projection: &ProfileConfigProjection, + profile: &str, +) -> Option { + let catalog = builtin_profile_catalog_snapshot(); + let builtin = if profile.starts_with("builtin:") { + profile.to_string() + } else { + format!("builtin:{profile}") + }; + if catalog.entrypoints.contains_key(&builtin) { + return Some(worker_runtime::catalog::ProfileSelector::Builtin(builtin)); + } + projection + .entries + .contains_key(profile) + .then(|| worker_runtime::catalog::ProfileSelector::Named(profile.into())) +} + fn validate_prompt_projection_matches_state( workspace_id: &str, state: &WorkspaceConfigState, @@ -401,8 +422,12 @@ pub fn build_virtual_profile_config_bundle_with_prompt_projection( ) -> Result> { validate_prompt_projection_matches_state(workspace_id, state, prompt_projection)?; let prompt_catalog = prompt_projection.catalog().clone(); - let profile_selector = selector_for_builtin_candidate(selector) - .unwrap_or_else(|| worker_runtime::catalog::ProfileSelector::Named(selector.to_string())); + let profile_selector = + selector_for_registered_profile(projection, selector).ok_or_else(|| { + Error::InvalidInput(format!( + "Profile `{selector}` is not a registered launch selector" + )) + })?; let archive = match projection.entries.get(selector) { Some(entry) => Some(build_virtual_profile_archive( selector, diff --git a/crates/workspace-server/src/runtime_subscription_tests.rs b/crates/workspace-server/src/runtime_subscription_tests.rs index 1ed9375b0..c07ca6450 100644 --- a/crates/workspace-server/src/runtime_subscription_tests.rs +++ b/crates/workspace-server/src/runtime_subscription_tests.rs @@ -74,6 +74,7 @@ fn create_request(name: &str) -> CreateWorkerRequest { language: "English".to_string(), }), subjektiv_attached: false, + backend_job: None, } } diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 6589f6230..84fb046e1 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1,3 +1,5 @@ +mod backend_job_profile; +mod subjektiv_job; mod worker_operations; pub(crate) use worker_operations::{ WorkerOperationContext, WorkspaceWorker, WorkspaceWorkerMethodSender, @@ -3500,6 +3502,7 @@ impl WorkspaceApi { &self, request: &BackendJobRequest, ) -> Result { + self.resolve_backend_job_profile(request)?; let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); let reservation = self.store @@ -3548,6 +3551,19 @@ impl WorkspaceApi { return Ok(reservation); } let request = &reservation.job.request; + let (profile, config_bundle) = match self.resolve_backend_job_profile(request) { + Ok(resolved) => resolved, + Err(error) => { + self.finish_backend_job_attempt( + &request.job_id, + &reservation.attempt.attempt_id, + BackendJobAttemptState::Failed, + "profile_requirements", + &error.to_string(), + )?; + return Err(error); + } + }; let worker_input = request.worker_input(&reservation.attempt.attempt_id)?; let control_operation = WorkerControlOperation { operation_id: allocation_key(&reservation.attempt.attempt_id), @@ -3562,22 +3578,24 @@ impl WorkspaceApi { job_id: request.job_id.clone(), attempt_id: reservation.attempt.attempt_id.clone(), purpose: request.purpose.clone(), + input_revision: request.input_revision.clone(), + subjektiv_consolidation: request.grants.subjektiv_consolidation.is_some(), }, acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { expected_segments: 0, }, - profile: ProfileSelector::Builtin(request.profile.clone()), + profile, ticket_assignment: None, initial_submit: Vec::new(), workdir_attachment_requests: Vec::new(), resolved_workdir_attachment_requests: Vec::new(), resolved_workdir_attachments: Vec::new(), - resolved_config_bundle: None, + resolved_config_bundle: Some(config_bundle), resolved_worker_observation_enabled: false, resolved_worker_observation_grants: Vec::new(), resolved_workspace_api: None, resolved_memory_settings: None, - resolved_subjektiv_attached: false, + resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some(), resolved_control_operation: Some(control_operation), }, ) { @@ -3647,6 +3665,7 @@ impl WorkspaceApi { job, attempt, replayed: true, + resource_reused: false, }); } @@ -3738,6 +3757,7 @@ impl WorkspaceApi { job, attempt, replayed: reservation.replayed, + resource_reused: reservation.resource_reused, }) } @@ -3801,6 +3821,7 @@ impl WorkspaceApi { job, attempt: attempt.clone(), replayed: true, + resource_reused: false, }) { Ok(reservation) => { made_progress |= @@ -4187,6 +4208,7 @@ impl WorkspaceApi { job, attempt, replayed: true, + resource_reused: false, }) { tracing::warn!(%error, "Backend Job reserved-attempt recovery failed"); @@ -4292,6 +4314,7 @@ impl WorkspaceApi { submission: &BackendJobResultSubmission, ) -> Result { let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let mut verified_submission = submission.clone(); if let Some(job) = self .store .get_backend_job(&self.config.workspace_id, &submission.job_id)? @@ -4334,10 +4357,41 @@ impl WorkspaceApi { return Err(error); } } + if let Some(job) = self + .store + .get_backend_job(&self.config.workspace_id, &submission.job_id)? + && job.request.grants.subjektiv_consolidation.is_some() + { + let attempt = self + .store + .get_backend_job_attempt( + &self.config.workspace_id, + &submission.job_id, + &submission.attempt_id, + )? + .ok_or_else(|| Error::InvalidInput("unknown consolidation attempt".into()))?; + if attempt.worker.as_ref() != Some(source_worker) + || job.current_attempt != attempt.attempt + || attempt.input_revision != submission.input_revision + || !matches!( + attempt.state, + BackendJobAttemptState::Dispatched | BackendJobAttemptState::Completed + ) + { + return Err(Error::WorkspacePermissionDenied( + "consolidation result requires the exact bound attempt".into(), + )); + } + if attempt.state != BackendJobAttemptState::Completed { + subjektiv_job::validate_result(self, &job.request, &attempt, &submission.result)?; + } + verified_submission.result = + subjektiv_job::with_dispositions(self, &job.request, &submission.result)?; + } let acceptance = self.store.accept_backend_job_result( &self.config.workspace_id, source_worker, - submission, + &verified_submission, &accepted_at, )?; if let Some(target) = acceptance.job.request.notification_target.as_ref() { @@ -18175,6 +18229,13 @@ fn subjektiv_subject_scope( ) })?; let worker = RuntimeWorkerRef::new(&source.runtime_id, worker_id); + if let Some(grant) = subjektiv_job::active_grant(api, workspace_id, &worker)? { + return Ok(( + grant.subject_id, + worker, + SubjektivWorkerAuthority::Consolidation, + )); + } let lease = api .store .require_current_worker_singleton_owner(workspace_id, &worker)? @@ -20006,7 +20067,8 @@ async fn scoped_subjektiv_memory_backend( Json(request): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, _, authority) = subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + let (subject_id, source_worker, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; let store = open_subjektiv_store(&api)?; let response = match request.operation { server_api::SubjektivMemoryBackendOperation::ResidentSummary(_) => { @@ -20118,9 +20180,30 @@ async fn scoped_subjektiv_memory_backend( "candidate listing", )?; let limit = bounded_subjektiv_limit(input.limit, SUBJEKTIV_QUERY_DEFAULT_LIMIT)?; - let mut candidates = store - .pending_staging_candidates(&subject_id, limit.saturating_add(1)) - .map_err(subjektiv_store_error)?; + let mut candidates = if let Some(grant) = + subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)? + { + let mut pending = Vec::new(); + for id in &grant.candidate_ids { + if store + .staging_resolution(&subject_id, id) + .map_err(subjektiv_store_error)? + .is_none() + { + if let Some(candidate) = store + .staging_candidate(&subject_id, id) + .map_err(subjektiv_store_error)? + { + pending.push(candidate); + } + } + } + pending + } else { + store + .pending_staging_candidates(&subject_id, limit.saturating_add(1)) + .map_err(subjektiv_store_error)? + }; let has_more = candidates.len() > limit; candidates.truncate(limit); server_api::SubjektivMemoryBackendResponse::Candidates( @@ -20139,6 +20222,12 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "candidate read", )?; + subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?; let candidate = store .staging_candidate(&subject_id, &input.candidate_id) .map_err(subjektiv_store_error)? @@ -20167,6 +20256,12 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "candidate decision", )?; + subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?; let receipt = store .decide_candidate(&subject_id, subjektiv_candidate_decision(input)?) .map_err(subjektiv_store_error)?; @@ -20180,12 +20275,42 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "surface generation preparation", )?; + if let Some(grant) = + subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)? + { + for id in &grant.candidate_ids { + if store + .staging_resolution(&subject_id, id) + .map_err(subjektiv_store_error)? + .is_none() + { + return Err(Error::InvalidInput(format!("surface generation requires a durable disposition for batch candidate `{id}`")).into()); + } + } + } + let binding = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|p| p.job); let generation = store - .prepare_surface_generation(&subject_id) + .prepare_job_surface_generation( + &subject_id, + binding + .as_ref() + .map(|b| (b.job_id.as_str(), b.attempt_id.as_str())), + ) .map_err(subjektiv_store_error)?; - server_api::SubjektivMemoryBackendResponse::SurfacePrepared( - subjektiv_surface_prepare_response(generation), - ) + let resident = store + .resident_surface(&subject_id) + .map_err(subjektiv_store_error)?; + let mut response = subjektiv_surface_prepare_response(generation); + if resident.availability == crate::subjektiv::SurfaceAvailability::Ready { + response.current_snapshot_id = resident + .snapshot + .filter(|s| s.built_from_store_revision == response.store_revision) + .map(|s| s.id); + } + server_api::SubjektivMemoryBackendResponse::SurfacePrepared(response) } server_api::SubjektivMemoryBackendOperation::PublishSurface(input) => { require_subjektiv_worker_authority( @@ -20193,6 +20318,20 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "surface publication", )?; + if let Some(binding) = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|p| p.job) + { + store + .require_job_surface_generation( + &subject_id, + &input.generation_id, + &binding.job_id, + &binding.attempt_id, + ) + .map_err(subjektiv_store_error)?; + } let points = input .points .into_iter() @@ -20225,13 +20364,52 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "surface generation failure", )?; + if let Some(binding) = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|p| p.job) + { + store + .require_job_surface_generation( + &subject_id, + &input.generation_id, + &binding.job_id, + &binding.attempt_id, + ) + .map_err(subjektiv_store_error)?; + } let revision = store .fail_surface_generation(&subject_id, &input.generation_id, &input.reason_code) .map_err(subjektiv_store_error)?; + let status = if let Some(binding) = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|p| p.job) + { + store + .validate_job_surface_outcome( + &subject_id, + &input.generation_id, + revision, + "failed", + None, + Some(&input.reason_code), + &binding.job_id, + &binding.attempt_id, + ) + .map_err(|e| { + Error::RepositoryConflict(format!( + "surface failure was not recorded for this attempt: {e}" + )) + })?; + "failed_confirmed" + } else { + "failed" + }; server_api::SubjektivMemoryBackendResponse::SurfaceFailed( server_api::SubjektivSurfaceFailureResponse { store_revision: revision, - status: "failed".into(), + status: status.into(), }, ) } @@ -20244,6 +20422,7 @@ fn subjektiv_surface_prepare_response( ) -> server_api::SubjektivSurfacePrepareResponse { server_api::SubjektivSurfacePrepareResponse { generation_id: generation.id, + current_snapshot_id: None, store_revision: generation.store_revision, active_memory_count: generation.active_memory_count, materials: generation @@ -21448,11 +21627,9 @@ async fn scoped_subjektiv_memory_consolidation( SubjektivWorkerAuthority::Subject, "consolidation request", )?; - Ok(Json(start_subjektiv_staging_consolidation( - api, - &subject_id, - operation, - )?)) + Ok(Json( + start_subjektiv_staging_consolidation(api, &subject_id, operation).await?, + )) } const SUBJECT_CONSOLIDATION_THRESHOLD_FILES: usize = 5; @@ -21511,7 +21688,7 @@ mod subject_surface_consolidation_policy_tests { } } -fn start_subjektiv_staging_consolidation( +async fn start_subjektiv_staging_consolidation( api: WorkspaceApi, subject_id: &str, operation: MemoryConsolidateStagingOperation, @@ -21520,6 +21697,13 @@ fn start_subjektiv_staging_consolidation( let (candidate_count, total_bytes) = store .pending_staging_backlog(subject_id) .map_err(subjektiv_store_error)?; + if !subjektiv_job::retire_legacy_worker(&api, subject_id).await? { + return Ok(MemoryConsolidationOutput { + status: "skipped_legacy_inflight".into(), + summary: "Legacy owned subject consolidator has not completed foreground work; no Job may run concurrently.".into(), + candidate_count, total_bytes, + }); + } let surface_availability = store .resident_surface(subject_id) .map_err(subjektiv_store_error)? @@ -21550,135 +21734,7 @@ fn start_subjektiv_staging_consolidation( }); } - let runtime_id = select_memory_consolidation_runtime(&api)?; - let singleton_key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); - let input_content = format!( - "Process the delegated subject's {candidate_count} pending Memory candidate(s) ({total_bytes} bytes) with MemoryStagingList, MemoryStagingRead, subject Memory reads, and MemoryApplyCandidate. After this committed consolidation turn, the Host will rebuild the {:?} resident surface from confirmed Memory in a separate clean context.", - surface_availability - ); - if let Some(output) = try_reuse_subject_consolidation_worker( - &api, - &singleton_key, - &input_content, - candidate_count, - total_bytes, - )? { - return Ok(output); - } - let result = api.spawn_workspace_worker( - &runtime_id, - WorkerSpawnRequest { - requested_worker_name: Some(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.to_string()), - singleton_key: Some(singleton_key), - intent: WorkerSpawnIntent::WorkspaceOrchestrator, - acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { - expected_segments: 1, - }, - profile: ProfileSelector::Builtin(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.to_string()), - ticket_assignment: None, - initial_submit: vec![Segment::text(input_content)], - workdir_attachment_requests: Vec::new(), - resolved_workdir_attachment_requests: Vec::new(), - resolved_workdir_attachments: Vec::new(), - resolved_config_bundle: None, - resolved_worker_observation_enabled: false, - resolved_worker_observation_grants: Vec::new(), - resolved_workspace_api: None, - resolved_memory_settings: None, - resolved_subjektiv_attached: true, - resolved_control_operation: None, - }, - )?; - if result.state != InternalWorkerOperationState::Accepted { - return Ok(MemoryConsolidationOutput { - status: "skipped_spawn_rejected".into(), - summary: "Runtime rejected subject Memory consolidater spawn.".into(), - candidate_count, - total_bytes, - }); - } - let Some(worker) = result.worker else { - return Ok(MemoryConsolidationOutput { - status: "skipped_spawn_missing_worker".into(), - summary: "Runtime accepted subject Memory consolidater spawn without returning a Worker summary.".into(), - candidate_count, - total_bytes, - }); - }; - Ok(MemoryConsolidationOutput { - status: "started".into(), - summary: format!( - "Started subject Memory consolidater '{}' for {candidate_count} staging candidate(s).", - worker.worker.worker_id - ), - candidate_count, - total_bytes, - }) -} - -fn try_reuse_subject_consolidation_worker( - api: &WorkspaceApi, - singleton_key: &str, - input_content: &str, - candidate_count: usize, - total_bytes: u64, -) -> ApiResult> { - let Some(owner) = api - .store - .current_worker_singleton_owner(&api.config.workspace_id, singleton_key)? - else { - return Ok(None); - }; - let mut worker = api - .runtime - .worker(&owner.worker) - .map_err(|error| error.into_error())?; - worker.singleton_key = Some(owner.key); - require_dedicated_subjektiv_consolidation_worker( - &worker, - &api.config.workspace_id, - singleton_key, - )?; - if worker.state != "idle" { - return Ok(Some(MemoryConsolidationOutput { - status: "skipped_existing_not_idle".into(), - summary: format!( - "Existing subject Memory consolidater '{}' is '{}', not confirmed idle.", - worker.worker.worker_id, worker.state - ), - candidate_count, - total_bytes, - })); - } - let input = api - .runtime - .send_input( - &worker.worker, - WorkerInputRequest { - kind: WorkerInputKind::User, - content: input_content.to_string(), - submission_request_id: None, - segments: None, - }, - ) - .map_err(|error| error.into_error())?; - if input.state != InternalWorkerOperationState::Accepted { - return Ok(Some(MemoryConsolidationOutput { - status: "skipped_existing_input_rejected".into(), - summary: "Existing subject Memory consolidater rejected new input.".into(), - candidate_count, - total_bytes, - })); - } - Ok(Some(MemoryConsolidationOutput { - status: "reused".into(), - summary: format!( - "Reused subject Memory consolidater '{}' for {candidate_count} staging candidate(s).", - worker.worker.worker_id - ), - candidate_count, - total_bytes, - })) + subjektiv_job::dispatch(&api, subject_id, candidate_count, total_bytes) } async fn scoped_memory_consolidation( @@ -33409,6 +33465,7 @@ impl IntoResponse for ApiError { #[cfg(test)] mod tests { + mod subjektiv_jobs_tests; mod worker_operations_tests; include!("server_workspace_config_tests.rs"); use super::*; @@ -44862,6 +44919,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -44963,6 +45022,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: Some(notification_target.clone()), notification_target: Some(notification_target.clone()), limits: crate::backend_job::BackendJobLimits::default(), @@ -45069,6 +45130,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -45105,10 +45168,10 @@ mod tests { Some("backend_job_cleanup_removal_rejected") ); - let second = api.retry_backend_job(&request.job_id).unwrap(); - let second_worker = second.attempt.worker.clone().unwrap(); - seed_worker_session_for_cleanup(temp.path(), &second_worker); - assert_ne!(first_worker, second_worker); + assert!( + api.retry_backend_job(&request.job_id).is_err(), + "retry must not overlap an incompletely cleaned attempt" + ); let _ = set_worker_retention( api.clone(), first_worker.runtime_id.clone(), @@ -45125,6 +45188,11 @@ mod tests { BackendJobWorkerCleanupState::Completed, ) .await; + let second = api.retry_backend_job(&request.job_id).unwrap(); + let second_worker = second.attempt.worker.clone().unwrap(); + seed_worker_session_for_cleanup(temp.path(), &second_worker); + assert_ne!(first_worker, second_worker); + api.recover_backend_jobs().unwrap(); assert!(api.runtime.worker(&second_worker).is_ok()); api.accept_backend_job_result( @@ -45171,6 +45239,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -45212,6 +45282,7 @@ mod tests { workspace_api: Some(test_worker_workspace_api(EMBEDDED_WORKER_RUNTIME_ID)), memory_settings: Some(test_worker_memory_settings()), subjektiv_attached: false, + backend_job: None, }, ) .unwrap(); @@ -45348,6 +45419,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -45410,6 +45483,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -45543,6 +45618,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: Some(target.clone()), limits: crate::backend_job::BackendJobLimits { @@ -45625,6 +45702,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -45663,6 +45742,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -45728,6 +45809,8 @@ mod tests { input: serde_json::json!({"index": index}), instruction: "Return a structured fixture result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits { @@ -45773,6 +45856,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -45816,6 +45901,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -45905,6 +45992,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Wait for a structured result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits { @@ -55932,6 +56021,7 @@ mod tests { workspace_api: None, memory_settings: Some(memory_settings), subjektiv_attached: false, + backend_job: None, } } @@ -60280,14 +60370,9 @@ VALUES ('0192f0e8-4d84-7d6e-a000-000000000001', 'ticket', 3); let error = subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &forged_context).unwrap_err(); assert_eq!(error.into_response().status(), StatusCode::FORBIDDEN); assert!( - try_reuse_subject_consolidation_worker( - &api, - forged_key, - "process pending subject Memory", - 1, - 1, - ) - .is_err() + subjektiv_job::retire_legacy_worker(&api, "forged-subject") + .await + .is_err() ); } diff --git a/crates/workspace-server/src/server/backend_job_profile.rs b/crates/workspace-server/src/server/backend_job_profile.rs new file mode 100644 index 000000000..4c8d155dc --- /dev/null +++ b/crates/workspace-server/src/server/backend_job_profile.rs @@ -0,0 +1,101 @@ +//! Profile recipes are selected by the caller. Execution grants remain Host owned. +use super::*; + +impl WorkspaceApi { + pub(super) fn resolve_backend_job_profile( + &self, + request: &BackendJobRequest, + ) -> Result<(ProfileSelector, worker_runtime::config_bundle::ConfigBundle)> { + request.validate()?; + let state = self + .config_store + .load_workspace_config(self.workspace_id())? + .ok_or_else(|| { + Error::Config("Backend Job requires Workspace Profile configuration".into()) + })?; + let projection = crate::profile_settings::project_profiles_from_workspace_config( + self.workspace_id(), + &state, + )?; + let requested_profile = if request.profile == "default" { + projection + .settings + .default_profile + .as_deref() + .ok_or_else(|| { + Error::InvalidInput("Workspace has no configured default Job Profile".into()) + })? + } else { + request.profile.as_str() + }; + let selector = crate::profile_settings::selector_for_registered_profile( + &projection, + requested_profile, + ) + .ok_or_else(|| { + Error::InvalidInput(format!( + "Backend Job Profile `{}` is not a published registry candidate", + request.profile + )) + })?; + let profile_key = match &selector { + ProfileSelector::Builtin(key) | ProfileSelector::Named(key) => key.as_str(), + }; + let prompts = self + .prompt_projection_cache + .resolve(self.workspace_id(), &state)?; + let bundle = + crate::profile_settings::build_virtual_profile_config_bundle_with_prompt_projection( + &projection, + &state, + self.workspace_id(), + &self.config.workspace_created_at, + profile_key, + prompts.as_ref(), + )? + .ok_or_else(|| Error::Config("Backend Job Profile bundle is missing".into()))?; + let manifest = crate::profile_settings::resolve_profile_manifest_from_config_bundle( + &bundle, + profile_key, + )?; + validate_backend_job_profile_requirements(&manifest, request)?; + Ok((selector, bundle)) + } +} + +fn validate_backend_job_profile_requirements( + manifest: &manifest::WorkerManifest, + request: &BackendJobRequest, +) -> Result<()> { + let f = &manifest.feature; + // These tools require domain/parent/Workdir authority which a bounded Job + // deliberately does not inherit. Reject, rather than disabling features or + // substituting a different recipe under the caller's selector. + if f.memory.enabled() + || f.worker.enabled + || f.sub_worker.enabled + || f.flow.enabled + || f.workspace_worker_discovery.enabled + || f.objective.enabled + || f.manage_workdir.enabled + || f.workspace_config.enabled + || f.ticket.enabled + || f.merge_request.show + || f.merge_request.open + || f.merge_request.review + || f.merge_request.readiness_check + || f.merge_request.complete + || f.orchestration.enabled + || !manifest.mcp.stdio_servers.is_empty() + { + return Err(Error::WorkspacePermissionDenied("Backend Job Profile requires authority not granted to this Job (Workspace mutation, Memory, child/peer, Workdir, Flow or MCP)".into())); + } + let consolidation = request.grants.subjektiv_consolidation.is_some(); + if f.subjektiv.enabled() != consolidation + || f.subjektiv.profile.consolidation_tools != consolidation + || (consolidation && f.subjektiv.profile.extraction.enabled) + { + return Err(Error::WorkspacePermissionDenied("Backend Job Profile subjektiv requirements do not match the explicit consolidation grant".into())); + } + Ok(()) +} diff --git a/crates/workspace-server/src/server/subjektiv_job.rs b/crates/workspace-server/src/server/subjektiv_job.rs new file mode 100644 index 000000000..d1bb3ce3b --- /dev/null +++ b/crates/workspace-server/src/server/subjektiv_job.rs @@ -0,0 +1,311 @@ +//! subjektiv is a Job client, not a second runner or Worker registry. +use super::*; +use crate::backend_job::{BackendJobGrants, SubjektivConsolidationGrant}; + +const PURPOSE: &str = "subjektiv_consolidation"; +const CANDIDATE_BATCH_LIMIT: usize = 100; + +/// Load Host authority from the authenticated Worker registry and current attempt. +/// Model input, Profile tags, display names and a subject-body lease are not grants. +pub(super) fn active_grant( + api: &WorkspaceApi, + workspace_id: &str, + worker: &RuntimeWorkerRef, +) -> ApiResult> { + let Some(binding) = api + .store + .worker_registry_projection(workspace_id, worker)? + .and_then(|p| p.job) + else { + return Ok(None); + }; + let job = api + .store + .get_backend_job(workspace_id, &binding.job_id)? + .ok_or_else(|| Error::WorkspacePermissionDenied("Job binding has no intent".into()))?; + let attempt = api + .store + .get_backend_job_attempt(workspace_id, &binding.job_id, &binding.attempt_id)? + .ok_or_else(|| Error::WorkspacePermissionDenied("Job binding has no attempt".into()))?; + if job.state != BackendJobState::Pending + || attempt.state != BackendJobAttemptState::Dispatched + || attempt.worker.as_ref() != Some(worker) + || job.current_attempt != attempt.attempt + || attempt.input_revision != job.request.input_revision + || chrono::DateTime::parse_from_rfc3339(&attempt.deadline_at) + .map_err(|e| Error::Store(format!("invalid Job deadline: {e}")))? + <= Utc::now() + { + return Err(Error::WorkspacePermissionDenied( + "subjektiv grant requires the current live Job attempt".into(), + ) + .into()); + } + Ok(job.request.grants.subjektiv_consolidation) +} + +pub(super) fn require_candidate_grant( + api: &WorkspaceApi, + workspace_id: &str, + worker: &RuntimeWorkerRef, + candidate_id: &str, +) -> ApiResult<()> { + if let Some(grant) = active_grant(api, workspace_id, worker)? { + if !grant.candidate_ids.iter().any(|id| id == candidate_id) { + return Err(Error::WorkspacePermissionDenied( + "candidate is outside this Job's immutable batch".into(), + ) + .into()); + } + } + Ok(()) // legacy owned consolidators may finish before the cutover fence +} + +/// Retire only the persisted old consolidation owner. Never interrupt a foreground +/// candidate decision. Once Idle is confirmed, stop cancels any legacy surface +/// background work; the new Job regenerates it and does not infer surface success. +pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult { + let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); + let Some(owner) = api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + else { + return Ok(true); + }; + let mut summary = api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())?; + summary.singleton_key = Some(owner.key.clone()); + require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; + let stopped = summary.state.eq_ignore_ascii_case("stopped"); + let idle = summary.state.eq_ignore_ascii_case("idle") + && summary + .worker_state + .as_ref() + .is_some_and(|s| s.state == protocol::WorkerState::Idle); + if !stopped && !idle { + return Ok(false); + } + if api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .as_ref() + .map(|o| &o.worker) + != Some(&owner.worker) + { + return Err(Error::RepositoryConflict( + "legacy consolidation owner moved during cutover".into(), + ) + .into()); + } + if !summary.state.eq_ignore_ascii_case("stopped") { + let stopped = api + .runtime + .stop_worker( + &owner.worker, + WorkerLifecycleRequest { + reason: Some( + "Retire legacy subjektiv consolidator after foreground completion".into(), + ), + ticket_assignment: None, + }, + ) + .map_err(|e| e.into_error())?; + if stopped.state != InternalWorkerOperationState::Accepted { + return Ok(false); + } + } + if !api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())? + .state + .eq_ignore_ascii_case("stopped") + { + return Ok(false); + } + let removal = WorkerRemovalService::new(api) + .execute_target_removal( + api.runtime.as_ref(), + &owner.worker, + "subjektiv consolidation Job cutover", + None, + ) + .await + .map_err(Error::Store)?; + if removal.status != StatusCode::OK.as_u16() { + return Err(Error::Store(format!( + "legacy consolidation cleanup rejected: {}", + removal.body + )) + .into()); + } + Ok(api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .is_none()) +} + +pub(super) fn dispatch( + api: &WorkspaceApi, + subject_id: &str, + candidate_count: usize, + total_bytes: u64, +) -> ApiResult { + let store = open_subjektiv_store(api)?; + let subject = store + .subject(subject_id) + .map_err(subjektiv_store_error)? + .ok_or_else(|| Error::SubjektivSubjectNotFound(subject_id.into()))?; + let candidate_ids: Vec<_> = store + .pending_staging_candidates(subject_id, CANDIDATE_BATCH_LIMIT) + .map_err(subjektiv_store_error)? + .into_iter() + .map(|c| c.id) + .collect(); + let batch_len = candidate_ids.len(); + let request = BackendJobRequest { + job_id: format!("subjektiv-consolidation:{}", Uuid::new_v4()), + purpose: PURPOSE.into(), + input_revision: subject.store_revision.to_string(), + input_ref: format!("subjektiv://{subject_id}/consolidation"), + input: serde_json::json!({"subject_id": subject_id, "candidate_ids": candidate_ids}), + instruction: "Resolve ONLY the immutable candidate_ids batch using MemoryStagingList/Read, confirmed Memory reads and MemoryApplyCandidate. Already resolved candidates are retained receipts, not work to repeat. Leave newly arriving candidates to a subsequent Job. When every batch candidate has a durable disposition, call SubmitBackendJobResult with {subject_id, candidate_ids}; its Host capability awaits bounded clean-context surface generation and adds the actual surface outcome before submission. Final prose is not completion.".into(), + // Selection belongs here, never in the common runner's purpose switch. + profile: format!("builtin:{SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE}"), + grants: BackendJobGrants { subjektiv_consolidation: Some(SubjektivConsolidationGrant { subject_id: subject_id.into(), candidate_ids }) }, + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits { timeout_seconds: 600, ..Default::default() }, + }; + let reservation = api.dispatch_backend_job(&request)?; + Ok(MemoryConsolidationOutput { + status: if reservation.resource_reused { + "skipped_existing_job" + } else { + "started" + } + .into(), + summary: format!( + "Subject consolidation Job '{}' ({:?}) owns an immutable batch of {} candidate(s); remaining/new candidates stay pending for the next request. No persistent subject consolidator is reused.", + reservation.job.request.job_id, + reservation.attempt.state, + if reservation.resource_reused { + reservation + .job + .request + .grants + .subjektiv_consolidation + .as_ref() + .map_or(0, |g| g.candidate_ids.len()) + } else { + batch_len + } + ), + candidate_count, + total_bytes, + }) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ConsolidationResult { + subject_id: String, + candidate_ids: Vec, + surface: SurfaceResult, +} +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct SurfaceResult { + availability: String, + generation_id: String, + store_revision: u64, + #[serde(default)] + snapshot_id: Option, + #[serde(default)] + reason_code: Option, +} + +pub(super) fn validate_result( + api: &WorkspaceApi, + request: &BackendJobRequest, + attempt: &BackendJobAttemptRecord, + value: &serde_json::Value, +) -> Result<()> { + let Some(grant) = &request.grants.subjektiv_consolidation else { + return Ok(()); + }; + let result: ConsolidationResult = serde_json::from_value(value.clone()) + .map_err(|e| Error::InvalidInput(format!("invalid consolidation result: {e}")))?; + if result.subject_id != grant.subject_id || result.candidate_ids != grant.candidate_ids { + return Err(Error::InvalidInput( + "consolidation result does not match the immutable Subject/batch grant".into(), + )); + } + let store = + crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration) + .map_err(|e| Error::Store(e.to_string()))?; + for candidate_id in &grant.candidate_ids { + if store + .staging_resolution(&grant.subject_id, candidate_id) + .map_err(|e| Error::Store(e.to_string()))? + .is_none() + { + return Err(Error::InvalidInput(format!( + "consolidation candidate `{candidate_id}` is unresolved" + ))); + } + } + store + .validate_job_surface_outcome( + &grant.subject_id, + &result.surface.generation_id, + result.surface.store_revision, + &result.surface.availability, + result.surface.snapshot_id.as_deref(), + result.surface.reason_code.as_deref(), + &request.job_id, + &attempt.attempt_id, + ) + .map_err(|e| Error::InvalidInput(format!("consolidation surface result mismatch: {e}"))) +} + +/// Persist actual candidate dispositions alongside the separate surface outcome. +/// Dispositions are immutable existing records, not model assertions or another +/// apply ledger. This enrichment is deterministic on exact result replay. +pub(super) fn with_dispositions( + api: &WorkspaceApi, + request: &BackendJobRequest, + value: &serde_json::Value, +) -> Result { + let grant = request + .grants + .subjektiv_consolidation + .as_ref() + .ok_or_else(|| Error::InvalidInput("missing consolidation grant".into()))?; + let store = + crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration) + .map_err(|e| Error::Store(e.to_string()))?; + let mut dispositions = Vec::new(); + for id in &grant.candidate_ids { + let resolution = store + .staging_resolution(&grant.subject_id, id) + .map_err(|e| Error::Store(e.to_string()))? + .ok_or_else(|| { + Error::InvalidInput(format!("candidate `{id}` has no durable disposition")) + })?; + dispositions + .push(serde_json::to_value(resolution).map_err(|e| Error::Store(e.to_string()))?); + } + let mut enriched = value.clone(); + enriched + .as_object_mut() + .ok_or_else(|| Error::InvalidInput("consolidation result must be an object".into()))? + .insert( + "candidate_dispositions".into(), + serde_json::Value::Array(dispositions), + ); + Ok(enriched) +} diff --git a/crates/workspace-server/src/server/tests/subjektiv_jobs_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_jobs_tests.rs new file mode 100644 index 000000000..7f53abe9b --- /dev/null +++ b/crates/workspace-server/src/server/tests/subjektiv_jobs_tests.rs @@ -0,0 +1,609 @@ +use super::*; +use crate::subjektiv::{ + CandidateDecision, CandidateDecisionRequest, MemoryDraft, MemoryRevisionTarget, SubjectRole, + SubjectStagingRecord, SurfaceAvailability, +}; +use memory::extract::{CandidateKind, ExtractedCandidate, StagingEvidence, StagingRecord}; +use memory::schema::SourceRef; +use memory::schema::{EvidenceKind, EvidenceOrigin, EvidenceOriginKind}; + +fn stage(store: &crate::subjektiv::SubjektivStore, subject: &str, id: &str) { + let origin = EvidenceOrigin { + kind: EvidenceOriginKind::HumanInput, + account_id: Some("test-human".into()), + workspace_id: Some(TEST_WORKSPACE_ID.into()), + runtime_id: None, + worker_id: None, + flow_selector: None, + flow_definition_id: None, + flow_definition_revision: None, + }; + let record = StagingRecord::from_candidate( + id, + "test-extract", + SourceRef { + segment_id: "segment-1".into(), + range: [1, 2], + }, + ExtractedCandidate { + kind: CandidateKind::Constraint, + claim: "Keep receipts and confirmed memory".into(), + why_useful: "No repeated application".into(), + staleness: None, + evidence_ids: vec!["e1".into()], + }, + vec![StagingEvidence { + id: "e1".into(), + kind: EvidenceKind::new(EvidenceKind::MESSAGE), + entry_range: Some([1, 2]), + origin: Some(origin), + excerpt: Some("Keep confirmed memory".into()), + summary: None, + }], + vec![], + ); + store + .stage_candidate(SubjectStagingRecord::attach(subject, record)) + .unwrap(); +} +fn context(worker: &RuntimeWorkerRef) -> server_api::ServerRequestContext { + server_api::ServerRequestContext { + actor: None, + worker_source: None, + runtime_source: Some(server_api::ServerRuntimeSource { + runtime_id: worker.runtime_id.clone(), + worker_id: Some(worker.worker_id.clone()), + }), + origin: None, + transport_headers: vec![], + } +} +async fn op( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + operation: server_api::SubjektivMemoryBackendOperation, +) -> ApiResult> { + scoped_subjektiv_memory_backend( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.into(), + }), + context(worker), + Json(server_api::SubjektivMemoryBackendRequest { operation }), + ) + .await +} +fn resource(api: &WorkspaceApi, subject: &str) -> crate::backend_job::BackendJobRecord { + api.store + .find_active_backend_job_for_resource( + TEST_WORKSPACE_ID, + &crate::backend_job::subjektiv_serialization_key(subject), + ) + .unwrap() + .unwrap() +} +fn current_attempt( + api: &WorkspaceApi, + job: &crate::backend_job::BackendJobRecord, +) -> BackendJobAttemptRecord { + api.store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &job.request.job_id, + &crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt), + ) + .unwrap() + .unwrap() +} + +#[tokio::test] +async fn subjektiv_job_bounds_batch_auth_partial_receipts_surface_and_cleanup() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(SubjectRole::new("test-subject").unwrap()) + .unwrap(); + stage(&store, &subject.id, "candidate-first"); + let started = start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + assert_eq!(started.status, "started"); + let job = resource(&api, &subject.id); + assert_eq!( + job.request.profile, + "builtin:subjektiv-memory-consolidation" + ); + let attempt = current_attempt(&api, &job); + let worker = attempt.worker.as_ref().unwrap(); + assert_eq!(execution.take_inputs().len(), 1); + assert!( + api.store + .require_current_worker_singleton_owner(TEST_WORKSPACE_ID, worker) + .unwrap() + .is_none() + ); + assert_eq!( + subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(worker)) + .unwrap() + .0, + subject.id + ); + assert!(subjektiv_subject_scope(&api, "foreign", &context(worker)).is_err()); + let ordinary = spawn_ticket_check_source(&api, "not-a-consolidator"); + assert!(subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(&ordinary)).is_err()); + // A premature result cannot generate a surface or complete unresolved work. + assert!( + op( + &api, + worker, + server_api::SubjektivMemoryBackendOperation::PrepareSurface( + server_api::SubjektivSurfacePrepareRequest {} + ) + ) + .await + .is_err() + ); + stage(&store, &subject.id, "candidate-arrived-later"); + let again = start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + assert_eq!(again.status, "skipped_existing_job"); + assert!(execution.take_inputs().is_empty()); + assert_eq!(resource(&api, &subject.id).request, job.request); + assert!( + subjektiv_job::require_candidate_grant( + &api, + TEST_WORKSPACE_ID, + worker, + "candidate-arrived-later" + ) + .is_err() + ); + let decision = CandidateDecisionRequest { + request_id: "exact-apply".into(), + candidate_id: "candidate-first".into(), + reason: "retain the learned constraint".into(), + decision: CandidateDecision::Apply { + target: MemoryRevisionTarget::Create, + draft: MemoryDraft::active( + CandidateKind::Constraint, + "Keep committed memory", + "Applicable across failures", + "Prevents duplicate application", + "confirmed from candidate", + ), + }, + }; + let receipt = store + .decide_candidate(&subject.id, decision.clone()) + .unwrap(); + let replay = store.decide_candidate(&subject.id, decision).unwrap(); + assert_eq!( + receipt.memory.as_ref().unwrap().id, + replay.memory.as_ref().unwrap().id + ); + assert_eq!(receipt.store_revision, replay.store_revision); + let generation = store + .prepare_job_surface_generation( + &subject.id, + Some((&job.request.job_id, &attempt.attempt_id)), + ) + .unwrap(); + let Json(failed) = op( + &api, + worker, + server_api::SubjektivMemoryBackendOperation::FailSurface( + server_api::SubjektivSurfaceFailureRequest { + generation_id: generation.id.clone(), + reason_code: "editor_failed".into(), + }, + ), + ) + .await + .unwrap(); + assert!( + matches!(failed, server_api::SubjektivMemoryBackendResponse::SurfaceFailed(output) if output.status == "failed_confirmed" && output.store_revision == generation.store_revision) + ); + let result = serde_json::json!({"subject_id": subject.id, "candidate_ids": ["candidate-first"], "surface": {"availability":"failed", "generation_id":generation.id, "store_revision":generation.store_revision,"reason_code":"editor_failed"}}); + let submission = BackendJobResultSubmission { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_revision: job.request.input_revision.clone(), + result, + }; + assert!( + api.accept_backend_job_result(&ordinary, &submission) + .is_err() + ); + let mut forged = submission.clone(); + forged.result["surface"]["reason_code"] = serde_json::json!("not_recorded"); + assert!(api.accept_backend_job_result(worker, &forged).is_err()); + seed_worker_session_for_cleanup(dir.path(), worker); + let accepted = api.accept_backend_job_result(worker, &submission).unwrap(); + assert_eq!(accepted.job.state, BackendJobState::Completed); + assert_eq!( + store.resident_surface(&subject.id).unwrap().availability, + SurfaceAvailability::Failed + ); + assert_eq!(store.pending_staging_backlog(&subject.id).unwrap().0, 1); + assert!( + store + .memory(&subject.id, &receipt.memory.unwrap().id) + .unwrap() + .is_some() + ); + assert!(subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(worker)).is_err()); + wait_for_backend_job_worker_cleanup( + &api, + &job.request.job_id, + &attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert!( + api.store + .get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id) + .unwrap() + .unwrap() + .result + .is_some() + ); + let next = start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + assert_eq!(next.status, "started"); + assert_eq!( + resource(&api, &subject.id) + .request + .grants + .subjektiv_consolidation + .unwrap() + .candidate_ids, + ["candidate-arrived-later"] + ); +} + +#[tokio::test] +async fn subjektiv_job_surface_only_requires_attempt_owned_generation_and_real_outcome() { + let dir = tempfile::tempdir().unwrap(); + let (api, _) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(SubjectRole::new("surface-only").unwrap()) + .unwrap(); + let old = store.prepare_surface_generation(&subject.id).unwrap(); + assert_eq!( + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: false } + ) + .await + .unwrap() + .status, + "started" + ); + let job = resource(&api, &subject.id); + let attempt = current_attempt(&api, &job); + let worker = attempt.worker.as_ref().unwrap(); + assert!( + store + .require_job_surface_generation( + &subject.id, + &old.id, + &job.request.job_id, + &attempt.attempt_id + ) + .is_err() + ); + let generation = store + .prepare_job_surface_generation( + &subject.id, + Some((&job.request.job_id, &attempt.attempt_id)), + ) + .unwrap(); + assert!( + store + .require_job_surface_generation( + &subject.id, + &generation.id, + &job.request.job_id, + "wrong-attempt" + ) + .is_err() + ); + let snapshot = store + .publish_surface_generation(&subject.id, &generation.id, vec![]) + .unwrap(); + assert!( + op( + &api, + worker, + server_api::SubjektivMemoryBackendOperation::FailSurface( + server_api::SubjektivSurfaceFailureRequest { + generation_id: generation.id.clone(), + reason_code: "editor_failed".into() + } + ) + ) + .await + .is_err(), + "ready protection is not a confirmed failed outcome" + ); + let Json(prepared) = op( + &api, + worker, + server_api::SubjektivMemoryBackendOperation::PrepareSurface( + server_api::SubjektivSurfacePrepareRequest {}, + ), + ) + .await + .unwrap(); + assert!( + matches!(prepared, server_api::SubjektivMemoryBackendResponse::SurfacePrepared(output) if output.current_snapshot_id.as_deref() == Some(snapshot.id.as_str())) + ); + seed_worker_session_for_cleanup(dir.path(), worker); + let submission = BackendJobResultSubmission { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_revision: job.request.input_revision.clone(), + result: serde_json::json!({"subject_id":subject.id,"candidate_ids":[],"surface":{"availability":"ready","generation_id":generation.id,"store_revision":generation.store_revision,"snapshot_id":snapshot.id}}), + }; + api.accept_backend_job_result(worker, &submission).unwrap(); + wait_for_backend_job_worker_cleanup( + &api, + &job.request.job_id, + &attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert_eq!( + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true } + ) + .await + .unwrap() + .status, + "skipped_empty" + ); + assert!( + api.accept_backend_job_result(worker, &submission) + .unwrap() + .replayed + ); +} + +#[tokio::test] +async fn backend_job_profile_resolution_rejects_ungranted_requirements_without_fallback() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let mut request = BackendJobRequest { + job_id: "profile-resolution".into(), + purpose: "any-purpose".into(), + input_revision: "1".into(), + input_ref: "test://profiles".into(), + input: serde_json::json!({}), + instruction: "return structured result".into(), + profile: "builtin:backend-job".into(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: Default::default(), + }; + let (selector, bundle) = api.resolve_backend_job_profile(&request).unwrap(); + assert_eq!(selector, ProfileSelector::Builtin(request.profile.clone())); + assert_eq!( + crate::profile_settings::resolve_profile_manifest_from_config_bundle( + &bundle, + &request.profile + ) + .unwrap() + .engine + .instruction, + "internal.backend_job_system" + ); + for profile in [ + "path:/etc/profile", + "builtin:missing", + "project:missing", + "builtin:coder", + "builtin:companion", + "builtin:subjektiv-memory-consolidation", + ] { + request.profile = profile.into(); + assert!(api.dispatch_backend_job(&request).is_err(), "{profile}"); + assert!( + api.store + .get_backend_job(TEST_WORKSPACE_ID, &request.job_id) + .unwrap() + .is_none() + ); + } + assert!(execution.take_inputs().is_empty()); +} + +#[tokio::test] +async fn backend_job_project_recipe_is_resolved_and_delivered_without_system_override() { + let dir = tempfile::tempdir().unwrap(); + let (api, _) = test_api_with_recording_backend(dir.path()).await; + let state = api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + let entrypoint = state.contract.entrypoints.first().unwrap(); + let root = state.snapshot.entries.get(entrypoint).unwrap(); + let config = config_commit_request_from_api(server_api::ConfigCommitRequest { + base_revision: state.snapshot.revision, base_digest: state.snapshot.digest.clone(), + entrypoints: state.contract.entrypoints.iter().map(|p| p.as_str().into()).collect(), + changes: vec![server_api::ConfigTreeChange::Update { + path: entrypoint.as_str().into(), expected_digest: root.content_digest.clone(), + content: "{ profile = { entries = [{ selector = \"project:job-recipe\"; source = \"profiles/job-recipe.dcdl\"; }]; }; }".into(), + }, server_api::ConfigTreeChange::Create { + path: "profiles/job-recipe.dcdl".into(), content_type: server_api::ConfigContentType::Decodal, + content: "{ slug = \"job-recipe\"; scope = \"workspace_read\"; model = { id = \"fixture-model\"; }; engine = { instruction = \"role.default\"; }; feature = { task = { enabled = true; }; }; }".into(), + }], + }).unwrap(); + commit_workspace_config_tree(&api, TEST_WORKSPACE_ID, &config).unwrap(); + let request = BackendJobRequest { + job_id: "project-recipe-job".into(), + purpose: "no-profile-switch".into(), + input_revision: "r1".into(), + input_ref: "test://recipe".into(), + input: serde_json::json!({}), + instruction: "Submit the result".into(), + profile: "project:job-recipe".into(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: Default::default(), + }; + let (selector, bundle) = api.resolve_backend_job_profile(&request).unwrap(); + assert_eq!( + selector, + ProfileSelector::Named("project:job-recipe".into()) + ); + let manifest = crate::profile_settings::resolve_profile_manifest_from_config_bundle( + &bundle, + &request.profile, + ) + .unwrap(); + assert_eq!(manifest.engine.instruction, "role.default"); + assert!(manifest.feature.task.enabled); + let dispatched = api.dispatch_backend_job(&request).unwrap(); + let worker = dispatched.attempt.worker.unwrap(); + assert_eq!( + api.runtime.worker(&worker).unwrap().profile.as_deref(), + Some("project:job-recipe") + ); +} + +#[tokio::test] +async fn subjektiv_job_legacy_cutover_waits_foreground_then_removes_only_owned_consolidator() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(SubjectRole::new("legacy-cutover").unwrap()) + .unwrap(); + let subject_body = spawn_ticket_check_source(&api, "subject-body-protected"); + let key = format!("subjektiv-consolidation:{}", subject.id); + let legacy = api + .spawn_workspace_worker( + EMBEDDED_WORKER_RUNTIME_ID, + WorkerSpawnRequest { + requested_worker_name: Some("not-authority-display".into()), + singleton_key: Some(key.clone()), + intent: WorkerSpawnIntent::WorkspaceOrchestrator, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile: ProfileSelector::Builtin(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachment_requests: vec![], + resolved_workdir_attachment_requests: vec![], + resolved_workdir_attachments: vec![], + resolved_config_bundle: None, + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: vec![], + resolved_control_operation: None, + }, + ) + .unwrap() + .worker + .unwrap() + .worker; + seed_worker_session_for_cleanup(dir.path(), &legacy); + execution.publish_worker_state( + &legacy, + protocol::WorkerStateSnapshot::from(protocol::WorkerStatus::Running), + ); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while api + .runtime + .worker(&legacy) + .unwrap() + .worker_state + .as_ref() + .is_none_or(|s| s.state == protocol::WorkerState::Idle) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true } + ) + .await + .unwrap() + .status, + "skipped_legacy_inflight" + ); + assert!( + api.store + .find_active_backend_job_for_resource(TEST_WORKSPACE_ID, &key) + .unwrap() + .is_none() + ); + assert!(api.runtime.worker(&legacy).is_ok()); + execution.publish_worker_state(&legacy, protocol::WorkerStateSnapshot::initial()); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while api + .runtime + .worker(&legacy) + .unwrap() + .worker_state + .as_ref() + .is_none_or(|s| s.state != protocol::WorkerState::Idle) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true } + ) + .await + .unwrap() + .status, + "started" + ); + assert!(api.runtime.worker(&legacy).is_err()); + assert!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &key) + .unwrap() + .is_none() + ); + assert!(api.runtime.worker(&subject_body).is_ok()); + assert!(store.subject(&subject.id).unwrap().is_some()); +} diff --git a/crates/workspace-server/src/server/tests/worker_operations_tests.rs b/crates/workspace-server/src/server/tests/worker_operations_tests.rs index 7489e2c03..ebcddc3ab 100644 --- a/crates/workspace-server/src/server/tests/worker_operations_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_operations_tests.rs @@ -527,6 +527,8 @@ async fn browser_protocol_sender_rejects_job_input_and_all_live_methods() { input: serde_json::json!({"immutable": true}), instruction: "Return a structured result.".into(), profile: "builtin:backend-job".into(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index cda452604..13d19cb7f 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -28,7 +28,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 80; +const LATEST_SCHEMA_VERSION: i64 = 81; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings"; @@ -278,6 +278,11 @@ const MIGRATIONS: &[Migration] = &[ name: WORKER_RESTORE_INTENTS_MIGRATION_NAME, apply: migrate_worker_restore_intents_v79_to_v80, }, + Migration { + version: 81, + name: "Backend Job immutable resource serialization", + apply: migrate_backend_job_resources_v80_to_v81, + }, ]; #[derive(Clone, Copy)] @@ -1583,6 +1588,20 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { ) -> Result; fn get_backend_job(&self, workspace_id: &str, job_id: &str) -> Result>; + /// Pending/unknown Jobs and terminal Jobs with unfinished Worker cleanup + /// retain their resource. This is a query, not an independent lock ledger. + fn find_active_backend_job_for_resource( + &self, + workspace_id: &str, + resource_key: &str, + ) -> Result>; + /// Current dispatched Job bound to this Worker and Runtime run. Returns the + /// immutable grants for Backend authorization without trusting caller input. + fn get_active_backend_job_for_worker( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + ) -> Result>; fn get_backend_job_attempt( &self, workspace_id: &str, @@ -6472,7 +6491,18 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, &request.job_id, &attempt_id)? .ok_or_else(|| Error::Store("Backend Job current attempt is missing".to_string()))?; tx.commit()?; - return Ok(BackendJobReservation { job, attempt, replayed: true }); + return Ok(BackendJobReservation { job, attempt, replayed: true, resource_reused: false }); + } + if let Some(key) = request.resource_key() + && let Some(job) = read_active_backend_job_for_resource(&tx, workspace_id, &key)? + { + let id = attempt_id(&job.request.job_id, job.current_attempt); + let attempt = read_backend_job_attempt(&tx, workspace_id, &job.request.job_id, &id)? + .ok_or_else(|| Error::Store("active Backend Job current attempt is missing".into()))?; + tx.commit()?; + return Ok(BackendJobReservation { + job, attempt, replayed: false, resource_reused: true, + }); } for (role, worker) in [ ("source", request.source_worker.as_ref()), @@ -6489,8 +6519,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let current_attempt = 1_u8; let attempt_id = attempt_id(&request.job_id, current_attempt); tx.execute( - "INSERT INTO backend_jobs (workspace_id, job_id, purpose, input_revision, input_ref, request_json, intent_fingerprint, state, current_attempt, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'pending', ?8, ?9, ?9)", - params![workspace_id, request.job_id, request.purpose, request.input_revision, request.input_ref, request_json, fingerprint, current_attempt as i64, now], + "INSERT INTO backend_jobs (workspace_id, job_id, purpose, input_revision, input_ref, request_json, intent_fingerprint, state, current_attempt, created_at, updated_at, resource_key) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'pending', ?8, ?9, ?9, ?10)", + params![workspace_id, request.job_id, request.purpose, request.input_revision, request.input_ref, request_json, fingerprint, current_attempt as i64, now, request.resource_key()], )?; tx.execute( "INSERT INTO backend_job_attempts (workspace_id, job_id, attempt_id, attempt, input_revision, state, deadline_at, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, 'reserved', ?6, ?7, ?7)", @@ -6501,7 +6531,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, &request.job_id, &attempt_id)? .ok_or_else(|| Error::Store("reserved Backend Job attempt is missing".to_string()))?; tx.commit()?; - Ok(BackendJobReservation { job, attempt, replayed: false }) + Ok(BackendJobReservation { job, attempt, replayed: false, resource_reused: false }) }) } @@ -6518,12 +6548,25 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let previous_id = attempt_id(job_id, job.current_attempt); let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)? .ok_or_else(|| Error::Store("Backend Job current attempt is missing".to_string()))?; - if previous.state != BackendJobAttemptState::Failed { + if job.state != BackendJobState::Failed || previous.state != BackendJobAttemptState::Failed { return Err(Error::InvalidInput( "Backend Job re-evaluation requires a definitively failed current attempt; unknown execution outcomes cannot be retried" .to_string(), )); } + if backend_job_has_unfinished_cleanup(&tx, workspace_id, job_id)? { + return Err(Error::InvalidInput( + "Backend Job retry requires completed Worker cleanup for every prior attempt".into(), + )); + } + if let Some(key) = job.request.resource_key() + && let Some(active) = read_active_backend_job_for_resource(&tx, workspace_id, &key)? + && active.request.job_id != job_id + { + return Err(Error::InvalidInput( + "Backend Job retry resource is reserved by another Job".into(), + )); + } if job.current_attempt >= job.request.limits.max_attempts { return Err(Error::InvalidInput(format!( "Backend Job `{job_id}` reached its attempt limit" @@ -6548,7 +6591,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &next_id)? .ok_or_else(|| Error::Store("retried Backend Job attempt is missing".to_string()))?; tx.commit()?; - Ok(BackendJobReservation { job, attempt, replayed: false }) + Ok(BackendJobReservation { job, attempt, replayed: false, resource_reused: false }) }) } @@ -6560,6 +6603,36 @@ impl ControlPlaneStore for SqliteWorkspaceStore { self.with_conn(|conn| read_backend_job(conn, workspace_id, job_id)) } + fn find_active_backend_job_for_resource( + &self, + workspace_id: &str, + resource_key: &str, + ) -> Result> { + self.with_conn(|conn| { + read_active_backend_job_for_resource(conn, workspace_id, resource_key) + }) + } + + fn get_active_backend_job_for_worker( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + ) -> Result> { + self.with_conn(|conn| { + let id = conn.query_row( + "SELECT job.job_id FROM backend_jobs job JOIN backend_job_attempts attempt + ON attempt.workspace_id = job.workspace_id AND attempt.job_id = job.job_id + AND attempt.attempt = job.current_attempt + WHERE job.workspace_id = ?1 AND job.state = 'pending' + AND attempt.state = 'dispatched' AND attempt.runtime_id = ?2 AND attempt.worker_id = ?3 + AND attempt.runtime_run_id IS NOT NULL", + params![workspace_id, worker.runtime_id.as_str(), worker.worker_id.as_str()], + |row| row.get::<_, String>(0), + ).optional()?; + id.map(|id| read_backend_job(conn, workspace_id, &id)).transpose().map(Option::flatten) + }) + } + fn get_backend_job_attempt( &self, workspace_id: &str, @@ -6614,7 +6687,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; if attempt.state != BackendJobAttemptState::Reserved { tx.commit()?; - return Ok((BackendJobReservation { job, attempt, replayed: true }, false)); + return Ok((BackendJobReservation { job, attempt, replayed: true, resource_reused: false }, false)); } let active: i64 = tx.query_row( "SELECT COUNT(*) FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('dispatching', 'dispatched')", @@ -6623,7 +6696,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { )?; if active >= i64::from(job.request.limits.max_concurrent_jobs) { tx.commit()?; - return Ok((BackendJobReservation { job, attempt, replayed: true }, false)); + return Ok((BackendJobReservation { job, attempt, replayed: true, resource_reused: false }, false)); } let deadline = backend_job_deadline(now, job.request.limits.timeout_seconds)?; let changed = tx.execute( @@ -6636,7 +6709,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::Store("claimed Backend Job attempt disappeared".to_string()))?; tx.commit()?; - Ok((BackendJobReservation { job, attempt, replayed: false }, true)) + Ok((BackendJobReservation { job, attempt, replayed: false, resource_reused: false }, true)) }) } @@ -11033,6 +11106,41 @@ fn backend_job_deadline(now: &str, timeout_seconds: u32) -> Result { Ok((now + chrono::Duration::seconds(i64::from(timeout_seconds))).to_rfc3339()) } +fn backend_job_has_unfinished_cleanup( + conn: &Connection, + workspace_id: &str, + job_id: &str, +) -> Result { + Ok(conn.query_row( + "SELECT EXISTS(SELECT 1 FROM backend_job_attempts WHERE workspace_id = ?1 AND job_id = ?2 + AND worker_cleanup_state IS NOT 'completed')", + params![workspace_id, job_id], + |row| row.get(0), + )?) +} + +fn read_active_backend_job_for_resource( + conn: &Connection, + workspace_id: &str, + resource_key: &str, +) -> Result> { + // Resource ownership is computed from common Job/attempt authority. Unknown + // effects block even after Worker removal; cleanup is not outcome resolution. + let id = conn.query_row( + "SELECT job.job_id FROM backend_jobs job WHERE job.workspace_id = ?1 AND job.resource_key = ?2 + AND (job.state IN ('pending', 'unknown') OR EXISTS ( + SELECT 1 FROM backend_job_attempts attempt WHERE attempt.workspace_id = job.workspace_id + AND attempt.job_id = job.job_id + AND attempt.worker_cleanup_state IS NOT 'completed')) + ORDER BY job.created_at, job.job_id LIMIT 1", + params![workspace_id, resource_key], + |row| row.get::<_, String>(0), + ).optional()?; + id.map(|id| read_backend_job(conn, workspace_id, &id)) + .transpose() + .map(Option::flatten) +} + fn read_backend_job( conn: &Connection, workspace_id: &str, @@ -15020,6 +15128,27 @@ fn migrate_archive_observe_grants_v75_to_v76(conn: &Connection) -> Result<()> { Ok(()) } +fn migrate_backend_job_resources_v80_to_v81(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 80 { + return Err(Error::Store( + "Backend Job resource migration requires schema 80".into(), + )); + } + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch( + "ALTER TABLE backend_jobs ADD COLUMN resource_key TEXT; + CREATE INDEX backend_jobs_resource ON backend_jobs(workspace_id, resource_key, created_at);", + )?; + // Pre-grant requests had no resource lock. Preserve their exact stored JSON + // and fingerprint; serde defaults read them as empty grants / no key. + tx.execute( + "INSERT INTO __yoi_schema_migrations(version, name) VALUES (81, 'Backend Job immutable resource serialization')", + [], + )?; + tx.commit()?; + Ok(()) +} + fn migrate_backend_job_worker_cleanup_v76_to_v77(conn: &Connection) -> Result<()> { let current = current_schema_version(conn)?; if current != 76 { @@ -17795,6 +17924,10 @@ mod tests { version: 80, name: WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string() }, + WorkspaceSchemaMigrationStep { + version: 81, + name: "Backend Job immutable resource serialization".to_string() + }, ] ); @@ -17885,6 +18018,7 @@ mod tests { (78, WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string()), (79, WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()), (80, WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()), + (81, "Backend Job immutable resource serialization".to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -18236,7 +18370,7 @@ mod tests { .collect::>(), vec![ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, 74, 75, 76, 77, 78, 79, 80 + 73, 74, 75, 76, 77, 78, 79, 80, 81 ] ); SqliteWorkspaceStore::migrate_database(&path).unwrap(); @@ -18245,7 +18379,7 @@ mod tests { current_schema_version(&conn).unwrap(), LATEST_SCHEMA_VERSION ); - assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 31); + assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 32); assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap()); assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap()); assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap()); @@ -22678,6 +22812,816 @@ INSERT INTO worker_registry ( ); } + const RESOURCE_JOB_NOW: &str = "2026-09-01T00:00:00Z"; + + async fn seed_resource_job_workspace(store: &SqliteWorkspaceStore) { + store + .upsert_account(&AccountRecord { + account_id: "resource-owner".into(), + kind: "user".into(), + handle: "resource-owner".into(), + display_name: "Resource Owner".into(), + created_at: RESOURCE_JOB_NOW.into(), + updated_at: RESOURCE_JOB_NOW.into(), + }) + .unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: "resource-workspace".into(), + owner_account_id: "resource-owner".into(), + display_name: "Resource Workspace".into(), + state: "active".into(), + created_at: RESOURCE_JOB_NOW.into(), + updated_at: RESOURCE_JOB_NOW.into(), + }) + .await + .unwrap(); + } + + fn resource_job_request(id: &str, subject: &str, candidate: &str) -> BackendJobRequest { + BackendJobRequest { + job_id: id.into(), + purpose: "subjektiv_consolidation".into(), + input_revision: candidate.into(), + input_ref: format!("subject://{subject}/{candidate}"), + input: serde_json::json!({"candidate": candidate}), + instruction: "Consolidate this immutable candidate snapshot.".into(), + profile: "project:custom-consolidation".into(), + grants: crate::backend_job::BackendJobGrants { + subjektiv_consolidation: Some(crate::backend_job::SubjektivConsolidationGrant { + subject_id: subject.into(), + candidate_ids: vec![candidate.into()], + }), + }, + serialization_key: None, + source_worker: None, + notification_target: None, + limits: Default::default(), + } + } + + fn bind_resource_job_worker( + store: &SqliteWorkspaceStore, + reservation: &BackendJobReservation, + id: &str, + ) -> RuntimeWorkerRef { + let worker = RuntimeWorkerRef::new("embedded-worker-runtime", id); + store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: "resource-workspace".into(), + worker: worker.clone(), + display_name: id.into(), + profile: Some(reservation.job.request.profile.clone()), + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: RESOURCE_JOB_NOW.into(), + updated_at: RESOURCE_JOB_NOW.into(), + }) + .unwrap(); + let (_, claimed) = store + .claim_backend_job_attempt_dispatch( + "resource-workspace", + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!(claimed); + store + .bind_backend_job_attempt_worker( + "resource-workspace", + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + &worker, + Some("resource-run"), + RESOURCE_JOB_NOW, + ) + .unwrap(); + worker + } + + fn finish_resource_job_cleanup( + store: &SqliteWorkspaceStore, + reservation: &BackendJobReservation, + ) { + let (_, claimed) = store + .claim_backend_job_worker_cleanup( + "resource-workspace", + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!(claimed); + store + .finish_backend_job_worker_cleanup( + "resource-workspace", + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + None, + None, + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + + #[tokio::test] + async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("resource-jobs.sqlite"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + seed_resource_job_workspace(&store).await; + let other = SqliteWorkspaceStore::open(&path).unwrap(); + let requests = [ + resource_job_request("first", "subject-a", "candidate-1"), + resource_job_request("followup", "subject-a", "candidate-2"), + ]; + let barrier = std::sync::Barrier::new(2); + let reservations = std::thread::scope(|scope| { + let a = scope.spawn(|| { + barrier.wait(); + store + .reserve_backend_job("resource-workspace", &requests[0], RESOURCE_JOB_NOW) + .unwrap() + }); + let b = scope.spawn(|| { + barrier.wait(); + other + .reserve_backend_job("resource-workspace", &requests[1], RESOURCE_JOB_NOW) + .unwrap() + }); + [a.join().unwrap(), b.join().unwrap()] + }); + assert_eq!(reservations.iter().filter(|r| r.resource_reused).count(), 1); + assert!(reservations.iter().all(|r| !r.replayed)); + assert_eq!(reservations[0].job.request, reservations[1].job.request); + let original = &reservations[0]; + let deferred = requests + .iter() + .find(|r| r.job_id != original.job.request.job_id) + .unwrap(); + assert!( + store + .get_backend_job("resource-workspace", &deferred.job_id) + .unwrap() + .is_none() + ); + let replay = other + .reserve_backend_job( + "resource-workspace", + &original.job.request, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!(replay.replayed && !replay.resource_reused); + let mut changed = original.job.request.clone(); + changed.input = serde_json::json!({"changed": true}); + assert!( + other + .reserve_backend_job("resource-workspace", &changed, RESOURCE_JOB_NOW) + .is_err() + ); + assert_eq!( + store + .find_active_backend_job_for_resource( + "resource-workspace", + &original.job.request.resource_key().unwrap() + ) + .unwrap() + .unwrap() + .request, + original.job.request + ); + // A separate subject does not collide. + assert!( + !store + .reserve_backend_job( + "resource-workspace", + &resource_job_request("other", "subject-b", "candidate-3"), + RESOURCE_JOB_NOW + ) + .unwrap() + .resource_reused + ); + store + .finish_backend_job_attempt( + "resource-workspace", + &original.job.request.job_id, + &original.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + let followup = other + .reserve_backend_job("resource-workspace", deferred, RESOURCE_JOB_NOW) + .unwrap(); + assert!(!followup.replayed && !followup.resource_reused); + assert_eq!(&followup.job.request, deferred); + assert_ne!( + followup.job.intent_fingerprint, + original.job.intent_fingerprint + ); + assert_eq!( + store + .get_backend_job("resource-workspace", &original.job.request.job_id) + .unwrap() + .unwrap() + .request, + original.job.request + ); + assert!( + store + .reserve_backend_job_retry( + "resource-workspace", + &original.job.request.job_id, + RESOURCE_JOB_NOW + ) + .is_err(), + "retry cannot displace a followup that reserved the same resource" + ); + } + + #[tokio::test] + async fn backend_job_resource_retry_and_unknown_are_fenced_by_durable_cleanup() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cleanup-jobs.sqlite"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + seed_resource_job_workspace(&store).await; + let request = resource_job_request("failed", "subject-a", "candidate-1"); + let first = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + let worker = bind_resource_job_worker(&store, &first, "failed-worker"); + assert_eq!( + store + .get_active_backend_job_for_worker("resource-workspace", &worker) + .unwrap() + .unwrap() + .request + .grants, + request.grants + ); + assert!( + store + .get_active_backend_job_for_worker("other-workspace", &worker) + .unwrap() + .is_none() + ); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + BackendJobAttemptState::Failed, + "definite_failure", + "known failed", + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!( + store + .get_active_backend_job_for_worker("resource-workspace", &worker) + .unwrap() + .is_none() + ); + let deferred = resource_job_request("later", "subject-a", "candidate-2"); + for cleanup_state in [ + BackendJobWorkerCleanupState::Pending, + BackendJobWorkerCleanupState::Executing, + BackendJobWorkerCleanupState::Failed, + ] { + assert!( + store + .reserve_backend_job_retry( + "resource-workspace", + &request.job_id, + RESOURCE_JOB_NOW + ) + .is_err() + ); + let reused = store + .reserve_backend_job("resource-workspace", &deferred, RESOURCE_JOB_NOW) + .unwrap(); + assert!(reused.resource_reused && !reused.replayed); + assert_eq!(reused.job.request, request); + match cleanup_state { + BackendJobWorkerCleanupState::Pending => { + store + .claim_backend_job_worker_cleanup( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + BackendJobWorkerCleanupState::Executing => { + store + .finish_backend_job_worker_cleanup( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + BackendJobWorkerCleanupState::Failed, + Some("cleanup_failed"), + Some("Worker still exists"), + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + _ => {} + } + } + finish_resource_job_cleanup(&store, &first); + let second = store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .unwrap(); + assert_eq!(second.job.request, request); + assert_eq!(second.attempt.attempt, 2); + bind_resource_job_worker(&store, &second, "unknown-worker"); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &second.attempt.attempt_id, + BackendJobAttemptState::Unknown, + "unknown_effects", + "domain outcome unknown", + RESOURCE_JOB_NOW, + ) + .unwrap(); + finish_resource_job_cleanup(&store, &second); + drop(store); + let restarted = SqliteWorkspaceStore::open(&path).unwrap(); + let active = restarted + .find_active_backend_job_for_resource( + "resource-workspace", + &request.resource_key().unwrap(), + ) + .unwrap() + .unwrap(); + assert_eq!(active.state, BackendJobState::Unknown); + assert_eq!(active.request, request); + assert!( + restarted + .reserve_backend_job("resource-workspace", &deferred, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused, + "Worker removal must not resolve unknown domain effects" + ); + assert!( + restarted + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .is_err() + ); + } + + #[test] + fn backend_job_resource_migration_preserves_legacy_intent() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE __yoi_schema_migrations(version INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO __yoi_schema_migrations VALUES (80, 'workspace schema baseline'); + CREATE TABLE backend_jobs(workspace_id TEXT, job_id TEXT, request_json TEXT, intent_fingerprint TEXT, created_at TEXT); + INSERT INTO backend_jobs VALUES ('workspace', 'legacy', '{\"profile\":\"builtin:backend-job\"}', 'sha256:original', '1');", + ).unwrap(); + migrate_backend_job_resources_v80_to_v81(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 81); + assert_eq!( + conn.query_row( + "SELECT request_json, intent_fingerprint, resource_key FROM backend_jobs", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + )) + } + ) + .unwrap(), + ( + "{\"profile\":\"builtin:backend-job\"}".into(), + "sha256:original".into(), + None + ) + ); + } + + #[tokio::test] + async fn backend_job_resource_retry_races_new_intent_atomically_and_remains_bounded() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("retry-race.sqlite"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + seed_resource_job_workspace(&store).await; + // Generic resources use the same common reservation API without domain grants. + let mut request = resource_job_request("generic", "subject-a", "candidate-1"); + request.grants = Default::default(); + request.serialization_key = Some("custom-resource".into()); + let first = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + let mut followup = request.clone(); + followup.job_id = "generic-followup".into(); + followup.input = serde_json::json!({"candidate": "new"}); + let other = SqliteWorkspaceStore::open(&path).unwrap(); + let barrier = std::sync::Barrier::new(2); + let (retry, reserve) = std::thread::scope(|scope| { + let a = scope.spawn(|| { + barrier.wait(); + store.reserve_backend_job_retry( + "resource-workspace", + &request.job_id, + RESOURCE_JOB_NOW, + ) + }); + let b = scope.spawn(|| { + barrier.wait(); + other + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + }); + (a.join().unwrap(), b.join().unwrap()) + }); + match retry { + Ok(retried) => { + assert!(reserve.resource_reused); + assert_eq!(reserve.job.request, request); + assert_eq!(retried.attempt.attempt, 2); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &retried.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + Err(_) => { + assert!(!reserve.resource_reused); + assert_eq!(reserve.job.request, followup); + store + .finish_backend_job_attempt( + "resource-workspace", + &followup.job_id, + &reserve.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + let retried = store + .reserve_backend_job_retry( + "resource-workspace", + &request.job_id, + RESOURCE_JOB_NOW, + ) + .unwrap(); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &retried.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + } + assert!( + store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .is_err(), + "retry limit cannot be reset by new candidates" + ); + assert!( + store + .find_active_backend_job_for_resource("resource-workspace", "custom-resource") + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn backend_job_all_older_attempt_cleanups_fence_retry_and_resource_release() { + let store = SqliteWorkspaceStore::in_memory().unwrap(); + seed_resource_job_workspace(&store).await; + let mut request = resource_job_request("legacy-overlap", "subject-a", "candidate-1"); + request.limits.max_attempts = 3; + let first = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + bind_resource_job_worker(&store, &first, "old-worker"); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + BackendJobAttemptState::Failed, + "partial_apply", + "some candidate effects durably applied", + RESOURCE_JOB_NOW, + ) + .unwrap(); + // Model a pre-fence durable journal: the old runner permitted attempt 2 + // before attempt 1's Worker cleanup. Current reservation must fail closed + // on this inherited state, even after the current attempt is cleaned. + let second_id = attempt_id(&request.job_id, 2); + store.with_conn(|conn| { + conn.execute("UPDATE backend_jobs SET state = 'pending', current_attempt = 2, completed_at = NULL WHERE workspace_id = 'resource-workspace' AND job_id = ?1", params![request.job_id])?; + conn.execute("INSERT INTO backend_job_attempts (workspace_id, job_id, attempt_id, attempt, input_revision, state, deadline_at, created_at, updated_at) VALUES ('resource-workspace', ?1, ?2, 2, ?3, 'reserved', ?4, ?4, ?4)", + params![request.job_id, second_id, request.input_revision, RESOURCE_JOB_NOW])?; + Ok(()) + }).unwrap(); + let second = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + assert_eq!(second.attempt.attempt, 2); + bind_resource_job_worker(&store, &second, "current-worker"); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &second.attempt.attempt_id, + BackendJobAttemptState::Failed, + "partial_apply", + "remaining candidate effects not yet applied", + RESOURCE_JOB_NOW, + ) + .unwrap(); + finish_resource_job_cleanup(&store, &second); + let followup = resource_job_request("new-batch", "subject-a", "candidate-2"); + assert!( + store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .is_err(), + "completed current cleanup must not hide incomplete attempt 1" + ); + assert!( + store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused + ); + assert_eq!( + store + .find_active_backend_job_for_resource( + "resource-workspace", + &request.resource_key().unwrap() + ) + .unwrap() + .unwrap() + .current_attempt, + 2 + ); + finish_resource_job_cleanup(&store, &first); + let third = store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .unwrap(); + assert_eq!( + third.job.request, request, + "partial apply retry keeps its original snapshot and grants" + ); + assert_eq!(third.attempt.attempt, 3); + let worker = bind_resource_job_worker(&store, &third, "converged-worker"); + let submission = BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: third.attempt.attempt_id.clone(), + input_revision: request.input_revision.clone(), + result: serde_json::json!({"converged": true}), + }; + store + .accept_backend_job_result("resource-workspace", &worker, &submission, RESOURCE_JOB_NOW) + .unwrap(); + finish_resource_job_cleanup(&store, &third); + // A legacy incomplete older cleanup also fences a completed current Job. + store.with_conn(|conn| { + conn.execute("UPDATE backend_job_attempts SET worker_cleanup_state = 'failed', worker_cleanup_completed_at = NULL WHERE workspace_id = 'resource-workspace' AND attempt_id = ?1", params![first.attempt.attempt_id])?; + Ok(()) + }).unwrap(); + assert!( + store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused, + "completed Job and current cleanup must not release an older unremoved Worker" + ); + finish_resource_job_cleanup(&store, &first); + assert!( + !store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused + ); + let replay = store + .accept_backend_job_result("resource-workspace", &worker, &submission, RESOURCE_JOB_NOW) + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.job.result, Some(submission.result)); + assert!( + store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .is_err(), + "converged intent cannot be reopened or exceed its bounded attempts" + ); + } + + #[tokio::test] + async fn backend_job_empty_batch_surface_refresh_locks_subject_and_defers_candidates() { + let store = SqliteWorkspaceStore::in_memory().unwrap(); + seed_resource_job_workspace(&store).await; + for surface_state in ["ungenerated", "stale", "failed"] { + let subject = format!("subject-{surface_state}"); + let mut refresh = resource_job_request( + &format!("surface-{surface_state}"), + &subject, + "surface-revision", + ); + refresh + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids + .clear(); + refresh.input = serde_json::json!({"subject_id": subject, "candidate_ids": [], "surface_state": surface_state}); + refresh.instruction = + "Regenerate only the Subject surface; this batch grants no candidate access." + .into(); + let first = store + .reserve_backend_job("resource-workspace", &refresh, RESOURCE_JOB_NOW) + .unwrap(); + assert_eq!(first.job.request, refresh); + assert!( + store + .reserve_backend_job("resource-workspace", &refresh, RESOURCE_JOB_NOW) + .unwrap() + .replayed + ); + let worker = bind_resource_job_worker( + &store, + &first, + &format!("surface-worker-{surface_state}"), + ); + let active = store + .get_active_backend_job_for_worker("resource-workspace", &worker) + .unwrap() + .unwrap(); + let grant = active + .request + .grants + .subjektiv_consolidation + .as_ref() + .unwrap(); + assert_eq!(grant.subject_id, subject); + assert!( + grant.candidate_ids.is_empty(), + "surface-only Job grants no candidate capability" + ); + assert!( + active + .request + .worker_input(&first.attempt.attempt_id) + .unwrap() + .contains("Regenerate only the Subject surface") + ); + assert_eq!( + store + .find_active_backend_job_for_resource( + "resource-workspace", + &refresh.resource_key().unwrap() + ) + .unwrap() + .unwrap() + .request, + refresh + ); + let followup = resource_job_request( + &format!("candidates-{surface_state}"), + &subject, + "new-candidate", + ); + let reused = store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap(); + assert!(reused.resource_reused && !reused.replayed); + assert_eq!( + reused.job.request, refresh, + "later candidates must not expand surface-only authority" + ); + store + .accept_backend_job_result( + "resource-workspace", + &worker, + &BackendJobResultSubmission { + job_id: refresh.job_id.clone(), + attempt_id: first.attempt.attempt_id.clone(), + input_revision: refresh.input_revision.clone(), + result: serde_json::json!({"surface": {"availability": "available"}}), + }, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!( + store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused, + "completed surface refresh still fences the subject until Worker cleanup" + ); + finish_resource_job_cleanup(&store, &first); + let next = store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap(); + assert!(!next.resource_reused); + assert_eq!(next.job.request, followup); + assert_eq!( + store + .get_backend_job("resource-workspace", &refresh.job_id) + .unwrap() + .unwrap() + .request, + refresh + ); + } + } + + #[tokio::test] + async fn backend_job_completed_resource_releases_only_after_worker_cleanup() { + let store = SqliteWorkspaceStore::in_memory().unwrap(); + seed_resource_job_workspace(&store).await; + let request = resource_job_request("completed", "subject-a", "candidate-1"); + let first = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + let worker = bind_resource_job_worker(&store, &first, "completed-worker"); + store + .accept_backend_job_result( + "resource-workspace", + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: first.attempt.attempt_id.clone(), + input_revision: request.input_revision.clone(), + result: serde_json::json!({"done": true}), + }, + RESOURCE_JOB_NOW, + ) + .unwrap(); + let followup = resource_job_request("new-candidates", "subject-a", "candidate-2"); + assert!( + store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused + ); + finish_resource_job_cleanup(&store, &first); + assert!( + store + .find_active_backend_job_for_resource( + "resource-workspace", + &request.resource_key().unwrap() + ) + .unwrap() + .is_none() + ); + let next = store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap(); + assert!(!next.resource_reused); + assert_eq!(next.job.request, followup); + assert_eq!( + store + .get_backend_job("resource-workspace", &request.job_id) + .unwrap() + .unwrap() + .request, + request + ); + } + #[tokio::test] async fn backend_jobs_fence_results_project_ownership_and_bound_retries() { let store = SqliteWorkspaceStore::in_memory().unwrap(); @@ -22765,6 +23709,8 @@ INSERT INTO worker_registry ( input: serde_json::json!({"title": "check"}), instruction: "Check the immutable input.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: Some(source.clone()), notification_target: Some(source.clone()), limits: crate::backend_job::BackendJobLimits { @@ -23185,6 +24131,35 @@ INSERT INTO worker_registry ( ) .is_err() ); + assert!( + store + .reserve_backend_job_retry( + workspace_id, + &retry_request.job_id, + "2026-09-01T00:01:04Z" + ) + .is_err(), + "failed attempt cannot retry before canonical Worker cleanup" + ); + store + .claim_backend_job_worker_cleanup( + workspace_id, + &retry_request.job_id, + &failed.attempt.attempt_id, + "2026-09-01T00:01:04Z", + ) + .unwrap(); + store + .finish_backend_job_worker_cleanup( + workspace_id, + &retry_request.job_id, + &failed.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + None, + None, + "2026-09-01T00:01:04Z", + ) + .unwrap(); let second = store .reserve_backend_job_retry(workspace_id, &retry_request.job_id, "2026-09-01T00:01:04Z") .unwrap(); diff --git a/crates/workspace-server/src/subjektiv.rs b/crates/workspace-server/src/subjektiv.rs index 7172f8736..83d4cfe63 100644 --- a/crates/workspace-server/src/subjektiv.rs +++ b/crates/workspace-server/src/subjektiv.rs @@ -1021,6 +1021,11 @@ ALTER TABLE subjects ADD COLUMN behavior_revision INTEGER NOT NULL DEFAULT 0 Ok(()) } +fn add_surface_job_provenance(transaction: &Transaction<'_>) -> crate::feature_storage::Result<()> { + transaction.execute_batch("ALTER TABLE surface_generation_runs ADD COLUMN job_id TEXT; ALTER TABLE surface_generation_runs ADD COLUMN attempt_id TEXT;")?; + Ok(()) +} + static MIGRATIONS: &[FeatureMigration] = &[ FeatureMigration::new(1, "create subjektiv subject memory store", create_schema), FeatureMigration::new( @@ -1039,6 +1044,11 @@ static MIGRATIONS: &[FeatureMigration] = &[ add_surface_generation_state, ), FeatureMigration::new(5, "add user-managed subject behavior", add_subject_behavior), + FeatureMigration::new( + 6, + "bind surface generation provenance to Job attempts", + add_surface_job_provenance, + ), ]; pub const REGISTRATION: FeatureRegistration = @@ -1958,6 +1968,14 @@ impl SubjektivStore { /// active Memory revisions. Kind order is fixed and selection is round-robin /// across kinds; within each kind, updated_at desc then memory id asc. pub fn prepare_surface_generation(&self, subject_id: &str) -> Result { + self.prepare_job_surface_generation(subject_id, None) + } + + pub(crate) fn prepare_job_surface_generation( + &self, + subject_id: &str, + job_attempt: Option<(&str, &str)>, + ) -> Result { self.database.try_transaction(|transaction| { let subject = require_active_subject(transaction, subject_id)?; let (active_memory_count, materials) = @@ -1973,21 +1991,71 @@ impl SubjektivStore { transaction.execute( "INSERT INTO surface_generation_runs ( subject_id, generation_id, store_revision, active_memory_count, - materials_json, created_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + materials_json, created_at, job_id, attempt_id + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", params![ subject_id, generation.id, to_i64(generation.store_revision)?, to_i64(generation.active_memory_count as u64)?, serde_json::to_string(&generation.materials)?, - generation.created_at + generation.created_at, + job_attempt.map(|(job_id, _)| job_id), + job_attempt.map(|(_, attempt_id)| attempt_id), ], )?; Ok(generation) }) } + /// Validate a Job's structured surface claim against the existing generation + /// and publication/failure records. This does not create another receipt or + /// roll back Memory. The generation must have begun inside this attempt. + #[allow(clippy::too_many_arguments)] + pub(crate) fn validate_job_surface_outcome( + &self, + subject_id: &str, + generation_id: &str, + store_revision: u64, + availability: &str, + snapshot_id: Option<&str>, + reason_code: Option<&str>, + job_id: &str, + attempt_id: &str, + ) -> Result<()> { + self.database.try_with_connection(|connection| { + let subject = require_subject_in_connection(connection, subject_id)?; + let generation_revision = require_job_generation(connection, subject_id, generation_id, job_id, attempt_id)?; + if generation_revision != store_revision || subject.store_revision != store_revision { + return Err(SubjektivError::InvalidRecord("surface generation is stale".into())); + } + let state = connection.query_row( + "SELECT store_revision, status, snapshot_id, reason_code FROM surface_generation_state WHERE subject_id = ?1", + [subject_id], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Option>(2)?, row.get::<_, Option>(3)?)), + ).optional()?.ok_or_else(|| SubjektivError::InvalidRecord("surface generation has no terminal outcome".into()))?; + if state.0 != to_i64(store_revision)? || state.1 != availability + || !matches!(availability, "ready" | "failed") + || state.2.as_deref() != snapshot_id || state.3.as_deref() != reason_code { + return Err(SubjektivError::InvalidRecord("surface claim differs from the persisted outcome".into())); + } + Ok(()) + }) + } + + pub(crate) fn require_job_surface_generation( + &self, + subject_id: &str, + generation_id: &str, + job_id: &str, + attempt_id: &str, + ) -> Result<()> { + self.database.try_with_connection(|connection| { + require_job_generation(connection, subject_id, generation_id, job_id, attempt_id)?; + Ok(()) + }) + } + /// Validates structured editor output against the exact material snapshot and /// atomically publishes it only while the subject store generation is current. pub fn publish_surface_generation( @@ -3605,6 +3673,26 @@ fn reject_duplicate_memory_ids(kind: &str, values: &[MemoryRevisionRef]) -> Resu Ok(()) } +fn require_job_generation( + connection: &Connection, + subject_id: &str, + generation_id: &str, + job_id: &str, + attempt_id: &str, +) -> Result { + let revision: Option = connection.query_row( + "SELECT store_revision FROM surface_generation_runs WHERE subject_id = ?1 AND generation_id = ?2 AND job_id = ?3 AND attempt_id = ?4", + params![subject_id, generation_id, job_id, attempt_id], |row| row.get(0), + ).optional()?; + revision + .and_then(|value| u64::try_from(value).ok()) + .ok_or_else(|| { + SubjektivError::InvalidRecord( + "surface generation is not owned by this Job attempt".into(), + ) + }) +} + fn to_i64(value: u64) -> Result { i64::try_from(value).map_err(|_| { SubjektivError::InvalidRecord(format!("revision value {value} exceeds SQLite range")) diff --git a/crates/workspace-server/src/ticket_item_checker.rs b/crates/workspace-server/src/ticket_item_checker.rs index 5e7178590..af592f9cc 100644 --- a/crates/workspace-server/src/ticket_item_checker.rs +++ b/crates/workspace-server/src/ticket_item_checker.rs @@ -129,6 +129,8 @@ pub fn request( })?, instruction, profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: Some(source_worker.clone()), notification_target: Some(source_worker), limits: BackendJobLimits { diff --git a/docs/design/durable-operations.md b/docs/design/durable-operations.md index 08616d3de..053d4b676 100644 --- a/docs/design/durable-operations.md +++ b/docs/design/durable-operations.md @@ -330,9 +330,30 @@ Exact dispatch retry reuses the deterministic attempt and Worker allocation key. Only explicit re-evaluation of a definitively failed attempt increments the bounded attempt number; unknown outcomes are not replayable. The runner uses the existing embedded Runtime spawn, Worker registry, Session input, -observation, usage, and lifecycle paths with no Workdir attachment. It uses the -dedicated `builtin:backend-job` profile, whose tools cannot mutate Workspace -state, recursively create Workers or Jobs, or inherit caller Workdir authority. +observation, usage, and lifecycle paths with no Workdir attachment. Each client +explicitly selects an existing registry Profile. The runner resolves the complete +Builtin/Workspace project catalog through the normal virtual config bundle and +prompt projection path; browser launch candidates remain a narrower UI policy. +Raw Profile source and filesystem/path selectors are not accepted. A selected +recipe's instruction and features are not replaced with `backend-job` defaults. +Profiles requiring ungranted Workspace mutation, ordinary Memory, child/peer, +Flow, Workdir or MCP authority fail explicitly; no caller scope is inherited. +The Ticket checker continues to select `builtin:backend-job` itself. + +Job identity/result capability comes from an immutable Host-authored create and +restore binding, not the Profile name, tag, instruction, Worker display name or +Runtime placement. Backend checks the exact live Job/attempt/input revision and +authenticated Worker before accepting results or domain operations. Profile +policy alone never confers a subject grant. + +A common optional resource key serializes Jobs using their existing durable +intent/attempt records (schema 81), not a second domain workflow table. Exact +request replay is distinct from a resource collision: the latter returns the +original active immutable request with `resource_reused`, and never claims that +new input was processed. Pending/unknown Jobs and **all** unfinished attempt +cleanups retain the fence. Retry requires definitive failure, remaining attempt +budget, completed prior cleanups and no competing resource owner. Unknown intent +remains attention-required even after its Worker is removed. Success is accepted only by the structured result boundary. Before Runtime input can execute, the attempt is fenced by Workspace, Job ID, attempt ID, input diff --git a/docs/design/subjektiv-consolidation-jobs.md b/docs/design/subjektiv-consolidation-jobs.md new file mode 100644 index 000000000..da7e20e82 --- /dev/null +++ b/docs/design/subjektiv-consolidation-jobs.md @@ -0,0 +1,97 @@ +# Backend subjektiv consolidation Jobs (T-704) + +subjektiv requests the common Backend Job runner with an explicit +`builtin:subjektiv-memory-consolidation` Profile. The runner has no purpose-based +Profile switch. Subject-body Workers and the independent Workspace Memory feature +are not Jobs and are not migrated or reset here. The shared/standalone adapters +planned in T-708/T-709 are outside this Backend change. + +## Input, grants and concurrency + +The existing backlog/byte thresholds and missing/stale/failed surface policy +remain the launch decision. A Job captures an immutable bounded batch (up to 100 +candidate IDs) plus the Subject store revision. An empty batch is valid for a +surface-only rebuild. The Backend explicitly binds the Subject and candidate +batch grant in durable Job intent before Worker execution. No subject-body +singleton ownership or Session attribution is transferred to the Job. + +Authenticated `(Workspace, Runtime, Worker)` registry binding and the exact +current dispatched attempt authorize each operation. Names, Profile tags and +model-provided `subject_id` are not authority. Candidate listing exposes only +unresolved IDs in the captured batch; reads/decisions outside it are denied. +Surface prepare requires all batch candidates to have durable dispositions. +Surface generation records preserve immutable Job/attempt provenance (subjektiv +schema migration 6); publication, failure and completion reject another attempt's +generation, including pre-Job/legacy generations. + +The common resource key `subjektiv-consolidation:` prevents parallel +Jobs for a Subject. Concurrent requests return the original active intent and +report `skipped_existing_job`; they do not expand its grants. New candidates and +items beyond the bounded batch stay in staging. A subsequent consolidation +request snapshots remaining work; it is not counted as processed by the first +Job. There is no domain-specific status ledger, pool or persistent idle Worker. +Definitive failures use the common bounded retry API with the original intent; +unknown execution stays fenced for attention. A later explicitly requested Job +may snapshot remaining candidates once all previous Workers are safely cleaned. + +## Memory and surface completion + +`MemoryApplyCandidate` still uses the existing expected-revision checks, atomic +apply/close and immutable decision receipts. Partial application survives model +failure, timeout, lost response, retry, Worker deletion and Backend recovery. +Job replay is not a second memory apply: resolved candidates retain their +existing dispositions, and exact candidate decisions converge on their receipts. + +`SubmitBackendJobResult` for a bound consolidation Job receives only the Subject +and captured candidate IDs. Its Host capability awaits the existing bounded +clean-context surface editor and publish-or-recorded-failure, then adds the actual +structured surface outcome. It does **not** schedule a Worker-owned post-Run task. +The result call serializes and caches the exact generated outcome for ambiguous +submission replay. The Backend verifies every batch disposition and the current +store revision, generation ownership and persisted ready/failed surface fields; +it enriches the durable result with actual immutable candidate dispositions. +Thus confirmed-memory consolidation and surface readiness are separate outcomes. +A recorded surface failure does not undo Memory. A stale/unfinished/fabricated +surface cannot be reported completed. Final prose or Idle is never success. + +Only after durable structured result acceptance does the existing terminal Job +cleanup run. Its tracked Run teardown fence lets the result tool finish before +stop/remove. Timeout/unknown paths cancel unfinished Worker execution through +that same cleanup; later operations/results are rejected. The next request can +regenerate failed/stale surfaces without touching confirmed memory. + +## Legacy cutover and recovery + +The subject-specific direct spawn and Idle resubmission paths are removed. +On a consolidation request the Backend checks the **persisted singleton owner** +for the old `subjektiv-consolidation:` key. It revalidates Workspace, +key and old dedicated recipe provenance, not the display name. A running/paused +legacy foreground operation is left alone and returns `skipped_legacy_inflight`; +no Job is launched alongside it. Legacy owned operations can finish during this +bounded cutover. + +After foreground Idle (or Stopped) is confirmed, Backend stop is confirmed before +canonical WorkerRemovalService removal. Any legacy surface background task is +cancelled at stop; the subsequent Job rebuilds from confirmed Memory and does not +infer successful surface completion from Idle. Busy/unobservable/mismatched +owners fail closed. Empty/ready Subjects also retire their idle legacy owner. +Removal failures remain diagnosable through existing durable removal recovery; +a subsequent request rechecks ownership instead of deleting storage directly. +Subject-body/ordinary/Workspace-Memory Workers are never selected by this key. + +The Job runner uses its existing durable dispatch allocation, tracked input, +T-701 registry-bound restore/recovery, deadlines, unknown outcomes, result replay, +delivery and T-690 cleanup. Job results, candidate receipts, Memory revisions, +provenance and surfaces remain in their existing Backend stores after Worker +removal. No live dogfood Worker or environment is stopped/updated by development +validation. + +## Validation boundary + +Server recording-backend fixtures prove reservation/binding/auth/result/cleanup +contracts, and existing store tests prove candidate CAS/atomic receipts and +surface freshness. Worker scripted-client tests prove the clean-context editor +completion-before-result boundary and result replay. They are not real-provider +LLM or live dogfood process evidence. See the T-704 validation report for exact +commands/results and inherited intermittent failures; later passes do not erase +those failures. diff --git a/docs/design/subjektiv-store.md b/docs/design/subjektiv-store.md index f9b1c2a00..fdce5859d 100644 --- a/docs/design/subjektiv-store.md +++ b/docs/design/subjektiv-store.md @@ -289,11 +289,15 @@ active/resolved→retracted, and resolved→active. Retraction remains terminal. ## Candidate consolidation and corrections (T-670) -The Backend starts a restricted `builtin:subjektiv-memory-consolidation` Worker -for exactly one authenticated subject. The Host binds the subject; model-visible -inputs cannot select a Workspace, subject, Runtime, Worker, or Session. Its tools -list/read pending immutable candidates, query/read confirmed revisions, and make -one candidate decision. Candidate text is evidence rather than authority. +subjektiv requests a bounded Backend-owned Job with the explicit +`builtin:subjektiv-memory-consolidation` Profile and an immutable Subject/candidate +batch grant. The common runner resolves the recipe; Profile names do not grant +permissions. The Host derives authority from the authenticated current +Job/attempt/Worker binding; model-visible inputs cannot select a Workspace, +subject, Runtime, Worker, or Session. Its tools list/read only pending candidates +in that batch, query/read confirmed revisions, and make one candidate decision. +Candidate text is evidence rather than authority. No subject-specific idle Worker +is resubmitted; see [Job execution and cutover](subjektiv-consolidation-jobs.md). Every decision carries a stable request ID. An exact retry returns the committed receipt, while reuse with different input is a conflict. Applying a candidate diff --git a/docs/report/2026-10-06-t704-validation.md b/docs/report/2026-10-06-t704-validation.md new file mode 100644 index 000000000..74bb97d40 --- /dev/null +++ b/docs/report/2026-10-06-t704-validation.md @@ -0,0 +1,90 @@ +# T-704 implementation and validation ledger + +Base: `6f9621ca` (clean detached assigned checkout). Parent created +`work/t-704-backend-subjektiv-jobs` without replacing a branch or discarding changes. +Parent owns Backend/Profile/Subject authorization, cutover, documentation and +integration. Two direct implementation SubWorkers supplied the common durable +Job/store and Worker/Runtime contracts; parent inspected their committed sessions, +diffs and test evidence, integrated them and corrected the boundaries below. +Independent MR review is separate from implementation delegation. + +## Implemented contracts + +- Caller-selected existing Builtin/project registry Profiles, normal virtual + config/prompt bundle resolution and explicit execution-requirement rejection. + Chosen instructions/features/model are not replaced with backend-job defaults. +- Immutable Host-authored Job/attempt create/restore binding and result capability, + independent of Profile names/instructions/tags. No inherited Workdir, parent or + subject-body execution authority. +- Common transactional resource serialization (Workspace schema 81), immutable + Subject/candidate batch grants, exact replay vs resource collision, unknown + fences and all-old-attempt cleanup fences before retry/followup. +- subjektiv requests its Profile; immutable bounded batches, empty surface-only + Jobs, candidate authorization, existing CAS/atomic receipts and dispositions. +- Synchronous bounded clean-context surface generation before result submission. + Preparation reports an already-current snapshot to avoid needless conflicting + regeneration. Job failure acknowledgements are `failed_confirmed` only after + checking the actual failed marker/reason/revision and generation provenance. + Subject feature migration 6 adds immutable generation Job/attempt provenance, + without changing Subject/Memory/candidate content. +- Durable results contain actual candidate dispositions and a separate verified + ready/failed surface outcome. Existing terminal cleanup retains results/memory. +- Removed subject direct spawn and Idle resubmission. Persisted legacy ownership + plus actual foreground WorkerState and confirmed stop/removal fence the cutover. + Busy legacy workers are not interrupted or duplicated; subject bodies and the + independent old Workspace Memory feature are not selected for removal. + +## Final validation (bounded `CARGO_BUILD_JOBS=2`) + +| Command/boundary | Result | +| --- | --- | +| `cargo test -p yoi-workspace-server --lib backend_job` | 33 passed (before final surface acknowledgement addition; final whole-crate run below covers it) | +| `cargo test -p yoi-workspace-server --lib subjektiv_jobs_tests` | 5 passed, including project recipe, batch/auth/receipts, actual failed acknowledgement, current surface and legacy cutover | +| `cargo test -p worker --lib memory_surface_lifecycle` | 16 passed, including publication/failure before result, cancellation, definitive correction, ambiguous exact retry and already-current surface | +| Root `cargo check` | Passed; normal TUI/Server/standalone compile closure | +| `cargo test -p worker -p worker-runtime -p manifest -p runtime-api -p server-api -- --test-threads=4` | Passed: manifest 185; runtime-api 9; server-api 70; worker 815 lib + 110 integration; runtime 335 lib + 34 binary tests | +| `cargo test -p yoi-workspace-server -- --test-threads=4` (final) | 637 lib + 11 binary passed, no skips | +| `cargo test -p server-api --features typescript` | 77 passed, including generated/OpenAPI/TypeScript contract freshness; existing ts-rs serde-attribute warnings | +| `cargo fmt --all -- --check` / `git diff --check HEAD` | Passed | + +These are recording-backend/store and scripted-client fixtures, plus real compiler +and test binaries. They are **not** real-provider LLM execution, a live Backend / +Runtime deployment, live dogfood consolidation or an environment update. No live +Worker/environment was stopped or updated. + +## Failed checks and fixes (not erased by later passes) + +- During integration, new request fields, resource replay metadata and surface + validation arguments initially failed compilation; all were corrected before + final validation. +- Initial Backend tests exposed the narrower browser builtin candidate allowlist; + Host launch now resolves the complete existing registry without widening browser + choices or adding a purpose-specific Profile allowlist. +- Initial surface-only test exposed rejection of empty candidate grants; empty + batches now retain the subject lock and grant no candidate access (owned store + tests 9 + contract tests 9 passed). +- Legacy fixture initially compared catalog compatibility state rather than + actual WorkerState. Cutover now requires the actual Idle snapshot, and the test + synchronizes on that authoritative observation before asserting behavior. +- First full Server run: **635 passed / 2 failed**, both stale migration-history + fixtures expecting schema 80. Their explicit retained chain/history is updated + to 81; migration filter passed 14 tests and subsequent whole-crate runs passed. +- Exploratory `cargo check -p worker-runtime --no-default-features` and + `--no-default-features --features fs-store` both **failed**, with 16 errors from + existing unconditional `workspace_request` / `reqwest` imports while HTTP is + disabled (plus unused-import warnings). The source/gates are present in the + original baseline and are recorded in the + [T-701 ledger](2026-10-06-t701-validation.md#validation-results). No isolated + baseline build was performed here. These remain failed exploratory unsupported + configurations, not feature passes or a T-704 feature-gating fix. The default + Runtime fs-store/ws-server configuration is fully tested. +- The inherited intermittent Ticket checker failure described in the + [T-702 ledger](2026-10-06-t702-stale-checker-validation.md) and T-701 report was + not observed in the final T-704 full Server runs, but is **not** claimed fixed. + +Raw parent logs are under `target/t704-validation/` (ignored local artifacts), +including `server-full-initial-migration-fixtures.log`, `server-full-final.log`, +`changed-crates.log`, `root-check.log`, `worker-surface-final.log`, +`server-subjektiv-final.log`, `server-api-typescript.log`, `runtime-no-default.log` +and `runtime-fs-only.log`. The durable report records their concrete results; +local log paths are not a substitute for the published immutable MR source. diff --git a/resources/profiles/subjektiv-memory-consolidation.dcdl b/resources/profiles/subjektiv-memory-consolidation.dcdl index b3c9ec16d..3549c4486 100644 --- a/resources/profiles/subjektiv-memory-consolidation.dcdl +++ b/resources/profiles/subjektiv-memory-consolidation.dcdl @@ -3,13 +3,13 @@ import "./base.dcdl" // { description = "Delegated subject Memory staging consolidation profile."; scope = "workspace_read"; model = { ref = "codex-oauth/gpt-5.6-luna"; }; - engine = { reasoning = "medium"; }; + engine = { reasoning = "medium"; instruction = "internal.subjektiv_memory_consolidation_system"; }; web = { enabled = false; }; feature = { task = { enabled = false; }; memory = { enabled = false; staging_tools = false; }; - subjektiv = { enabled = true; extraction = { enabled = false; }; }; + subjektiv = { enabled = true; consolidation_tools = true; extraction = { enabled = false; }; }; web = { enabled = false; }; image = { enabled = false; }; sub_worker = { enabled = false; }; diff --git a/resources/prompts/internal/subjektiv_memory_consolidation_system.md b/resources/prompts/internal/subjektiv_memory_consolidation_system.md index 7cb150dc7..2ca32f404 100644 --- a/resources/prompts/internal/subjektiv_memory_consolidation_system.md +++ b/resources/prompts/internal/subjektiv_memory_consolidation_system.md @@ -1,6 +1,6 @@ # Memory staging consolidater -You are the delegated internal Memory staging consolidater for one subject. +You are the Backend Job Memory staging consolidater for one subject. Your job is to consume that subject's pending Memory candidates through tools. The host binds every tool call to the subject; never infer or provide a subject identifier yourself. @@ -26,7 +26,10 @@ Rules: - `source_candidate_ids` are host-derived from the applied candidate. Do not attempt to author them. - `derived_from` must contain only exact existing Memory revision refs returned by reads. - Applied responses return affected refs with `{memory_id, revision, operation}`. `duplicate` and `already_covered` may reference exact existing revisions; `discarded` and `invalid` require a concrete reason. -- Surface generation is a separate clean-context Host lifecycle that runs once after this consolidation turn commits. `surface_dirty: true` means confirmed Memory changed; never treat later surface generation failure as failure of an already returned committed decision. +- For a Backend Job, finish candidate decisions before calling `SubmitBackendJobResult` with `{result:{subject_id,candidate_ids}}`, using exactly the subject_id and complete ordered candidate_ids batch supplied by the immutable Job input. Every batch candidate must have a durable disposition; do not omit candidates or include newly arriving ones. Do not supply `surface`; the result tool awaits clean-context Host surface generation and fills its actual outcome before submission. Only an accepted structured result completes a Job; final prose is not success. +- `surface_dirty: true` means confirmed Memory changed. Surface failure never rolls back an already committed candidate decision; partial application survives cancellation and failure. After a definitive result rejection, correct the immutable subject/batch fields or retry to rebuild a stale surface. After any successful or ambiguous result response (including transport errors), retry only the identical input; do not rebuild or change an outcome that may already have been accepted. +- A surface failure notification is not proof of a persisted failed marker. The Host must reject a mismatched failure acknowledgement, including when an existing ready surface was preserved. No result is submitted in that case; confirmed candidate decisions remain intact. +- Legacy non-Job consolidation retains its separate post-commit Host surface lifecycle. - An empty model response or transport failure is not a candidate disposition. Only a successful `MemoryApplyCandidate` response closes a candidate. Valid non-applied close actions are `discarded`, `invalid`, `duplicate`, and `already_covered`. Always provide a specific reason.