feat: caller-selected Backend Jobs and bounded subjektiv consolidation (T-704)
This commit is contained in:
@@ -288,6 +288,8 @@ pub struct SubjektivFeatureConfigPartial {
|
||||
#[serde(default)]
|
||||
pub enabled: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub consolidation_tools: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub extraction: Option<MemoryExtractionProfileConfigPartial>,
|
||||
}
|
||||
|
||||
@@ -295,6 +297,7 @@ impl SubjektivFeatureConfigPartial {
|
||||
fn merge(self, other: Self) -> Self {
|
||||
Self {
|
||||
enabled: other.enabled.or(self.enabled),
|
||||
consolidation_tools: other.consolidation_tools.or(self.consolidation_tools),
|
||||
extraction: merge_option(
|
||||
self.extraction,
|
||||
other.extraction,
|
||||
@@ -544,6 +547,7 @@ impl From<SubjektivFeatureConfigPartial> for ResolvedSubjektivFeatureConfig {
|
||||
Self {
|
||||
profile: SubjektivFeatureProfileConfig {
|
||||
enabled: value.enabled.unwrap_or_default(),
|
||||
consolidation_tools: value.consolidation_tools.unwrap_or_default(),
|
||||
extraction: MemoryExtractionProfileConfig {
|
||||
enabled: extraction.enabled.unwrap_or(true),
|
||||
model: extraction.model,
|
||||
@@ -563,6 +567,7 @@ impl From<ResolvedSubjektivFeatureConfig> for SubjektivFeatureConfigPartial {
|
||||
fn from(value: ResolvedSubjektivFeatureConfig) -> Self {
|
||||
Self {
|
||||
enabled: Some(value.profile.enabled),
|
||||
consolidation_tools: Some(value.profile.consolidation_tools),
|
||||
extraction: Some(MemoryExtractionProfileConfigPartial {
|
||||
enabled: Some(value.profile.extraction.enabled),
|
||||
model: value.profile.extraction.model,
|
||||
|
||||
@@ -402,6 +402,8 @@ impl ResolvedMemoryFeatureConfig {
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub struct SubjektivFeatureProfileConfig {
|
||||
pub enabled: bool,
|
||||
/// Explicit consolidation tool policy; never grants Job or subject authority.
|
||||
pub consolidation_tools: bool,
|
||||
pub extraction: MemoryExtractionProfileConfig,
|
||||
}
|
||||
|
||||
@@ -409,6 +411,7 @@ impl Default for SubjektivFeatureProfileConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
consolidation_tools: false,
|
||||
extraction: MemoryExtractionProfileConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -420,6 +420,18 @@ pub struct WorkerInput {
|
||||
pub segments: Option<Vec<Segment>>,
|
||||
}
|
||||
|
||||
/// Host-authored Job/attempt identity, independent of Profile selection.
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobExecutionBinding {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_revision: Option<String>,
|
||||
#[serde(default)]
|
||||
pub subjektiv_consolidation: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub struct CreateWorkerRequest {
|
||||
pub worker_id: WorkerId,
|
||||
@@ -448,6 +460,8 @@ pub struct CreateWorkerRequest {
|
||||
/// never activates subjektiv for an ordinary Workspace Worker.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub subjektiv_attached: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub backend_job: Option<BackendJobExecutionBinding>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
|
||||
@@ -5530,6 +5530,9 @@ pub struct SubjektivSurfacePrepareRequest {}
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjektivSurfacePrepareResponse {
|
||||
pub generation_id: String,
|
||||
/// Existing ready surface at this exact store revision, if no rebuild is needed.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub current_snapshot_id: Option<String>,
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||
pub store_revision: u64,
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))]
|
||||
|
||||
@@ -258,6 +258,8 @@ impl std::fmt::Debug for WorkspaceApiRef {
|
||||
}
|
||||
}
|
||||
|
||||
pub use worker::BackendJobExecutionBinding;
|
||||
|
||||
/// Canonical Runtime Worker creation request.
|
||||
///
|
||||
/// Browser/product launch semantics are resolved by a backend before this
|
||||
@@ -302,6 +304,9 @@ pub struct CreateWorkerRequest {
|
||||
/// may enable subjektiv policy, but only this host-owned bit activates it.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub subjektiv_attached: bool,
|
||||
/// Backend-only immutable Job/attempt capability. Never inferred from Profile policy.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub backend_job: Option<BackendJobExecutionBinding>,
|
||||
}
|
||||
|
||||
/// Last persisted Worker lifecycle status.
|
||||
|
||||
@@ -3129,6 +3129,7 @@ mod tests {
|
||||
language: "English".to_string(),
|
||||
}),
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4685,6 +4686,7 @@ mod ws_tests {
|
||||
language: "English".to_string(),
|
||||
}),
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -898,7 +898,9 @@ impl Runtime {
|
||||
message: "persisted Worker restore request is unavailable".to_string(),
|
||||
}
|
||||
})?;
|
||||
if existing_request.create_fingerprint != request.create_fingerprint {
|
||||
if existing_request.create_fingerprint != request.create_fingerprint
|
||||
|| existing_request.backend_job != request.backend_job
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(format!(
|
||||
"worker {} was already created with a different fingerprint",
|
||||
request.worker_id
|
||||
@@ -1068,7 +1070,9 @@ impl Runtime {
|
||||
message: "persisted Worker restore request is unavailable".to_string(),
|
||||
}
|
||||
})?;
|
||||
if existing_request.create_fingerprint != request.create_fingerprint {
|
||||
if existing_request.create_fingerprint != request.create_fingerprint
|
||||
|| existing_request.backend_job != request.backend_job
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(format!(
|
||||
"worker {} was already created with a different fingerprint",
|
||||
request.worker_id
|
||||
@@ -6242,6 +6246,16 @@ fn validate_logical_workdir_attachments(
|
||||
}
|
||||
|
||||
fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), RuntimeError> {
|
||||
if let Some(binding) = &request.backend_job {
|
||||
binding
|
||||
.validate()
|
||||
.map_err(|message| RuntimeError::InvalidRequest(message.into()))?;
|
||||
if request.workspace_api.is_none()
|
||||
|| (binding.subjektiv_consolidation && !request.subjektiv_attached)
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest("Backend Job capability requires Workspace authority and consolidation requires trusted subject attachment".into()));
|
||||
}
|
||||
}
|
||||
if request.create_fingerprint.trim().is_empty() {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"create_fingerprint must not be empty".to_string(),
|
||||
@@ -7112,9 +7126,95 @@ mod tests {
|
||||
language: "English".to_string(),
|
||||
}),
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn backend_job_binding_survives_restart_and_restore_without_profile_inference() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let options = crate::fs_store::FsRuntimeStoreOptions {
|
||||
root: dir.path().join("runtime"),
|
||||
runtime_id: "job-runtime".into(),
|
||||
display_name: None,
|
||||
};
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
let runtime =
|
||||
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let mut request = task_request("bound Job");
|
||||
request.workspace_api = Some(WorkspaceApiRef {
|
||||
workspace_id: "local".into(),
|
||||
base_url: "https://backend.invalid".into(),
|
||||
});
|
||||
request.backend_job = Some(crate::catalog::BackendJobExecutionBinding {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "attempt-1".into(),
|
||||
input_revision: Some("revision-1".into()),
|
||||
subjektiv_consolidation: false,
|
||||
});
|
||||
let owner = scope("local", "server-job-test");
|
||||
let created = runtime
|
||||
.create_worker_scoped(&owner, request.clone())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
backend.job_bindings.lock().unwrap().as_slice(),
|
||||
&[request.backend_job.clone()]
|
||||
);
|
||||
let mut changed = request.clone();
|
||||
changed.backend_job.as_mut().unwrap().attempt_id = "attempt-other".into();
|
||||
assert!(runtime.create_worker_scoped(&owner, changed).is_err());
|
||||
drop(runtime);
|
||||
let restored_backend = Arc::new(TestExecutionBackend::default());
|
||||
let restored =
|
||||
Runtime::with_fs_store_and_execution_backend(options, restored_backend.clone())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
restored
|
||||
.lock()
|
||||
.unwrap()
|
||||
.worker(&created.worker_ref)
|
||||
.unwrap()
|
||||
.request
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.backend_job,
|
||||
request.backend_job
|
||||
);
|
||||
restored
|
||||
.restore_worker(
|
||||
&created.worker_ref,
|
||||
restored.test_restore_request(&created.worker_ref),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
restored_backend.job_bindings.lock().unwrap().as_slice(),
|
||||
&[request.backend_job]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backend_job_binding_requires_workspace_and_consolidation_attachment() {
|
||||
let mut request = task_request("Job binding validation");
|
||||
request.backend_job = Some(crate::catalog::BackendJobExecutionBinding {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "attempt-1".into(),
|
||||
input_revision: None,
|
||||
subjektiv_consolidation: true,
|
||||
});
|
||||
assert!(validate_create_worker_request(&request).is_err());
|
||||
request.workspace_api = Some(WorkspaceApiRef {
|
||||
workspace_id: "local".into(),
|
||||
base_url: "https://backend.invalid".into(),
|
||||
});
|
||||
assert!(validate_create_worker_request(&request).is_err());
|
||||
request.subjektiv_attached = true;
|
||||
assert!(validate_create_worker_request(&request).is_ok());
|
||||
request.backend_job.as_mut().unwrap().attempt_id.clear();
|
||||
assert!(validate_create_worker_request(&request).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_worker_request_omits_repository_credentials() {
|
||||
let mut request = task_request("worker-secret-redaction");
|
||||
@@ -7537,6 +7637,7 @@ mod tests {
|
||||
#[derive(Default)]
|
||||
struct TestExecutionBackend {
|
||||
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
||||
job_bindings: Mutex<Vec<Option<crate::catalog::BackendJobExecutionBinding>>>,
|
||||
dispatch_result: Mutex<Option<WorkerExecutionResult>>,
|
||||
stop_result: Mutex<Option<WorkerExecutionResult>>,
|
||||
stop_gate: Mutex<Option<Arc<RestoreGate>>>,
|
||||
@@ -7653,6 +7754,10 @@ mod tests {
|
||||
}
|
||||
|
||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||
self.job_bindings
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(request.request.backend_job.clone());
|
||||
if let Some(result) = self.spawn_result.lock().unwrap().take() {
|
||||
return result;
|
||||
}
|
||||
@@ -7726,6 +7831,10 @@ mod tests {
|
||||
&self,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
self.job_bindings
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(request.request.backend_job.clone());
|
||||
*self.restore_count.lock().unwrap() += 1;
|
||||
self.restore_operation_ids
|
||||
.lock()
|
||||
|
||||
@@ -1086,10 +1086,26 @@ fn restored_workdir_router(
|
||||
Ok(router)
|
||||
}
|
||||
|
||||
fn validate_backend_job_profile(
|
||||
manifest: &manifest::WorkerManifest,
|
||||
request: &CreateWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
if let Some(binding) = &request.backend_job {
|
||||
let policy = &manifest.feature.subjektiv.profile;
|
||||
if policy.consolidation_tools != binding.subjektiv_consolidation
|
||||
|| (binding.subjektiv_consolidation && policy.extraction.enabled)
|
||||
{
|
||||
return Err("Backend Job consolidation_tools must match its trusted consolidation grant and extraction must be disabled".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bind_workspace_memory_settings(
|
||||
manifest: &mut manifest::WorkerManifest,
|
||||
request: &CreateWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
validate_backend_job_profile(manifest, request)?;
|
||||
let Some(snapshot) = request.memory_settings.as_ref() else {
|
||||
if request.workspace_api.is_some() || request.subjektiv_attached {
|
||||
return Err(
|
||||
@@ -1143,6 +1159,7 @@ fn validate_worker_memory_settings(
|
||||
manifest: &manifest::WorkerManifest,
|
||||
request: &CreateWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
validate_backend_job_profile(manifest, request)?;
|
||||
let Some(expected) = request.memory_settings.as_ref() else {
|
||||
if request.subjektiv_attached || request.workspace_api.is_some() {
|
||||
return Err(
|
||||
@@ -1419,6 +1436,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
}
|
||||
})?;
|
||||
let worker = prepared.worker_mut();
|
||||
if let Some(binding) = request.request.backend_job.as_ref() {
|
||||
worker
|
||||
.bind_backend_job(binding.clone())
|
||||
.map_err(str::to_string)?;
|
||||
}
|
||||
validate_worker_memory_settings(worker.manifest(), &request.request)?;
|
||||
worker
|
||||
.finalize_subjektiv_session_attribution(
|
||||
@@ -1684,6 +1706,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
}
|
||||
Err(err) => return Err(format!("failed to restore Worker from metadata: {err}")),
|
||||
};
|
||||
if let Some(binding) = request.request.backend_job.as_ref() {
|
||||
worker
|
||||
.bind_backend_job(binding.clone())
|
||||
.map_err(str::to_string)?;
|
||||
}
|
||||
validate_worker_memory_settings(worker.manifest(), &request.request)?;
|
||||
worker
|
||||
.finalize_subjektiv_session_attribution(
|
||||
@@ -7221,6 +7248,58 @@ mod tests {
|
||||
workspace_api: None,
|
||||
memory_settings: None,
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backend_job_profile_policy_matches_trusted_grant_without_implicit_changes() {
|
||||
let mut manifest = WorkerManifest::from_toml(
|
||||
r#"
|
||||
[worker]
|
||||
name = "job-policy-test"
|
||||
pwd = "./"
|
||||
[model]
|
||||
scheme = "anthropic"
|
||||
model_id = "chosen-model"
|
||||
auth = { kind = "none" }
|
||||
[engine]
|
||||
instruction = "default"
|
||||
[[scope.allow]]
|
||||
target = "./"
|
||||
permission = "read"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let mut request = create_request("policy");
|
||||
for granted in [false, true] {
|
||||
request.backend_job = Some(crate::catalog::BackendJobExecutionBinding {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "attempt-1".into(),
|
||||
input_revision: None,
|
||||
subjektiv_consolidation: granted,
|
||||
});
|
||||
for tools in [false, true] {
|
||||
for extraction in [false, true] {
|
||||
manifest.feature.subjektiv.profile.consolidation_tools = tools;
|
||||
manifest.feature.subjektiv.profile.extraction.enabled = extraction;
|
||||
let expected = tools == granted && (!granted || !extraction);
|
||||
let before = serde_json::to_value(&manifest).unwrap();
|
||||
assert_eq!(
|
||||
validate_backend_job_profile(&manifest, &request).is_ok(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
bind_workspace_memory_settings(&mut manifest, &request).is_ok(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
validate_worker_memory_settings(&manifest, &request).is_ok(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(serde_json::to_value(&manifest).unwrap(), before);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
//! Host-authored Job authority, separate from reusable Profile policy.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Immutable execution capability delivered only by a trusted Backend host.
|
||||
/// Names, instructions and Profiles never confer this capability.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobExecutionBinding {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_revision: Option<String>,
|
||||
#[serde(default)]
|
||||
pub subjektiv_consolidation: bool,
|
||||
}
|
||||
|
||||
impl BackendJobExecutionBinding {
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
if [&self.job_id, &self.attempt_id]
|
||||
.into_iter()
|
||||
.chain(self.input_revision.iter())
|
||||
.any(|value| value.trim().is_empty() || value.chars().any(char::is_control))
|
||||
{
|
||||
return Err("Backend Job binding contains an invalid identity or input revision");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1376,10 +1376,12 @@ where
|
||||
),
|
||||
None => durable_parent_notifications,
|
||||
};
|
||||
let backend_job_profile = worker.manifest().engine.instruction == "internal.backend_job_system";
|
||||
let backend_job = worker.backend_job().cloned();
|
||||
// Bounded Jobs complete their lifecycle synchronously, never by post-run background work.
|
||||
let lifecycle_enabled = worker.manifest_lifecycle_features_enabled() && backend_job.is_none();
|
||||
let prompts = worker.prompts().clone();
|
||||
let tracker = tools::Tracker::new();
|
||||
if !backend_job_profile {
|
||||
{
|
||||
let paste_store = worker.store().clone();
|
||||
let paste_session_id = worker.session_id();
|
||||
worker
|
||||
@@ -1426,11 +1428,9 @@ where
|
||||
let worker_enabled = feature_config.worker.enabled;
|
||||
let sub_worker_enabled = feature_config.sub_worker.enabled;
|
||||
let mut feature_registry = FeatureRegistryBuilder::new();
|
||||
if !backend_job_profile {
|
||||
feature_registry
|
||||
.add_module(crate::feature::builtin::chat_invocation::AttachmentInvocationFeature);
|
||||
}
|
||||
if backend_job_profile {
|
||||
feature_registry
|
||||
.add_module(crate::feature::builtin::chat_invocation::AttachmentInvocationFeature);
|
||||
if let Some(binding) = backend_job.as_ref() {
|
||||
let workspace_client = worker.workspace_client_handle();
|
||||
if !workspace_client.is_available() || workspace_client.workspace_id().is_none() {
|
||||
return Err(std::io::Error::new(
|
||||
@@ -1438,11 +1438,18 @@ where
|
||||
"Backend Job result capability requires Backend Workspace API authority",
|
||||
));
|
||||
}
|
||||
feature_registry.add_module(
|
||||
let mut result_feature =
|
||||
crate::feature::builtin::backend_job_result::BackendJobResultFeature::new(
|
||||
workspace_client,
|
||||
),
|
||||
);
|
||||
workspace_client.clone(),
|
||||
);
|
||||
if binding.subjektiv_consolidation {
|
||||
let surface = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::for_job(
|
||||
workspace_client, spawner_manifest.clone(), worker.llm_client_handle(),
|
||||
prompts.clone(), spawner_workspace_context.clone(),
|
||||
)?;
|
||||
result_feature = result_feature.with_surface(surface);
|
||||
}
|
||||
feature_registry.add_module(result_feature);
|
||||
}
|
||||
let memory_profile = &worker.manifest().feature.memory.profile;
|
||||
let subjektiv_profile = &worker.manifest().feature.subjektiv.profile;
|
||||
@@ -1455,7 +1462,11 @@ where
|
||||
let mut feature_prompt_contribution = memory_install_plan.as_ref().map(|plan| {
|
||||
(
|
||||
plan.resident_summary_source.clone(),
|
||||
plan.system_prompt_override.clone(),
|
||||
if backend_job.is_none() {
|
||||
plan.system_prompt_override.clone()
|
||||
} else {
|
||||
None
|
||||
},
|
||||
)
|
||||
});
|
||||
let memory_lifecycle_config = worker.manifest().feature.memory.clone();
|
||||
@@ -1469,7 +1480,9 @@ where
|
||||
worker.prompts().load_full(),
|
||||
)?;
|
||||
if let Some(plan) = subjektiv_consolidation_plan {
|
||||
feature_prompt_contribution = Some((None, Some(plan.system_prompt_override)));
|
||||
if backend_job.is_none() {
|
||||
feature_prompt_contribution = Some((None, Some(plan.system_prompt_override)));
|
||||
}
|
||||
feature_registry.add_module(plan.module);
|
||||
}
|
||||
let ordinary_subjektiv_features_enabled =
|
||||
@@ -1488,7 +1501,7 @@ where
|
||||
.to_string();
|
||||
if let Some(refresh_feature) =
|
||||
crate::feature::builtin::memory::SubjektivResidentRestoreRefreshFeature::for_host(
|
||||
worker.manifest_lifecycle_features_enabled(),
|
||||
lifecycle_enabled,
|
||||
Arc::clone(&resident_summary_source),
|
||||
worker.prompts(),
|
||||
workspace_id,
|
||||
@@ -1519,7 +1532,7 @@ where
|
||||
}
|
||||
if let Some(memory_lifecycle) =
|
||||
crate::feature::builtin::memory_lifecycle::MemoryLifecycleFeature::from_resolved_config(
|
||||
worker.manifest_lifecycle_features_enabled(),
|
||||
lifecycle_enabled,
|
||||
memory_lifecycle_config,
|
||||
worker.committed_session_capture_handle(),
|
||||
worker.session_extension_handle(),
|
||||
@@ -1533,8 +1546,8 @@ where
|
||||
{
|
||||
feature_registry.add_module(memory_lifecycle);
|
||||
}
|
||||
if let Some(surface_lifecycle) = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::from_manifest(
|
||||
worker.manifest_lifecycle_features_enabled(),
|
||||
if backend_job.is_none() && let Some(surface_lifecycle) = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::from_manifest(
|
||||
lifecycle_enabled,
|
||||
worker.workspace_client_handle(),
|
||||
spawner_manifest.clone(),
|
||||
worker.llm_client_handle(),
|
||||
@@ -1546,7 +1559,7 @@ where
|
||||
if ordinary_subjektiv_features_enabled
|
||||
&& let Some(subjektiv_lifecycle) =
|
||||
crate::feature::builtin::memory_lifecycle::SubjektivLifecycleFeature::from_resolved_config(
|
||||
worker.manifest_lifecycle_features_enabled(),
|
||||
lifecycle_enabled,
|
||||
worker.manifest().feature.subjektiv.clone(),
|
||||
worker.committed_session_capture_handle(),
|
||||
worker.session_extension_handle(),
|
||||
@@ -1721,7 +1734,7 @@ where
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if wip_mode && !backend_job_profile {
|
||||
if wip_mode {
|
||||
crate::checkout::mount_checkouts(
|
||||
&mut wip_mount_registry,
|
||||
worker.workdir_sessions(),
|
||||
|
||||
@@ -190,7 +190,7 @@ impl BackgroundTaskCancellation {
|
||||
}
|
||||
}
|
||||
|
||||
fn cancel(&self) {
|
||||
pub(crate) fn cancel(&self) {
|
||||
self.sender.send_replace(true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
//! identity come from the authenticated Workspace client, while Job, attempt,
|
||||
//! and input revision are resolved from the Backend-owned binding.
|
||||
|
||||
use std::sync::Arc;
|
||||
use crate::feature::background::BackgroundTaskCancellation;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
|
||||
use async_trait::async_trait;
|
||||
@@ -12,6 +14,7 @@ use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature;
|
||||
use crate::feature::{
|
||||
FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureModule, ToolContribution,
|
||||
ToolDeclaration, ToolDefinition,
|
||||
@@ -27,9 +30,61 @@ struct SubmitResultInput {
|
||||
result: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ConsolidationResult {
|
||||
subject_id: String,
|
||||
candidate_ids: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SubmitConsolidationResultInput {
|
||||
/// Closed candidate ids only. Host awaits and adds the actual surface outcome.
|
||||
result: ConsolidationResult,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ResultSubmissionState {
|
||||
sealed: Option<SealedResult>,
|
||||
reusable_surface: Option<Value>,
|
||||
}
|
||||
|
||||
struct SealedResult {
|
||||
original: Value,
|
||||
completed: Value,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SubmitResultTool {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
surface: Option<SubjektivSurfaceLifecycleFeature>,
|
||||
submission: Arc<tokio::sync::Mutex<ResultSubmissionState>>,
|
||||
active: Arc<Mutex<HashMap<String, BackgroundTaskCancellation>>>,
|
||||
}
|
||||
|
||||
struct ActiveResultGuard {
|
||||
id: String,
|
||||
active: Arc<Mutex<HashMap<String, BackgroundTaskCancellation>>>,
|
||||
cancellation: BackgroundTaskCancellation,
|
||||
}
|
||||
impl Drop for ActiveResultGuard {
|
||||
fn drop(&mut self) {
|
||||
self.cancellation.cancel();
|
||||
self.active
|
||||
.lock()
|
||||
.expect("Job result cancellation state poisoned")
|
||||
.remove(&self.id);
|
||||
}
|
||||
}
|
||||
|
||||
fn cancelled_output() -> ToolError {
|
||||
ToolError::Cancelled(ToolOutput {
|
||||
summary: "Job result submission cancelled; committed candidate decisions are preserved"
|
||||
.into(),
|
||||
content: None,
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -37,27 +92,128 @@ impl Tool for SubmitResultTool {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
_context: ToolExecutionContext,
|
||||
context: ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let input: SubmitResultInput = serde_json::from_str(input_json)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
let id = context.execution_id();
|
||||
let cancellation = self
|
||||
.active
|
||||
.lock()
|
||||
.expect("Job result cancellation state poisoned")
|
||||
.entry(id.clone())
|
||||
.or_default()
|
||||
.clone();
|
||||
let _active = ActiveResultGuard {
|
||||
id,
|
||||
active: self.active.clone(),
|
||||
cancellation: cancellation.clone(),
|
||||
};
|
||||
// Serialize result calls and retain the exact lifecycle outcome for ambiguous retries.
|
||||
let mut submission = tokio::select! {
|
||||
guard = self.submission.lock() => guard,
|
||||
_ = cancellation.cancelled() => return Err(cancelled_output()),
|
||||
};
|
||||
// An existing seal may already have had an effect, even if this replay
|
||||
// is now definitively rejected. It must never be reopened.
|
||||
let was_previously_sealed = submission.sealed.is_some();
|
||||
let result = if let Some(sealed) = submission.sealed.as_ref() {
|
||||
if sealed.original != input.result {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"Job result is already sealed; retry the identical result".into(),
|
||||
));
|
||||
}
|
||||
sealed.completed.clone()
|
||||
} else {
|
||||
let mut result = input.result.clone();
|
||||
if let Some(surface) = &self.surface {
|
||||
let checked = serde_json::from_str::<SubmitConsolidationResultInput>(input_json)
|
||||
.map_err(|error| {
|
||||
ToolError::InvalidArgument(format!(
|
||||
"invalid consolidation result (surface is Host-generated): {error}"
|
||||
))
|
||||
})?
|
||||
.result;
|
||||
if checked.subject_id.trim().is_empty()
|
||||
|| checked.candidate_ids.iter().any(|id| id.trim().is_empty())
|
||||
{
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"subject_id and candidate ids must not be empty".into(),
|
||||
));
|
||||
}
|
||||
let object = result
|
||||
.as_object_mut()
|
||||
.expect("typed consolidation result is an object");
|
||||
let outcome = surface
|
||||
.complete_for_job(cancellation.clone(), submission.reusable_surface.as_ref())
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if cancellation.is_cancelled() {
|
||||
cancelled_output()
|
||||
} else {
|
||||
ToolError::ExecutionFailed(error.to_string())
|
||||
}
|
||||
})?;
|
||||
submission.reusable_surface = Some(outcome.clone());
|
||||
object.insert("surface".into(), outcome);
|
||||
}
|
||||
result
|
||||
};
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(cancelled_output());
|
||||
}
|
||||
let workspace_id = self.client.workspace_id().ok_or_else(|| {
|
||||
ToolError::ExecutionFailed(
|
||||
"Backend Job result submission requires Workspace identity".to_string(),
|
||||
)
|
||||
})?;
|
||||
let response = self
|
||||
.client
|
||||
.execute(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
format!("/api/w/{workspace_id}/workers/self/backend-job-result"),
|
||||
serde_json::to_string(&serde_json::json!({ "result": input.result }))
|
||||
.map_err(|error| ToolError::Internal(error.to_string()))?,
|
||||
))
|
||||
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||
let request = WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
format!("/api/w/{workspace_id}/workers/self/backend-job-result"),
|
||||
serde_json::to_string(&serde_json::json!({ "result": result }))
|
||||
.map_err(|error| ToolError::Internal(error.to_string()))?,
|
||||
);
|
||||
// Seal before dispatch: a transport failure may follow durable acceptance.
|
||||
if submission.sealed.is_none() {
|
||||
submission.sealed = Some(SealedResult {
|
||||
original: input.result,
|
||||
completed: result,
|
||||
});
|
||||
submission.reusable_surface = None;
|
||||
}
|
||||
let response = match self.client.execute(request) {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Backend Job result acceptance is unknown; retry the identical result: {error}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let definitive_rejection = matches!(response.status, 400 | 403 | 409 | 422);
|
||||
if !response.is_success() {
|
||||
let sealed = submission.sealed.as_mut().expect("sealed before dispatch");
|
||||
if definitive_rejection && !was_previously_sealed {
|
||||
// Only a definitively unaccepted request can be corrected. Reuse
|
||||
// a ready snapshot after a new preparation confirms its revision;
|
||||
// a conflict invalidates even that snapshot. Failed outcomes are
|
||||
// rebuilt, since another generation can replace their marker.
|
||||
submission.reusable_surface = if response.status != 409 {
|
||||
sealed
|
||||
.completed
|
||||
.get("surface")
|
||||
.filter(|surface| surface["availability"] == "ready")
|
||||
.cloned()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
submission.sealed = None;
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Backend Job result was definitively rejected with HTTP {}; correct result fields or retry to rebuild a stale surface: {}",
|
||||
response.status, response.body
|
||||
)));
|
||||
}
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Backend Job result submission was rejected with HTTP {}: {}",
|
||||
"Backend Job result acceptance remains sealed; retry the identical result (HTTP {}): {}",
|
||||
response.status, response.body
|
||||
)));
|
||||
}
|
||||
@@ -67,11 +223,30 @@ impl Tool for SubmitResultTool {
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn cancel_execution(&self, context: &ToolExecutionContext) -> Result<(), ToolError> {
|
||||
self.active
|
||||
.lock()
|
||||
.expect("Job result cancellation state poisoned")
|
||||
.entry(context.execution_id())
|
||||
.or_default()
|
||||
.cancel();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn definition(client: Arc<dyn WorkspaceClient>) -> ToolDefinition {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(SubmitResultInput))
|
||||
.expect("Backend Job result schema must serialize");
|
||||
fn definition(
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
surface: Option<SubjektivSurfaceLifecycleFeature>,
|
||||
) -> ToolDefinition {
|
||||
let submission = Arc::new(tokio::sync::Mutex::new(ResultSubmissionState::default()));
|
||||
let active = Arc::new(Mutex::new(HashMap::new()));
|
||||
let schema = if surface.is_some() {
|
||||
serde_json::to_value(schemars::schema_for!(SubmitConsolidationResultInput))
|
||||
} else {
|
||||
serde_json::to_value(schemars::schema_for!(SubmitResultInput))
|
||||
}
|
||||
.expect("Backend Job result schema must serialize");
|
||||
Arc::new(move || {
|
||||
(
|
||||
ToolMeta::new(TOOL_NAME)
|
||||
@@ -79,6 +254,9 @@ fn definition(client: Arc<dyn WorkspaceClient>) -> ToolDefinition {
|
||||
.input_schema(schema.clone()),
|
||||
Arc::new(SubmitResultTool {
|
||||
client: client.clone(),
|
||||
surface: surface.clone(),
|
||||
submission: submission.clone(),
|
||||
active: active.clone(),
|
||||
}) as Arc<dyn Tool>,
|
||||
)
|
||||
})
|
||||
@@ -87,11 +265,22 @@ fn definition(client: Arc<dyn WorkspaceClient>) -> ToolDefinition {
|
||||
#[derive(Clone)]
|
||||
pub struct BackendJobResultFeature {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
surface: Option<SubjektivSurfaceLifecycleFeature>,
|
||||
}
|
||||
|
||||
impl BackendJobResultFeature {
|
||||
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
|
||||
Self { client }
|
||||
Self {
|
||||
client,
|
||||
surface: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendJobResultFeature {
|
||||
pub(crate) fn with_surface(mut self, surface: SubjektivSurfaceLifecycleFeature) -> Self {
|
||||
self.surface = Some(surface);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,7 +294,7 @@ impl FeatureModule for BackendJobResultFeature {
|
||||
fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> {
|
||||
context.tools().register(ToolContribution::new(
|
||||
TOOL_NAME,
|
||||
definition(self.client.clone()),
|
||||
definition(self.client.clone(), self.surface.clone()),
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -172,4 +361,14 @@ mod tests {
|
||||
.ends_with("/workers/self/backend-job-result")
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn cancellation_before_execution_never_submits_a_result() {
|
||||
let client = Arc::new(RecordingClient::default());
|
||||
let (_, tool) = definition(client.clone(), None)();
|
||||
let context = ToolExecutionContext::new("result-call", "batch", 0);
|
||||
tool.cancel_execution(&context).await.unwrap();
|
||||
let outcome = tool.execute(r#"{"result":{"valid":true}}"#, context).await;
|
||||
assert!(matches!(outcome, Err(ToolError::Cancelled(_))));
|
||||
assert!(client.requests.lock().unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1115,7 +1115,7 @@ pub(crate) fn is_builtin_subjektiv_consolidation_profile(
|
||||
|
||||
pub(crate) fn ordinary_subjektiv_features_enabled(manifest: &manifest::WorkerManifest) -> bool {
|
||||
manifest.feature.subjektiv.execution_enabled()
|
||||
&& !is_builtin_subjektiv_consolidation_profile(manifest)
|
||||
&& !manifest.feature.subjektiv.profile.consolidation_tools
|
||||
}
|
||||
|
||||
pub(crate) fn ordinary_subjektiv_resident_summary_source(
|
||||
@@ -1175,7 +1175,7 @@ impl SubjektivConsolidationFeatureInstallPlan {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
prompts: Arc<crate::prompt::catalog::PromptCatalog>,
|
||||
) -> std::io::Result<Option<Self>> {
|
||||
if !is_builtin_subjektiv_consolidation_profile(manifest) {
|
||||
if !manifest.feature.subjektiv.profile.consolidation_tools {
|
||||
return Ok(None);
|
||||
}
|
||||
let config = &manifest.feature.subjektiv;
|
||||
|
||||
@@ -104,6 +104,23 @@ impl SubjektivSurfaceLifecycleFeature {
|
||||
if !lifecycle_enabled || !dedicated {
|
||||
return Ok(None);
|
||||
}
|
||||
Self::for_job(
|
||||
workspace_client,
|
||||
manifest,
|
||||
client,
|
||||
prompts,
|
||||
workspace_context,
|
||||
)
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn for_job(
|
||||
workspace_client: Arc<dyn WorkspaceClient>,
|
||||
manifest: WorkerManifest,
|
||||
client: Box<dyn LlmClient>,
|
||||
prompts: Arc<ArcSwap<PromptCatalog>>,
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
) -> std::io::Result<Self> {
|
||||
manifest
|
||||
.feature
|
||||
.subjektiv
|
||||
@@ -121,7 +138,7 @@ impl SubjektivSurfaceLifecycleFeature {
|
||||
"subjektiv surface generation requires Backend Workspace API authority",
|
||||
));
|
||||
}
|
||||
Ok(Some(Self {
|
||||
Ok(Self {
|
||||
task: SubjektivSurfaceLifecycleTask {
|
||||
workspace_client,
|
||||
manifest,
|
||||
@@ -129,7 +146,18 @@ impl SubjektivSurfaceLifecycleFeature {
|
||||
prompts,
|
||||
workspace_context,
|
||||
},
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
/// The Job tool awaits this clean-context lifecycle before submitting its result.
|
||||
pub(crate) async fn complete_for_job(
|
||||
&self,
|
||||
cancellation: BackgroundTaskCancellation,
|
||||
reusable_surface: Option<&serde_json::Value>,
|
||||
) -> Result<serde_json::Value, HookError> {
|
||||
self.task
|
||||
.generate(None, cancellation, reusable_surface)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,80 +210,125 @@ impl FeatureBackgroundTask for SubjektivSurfaceLifecycleTask {
|
||||
context: BackgroundTaskContext,
|
||||
cancellation: BackgroundTaskCancellation,
|
||||
) -> Result<(), HookError> {
|
||||
self.generate(Some(&context), cancellation, None)
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
impl SubjektivSurfaceLifecycleTask {
|
||||
async fn generate(
|
||||
&self,
|
||||
context: Option<&BackgroundTaskContext>,
|
||||
cancellation: BackgroundTaskCancellation,
|
||||
reusable_surface: Option<&serde_json::Value>,
|
||||
) -> Result<serde_json::Value, HookError> {
|
||||
for attempt in 0..MAX_GENERATION_ATTEMPTS {
|
||||
context.generation_fence.ensure_current()?;
|
||||
if let Some(context) = context {
|
||||
context.generation_fence.ensure_current()?;
|
||||
}
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
return Err(HookError::new(
|
||||
HookErrorCategory::Cancelled,
|
||||
"surface generation cancelled",
|
||||
));
|
||||
}
|
||||
let generation = self
|
||||
.workspace_client
|
||||
.prepare_subjektiv_memory_surface()
|
||||
.await
|
||||
.map_err(surface_hook_error)?;
|
||||
if context.is_none() {
|
||||
if let Some(snapshot_id) = &generation.current_snapshot_id {
|
||||
return Ok(serde_json::json!({
|
||||
"availability": "ready", "generation_id": generation.generation_id,
|
||||
"store_revision": generation.store_revision, "snapshot_id": snapshot_id,
|
||||
}));
|
||||
}
|
||||
}
|
||||
// Preparation is Backend-fenced to the live attempt and immutable batch.
|
||||
// Only a ready snapshot is immutable within its store revision. Never
|
||||
// reuse a failed marker, which a later generation can replace.
|
||||
if let Some(surface) = reusable_surface.filter(|surface| {
|
||||
surface["availability"] == "ready"
|
||||
&& surface["store_revision"].as_u64() == Some(generation.store_revision)
|
||||
}) {
|
||||
return Ok(surface.clone());
|
||||
}
|
||||
if generation.materials.is_empty() && generation.active_memory_count > 0 {
|
||||
self.record_failure(&generation.generation_id, "input_budget_exhausted")
|
||||
return self
|
||||
.record_failure(&generation, "input_budget_exhausted", context.is_none())
|
||||
.await;
|
||||
tracing::warn!(
|
||||
active_memory_count = generation.active_memory_count,
|
||||
"subject Memory surface materials could not fit the bounded input"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let points = if generation.materials.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
match self.edit_surface(&generation, cancellation.clone()).await {
|
||||
Ok(points) => points,
|
||||
Err(failure) => {
|
||||
self.record_failure(&generation.generation_id, failure.reason_code)
|
||||
.await;
|
||||
match tokio::time::timeout(
|
||||
TASK_TIMEOUT,
|
||||
self.edit_surface(&generation, cancellation.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(points)) => points,
|
||||
Ok(Err(failure)) => {
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(HookError::new(
|
||||
HookErrorCategory::Cancelled,
|
||||
"surface generation cancelled",
|
||||
));
|
||||
}
|
||||
tracing::warn!(error = %failure.error, reason_code = failure.reason_code, "subject Memory surface editor failed");
|
||||
return Ok(());
|
||||
return self
|
||||
.record_failure(&generation, failure.reason_code, context.is_none())
|
||||
.await;
|
||||
}
|
||||
Err(_) => {
|
||||
return self
|
||||
.record_failure(&generation, "editor_timeout", context.is_none())
|
||||
.await;
|
||||
}
|
||||
}
|
||||
};
|
||||
context.generation_fence.ensure_current()?;
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
if let Some(context) = context {
|
||||
context.generation_fence.ensure_current()?;
|
||||
}
|
||||
let publish = self
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(HookError::new(
|
||||
HookErrorCategory::Cancelled,
|
||||
"surface generation cancelled",
|
||||
));
|
||||
}
|
||||
match self
|
||||
.workspace_client
|
||||
.publish_subjektiv_memory_surface(server_api::SubjektivSurfacePublishRequest {
|
||||
generation_id: generation.generation_id.clone(),
|
||||
points,
|
||||
})
|
||||
.await;
|
||||
match publish {
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
tracing::debug!(
|
||||
snapshot_id = output.snapshot_id,
|
||||
store_revision = output.built_from_store_revision,
|
||||
empty = output.empty,
|
||||
"published subject Memory surface"
|
||||
);
|
||||
return Ok(());
|
||||
return Ok(serde_json::json!({
|
||||
"availability": "ready", "generation_id": generation.generation_id,
|
||||
"store_revision": output.built_from_store_revision, "snapshot_id": output.snapshot_id,
|
||||
}));
|
||||
}
|
||||
Err(WorkspaceMemoryBackendError::Http { status, .. })
|
||||
if status == reqwest::StatusCode::CONFLICT
|
||||
&& attempt + 1 < MAX_GENERATION_ATTEMPTS =>
|
||||
{
|
||||
// Confirmed Memory moved while editing. Discard the stale
|
||||
// output and rebuild once from a fresh bounded generation.
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
self.record_failure(&generation.generation_id, "publish_failed")
|
||||
.await;
|
||||
tracing::warn!(%error, "subject Memory surface publication failed");
|
||||
return Ok(());
|
||||
return self
|
||||
.record_failure(&generation, "publish_failed", context.is_none())
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
unreachable!("bounded generation loop always returns on its last attempt")
|
||||
}
|
||||
}
|
||||
|
||||
impl SubjektivSurfaceLifecycleTask {
|
||||
async fn edit_surface(
|
||||
&self,
|
||||
generation: &server_api::SubjektivSurfacePrepareResponse,
|
||||
@@ -344,17 +417,39 @@ impl SubjektivSurfaceLifecycleTask {
|
||||
.map_err(SurfaceEditorFailure::editor)
|
||||
}
|
||||
|
||||
async fn record_failure(&self, generation_id: &str, reason_code: &str) {
|
||||
if let Err(error) = self
|
||||
async fn record_failure(
|
||||
&self,
|
||||
generation: &server_api::SubjektivSurfacePrepareResponse,
|
||||
reason_code: &str,
|
||||
job_confirmation_required: bool,
|
||||
) -> Result<serde_json::Value, HookError> {
|
||||
let failure = self
|
||||
.workspace_client
|
||||
.fail_subjektiv_memory_surface(server_api::SubjektivSurfaceFailureRequest {
|
||||
generation_id: generation_id.to_string(),
|
||||
generation_id: generation.generation_id.clone(),
|
||||
reason_code: reason_code.to_string(),
|
||||
})
|
||||
.await
|
||||
.map_err(surface_hook_error)?;
|
||||
// Success must attest the actual persisted marker, not merely receipt of
|
||||
// a failure notification. A protected ready surface or moved revision
|
||||
// must yield a Backend rejection/non-failed status, never a failed claim.
|
||||
let confirmed_status = if job_confirmation_required {
|
||||
"failed_confirmed"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
if failure.status != confirmed_status || failure.store_revision != generation.store_revision
|
||||
{
|
||||
tracing::debug!(%error, "could not record subject Memory surface failure");
|
||||
return Err(HookError::new(
|
||||
HookErrorCategory::Internal,
|
||||
"Backend did not confirm a failed surface for this generation's revision; no Job result was submitted",
|
||||
));
|
||||
}
|
||||
Ok(serde_json::json!({
|
||||
"availability": "failed", "generation_id": generation.generation_id,
|
||||
"store_revision": failure.store_revision, "reason_code": reason_code,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -537,6 +632,7 @@ mod tests {
|
||||
) -> server_api::SubjektivSurfacePrepareResponse {
|
||||
server_api::SubjektivSurfacePrepareResponse {
|
||||
generation_id: "generation-1".into(),
|
||||
current_snapshot_id: None,
|
||||
store_revision: 1,
|
||||
active_memory_count: materials.len(),
|
||||
materials,
|
||||
@@ -561,6 +657,13 @@ mod tests {
|
||||
prepare_calls: AtomicUsize,
|
||||
publish_calls: AtomicUsize,
|
||||
failure_reasons: Mutex<Vec<String>>,
|
||||
job_results: Mutex<Vec<serde_json::Value>>,
|
||||
result_responses:
|
||||
Mutex<std::collections::VecDeque<Result<WorkspaceResponse, WorkspaceClientError>>>,
|
||||
current_revision: AtomicUsize,
|
||||
failure_status: Mutex<String>,
|
||||
failure_revision: AtomicUsize,
|
||||
ready_snapshot: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
impl SurfaceWorkspaceClient {
|
||||
@@ -571,6 +674,12 @@ mod tests {
|
||||
prepare_calls: AtomicUsize::new(0),
|
||||
publish_calls: AtomicUsize::new(0),
|
||||
failure_reasons: Mutex::new(Vec::new()),
|
||||
job_results: Mutex::new(Vec::new()),
|
||||
result_responses: Mutex::new(Default::default()),
|
||||
current_revision: AtomicUsize::new(0),
|
||||
failure_status: Mutex::new("failed".into()),
|
||||
failure_revision: AtomicUsize::new(0),
|
||||
ready_snapshot: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -581,6 +690,15 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn revision(&self, fallback: usize) -> u64 {
|
||||
let current = self.current_revision.load(Ordering::SeqCst);
|
||||
if current == 0 {
|
||||
fallback as u64
|
||||
} else {
|
||||
current as u64
|
||||
}
|
||||
}
|
||||
|
||||
fn response(value: server_api::SubjektivMemoryBackendResponse) -> WorkspaceResponse {
|
||||
WorkspaceResponse {
|
||||
status: 200,
|
||||
@@ -606,6 +724,28 @@ mod tests {
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
if request.path.ends_with("/workers/self/backend-job-result") {
|
||||
assert!(
|
||||
self.publish_calls.load(Ordering::SeqCst) > 0
|
||||
|| !self.failure_reasons.lock().unwrap().is_empty(),
|
||||
"result must follow publication or recorded failure"
|
||||
);
|
||||
self.job_results
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(serde_json::from_str(request.body.as_deref().unwrap()).unwrap());
|
||||
return self
|
||||
.result_responses
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.unwrap_or_else(|| {
|
||||
Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: "{\"replayed\":false}".into(),
|
||||
})
|
||||
});
|
||||
}
|
||||
let request: server_api::SubjektivMemoryBackendRequest =
|
||||
serde_json::from_str(request.body.as_deref().unwrap_or_default())
|
||||
.map_err(|error| WorkspaceClientError::Request(error.to_string()))?;
|
||||
@@ -615,7 +755,8 @@ mod tests {
|
||||
let mut prepared =
|
||||
generation(self.input_token_budget, vec![material("memory-1", 32)]);
|
||||
prepared.generation_id = format!("generation-{call}");
|
||||
prepared.store_revision = call as u64;
|
||||
prepared.store_revision = self.revision(call);
|
||||
prepared.current_snapshot_id = self.ready_snapshot.lock().unwrap().clone();
|
||||
Ok(Self::response(
|
||||
server_api::SubjektivMemoryBackendResponse::SurfacePrepared(prepared),
|
||||
))
|
||||
@@ -633,7 +774,7 @@ mod tests {
|
||||
server_api::SubjektivMemoryBackendResponse::SurfacePublished(
|
||||
server_api::SubjektivSurfacePublishResponse {
|
||||
snapshot_id: format!("surface-{call}"),
|
||||
built_from_store_revision: call as u64,
|
||||
built_from_store_revision: self.revision(call),
|
||||
empty: false,
|
||||
},
|
||||
),
|
||||
@@ -644,8 +785,13 @@ mod tests {
|
||||
Ok(Self::response(
|
||||
server_api::SubjektivMemoryBackendResponse::SurfaceFailed(
|
||||
server_api::SubjektivSurfaceFailureResponse {
|
||||
store_revision: self.prepare_calls.load(Ordering::SeqCst) as u64,
|
||||
status: "failed".into(),
|
||||
store_revision: if self.failure_revision.load(Ordering::SeqCst) > 0
|
||||
{
|
||||
self.failure_revision.load(Ordering::SeqCst) as u64
|
||||
} else {
|
||||
self.revision(self.prepare_calls.load(Ordering::SeqCst))
|
||||
},
|
||||
status: self.failure_status.lock().unwrap().clone(),
|
||||
},
|
||||
),
|
||||
))
|
||||
@@ -974,4 +1120,340 @@ permission = "write"
|
||||
let error = bounded_editor_input("system policy", &generation).unwrap_err();
|
||||
assert!(error.to_string().contains("input budget"));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn job_result_waits_for_surface_and_retries_the_exact_outcome() {
|
||||
for failed in [false, true] {
|
||||
let workspace = Arc::new(if failed {
|
||||
SurfaceWorkspaceClient::with_input_token_budget(1)
|
||||
} else {
|
||||
SurfaceWorkspaceClient::new(0)
|
||||
});
|
||||
*workspace.failure_status.lock().unwrap() = "failed_confirmed".into();
|
||||
let lifecycle = SubjektivSurfaceLifecycleFeature {
|
||||
task: test_task(workspace.clone(), Arc::new(AtomicUsize::new(0))),
|
||||
};
|
||||
let mut pending = Vec::new();
|
||||
let mut hooks = crate::hook::HookRegistryBuilder::default();
|
||||
FeatureRegistryBuilder::new()
|
||||
.with_module(
|
||||
crate::feature::builtin::backend_job_result::BackendJobResultFeature::new(
|
||||
workspace.clone(),
|
||||
)
|
||||
.with_surface(lifecycle),
|
||||
)
|
||||
.install_into_pending(&mut pending, &mut hooks);
|
||||
let (_, tool) = pending.remove(0)();
|
||||
let input = r#"{"result":{"subject_id":"subject-1","candidate_ids":["candidate-1"]}}"#;
|
||||
for _ in 0..2 {
|
||||
tool.execute(input, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let results = workspace.job_results.lock().unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
assert_eq!(results[0], results[1]);
|
||||
assert_eq!(
|
||||
results[0]["result"]["surface"]["availability"],
|
||||
if failed { "failed" } else { "ready" }
|
||||
);
|
||||
assert_eq!(
|
||||
results[0]["result"]["surface"]["generation_id"],
|
||||
"generation-1"
|
||||
);
|
||||
assert_eq!(results[0]["result"]["surface"]["store_revision"], 1);
|
||||
assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 1);
|
||||
assert!(
|
||||
tool.execute(
|
||||
r#"{"result":{"subject_id":"subject-1","candidate_ids":[]}}"#,
|
||||
agen::tool::ToolExecutionContext::direct()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_uses_backend_confirmed_current_surface_without_regeneration() {
|
||||
let workspace = Arc::new(SurfaceWorkspaceClient::new(0));
|
||||
*workspace.ready_snapshot.lock().unwrap() = Some("already-current".into());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let task = test_task(workspace.clone(), calls.clone());
|
||||
let outcome = task
|
||||
.generate(None, BackgroundTaskCancellation::default(), None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(outcome["availability"], "ready");
|
||||
assert_eq!(outcome["generation_id"], "generation-1");
|
||||
assert_eq!(outcome["snapshot_id"], "already-current");
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 0);
|
||||
assert!(workspace.failure_reasons.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_result_rejects_model_surface_claims_before_generation() {
|
||||
let workspace = Arc::new(SurfaceWorkspaceClient::new(0));
|
||||
let lifecycle = SubjektivSurfaceLifecycleFeature {
|
||||
task: test_task(workspace.clone(), Arc::new(AtomicUsize::new(0))),
|
||||
};
|
||||
let mut pending = Vec::new();
|
||||
let mut hooks = crate::hook::HookRegistryBuilder::default();
|
||||
FeatureRegistryBuilder::new()
|
||||
.with_module(
|
||||
crate::feature::builtin::backend_job_result::BackendJobResultFeature::new(
|
||||
workspace.clone(),
|
||||
)
|
||||
.with_surface(lifecycle),
|
||||
)
|
||||
.install_into_pending(&mut pending, &mut hooks);
|
||||
let (_, tool) = pending.remove(0)();
|
||||
assert!(tool.execute(r#"{"result":{"subject_id":"subject-1","candidate_ids":[],"surface":{"availability":"ready"}}}"#, agen::tool::ToolExecutionContext::direct()).await.is_err());
|
||||
assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 0);
|
||||
assert!(workspace.job_results.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct PendingSurfaceEditorClient(Arc<tokio::sync::Notify>);
|
||||
#[async_trait]
|
||||
impl LlmClient for PendingSurfaceEditorClient {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
async fn stream(
|
||||
&self,
|
||||
_request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
self.0.notify_one();
|
||||
Ok(Box::pin(futures::stream::pending()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_job_surface_waits_for_editor_and_never_submits_result() {
|
||||
let workspace = Arc::new(SurfaceWorkspaceClient::new(0));
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let mut task = test_task(workspace.clone(), Arc::new(AtomicUsize::new(0)));
|
||||
task.client = Box::new(PendingSurfaceEditorClient(entered.clone()));
|
||||
let mut pending = Vec::new();
|
||||
let mut hooks = crate::hook::HookRegistryBuilder::default();
|
||||
FeatureRegistryBuilder::new()
|
||||
.with_module(
|
||||
crate::feature::builtin::backend_job_result::BackendJobResultFeature::new(
|
||||
workspace.clone(),
|
||||
)
|
||||
.with_surface(SubjektivSurfaceLifecycleFeature { task }),
|
||||
)
|
||||
.install_into_pending(&mut pending, &mut hooks);
|
||||
let (_, tool) = pending.remove(0)();
|
||||
let running_tool = tool.clone();
|
||||
let context = agen::tool::ToolExecutionContext::new("result-call", "batch", 0);
|
||||
let running_context = context.clone();
|
||||
let running = tokio::spawn(async move {
|
||||
running_tool
|
||||
.execute(
|
||||
r#"{"result":{"subject_id":"subject-1","candidate_ids":[]}}"#,
|
||||
running_context,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
tool.cancel_execution(&context).await.unwrap();
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(5), running)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(matches!(outcome, Err(agen::tool::ToolError::Cancelled(_))));
|
||||
assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 0);
|
||||
assert!(workspace.job_results.lock().unwrap().is_empty());
|
||||
assert!(workspace.failure_reasons.lock().unwrap().is_empty());
|
||||
}
|
||||
fn job_result_tool(workspace: Arc<SurfaceWorkspaceClient>) -> Arc<dyn agen::tool::Tool> {
|
||||
*workspace.failure_status.lock().unwrap() = "failed_confirmed".into();
|
||||
let mut pending = Vec::new();
|
||||
let mut hooks = crate::hook::HookRegistryBuilder::default();
|
||||
FeatureRegistryBuilder::new()
|
||||
.with_module(
|
||||
crate::feature::builtin::backend_job_result::BackendJobResultFeature::new(
|
||||
workspace.clone(),
|
||||
)
|
||||
.with_surface(SubjektivSurfaceLifecycleFeature {
|
||||
task: test_task(workspace, Arc::new(AtomicUsize::new(0))),
|
||||
}),
|
||||
)
|
||||
.install_into_pending(&mut pending, &mut hooks);
|
||||
pending.remove(0)().1
|
||||
}
|
||||
|
||||
const CORRECT_RESULT: &str =
|
||||
r#"{"result":{"subject_id":"subject-1","candidate_ids":["candidate-1"]}}"#;
|
||||
const WRONG_RESULT: &str = r#"{"result":{"subject_id":"wrong-subject","candidate_ids":[]}}"#;
|
||||
|
||||
fn rejection(status: u16) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
Ok(WorkspaceResponse {
|
||||
status,
|
||||
body: "explicit rejection before acceptance".into(),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_result_definitive_rejection_allows_correction_and_reuses_current_ready_surface() {
|
||||
for status in [400, 403, 422] {
|
||||
let workspace = Arc::new(SurfaceWorkspaceClient::new(0));
|
||||
workspace.current_revision.store(1, Ordering::SeqCst);
|
||||
workspace
|
||||
.result_responses
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push_back(rejection(status));
|
||||
let tool = job_result_tool(workspace.clone());
|
||||
assert!(
|
||||
tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.unwrap();
|
||||
let results = workspace.job_results.lock().unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
assert_ne!(
|
||||
results[0]["result"]["subject_id"],
|
||||
results[1]["result"]["subject_id"]
|
||||
);
|
||||
assert_eq!(
|
||||
results[0]["result"]["surface"],
|
||||
results[1]["result"]["surface"]
|
||||
);
|
||||
assert_eq!(
|
||||
workspace.prepare_calls.load(Ordering::SeqCst),
|
||||
2,
|
||||
"reuse must verify current revision"
|
||||
);
|
||||
assert_eq!(
|
||||
workspace.publish_calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"correction must not duplicate publication"
|
||||
);
|
||||
drop(results);
|
||||
assert!(
|
||||
tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err(),
|
||||
"accepted results stay sealed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_result_rejection_rebuilds_conflicting_or_stale_surfaces() {
|
||||
for status in [400, 409] {
|
||||
let workspace = Arc::new(SurfaceWorkspaceClient::new(0));
|
||||
workspace.current_revision.store(1, Ordering::SeqCst);
|
||||
workspace
|
||||
.result_responses
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push_back(rejection(status));
|
||||
let tool = job_result_tool(workspace.clone());
|
||||
assert!(
|
||||
tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
workspace.current_revision.store(2, Ordering::SeqCst);
|
||||
tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.unwrap();
|
||||
let results = workspace.job_results.lock().unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
assert_eq!(results[0]["result"]["surface"]["store_revision"], 1);
|
||||
assert_eq!(results[1]["result"]["surface"]["store_revision"], 2);
|
||||
assert_ne!(
|
||||
results[0]["result"]["surface"]["generation_id"],
|
||||
results[1]["result"]["surface"]["generation_id"]
|
||||
);
|
||||
assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_result_ambiguous_acceptance_never_reopens_even_after_later_rejection() {
|
||||
for first in [
|
||||
Err(WorkspaceClientError::Request(
|
||||
"accepted but response lost".into(),
|
||||
)),
|
||||
rejection(500),
|
||||
] {
|
||||
let workspace = Arc::new(SurfaceWorkspaceClient::new(0));
|
||||
workspace
|
||||
.result_responses
|
||||
.lock()
|
||||
.unwrap()
|
||||
.extend([first, rejection(403)]);
|
||||
let tool = job_result_tool(workspace.clone());
|
||||
assert!(
|
||||
tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.unwrap();
|
||||
let results = workspace.job_results.lock().unwrap();
|
||||
assert_eq!(results.len(), 3);
|
||||
assert_eq!(results[0], results[1]);
|
||||
assert_eq!(results[1], results[2]);
|
||||
assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn job_result_never_submits_unconfirmed_failure_or_wrong_revision() {
|
||||
for (status, revision) in [
|
||||
("failed", 0),
|
||||
("ready", 0),
|
||||
("stale", 0),
|
||||
("failed_confirmed", 2),
|
||||
] {
|
||||
let workspace = Arc::new(SurfaceWorkspaceClient::with_input_token_budget(1));
|
||||
workspace.current_revision.store(1, Ordering::SeqCst);
|
||||
let tool = job_result_tool(workspace.clone());
|
||||
*workspace.failure_status.lock().unwrap() = status.into();
|
||||
workspace.failure_revision.store(revision, Ordering::SeqCst);
|
||||
assert!(
|
||||
tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(workspace.job_results.lock().unwrap().is_empty());
|
||||
// Nothing was dispatched/sealed: a real confirmed outcome remains retryable.
|
||||
*workspace.failure_status.lock().unwrap() = "failed_confirmed".into();
|
||||
workspace.failure_revision.store(0, Ordering::SeqCst);
|
||||
tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(workspace.job_results.lock().unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
mod backend_job;
|
||||
pub use backend_job::BackendJobExecutionBinding;
|
||||
pub mod bootstrap;
|
||||
pub mod checkout;
|
||||
pub mod compact;
|
||||
|
||||
@@ -2599,6 +2599,8 @@ struct PendingCompactionCleanup {
|
||||
/// `session-store` functions after each turn.
|
||||
pub struct Worker<C: LlmClient, St: Store> {
|
||||
manifest: WorkerManifest,
|
||||
/// Host-bound once before Feature installation; never inherited by child Workers.
|
||||
backend_job: std::sync::OnceLock<Option<crate::BackendJobExecutionBinding>>,
|
||||
/// Always `Some` outside of `run()`/`resume()`.
|
||||
engine: Option<Engine<C, Mutable, SessionHistoryMetadata>>,
|
||||
/// Sole live authority for committed model-visible history.
|
||||
@@ -2965,6 +2967,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
let scope = SharedScope::new(scope);
|
||||
let workdir_sessions = workdir_sessions_from_authority(&filesystem_authority, &scope);
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
manifest,
|
||||
engine: Some(worker),
|
||||
session: WorkerSession::new(session_id, Vec::new()),
|
||||
@@ -3128,6 +3131,30 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
self.segment_state.session_id()
|
||||
}
|
||||
|
||||
/// Immutable host-owned Job authority, absent for ordinary and child Workers.
|
||||
pub fn backend_job(&self) -> Option<&crate::BackendJobExecutionBinding> {
|
||||
self.backend_job.get().and_then(Option::as_ref)
|
||||
}
|
||||
|
||||
/// Trusted host seam; immutable after binding and unavailable after Feature installation.
|
||||
pub fn bind_backend_job(
|
||||
&mut self,
|
||||
binding: crate::BackendJobExecutionBinding,
|
||||
) -> Result<(), &'static str> {
|
||||
binding.validate()?;
|
||||
if self.interceptor_installed || self.backend_job.get().is_some() {
|
||||
return Err("Backend Job capability is already sealed");
|
||||
}
|
||||
if !self.workspace_client().is_available()
|
||||
|| self.workspace_client().workspace_id().is_none()
|
||||
{
|
||||
return Err("Backend Job capability requires Backend Workspace API authority");
|
||||
}
|
||||
self.backend_job
|
||||
.set(Some(binding))
|
||||
.map_err(|_| "Backend Job capability is already bound")
|
||||
}
|
||||
|
||||
/// The Worker's manifest.
|
||||
pub fn manifest(&self) -> &WorkerManifest {
|
||||
&self.manifest
|
||||
@@ -3398,6 +3425,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
&mut self,
|
||||
registry: FeatureRegistryBuilder,
|
||||
) -> FeatureRegistryInstallReport {
|
||||
self.backend_job.get_or_init(|| None);
|
||||
let worker = self.engine.as_mut().expect("worker taken during run");
|
||||
let report = registry.install_into_engine(worker, &mut self.hook_builder);
|
||||
if report.has_errors() {
|
||||
@@ -7249,6 +7277,7 @@ where
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
manifest,
|
||||
engine: Some(worker),
|
||||
session: WorkerSession::new(session_id, Vec::new()),
|
||||
@@ -7350,6 +7379,18 @@ where
|
||||
});
|
||||
}
|
||||
|
||||
if self
|
||||
.backend_job()
|
||||
.is_some_and(|binding| binding.subjektiv_consolidation)
|
||||
{
|
||||
if metadata.subjektiv_session_attribution.is_some() {
|
||||
return Err(WorkerError::SubjektivSessionAttribution {
|
||||
state: "job_attribution_forbidden",
|
||||
message: "consolidation Job execution must not carry subject-body Session attribution".into(),
|
||||
});
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if !subjektiv_attached {
|
||||
if metadata.subjektiv_session_attribution.is_some() {
|
||||
return Err(WorkerError::SubjektivSessionAttribution {
|
||||
@@ -7528,6 +7569,7 @@ where
|
||||
let workdir_sessions =
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
manifest,
|
||||
engine: Some(engine),
|
||||
session: WorkerSession::new(session_id, Vec::new()),
|
||||
@@ -7655,6 +7697,7 @@ where
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
manifest,
|
||||
engine: Some(worker),
|
||||
session: WorkerSession::new(session_id, Vec::new()),
|
||||
@@ -8036,6 +8079,7 @@ where
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
manifest,
|
||||
engine: Some(worker),
|
||||
session: WorkerSession::new(session_id, restored_history_entries),
|
||||
@@ -10485,6 +10529,56 @@ mod build_summary_prompt_tests {
|
||||
(worker, store, temp)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_worker_cannot_gain_job_authority_after_feature_installation() {
|
||||
let client = Arc::new(AttributionLifecycleClient::new([]));
|
||||
let (mut worker, _store, _temp) = attribution_lifecycle_worker(client).await;
|
||||
worker.install_features(FeatureRegistryBuilder::new());
|
||||
assert!(
|
||||
worker
|
||||
.bind_backend_job(crate::BackendJobExecutionBinding {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "attempt-1".into(),
|
||||
input_revision: None,
|
||||
subjektiv_consolidation: false
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert!(worker.backend_job().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn consolidation_job_binding_is_immutable_and_has_no_subject_body_attribution() {
|
||||
let client = Arc::new(AttributionLifecycleClient::new([]));
|
||||
let (mut worker, store, _temp) = attribution_lifecycle_worker(client.clone()).await;
|
||||
let binding = crate::BackendJobExecutionBinding {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "attempt-1".into(),
|
||||
input_revision: Some("revision-1".into()),
|
||||
subjektiv_consolidation: true,
|
||||
};
|
||||
worker.bind_backend_job(binding.clone()).unwrap();
|
||||
assert_eq!(worker.backend_job(), Some(&binding));
|
||||
assert!(worker.bind_backend_job(binding).is_err());
|
||||
for lifecycle in [
|
||||
SubjektivSessionAttributionLifecycle::NewSession,
|
||||
SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||
] {
|
||||
worker
|
||||
.finalize_subjektiv_session_attribution(true, lifecycle)
|
||||
.unwrap();
|
||||
}
|
||||
assert!(client.requests.lock().unwrap().is_empty());
|
||||
assert!(
|
||||
store
|
||||
.read_by_name(&worker.manifest().worker.name)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.subjektiv_session_attribution
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_attribution_finalizes_once_before_runs_and_persists_subject() {
|
||||
let client = Arc::new(AttributionLifecycleClient::new([
|
||||
|
||||
@@ -838,36 +838,126 @@ async fn feature_flags_default_to_core_tool_surface_only() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_profile_exposes_only_structured_result_capability() {
|
||||
async fn backend_job_capability_is_explicit_and_preserves_profile_features() {
|
||||
for bound in [false, true] {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let mut resolved = ProfileResolver::new()
|
||||
.with_workspace_base(workspace.path())
|
||||
.resolve(
|
||||
&ProfileSelector::source_named(ProfileRegistrySource::Builtin, "backend-job"),
|
||||
ProfileResolveOptions::with_worker_name("backend-job-worker"),
|
||||
)
|
||||
.unwrap();
|
||||
// A custom instruction and enabled feature must survive trusted Job binding.
|
||||
if bound {
|
||||
resolved.manifest.engine.instruction = "default".into();
|
||||
resolved.manifest.feature.task.enabled = true;
|
||||
}
|
||||
let instruction = resolved.manifest.engine.instruction.clone();
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let client_for_assert = client.clone();
|
||||
let (mut worker, _pwd) = make_worker_with_manifest_and_workspace_context(
|
||||
client,
|
||||
resolved.manifest,
|
||||
WorkerWorkspaceContext::with_client(None, Arc::new(AvailableWorkspaceClient)),
|
||||
)
|
||||
.await;
|
||||
if bound {
|
||||
worker
|
||||
.bind_backend_job(worker::BackendJobExecutionBinding {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "attempt-1".into(),
|
||||
input_revision: Some("input-1".into()),
|
||||
subjektiv_consolidation: false,
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(worker.manifest().engine.instruction, instruction);
|
||||
let handle = spawn_controller(worker).await;
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Run the bounded Backend Job.",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
let names = request_tool_names(&request);
|
||||
assert_eq!(
|
||||
names.iter().any(|name| name == "SubmitBackendJobResult"),
|
||||
bound
|
||||
);
|
||||
assert_eq!(names.iter().any(|name| name == "TaskCreate"), bound);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn consolidation_job_preserves_custom_instruction_and_explicit_tool_policy() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let resolved = ProfileResolver::new()
|
||||
let mut manifest = ProfileResolver::new()
|
||||
.with_workspace_base(workspace.path())
|
||||
.resolve(
|
||||
&ProfileSelector::source_named(ProfileRegistrySource::Builtin, "backend-job"),
|
||||
ProfileResolveOptions::with_worker_name("backend-job-worker"),
|
||||
&ProfileSelector::source_named(
|
||||
ProfileRegistrySource::Builtin,
|
||||
"subjektiv-memory-consolidation",
|
||||
),
|
||||
ProfileResolveOptions::with_worker_name("custom-consolidation-job"),
|
||||
)
|
||||
.unwrap()
|
||||
.manifest;
|
||||
manifest.engine.instruction = "default".into();
|
||||
manifest.feature.task.enabled = true;
|
||||
manifest
|
||||
.feature
|
||||
.subjektiv
|
||||
.bind_workspace_settings(manifest::WorkspaceMemorySettingsSnapshot {
|
||||
workspace_id: "workspace-1".into(),
|
||||
settings_revision: 1,
|
||||
language: "English".into(),
|
||||
})
|
||||
.unwrap();
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let client_for_assert = client.clone();
|
||||
let (worker, _pwd) = make_worker_with_manifest_and_workspace_context(
|
||||
let captured = client.clone();
|
||||
let (mut worker, _pwd) = make_worker_with_manifest_and_workspace_context(
|
||||
client,
|
||||
resolved.manifest,
|
||||
manifest,
|
||||
WorkerWorkspaceContext::with_client(None, Arc::new(AvailableWorkspaceClient)),
|
||||
)
|
||||
.await;
|
||||
worker
|
||||
.engine_mut()
|
||||
.set_system_prompt("chosen custom instruction");
|
||||
worker
|
||||
.bind_backend_job(worker::BackendJobExecutionBinding {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "attempt-1".into(),
|
||||
input_revision: None,
|
||||
subjektiv_consolidation: true,
|
||||
})
|
||||
.unwrap();
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Run the bounded Backend Job.",
|
||||
"Inspect the bounded batch.",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
assert_eq!(request_tool_names(&request), vec!["SubmitBackendJobResult"]);
|
||||
let request = wait_for_captured_request(&captured).await;
|
||||
assert_eq!(
|
||||
request.system_prompt.as_deref(),
|
||||
Some("chosen custom instruction")
|
||||
);
|
||||
let tools = request_tool_names(&request);
|
||||
for tool in [
|
||||
"SubmitBackendJobResult",
|
||||
"MemoryApplyCandidate",
|
||||
"TaskCreate",
|
||||
] {
|
||||
assert!(tools.iter().any(|name| name == tool));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -199,6 +199,34 @@ const fn default_max_attempts() -> u8 {
|
||||
DEFAULT_MAX_ATTEMPTS
|
||||
}
|
||||
|
||||
/// Explicit Backend-issued domain capabilities, never inferred from purpose or profile.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobGrants {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub subjektiv_consolidation: Option<SubjektivConsolidationGrant>,
|
||||
}
|
||||
|
||||
impl BackendJobGrants {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.subjektiv_consolidation.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjektivConsolidationGrant {
|
||||
pub subject_id: String,
|
||||
/// Exact immutable candidate scope. Empty is a surface-only refresh grant:
|
||||
/// the subject is still locked, but no candidate access is authorized.
|
||||
pub candidate_ids: Vec<String>,
|
||||
}
|
||||
|
||||
/// Common runner lock identity for a subject, independent of the candidate snapshot.
|
||||
pub fn subjektiv_serialization_key(subject_id: &str) -> String {
|
||||
format!("subjektiv-consolidation:{subject_id}")
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobRequest {
|
||||
@@ -216,6 +244,13 @@ pub struct BackendJobRequest {
|
||||
pub instruction: String,
|
||||
/// Existing configured profile selector. Job dispatch never accepts raw profile source.
|
||||
pub profile: String,
|
||||
/// Immutable domain grants; profile resolution alone never grants access.
|
||||
#[serde(default, skip_serializing_if = "BackendJobGrants::is_empty")]
|
||||
pub grants: BackendJobGrants,
|
||||
/// Optional common-runner single-flight resource, not a separate domain state machine.
|
||||
/// Consolidation grants always derive their subject lock; an explicit key must match it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub serialization_key: Option<String>,
|
||||
/// Writer provenance and optional advisory destination, not parent ownership.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub source_worker: Option<RuntimeWorkerRef>,
|
||||
@@ -233,10 +268,34 @@ impl BackendJobRequest {
|
||||
validate_bounded_text("input_ref", &self.input_ref, MAX_JOB_REFERENCE_BYTES)?;
|
||||
validate_bounded_text("instruction", &self.instruction, MAX_JOB_INSTRUCTION_BYTES)?;
|
||||
validate_identifier("profile", &self.profile, 256)?;
|
||||
if self.profile != "builtin:backend-job" {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Jobs must use the dedicated builtin:backend-job profile".to_string(),
|
||||
));
|
||||
self.profile_selector()?;
|
||||
if let Some(key) = &self.serialization_key {
|
||||
validate_identifier("serialization_key", key, MAX_JOB_REFERENCE_BYTES)?;
|
||||
}
|
||||
if let Some(grant) = &self.grants.subjektiv_consolidation {
|
||||
validate_identifier("grant subject_id", &grant.subject_id, 256)?;
|
||||
if grant.candidate_ids.len() > 256 {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Job consolidation grant requires 0..=256 candidate ids".into(),
|
||||
));
|
||||
}
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for id in &grant.candidate_ids {
|
||||
validate_identifier("grant candidate_id", id, 256)?;
|
||||
if !seen.insert(id) {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Job consolidation grant has duplicate candidate ids".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(key) = &self.serialization_key
|
||||
&& *key != subjektiv_serialization_key(&grant.subject_id)
|
||||
{
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Job serialization_key must match the consolidation subject lock"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let input = serde_json::to_vec(&self.input).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job input: {error}"))
|
||||
@@ -249,6 +308,44 @@ impl BackendJobRequest {
|
||||
self.limits.validate()
|
||||
}
|
||||
|
||||
/// Validate selector syntax only. Registry existence and resolution are dispatch authority.
|
||||
pub fn profile_selector(&self) -> Result<manifest::ProfileSelector> {
|
||||
use manifest::ProfileSelector;
|
||||
validate_identifier("profile", &self.profile, 256)?;
|
||||
let selector = ProfileSelector::parse_cli(&self.profile);
|
||||
let valid = match &selector {
|
||||
ProfileSelector::Default => true,
|
||||
ProfileSelector::Path { .. } => false,
|
||||
ProfileSelector::Named { source, name } => {
|
||||
!name.is_empty()
|
||||
&& !name.starts_with('.')
|
||||
&& !name.contains(['/', '\\', ':'])
|
||||
&& name
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.'))
|
||||
&& ![".toml", ".json", ".dcdl", ".nix"]
|
||||
.iter()
|
||||
.any(|suffix| name.ends_with(suffix))
|
||||
&& (source.is_some() || name != "inherit")
|
||||
}
|
||||
};
|
||||
if !valid {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Job profile must be a manifest registry selector, not raw source or a path".into(),
|
||||
));
|
||||
}
|
||||
Ok(selector)
|
||||
}
|
||||
|
||||
/// Effective single-flight key. Typed grants cannot bypass their subject fence.
|
||||
pub fn resource_key(&self) -> Option<String> {
|
||||
self.grants
|
||||
.subjektiv_consolidation
|
||||
.as_ref()
|
||||
.map(|grant| subjektiv_serialization_key(&grant.subject_id))
|
||||
.or_else(|| self.serialization_key.clone())
|
||||
}
|
||||
|
||||
pub fn fingerprint(&self) -> Result<String> {
|
||||
self.validate()?;
|
||||
let encoded = serde_json::to_vec(self).map_err(|error| {
|
||||
@@ -346,7 +443,12 @@ pub struct BackendJobAttemptRecord {
|
||||
pub struct BackendJobReservation {
|
||||
pub job: BackendJobRecord,
|
||||
pub attempt: BackendJobAttemptRecord,
|
||||
/// Exact replay of the caller-selected immutable intent.
|
||||
pub replayed: bool,
|
||||
/// A different intent already fences this resource. The returned request is
|
||||
/// the original snapshot; the caller must defer its new input to a followup.
|
||||
#[serde(default)]
|
||||
pub resource_reused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
@@ -486,6 +588,8 @@ mod tests {
|
||||
input: serde_json::json!({"title": "Check me"}),
|
||||
instruction: "Check this immutable Ticket snapshot.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: BackendJobGrants::default(),
|
||||
serialization_key: None,
|
||||
source_worker: Some(RuntimeWorkerRef::new("runtime-a", "worker-a")),
|
||||
notification_target: Some(RuntimeWorkerRef::new("runtime-a", "worker-a")),
|
||||
limits: BackendJobLimits::default(),
|
||||
@@ -509,6 +613,133 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_profile_syntax_accepts_configured_selectors_without_an_allowlist() {
|
||||
for profile in [
|
||||
"builtin:backend-job",
|
||||
"builtin:subjektiv-memory-consolidation",
|
||||
"project:custom",
|
||||
"user:custom",
|
||||
"custom",
|
||||
"default",
|
||||
"builtin:not-yet-configured",
|
||||
] {
|
||||
let mut request = request();
|
||||
request.profile = profile.into();
|
||||
request.validate().unwrap();
|
||||
}
|
||||
for profile in [
|
||||
"",
|
||||
" builtin:backend-job",
|
||||
"path:custom",
|
||||
"./custom.toml",
|
||||
"/tmp/custom",
|
||||
"project:../custom",
|
||||
"project:custom.toml",
|
||||
"builtin:",
|
||||
"unknown:custom",
|
||||
"project:a:b",
|
||||
"C:\\custom",
|
||||
"{\"worker\":{}}",
|
||||
"inherit",
|
||||
] {
|
||||
let mut request = request();
|
||||
request.profile = profile.into();
|
||||
assert!(request.validate().is_err(), "accepted {profile}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_grants_are_immutable_bounded_and_derive_a_subject_fence() {
|
||||
let mut request = request();
|
||||
let original = request.fingerprint().unwrap();
|
||||
request.grants.subjektiv_consolidation = Some(SubjektivConsolidationGrant {
|
||||
subject_id: "subject-a".into(),
|
||||
candidate_ids: vec!["candidate-1".into()],
|
||||
});
|
||||
assert_ne!(original, request.fingerprint().unwrap());
|
||||
assert_eq!(
|
||||
request.resource_key(),
|
||||
Some(subjektiv_serialization_key("subject-a"))
|
||||
);
|
||||
let granted = request.fingerprint().unwrap();
|
||||
request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.candidate_ids
|
||||
.push("candidate-2".into());
|
||||
assert_ne!(granted, request.fingerprint().unwrap());
|
||||
request.serialization_key = Some("another-subject".into());
|
||||
assert!(request.validate().is_err());
|
||||
request.serialization_key = None;
|
||||
request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.candidate_ids
|
||||
.push("candidate-2".into());
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_consolidation_grant_keeps_subject_lock_without_candidate_authority() {
|
||||
let mut request = request();
|
||||
request.grants.subjektiv_consolidation = Some(SubjektivConsolidationGrant {
|
||||
subject_id: "subject-a".into(),
|
||||
candidate_ids: vec![],
|
||||
});
|
||||
request.validate().unwrap();
|
||||
assert!(
|
||||
!request.grants.is_empty(),
|
||||
"empty candidate scope still carries a subject grant"
|
||||
);
|
||||
assert_eq!(
|
||||
request.resource_key(),
|
||||
Some(subjektiv_serialization_key("subject-a"))
|
||||
);
|
||||
let encoded = serde_json::to_string(&request).unwrap();
|
||||
assert!(encoded.contains("\"candidate_ids\":[]"));
|
||||
let decoded: BackendJobRequest = serde_json::from_str(&encoded).unwrap();
|
||||
assert_eq!(decoded, request);
|
||||
let surface_only = request.fingerprint().unwrap();
|
||||
request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.candidate_ids = (0..256).map(|index| format!("candidate-{index}")).collect();
|
||||
request.validate().unwrap();
|
||||
assert_ne!(surface_only, request.fingerprint().unwrap());
|
||||
request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.candidate_ids
|
||||
.push("candidate-256".into());
|
||||
assert!(
|
||||
request.validate().is_err(),
|
||||
"candidate batch remains bounded"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_request_replay_keeps_its_exact_serialization() {
|
||||
let request = request();
|
||||
let original_json = serde_json::to_string(&request).unwrap();
|
||||
assert!(!original_json.contains("grants"));
|
||||
assert!(!original_json.contains("serialization_key"));
|
||||
let decoded: BackendJobRequest = serde_json::from_str(&original_json).unwrap();
|
||||
assert_eq!(decoded.grants, BackendJobGrants::default());
|
||||
assert_eq!(
|
||||
request.fingerprint().unwrap(),
|
||||
decoded.fingerprint().unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_runtime_ids_are_fixed_size_for_maximum_job_ids() {
|
||||
let job_id = "j".repeat(256);
|
||||
|
||||
@@ -564,6 +564,8 @@ pub enum WorkerSpawnIntent {
|
||||
job_id: String,
|
||||
attempt_id: String,
|
||||
purpose: String,
|
||||
input_revision: String,
|
||||
subjektiv_consolidation: bool,
|
||||
},
|
||||
TicketRole {
|
||||
ticket_id: String,
|
||||
@@ -5708,6 +5710,21 @@ fn runtime_create_worker_request(
|
||||
workspace_api: Some(workspace_api),
|
||||
memory_settings: request.resolved_memory_settings.clone(),
|
||||
subjektiv_attached: request.resolved_subjektiv_attached,
|
||||
backend_job: match &request.intent {
|
||||
WorkerSpawnIntent::BackendJob {
|
||||
job_id,
|
||||
attempt_id,
|
||||
input_revision,
|
||||
subjektiv_consolidation,
|
||||
..
|
||||
} => Some(worker_runtime::catalog::BackendJobExecutionBinding {
|
||||
job_id: job_id.clone(),
|
||||
attempt_id: attempt_id.clone(),
|
||||
input_revision: Some(input_revision.clone()),
|
||||
subjektiv_consolidation: *subjektiv_consolidation,
|
||||
}),
|
||||
_ => None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -845,6 +845,7 @@ CREATE TABLE backend_jobs (
|
||||
purpose TEXT NOT NULL,
|
||||
input_revision TEXT NOT NULL,
|
||||
input_ref TEXT NOT NULL,
|
||||
resource_key TEXT,
|
||||
request_json TEXT NOT NULL,
|
||||
intent_fingerprint TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown')),
|
||||
@@ -863,6 +864,8 @@ CREATE TABLE backend_jobs (
|
||||
);
|
||||
CREATE INDEX backend_jobs_active
|
||||
ON backend_jobs(workspace_id, state, updated_at);
|
||||
CREATE INDEX backend_jobs_resource
|
||||
ON backend_jobs(workspace_id, resource_key, created_at);
|
||||
CREATE TABLE backend_job_attempts (
|
||||
workspace_id TEXT NOT NULL,
|
||||
job_id TEXT NOT NULL,
|
||||
|
||||
@@ -251,6 +251,27 @@ pub fn selector_for_workspace_candidate(
|
||||
.then(|| worker_runtime::catalog::ProfileSelector::Named(profile.to_string()))
|
||||
}
|
||||
|
||||
/// Resolve a Host-owned launch selector against the complete existing registry.
|
||||
/// Browser candidates remain a narrower UI policy; this does not widen that surface.
|
||||
pub(crate) fn selector_for_registered_profile(
|
||||
projection: &ProfileConfigProjection,
|
||||
profile: &str,
|
||||
) -> Option<worker_runtime::catalog::ProfileSelector> {
|
||||
let catalog = builtin_profile_catalog_snapshot();
|
||||
let builtin = if profile.starts_with("builtin:") {
|
||||
profile.to_string()
|
||||
} else {
|
||||
format!("builtin:{profile}")
|
||||
};
|
||||
if catalog.entrypoints.contains_key(&builtin) {
|
||||
return Some(worker_runtime::catalog::ProfileSelector::Builtin(builtin));
|
||||
}
|
||||
projection
|
||||
.entries
|
||||
.contains_key(profile)
|
||||
.then(|| worker_runtime::catalog::ProfileSelector::Named(profile.into()))
|
||||
}
|
||||
|
||||
fn validate_prompt_projection_matches_state(
|
||||
workspace_id: &str,
|
||||
state: &WorkspaceConfigState,
|
||||
@@ -401,8 +422,12 @@ pub fn build_virtual_profile_config_bundle_with_prompt_projection(
|
||||
) -> Result<Option<ConfigBundle>> {
|
||||
validate_prompt_projection_matches_state(workspace_id, state, prompt_projection)?;
|
||||
let prompt_catalog = prompt_projection.catalog().clone();
|
||||
let profile_selector = selector_for_builtin_candidate(selector)
|
||||
.unwrap_or_else(|| worker_runtime::catalog::ProfileSelector::Named(selector.to_string()));
|
||||
let profile_selector =
|
||||
selector_for_registered_profile(projection, selector).ok_or_else(|| {
|
||||
Error::InvalidInput(format!(
|
||||
"Profile `{selector}` is not a registered launch selector"
|
||||
))
|
||||
})?;
|
||||
let archive = match projection.entries.get(selector) {
|
||||
Some(entry) => Some(build_virtual_profile_archive(
|
||||
selector,
|
||||
|
||||
@@ -74,6 +74,7 @@ fn create_request(name: &str) -> CreateWorkerRequest {
|
||||
language: "English".to_string(),
|
||||
}),
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
mod backend_job_profile;
|
||||
mod subjektiv_job;
|
||||
mod worker_operations;
|
||||
pub(crate) use worker_operations::{
|
||||
WorkerOperationContext, WorkspaceWorker, WorkspaceWorkerMethodSender,
|
||||
@@ -3500,6 +3502,7 @@ impl WorkspaceApi {
|
||||
&self,
|
||||
request: &BackendJobRequest,
|
||||
) -> Result<BackendJobReservation> {
|
||||
self.resolve_backend_job_profile(request)?;
|
||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let reservation =
|
||||
self.store
|
||||
@@ -3548,6 +3551,19 @@ impl WorkspaceApi {
|
||||
return Ok(reservation);
|
||||
}
|
||||
let request = &reservation.job.request;
|
||||
let (profile, config_bundle) = match self.resolve_backend_job_profile(request) {
|
||||
Ok(resolved) => resolved,
|
||||
Err(error) => {
|
||||
self.finish_backend_job_attempt(
|
||||
&request.job_id,
|
||||
&reservation.attempt.attempt_id,
|
||||
BackendJobAttemptState::Failed,
|
||||
"profile_requirements",
|
||||
&error.to_string(),
|
||||
)?;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let worker_input = request.worker_input(&reservation.attempt.attempt_id)?;
|
||||
let control_operation = WorkerControlOperation {
|
||||
operation_id: allocation_key(&reservation.attempt.attempt_id),
|
||||
@@ -3562,22 +3578,24 @@ impl WorkspaceApi {
|
||||
job_id: request.job_id.clone(),
|
||||
attempt_id: reservation.attempt.attempt_id.clone(),
|
||||
purpose: request.purpose.clone(),
|
||||
input_revision: request.input_revision.clone(),
|
||||
subjektiv_consolidation: request.grants.subjektiv_consolidation.is_some(),
|
||||
},
|
||||
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
|
||||
expected_segments: 0,
|
||||
},
|
||||
profile: ProfileSelector::Builtin(request.profile.clone()),
|
||||
profile,
|
||||
ticket_assignment: None,
|
||||
initial_submit: Vec::new(),
|
||||
workdir_attachment_requests: Vec::new(),
|
||||
resolved_workdir_attachment_requests: Vec::new(),
|
||||
resolved_workdir_attachments: Vec::new(),
|
||||
resolved_config_bundle: None,
|
||||
resolved_config_bundle: Some(config_bundle),
|
||||
resolved_worker_observation_enabled: false,
|
||||
resolved_worker_observation_grants: Vec::new(),
|
||||
resolved_workspace_api: None,
|
||||
resolved_memory_settings: None,
|
||||
resolved_subjektiv_attached: false,
|
||||
resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some(),
|
||||
resolved_control_operation: Some(control_operation),
|
||||
},
|
||||
) {
|
||||
@@ -3647,6 +3665,7 @@ impl WorkspaceApi {
|
||||
job,
|
||||
attempt,
|
||||
replayed: true,
|
||||
resource_reused: false,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3738,6 +3757,7 @@ impl WorkspaceApi {
|
||||
job,
|
||||
attempt,
|
||||
replayed: reservation.replayed,
|
||||
resource_reused: reservation.resource_reused,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3801,6 +3821,7 @@ impl WorkspaceApi {
|
||||
job,
|
||||
attempt: attempt.clone(),
|
||||
replayed: true,
|
||||
resource_reused: false,
|
||||
}) {
|
||||
Ok(reservation) => {
|
||||
made_progress |=
|
||||
@@ -4187,6 +4208,7 @@ impl WorkspaceApi {
|
||||
job,
|
||||
attempt,
|
||||
replayed: true,
|
||||
resource_reused: false,
|
||||
})
|
||||
{
|
||||
tracing::warn!(%error, "Backend Job reserved-attempt recovery failed");
|
||||
@@ -4292,6 +4314,7 @@ impl WorkspaceApi {
|
||||
submission: &BackendJobResultSubmission,
|
||||
) -> Result<BackendJobResultAcceptance> {
|
||||
let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let mut verified_submission = submission.clone();
|
||||
if let Some(job) = self
|
||||
.store
|
||||
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
|
||||
@@ -4334,10 +4357,41 @@ impl WorkspaceApi {
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
if let Some(job) = self
|
||||
.store
|
||||
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
|
||||
&& job.request.grants.subjektiv_consolidation.is_some()
|
||||
{
|
||||
let attempt = self
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
&self.config.workspace_id,
|
||||
&submission.job_id,
|
||||
&submission.attempt_id,
|
||||
)?
|
||||
.ok_or_else(|| Error::InvalidInput("unknown consolidation attempt".into()))?;
|
||||
if attempt.worker.as_ref() != Some(source_worker)
|
||||
|| job.current_attempt != attempt.attempt
|
||||
|| attempt.input_revision != submission.input_revision
|
||||
|| !matches!(
|
||||
attempt.state,
|
||||
BackendJobAttemptState::Dispatched | BackendJobAttemptState::Completed
|
||||
)
|
||||
{
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"consolidation result requires the exact bound attempt".into(),
|
||||
));
|
||||
}
|
||||
if attempt.state != BackendJobAttemptState::Completed {
|
||||
subjektiv_job::validate_result(self, &job.request, &attempt, &submission.result)?;
|
||||
}
|
||||
verified_submission.result =
|
||||
subjektiv_job::with_dispositions(self, &job.request, &submission.result)?;
|
||||
}
|
||||
let acceptance = self.store.accept_backend_job_result(
|
||||
&self.config.workspace_id,
|
||||
source_worker,
|
||||
submission,
|
||||
&verified_submission,
|
||||
&accepted_at,
|
||||
)?;
|
||||
if let Some(target) = acceptance.job.request.notification_target.as_ref() {
|
||||
@@ -18175,6 +18229,13 @@ fn subjektiv_subject_scope(
|
||||
)
|
||||
})?;
|
||||
let worker = RuntimeWorkerRef::new(&source.runtime_id, worker_id);
|
||||
if let Some(grant) = subjektiv_job::active_grant(api, workspace_id, &worker)? {
|
||||
return Ok((
|
||||
grant.subject_id,
|
||||
worker,
|
||||
SubjektivWorkerAuthority::Consolidation,
|
||||
));
|
||||
}
|
||||
let lease = api
|
||||
.store
|
||||
.require_current_worker_singleton_owner(workspace_id, &worker)?
|
||||
@@ -20006,7 +20067,8 @@ async fn scoped_subjektiv_memory_backend(
|
||||
Json(request): Json<server_api::SubjektivMemoryBackendRequest>,
|
||||
) -> ApiResult<Json<server_api::SubjektivMemoryBackendResponse>> {
|
||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
||||
let (subject_id, _, authority) = subjektiv_subject_scope(&api, &path.workspace_id, &context)?;
|
||||
let (subject_id, source_worker, authority) =
|
||||
subjektiv_subject_scope(&api, &path.workspace_id, &context)?;
|
||||
let store = open_subjektiv_store(&api)?;
|
||||
let response = match request.operation {
|
||||
server_api::SubjektivMemoryBackendOperation::ResidentSummary(_) => {
|
||||
@@ -20118,9 +20180,30 @@ async fn scoped_subjektiv_memory_backend(
|
||||
"candidate listing",
|
||||
)?;
|
||||
let limit = bounded_subjektiv_limit(input.limit, SUBJEKTIV_QUERY_DEFAULT_LIMIT)?;
|
||||
let mut candidates = store
|
||||
.pending_staging_candidates(&subject_id, limit.saturating_add(1))
|
||||
.map_err(subjektiv_store_error)?;
|
||||
let mut candidates = if let Some(grant) =
|
||||
subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?
|
||||
{
|
||||
let mut pending = Vec::new();
|
||||
for id in &grant.candidate_ids {
|
||||
if store
|
||||
.staging_resolution(&subject_id, id)
|
||||
.map_err(subjektiv_store_error)?
|
||||
.is_none()
|
||||
{
|
||||
if let Some(candidate) = store
|
||||
.staging_candidate(&subject_id, id)
|
||||
.map_err(subjektiv_store_error)?
|
||||
{
|
||||
pending.push(candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
pending
|
||||
} else {
|
||||
store
|
||||
.pending_staging_candidates(&subject_id, limit.saturating_add(1))
|
||||
.map_err(subjektiv_store_error)?
|
||||
};
|
||||
let has_more = candidates.len() > limit;
|
||||
candidates.truncate(limit);
|
||||
server_api::SubjektivMemoryBackendResponse::Candidates(
|
||||
@@ -20139,6 +20222,12 @@ async fn scoped_subjektiv_memory_backend(
|
||||
SubjektivWorkerAuthority::Consolidation,
|
||||
"candidate read",
|
||||
)?;
|
||||
subjektiv_job::require_candidate_grant(
|
||||
&api,
|
||||
&path.workspace_id,
|
||||
&source_worker,
|
||||
&input.candidate_id,
|
||||
)?;
|
||||
let candidate = store
|
||||
.staging_candidate(&subject_id, &input.candidate_id)
|
||||
.map_err(subjektiv_store_error)?
|
||||
@@ -20167,6 +20256,12 @@ async fn scoped_subjektiv_memory_backend(
|
||||
SubjektivWorkerAuthority::Consolidation,
|
||||
"candidate decision",
|
||||
)?;
|
||||
subjektiv_job::require_candidate_grant(
|
||||
&api,
|
||||
&path.workspace_id,
|
||||
&source_worker,
|
||||
&input.candidate_id,
|
||||
)?;
|
||||
let receipt = store
|
||||
.decide_candidate(&subject_id, subjektiv_candidate_decision(input)?)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
@@ -20180,12 +20275,42 @@ async fn scoped_subjektiv_memory_backend(
|
||||
SubjektivWorkerAuthority::Consolidation,
|
||||
"surface generation preparation",
|
||||
)?;
|
||||
if let Some(grant) =
|
||||
subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)?
|
||||
{
|
||||
for id in &grant.candidate_ids {
|
||||
if store
|
||||
.staging_resolution(&subject_id, id)
|
||||
.map_err(subjektiv_store_error)?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::InvalidInput(format!("surface generation requires a durable disposition for batch candidate `{id}`")).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
let binding = api
|
||||
.store
|
||||
.worker_registry_projection(&path.workspace_id, &source_worker)?
|
||||
.and_then(|p| p.job);
|
||||
let generation = store
|
||||
.prepare_surface_generation(&subject_id)
|
||||
.prepare_job_surface_generation(
|
||||
&subject_id,
|
||||
binding
|
||||
.as_ref()
|
||||
.map(|b| (b.job_id.as_str(), b.attempt_id.as_str())),
|
||||
)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
server_api::SubjektivMemoryBackendResponse::SurfacePrepared(
|
||||
subjektiv_surface_prepare_response(generation),
|
||||
)
|
||||
let resident = store
|
||||
.resident_surface(&subject_id)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
let mut response = subjektiv_surface_prepare_response(generation);
|
||||
if resident.availability == crate::subjektiv::SurfaceAvailability::Ready {
|
||||
response.current_snapshot_id = resident
|
||||
.snapshot
|
||||
.filter(|s| s.built_from_store_revision == response.store_revision)
|
||||
.map(|s| s.id);
|
||||
}
|
||||
server_api::SubjektivMemoryBackendResponse::SurfacePrepared(response)
|
||||
}
|
||||
server_api::SubjektivMemoryBackendOperation::PublishSurface(input) => {
|
||||
require_subjektiv_worker_authority(
|
||||
@@ -20193,6 +20318,20 @@ async fn scoped_subjektiv_memory_backend(
|
||||
SubjektivWorkerAuthority::Consolidation,
|
||||
"surface publication",
|
||||
)?;
|
||||
if let Some(binding) = api
|
||||
.store
|
||||
.worker_registry_projection(&path.workspace_id, &source_worker)?
|
||||
.and_then(|p| p.job)
|
||||
{
|
||||
store
|
||||
.require_job_surface_generation(
|
||||
&subject_id,
|
||||
&input.generation_id,
|
||||
&binding.job_id,
|
||||
&binding.attempt_id,
|
||||
)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
}
|
||||
let points = input
|
||||
.points
|
||||
.into_iter()
|
||||
@@ -20225,13 +20364,52 @@ async fn scoped_subjektiv_memory_backend(
|
||||
SubjektivWorkerAuthority::Consolidation,
|
||||
"surface generation failure",
|
||||
)?;
|
||||
if let Some(binding) = api
|
||||
.store
|
||||
.worker_registry_projection(&path.workspace_id, &source_worker)?
|
||||
.and_then(|p| p.job)
|
||||
{
|
||||
store
|
||||
.require_job_surface_generation(
|
||||
&subject_id,
|
||||
&input.generation_id,
|
||||
&binding.job_id,
|
||||
&binding.attempt_id,
|
||||
)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
}
|
||||
let revision = store
|
||||
.fail_surface_generation(&subject_id, &input.generation_id, &input.reason_code)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
let status = if let Some(binding) = api
|
||||
.store
|
||||
.worker_registry_projection(&path.workspace_id, &source_worker)?
|
||||
.and_then(|p| p.job)
|
||||
{
|
||||
store
|
||||
.validate_job_surface_outcome(
|
||||
&subject_id,
|
||||
&input.generation_id,
|
||||
revision,
|
||||
"failed",
|
||||
None,
|
||||
Some(&input.reason_code),
|
||||
&binding.job_id,
|
||||
&binding.attempt_id,
|
||||
)
|
||||
.map_err(|e| {
|
||||
Error::RepositoryConflict(format!(
|
||||
"surface failure was not recorded for this attempt: {e}"
|
||||
))
|
||||
})?;
|
||||
"failed_confirmed"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
server_api::SubjektivMemoryBackendResponse::SurfaceFailed(
|
||||
server_api::SubjektivSurfaceFailureResponse {
|
||||
store_revision: revision,
|
||||
status: "failed".into(),
|
||||
status: status.into(),
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -20244,6 +20422,7 @@ fn subjektiv_surface_prepare_response(
|
||||
) -> server_api::SubjektivSurfacePrepareResponse {
|
||||
server_api::SubjektivSurfacePrepareResponse {
|
||||
generation_id: generation.id,
|
||||
current_snapshot_id: None,
|
||||
store_revision: generation.store_revision,
|
||||
active_memory_count: generation.active_memory_count,
|
||||
materials: generation
|
||||
@@ -21448,11 +21627,9 @@ async fn scoped_subjektiv_memory_consolidation(
|
||||
SubjektivWorkerAuthority::Subject,
|
||||
"consolidation request",
|
||||
)?;
|
||||
Ok(Json(start_subjektiv_staging_consolidation(
|
||||
api,
|
||||
&subject_id,
|
||||
operation,
|
||||
)?))
|
||||
Ok(Json(
|
||||
start_subjektiv_staging_consolidation(api, &subject_id, operation).await?,
|
||||
))
|
||||
}
|
||||
|
||||
const SUBJECT_CONSOLIDATION_THRESHOLD_FILES: usize = 5;
|
||||
@@ -21511,7 +21688,7 @@ mod subject_surface_consolidation_policy_tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn start_subjektiv_staging_consolidation(
|
||||
async fn start_subjektiv_staging_consolidation(
|
||||
api: WorkspaceApi,
|
||||
subject_id: &str,
|
||||
operation: MemoryConsolidateStagingOperation,
|
||||
@@ -21520,6 +21697,13 @@ fn start_subjektiv_staging_consolidation(
|
||||
let (candidate_count, total_bytes) = store
|
||||
.pending_staging_backlog(subject_id)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
if !subjektiv_job::retire_legacy_worker(&api, subject_id).await? {
|
||||
return Ok(MemoryConsolidationOutput {
|
||||
status: "skipped_legacy_inflight".into(),
|
||||
summary: "Legacy owned subject consolidator has not completed foreground work; no Job may run concurrently.".into(),
|
||||
candidate_count, total_bytes,
|
||||
});
|
||||
}
|
||||
let surface_availability = store
|
||||
.resident_surface(subject_id)
|
||||
.map_err(subjektiv_store_error)?
|
||||
@@ -21550,135 +21734,7 @@ fn start_subjektiv_staging_consolidation(
|
||||
});
|
||||
}
|
||||
|
||||
let runtime_id = select_memory_consolidation_runtime(&api)?;
|
||||
let singleton_key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
|
||||
let input_content = format!(
|
||||
"Process the delegated subject's {candidate_count} pending Memory candidate(s) ({total_bytes} bytes) with MemoryStagingList, MemoryStagingRead, subject Memory reads, and MemoryApplyCandidate. After this committed consolidation turn, the Host will rebuild the {:?} resident surface from confirmed Memory in a separate clean context.",
|
||||
surface_availability
|
||||
);
|
||||
if let Some(output) = try_reuse_subject_consolidation_worker(
|
||||
&api,
|
||||
&singleton_key,
|
||||
&input_content,
|
||||
candidate_count,
|
||||
total_bytes,
|
||||
)? {
|
||||
return Ok(output);
|
||||
}
|
||||
let result = api.spawn_workspace_worker(
|
||||
&runtime_id,
|
||||
WorkerSpawnRequest {
|
||||
requested_worker_name: Some(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.to_string()),
|
||||
singleton_key: Some(singleton_key),
|
||||
intent: WorkerSpawnIntent::WorkspaceOrchestrator,
|
||||
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
|
||||
expected_segments: 1,
|
||||
},
|
||||
profile: ProfileSelector::Builtin(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.to_string()),
|
||||
ticket_assignment: None,
|
||||
initial_submit: vec![Segment::text(input_content)],
|
||||
workdir_attachment_requests: Vec::new(),
|
||||
resolved_workdir_attachment_requests: Vec::new(),
|
||||
resolved_workdir_attachments: Vec::new(),
|
||||
resolved_config_bundle: None,
|
||||
resolved_worker_observation_enabled: false,
|
||||
resolved_worker_observation_grants: Vec::new(),
|
||||
resolved_workspace_api: None,
|
||||
resolved_memory_settings: None,
|
||||
resolved_subjektiv_attached: true,
|
||||
resolved_control_operation: None,
|
||||
},
|
||||
)?;
|
||||
if result.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(MemoryConsolidationOutput {
|
||||
status: "skipped_spawn_rejected".into(),
|
||||
summary: "Runtime rejected subject Memory consolidater spawn.".into(),
|
||||
candidate_count,
|
||||
total_bytes,
|
||||
});
|
||||
}
|
||||
let Some(worker) = result.worker else {
|
||||
return Ok(MemoryConsolidationOutput {
|
||||
status: "skipped_spawn_missing_worker".into(),
|
||||
summary: "Runtime accepted subject Memory consolidater spawn without returning a Worker summary.".into(),
|
||||
candidate_count,
|
||||
total_bytes,
|
||||
});
|
||||
};
|
||||
Ok(MemoryConsolidationOutput {
|
||||
status: "started".into(),
|
||||
summary: format!(
|
||||
"Started subject Memory consolidater '{}' for {candidate_count} staging candidate(s).",
|
||||
worker.worker.worker_id
|
||||
),
|
||||
candidate_count,
|
||||
total_bytes,
|
||||
})
|
||||
}
|
||||
|
||||
fn try_reuse_subject_consolidation_worker(
|
||||
api: &WorkspaceApi,
|
||||
singleton_key: &str,
|
||||
input_content: &str,
|
||||
candidate_count: usize,
|
||||
total_bytes: u64,
|
||||
) -> ApiResult<Option<MemoryConsolidationOutput>> {
|
||||
let Some(owner) = api
|
||||
.store
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, singleton_key)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut worker = api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|error| error.into_error())?;
|
||||
worker.singleton_key = Some(owner.key);
|
||||
require_dedicated_subjektiv_consolidation_worker(
|
||||
&worker,
|
||||
&api.config.workspace_id,
|
||||
singleton_key,
|
||||
)?;
|
||||
if worker.state != "idle" {
|
||||
return Ok(Some(MemoryConsolidationOutput {
|
||||
status: "skipped_existing_not_idle".into(),
|
||||
summary: format!(
|
||||
"Existing subject Memory consolidater '{}' is '{}', not confirmed idle.",
|
||||
worker.worker.worker_id, worker.state
|
||||
),
|
||||
candidate_count,
|
||||
total_bytes,
|
||||
}));
|
||||
}
|
||||
let input = api
|
||||
.runtime
|
||||
.send_input(
|
||||
&worker.worker,
|
||||
WorkerInputRequest {
|
||||
kind: WorkerInputKind::User,
|
||||
content: input_content.to_string(),
|
||||
submission_request_id: None,
|
||||
segments: None,
|
||||
},
|
||||
)
|
||||
.map_err(|error| error.into_error())?;
|
||||
if input.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(Some(MemoryConsolidationOutput {
|
||||
status: "skipped_existing_input_rejected".into(),
|
||||
summary: "Existing subject Memory consolidater rejected new input.".into(),
|
||||
candidate_count,
|
||||
total_bytes,
|
||||
}));
|
||||
}
|
||||
Ok(Some(MemoryConsolidationOutput {
|
||||
status: "reused".into(),
|
||||
summary: format!(
|
||||
"Reused subject Memory consolidater '{}' for {candidate_count} staging candidate(s).",
|
||||
worker.worker.worker_id
|
||||
),
|
||||
candidate_count,
|
||||
total_bytes,
|
||||
}))
|
||||
subjektiv_job::dispatch(&api, subject_id, candidate_count, total_bytes)
|
||||
}
|
||||
|
||||
async fn scoped_memory_consolidation(
|
||||
@@ -33409,6 +33465,7 @@ impl IntoResponse for ApiError {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod subjektiv_jobs_tests;
|
||||
mod worker_operations_tests;
|
||||
include!("server_workspace_config_tests.rs");
|
||||
use super::*;
|
||||
@@ -44862,6 +44919,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -44963,6 +45022,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: Some(notification_target.clone()),
|
||||
notification_target: Some(notification_target.clone()),
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45069,6 +45130,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45105,10 +45168,10 @@ mod tests {
|
||||
Some("backend_job_cleanup_removal_rejected")
|
||||
);
|
||||
|
||||
let second = api.retry_backend_job(&request.job_id).unwrap();
|
||||
let second_worker = second.attempt.worker.clone().unwrap();
|
||||
seed_worker_session_for_cleanup(temp.path(), &second_worker);
|
||||
assert_ne!(first_worker, second_worker);
|
||||
assert!(
|
||||
api.retry_backend_job(&request.job_id).is_err(),
|
||||
"retry must not overlap an incompletely cleaned attempt"
|
||||
);
|
||||
let _ = set_worker_retention(
|
||||
api.clone(),
|
||||
first_worker.runtime_id.clone(),
|
||||
@@ -45125,6 +45188,11 @@ mod tests {
|
||||
BackendJobWorkerCleanupState::Completed,
|
||||
)
|
||||
.await;
|
||||
let second = api.retry_backend_job(&request.job_id).unwrap();
|
||||
let second_worker = second.attempt.worker.clone().unwrap();
|
||||
seed_worker_session_for_cleanup(temp.path(), &second_worker);
|
||||
assert_ne!(first_worker, second_worker);
|
||||
api.recover_backend_jobs().unwrap();
|
||||
assert!(api.runtime.worker(&second_worker).is_ok());
|
||||
|
||||
api.accept_backend_job_result(
|
||||
@@ -45171,6 +45239,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45212,6 +45282,7 @@ mod tests {
|
||||
workspace_api: Some(test_worker_workspace_api(EMBEDDED_WORKER_RUNTIME_ID)),
|
||||
memory_settings: Some(test_worker_memory_settings()),
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -45348,6 +45419,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45410,6 +45483,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45543,6 +45618,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: Some(target.clone()),
|
||||
limits: crate::backend_job::BackendJobLimits {
|
||||
@@ -45625,6 +45702,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45663,6 +45742,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45728,6 +45809,8 @@ mod tests {
|
||||
input: serde_json::json!({"index": index}),
|
||||
instruction: "Return a structured fixture result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits {
|
||||
@@ -45773,6 +45856,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45816,6 +45901,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Return a structured check result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
@@ -45905,6 +45992,8 @@ mod tests {
|
||||
input: serde_json::json!({"value": "immutable"}),
|
||||
instruction: "Wait for a structured result.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits {
|
||||
@@ -55932,6 +56021,7 @@ mod tests {
|
||||
workspace_api: None,
|
||||
memory_settings: Some(memory_settings),
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60280,14 +60370,9 @@ VALUES ('0192f0e8-4d84-7d6e-a000-000000000001', 'ticket', 3);
|
||||
let error = subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &forged_context).unwrap_err();
|
||||
assert_eq!(error.into_response().status(), StatusCode::FORBIDDEN);
|
||||
assert!(
|
||||
try_reuse_subject_consolidation_worker(
|
||||
&api,
|
||||
forged_key,
|
||||
"process pending subject Memory",
|
||||
1,
|
||||
1,
|
||||
)
|
||||
.is_err()
|
||||
subjektiv_job::retire_legacy_worker(&api, "forged-subject")
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
//! Profile recipes are selected by the caller. Execution grants remain Host owned.
|
||||
use super::*;
|
||||
|
||||
impl WorkspaceApi {
|
||||
pub(super) fn resolve_backend_job_profile(
|
||||
&self,
|
||||
request: &BackendJobRequest,
|
||||
) -> Result<(ProfileSelector, worker_runtime::config_bundle::ConfigBundle)> {
|
||||
request.validate()?;
|
||||
let state = self
|
||||
.config_store
|
||||
.load_workspace_config(self.workspace_id())?
|
||||
.ok_or_else(|| {
|
||||
Error::Config("Backend Job requires Workspace Profile configuration".into())
|
||||
})?;
|
||||
let projection = crate::profile_settings::project_profiles_from_workspace_config(
|
||||
self.workspace_id(),
|
||||
&state,
|
||||
)?;
|
||||
let requested_profile = if request.profile == "default" {
|
||||
projection
|
||||
.settings
|
||||
.default_profile
|
||||
.as_deref()
|
||||
.ok_or_else(|| {
|
||||
Error::InvalidInput("Workspace has no configured default Job Profile".into())
|
||||
})?
|
||||
} else {
|
||||
request.profile.as_str()
|
||||
};
|
||||
let selector = crate::profile_settings::selector_for_registered_profile(
|
||||
&projection,
|
||||
requested_profile,
|
||||
)
|
||||
.ok_or_else(|| {
|
||||
Error::InvalidInput(format!(
|
||||
"Backend Job Profile `{}` is not a published registry candidate",
|
||||
request.profile
|
||||
))
|
||||
})?;
|
||||
let profile_key = match &selector {
|
||||
ProfileSelector::Builtin(key) | ProfileSelector::Named(key) => key.as_str(),
|
||||
};
|
||||
let prompts = self
|
||||
.prompt_projection_cache
|
||||
.resolve(self.workspace_id(), &state)?;
|
||||
let bundle =
|
||||
crate::profile_settings::build_virtual_profile_config_bundle_with_prompt_projection(
|
||||
&projection,
|
||||
&state,
|
||||
self.workspace_id(),
|
||||
&self.config.workspace_created_at,
|
||||
profile_key,
|
||||
prompts.as_ref(),
|
||||
)?
|
||||
.ok_or_else(|| Error::Config("Backend Job Profile bundle is missing".into()))?;
|
||||
let manifest = crate::profile_settings::resolve_profile_manifest_from_config_bundle(
|
||||
&bundle,
|
||||
profile_key,
|
||||
)?;
|
||||
validate_backend_job_profile_requirements(&manifest, request)?;
|
||||
Ok((selector, bundle))
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_backend_job_profile_requirements(
|
||||
manifest: &manifest::WorkerManifest,
|
||||
request: &BackendJobRequest,
|
||||
) -> Result<()> {
|
||||
let f = &manifest.feature;
|
||||
// These tools require domain/parent/Workdir authority which a bounded Job
|
||||
// deliberately does not inherit. Reject, rather than disabling features or
|
||||
// substituting a different recipe under the caller's selector.
|
||||
if f.memory.enabled()
|
||||
|| f.worker.enabled
|
||||
|| f.sub_worker.enabled
|
||||
|| f.flow.enabled
|
||||
|| f.workspace_worker_discovery.enabled
|
||||
|| f.objective.enabled
|
||||
|| f.manage_workdir.enabled
|
||||
|| f.workspace_config.enabled
|
||||
|| f.ticket.enabled
|
||||
|| f.merge_request.show
|
||||
|| f.merge_request.open
|
||||
|| f.merge_request.review
|
||||
|| f.merge_request.readiness_check
|
||||
|| f.merge_request.complete
|
||||
|| f.orchestration.enabled
|
||||
|| !manifest.mcp.stdio_servers.is_empty()
|
||||
{
|
||||
return Err(Error::WorkspacePermissionDenied("Backend Job Profile requires authority not granted to this Job (Workspace mutation, Memory, child/peer, Workdir, Flow or MCP)".into()));
|
||||
}
|
||||
let consolidation = request.grants.subjektiv_consolidation.is_some();
|
||||
if f.subjektiv.enabled() != consolidation
|
||||
|| f.subjektiv.profile.consolidation_tools != consolidation
|
||||
|| (consolidation && f.subjektiv.profile.extraction.enabled)
|
||||
{
|
||||
return Err(Error::WorkspacePermissionDenied("Backend Job Profile subjektiv requirements do not match the explicit consolidation grant".into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
//! subjektiv is a Job client, not a second runner or Worker registry.
|
||||
use super::*;
|
||||
use crate::backend_job::{BackendJobGrants, SubjektivConsolidationGrant};
|
||||
|
||||
const PURPOSE: &str = "subjektiv_consolidation";
|
||||
const CANDIDATE_BATCH_LIMIT: usize = 100;
|
||||
|
||||
/// Load Host authority from the authenticated Worker registry and current attempt.
|
||||
/// Model input, Profile tags, display names and a subject-body lease are not grants.
|
||||
pub(super) fn active_grant(
|
||||
api: &WorkspaceApi,
|
||||
workspace_id: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ApiResult<Option<SubjektivConsolidationGrant>> {
|
||||
let Some(binding) = api
|
||||
.store
|
||||
.worker_registry_projection(workspace_id, worker)?
|
||||
.and_then(|p| p.job)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let job = api
|
||||
.store
|
||||
.get_backend_job(workspace_id, &binding.job_id)?
|
||||
.ok_or_else(|| Error::WorkspacePermissionDenied("Job binding has no intent".into()))?;
|
||||
let attempt = api
|
||||
.store
|
||||
.get_backend_job_attempt(workspace_id, &binding.job_id, &binding.attempt_id)?
|
||||
.ok_or_else(|| Error::WorkspacePermissionDenied("Job binding has no attempt".into()))?;
|
||||
if job.state != BackendJobState::Pending
|
||||
|| attempt.state != BackendJobAttemptState::Dispatched
|
||||
|| attempt.worker.as_ref() != Some(worker)
|
||||
|| job.current_attempt != attempt.attempt
|
||||
|| attempt.input_revision != job.request.input_revision
|
||||
|| chrono::DateTime::parse_from_rfc3339(&attempt.deadline_at)
|
||||
.map_err(|e| Error::Store(format!("invalid Job deadline: {e}")))?
|
||||
<= Utc::now()
|
||||
{
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"subjektiv grant requires the current live Job attempt".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(job.request.grants.subjektiv_consolidation)
|
||||
}
|
||||
|
||||
pub(super) fn require_candidate_grant(
|
||||
api: &WorkspaceApi,
|
||||
workspace_id: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
candidate_id: &str,
|
||||
) -> ApiResult<()> {
|
||||
if let Some(grant) = active_grant(api, workspace_id, worker)? {
|
||||
if !grant.candidate_ids.iter().any(|id| id == candidate_id) {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"candidate is outside this Job's immutable batch".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
Ok(()) // legacy owned consolidators may finish before the cutover fence
|
||||
}
|
||||
|
||||
/// Retire only the persisted old consolidation owner. Never interrupt a foreground
|
||||
/// candidate decision. Once Idle is confirmed, stop cancels any legacy surface
|
||||
/// background work; the new Job regenerates it and does not infer surface success.
|
||||
pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult<bool> {
|
||||
let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}");
|
||||
let Some(owner) = api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
else {
|
||||
return Ok(true);
|
||||
};
|
||||
let mut summary = api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?;
|
||||
summary.singleton_key = Some(owner.key.clone());
|
||||
require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?;
|
||||
let stopped = summary.state.eq_ignore_ascii_case("stopped");
|
||||
let idle = summary.state.eq_ignore_ascii_case("idle")
|
||||
&& summary
|
||||
.worker_state
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.state == protocol::WorkerState::Idle);
|
||||
if !stopped && !idle {
|
||||
return Ok(false);
|
||||
}
|
||||
if api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.as_ref()
|
||||
.map(|o| &o.worker)
|
||||
!= Some(&owner.worker)
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"legacy consolidation owner moved during cutover".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if !summary.state.eq_ignore_ascii_case("stopped") {
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(
|
||||
&owner.worker,
|
||||
WorkerLifecycleRequest {
|
||||
reason: Some(
|
||||
"Retire legacy subjektiv consolidator after foreground completion".into(),
|
||||
),
|
||||
ticket_assignment: None,
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?;
|
||||
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
if !api
|
||||
.runtime
|
||||
.worker(&owner.worker)
|
||||
.map_err(|e| e.into_error())?
|
||||
.state
|
||||
.eq_ignore_ascii_case("stopped")
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let removal = WorkerRemovalService::new(api)
|
||||
.execute_target_removal(
|
||||
api.runtime.as_ref(),
|
||||
&owner.worker,
|
||||
"subjektiv consolidation Job cutover",
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::Store)?;
|
||||
if removal.status != StatusCode::OK.as_u16() {
|
||||
return Err(Error::Store(format!(
|
||||
"legacy consolidation cleanup rejected: {}",
|
||||
removal.body
|
||||
))
|
||||
.into());
|
||||
}
|
||||
Ok(api
|
||||
.store
|
||||
.current_worker_singleton_owner(api.workspace_id(), &key)?
|
||||
.is_none())
|
||||
}
|
||||
|
||||
pub(super) fn dispatch(
|
||||
api: &WorkspaceApi,
|
||||
subject_id: &str,
|
||||
candidate_count: usize,
|
||||
total_bytes: u64,
|
||||
) -> ApiResult<MemoryConsolidationOutput> {
|
||||
let store = open_subjektiv_store(api)?;
|
||||
let subject = store
|
||||
.subject(subject_id)
|
||||
.map_err(subjektiv_store_error)?
|
||||
.ok_or_else(|| Error::SubjektivSubjectNotFound(subject_id.into()))?;
|
||||
let candidate_ids: Vec<_> = store
|
||||
.pending_staging_candidates(subject_id, CANDIDATE_BATCH_LIMIT)
|
||||
.map_err(subjektiv_store_error)?
|
||||
.into_iter()
|
||||
.map(|c| c.id)
|
||||
.collect();
|
||||
let batch_len = candidate_ids.len();
|
||||
let request = BackendJobRequest {
|
||||
job_id: format!("subjektiv-consolidation:{}", Uuid::new_v4()),
|
||||
purpose: PURPOSE.into(),
|
||||
input_revision: subject.store_revision.to_string(),
|
||||
input_ref: format!("subjektiv://{subject_id}/consolidation"),
|
||||
input: serde_json::json!({"subject_id": subject_id, "candidate_ids": candidate_ids}),
|
||||
instruction: "Resolve ONLY the immutable candidate_ids batch using MemoryStagingList/Read, confirmed Memory reads and MemoryApplyCandidate. Already resolved candidates are retained receipts, not work to repeat. Leave newly arriving candidates to a subsequent Job. When every batch candidate has a durable disposition, call SubmitBackendJobResult with {subject_id, candidate_ids}; its Host capability awaits bounded clean-context surface generation and adds the actual surface outcome before submission. Final prose is not completion.".into(),
|
||||
// Selection belongs here, never in the common runner's purpose switch.
|
||||
profile: format!("builtin:{SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE}"),
|
||||
grants: BackendJobGrants { subjektiv_consolidation: Some(SubjektivConsolidationGrant { subject_id: subject_id.into(), candidate_ids }) },
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits { timeout_seconds: 600, ..Default::default() },
|
||||
};
|
||||
let reservation = api.dispatch_backend_job(&request)?;
|
||||
Ok(MemoryConsolidationOutput {
|
||||
status: if reservation.resource_reused {
|
||||
"skipped_existing_job"
|
||||
} else {
|
||||
"started"
|
||||
}
|
||||
.into(),
|
||||
summary: format!(
|
||||
"Subject consolidation Job '{}' ({:?}) owns an immutable batch of {} candidate(s); remaining/new candidates stay pending for the next request. No persistent subject consolidator is reused.",
|
||||
reservation.job.request.job_id,
|
||||
reservation.attempt.state,
|
||||
if reservation.resource_reused {
|
||||
reservation
|
||||
.job
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.as_ref()
|
||||
.map_or(0, |g| g.candidate_ids.len())
|
||||
} else {
|
||||
batch_len
|
||||
}
|
||||
),
|
||||
candidate_count,
|
||||
total_bytes,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ConsolidationResult {
|
||||
subject_id: String,
|
||||
candidate_ids: Vec<String>,
|
||||
surface: SurfaceResult,
|
||||
}
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SurfaceResult {
|
||||
availability: String,
|
||||
generation_id: String,
|
||||
store_revision: u64,
|
||||
#[serde(default)]
|
||||
snapshot_id: Option<String>,
|
||||
#[serde(default)]
|
||||
reason_code: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) fn validate_result(
|
||||
api: &WorkspaceApi,
|
||||
request: &BackendJobRequest,
|
||||
attempt: &BackendJobAttemptRecord,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<()> {
|
||||
let Some(grant) = &request.grants.subjektiv_consolidation else {
|
||||
return Ok(());
|
||||
};
|
||||
let result: ConsolidationResult = serde_json::from_value(value.clone())
|
||||
.map_err(|e| Error::InvalidInput(format!("invalid consolidation result: {e}")))?;
|
||||
if result.subject_id != grant.subject_id || result.candidate_ids != grant.candidate_ids {
|
||||
return Err(Error::InvalidInput(
|
||||
"consolidation result does not match the immutable Subject/batch grant".into(),
|
||||
));
|
||||
}
|
||||
let store =
|
||||
crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration)
|
||||
.map_err(|e| Error::Store(e.to_string()))?;
|
||||
for candidate_id in &grant.candidate_ids {
|
||||
if store
|
||||
.staging_resolution(&grant.subject_id, candidate_id)
|
||||
.map_err(|e| Error::Store(e.to_string()))?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"consolidation candidate `{candidate_id}` is unresolved"
|
||||
)));
|
||||
}
|
||||
}
|
||||
store
|
||||
.validate_job_surface_outcome(
|
||||
&grant.subject_id,
|
||||
&result.surface.generation_id,
|
||||
result.surface.store_revision,
|
||||
&result.surface.availability,
|
||||
result.surface.snapshot_id.as_deref(),
|
||||
result.surface.reason_code.as_deref(),
|
||||
&request.job_id,
|
||||
&attempt.attempt_id,
|
||||
)
|
||||
.map_err(|e| Error::InvalidInput(format!("consolidation surface result mismatch: {e}")))
|
||||
}
|
||||
|
||||
/// Persist actual candidate dispositions alongside the separate surface outcome.
|
||||
/// Dispositions are immutable existing records, not model assertions or another
|
||||
/// apply ledger. This enrichment is deterministic on exact result replay.
|
||||
pub(super) fn with_dispositions(
|
||||
api: &WorkspaceApi,
|
||||
request: &BackendJobRequest,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<serde_json::Value> {
|
||||
let grant = request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::InvalidInput("missing consolidation grant".into()))?;
|
||||
let store =
|
||||
crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration)
|
||||
.map_err(|e| Error::Store(e.to_string()))?;
|
||||
let mut dispositions = Vec::new();
|
||||
for id in &grant.candidate_ids {
|
||||
let resolution = store
|
||||
.staging_resolution(&grant.subject_id, id)
|
||||
.map_err(|e| Error::Store(e.to_string()))?
|
||||
.ok_or_else(|| {
|
||||
Error::InvalidInput(format!("candidate `{id}` has no durable disposition"))
|
||||
})?;
|
||||
dispositions
|
||||
.push(serde_json::to_value(resolution).map_err(|e| Error::Store(e.to_string()))?);
|
||||
}
|
||||
let mut enriched = value.clone();
|
||||
enriched
|
||||
.as_object_mut()
|
||||
.ok_or_else(|| Error::InvalidInput("consolidation result must be an object".into()))?
|
||||
.insert(
|
||||
"candidate_dispositions".into(),
|
||||
serde_json::Value::Array(dispositions),
|
||||
);
|
||||
Ok(enriched)
|
||||
}
|
||||
@@ -0,0 +1,609 @@
|
||||
use super::*;
|
||||
use crate::subjektiv::{
|
||||
CandidateDecision, CandidateDecisionRequest, MemoryDraft, MemoryRevisionTarget, SubjectRole,
|
||||
SubjectStagingRecord, SurfaceAvailability,
|
||||
};
|
||||
use memory::extract::{CandidateKind, ExtractedCandidate, StagingEvidence, StagingRecord};
|
||||
use memory::schema::SourceRef;
|
||||
use memory::schema::{EvidenceKind, EvidenceOrigin, EvidenceOriginKind};
|
||||
|
||||
fn stage(store: &crate::subjektiv::SubjektivStore, subject: &str, id: &str) {
|
||||
let origin = EvidenceOrigin {
|
||||
kind: EvidenceOriginKind::HumanInput,
|
||||
account_id: Some("test-human".into()),
|
||||
workspace_id: Some(TEST_WORKSPACE_ID.into()),
|
||||
runtime_id: None,
|
||||
worker_id: None,
|
||||
flow_selector: None,
|
||||
flow_definition_id: None,
|
||||
flow_definition_revision: None,
|
||||
};
|
||||
let record = StagingRecord::from_candidate(
|
||||
id,
|
||||
"test-extract",
|
||||
SourceRef {
|
||||
segment_id: "segment-1".into(),
|
||||
range: [1, 2],
|
||||
},
|
||||
ExtractedCandidate {
|
||||
kind: CandidateKind::Constraint,
|
||||
claim: "Keep receipts and confirmed memory".into(),
|
||||
why_useful: "No repeated application".into(),
|
||||
staleness: None,
|
||||
evidence_ids: vec!["e1".into()],
|
||||
},
|
||||
vec![StagingEvidence {
|
||||
id: "e1".into(),
|
||||
kind: EvidenceKind::new(EvidenceKind::MESSAGE),
|
||||
entry_range: Some([1, 2]),
|
||||
origin: Some(origin),
|
||||
excerpt: Some("Keep confirmed memory".into()),
|
||||
summary: None,
|
||||
}],
|
||||
vec![],
|
||||
);
|
||||
store
|
||||
.stage_candidate(SubjectStagingRecord::attach(subject, record))
|
||||
.unwrap();
|
||||
}
|
||||
fn context(worker: &RuntimeWorkerRef) -> server_api::ServerRequestContext {
|
||||
server_api::ServerRequestContext {
|
||||
actor: None,
|
||||
worker_source: None,
|
||||
runtime_source: Some(server_api::ServerRuntimeSource {
|
||||
runtime_id: worker.runtime_id.clone(),
|
||||
worker_id: Some(worker.worker_id.clone()),
|
||||
}),
|
||||
origin: None,
|
||||
transport_headers: vec![],
|
||||
}
|
||||
}
|
||||
async fn op(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
operation: server_api::SubjektivMemoryBackendOperation,
|
||||
) -> ApiResult<Json<server_api::SubjektivMemoryBackendResponse>> {
|
||||
scoped_subjektiv_memory_backend(
|
||||
State(api.clone()),
|
||||
AxumPath(ScopedWorkspacePath {
|
||||
workspace_id: TEST_WORKSPACE_ID.into(),
|
||||
}),
|
||||
context(worker),
|
||||
Json(server_api::SubjektivMemoryBackendRequest { operation }),
|
||||
)
|
||||
.await
|
||||
}
|
||||
fn resource(api: &WorkspaceApi, subject: &str) -> crate::backend_job::BackendJobRecord {
|
||||
api.store
|
||||
.find_active_backend_job_for_resource(
|
||||
TEST_WORKSPACE_ID,
|
||||
&crate::backend_job::subjektiv_serialization_key(subject),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
fn current_attempt(
|
||||
api: &WorkspaceApi,
|
||||
job: &crate::backend_job::BackendJobRecord,
|
||||
) -> BackendJobAttemptRecord {
|
||||
api.store
|
||||
.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&job.request.job_id,
|
||||
&crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subjektiv_job_bounds_batch_auth_partial_receipts_surface_and_cleanup() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(SubjectRole::new("test-subject").unwrap())
|
||||
.unwrap();
|
||||
stage(&store, &subject.id, "candidate-first");
|
||||
let started = start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: true },
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(started.status, "started");
|
||||
let job = resource(&api, &subject.id);
|
||||
assert_eq!(
|
||||
job.request.profile,
|
||||
"builtin:subjektiv-memory-consolidation"
|
||||
);
|
||||
let attempt = current_attempt(&api, &job);
|
||||
let worker = attempt.worker.as_ref().unwrap();
|
||||
assert_eq!(execution.take_inputs().len(), 1);
|
||||
assert!(
|
||||
api.store
|
||||
.require_current_worker_singleton_owner(TEST_WORKSPACE_ID, worker)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(worker))
|
||||
.unwrap()
|
||||
.0,
|
||||
subject.id
|
||||
);
|
||||
assert!(subjektiv_subject_scope(&api, "foreign", &context(worker)).is_err());
|
||||
let ordinary = spawn_ticket_check_source(&api, "not-a-consolidator");
|
||||
assert!(subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(&ordinary)).is_err());
|
||||
// A premature result cannot generate a surface or complete unresolved work.
|
||||
assert!(
|
||||
op(
|
||||
&api,
|
||||
worker,
|
||||
server_api::SubjektivMemoryBackendOperation::PrepareSurface(
|
||||
server_api::SubjektivSurfacePrepareRequest {}
|
||||
)
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
stage(&store, &subject.id, "candidate-arrived-later");
|
||||
let again = start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: true },
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(again.status, "skipped_existing_job");
|
||||
assert!(execution.take_inputs().is_empty());
|
||||
assert_eq!(resource(&api, &subject.id).request, job.request);
|
||||
assert!(
|
||||
subjektiv_job::require_candidate_grant(
|
||||
&api,
|
||||
TEST_WORKSPACE_ID,
|
||||
worker,
|
||||
"candidate-arrived-later"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let decision = CandidateDecisionRequest {
|
||||
request_id: "exact-apply".into(),
|
||||
candidate_id: "candidate-first".into(),
|
||||
reason: "retain the learned constraint".into(),
|
||||
decision: CandidateDecision::Apply {
|
||||
target: MemoryRevisionTarget::Create,
|
||||
draft: MemoryDraft::active(
|
||||
CandidateKind::Constraint,
|
||||
"Keep committed memory",
|
||||
"Applicable across failures",
|
||||
"Prevents duplicate application",
|
||||
"confirmed from candidate",
|
||||
),
|
||||
},
|
||||
};
|
||||
let receipt = store
|
||||
.decide_candidate(&subject.id, decision.clone())
|
||||
.unwrap();
|
||||
let replay = store.decide_candidate(&subject.id, decision).unwrap();
|
||||
assert_eq!(
|
||||
receipt.memory.as_ref().unwrap().id,
|
||||
replay.memory.as_ref().unwrap().id
|
||||
);
|
||||
assert_eq!(receipt.store_revision, replay.store_revision);
|
||||
let generation = store
|
||||
.prepare_job_surface_generation(
|
||||
&subject.id,
|
||||
Some((&job.request.job_id, &attempt.attempt_id)),
|
||||
)
|
||||
.unwrap();
|
||||
let Json(failed) = op(
|
||||
&api,
|
||||
worker,
|
||||
server_api::SubjektivMemoryBackendOperation::FailSurface(
|
||||
server_api::SubjektivSurfaceFailureRequest {
|
||||
generation_id: generation.id.clone(),
|
||||
reason_code: "editor_failed".into(),
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(failed, server_api::SubjektivMemoryBackendResponse::SurfaceFailed(output) if output.status == "failed_confirmed" && output.store_revision == generation.store_revision)
|
||||
);
|
||||
let result = serde_json::json!({"subject_id": subject.id, "candidate_ids": ["candidate-first"], "surface": {"availability":"failed", "generation_id":generation.id, "store_revision":generation.store_revision,"reason_code":"editor_failed"}});
|
||||
let submission = BackendJobResultSubmission {
|
||||
job_id: job.request.job_id.clone(),
|
||||
attempt_id: attempt.attempt_id.clone(),
|
||||
input_revision: job.request.input_revision.clone(),
|
||||
result,
|
||||
};
|
||||
assert!(
|
||||
api.accept_backend_job_result(&ordinary, &submission)
|
||||
.is_err()
|
||||
);
|
||||
let mut forged = submission.clone();
|
||||
forged.result["surface"]["reason_code"] = serde_json::json!("not_recorded");
|
||||
assert!(api.accept_backend_job_result(worker, &forged).is_err());
|
||||
seed_worker_session_for_cleanup(dir.path(), worker);
|
||||
let accepted = api.accept_backend_job_result(worker, &submission).unwrap();
|
||||
assert_eq!(accepted.job.state, BackendJobState::Completed);
|
||||
assert_eq!(
|
||||
store.resident_surface(&subject.id).unwrap().availability,
|
||||
SurfaceAvailability::Failed
|
||||
);
|
||||
assert_eq!(store.pending_staging_backlog(&subject.id).unwrap().0, 1);
|
||||
assert!(
|
||||
store
|
||||
.memory(&subject.id, &receipt.memory.unwrap().id)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert!(subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(worker)).is_err());
|
||||
wait_for_backend_job_worker_cleanup(
|
||||
&api,
|
||||
&job.request.job_id,
|
||||
&attempt.attempt_id,
|
||||
BackendJobWorkerCleanupState::Completed,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
api.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.result
|
||||
.is_some()
|
||||
);
|
||||
let next = start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: true },
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(next.status, "started");
|
||||
assert_eq!(
|
||||
resource(&api, &subject.id)
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.unwrap()
|
||||
.candidate_ids,
|
||||
["candidate-arrived-later"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subjektiv_job_surface_only_requires_attempt_owned_generation_and_real_outcome() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, _) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(SubjectRole::new("surface-only").unwrap())
|
||||
.unwrap();
|
||||
let old = store.prepare_surface_generation(&subject.id).unwrap();
|
||||
assert_eq!(
|
||||
start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: false }
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.status,
|
||||
"started"
|
||||
);
|
||||
let job = resource(&api, &subject.id);
|
||||
let attempt = current_attempt(&api, &job);
|
||||
let worker = attempt.worker.as_ref().unwrap();
|
||||
assert!(
|
||||
store
|
||||
.require_job_surface_generation(
|
||||
&subject.id,
|
||||
&old.id,
|
||||
&job.request.job_id,
|
||||
&attempt.attempt_id
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let generation = store
|
||||
.prepare_job_surface_generation(
|
||||
&subject.id,
|
||||
Some((&job.request.job_id, &attempt.attempt_id)),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.require_job_surface_generation(
|
||||
&subject.id,
|
||||
&generation.id,
|
||||
&job.request.job_id,
|
||||
"wrong-attempt"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let snapshot = store
|
||||
.publish_surface_generation(&subject.id, &generation.id, vec![])
|
||||
.unwrap();
|
||||
assert!(
|
||||
op(
|
||||
&api,
|
||||
worker,
|
||||
server_api::SubjektivMemoryBackendOperation::FailSurface(
|
||||
server_api::SubjektivSurfaceFailureRequest {
|
||||
generation_id: generation.id.clone(),
|
||||
reason_code: "editor_failed".into()
|
||||
}
|
||||
)
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"ready protection is not a confirmed failed outcome"
|
||||
);
|
||||
let Json(prepared) = op(
|
||||
&api,
|
||||
worker,
|
||||
server_api::SubjektivMemoryBackendOperation::PrepareSurface(
|
||||
server_api::SubjektivSurfacePrepareRequest {},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(prepared, server_api::SubjektivMemoryBackendResponse::SurfacePrepared(output) if output.current_snapshot_id.as_deref() == Some(snapshot.id.as_str()))
|
||||
);
|
||||
seed_worker_session_for_cleanup(dir.path(), worker);
|
||||
let submission = BackendJobResultSubmission {
|
||||
job_id: job.request.job_id.clone(),
|
||||
attempt_id: attempt.attempt_id.clone(),
|
||||
input_revision: job.request.input_revision.clone(),
|
||||
result: serde_json::json!({"subject_id":subject.id,"candidate_ids":[],"surface":{"availability":"ready","generation_id":generation.id,"store_revision":generation.store_revision,"snapshot_id":snapshot.id}}),
|
||||
};
|
||||
api.accept_backend_job_result(worker, &submission).unwrap();
|
||||
wait_for_backend_job_worker_cleanup(
|
||||
&api,
|
||||
&job.request.job_id,
|
||||
&attempt.attempt_id,
|
||||
BackendJobWorkerCleanupState::Completed,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: true }
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.status,
|
||||
"skipped_empty"
|
||||
);
|
||||
assert!(
|
||||
api.accept_backend_job_result(worker, &submission)
|
||||
.unwrap()
|
||||
.replayed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_profile_resolution_rejects_ungranted_requirements_without_fallback() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let mut request = BackendJobRequest {
|
||||
job_id: "profile-resolution".into(),
|
||||
purpose: "any-purpose".into(),
|
||||
input_revision: "1".into(),
|
||||
input_ref: "test://profiles".into(),
|
||||
input: serde_json::json!({}),
|
||||
instruction: "return structured result".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: Default::default(),
|
||||
};
|
||||
let (selector, bundle) = api.resolve_backend_job_profile(&request).unwrap();
|
||||
assert_eq!(selector, ProfileSelector::Builtin(request.profile.clone()));
|
||||
assert_eq!(
|
||||
crate::profile_settings::resolve_profile_manifest_from_config_bundle(
|
||||
&bundle,
|
||||
&request.profile
|
||||
)
|
||||
.unwrap()
|
||||
.engine
|
||||
.instruction,
|
||||
"internal.backend_job_system"
|
||||
);
|
||||
for profile in [
|
||||
"path:/etc/profile",
|
||||
"builtin:missing",
|
||||
"project:missing",
|
||||
"builtin:coder",
|
||||
"builtin:companion",
|
||||
"builtin:subjektiv-memory-consolidation",
|
||||
] {
|
||||
request.profile = profile.into();
|
||||
assert!(api.dispatch_backend_job(&request).is_err(), "{profile}");
|
||||
assert!(
|
||||
api.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &request.job_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(execution.take_inputs().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_project_recipe_is_resolved_and_delivered_without_system_override() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, _) = test_api_with_recording_backend(dir.path()).await;
|
||||
let state = api
|
||||
.config_store
|
||||
.load_workspace_config(TEST_WORKSPACE_ID)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let entrypoint = state.contract.entrypoints.first().unwrap();
|
||||
let root = state.snapshot.entries.get(entrypoint).unwrap();
|
||||
let config = config_commit_request_from_api(server_api::ConfigCommitRequest {
|
||||
base_revision: state.snapshot.revision, base_digest: state.snapshot.digest.clone(),
|
||||
entrypoints: state.contract.entrypoints.iter().map(|p| p.as_str().into()).collect(),
|
||||
changes: vec![server_api::ConfigTreeChange::Update {
|
||||
path: entrypoint.as_str().into(), expected_digest: root.content_digest.clone(),
|
||||
content: "{ profile = { entries = [{ selector = \"project:job-recipe\"; source = \"profiles/job-recipe.dcdl\"; }]; }; }".into(),
|
||||
}, server_api::ConfigTreeChange::Create {
|
||||
path: "profiles/job-recipe.dcdl".into(), content_type: server_api::ConfigContentType::Decodal,
|
||||
content: "{ slug = \"job-recipe\"; scope = \"workspace_read\"; model = { id = \"fixture-model\"; }; engine = { instruction = \"role.default\"; }; feature = { task = { enabled = true; }; }; }".into(),
|
||||
}],
|
||||
}).unwrap();
|
||||
commit_workspace_config_tree(&api, TEST_WORKSPACE_ID, &config).unwrap();
|
||||
let request = BackendJobRequest {
|
||||
job_id: "project-recipe-job".into(),
|
||||
purpose: "no-profile-switch".into(),
|
||||
input_revision: "r1".into(),
|
||||
input_ref: "test://recipe".into(),
|
||||
input: serde_json::json!({}),
|
||||
instruction: "Submit the result".into(),
|
||||
profile: "project:job-recipe".into(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: Default::default(),
|
||||
};
|
||||
let (selector, bundle) = api.resolve_backend_job_profile(&request).unwrap();
|
||||
assert_eq!(
|
||||
selector,
|
||||
ProfileSelector::Named("project:job-recipe".into())
|
||||
);
|
||||
let manifest = crate::profile_settings::resolve_profile_manifest_from_config_bundle(
|
||||
&bundle,
|
||||
&request.profile,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(manifest.engine.instruction, "role.default");
|
||||
assert!(manifest.feature.task.enabled);
|
||||
let dispatched = api.dispatch_backend_job(&request).unwrap();
|
||||
let worker = dispatched.attempt.worker.unwrap();
|
||||
assert_eq!(
|
||||
api.runtime.worker(&worker).unwrap().profile.as_deref(),
|
||||
Some("project:job-recipe")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subjektiv_job_legacy_cutover_waits_foreground_then_removes_only_owned_consolidator() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(SubjectRole::new("legacy-cutover").unwrap())
|
||||
.unwrap();
|
||||
let subject_body = spawn_ticket_check_source(&api, "subject-body-protected");
|
||||
let key = format!("subjektiv-consolidation:{}", subject.id);
|
||||
let legacy = api
|
||||
.spawn_workspace_worker(
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
WorkerSpawnRequest {
|
||||
requested_worker_name: Some("not-authority-display".into()),
|
||||
singleton_key: Some(key.clone()),
|
||||
intent: WorkerSpawnIntent::WorkspaceOrchestrator,
|
||||
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
|
||||
expected_segments: 0,
|
||||
},
|
||||
profile: ProfileSelector::Builtin(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.into()),
|
||||
ticket_assignment: None,
|
||||
initial_submit: vec![],
|
||||
workdir_attachment_requests: vec![],
|
||||
resolved_workdir_attachment_requests: vec![],
|
||||
resolved_workdir_attachments: vec![],
|
||||
resolved_config_bundle: None,
|
||||
resolved_workspace_api: None,
|
||||
resolved_memory_settings: None,
|
||||
resolved_subjektiv_attached: false,
|
||||
resolved_worker_observation_enabled: false,
|
||||
resolved_worker_observation_grants: vec![],
|
||||
resolved_control_operation: None,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.worker
|
||||
.unwrap()
|
||||
.worker;
|
||||
seed_worker_session_for_cleanup(dir.path(), &legacy);
|
||||
execution.publish_worker_state(
|
||||
&legacy,
|
||||
protocol::WorkerStateSnapshot::from(protocol::WorkerStatus::Running),
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while api
|
||||
.runtime
|
||||
.worker(&legacy)
|
||||
.unwrap()
|
||||
.worker_state
|
||||
.as_ref()
|
||||
.is_none_or(|s| s.state == protocol::WorkerState::Idle)
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: true }
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.status,
|
||||
"skipped_legacy_inflight"
|
||||
);
|
||||
assert!(
|
||||
api.store
|
||||
.find_active_backend_job_for_resource(TEST_WORKSPACE_ID, &key)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(api.runtime.worker(&legacy).is_ok());
|
||||
execution.publish_worker_state(&legacy, protocol::WorkerStateSnapshot::initial());
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while api
|
||||
.runtime
|
||||
.worker(&legacy)
|
||||
.unwrap()
|
||||
.worker_state
|
||||
.as_ref()
|
||||
.is_none_or(|s| s.state != protocol::WorkerState::Idle)
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: true }
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.status,
|
||||
"started"
|
||||
);
|
||||
assert!(api.runtime.worker(&legacy).is_err());
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(api.runtime.worker(&subject_body).is_ok());
|
||||
assert!(store.subject(&subject.id).unwrap().is_some());
|
||||
}
|
||||
@@ -527,6 +527,8 @@ async fn browser_protocol_sender_rejects_job_input_and_all_live_methods() {
|
||||
input: serde_json::json!({"immutable": true}),
|
||||
instruction: "Return a structured result.".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits::default(),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1021,6 +1021,11 @@ ALTER TABLE subjects ADD COLUMN behavior_revision INTEGER NOT NULL DEFAULT 0
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_surface_job_provenance(transaction: &Transaction<'_>) -> crate::feature_storage::Result<()> {
|
||||
transaction.execute_batch("ALTER TABLE surface_generation_runs ADD COLUMN job_id TEXT; ALTER TABLE surface_generation_runs ADD COLUMN attempt_id TEXT;")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
static MIGRATIONS: &[FeatureMigration] = &[
|
||||
FeatureMigration::new(1, "create subjektiv subject memory store", create_schema),
|
||||
FeatureMigration::new(
|
||||
@@ -1039,6 +1044,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
|
||||
add_surface_generation_state,
|
||||
),
|
||||
FeatureMigration::new(5, "add user-managed subject behavior", add_subject_behavior),
|
||||
FeatureMigration::new(
|
||||
6,
|
||||
"bind surface generation provenance to Job attempts",
|
||||
add_surface_job_provenance,
|
||||
),
|
||||
];
|
||||
|
||||
pub const REGISTRATION: FeatureRegistration =
|
||||
@@ -1958,6 +1968,14 @@ impl SubjektivStore {
|
||||
/// active Memory revisions. Kind order is fixed and selection is round-robin
|
||||
/// across kinds; within each kind, updated_at desc then memory id asc.
|
||||
pub fn prepare_surface_generation(&self, subject_id: &str) -> Result<SurfaceGeneration> {
|
||||
self.prepare_job_surface_generation(subject_id, None)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_job_surface_generation(
|
||||
&self,
|
||||
subject_id: &str,
|
||||
job_attempt: Option<(&str, &str)>,
|
||||
) -> Result<SurfaceGeneration> {
|
||||
self.database.try_transaction(|transaction| {
|
||||
let subject = require_active_subject(transaction, subject_id)?;
|
||||
let (active_memory_count, materials) =
|
||||
@@ -1973,21 +1991,71 @@ impl SubjektivStore {
|
||||
transaction.execute(
|
||||
"INSERT INTO surface_generation_runs (
|
||||
subject_id, generation_id, store_revision, active_memory_count,
|
||||
materials_json, created_at
|
||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
materials_json, created_at, job_id, attempt_id
|
||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||
params![
|
||||
subject_id,
|
||||
generation.id,
|
||||
to_i64(generation.store_revision)?,
|
||||
to_i64(generation.active_memory_count as u64)?,
|
||||
serde_json::to_string(&generation.materials)?,
|
||||
generation.created_at
|
||||
generation.created_at,
|
||||
job_attempt.map(|(job_id, _)| job_id),
|
||||
job_attempt.map(|(_, attempt_id)| attempt_id),
|
||||
],
|
||||
)?;
|
||||
Ok(generation)
|
||||
})
|
||||
}
|
||||
|
||||
/// Validate a Job's structured surface claim against the existing generation
|
||||
/// and publication/failure records. This does not create another receipt or
|
||||
/// roll back Memory. The generation must have begun inside this attempt.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn validate_job_surface_outcome(
|
||||
&self,
|
||||
subject_id: &str,
|
||||
generation_id: &str,
|
||||
store_revision: u64,
|
||||
availability: &str,
|
||||
snapshot_id: Option<&str>,
|
||||
reason_code: Option<&str>,
|
||||
job_id: &str,
|
||||
attempt_id: &str,
|
||||
) -> Result<()> {
|
||||
self.database.try_with_connection(|connection| {
|
||||
let subject = require_subject_in_connection(connection, subject_id)?;
|
||||
let generation_revision = require_job_generation(connection, subject_id, generation_id, job_id, attempt_id)?;
|
||||
if generation_revision != store_revision || subject.store_revision != store_revision {
|
||||
return Err(SubjektivError::InvalidRecord("surface generation is stale".into()));
|
||||
}
|
||||
let state = connection.query_row(
|
||||
"SELECT store_revision, status, snapshot_id, reason_code FROM surface_generation_state WHERE subject_id = ?1",
|
||||
[subject_id],
|
||||
|row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Option<String>>(2)?, row.get::<_, Option<String>>(3)?)),
|
||||
).optional()?.ok_or_else(|| SubjektivError::InvalidRecord("surface generation has no terminal outcome".into()))?;
|
||||
if state.0 != to_i64(store_revision)? || state.1 != availability
|
||||
|| !matches!(availability, "ready" | "failed")
|
||||
|| state.2.as_deref() != snapshot_id || state.3.as_deref() != reason_code {
|
||||
return Err(SubjektivError::InvalidRecord("surface claim differs from the persisted outcome".into()));
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn require_job_surface_generation(
|
||||
&self,
|
||||
subject_id: &str,
|
||||
generation_id: &str,
|
||||
job_id: &str,
|
||||
attempt_id: &str,
|
||||
) -> Result<()> {
|
||||
self.database.try_with_connection(|connection| {
|
||||
require_job_generation(connection, subject_id, generation_id, job_id, attempt_id)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Validates structured editor output against the exact material snapshot and
|
||||
/// atomically publishes it only while the subject store generation is current.
|
||||
pub fn publish_surface_generation(
|
||||
@@ -3605,6 +3673,26 @@ fn reject_duplicate_memory_ids(kind: &str, values: &[MemoryRevisionRef]) -> Resu
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_job_generation(
|
||||
connection: &Connection,
|
||||
subject_id: &str,
|
||||
generation_id: &str,
|
||||
job_id: &str,
|
||||
attempt_id: &str,
|
||||
) -> Result<u64> {
|
||||
let revision: Option<i64> = connection.query_row(
|
||||
"SELECT store_revision FROM surface_generation_runs WHERE subject_id = ?1 AND generation_id = ?2 AND job_id = ?3 AND attempt_id = ?4",
|
||||
params![subject_id, generation_id, job_id, attempt_id], |row| row.get(0),
|
||||
).optional()?;
|
||||
revision
|
||||
.and_then(|value| u64::try_from(value).ok())
|
||||
.ok_or_else(|| {
|
||||
SubjektivError::InvalidRecord(
|
||||
"surface generation is not owned by this Job attempt".into(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn to_i64(value: u64) -> Result<i64> {
|
||||
i64::try_from(value).map_err(|_| {
|
||||
SubjektivError::InvalidRecord(format!("revision value {value} exceeds SQLite range"))
|
||||
|
||||
@@ -129,6 +129,8 @@ pub fn request(
|
||||
})?,
|
||||
instruction,
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: Some(source_worker.clone()),
|
||||
notification_target: Some(source_worker),
|
||||
limits: BackendJobLimits {
|
||||
|
||||
@@ -330,9 +330,30 @@ Exact dispatch retry reuses the deterministic attempt and Worker allocation key.
|
||||
Only explicit re-evaluation of a definitively failed attempt increments the
|
||||
bounded attempt number; unknown outcomes are not replayable. The runner
|
||||
uses the existing embedded Runtime spawn, Worker registry, Session input,
|
||||
observation, usage, and lifecycle paths with no Workdir attachment. It uses the
|
||||
dedicated `builtin:backend-job` profile, whose tools cannot mutate Workspace
|
||||
state, recursively create Workers or Jobs, or inherit caller Workdir authority.
|
||||
observation, usage, and lifecycle paths with no Workdir attachment. Each client
|
||||
explicitly selects an existing registry Profile. The runner resolves the complete
|
||||
Builtin/Workspace project catalog through the normal virtual config bundle and
|
||||
prompt projection path; browser launch candidates remain a narrower UI policy.
|
||||
Raw Profile source and filesystem/path selectors are not accepted. A selected
|
||||
recipe's instruction and features are not replaced with `backend-job` defaults.
|
||||
Profiles requiring ungranted Workspace mutation, ordinary Memory, child/peer,
|
||||
Flow, Workdir or MCP authority fail explicitly; no caller scope is inherited.
|
||||
The Ticket checker continues to select `builtin:backend-job` itself.
|
||||
|
||||
Job identity/result capability comes from an immutable Host-authored create and
|
||||
restore binding, not the Profile name, tag, instruction, Worker display name or
|
||||
Runtime placement. Backend checks the exact live Job/attempt/input revision and
|
||||
authenticated Worker before accepting results or domain operations. Profile
|
||||
policy alone never confers a subject grant.
|
||||
|
||||
A common optional resource key serializes Jobs using their existing durable
|
||||
intent/attempt records (schema 81), not a second domain workflow table. Exact
|
||||
request replay is distinct from a resource collision: the latter returns the
|
||||
original active immutable request with `resource_reused`, and never claims that
|
||||
new input was processed. Pending/unknown Jobs and **all** unfinished attempt
|
||||
cleanups retain the fence. Retry requires definitive failure, remaining attempt
|
||||
budget, completed prior cleanups and no competing resource owner. Unknown intent
|
||||
remains attention-required even after its Worker is removed.
|
||||
|
||||
Success is accepted only by the structured result boundary. Before Runtime
|
||||
input can execute, the attempt is fenced by Workspace, Job ID, attempt ID, input
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
# Backend subjektiv consolidation Jobs (T-704)
|
||||
|
||||
subjektiv requests the common Backend Job runner with an explicit
|
||||
`builtin:subjektiv-memory-consolidation` Profile. The runner has no purpose-based
|
||||
Profile switch. Subject-body Workers and the independent Workspace Memory feature
|
||||
are not Jobs and are not migrated or reset here. The shared/standalone adapters
|
||||
planned in T-708/T-709 are outside this Backend change.
|
||||
|
||||
## Input, grants and concurrency
|
||||
|
||||
The existing backlog/byte thresholds and missing/stale/failed surface policy
|
||||
remain the launch decision. A Job captures an immutable bounded batch (up to 100
|
||||
candidate IDs) plus the Subject store revision. An empty batch is valid for a
|
||||
surface-only rebuild. The Backend explicitly binds the Subject and candidate
|
||||
batch grant in durable Job intent before Worker execution. No subject-body
|
||||
singleton ownership or Session attribution is transferred to the Job.
|
||||
|
||||
Authenticated `(Workspace, Runtime, Worker)` registry binding and the exact
|
||||
current dispatched attempt authorize each operation. Names, Profile tags and
|
||||
model-provided `subject_id` are not authority. Candidate listing exposes only
|
||||
unresolved IDs in the captured batch; reads/decisions outside it are denied.
|
||||
Surface prepare requires all batch candidates to have durable dispositions.
|
||||
Surface generation records preserve immutable Job/attempt provenance (subjektiv
|
||||
schema migration 6); publication, failure and completion reject another attempt's
|
||||
generation, including pre-Job/legacy generations.
|
||||
|
||||
The common resource key `subjektiv-consolidation:<subject>` prevents parallel
|
||||
Jobs for a Subject. Concurrent requests return the original active intent and
|
||||
report `skipped_existing_job`; they do not expand its grants. New candidates and
|
||||
items beyond the bounded batch stay in staging. A subsequent consolidation
|
||||
request snapshots remaining work; it is not counted as processed by the first
|
||||
Job. There is no domain-specific status ledger, pool or persistent idle Worker.
|
||||
Definitive failures use the common bounded retry API with the original intent;
|
||||
unknown execution stays fenced for attention. A later explicitly requested Job
|
||||
may snapshot remaining candidates once all previous Workers are safely cleaned.
|
||||
|
||||
## Memory and surface completion
|
||||
|
||||
`MemoryApplyCandidate` still uses the existing expected-revision checks, atomic
|
||||
apply/close and immutable decision receipts. Partial application survives model
|
||||
failure, timeout, lost response, retry, Worker deletion and Backend recovery.
|
||||
Job replay is not a second memory apply: resolved candidates retain their
|
||||
existing dispositions, and exact candidate decisions converge on their receipts.
|
||||
|
||||
`SubmitBackendJobResult` for a bound consolidation Job receives only the Subject
|
||||
and captured candidate IDs. Its Host capability awaits the existing bounded
|
||||
clean-context surface editor and publish-or-recorded-failure, then adds the actual
|
||||
structured surface outcome. It does **not** schedule a Worker-owned post-Run task.
|
||||
The result call serializes and caches the exact generated outcome for ambiguous
|
||||
submission replay. The Backend verifies every batch disposition and the current
|
||||
store revision, generation ownership and persisted ready/failed surface fields;
|
||||
it enriches the durable result with actual immutable candidate dispositions.
|
||||
Thus confirmed-memory consolidation and surface readiness are separate outcomes.
|
||||
A recorded surface failure does not undo Memory. A stale/unfinished/fabricated
|
||||
surface cannot be reported completed. Final prose or Idle is never success.
|
||||
|
||||
Only after durable structured result acceptance does the existing terminal Job
|
||||
cleanup run. Its tracked Run teardown fence lets the result tool finish before
|
||||
stop/remove. Timeout/unknown paths cancel unfinished Worker execution through
|
||||
that same cleanup; later operations/results are rejected. The next request can
|
||||
regenerate failed/stale surfaces without touching confirmed memory.
|
||||
|
||||
## Legacy cutover and recovery
|
||||
|
||||
The subject-specific direct spawn and Idle resubmission paths are removed.
|
||||
On a consolidation request the Backend checks the **persisted singleton owner**
|
||||
for the old `subjektiv-consolidation:<subject>` key. It revalidates Workspace,
|
||||
key and old dedicated recipe provenance, not the display name. A running/paused
|
||||
legacy foreground operation is left alone and returns `skipped_legacy_inflight`;
|
||||
no Job is launched alongside it. Legacy owned operations can finish during this
|
||||
bounded cutover.
|
||||
|
||||
After foreground Idle (or Stopped) is confirmed, Backend stop is confirmed before
|
||||
canonical WorkerRemovalService removal. Any legacy surface background task is
|
||||
cancelled at stop; the subsequent Job rebuilds from confirmed Memory and does not
|
||||
infer successful surface completion from Idle. Busy/unobservable/mismatched
|
||||
owners fail closed. Empty/ready Subjects also retire their idle legacy owner.
|
||||
Removal failures remain diagnosable through existing durable removal recovery;
|
||||
a subsequent request rechecks ownership instead of deleting storage directly.
|
||||
Subject-body/ordinary/Workspace-Memory Workers are never selected by this key.
|
||||
|
||||
The Job runner uses its existing durable dispatch allocation, tracked input,
|
||||
T-701 registry-bound restore/recovery, deadlines, unknown outcomes, result replay,
|
||||
delivery and T-690 cleanup. Job results, candidate receipts, Memory revisions,
|
||||
provenance and surfaces remain in their existing Backend stores after Worker
|
||||
removal. No live dogfood Worker or environment is stopped/updated by development
|
||||
validation.
|
||||
|
||||
## Validation boundary
|
||||
|
||||
Server recording-backend fixtures prove reservation/binding/auth/result/cleanup
|
||||
contracts, and existing store tests prove candidate CAS/atomic receipts and
|
||||
surface freshness. Worker scripted-client tests prove the clean-context editor
|
||||
completion-before-result boundary and result replay. They are not real-provider
|
||||
LLM or live dogfood process evidence. See the T-704 validation report for exact
|
||||
commands/results and inherited intermittent failures; later passes do not erase
|
||||
those failures.
|
||||
@@ -289,11 +289,15 @@ active/resolved→retracted, and resolved→active. Retraction remains terminal.
|
||||
|
||||
## Candidate consolidation and corrections (T-670)
|
||||
|
||||
The Backend starts a restricted `builtin:subjektiv-memory-consolidation` Worker
|
||||
for exactly one authenticated subject. The Host binds the subject; model-visible
|
||||
inputs cannot select a Workspace, subject, Runtime, Worker, or Session. Its tools
|
||||
list/read pending immutable candidates, query/read confirmed revisions, and make
|
||||
one candidate decision. Candidate text is evidence rather than authority.
|
||||
subjektiv requests a bounded Backend-owned Job with the explicit
|
||||
`builtin:subjektiv-memory-consolidation` Profile and an immutable Subject/candidate
|
||||
batch grant. The common runner resolves the recipe; Profile names do not grant
|
||||
permissions. The Host derives authority from the authenticated current
|
||||
Job/attempt/Worker binding; model-visible inputs cannot select a Workspace,
|
||||
subject, Runtime, Worker, or Session. Its tools list/read only pending candidates
|
||||
in that batch, query/read confirmed revisions, and make one candidate decision.
|
||||
Candidate text is evidence rather than authority. No subject-specific idle Worker
|
||||
is resubmitted; see [Job execution and cutover](subjektiv-consolidation-jobs.md).
|
||||
|
||||
Every decision carries a stable request ID. An exact retry returns the committed
|
||||
receipt, while reuse with different input is a conflict. Applying a candidate
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# T-704 implementation and validation ledger
|
||||
|
||||
Base: `6f9621ca` (clean detached assigned checkout). Parent created
|
||||
`work/t-704-backend-subjektiv-jobs` without replacing a branch or discarding changes.
|
||||
Parent owns Backend/Profile/Subject authorization, cutover, documentation and
|
||||
integration. Two direct implementation SubWorkers supplied the common durable
|
||||
Job/store and Worker/Runtime contracts; parent inspected their committed sessions,
|
||||
diffs and test evidence, integrated them and corrected the boundaries below.
|
||||
Independent MR review is separate from implementation delegation.
|
||||
|
||||
## Implemented contracts
|
||||
|
||||
- Caller-selected existing Builtin/project registry Profiles, normal virtual
|
||||
config/prompt bundle resolution and explicit execution-requirement rejection.
|
||||
Chosen instructions/features/model are not replaced with backend-job defaults.
|
||||
- Immutable Host-authored Job/attempt create/restore binding and result capability,
|
||||
independent of Profile names/instructions/tags. No inherited Workdir, parent or
|
||||
subject-body execution authority.
|
||||
- Common transactional resource serialization (Workspace schema 81), immutable
|
||||
Subject/candidate batch grants, exact replay vs resource collision, unknown
|
||||
fences and all-old-attempt cleanup fences before retry/followup.
|
||||
- subjektiv requests its Profile; immutable bounded batches, empty surface-only
|
||||
Jobs, candidate authorization, existing CAS/atomic receipts and dispositions.
|
||||
- Synchronous bounded clean-context surface generation before result submission.
|
||||
Preparation reports an already-current snapshot to avoid needless conflicting
|
||||
regeneration. Job failure acknowledgements are `failed_confirmed` only after
|
||||
checking the actual failed marker/reason/revision and generation provenance.
|
||||
Subject feature migration 6 adds immutable generation Job/attempt provenance,
|
||||
without changing Subject/Memory/candidate content.
|
||||
- Durable results contain actual candidate dispositions and a separate verified
|
||||
ready/failed surface outcome. Existing terminal cleanup retains results/memory.
|
||||
- Removed subject direct spawn and Idle resubmission. Persisted legacy ownership
|
||||
plus actual foreground WorkerState and confirmed stop/removal fence the cutover.
|
||||
Busy legacy workers are not interrupted or duplicated; subject bodies and the
|
||||
independent old Workspace Memory feature are not selected for removal.
|
||||
|
||||
## Final validation (bounded `CARGO_BUILD_JOBS=2`)
|
||||
|
||||
| Command/boundary | Result |
|
||||
| --- | --- |
|
||||
| `cargo test -p yoi-workspace-server --lib backend_job` | 33 passed (before final surface acknowledgement addition; final whole-crate run below covers it) |
|
||||
| `cargo test -p yoi-workspace-server --lib subjektiv_jobs_tests` | 5 passed, including project recipe, batch/auth/receipts, actual failed acknowledgement, current surface and legacy cutover |
|
||||
| `cargo test -p worker --lib memory_surface_lifecycle` | 16 passed, including publication/failure before result, cancellation, definitive correction, ambiguous exact retry and already-current surface |
|
||||
| Root `cargo check` | Passed; normal TUI/Server/standalone compile closure |
|
||||
| `cargo test -p worker -p worker-runtime -p manifest -p runtime-api -p server-api -- --test-threads=4` | Passed: manifest 185; runtime-api 9; server-api 70; worker 815 lib + 110 integration; runtime 335 lib + 34 binary tests |
|
||||
| `cargo test -p yoi-workspace-server -- --test-threads=4` (final) | 637 lib + 11 binary passed, no skips |
|
||||
| `cargo test -p server-api --features typescript` | 77 passed, including generated/OpenAPI/TypeScript contract freshness; existing ts-rs serde-attribute warnings |
|
||||
| `cargo fmt --all -- --check` / `git diff --check HEAD` | Passed |
|
||||
|
||||
These are recording-backend/store and scripted-client fixtures, plus real compiler
|
||||
and test binaries. They are **not** real-provider LLM execution, a live Backend /
|
||||
Runtime deployment, live dogfood consolidation or an environment update. No live
|
||||
Worker/environment was stopped or updated.
|
||||
|
||||
## Failed checks and fixes (not erased by later passes)
|
||||
|
||||
- During integration, new request fields, resource replay metadata and surface
|
||||
validation arguments initially failed compilation; all were corrected before
|
||||
final validation.
|
||||
- Initial Backend tests exposed the narrower browser builtin candidate allowlist;
|
||||
Host launch now resolves the complete existing registry without widening browser
|
||||
choices or adding a purpose-specific Profile allowlist.
|
||||
- Initial surface-only test exposed rejection of empty candidate grants; empty
|
||||
batches now retain the subject lock and grant no candidate access (owned store
|
||||
tests 9 + contract tests 9 passed).
|
||||
- Legacy fixture initially compared catalog compatibility state rather than
|
||||
actual WorkerState. Cutover now requires the actual Idle snapshot, and the test
|
||||
synchronizes on that authoritative observation before asserting behavior.
|
||||
- First full Server run: **635 passed / 2 failed**, both stale migration-history
|
||||
fixtures expecting schema 80. Their explicit retained chain/history is updated
|
||||
to 81; migration filter passed 14 tests and subsequent whole-crate runs passed.
|
||||
- Exploratory `cargo check -p worker-runtime --no-default-features` and
|
||||
`--no-default-features --features fs-store` both **failed**, with 16 errors from
|
||||
existing unconditional `workspace_request` / `reqwest` imports while HTTP is
|
||||
disabled (plus unused-import warnings). The source/gates are present in the
|
||||
original baseline and are recorded in the
|
||||
[T-701 ledger](2026-10-06-t701-validation.md#validation-results). No isolated
|
||||
baseline build was performed here. These remain failed exploratory unsupported
|
||||
configurations, not feature passes or a T-704 feature-gating fix. The default
|
||||
Runtime fs-store/ws-server configuration is fully tested.
|
||||
- The inherited intermittent Ticket checker failure described in the
|
||||
[T-702 ledger](2026-10-06-t702-stale-checker-validation.md) and T-701 report was
|
||||
not observed in the final T-704 full Server runs, but is **not** claimed fixed.
|
||||
|
||||
Raw parent logs are under `target/t704-validation/` (ignored local artifacts),
|
||||
including `server-full-initial-migration-fixtures.log`, `server-full-final.log`,
|
||||
`changed-crates.log`, `root-check.log`, `worker-surface-final.log`,
|
||||
`server-subjektiv-final.log`, `server-api-typescript.log`, `runtime-no-default.log`
|
||||
and `runtime-fs-only.log`. The durable report records their concrete results;
|
||||
local log paths are not a substitute for the published immutable MR source.
|
||||
@@ -3,13 +3,13 @@ import "./base.dcdl" // {
|
||||
description = "Delegated subject Memory staging consolidation profile.";
|
||||
scope = "workspace_read";
|
||||
model = { ref = "codex-oauth/gpt-5.6-luna"; };
|
||||
engine = { reasoning = "medium"; };
|
||||
engine = { reasoning = "medium"; instruction = "internal.subjektiv_memory_consolidation_system"; };
|
||||
web = { enabled = false; };
|
||||
|
||||
feature = {
|
||||
task = { enabled = false; };
|
||||
memory = { enabled = false; staging_tools = false; };
|
||||
subjektiv = { enabled = true; extraction = { enabled = false; }; };
|
||||
subjektiv = { enabled = true; consolidation_tools = true; extraction = { enabled = false; }; };
|
||||
web = { enabled = false; };
|
||||
image = { enabled = false; };
|
||||
sub_worker = { enabled = false; };
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Memory staging consolidater
|
||||
|
||||
You are the delegated internal Memory staging consolidater for one subject.
|
||||
You are the Backend Job Memory staging consolidater for one subject.
|
||||
|
||||
Your job is to consume that subject's pending Memory candidates through tools. The host binds every tool call to the subject; never infer or provide a subject identifier yourself.
|
||||
|
||||
@@ -26,7 +26,10 @@ Rules:
|
||||
- `source_candidate_ids` are host-derived from the applied candidate. Do not attempt to author them.
|
||||
- `derived_from` must contain only exact existing Memory revision refs returned by reads.
|
||||
- Applied responses return affected refs with `{memory_id, revision, operation}`. `duplicate` and `already_covered` may reference exact existing revisions; `discarded` and `invalid` require a concrete reason.
|
||||
- Surface generation is a separate clean-context Host lifecycle that runs once after this consolidation turn commits. `surface_dirty: true` means confirmed Memory changed; never treat later surface generation failure as failure of an already returned committed decision.
|
||||
- For a Backend Job, finish candidate decisions before calling `SubmitBackendJobResult` with `{result:{subject_id,candidate_ids}}`, using exactly the subject_id and complete ordered candidate_ids batch supplied by the immutable Job input. Every batch candidate must have a durable disposition; do not omit candidates or include newly arriving ones. Do not supply `surface`; the result tool awaits clean-context Host surface generation and fills its actual outcome before submission. Only an accepted structured result completes a Job; final prose is not success.
|
||||
- `surface_dirty: true` means confirmed Memory changed. Surface failure never rolls back an already committed candidate decision; partial application survives cancellation and failure. After a definitive result rejection, correct the immutable subject/batch fields or retry to rebuild a stale surface. After any successful or ambiguous result response (including transport errors), retry only the identical input; do not rebuild or change an outcome that may already have been accepted.
|
||||
- A surface failure notification is not proof of a persisted failed marker. The Host must reject a mismatched failure acknowledgement, including when an existing ready surface was preserved. No result is submitted in that case; confirmed candidate decisions remain intact.
|
||||
- Legacy non-Job consolidation retains its separate post-commit Host surface lifecycle.
|
||||
- An empty model response or transport failure is not a candidate disposition. Only a successful `MemoryApplyCandidate` response closes a candidate.
|
||||
|
||||
Valid non-applied close actions are `discarded`, `invalid`, `duplicate`, and `already_covered`. Always provide a specific reason.
|
||||
|
||||
Reference in New Issue
Block a user