Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: record confirmed public Session history through durable Journal Jobs

Browse Source
This commit is contained in:
Hare
2026-10-16 02:19:24 +09:00
parent 18384482f6
commit 81af1f085c
62 changed files with 14553 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/backend_job.rs
+51 -1
View File
@@ -259,11 +259,13 @@ const fn default_max_attempts() -> u8 {
pub struct BackendJobGrants {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub subjektiv_consolidation: Option<SubjektivConsolidationGrant>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub subjektiv_journal: Option<SubjektivJournalGrant>,
}
impl BackendJobGrants {
pub fn is_empty(&self) -> bool {
self.subjektiv_consolidation.is_none()
self.subjektiv_consolidation.is_none() && self.subjektiv_journal.is_none()
}
}
@@ -276,6 +278,20 @@ pub struct SubjektivConsolidationGrant {
pub candidate_ids: Vec<String>,
}
/// A frozen public-source capability, separate from subject Memory administration.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct SubjektivJournalGrant {
pub subject_id: String,
pub session_id: String,
pub journal_id: String,
pub selected_ranges: Vec<subjektiv::journal::JournalSourceRange>,
}
pub fn journal_serialization_key(journal_id: &str) -> String {
format!("subjektiv-journal:{journal_id}")
}
/// Common runner lock identity for a subject, independent of the candidate snapshot.
pub fn subjektiv_serialization_key(subject_id: &str) -> String {
format!("subjektiv-consolidation:{subject_id}")
@@ -360,6 +376,34 @@ impl BackendJobRequest {
));
}
}
if let Some(grant) = &self.grants.subjektiv_journal {
if self.grants.subjektiv_consolidation.is_some() {
return Err(Error::InvalidInput(
"Journal and Memory grants are mutually exclusive".into(),
));
}
for (name, id) in [
("journal subject", &grant.subject_id),
("journal session", &grant.session_id),
("journal id", &grant.journal_id),
] {
validate_identifier(name, id, 256)?;
}
if grant.selected_ranges.is_empty()
|| grant.selected_ranges.len() > subjektiv::journal::MAX_JOURNAL_RANGES
|| grant.selected_ranges.iter().any(|r| {
r.source.session_id != grant.session_id
|| r.bytes.start >= r.bytes.end
|| r.bytes.end > r.source.total_bytes
})
|| self.serialization_key.as_deref()
!= Some(journal_serialization_key(&grant.journal_id).as_str())
{
return Err(Error::InvalidInput(
"invalid Journal range capability or serialization key".into(),
));
}
}
request.validate_input().map_err(backend_error)?;
self.limits.validate()
}
@@ -391,6 +435,12 @@ impl BackendJobRequest {
.subjektiv_consolidation
.as_ref()
.map(|grant| subjektiv_serialization_key(&grant.subject_id))
.or_else(|| {
self.grants
.subjektiv_journal
.as_ref()
.map(|g| journal_serialization_key(&g.journal_id))
})
.or_else(|| self.serialization_key.clone())
}
crates/workspace-server/src/hosts.rs
+205
View File
@@ -581,6 +581,7 @@ pub enum WorkerSpawnIntent {
purpose: String,
input_digest: String,
subjektiv_consolidation: bool,
subjektiv_journal: bool,
},
TicketRole {
ticket_id: String,
@@ -1287,6 +1288,32 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
})
}
fn session_material_pin(
&self,
_request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, String> {
Err("Session source retention pin is unavailable on this Runtime".into())
}
fn session_material_release(
&self,
_request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, String> {
Err("Session source retention release is unavailable on this Runtime".into())
}
fn session_material_catalog(
&self,
_request: runtime_api::SessionMaterialCatalogRequest,
) -> Result<runtime_api::SessionMaterialCatalogAvailability, String> {
Err("canonical public material catalog is unavailable on this Runtime".into())
}
fn session_material_read(
&self,
_request: runtime_api::SessionMaterialReadRequest,
) -> Result<runtime_api::SessionMaterialReadAvailability, String> {
Err("canonical public material read is unavailable on this Runtime".into())
}
fn session_public_search(
&self,
_request: runtime_api::SessionPublicSearchRequest,
@@ -2210,6 +2237,64 @@ impl RuntimeRegistry {
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
}
pub fn session_material_pin(
&self,
runtime_id: &str,
request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.session_material_pin(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "session_material_pin_failed".into(),
message,
})
}
pub fn session_material_release(
&self,
runtime_id: &str,
request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.session_material_release(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "session_material_release_failed".into(),
message,
})
}
pub fn session_material_catalog(
&self,
runtime_id: &str,
request: runtime_api::SessionMaterialCatalogRequest,
) -> Result<runtime_api::SessionMaterialCatalogAvailability, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.session_material_catalog(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "session_material_catalog_failed".into(),
message,
})
}
pub fn session_material_read(
&self,
runtime_id: &str,
request: runtime_api::SessionMaterialReadRequest,
) -> Result<runtime_api::SessionMaterialReadAvailability, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.session_material_read(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "session_material_read_failed".into(),
message,
})
}
pub fn session_public_search(
&self,
runtime_id: &str,
@@ -3190,6 +3275,78 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
})
}
fn session_material_pin(
&self,
request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, String> {
if request.workspace_id != self.workspace_id {
return Err("Session pin Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
let pin = session_store::retention::SessionRetentionPin(
uuid::Uuid::parse_str(&request.pin_id).map_err(|e| e.to_string())?,
);
self.runtime
.pin_session_public_source_scoped(
&scope,
&request.source,
&request.expected_session_id,
pin,
)
.map_err(|e| e.to_string())?;
Ok(runtime_api::SessionMaterialPinResponse {
session_id: request.expected_session_id,
pin_id: request.pin_id,
})
}
fn session_material_release(
&self,
request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, String> {
if request.workspace_id != self.workspace_id {
return Err("Session release Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
let pin = session_store::retention::SessionRetentionPin(
uuid::Uuid::parse_str(&request.pin_id).map_err(|e| e.to_string())?,
);
self.runtime
.release_session_public_source_pin_scoped(
&scope,
&request.source,
&request.expected_session_id,
pin,
)
.map_err(|e| e.to_string())?;
Ok(runtime_api::SessionMaterialPinResponse {
session_id: request.expected_session_id,
pin_id: request.pin_id,
})
}
fn session_material_catalog(
&self,
request: runtime_api::SessionMaterialCatalogRequest,
) -> Result<runtime_api::SessionMaterialCatalogAvailability, String> {
if request.workspace_id != self.workspace_id {
return Err("canonical public material Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
Ok(self
.runtime
.session_material_catalog_scoped(&scope, &request))
}
fn session_material_read(
&self,
request: runtime_api::SessionMaterialReadRequest,
) -> Result<runtime_api::SessionMaterialReadAvailability, String> {
if request.workspace_id != self.workspace_id {
return Err("canonical public material Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
Ok(self.runtime.session_material_read_scoped(&scope, &request))
}
fn session_public_search(
&self,
request: runtime_api::SessionPublicSearchRequest,
@@ -5323,6 +5480,52 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
})
}
fn session_material_pin(
&self,
request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.session_material_pin(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn session_material_release(
&self,
request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.session_material_release(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn session_material_catalog(
&self,
request: runtime_api::SessionMaterialCatalogRequest,
) -> Result<runtime_api::SessionMaterialCatalogAvailability, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.session_material_catalog(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn session_material_read(
&self,
request: runtime_api::SessionMaterialReadRequest,
) -> Result<runtime_api::SessionMaterialReadAvailability, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.session_material_read(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn session_public_search(
&self,
request: runtime_api::SessionPublicSearchRequest,
@@ -5602,12 +5805,14 @@ fn runtime_create_worker_request(
attempt_id,
input_digest,
subjektiv_consolidation,
subjektiv_journal,
..
} => Some(worker_runtime::catalog::BackendJobExecutionBinding {
job_id: job_id.clone(),
attempt_id: attempt_id.clone(),
input_digest: Some(input_digest.clone()),
subjektiv_consolidation: *subjektiv_consolidation,
subjektiv_journal: *subjektiv_journal,
}),
_ => None,
},
crates/workspace-server/src/retention.rs
+34 -2
View File
@@ -226,7 +226,7 @@ impl SqliteWorkspaceStore {
let now = Utc::now().to_rfc3339();
self.with_conn_mut(|conn| {
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let policy=load_policy(&tx,&req.workspace_id)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
Some(v)=>v,
None=>{
@@ -272,7 +272,7 @@ impl SqliteWorkspaceStore {
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
}
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
let policy=load_policy(&tx,workspace_id)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
let policy=load_removal_policy(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
return Err(stale_error(&plan,"policy content changed"));
@@ -785,6 +785,36 @@ struct WorkerRow {
fn load_worker(c: &Connection, w: &str, r: &RuntimeWorkerRef) -> crate::Result<Option<WorkerRow>> {
c.query_row("SELECT display_name,profile,retention_state,created_at,updated_at FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",params![w,r.runtime_id,r.worker_id],|x|Ok(WorkerRow{display_name:x.get(0)?,profile:x.get(1)?,retention_state:x.get(2)?,created_at:x.get(3)?,updated_at:x.get(4)?})).optional().map_err(StoreError::from)
}
/// Recording Sessions may contain original text in tool results. Only a terminal,
/// Host-bound Journal attempt grants this exception; profile names grant nothing.
/// This plan-local policy never changes the Workspace conversation policy.
fn load_removal_policy(
c: &Connection,
w: &str,
worker: &RuntimeWorkerRef,
) -> crate::Result<Option<WorkerRetentionPolicy>> {
let Some(mut policy) = load_policy(c, w)? else {
return Ok(None);
};
if crate::store::terminal_journal_job_worker(c, w, worker)? {
let update = WorkerRetentionPolicyUpdate {
policy_id: "backend-journal-ephemeral-session-v1".into(),
session_disposition: SessionDisposition::Purge,
metadata_disposition: MetadataDisposition::Tombstone,
archive_retention: ArchiveRetention::Forever,
diagnostics_disposition: DiagnosticsDisposition::Purge,
diagnostics_retention_seconds: None,
};
policy.policy_id = update.policy_id.clone();
policy.policy_digest = worker_retention_policy_digest(&update);
policy.session_disposition = update.session_disposition;
policy.metadata_disposition = update.metadata_disposition;
policy.archive_retention = update.archive_retention;
policy.diagnostics_disposition = update.diagnostics_disposition;
policy.diagnostics_retention_seconds = None;
}
Ok(Some(policy))
}
fn load_policy(c: &Connection, w: &str) -> crate::Result<Option<WorkerRetentionPolicy>> {
c.query_row(
"SELECT p.policy_id,p.policy_digest,r.session_disposition,r.metadata_disposition,
@@ -1424,6 +1454,7 @@ mod tests {
source_runtime_id: "r".into(),
source_worker_id: worker_id(),
source_session_id: "s".into(),
source_active_segment_id: None,
segment_ids: vec!["a".into()],
source_created_at: "created".into(),
removed_at: "removed".into(),
@@ -2361,6 +2392,7 @@ mod tests {
source_runtime_id: "r".into(),
source_worker_id: worker_id(),
source_session_id: "s".into(),
source_active_segment_id: None,
segment_ids: vec!["a".into()],
source_created_at: "created".into(),
removed_at: "removed".into(),
crates/workspace-server/src/server.rs
+217 -10
View File
@@ -27,6 +27,7 @@
use subjektiv::api::*;
mod backend_job_profile;
mod subjektiv_job;
mod subjektiv_journal;
mod worker_operations;
mod worker_protocol;
pub(crate) use worker_operations::{
@@ -1650,6 +1651,7 @@ pub struct WorkspaceApi {
Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<Mutex<OrchestratorAttentionState>>>>>,
notification_delivery_lifecycle: Arc<notification_delivery::NotificationDeliveryLifecycle>,
backend_job_drain_requests: Arc<AtomicU64>,
journal_dispatch_owner: Option<Arc<WorkspaceApi>>,
observation_proxy: BackendObservationProxy,
runtime_subscription_broker: RuntimeSubscriptionBroker,
pub(crate) worker_projection: Arc<crate::worker_projection::WorkerProjectionService>,
@@ -1728,6 +1730,9 @@ impl ServerApiContractService {
#[derive(Clone)]
struct WorkerRemovalService {
workspace_id: String,
journal_storage: crate::WorkspaceFeatureStorage,
journal_registration: crate::RegisteredFeature,
journal_dispatch_api: Weak<WorkspaceApi>,
store: Arc<dyn ControlPlaneStore>,
host_executors: HostExecutors,
runtime: Weak<RuntimeRegistry>,
@@ -1742,6 +1747,12 @@ impl WorkerRemovalService {
fn new(api: &WorkspaceApi) -> Self {
Self {
workspace_id: api.config.workspace_id.clone(),
journal_storage: api.feature_storage.clone(),
journal_registration: api.subjektiv_registration.clone(),
journal_dispatch_api: api
.journal_dispatch_owner
.as_ref()
.map_or_else(Weak::new, Arc::downgrade),
store: api.store.clone(),
host_executors: api.host_executors.clone(),
runtime: Arc::downgrade(&api.runtime),
@@ -1877,6 +1888,13 @@ impl WorkerRemovalService {
target: &RuntimeWorkerRef,
prepared: crate::retention::PreparedWorkerRemoval,
) -> std::result::Result<worker::WorkspaceResponse, String> {
if let Err(error) = subjektiv_journal::validate_removal_retention(self, target, &prepared) {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"journal_retention_unconfirmed",
&error.to_string(),
));
}
let result =
match runtime.execute_worker_retention(target, prepared.runtime_request.clone()) {
Ok(result) => result,
@@ -1898,7 +1916,10 @@ impl WorkerRemovalService {
)
.map(|commit| ((), commit.catalog))
}) {
Ok(()) => Ok(worker_remove_success_response(target)),
Ok(()) => {
subjektiv_journal::after_worker_removal(self, &result);
Ok(worker_remove_success_response(target))
}
Err(error) => Ok(worker_retention_error_response(error)),
}
}
@@ -2166,6 +2187,13 @@ impl WorkerRemovalService {
));
}
if let Err(error) = subjektiv_journal::prepare_worker_removal(self, runtime, target) {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"journal_handoff_incomplete",
&error.to_string(),
));
}
let inventory = match runtime.worker_retention_inventory(&target) {
Ok(inventory) => inventory,
Err(_) => {
@@ -2238,6 +2266,13 @@ impl WorkerRemovalService {
));
}
if let Err(error) = subjektiv_journal::validate_removal_retention(self, target, &prepared) {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"journal_retention_unconfirmed",
&error.to_string(),
));
}
let retention_result =
match runtime.execute_worker_retention(&target, prepared.runtime_request.clone()) {
Ok(result) => result,
@@ -2276,6 +2311,7 @@ impl WorkerRemovalService {
return Ok(worker_retention_error_response(error));
}
};
subjektiv_journal::after_worker_removal(self, &retention_result);
Ok(worker_remove_success_response(&target))
}
}
@@ -3896,6 +3932,7 @@ impl WorkspaceApi {
notification_delivery_lifecycle: Arc::new(Default::default()),
orchestrator_attention: Arc::new(Mutex::new(HashMap::new())),
backend_job_drain_requests: Arc::new(AtomicU64::new(0)),
journal_dispatch_owner: None,
observation_proxy,
runtime_subscription_broker,
worker_projection,
@@ -3927,6 +3964,9 @@ impl WorkspaceApi {
api.authority = api.authority.with_source_commit_resolver(Arc::new(
RuntimeTicketSourceCommitResolver { api: observer_api },
));
// The owner snapshot does not retain itself. Runtime removal dispatchers hold only
// Weak to it, so durable handoff can wake the existing runner without a Runtime cycle.
api.journal_dispatch_owner = Some(Arc::new(api.clone()));
if let Some(dispatcher) = worker_remove_dispatcher {
dispatcher
.install_executor(Arc::new(
@@ -3979,6 +4019,7 @@ impl WorkspaceApi {
.run(move || {
notification_delivery::recover_job_notifications(&api);
api.recover_backend_jobs()?;
subjektiv_journal::recover(&api)?;
if let Some(orchestrator) = find_online_workspace_orchestrator(&api) {
dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker);
}
@@ -4037,6 +4078,7 @@ impl WorkspaceApi {
{
return Ok(reservation);
}
subjektiv_journal::prepare(self, &reservation)?;
let (reservation, claimed) = self.store.claim_backend_job_attempt_dispatch(
&self.config.workspace_id,
&reservation.job.request.job_id,
@@ -4076,6 +4118,7 @@ impl WorkspaceApi {
purpose: request.purpose.clone(),
input_digest: request.input_digest()?,
subjektiv_consolidation: request.grants.subjektiv_consolidation.is_some(),
subjektiv_journal: request.grants.subjektiv_journal.is_some(),
},
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
expected_segments: 0,
@@ -4091,7 +4134,8 @@ impl WorkspaceApi {
resolved_worker_observation_grants: Vec::new(),
resolved_workspace_api: None,
resolved_memory_settings: None,
resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some(),
resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some()
|| request.grants.subjektiv_journal.is_some(),
resolved_control_operation: Some(control_operation),
},
) {
@@ -4354,6 +4398,17 @@ impl WorkspaceApi {
failure_category: &str,
failure_detail: &str,
) -> Result<BackendJobAttemptRecord> {
if let Some(job) = self
.store
.get_backend_job(&self.config.workspace_id, job_id)?
&& job.request.grants.subjektiv_journal.is_some()
&& state != BackendJobAttemptState::Unknown
&& let Some(attempt) =
self.store
.get_backend_job_attempt(&self.config.workspace_id, job_id, attempt_id)?
{
subjektiv_journal::close(self, &job.request, &attempt)?;
}
let attempt = self.store.finish_backend_job_attempt(
&self.config.workspace_id,
job_id,
@@ -4363,8 +4418,21 @@ impl WorkspaceApi {
failure_detail,
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
)?;
let _ = self.schedule_backend_job_worker_cleanup(&attempt);
Ok(attempt)
// A retained checkpoint can prove the Journal result even if teardown is
// blocked. Do not make outcome resolution depend on successful Session cleanup.
if attempt.state == BackendJobAttemptState::Unknown
&& let Some(job) = self.store.get_backend_job(self.workspace_id(), job_id)?
&& job.request.grants.subjektiv_journal.is_some()
&& let Err(error) = subjektiv_journal::reconcile_unknown(self, &job, &attempt)
{
tracing::warn!(%error,%job_id,%attempt_id,"Journal Unknown outcome remains recoverable");
}
let current = self
.store
.get_backend_job_attempt(self.workspace_id(), job_id, attempt_id)?
.ok_or_else(|| Error::Store("finished Backend Job attempt disappeared".into()))?;
let _ = self.schedule_backend_job_worker_cleanup(&current);
Ok(current)
}
/// Admit on the independent cleanup lane before claiming durable work. Rejection leaves
@@ -4434,7 +4502,22 @@ impl WorkspaceApi {
// the safety boundary for those legacy leftovers.
return Ok(());
};
let journal_request = self
.store
.get_backend_job(self.workspace_id(), &attempt.job_id)?
.filter(|job| job.request.grants.subjektiv_journal.is_some())
.map(|job| job.request);
let deadline = chrono::DateTime::parse_from_rfc3339(&attempt.deadline_at)
.map_err(|e| Error::Store(e.to_string()))?;
loop {
if let Some(request) = &journal_request {
// Preserve normal result-response teardown ordering, but an explicit
// erasure or the immutable Job deadline must not wait on a stuck Run.
// Actual stop and canonical Purge still happen below this wait.
if Utc::now() >= deadline || subjektiv_journal::session_erased(self, request)? {
return Ok(());
}
}
match self.runtime.worker(worker) {
Ok(detail)
if detail.state.eq_ignore_ascii_case("stopped")
@@ -4525,6 +4608,27 @@ impl WorkspaceApi {
})
};
let complete = || -> Result<()> {
let journal = self
.store
.get_backend_job(self.workspace_id(), job_id)?
.is_some_and(|job| job.request.grants.subjektiv_journal.is_some());
if journal {
let removal = self
.config_store
.recover_worker_removal_execution(self.workspace_id(), &worker)
.map_err(|e| Error::Store(e.to_string()))?;
if !removal.is_some_and(|p| {
p.plan.state == crate::retention::WorkerRemovalPlanState::Succeeded
&& p.runtime_request.session_disposition
== worker_runtime::retention::SessionDisposition::Purge
}) {
return fail(
"journal_job_purge_unconfirmed",
"Journal execution Session requires a committed canonical Purge receipt"
.into(),
);
}
}
self.store.finish_backend_job_worker_cleanup(
&self.config.workspace_id,
job_id,
@@ -4534,6 +4638,26 @@ impl WorkspaceApi {
None,
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
)?;
if let Some(job) = self
.store
.get_backend_job(&self.config.workspace_id, job_id)?
&& job.request.grants.subjektiv_journal.is_some()
{
// Cleanup completion is a durable stop/removal fence. Unknown is
// reconciled with retained domain receipts before closing its generation.
let current = self
.store
.get_backend_job_attempt(self.workspace_id(), job_id, attempt_id)?
.ok_or_else(|| Error::Store("cleaned Journal attempt missing".into()))?;
if current.state == BackendJobAttemptState::Unknown {
subjektiv_journal::reconcile_unknown(self, &job, &current)?;
} else if !subjektiv_journal::session_erased(self, &job.request)? {
subjektiv_journal::close(self, &job.request, &current)?;
if current.state == BackendJobAttemptState::Completed {
subjektiv_journal::drain_after_completion(self, &job.request);
}
}
}
Ok(())
};
@@ -4700,10 +4824,28 @@ impl WorkspaceApi {
)?;
continue;
}
let worker_known = attempt
let observed = attempt
.worker
.as_ref()
.is_some_and(|worker| self.runtime.worker(worker).is_ok());
.map(|worker| self.runtime.worker(worker));
if observed.as_ref().is_some_and(|result| {
result.as_ref().is_ok_and(|detail| {
detail.state.eq_ignore_ascii_case("stopped")
|| detail.execution_reconstructed
})
}) && self
.store
.get_backend_job(self.workspace_id(), &attempt.job_id)?
.is_some_and(|job| job.request.grants.subjektiv_journal.is_some())
{
// This is not success evidence. Move to Unknown to fence tools;
// only domain completion or confirmed canonical cleanup resolves it.
self.finish_backend_job_attempt(&attempt.job_id, &attempt.attempt_id,
BackendJobAttemptState::Unknown, "recovery_journal_execution_ended",
"Runtime reports the prior Journal execution ended; reconciling saved domain receipts")?;
continue;
}
let worker_known = observed.as_ref().is_some_and(|result| result.is_ok());
if !worker_known {
self.finish_backend_job_attempt(
&attempt.job_id,
@@ -4725,6 +4867,7 @@ impl WorkspaceApi {
| BackendJobAttemptState::Unknown => {}
}
}
subjektiv_journal::recover_unknowns(self)?;
self.recover_backend_job_worker_cleanups()?;
self.drain_reserved_backend_jobs();
Ok(())
@@ -4859,12 +5002,64 @@ impl WorkspaceApi {
verified_submission.result =
subjektiv_job::with_dispositions(self, &job.request, &submission.result)?;
}
if let Some(job) = self
.store
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
&& job.request.grants.subjektiv_journal.is_some()
{
let attempt = self
.store
.get_backend_job_attempt(
&self.config.workspace_id,
&submission.job_id,
&submission.attempt_id,
)?
.ok_or_else(|| {
Error::WorkspacePermissionDenied("Journal attempt missing".into())
})?;
if attempt.worker.as_ref() != Some(source_worker)
|| job.current_attempt != attempt.attempt
|| attempt.input_digest != submission.input_digest
|| !matches!(
attempt.state,
BackendJobAttemptState::Dispatched | BackendJobAttemptState::Completed
)
{
return Err(Error::WorkspacePermissionDenied(
"Journal result requires the bound live attempt".into(),
));
}
if attempt.state != BackendJobAttemptState::Completed {
if chrono::DateTime::parse_from_rfc3339(&attempt.deadline_at)
.map_err(|e| Error::Store(e.to_string()))?
<= Utc::now()
{
return Err(Error::WorkspacePermissionDenied(
"Journal attempt expired".into(),
));
}
subjektiv_journal::validate_result(
self,
&job.request,
&attempt,
&submission.result,
)?;
}
}
let acceptance = self.store.accept_backend_job_result(
&self.config.workspace_id,
source_worker,
&verified_submission,
&accepted_at,
)?;
if let Some(job) = self
.store
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
&& job.request.grants.subjektiv_journal.is_some()
{
subjektiv_journal::close(self, &job.request, &acceptance.attempt)?;
subjektiv_journal::drain_after_completion(self, &job.request);
}
notification_delivery::deliver_job_result(self, &acceptance);
let _ = self.schedule_backend_job_worker_cleanup(&acceptance.attempt);
self.drain_reserved_backend_jobs();
@@ -4907,13 +5102,21 @@ impl WorkspaceApi {
.register_remote_runtime(remote_config);
let projection = self.worker_projection.clone();
let broker = self.runtime_subscription_broker.clone();
let journal_api = self.clone();
tokio::spawn(async move {
if let Err(error) = projection
.attach_runtime(&broker, runtime_id.as_str())
.await
{
tracing::warn!(%runtime_id, %error, "failed to attach Worker projection Runtime");
return;
}
let recovery = journal_api.clone();
if let Err(error)=journal_api.host_executors.host().and_then(|executor|executor.dispatch(move || {
if let Err(error)=subjektiv_journal::recover(&recovery) {
tracing::warn!(%error,"Journal reconnect reconciliation blocked; saved requests retained");
}
})) {tracing::warn!(%error,"Journal reconnect reconciliation not admitted");}
});
Ok(())
}
@@ -12417,6 +12620,9 @@ fn build_inner_router(api: WorkspaceApi) -> Router {
"/api/w/{workspace_id}/workers/self/backend-job-result",
post(scoped_submit_backend_job_result),
)
.route("/api/w/{workspace_id}/subjektiv/journal/recording", post(subjektiv_journal::recording_route))
.route("/api/w/{workspace_id}/subjektiv/journal/job", post(subjektiv_journal::job_route))
.route("/api/w/{workspace_id}/subjektiv/journal/job/finish", post(subjektiv_journal::finish_route))
.fallback(get(static_or_spa_fallback))
.with_state(api)
.merge(generated_workspace_contract_router(contract_service));
@@ -35878,6 +36084,7 @@ mod tests {
mod protocol_backpressure_tests;
mod subject_spawn_tests;
mod subjektiv_jobs_tests;
mod subjektiv_journal_tests;
mod ticket_evidence_tests;
mod ticket_worker_claim_tests;
mod ticket_worker_launch_tests;
@@ -46845,7 +47052,7 @@ mod tests {
worker_root.join("session/session.json"),
serde_json::to_vec_pretty(&json!({
"schema_version": 1,
"session_id": format!("backend-job-cleanup-{}", worker.worker_id)
"session_id": session_store::new_session_id()
}))
.unwrap(),
)
@@ -55151,7 +55358,7 @@ mod tests {
worker_root.join("session/session.json"),
serde_json::to_vec_pretty(&json!({
"schema_version": 1,
"session_id": "worker-remove-session"
"session_id": session_store::new_session_id()
}))
.unwrap(),
)
@@ -58360,7 +58567,7 @@ mod tests {
worker_root.join("session/session.json"),
serde_json::to_vec_pretty(&json!({
"schema_version": 1,
"session_id": "singleton-cleanup-session"
"session_id": session_store::new_session_id()
}))
.unwrap(),
)
@@ -58466,7 +58673,7 @@ mod tests {
worker_root.join("session/session.json"),
serde_json::to_vec_pretty(&json!({
"schema_version": 1,
"session_id": "cleanup-removal-session"
"session_id": session_store::new_session_id()
}))
.unwrap(),
)
crates/workspace-server/src/server/subjektiv_job.rs
+1 -1
View File
@@ -204,7 +204,7 @@ pub(super) fn dispatch(
instruction: "Resolve ONLY the immutable candidate_ids batch using MemoryStagingList/Read, confirmed Memory reads and MemoryApplyCandidate. Already resolved candidates are retained receipts, not work to repeat. Leave newly arriving candidates to a subsequent Job. When every batch candidate has a durable disposition, call SubmitBackendJobResult with {subject_id, candidate_ids}; its Host capability awaits bounded clean-context surface generation and adds the actual surface outcome before submission. Final prose is not completion.".into(),
// Selection belongs here, never in the common runner's purpose switch.
profile: format!("builtin:{SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE}"),
grants: BackendJobGrants { subjektiv_consolidation: Some(SubjektivConsolidationGrant { subject_id: subject_id.into(), candidate_ids }) },
grants: BackendJobGrants { subjektiv_consolidation: Some(SubjektivConsolidationGrant { subject_id: subject_id.into(), candidate_ids }), ..Default::default() },
serialization_key: None,
source_worker: None,
notification_target: None,
crates/workspace-server/src/server/subjektiv_journal.rs
+1397
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs
+1499
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/store.rs
+125
View File
@@ -1679,6 +1679,29 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
submission: &BackendJobResultSubmission,
now: &str,
) -> Result<BackendJobResultAcceptance>;
/// Host-only reconciliation of a saved domain completion, never model claims.
/// None requires completed canonical cleanup, not Worker liveness inference.
fn reconcile_backend_journal_attempt(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
completion: Option<&BackendJobResultSubmission>,
now: &str,
) -> Result<BackendJobAttemptRecord>;
/// Bounded keyset over current Journal Unknown attempts, independent of source liveness.
fn list_backend_journal_unknown_attempts(
&self,
workspace_id: &str,
after_job_id: Option<&str>,
limit: usize,
) -> Result<Vec<BackendJobAttemptRecord>>;
fn list_backend_journal_jobs(
&self,
workspace_id: &str,
session_id: &str,
after: Option<&str>,
limit: usize,
) -> Result<Vec<BackendJobRecord>>;
fn finish_backend_job_attempt(
&self,
workspace_id: &str,
@@ -7026,6 +7049,82 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
})
}
fn list_backend_journal_unknown_attempts(
&self,
workspace_id: &str,
after_job_id: Option<&str>,
limit: usize,
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn(|conn| {
let mut stmt = conn.prepare("SELECT a.job_id,a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='unknown' AND j.state='unknown' AND a.attempt=j.current_attempt AND json_type(j.request_json,'$.grants.subjektiv_journal')='object' AND (?2 IS NULL OR a.job_id>?2) ORDER BY a.job_id LIMIT ?3")?;
let ids = stmt.query_map(params![workspace_id,after_job_id,limit.clamp(1,100) as i64], |r| Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
ids.into_iter().map(|(job,id)| read_backend_job_attempt(conn,workspace_id,&job,&id)?.ok_or_else(|| Error::Store("Journal Unknown attempt disappeared".into()))).collect()
})
}
fn list_backend_journal_jobs(
&self,
workspace_id: &str,
session_id: &str,
after: Option<&str>,
limit: usize,
) -> Result<Vec<BackendJobRecord>> {
self.with_conn(|conn| {
let mut stmt = conn.prepare("SELECT job_id FROM backend_jobs WHERE workspace_id=?1 AND json_extract(request_json,'$.grants.subjektiv_journal.session_id')=?2 AND (?3 IS NULL OR job_id>?3) ORDER BY job_id LIMIT ?4")?;
let ids = stmt.query_map(params![workspace_id,session_id,after,limit.clamp(1,100) as i64], |r| r.get::<_,String>(0))?.collect::<rusqlite::Result<Vec<_>>>()?;
ids.iter().map(|id| read_backend_job(conn, workspace_id, id)?.ok_or_else(|| Error::Store("Journal Job disappeared".into()))).collect()
})
}
fn reconcile_backend_journal_attempt(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
completion: Option<&BackendJobResultSubmission>,
now: &str,
) -> Result<BackendJobAttemptRecord> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?;
let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?;
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
if &attempt != expected || job.current_attempt != attempt.attempt
|| job.request.grants.subjektiv_journal.is_none()
|| attempt.input_digest != job.request.input_digest()?
|| !matches!(job.state, BackendJobState::Unknown | BackendJobState::Completed | BackendJobState::Failed) {
return Err(Error::InvalidInput("Journal reconciliation binding changed".into()));
}
if let Some(submission) = completion {
if submission.job_id != expected.job_id || submission.attempt_id != expected.attempt_id || submission.input_digest != expected.input_digest {
return Err(Error::InvalidInput("Journal completion binding mismatch".into()));
}
let (digest, json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
if attempt.state == BackendJobAttemptState::Completed && attempt.result_digest.as_deref() == Some(digest.as_str()) { tx.commit()?; return Ok(attempt); }
if attempt.state != BackendJobAttemptState::Unknown || job.state != BackendJobState::Unknown {
return Err(Error::InvalidInput("Journal completion cannot replace definitive outcome".into()));
}
tx.execute("UPDATE backend_job_attempts SET state='completed',result_json=?4,result_digest=?5,failure_category=NULL,failure_detail=NULL,updated_at=?6,completed_at=?6 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='unknown'",params![workspace_id,expected.job_id,expected.attempt_id,json,digest,now])?;
tx.execute("UPDATE backend_jobs SET state='completed',result_json=?3,result_digest=?4,failure_category=NULL,failure_detail=NULL,updated_at=?5,completed_at=?5 WHERE workspace_id=?1 AND job_id=?2 AND state='unknown'",params![workspace_id,expected.job_id,json,digest,now])?;
notification_routing::jobs::capture(&tx,workspace_id,&expected.job_id,&expected.attempt_id)?;
} else {
if attempt.state == BackendJobAttemptState::Failed && attempt.failure_category.as_deref() == Some("journal_attempt_fenced") { tx.commit()?; return Ok(attempt); }
if attempt.state != BackendJobAttemptState::Unknown || job.state != BackendJobState::Unknown
|| attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed)
|| attempt.worker_cleanup_completed_at.is_none() {
return Err(Error::InvalidInput("partial Journal Unknown requires completed canonical cleanup fence".into()));
}
let purged = if let Some(worker) = &attempt.worker {
tx.query_row("SELECT EXISTS(SELECT 1 FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='succeeded' AND session_disposition='purge')",params![workspace_id,worker.runtime_id,worker.worker_id],|r|r.get::<_,bool>(0))?
} else { false };
if !purged { return Err(Error::InvalidInput("partial Journal Unknown has no committed stop/Purge removal receipt".into())); }
tx.execute("UPDATE backend_job_attempts SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed; retained operations remain valid',updated_at=?4 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='unknown'",params![workspace_id,expected.job_id,expected.attempt_id,now])?;
tx.execute("UPDATE backend_jobs SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed',updated_at=?3 WHERE workspace_id=?1 AND job_id=?2 AND state='unknown'",params![workspace_id,expected.job_id,now])?;
}
let result = read_backend_job_attempt(&tx,workspace_id,&expected.job_id,&expected.attempt_id)?.ok_or_else(|| Error::Store("Reconciled Journal attempt missing".into()))?;
tx.commit()?; Ok(result)
})
}
fn finish_backend_job_attempt(
&self,
workspace_id: &str,
@@ -11357,6 +11456,31 @@ fn read_active_backend_job_for_resource(
.map(Option::flatten)
}
/// Purge authority comes from the immutable request and exact attempt/Worker/input
/// binding, never from a profile or purpose string.
pub(crate) fn terminal_journal_job_worker(
conn: &Connection,
workspace: &str,
worker: &RuntimeWorkerRef,
) -> Result<bool> {
let identity: Option<(String, String)> = conn.query_row(
"SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
params![workspace, worker.runtime_id, worker.worker_id], |r| Ok((r.get(0)?, r.get(1)?)),
).optional()?;
let Some((job_id, attempt_id)) = identity else {
return Ok(false);
};
let job = read_backend_job(conn, workspace, &job_id)?
.ok_or_else(|| Error::Store("Journal cleanup Job missing".into()))?;
let attempt = read_backend_job_attempt(conn, workspace, &job_id, &attempt_id)?
.ok_or_else(|| Error::Store("Journal cleanup attempt missing".into()))?;
job.request.validate()?;
Ok(job.request.grants.subjektiv_journal.is_some()
&& attempt.state.terminal()
&& attempt.worker.as_ref() == Some(worker)
&& attempt.input_digest == job.request.input_digest()?)
}
fn read_backend_job(
conn: &Connection,
workspace_id: &str,
@@ -24091,6 +24215,7 @@ INSERT INTO worker_registry (
subject_id: subject.into(),
candidate_ids: vec![candidate.into()],
}),
..Default::default()
},
serialization_key: None,
source_worker: None,