Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: fence Journal originals before publication and reconcile removal intent

Browse Source 
Protect attributed originals before the first public commit; distinguish pending Session material from missing committed sources. Gate removal admission by the Journal Feature and resolve Subject ownership from durable singleton authority. Resume explicit erasure through signed delete transport and proof-bound purge plans without creating a Journal Job or archive.

Validation: changed-crate suites, root check, doc/fmt/diff/naming checks, Server/Runtime builds and worker_environment isolation suite passed. Public standalone scripted dialog connects Host, automatic Job, sequential tools and drain while dialog continues. Backend public embedded WorkerRemove tests connect removal forwarding, durable Jobs, retention, Runtime and SQLite; Session/attribution/profile fixture setup bypasses creation/auth routes. Runtime signed generated-client tests separately exercise request authentication.

Limits: real-model semantic acceptance condition 6 remains unevaluated without a new explicit model/credential/budget grant. Conditions 1/7/8/11 do not have complete product E2E coverage: worker_environment only verifies isolated startup/connection/cleanup. Compaction tests connect local hooks/reader/store rather than the full public dialog-to-Journal chain. Erasure racing a previously prepared Archive removal plan fails closed with conflict; automatic stale-plan replacement is not implemented.
This commit is contained in:
Hare
2026-10-16 03:59:45 +09:00
parent 9debf43645
commit 5bd49760e5
19 changed files with 1846 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/hosts.rs
+101
View File
@@ -1301,6 +1301,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
Err("Session source retention release is unavailable on this Runtime".into())
}
fn session_material_erase(
&self,
_request: runtime_api::SessionMaterialEraseRequest,
) -> Result<runtime_api::SessionMaterialEraseResponse, String> {
Err("Session source permanent erasure is unavailable on this Runtime".into())
}
fn session_material_catalog(
&self,
_request: runtime_api::SessionMaterialCatalogRequest,
@@ -2266,6 +2273,21 @@ impl RuntimeRegistry {
})
}
pub fn session_material_erase(
&self,
runtime_id: &str,
request: runtime_api::SessionMaterialEraseRequest,
) -> Result<runtime_api::SessionMaterialEraseResponse, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.session_material_erase(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "session_material_erase_failed".into(),
message,
})
}
pub fn session_material_catalog(
&self,
runtime_id: &str,
@@ -3324,6 +3346,27 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
})
}
fn session_material_erase(
&self,
request: runtime_api::SessionMaterialEraseRequest,
) -> Result<runtime_api::SessionMaterialEraseResponse, String> {
if request.workspace_id != self.workspace_id {
return Err("Session erase Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
self.runtime
.delete_session_public_source_scoped(
&scope,
&request.source,
&request.expected_session_id,
session_store::retention::SessionDeletionIntent::ErasePermanently,
)
.map_err(|e| e.to_string())?;
Ok(runtime_api::SessionMaterialEraseResponse {
session_id: request.expected_session_id,
})
}
fn session_material_catalog(
&self,
request: runtime_api::SessionMaterialCatalogRequest,
@@ -5503,6 +5546,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
.map_err(|diagnostic| diagnostic.message)
}
fn session_material_erase(
&self,
request: runtime_api::SessionMaterialEraseRequest,
) -> Result<runtime_api::SessionMaterialEraseResponse, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.session_material_erase(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn session_material_catalog(
&self,
request: runtime_api::SessionMaterialCatalogRequest,
@@ -8262,12 +8317,58 @@ mod tests {
),
"workers:read"
);
assert_eq!(
workspace_runtime_operation("POST", "/v1/session-public/material/retention/erase"),
"workers:delete"
);
assert_eq!(
workspace_runtime_operation("POST", SSH_HOST_KEY_PROBE_PATH),
SSH_HOST_KEY_PROBE_OPERATION
);
}
#[test]
fn remote_session_erasure_uses_the_generated_management_transport() {
// The loopback fixture substitutes only the remote Runtime. This covers
// Registry/remote-adapter DTO and route wiring, not Runtime authorization or deletion.
let session_id = uuid::Uuid::new_v4().to_string();
let (base_url, server) = serve_mock_http(vec![mock_response(
"POST",
"/v1/session-public/material/retention/erase",
true,
200,
json!({"session_id": session_id}).to_string(),
)]);
let registry = RuntimeRegistry::new(Vec::new());
registry.register(
RemoteWorkerRuntime::new(
RemoteRuntimeConfig::new(
"remote:erase",
"Erase transport fixture",
base_url,
Some("secret-token".into()),
),
"workspace-test".into(),
"http://127.0.0.1:8787".into(),
)
.unwrap(),
);
let response = registry
.session_material_erase(
"remote:erase",
runtime_api::SessionMaterialEraseRequest {
workspace_id: "workspace-test".into(),
source: runtime_api::SessionPublicSource::Retained {
worker_id: EmbeddedWorkerId::from_legacy_u64(1),
},
expected_session_id: session_id.clone(),
},
)
.unwrap();
assert_eq!(response.session_id, session_id);
server.join().unwrap();
}
#[test]
fn remote_runtime_registry_routes_commands_without_browser_secret_leaks() {
let worker_id = EmbeddedWorkerId::from_legacy_u64(1).to_string();
crates/workspace-server/src/retention.rs
+70 -6
View File
@@ -63,6 +63,51 @@ pub struct WorkerRemovalPlanRequest {
pub workspace_id: String,
pub worker: RuntimeWorkerRef,
pub reason: String,
/// Internal evidence cannot be supplied through serialized deletion requests.
#[serde(skip)]
pub(crate) journal_erasure: Option<ErasedJournalSession>,
}
const ERASED_JOURNAL_POLICY_ID: &str = "backend-journal-erased-session-v1";
/// Monotonic domain erasure proof, bound to the exact source Worker and Session.
/// The only constructor rereads the Journal tombstone; ordinary Purge grants no erase authority.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct ErasedJournalSession {
workspace_id: String,
worker: RuntimeWorkerRef,
session_id: String,
}
impl ErasedJournalSession {
pub(crate) fn validated(
workspace_id: &str,
store: &crate::subjektiv::SubjektivStore,
attr: &crate::subjektiv::SubjectSessionAttribution,
) -> crate::Result<Option<Self>> {
let host = subjektiv::journal::JournalHostContext::validated_session(store, attr.clone())
.map_err(|e| StoreError::Store(e.to_string()))?;
Ok(store
.journal_session_erased(&host)
.map_err(|e| StoreError::Store(e.to_string()))?
.then(|| Self {
workspace_id: workspace_id.into(),
worker: RuntimeWorkerRef::new(&attr.runtime_id, &attr.worker_id),
session_id: attr.session_id.clone(),
}))
}
fn matches(
&self,
req: &WorkerRemovalPlanRequest,
inventory: &WorkerRetentionInventory,
) -> bool {
self.workspace_id == req.workspace_id
&& self.worker == req.worker
&& inventory
.session_id
.as_ref()
.is_none_or(|id| *id == self.session_id)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -223,10 +268,20 @@ impl SqliteWorkspaceStore {
inv: &WorkerRetentionInventory,
) -> Result<WorkerRemovalPlan, WorkerRetentionError> {
validate_plan(req, inv)?;
if req
.journal_erasure
.as_ref()
.is_some_and(|proof| !proof.matches(req, inv))
{
return Err(WorkerRetentionError::Invalid(
"Journal erasure proof does not match removal source".into(),
));
}
let erased_journal = req.journal_erasure.is_some();
let now = Utc::now().to_rfc3339();
self.with_conn_mut(|conn| {
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
Some(v)=>v,
None=>{
@@ -272,7 +327,7 @@ impl SqliteWorkspaceStore {
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
}
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
let policy=load_removal_policy(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
return Err(stale_error(&plan,"policy content changed"));
@@ -785,20 +840,28 @@ struct WorkerRow {
fn load_worker(c: &Connection, w: &str, r: &RuntimeWorkerRef) -> crate::Result<Option<WorkerRow>> {
c.query_row("SELECT display_name,profile,retention_state,created_at,updated_at FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",params![w,r.runtime_id,r.worker_id],|x|Ok(WorkerRow{display_name:x.get(0)?,profile:x.get(1)?,retention_state:x.get(2)?,created_at:x.get(3)?,updated_at:x.get(4)?})).optional().map_err(StoreError::from)
}
/// Recording Sessions may contain original text in tool results. Only a terminal,
/// Host-bound Journal attempt grants this exception; profile names grant nothing.
/// Recording Sessions may contain original text in tool results. A terminal,
/// Host-bound Journal attempt or a validated source-erasure proof grants this
/// exception; profile names and serialized deletion input grant nothing.
/// A saved erasure plan retains the monotonic decision across restart, but creates
/// no Runtime erase authority: the domain/physical tombstones must already exist.
/// This plan-local policy never changes the Workspace conversation policy.
fn load_removal_policy(
c: &Connection,
w: &str,
worker: &RuntimeWorkerRef,
erased_journal: bool,
) -> crate::Result<Option<WorkerRetentionPolicy>> {
let Some(mut policy) = load_policy(c, w)? else {
return Ok(None);
};
if crate::store::terminal_journal_job_worker(c, w, worker)? {
if erased_journal || crate::store::terminal_journal_job_worker(c, w, worker)? {
let update = WorkerRetentionPolicyUpdate {
policy_id: "backend-journal-ephemeral-session-v1".into(),
policy_id: if erased_journal {
ERASED_JOURNAL_POLICY_ID.into()
} else {
"backend-journal-ephemeral-session-v1".into()
},
session_disposition: SessionDisposition::Purge,
metadata_disposition: MetadataDisposition::Tombstone,
archive_retention: ArchiveRetention::Forever,
@@ -1194,6 +1257,7 @@ mod tests {
worker_id: worker_id().to_string(),
},
reason: "cleanup".into(),
journal_erasure: None,
}
}
#[test]
crates/workspace-server/src/server.rs
+12 -7
View File
@@ -2190,13 +2190,17 @@ impl WorkerRemovalService {
));
}
if let Err(error) = subjektiv_journal::prepare_worker_removal(self, runtime, target) {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"journal_handoff_incomplete",
&error.to_string(),
));
}
let journal_erasure = match subjektiv_journal::prepare_worker_removal(self, runtime, target)
{
Ok(proof) => proof,
Err(error) => {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"journal_handoff_incomplete",
&error.to_string(),
));
}
};
let inventory = match runtime.worker_retention_inventory(&target) {
Ok(inventory) => inventory,
Err(_) => {
@@ -2211,6 +2215,7 @@ impl WorkerRemovalService {
workspace_id: self.workspace_id.clone(),
worker: target.clone(),
reason: reason.to_string(),
journal_erasure,
};
let plan = match self.store.plan_worker_removal(&request, &inventory) {
Ok(plan) => plan,
crates/workspace-server/src/server/subjektiv_journal.rs
+151 -73
View File
@@ -320,6 +320,22 @@ fn admit(
));
}
let mut id = store.journal_for_session(&host).map_err(domain)?;
let reader = material_reader(api, &attribution)?;
if reader.not_materialized {
if store.journal_retention_required(&host).map_err(domain)? {
return Err(Error::RepositoryConflict(
"Admitted Journal original Session is not materialized".into(),
));
}
retain_source(
api.runtime.as_ref(),
api.workspace_id(),
&attribution,
material_source(api, &attribution)?,
true,
)?;
return Ok(JournalRecordingResponse::NoNewSources);
}
retain_source(
api.runtime.as_ref(),
api.workspace_id(),
@@ -327,7 +343,10 @@ fn admit(
material_source(api, &attribution)?,
false,
)?;
let sources = canonical_catalog(api, &attribution)?;
let sources: Vec<_> = reader
.descriptors()
.map(|d| source_from_descriptor(&attribution.session_id, d))
.collect();
for batch in sources.chunks(MAX_JOURNAL_RANGES) {
id = Some(
store
@@ -374,7 +393,7 @@ pub(super) async fn recording_route(
)
.into());
}
require_enabled(api.clone(), &worker)?;
require_enabled(&api, &worker)?;
let store = store(&api)?;
let attributions = store
.subject_session_attributions(subject)
@@ -424,7 +443,16 @@ pub(super) async fn job_route(
.await
}
fn require_enabled(api: WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<()> {
fn require_enabled(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<()> {
if !enabled(api, worker)? {
return Err(Error::WorkspacePermissionDenied(
"Journal Feature is disabled".into(),
));
}
Ok(())
}
fn enabled(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<bool> {
let projection = api
.store
.worker_registry_projection(api.workspace_id(), worker)?
@@ -459,12 +487,7 @@ fn require_enabled(api: WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<()> {
.ok_or_else(|| Error::Config("Journal Worker Profile bundle missing".into()))?;
let manifest =
crate::profile_settings::resolve_profile_manifest_from_config_bundle(&bundle, key)?;
if !manifest.feature.subjektiv.enabled() {
return Err(Error::WorkspacePermissionDenied(
"Journal Feature is disabled".into(),
));
}
Ok(())
Ok(manifest.feature.subjektiv.enabled())
}
pub(super) fn drain_after_completion(api: &WorkspaceApi, request: &BackendJobRequest) {
@@ -539,16 +562,6 @@ fn source_from_descriptor(
},
}
}
fn canonical_catalog(
api: &WorkspaceApi,
attribution: &crate::subjektiv::SubjectSessionAttribution,
) -> Result<Vec<JournalSourceRange>> {
let reader = material_reader(api, attribution)?;
Ok(reader
.descriptors()
.map(|d| source_from_descriptor(&attribution.session_id, d))
.collect())
}
fn read_canonical(
reader: &MaterialReader<'_>,
source: &JournalSource,
@@ -626,20 +639,14 @@ pub(super) fn recover(api: &WorkspaceApi) -> Result<()> {
}
if erased {
if let Err(error) = material_source(api, attr).and_then(|source| {
retain_source(
api.runtime.as_ref(),
api.workspace_id(),
attr,
source,
true,
)
erase_source(api.runtime.as_ref(), api.workspace_id(), attr, source)
}) {
tracing::warn!(%error,session_id=%attr.session_id,"Erased Session source pin release remains recoverable");
tracing::warn!(%error,session_id=%attr.session_id,"Erased Session physical cleanup remains recoverable");
}
continue;
}
let source = RuntimeWorkerRef::new(&attr.runtime_id, &attr.worker_id);
if require_enabled(api.clone(), &source).is_err() {
if require_enabled(api, &source).is_err() {
// Disablement or removal does not erase previously admitted work, but
// also does not authorize collecting a new disabled Worker's history.
let host = JournalHostContext::validated_session(&store, attr.clone())
@@ -710,16 +717,13 @@ fn removal_attribution(
service: &WorkerRemovalService,
target: &RuntimeWorkerRef,
) -> Result<Option<crate::subjektiv::SubjectSessionAttribution>> {
let Some(p) = service
let Some(lease) = service
.store
.worker_registry_projection(&service.workspace_id, target)?
.require_current_worker_singleton_owner(&service.workspace_id, target)?
else {
return Ok(None);
};
let Some(key) = p.resource_key.as_deref() else {
return Ok(None);
};
let Some(subject) = key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX) else {
let Some(subject) = lease.key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX) else {
return Ok(None);
};
let store = crate::subjektiv::SubjektivStore::open(
@@ -767,9 +771,9 @@ pub(super) fn prepare_worker_removal(
service: &WorkerRemovalService,
runtime: &RuntimeRegistry,
target: &RuntimeWorkerRef,
) -> Result<()> {
) -> Result<Option<crate::retention::ErasedJournalSession>> {
let Some(attr) = removal_attribution(service, target)? else {
return Ok(());
return Ok(None);
};
let store = crate::subjektiv::SubjektivStore::open(
&service.journal_storage,
@@ -781,31 +785,59 @@ pub(super) fn prepare_worker_removal(
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
};
retain_source(runtime, &service.workspace_id, &attr, source.clone(), false)?;
let api = service.journal_dispatch_api.upgrade().ok_or_else(|| {
Error::Store("Workspace API unavailable for Journal removal Feature gate".into())
})?;
if let Some(proof) =
crate::retention::ErasedJournalSession::validated(&service.workspace_id, &store, &attr)?
{
// Erasure is monotonic. Retry the same physical intent, never admission or generation.
reconcile_session_jobs(&api, &attr, true)?;
erase_source(runtime, &service.workspace_id, &attr, source)?;
return Ok(Some(proof));
}
let collect_new_sources = enabled(&api, target)?;
let mut id = store.journal_for_session(&host).map_err(domain)?;
// Disablement forbids new admission, not recovery of an already admitted prefix.
// With no admitted remainder there is no source reader or Job to create at removal.
if !collect_new_sources && !store.journal_retention_required(&host).map_err(domain)? {
return Ok(None);
}
let reader = MaterialReader::open(
runtime,
&service.workspace_id,
&target.runtime_id,
source,
source.clone(),
&attr.session_id,
)?;
let ranges: Vec<_> = reader
.descriptors()
.map(|d| source_from_descriptor(&attr.session_id, d))
.collect();
let mut id = store.journal_for_session(&host).map_err(domain)?;
for batch in ranges.chunks(MAX_JOURNAL_RANGES) {
id = Some(
store
.accept_journal_sources(&host, "Session activity", batch)
.map_err(domain)?,
);
if reader.not_materialized {
if store.journal_retention_required(&host).map_err(domain)? {
return Err(Error::RepositoryConflict(
"Admitted Journal original Session is not materialized".into(),
));
}
retain_source(runtime, &service.workspace_id, &attr, source, true)?;
return Ok(None);
}
retain_source(runtime, &service.workspace_id, &attr, source, false)?;
if collect_new_sources {
let ranges: Vec<_> = reader
.descriptors()
.map(|d| source_from_descriptor(&attr.session_id, d))
.collect();
for batch in ranges.chunks(MAX_JOURNAL_RANGES) {
id = Some(
store
.accept_journal_sources(&host, "Session activity", batch)
.map_err(domain)?,
);
}
}
let Some(id) = id else {
return Ok(());
return Ok(None);
};
if !store.journal_retention_required(&host).map_err(domain)? {
return Ok(());
return Ok(None);
}
if let Some(g) = store
.active_journal_generation(&host, &id)
@@ -817,7 +849,7 @@ pub(super) fn prepare_worker_removal(
.get_backend_job(&service.workspace_id, &g.job.job_id)?
.is_some()
{
return Ok(());
return Ok(None);
}
return Err(Error::Store(
"Journal generation's existing Job intent is missing".into(),
@@ -843,7 +875,7 @@ pub(super) fn prepare_worker_removal(
// only starts the existing bounded Job; it never waits for Journal generation.
api.drain_reserved_backend_jobs();
}
Ok(())
Ok(None)
}
pub(super) fn validate_removal_retention(
@@ -860,6 +892,22 @@ pub(super) fn validate_removal_retention(
)
.map_err(|e| Error::Store(e.to_string()))?;
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
if store.journal_session_erased(&host).map_err(domain)? {
if prepared.runtime_request.session_disposition
!= worker_runtime::retention::SessionDisposition::Purge
|| prepared.runtime_request.diagnostics_disposition
!= worker_runtime::retention::DiagnosticsDisposition::Purge
|| prepared.runtime_request.archive_id.is_some()
{
return Err(Error::RepositoryConflict(
"Erased Journal Session requires source and diagnostics Purge, not an archive"
.into(),
));
}
// Physical erasure precedes the plan. Runtime resumes aggregate cleanup with
// its retained permanent fence, including recovery after the aggregate is gone.
return Ok(());
}
let source = runtime_api::SessionPublicSource::Retained {
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
@@ -926,6 +974,7 @@ struct MaterialReader<'a> {
runtime_id: String,
source: runtime_api::SessionPublicSource,
session_id: String,
not_materialized: bool,
descriptors: Vec<session_store::public_material::PublicMaterialDescriptor>,
}
fn material_descriptor(
@@ -968,6 +1017,25 @@ impl<'a> MaterialReader<'a> {
)
.map_err(|e| e.into_error())?
{
runtime_api::SessionMaterialCatalogAvailability::NotMaterialized {
session_id: observed,
} => {
if observed != session_id || offset != 0 || condition.is_some() {
return Err(Error::RepositoryConflict(
"Journal Session materialization proof changed during catalog read"
.into(),
));
}
return Ok(Self {
runtime,
workspace_id,
runtime_id: runtime_id.into(),
source,
session_id: session_id.into(),
not_materialized: true,
descriptors,
});
}
runtime_api::SessionMaterialCatalogAvailability::Page { page } => page,
runtime_api::SessionMaterialCatalogAvailability::Unavailable {
reason,
@@ -1015,6 +1083,7 @@ impl<'a> MaterialReader<'a> {
runtime_id: runtime_id.into(),
source,
session_id: session_id.into(),
not_materialized: false,
descriptors,
})
}
@@ -1075,24 +1144,30 @@ impl<'a> MaterialReader<'a> {
}
}
pub(super) fn source_pin_id(
fn erase_source(
runtime: &RuntimeRegistry,
workspace: &str,
attr: &crate::subjektiv::SubjectSessionAttribution,
) -> String {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(
serde_json::to_vec(&(
"subjektiv-journal-retention-v1",
workspace,
&attr.subject_id,
&attr.session_id,
))
.expect("string tuple serialization"),
);
let mut bytes = [0; 16];
bytes.copy_from_slice(&digest[..16]);
Uuid::from_bytes(bytes).to_string()
source: runtime_api::SessionPublicSource,
) -> Result<()> {
let response = runtime
.session_material_erase(
&attr.runtime_id,
runtime_api::SessionMaterialEraseRequest {
workspace_id: workspace.into(),
source,
expected_session_id: attr.session_id.clone(),
},
)
.map_err(|e| e.into_error())?;
if response.session_id != attr.session_id {
return Err(Error::RepositoryConflict(
"Session erase receipt differs from the requested identity".into(),
));
}
Ok(())
}
fn retain_source(
runtime: &RuntimeRegistry,
workspace: &str,
@@ -1100,7 +1175,9 @@ fn retain_source(
source: runtime_api::SessionPublicSource,
release: bool,
) -> Result<()> {
let pin_id = source_pin_id(workspace, attr);
let pin_id =
subjektiv::journal::source_retention_pin(workspace, &attr.subject_id, &attr.session_id)
.to_string();
let request = runtime_api::SessionMaterialPinRequest {
workspace_id: workspace.into(),
source,
@@ -1340,8 +1417,10 @@ fn reconcile_session_jobs(
}
impl WorkspaceApi {
/// Host-only explicit erasure hook. The existing domain tombstone is the durable
/// cancellation intent; interrupted Worker Purge is retried by ordinary Job cleanup.
/// Host-only erasure saves the domain fence, cancels Jobs, then erases originals.
/// Runtime requires a stopped source Worker. Physical failure leaves the domain
/// tombstone and cancellations committed for exact retry or removal preflight;
/// terminal Job execution Sessions remain owned by ordinary Job cleanup.
pub fn erase_subjektiv_journal_session(
&self,
session_id: &str,
@@ -1359,12 +1438,11 @@ impl WorkspaceApi {
.erase_journal_session(&host, request_id)
.map_err(domain)?;
reconcile_session_jobs(self, &attr, true)?;
retain_source(
erase_source(
self.runtime.as_ref(),
self.workspace_id(),
&attr,
material_source(self, &attr)?,
true,
)
}
}
crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs
+657 -13
View File
@@ -394,6 +394,24 @@ fn seed_session(
session: session_store::SessionId,
item: session_store::LoggedItem,
origin: session_store::LoggedSessionHistoryOrigin,
) {
seed_session_history(
root,
session,
vec![session_store::LoggedHistoryEntry {
item,
metadata: session_store::LoggedSessionHistoryMetadata {
entry_id: session_store::LoggedSessionHistoryEntryId("read-source-original".into()),
origin,
derivation: None,
},
}],
);
}
fn seed_session_history(
root: &std::path::Path,
session: session_store::SessionId,
history: Vec<session_store::LoggedHistoryEntry>,
) {
use session_store::{LogEntry, Store as _, WorkerSessionStore};
let segment = Uuid::new_v4();
@@ -407,16 +425,7 @@ fn seed_session(
session_id: session,
system_prompt: None,
config: agen::llm_client::RequestConfig::default(),
history: vec![session_store::LoggedHistoryEntry {
item,
metadata: session_store::LoggedSessionHistoryMetadata {
entry_id: session_store::LoggedSessionHistoryEntryId(
"read-source-original".into(),
),
origin,
derivation: None,
},
}],
history,
forked_from: None,
compacted_from: None,
}],
@@ -486,7 +495,12 @@ impl RuntimeFixture {
workspace_id: TEST_WORKSPACE_ID.into(),
source: src.clone(),
expected_session_id: session.to_string(),
pin_id: subjektiv_journal::source_pin_id(TEST_WORKSPACE_ID, &attr),
pin_id: subjektiv::journal::source_retention_pin(
TEST_WORKSPACE_ID,
&attr.subject_id,
&attr.session_id,
)
.to_string(),
},
)
.unwrap();
@@ -559,6 +573,18 @@ impl RuntimeFixture {
id,
}
}
fn stop_source(&self) {
self.api
.runtime
.stop_worker(
&self.source,
WorkerLifecycleRequest {
reason: Some("settle original source before permanent erase".into()),
ticket_assignment: None,
},
)
.unwrap();
}
fn dispatch_and_read(&self) -> BackendJobReservation {
let dispatched = self
.api
@@ -664,9 +690,10 @@ impl RuntimeFixture {
.unwrap()
.is_none()
);
assert!(
assert_eq!(
session_root(self.dir.path(), &self.source).exists(),
"the original conversation was not purged"
!self.store.journal_session_erased(&self.host).unwrap(),
"only explicit source erasure, never Job cleanup, removes the original conversation"
);
}
}
@@ -713,6 +740,7 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang
.unwrap();
}
"erased" => {
f.stop_source();
f.api
.erase_subjektiv_journal_session(
&f.session.to_string(),
@@ -1126,6 +1154,7 @@ async fn journal_profile_name_without_host_job_binding_keeps_normal_archive_poli
workspace_id: TEST_WORKSPACE_ID.into(),
worker: f.source,
reason: "profile is not purge authority".into(),
journal_erasure: None,
},
&inventory,
)
@@ -1364,6 +1393,7 @@ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_fail
)
.unwrap();
if outcome == "erased" {
f.stop_source();
f.api
.erase_subjektiv_journal_session(
&f.session.to_string(),
@@ -1472,6 +1502,7 @@ async fn explicit_erasure_interrupts_completed_journal_cleanup_waiting_for_run_t
session_root(f.dir.path(), worker).exists(),
"normal cleanup still waits for tracked response teardown"
);
f.stop_source();
f.api
.erase_subjektiv_journal_session(&f.session.to_string(), "erase-while-completed-run-busy")
.unwrap();
@@ -1497,3 +1528,616 @@ async fn explicit_erasure_interrupts_completed_journal_cleanup_waiting_for_run_t
assert!(f.store.journal_session_erased(&f.host).unwrap());
assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err());
}
/// Removal uses the production embedded WorkerRemove forwarder, target-bound proof,
/// common reception, catalog/retention planner, Runtime and SQLite. Source creation
/// calls the create handler directly (no HTTP auth); canonical Session contents,
/// attribution and profile disablement are fixtures, not model execution or proofs
/// of the normal publication/attribution connection.
struct SubjectRemovalFixture {
dir: tempfile::TempDir,
api: WorkspaceApi,
dispatcher: Arc<crate::worker_source::EmbeddedServerWorkerMutationDispatcher>,
controller: RuntimeWorkerRef,
target: RuntimeWorkerRef,
store: crate::subjektiv::SubjektivStore,
attr: crate::subjektiv::SubjectSessionAttribution,
host: JournalHostContext,
}
impl SubjectRemovalFixture {
async fn new() -> Self {
Self::with_materialized_source(true).await
}
async fn with_materialized_source(materialized: bool) -> Self {
let dir = tempfile::tempdir().unwrap();
let config = test_server_config(dir.path());
let control = Arc::new(test_control_store(&config));
seed_test_registered_workspace(control.as_ref(), &config)
.await
.unwrap();
let dispatcher = Arc::new(
crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
config.clone(),
control.clone(),
),
);
let api = WorkspaceApi::new_with_execution_backend_and_broker(
config,
control,
Arc::new(DeterministicExecutionBackend::default()),
BackendResourceBroker::default(),
HostExecutors::default(),
Some(dispatcher.clone()),
)
.await
.unwrap();
let Json(orchestrator) = scoped_start_workspace_orchestrator(
State(api.clone()),
AxumPath(ScopedWorkspacePath {
workspace_id: TEST_WORKSPACE_ID.into(),
}),
)
.await
.unwrap();
let controller = orchestrator.worker.unwrap();
let controller = RuntimeWorkerRef::new(controller.runtime_id, controller.worker_id);
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("removal-source").unwrap())
.unwrap();
let Json(created) = scoped_create_workspace_worker(
State(api.clone()),
AxumPath(ScopedWorkspacePath {
workspace_id: TEST_WORKSPACE_ID.into(),
}),
HeaderMap::new(),
Json(CreateWorkspaceWorkerRequest {
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(),
display_name: "removal-source".into(),
singleton_key: None,
profile: Some("builtin:companion".into()),
ticket_assignment: None,
initial_submit: vec![],
workdir_attachments: vec![],
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest {
subject_id: subject.id.clone(),
}),
},
control_operation_id: None,
}),
)
.await
.unwrap();
let target = RuntimeWorkerRef::new(created.runtime_id, created.worker_id);
let session = store
.subject_session_attributions(&subject.id)
.unwrap()
.first()
.map(|attr| attr.session_id.parse().unwrap())
.unwrap_or_else(session_store::new_session_id);
if materialized {
seed_session_history(
&session_root(dir.path(), &target),
session,
["admitted-prefix", "disabled-new-history"]
.into_iter()
.map(|text| session_store::LoggedHistoryEntry {
item: message(text),
metadata: session_store::LoggedSessionHistoryMetadata {
entry_id: session_store::LoggedSessionHistoryEntryId(text.into()),
origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None,
},
})
.collect(),
);
} else {
use session_store::WorkerMetadataStore as _;
let root = session_root(dir.path(), &target);
let aggregate = root.parent().unwrap();
let name = format!("worker-runtime-{}", target.worker_id);
session_store::WorkerAggregateStore::new(aggregate, &name)
.unwrap()
.write(
&session_store::WorkerMetadata::new(
&name,
Some(session_store::WorkerActiveSegmentRef::pending_segment(
session,
)),
)
.with_workspace_id(TEST_WORKSPACE_ID),
)
.unwrap();
}
let attr = store
.record_session_attribution(
crate::subjektiv::SubjectSessionAttribution::new(
&subject.id,
&target.runtime_id,
&target.worker_id,
session.to_string(),
)
.unwrap(),
)
.unwrap();
let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap();
seed_worker_control_grant(&api, &controller, &target, "subject-removal-fixture");
api.runtime
.stop_worker(
&target,
WorkerLifecycleRequest {
reason: Some("test stopped Subject removal".into()),
ticket_assignment: None,
},
)
.unwrap();
let summary = api.runtime.worker(&target).unwrap();
assert_eq!(summary.state, "stopped");
sync_worker_observation(&api, &summary).unwrap();
Self {
dir,
api,
dispatcher,
controller,
target,
store,
attr,
host,
}
}
fn disable(&self) {
let mut registry = self
.api
.store
.get_worker_registry(TEST_WORKSPACE_ID, &self.target)
.unwrap()
.unwrap();
registry.profile = Some("builtin:default".into());
self.api.store.upsert_worker_registry(&registry).unwrap();
}
fn remove(&self) -> worker::WorkspaceResponse {
let key = self
.api
.store
.worker_resource_key(TEST_WORKSPACE_ID, &self.target)
.unwrap()
.unwrap();
worker_runtime::worker_source::RuntimeWorkerMutationForwarder::embedded(
&self.controller.runtime_id,
worker_runtime::RuntimeWorkspaceScope::new(TEST_WORKSPACE_ID, "embedded-backend"),
&self.controller.worker_id,
self.dispatcher.clone(),
)
.execute_worker_remove(&self.target.runtime_id, &key, "remove stopped Subject")
.unwrap()
}
}
#[tokio::test]
async fn disabled_subject_removal_does_not_admit_new_public_history_or_reserve_a_job() {
let f = SubjectRemovalFixture::new().await;
f.disable();
let response = f.remove();
assert_eq!(
response.status,
StatusCode::OK.as_u16(),
"{}",
response.body
);
assert!(
f.api
.store
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
.unwrap()
.is_none()
);
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
assert!(
f.api
.store
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
.unwrap()
.is_empty()
);
}
#[tokio::test]
async fn disabled_subject_removal_retains_only_the_already_admitted_prefix() {
let f = SubjectRemovalFixture::new().await;
let runtime_api::SessionMaterialCatalogAvailability::Page { page } = f
.api
.runtime
.session_material_catalog(
&f.target.runtime_id,
runtime_api::SessionMaterialCatalogRequest {
workspace_id: TEST_WORKSPACE_ID.into(),
source: runtime_api::SessionPublicSource::Retained {
worker_id: worker_runtime::identity::WorkerId::parse(&f.target.worker_id)
.unwrap(),
},
expected_session_id: f.attr.session_id.clone(),
expected_catalog_sha256: None,
offset: 0,
limit: 100,
},
)
.unwrap()
else {
panic!("canonical source catalog required")
};
let ranges: Vec<_> = page
.descriptors
.into_iter()
.map(|d| JournalSourceRange {
source: JournalSource {
session_id: f.attr.session_id.clone(),
pointer: JournalSourcePointer {
segment_id: d.segment_id.parse().unwrap(),
entry_ref: d.entry_ref,
},
order: d.canonical_order,
content_sha256: d.sha256,
total_bytes: d.total_bytes,
},
bytes: JournalByteRange {
start: 0,
end: d.total_bytes,
},
})
.collect();
assert_eq!(ranges.len(), 2);
let id = f
.store
.accept_journal_sources(&f.host, "accepted before disablement", &ranges[..1])
.unwrap();
f.disable();
let response = f.remove();
assert_eq!(
response.status,
StatusCode::OK.as_u16(),
"{}",
response.body
);
assert!(
f.api
.store
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
.unwrap()
.is_none()
);
let jobs = f
.api
.store
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
.unwrap();
assert_eq!(jobs.len(), 1);
assert_eq!(
jobs[0]
.request
.grants
.subjektiv_journal
.as_ref()
.unwrap()
.selected_ranges,
ranges[..1]
);
assert_eq!(
f.store.journal_for_session(&f.host).unwrap().as_deref(),
Some(id.as_str())
);
assert!(f.store.journal_retention_required(&f.host).unwrap());
let archive = f
.api
.config_store
.worker_session_archive(TEST_WORKSPACE_ID, &f.target, &f.attr.session_id)
.unwrap()
.unwrap();
assert!(!session_root(f.dir.path(), &f.target).exists());
let source = runtime_api::SessionPublicSource::Archived {
archive_id: archive.archive_id,
};
let response = f
.api
.runtime
.session_material_catalog(
&f.target.runtime_id,
runtime_api::SessionMaterialCatalogRequest {
workspace_id: TEST_WORKSPACE_ID.into(),
source,
expected_session_id: f.attr.session_id.clone(),
expected_catalog_sha256: None,
offset: 0,
limit: 100,
},
)
.unwrap();
assert!(matches!(
response,
runtime_api::SessionMaterialCatalogAvailability::Page { .. }
));
}
#[tokio::test]
async fn attributed_unmaterialized_subject_removal_needs_no_journal_job() {
let f = SubjectRemovalFixture::with_materialized_source(false).await;
let retention = source_retention(f.dir.path());
let session = f.attr.session_id.parse().unwrap();
// Fixture the pre-publication pin without materializing the Session.
retention
.pin(
session,
session_store::retention::SessionRetentionPin(
subjektiv::journal::source_retention_pin(
TEST_WORKSPACE_ID,
&f.attr.subject_id,
&f.attr.session_id,
)
.to_string()
.parse()
.unwrap(),
),
|| Ok(true),
)
.unwrap();
assert_eq!(retention.pins(session).unwrap().len(), 1);
let response = f.remove();
assert!(retention.pins(session).unwrap().is_empty());
assert_eq!(
response.status,
StatusCode::OK.as_u16(),
"{}",
response.body
);
assert!(
f.api
.store
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
.unwrap()
.is_none()
);
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
assert!(
f.api
.store
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
.unwrap()
.is_empty()
);
}
#[tokio::test]
async fn missing_materialized_subject_source_blocks_removal_without_marking_it_empty() {
let f = SubjectRemovalFixture::new().await;
std::fs::remove_dir_all(session_root(f.dir.path(), &f.target)).unwrap();
let response = f.remove();
assert_eq!(
response.status,
StatusCode::CONFLICT.as_u16(),
"{}",
response.body
);
assert!(response.body.contains("journal_handoff_incomplete"));
assert!(
f.api
.store
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
.unwrap()
.is_some()
);
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
assert!(
f.api
.store
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
.unwrap()
.is_empty()
);
}
fn source_retention(root: &std::path::Path) -> session_store::retention::SessionRetentionStore {
session_store::retention::SessionRetentionStore::new(
root.join(".test-embedded-runtime-store/retention/session-sources"),
)
}
#[tokio::test]
async fn explicitly_erased_subject_removal_purges_without_generating_or_archiving() {
let f = SubjectRemovalFixture::new().await;
let policy = f
.api
.config_store
.worker_retention_policy(TEST_WORKSPACE_ID)
.unwrap();
f.api
.erase_subjektiv_journal_session(&f.attr.session_id, "erase-before-normal-remove")
.unwrap();
assert!(f.store.journal_session_erased(&f.host).unwrap());
assert!(!session_root(f.dir.path(), &f.target).exists());
// Exact erase replay must resume the same permanent intent even after source removal.
f.api
.erase_subjektiv_journal_session(&f.attr.session_id, "erase-before-normal-remove")
.unwrap();
let response = f.remove();
assert_eq!(
response.status,
StatusCode::OK.as_u16(),
"{}",
response.body
);
assert!(
f.api
.store
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
.unwrap()
.is_none()
);
let saved = f
.api
.config_store
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
.unwrap()
.unwrap();
assert_eq!(
saved.plan.session_disposition,
worker_runtime::retention::SessionDisposition::Purge
);
assert_eq!(
saved.plan.diagnostics_disposition,
worker_runtime::retention::DiagnosticsDisposition::Purge
);
assert!(saved.plan.archive_id.is_none());
assert!(
f.api
.config_store
.worker_session_archive(TEST_WORKSPACE_ID, &f.target, &f.attr.session_id)
.unwrap()
.is_none()
);
assert!(
f.api
.store
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
.unwrap()
.is_empty()
);
assert_eq!(
f.api
.config_store
.worker_retention_policy(TEST_WORKSPACE_ID)
.unwrap(),
policy
);
assert!(matches!(
source_retention(f.dir.path()).ensure_live(f.attr.session_id.parse().unwrap()),
Err(session_store::StoreError::SessionErased(_))
));
}
#[tokio::test]
async fn erased_subject_removal_retries_interrupted_physical_erasure_under_the_permanent_fence() {
let f = SubjectRemovalFixture::new().await;
// Fixture boundary: interrupt the real retention deletion after its durable fence,
// before physical removal. WorkerRemove below uses the real Backend/Runtime erase path.
f.store
.erase_journal_session(&f.host, "erase-interrupted")
.unwrap();
let retention = source_retention(f.dir.path());
let session = f.attr.session_id.parse().unwrap();
assert!(
retention
.delete(
session,
session_store::retention::SessionDeletionIntent::ErasePermanently,
|| {
Err::<(), _>(session_store::StoreError::Io(std::io::Error::other(
"injected physical cleanup failure",
)))
}
)
.is_err()
);
assert!(session_root(f.dir.path(), &f.target).exists());
let response = f.remove();
assert_eq!(
response.status,
StatusCode::OK.as_u16(),
"{}",
response.body
);
assert!(
!session_root(f.dir.path(), &f.target)
.parent()
.unwrap()
.exists()
);
assert!(
f.api
.store
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
.unwrap()
.is_none()
);
assert!(
f.api
.config_store
.worker_session_archive(TEST_WORKSPACE_ID, &f.target, &f.attr.session_id)
.unwrap()
.is_none()
);
assert!(
f.api
.store
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
.unwrap()
.is_empty()
);
assert!(matches!(
retention.ensure_live(session),
Err(session_store::StoreError::SessionErased(_))
));
}
#[tokio::test]
async fn physical_erase_failure_keeps_domain_tombstone_and_job_cancellation_for_exact_retry() {
let f = RuntimeFixture::new().await;
let dispatched = f.dispatch_and_read();
// Original source is not stopped: Runtime must reject physical erasure, after
// the Backend has durably fenced all Journal adoption and cancelled its Job.
assert!(
f.api
.erase_subjektiv_journal_session(&f.session.to_string(), "erase-unsettled-source")
.is_err()
);
assert!(f.store.journal_session_erased(&f.host).unwrap());
assert!(session_root(f.dir.path(), &f.source).exists());
let job = f
.api
.store
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
.unwrap()
.unwrap();
assert_eq!(job.state, BackendJobState::Failed);
assert!(
subjektiv_journal::execute(
&f.api,
dispatched.attempt.worker.as_ref().unwrap(),
JournalJobRequest::ReadState
)
.is_err()
);
f.stop_source();
f.api
.erase_subjektiv_journal_session(&f.session.to_string(), "erase-unsettled-source")
.unwrap();
assert!(!session_root(f.dir.path(), &f.source).exists());
assert_eq!(
f.api
.store
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.session.to_string(), None, 100)
.unwrap()
.len(),
1
);
}
#[tokio::test]
async fn serialized_removal_input_cannot_supply_journal_erasure_authority() {
let f = SubjectRemovalFixture::new().await;
let request: crate::retention::WorkerRemovalPlanRequest = serde_json::from_value(json!({
"workspace_id": TEST_WORKSPACE_ID, "worker": f.target, "reason": "untrusted erase claim",
"journal_erasure": {"workspace_id": TEST_WORKSPACE_ID, "worker": f.target, "session_id": f.attr.session_id},
})).unwrap();
let inventory = f.api.runtime.worker_retention_inventory(&f.target).unwrap();
let plan = f
.api
.store
.plan_worker_removal(&request, &inventory)
.unwrap();
assert_eq!(
plan.session_disposition,
worker_runtime::retention::SessionDisposition::Archive
);
assert!(!f.store.journal_session_erased(&f.host).unwrap());
assert!(session_root(f.dir.path(), &f.target).exists());
}