Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: fence Journal originals before publication and reconcile removal intent

Browse Source 
Protect attributed originals before the first public commit; distinguish pending Session material from missing committed sources. Gate removal admission by the Journal Feature and resolve Subject ownership from durable singleton authority. Resume explicit erasure through signed delete transport and proof-bound purge plans without creating a Journal Job or archive.

Validation: changed-crate suites, root check, doc/fmt/diff/naming checks, Server/Runtime builds and worker_environment isolation suite passed. Public standalone scripted dialog connects Host, automatic Job, sequential tools and drain while dialog continues. Backend public embedded WorkerRemove tests connect removal forwarding, durable Jobs, retention, Runtime and SQLite; Session/attribution/profile fixture setup bypasses creation/auth routes. Runtime signed generated-client tests separately exercise request authentication.

Limits: real-model semantic acceptance condition 6 remains unevaluated without a new explicit model/credential/budget grant. Conditions 1/7/8/11 do not have complete product E2E coverage: worker_environment only verifies isolated startup/connection/cleanup. Compaction tests connect local hooks/reader/store rather than the full public dialog-to-Journal chain. Erasure racing a previously prepared Archive removal plan fails closed with conflict; automatic stale-plan replacement is not implemented.
This commit is contained in:
Hare
2026-10-16 03:59:45 +09:00
parent 9debf43645
commit 5bd49760e5
19 changed files with 1846 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/worker-runtime/src/http_server/runtime_management_api.rs
+37
View File
@@ -369,6 +369,43 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
.map_err(api_error)
}
async fn session_material_erase(
&self,
request: runtime_api::SessionMaterialEraseRequest,
) -> Result<runtime_api::SessionMaterialEraseResponse, runtime_api::RuntimeApiError> {
let scope = material_scope(&self.state, &request.workspace_id)?;
#[cfg(feature = "fs-store")]
{
self.state
.runtime
.delete_session_public_source_scoped(
&scope,
&request.source,
&request.expected_session_id,
session_store::retention::SessionDeletionIntent::ErasePermanently,
)
.map_err(|_| {
runtime_api::RuntimeApiError::new(
409,
"session_erasure_incomplete",
"Session erasure could not complete; retain erasure intent and retry",
)
})?;
Ok(runtime_api::SessionMaterialEraseResponse {
session_id: request.expected_session_id,
})
}
#[cfg(not(feature = "fs-store"))]
{
let _ = scope;
Err(runtime_api::RuntimeApiError::new(
503,
"session_retention_unavailable",
"Session erasure requires filesystem storage",
))
}
}
async fn session_material_pin(
&self,
request: runtime_api::SessionMaterialPinRequest,
crates/worker-runtime/src/http_server/tests/session_material_tests.rs
+169 -8
View File
@@ -194,6 +194,101 @@ impl Fixture {
}
}
#[test]
fn pending_session_proof_is_not_missing_material_and_pin_release_allows_empty_archive() {
let f = Fixture::new();
std::fs::remove_dir_all(f.root()).unwrap();
let aggregate = f.root().parent().unwrap().to_path_buf();
let metadata =
WorkerAggregateStore::new(&aggregate, format!("worker-runtime-{}", f.worker.worker_id))
.unwrap();
metadata
.write(
&session_store::WorkerMetadata::new(
format!("worker-runtime-{}", f.worker.worker_id),
Some(session_store::WorkerActiveSegmentRef::pending_segment(
f.session,
)),
)
.with_workspace_id(&f.scope.workspace_id),
)
.unwrap();
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
let retention = runtime_session_retention(f.temp.path());
retention.retain_before_publication(f.session, pin).unwrap();
assert!(
matches!(f.runtime.session_material_catalog_scoped(&f.scope, &f.request()), Catalog::NotMaterialized { session_id } if session_id == f.session.to_string())
);
assert!(
f.runtime
.pin_session_public_source_scoped(&f.scope, &f.source(), &f.session.to_string(), pin)
.is_err(),
"retention intent is not source availability"
);
// A missing committed source is never the same fact as a pending pointer.
metadata
.write(
&session_store::WorkerMetadata::new(
format!("worker-runtime-{}", f.worker.worker_id),
Some(session_store::WorkerActiveSegmentRef::active_segment(
f.session, f.segment,
)),
)
.with_workspace_id(&f.scope.workspace_id),
)
.unwrap();
assert!(matches!(
f.runtime
.session_material_catalog_scoped(&f.scope, &f.request()),
Catalog::Unavailable { .. }
));
metadata
.write(
&session_store::WorkerMetadata::new(
format!("worker-runtime-{}", f.worker.worker_id),
Some(session_store::WorkerActiveSegmentRef::pending_segment(
f.session,
)),
)
.with_workspace_id(&f.scope.workspace_id),
)
.unwrap();
// Any unexpected source file invalidates the empty proof, too.
std::fs::create_dir_all(f.root()).unwrap();
std::fs::write(f.root().join("unattested-original"), "do not discard").unwrap();
assert!(matches!(
f.runtime
.session_material_catalog_scoped(&f.scope, &f.request()),
Catalog::Unavailable { .. }
));
std::fs::remove_file(f.root().join("unattested-original")).unwrap();
f.runtime
.release_session_public_source_pin_scoped(
&f.scope,
&f.source(),
&f.session.to_string(),
pin,
)
.unwrap();
assert!(retention.pins(f.session).unwrap().is_empty());
f.stop();
let inventory = f
.runtime
.worker_retention_inventory(&f.scope.workspace_id, &f.worker)
.unwrap();
assert_eq!(inventory.session_id, None);
assert_eq!(inventory.session_bytes, 0);
let removed = f
.runtime
.execute_worker_retention(&f.retention_request(SessionDisposition::Archive))
.unwrap();
let archive = removed.archive.unwrap();
assert!(removed.source_removed);
assert_eq!(archive.source_session_id, f.session.to_string());
assert_eq!(archive.content_file_count, 0);
assert!(archive.source_active_segment_id.is_none());
}
#[test]
fn canonical_pages_read_frozen_originals_after_tail_append_and_reject_forged_descriptors() {
let f = Fixture::new();
@@ -524,14 +619,10 @@ fn explicit_erase_invalidates_pins_and_fences_old_readers_publishers_and_retry()
SessionDeletionIntent::ErasePermanently,
)
.unwrap();
// Removing the remaining Worker registration/metadata still requires the
// caller's explicit erase intent; ordinary deletion may not infer it.
// Ordinary registration removal can resume a pre-existing permanent erase
// decision. It cannot establish one or recreate original Session material.
f.runtime
.delete_worker_with_intent_scoped(
&f.scope,
&f.worker,
SessionDeletionIntent::ErasePermanently,
)
.delete_worker_with_intent_scoped(&f.scope, &f.worker, SessionDeletionIntent::Ordinary)
.unwrap();
assert!(!f.root().parent().unwrap().exists());
assert!(matches!(
@@ -808,13 +899,47 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
.len(),
1
);
client.session_material_release(pin).await.unwrap();
client.session_material_release(pin.clone()).await.unwrap();
assert!(
runtime_session_retention(f.temp.path())
.pins(f.session)
.unwrap()
.is_empty()
);
client.session_material_pin(pin).await.unwrap();
f.stop();
let erase = runtime_api::SessionMaterialEraseRequest {
workspace_id: f.scope.workspace_id.clone(),
source: f.source(),
expected_session_id: f.session.to_string(),
};
for _ in 0..2 {
assert_eq!(
client
.session_material_erase(erase.clone())
.await
.unwrap()
.session_id,
f.session.to_string()
);
}
assert!(!f.root().exists());
assert!(matches!(
client.session_material_catalog(f.request()).await.unwrap(),
Catalog::Unavailable { .. }
));
assert!(matches!(
runtime_session_retention(f.temp.path()).ensure_live(f.session),
Err(session_store::StoreError::SessionErased(_))
));
// Register/lifecycle cleanup after erasure must not need a fresh archive or Job.
let result = f
.runtime
.execute_worker_retention(&f.retention_request(SessionDisposition::Purge))
.unwrap();
assert!(result.source_removed);
assert!(result.archive.is_none());
assert!(!f.root().parent().unwrap().exists());
})
.await
.expect("bounded signed client integration");
@@ -866,3 +991,39 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
.unwrap();
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn explicit_material_erasure_requires_delete_capability_and_matching_scope() {
let f = Fixture::new();
f.stop();
let identity = RuntimeIdentityMaterial::generate("erase-key").unwrap();
let trust = workspace_trust(&identity);
let app = runtime_http_router_with_workspace_auth(
f.runtime.clone(),
None,
workspace_test_auth(vec![trust.clone()]),
);
let path = "/v1/session-public/material/retention/erase";
let mut request = runtime_api::SessionMaterialEraseRequest {
workspace_id: f.scope.workspace_id.clone(),
source: f.source(),
expected_session_id: f.session.to_string(),
};
let body = serde_json::to_vec(&request).unwrap();
let mut claims = signed_claims(&trust, "POST", path, &body);
claims.operation = "workers:read".into();
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
assert!(f.root().exists());
request.workspace_id = "other-workspace".into();
let body = serde_json::to_vec(&request).unwrap();
let claims = signed_claims(&trust, "POST", path, &body);
let response = signed_request(app, &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::FORBIDDEN);
assert!(
runtime_session_retention(f.temp.path())
.ensure_live(f.session)
.is_ok()
);
assert!(f.root().exists());
}
crates/worker-runtime/src/retention.rs
+83 -33
View File
@@ -178,6 +178,40 @@ pub(crate) trait WorkerRetentionProvider: Send + Sync {
pub use crate::session_retention::runtime_session_retention;
/// Prove the legitimate pre-publication state, rather than interpreting a missing
/// source as empty. A committed active pointer or any source file defeats this proof.
pub(crate) fn unmaterialized_session(
worker_dir: &Path,
workspace_id: &str,
) -> Result<Option<session_store::SessionId>, RuntimeError> {
let metadata_path = worker_dir.join("metadata.json");
let bytes = match fs::read(&metadata_path) {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(source) => {
return Err(RuntimeError::StoreIo {
operation: "inspect pending Session",
path: metadata_path,
source,
});
}
};
let metadata: session_store::WorkerMetadata = serde_json::from_slice(&bytes)
.map_err(|_| RuntimeError::InvalidRequest("Pending Session metadata is invalid".into()))?;
if metadata.workspace_id.as_deref() != Some(workspace_id) {
return Ok(None);
}
let Some(active) = metadata.active.filter(|active| active.segment_id.is_none()) else {
return Ok(None);
};
let root = worker_dir.join("session");
if collect_files(&root, "inspect pending Session")?.is_empty() {
Ok(Some(active.session_id))
} else {
Ok(None)
}
}
pub(crate) fn aggregate_session_id(
worker_dir: &Path,
) -> Result<Option<session_store::SessionId>, RuntimeError> {
@@ -303,6 +337,19 @@ pub(crate) fn delete_worker_aggregate(
intent: session_store::retention::SessionDeletionIntent,
) -> Result<(), RuntimeError> {
let session = aggregate_session_id(worker_dir)?;
let retention = runtime_session_retention(runtime_root);
// Ordinary policy never creates erase authority. A pre-existing permanent
// tombstone is an irrevocable decision whose physical cleanup may be resumed.
let intent = match session
.map(|session| retention.ensure_live(session))
.transpose()
{
Err(session_store::StoreError::SessionErased(_)) => {
session_store::retention::SessionDeletionIntent::ErasePermanently
}
Err(error) => return Err(retention_error(error)),
Ok(_) => intent,
};
let remove = || -> Result<(), session_store::StoreError> {
if intent == session_store::retention::SessionDeletionIntent::ErasePermanently {
if let Some(session) = session {
@@ -561,33 +608,36 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
return Err(RuntimeError::WorkerNotFound { worker_id });
}
let session_dir = worker_dir.join("session");
let (session_id, segment_ids, session_bytes) = if session_dir.is_dir() {
let manifest: CanonicalSessionManifest = read_json(
&session_dir.join("session.json"),
"inventory Worker retention",
)?;
let files = collect_files(&session_dir, "inventory Worker retention")?;
let mut segment_ids = BTreeSet::new();
let mut bytes = 0_u64;
for (relative, path) in files {
bytes = bytes.saturating_add(file_len(&path, "inventory Worker retention")?);
if let Some(name) = relative.file_name().and_then(|value| value.to_str()) {
let segment = name
.strip_suffix(".trace.jsonl")
.or_else(|| name.strip_suffix(".jsonl"));
if let Some(segment) = segment {
segment_ids.insert(segment.to_string());
let (session_id, segment_ids, session_bytes) =
if unmaterialized_session(&worker_dir, workspace_id)?.is_some() {
(None, Vec::new(), 0)
} else if session_dir.is_dir() {
let manifest: CanonicalSessionManifest = read_json(
&session_dir.join("session.json"),
"inventory Worker retention",
)?;
let files = collect_files(&session_dir, "inventory Worker retention")?;
let mut segment_ids = BTreeSet::new();
let mut bytes = 0_u64;
for (relative, path) in files {
bytes = bytes.saturating_add(file_len(&path, "inventory Worker retention")?);
if let Some(name) = relative.file_name().and_then(|value| value.to_str()) {
let segment = name
.strip_suffix(".trace.jsonl")
.or_else(|| name.strip_suffix(".jsonl"));
if let Some(segment) = segment {
segment_ids.insert(segment.to_string());
}
}
}
}
(
Some(manifest.session_id),
segment_ids.into_iter().collect(),
bytes,
)
} else {
(None, Vec::new(), 0)
};
(
Some(manifest.session_id),
segment_ids.into_iter().collect(),
bytes,
)
} else {
(None, Vec::new(), 0)
};
let diagnostics_bytes = diagnostics_files(&worker_dir, "inventory Worker retention")?
.into_iter()
.try_fold(0_u64, |total, path| {
@@ -909,14 +959,14 @@ fn commit_session_archive(
RuntimeError::InvalidRequest("archive disposition requires archive_id".to_string())
})?;
let session_dir = worker_dir.join("session");
if !session_dir.is_dir() {
return Err(RuntimeError::StoreMissing {
operation: "archive Worker Session",
path: session_dir,
});
}
let session: CanonicalSessionManifest =
read_json(&session_dir.join("session.json"), "archive Worker Session")?;
let pending = unmaterialized_session(worker_dir, &request.workspace_id)?;
let session: CanonicalSessionManifest = if let Some(session) = pending {
CanonicalSessionManifest {
session_id: session.to_string(),
}
} else {
read_json(&session_dir.join("session.json"), "archive Worker Session")?
};
let source_files = collect_files(&session_dir, "archive Worker Session")?;
let (checksum, bytes, count) = checksum_files(&source_files, "archive Worker Session")?;
let mut segment_ids = BTreeSet::new();
crates/worker-runtime/src/runtime.rs
+38 -6
View File
@@ -4842,15 +4842,20 @@ impl Runtime {
expected_session_id: &str,
pin: session_store::retention::SessionRetentionPin,
) -> Result<(), RuntimeError> {
let (root, _) = self.resolve_session_public_source(scope, source)?;
let (root, archive) =
self.resolve_session_public_source_with_validation(scope, source, false)?;
let session = expected_session_id
.parse::<session_store::SessionId>()
.map_err(|_| RuntimeError::InvalidRequest("Invalid Session identity".into()))?;
if session_store::WorkerSessionStore::open_read_only(&root)
.and_then(|store| store.session_id())
.map_err(crate::retention::retention_error)?
!= Some(session)
{
let owned_session = match archive {
Some(manifest) => manifest.source_session_id.parse().ok(),
None => {
crate::retention::aggregate_session_id(root.parent().ok_or_else(|| {
RuntimeError::InvalidRequest("Session root unavailable".into())
})?)?
}
};
if owned_session != Some(session) {
return Err(RuntimeError::InvalidRequest(
"Session source identity mismatch".into(),
));
@@ -4995,6 +5000,33 @@ impl Runtime {
))
}
/// Prove an attributed but not yet materialized Session without treating a
/// missing committed log as empty. The source/Workspace binding is resolved first.
#[cfg(feature = "fs-store")]
pub(crate) fn session_not_materialized_scoped(
&self,
scope: &RuntimeWorkspaceScope,
source: &runtime_api::SessionPublicSource,
expected_session_id: &str,
) -> Result<bool, RuntimeError> {
let (root, archive) = self.resolve_session_public_source(scope, source)?;
if archive.is_some() {
return Ok(false);
}
let session = expected_session_id
.parse::<session_store::SessionId>()
.map_err(|_| RuntimeError::InvalidRequest("Invalid Session identity".into()))?;
let retention = self.session_retention_store()?;
let _guard = retention
.lock_live(session)
.map_err(crate::retention::retention_error)?;
Ok(crate::retention::unmaterialized_session(
root.parent()
.ok_or_else(|| RuntimeError::InvalidRequest("Session root unavailable".into()))?,
&scope.workspace_id,
)? == Some(session))
}
/// Resolve the Backend-selected source without exposing local storage paths.
/// The saved active pointer, never a caller-provided Segment, selects lineage.
#[cfg(feature = "fs-store")]
crates/worker-runtime/src/session_material.rs
+12
View File
@@ -57,6 +57,18 @@ pub(crate) fn catalog(
scope: &RuntimeWorkspaceScope,
request: &SessionMaterialCatalogRequest,
) -> SessionMaterialCatalogAvailability {
if scope.workspace_id == request.workspace_id
&& (1..=100).contains(&request.limit)
&& request.offset == 0
&& request.expected_catalog_sha256.is_none()
&& runtime
.session_not_materialized_scoped(scope, &request.source, &request.expected_session_id)
.is_ok_and(|empty| empty)
{
return SessionMaterialCatalogAvailability::NotMaterialized {
session_id: request.expected_session_id.clone(),
};
}
let result = (|| -> Result<_, SessionMaterialSourceError> {
if scope.workspace_id != request.workspace_id {
return Err("Session Workspace scope mismatch".into());