Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: record confirmed public Session history through durable Journal Jobs

Browse Source
This commit is contained in:
Hare
2026-10-16 02:19:24 +09:00
parent 18384482f6
commit 81af1f085c
62 changed files with 14553 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/worker-runtime/src/fs_store.rs
+12 -5
View File
@@ -256,15 +256,22 @@ impl FsRuntimeStore {
}
pub(crate) fn delete_worker_record(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> {
self.delete_worker_record_with_intent(
worker_id,
session_store::retention::SessionDeletionIntent::Ordinary,
)
}
pub(crate) fn delete_worker_record_with_intent(
&self,
worker_id: &WorkerId,
intent: session_store::retention::SessionDeletionIntent,
) -> Result<(), RuntimeError> {
let worker_dir = self.worker_dir(worker_id);
if !worker_dir.exists() {
return Ok(());
}
fs::remove_dir_all(&worker_dir).map_err(|source| RuntimeError::StoreIo {
operation: "delete worker store",
path: worker_dir,
source,
})
crate::retention::delete_worker_aggregate(&self.root, &worker_dir, intent)
}
pub(crate) fn load_runtime_state(&self) -> Result<PersistedRuntimeState, RuntimeError> {
crates/worker-runtime/src/http_server.rs
+3
View File
@@ -2420,6 +2420,9 @@ pub enum RuntimeHttpServerError {
#[cfg(test)]
mod tests {
#[cfg(feature = "fs-store")]
mod session_material_tests;
use super::*;
use crate::auth::RuntimeIdentityMaterial;
use crate::catalog::{ConfigBundleRef, ProfileSelector, WorkspaceApiRef};
crates/worker-runtime/src/http_server/runtime_management_api.rs
+140
View File
@@ -35,6 +35,86 @@ fn required_auth() -> Result<Extension<RuntimeAuthContext>, runtime_api::Runtime
})
}
fn material_scope(
state: &RuntimeHttpState,
workspace_id: &str,
) -> Result<RuntimeWorkspaceScope, runtime_api::RuntimeApiError> {
let scope = auth_workspace_scope(state, auth_extension().as_ref())
.map_err(api_error)?
.ok_or_else(|| {
runtime_api::RuntimeApiError::new(
403,
"runtime_session_material_scope_required",
"Canonical Session material requires a Workspace-scoped capability",
)
})?;
if scope.workspace_id != workspace_id {
return Err(runtime_api::RuntimeApiError::new(
403,
"runtime_session_material_scope_mismatch",
"Canonical Session Workspace does not match the authenticated capability",
));
}
Ok(scope)
}
fn material_pin(
state: &RuntimeHttpState,
request: runtime_api::SessionMaterialPinRequest,
release: bool,
) -> Result<runtime_api::SessionMaterialPinResponse, runtime_api::RuntimeApiError> {
let scope = material_scope(state, &request.workspace_id)?;
#[cfg(feature = "fs-store")]
{
let pin = request
.pin_id
.parse::<uuid::Uuid>()
.map(session_store::retention::SessionRetentionPin)
.map_err(|_| {
runtime_api::RuntimeApiError::new(
400,
"invalid_session_retention_pin",
"Invalid Session retention pin identity",
)
})?;
if release {
state.runtime.release_session_public_source_pin_scoped(
&scope,
&request.source,
&request.expected_session_id,
pin,
)
} else {
state.runtime.pin_session_public_source_scoped(
&scope,
&request.source,
&request.expected_session_id,
pin,
)
}
.map_err(|_| {
runtime_api::RuntimeApiError::new(
409,
"session_retention_rejected",
"Session retention operation could not be verified",
)
})?;
Ok(runtime_api::SessionMaterialPinResponse {
session_id: request.expected_session_id,
pin_id: request.pin_id,
})
}
#[cfg(not(feature = "fs-store"))]
{
let _ = (scope, release);
Err(runtime_api::RuntimeApiError::new(
503,
"session_retention_unavailable",
"Session retention requires filesystem storage",
))
}
}
#[derive(Clone)]
struct RuntimeManagementApi {
state: RuntimeHttpState,
@@ -289,6 +369,66 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
.map_err(api_error)
}
async fn session_material_pin(
&self,
request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, runtime_api::RuntimeApiError> {
material_pin(&self.state, request, false)
}
async fn session_material_release(
&self,
request: runtime_api::SessionMaterialPinRequest,
) -> Result<runtime_api::SessionMaterialPinResponse, runtime_api::RuntimeApiError> {
material_pin(&self.state, request, true)
}
async fn session_material_catalog(
&self,
request: runtime_api::SessionMaterialCatalogRequest,
) -> Result<runtime_api::SessionMaterialCatalogAvailability, runtime_api::RuntimeApiError> {
let scope = material_scope(&self.state, &request.workspace_id)?;
#[cfg(feature = "fs-store")]
{
Ok(self
.state
.runtime
.session_material_catalog_scoped(&scope, &request))
}
#[cfg(not(feature = "fs-store"))]
{
let _ = scope;
Ok(
runtime_api::SessionMaterialCatalogAvailability::Unavailable {
reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable,
message: "Canonical Session storage unavailable".into(),
},
)
}
}
async fn session_material_read(
&self,
request: runtime_api::SessionMaterialReadRequest,
) -> Result<runtime_api::SessionMaterialReadAvailability, runtime_api::RuntimeApiError> {
let scope = material_scope(&self.state, &request.workspace_id)?;
#[cfg(feature = "fs-store")]
{
Ok(self
.state
.runtime
.session_material_read_scoped(&scope, &request))
}
#[cfg(not(feature = "fs-store"))]
{
let _ = scope;
Ok(runtime_api::SessionMaterialReadAvailability::Unavailable {
reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable,
message: "Canonical Session storage unavailable".into(),
})
}
}
async fn session_public_search(
&self,
request: runtime_api::SessionPublicSearchRequest,
crates/worker-runtime/src/http_server/tests/session_material_tests.rs
+868
View File
@@ -0,0 +1,868 @@
//! Runtime/Session-store/retention and signed HTTP/generated-client integration.
//! Production catalog, filesystem stores, canonical reader, router and signature
//! verifier are connected. Only controller execution is replaced by AcceptingBackend;
//! no model, external server, user credentials or running Runtime is used.
use super::*;
use crate::retention::{DiagnosticsDisposition, SessionDisposition, runtime_session_retention};
use runtime_api::{
SessionMaterialCatalogAvailability as Catalog, SessionMaterialReadAvailability as Read,
SessionPublicSource as Source, SessionPublicUnavailableReason as Reason,
};
use session_store::retention::{SessionDeletionIntent, SessionRetentionPin};
use session_store::{
LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole,
LoggedSessionHistoryEntryId, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, Store,
WorkerAggregateStore, WorkerMetadataStore, WorkerSessionStore,
};
use std::io::{Seek, SeekFrom, Write};
use std::path::PathBuf;
use std::time::Duration;
struct Fixture {
temp: tempfile::TempDir,
runtime: Runtime,
scope: RuntimeWorkspaceScope,
worker: WorkerRef,
session: session_store::SessionId,
segment: session_store::SegmentId,
store: WorkerSessionStore,
}
fn message(id: &str, role: LoggedRole, text: &str) -> LoggedHistoryEntry {
LoggedHistoryEntry {
item: LoggedItem::Message {
role,
content: vec![LoggedContentPart::Text { text: text.into() }],
},
metadata: LoggedSessionHistoryMetadata {
entry_id: LoggedSessionHistoryEntryId(id.into()),
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None,
},
}
}
impl Fixture {
fn new() -> Self {
let temp = tempfile::tempdir().unwrap();
let runtime = Runtime::with_fs_store_and_execution_backend(
crate::FsRuntimeStoreOptions::new(temp.path()).with_runtime_id("runtime-test"),
Arc::new(AcceptingBackend),
)
.unwrap();
runtime.bind_runtime_identity("runtime-test").unwrap();
let scope = RuntimeWorkspaceScope::new("workspace-a", "https://backend.test");
let mut request = task_request("canonical Session fixture");
request.config_bundle = None;
request.memory_settings.as_mut().unwrap().workspace_id = scope.workspace_id.clone();
let worker = runtime
.create_worker_scoped(&scope, request)
.unwrap()
.worker_ref;
let aggregate = temp
.path()
.join("workers")
.join(worker.worker_id.to_string());
let session = session_store::new_session_id();
let segment = session_store::new_segment_id();
let store = WorkerSessionStore::new_with_retention(
aggregate.join("session"),
runtime_session_retention(temp.path()),
)
.unwrap();
store
.create_segment(
session,
segment,
&[
LogEntry::AnnotatedSegmentStart {
ts: 1,
session_id: session,
system_prompt: Some("private-system-prompt".into()),
config: agen::llm_client::RequestConfig::default(),
history: vec![],
forked_from: None,
compacted_from: None,
},
LogEntry::Invoke {
ts: 2,
trigger: protocol::InvokeKind::UserSend,
},
LogEntry::AnnotatedUserInput {
ts: 3,
segments: vec![],
extensions: vec![],
history: vec![message("user-1", LoggedRole::User, "公開の依頼")],
},
LogEntry::AnnotatedAssistantItem {
ts: 4,
entry: message("answer-1", LoggedRole::Assistant, "公開の回答🍎"),
},
LogEntry::RunCancelled {
ts: 5,
entry_id: None,
},
],
)
.unwrap();
WorkerAggregateStore::new(&aggregate, format!("worker-runtime-{}", worker.worker_id))
.unwrap()
.write(
&session_store::WorkerMetadata::new(
format!("worker-runtime-{}", worker.worker_id),
Some(session_store::WorkerActiveSegmentRef::active_segment(
session, segment,
)),
)
.with_workspace_id(&scope.workspace_id),
)
.unwrap();
Self {
temp,
runtime,
scope,
worker,
session,
segment,
store,
}
}
fn root(&self) -> PathBuf {
self.temp
.path()
.join("workers")
.join(self.worker.worker_id.to_string())
.join("session")
}
fn source(&self) -> Source {
Source::Retained {
worker_id: self.worker.worker_id,
}
}
fn request(&self) -> runtime_api::SessionMaterialCatalogRequest {
runtime_api::SessionMaterialCatalogRequest {
workspace_id: self.scope.workspace_id.clone(),
source: self.source(),
expected_session_id: self.session.to_string(),
expected_catalog_sha256: None,
offset: 0,
limit: 1,
}
}
fn catalog(&self) -> runtime_api::SessionMaterialCatalogPage {
match self
.runtime
.session_material_catalog_scoped(&self.scope, &self.request())
{
Catalog::Page { page } => page,
other => panic!("{other:?}"),
}
}
fn stop(&self) {
self.runtime
.stop_worker_scoped(&self.scope, &self.worker, None)
.unwrap();
}
fn retention_request(
&self,
disposition: SessionDisposition,
) -> WorkerRetentionExecutionRequest {
WorkerRetentionExecutionRequest {
operation_id: "remove-source".into(),
input_fingerprint: "source-and-archive".into(),
archive_id: (disposition == SessionDisposition::Archive)
.then(|| "canonical-archive".into()),
workspace_id: self.scope.workspace_id.clone(),
source_runtime_id: "runtime-test".into(),
worker_id: self.worker.worker_id,
source_created_at: "test".into(),
removed_at: "test".into(),
effective_profile: None,
retention_class: None,
policy_id: "test-policy".into(),
session_disposition: disposition,
diagnostics_disposition: DiagnosticsDisposition::Purge,
}
}
fn reader(
&self,
source: &Source,
) -> session_store::public_material::SessionPublicMaterialReader {
self.runtime
.session_public_material_reader_scoped(&self.scope, source, &self.session.to_string())
.unwrap()
}
}
#[test]
fn canonical_pages_read_frozen_originals_after_tail_append_and_reject_forged_descriptors() {
let f = Fixture::new();
let first = f.catalog();
assert_eq!(first.descriptors.len(), 1);
assert!(first.next_offset.is_some());
let metadata_json = serde_json::to_string(&first).unwrap();
assert!(!metadata_json.contains("公開の依頼"));
assert!(!metadata_json.contains("private-system-prompt"));
let mut continuation = f.request();
continuation.offset = first.next_offset.unwrap();
continuation.expected_catalog_sha256 = Some(first.catalog_sha256.clone());
let second = match f
.runtime
.session_material_catalog_scoped(&f.scope, &continuation)
{
Catalog::Page { page } => page,
other => panic!("{other:?}"),
};
assert_eq!(second.descriptors[0].canonical_order, 1);
continuation.offset = second
.next_offset
.expect("saved cancellation boundary is catalogued");
let boundary = match f
.runtime
.session_material_catalog_scoped(&f.scope, &continuation)
{
Catalog::Page { page } => page,
other => panic!("{other:?}"),
};
assert!(boundary.next_offset.is_none());
let boundary_origin: serde_json::Value =
serde_json::from_str(&boundary.descriptors[0].origin_json).unwrap();
assert_eq!(boundary_origin["kind"], "host_run_boundary");
let boundary_read = f.runtime.session_material_read_scoped(
&f.scope,
&runtime_api::SessionMaterialReadRequest {
workspace_id: f.scope.workspace_id.clone(),
source: f.source(),
expected_session_id: f.session.to_string(),
expected_descriptor: boundary.descriptors[0].clone(),
byte_offset: 0,
max_content_bytes: 16384,
},
);
let Read::Page {
page: boundary_text,
} = boundary_read
else {
panic!("{boundary_read:?}")
};
assert!(boundary_text.content.contains("cancelled"));
let expected = second.descriptors[0].clone();
f.store
.append(
f.session,
f.segment,
&LogEntry::AnnotatedAssistantItem {
ts: 6,
entry: message("tail", LoggedRole::Assistant, "new tail, not Job input"),
},
)
.unwrap();
assert!(matches!(
f.runtime
.session_material_catalog_scoped(&f.scope, &continuation),
Catalog::Unavailable {
reason: Reason::InvalidCursor,
..
}
));
let mut request = runtime_api::SessionMaterialReadRequest {
workspace_id: f.scope.workspace_id.clone(),
source: f.source(),
expected_session_id: f.session.to_string(),
expected_descriptor: expected.clone(),
byte_offset: 0,
max_content_bytes: 5,
};
let mut body = String::new();
while request.byte_offset < expected.total_bytes {
let page = match f.runtime.session_material_read_scoped(&f.scope, &request) {
Read::Page { page } => page,
other => panic!("{other:?}"),
};
assert_eq!(page.descriptor, expected);
assert!(page.content.len() <= 5);
assert!(page.next_offset > request.byte_offset);
body.push_str(&page.content);
request.byte_offset = page.next_offset;
}
assert_eq!(body.len() as u64, expected.total_bytes);
assert_eq!(
crate::workspace_issuer::hex_lower(&sha2::Sha256::digest(body.as_bytes())),
expected.sha256
);
assert!(body.contains("公開の回答🍎"));
assert!(!body.contains("new tail"));
request.byte_offset = body.find('公').unwrap() as u64 + 1;
assert!(matches!(
f.runtime.session_material_read_scoped(&f.scope, &request),
Read::Unavailable {
reason: Reason::InvalidCursor,
..
}
));
request.byte_offset = 0;
for forged in 0..5 {
request.expected_descriptor = expected.clone();
match forged {
0 => request.expected_descriptor.sha256 = "forged".into(),
1 => request.expected_descriptor.canonical_order += 1,
2 => request.expected_descriptor.total_bytes += 1,
3 => request.expected_descriptor.origin_json = "{}".into(),
_ => {
request.expected_descriptor.segment_id = session_store::new_segment_id().to_string()
}
}
assert!(matches!(
f.runtime.session_material_read_scoped(&f.scope, &request),
Read::Unavailable { .. }
));
}
}
#[test]
fn source_scope_session_and_active_pointer_mismatches_never_return_material() {
let f = Fixture::new();
assert!(
f.runtime
.session_public_material_reader_scoped(
&RuntimeWorkspaceScope::new("other", "server"),
&f.source(),
&f.session.to_string()
)
.is_err()
);
assert!(
f.runtime
.session_public_material_reader_scoped(
&f.scope,
&f.source(),
&session_store::new_session_id().to_string()
)
.is_err()
);
let mut request = f.request();
request.workspace_id = "other".into();
assert!(matches!(
f.runtime
.session_material_catalog_scoped(&f.scope, &request),
Catalog::Unavailable { .. }
));
let metadata = WorkerAggregateStore::new(
f.root().parent().unwrap(),
format!("worker-runtime-{}", f.worker.worker_id),
)
.unwrap();
metadata
.set_active(
&format!("worker-runtime-{}", f.worker.worker_id),
Some(session_store::WorkerActiveSegmentRef::active_segment(
session_store::new_session_id(),
f.segment,
)),
None,
)
.unwrap();
assert!(
f.runtime
.session_public_material_reader_scoped(&f.scope, &f.source(), &f.session.to_string())
.is_err()
);
assert!(
f.runtime
.session_public_material_reader_scoped(
&f.scope,
&Source::Archived {
archive_id: "../escape".into()
},
&f.session.to_string()
)
.is_err()
);
}
#[test]
fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() {
let f = Fixture::new();
let original: Vec<_> = f.reader(&f.source()).descriptors().cloned().collect();
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
f.runtime
.pin_session_public_source_scoped(&f.scope, &f.source(), &f.session.to_string(), pin)
.unwrap();
f.stop();
assert!(f.runtime.delete_worker_scoped(&f.scope, &f.worker).is_err());
assert!(f.root().is_dir());
assert!(
f.runtime
.execute_worker_retention(&f.retention_request(SessionDisposition::Purge))
.is_err()
);
assert!(f.root().is_dir());
// Use another operation identity after the deliberately different Purge request.
let mut request = f.retention_request(SessionDisposition::Archive);
request.operation_id = "archive-source".into();
let result = f.runtime.execute_worker_retention(&request).unwrap();
assert!(result.source_removed);
assert_eq!(
result.archive.as_ref().unwrap().source_active_segment_id,
Some(f.segment)
);
assert_eq!(
f.runtime.execute_worker_retention(&request).unwrap(),
result
);
assert!(!f.root().exists());
assert!(f.runtime.worker_detail(&f.worker).is_err());
assert!(
f.runtime
.session_public_material_reader_scoped(&f.scope, &f.source(), &f.session.to_string())
.is_err()
);
let archived = Source::Archived {
archive_id: "canonical-archive".into(),
};
let reader = f.reader(&archived);
assert_eq!(reader.descriptors().cloned().collect::<Vec<_>>(), original);
let body = reader.read_content(&original[0], 0, 16384).unwrap();
assert!(body.content.contains("公開の依頼"));
assert!(
f.runtime
.delete_session_public_source_scoped(
&f.scope,
&archived,
&f.session.to_string(),
SessionDeletionIntent::Ordinary
)
.is_err()
);
f.runtime
.release_session_public_source_pin_scoped(&f.scope, &archived, &f.session.to_string(), pin)
.unwrap();
f.runtime
.delete_session_public_source_scoped(
&f.scope,
&archived,
&f.session.to_string(),
SessionDeletionIntent::Ordinary,
)
.unwrap();
assert!(
f.runtime
.pin_session_public_source_scoped(&f.scope, &archived, &f.session.to_string(), pin)
.is_err()
);
assert!(f.temp.path().join("retention/session-sources").is_dir());
}
#[test]
fn explicit_erase_invalidates_pins_and_fences_old_readers_publishers_and_retry() {
let f = Fixture::new();
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
f.runtime
.pin_session_public_source_scoped(&f.scope, &f.source(), &f.session.to_string(), pin)
.unwrap();
let reader = f.reader(&f.source());
let descriptor = reader.descriptors().next().unwrap().clone();
f.stop();
let unrelated = session_store::new_session_id();
assert!(
f.runtime
.delete_session_public_source_scoped(
&f.scope,
&f.source(),
&unrelated.to_string(),
SessionDeletionIntent::ErasePermanently
)
.is_err()
);
runtime_session_retention(f.temp.path())
.ensure_live(unrelated)
.unwrap();
f.runtime
.delete_session_public_source_scoped(
&f.scope,
&f.source(),
&f.session.to_string(),
SessionDeletionIntent::ErasePermanently,
)
.unwrap();
assert!(!f.root().exists());
assert!(matches!(
reader.read_content(&descriptor, 0, 64),
Err(session_store::public_material::PublicMaterialError::Erased)
));
assert!(matches!(
f.store.create_segment(f.session, f.segment, &[]),
Err(session_store::StoreError::SessionErased(_))
));
assert!(matches!(
f.store.append(
f.session,
f.segment,
&LogEntry::TurnEnd {
ts: 10,
turn_count: 1
}
),
Err(session_store::StoreError::SessionErased(_))
));
assert!(
runtime_session_retention(f.temp.path())
.pins(f.session)
.unwrap()
.is_empty()
);
assert!(
f.runtime
.pin_session_public_source_scoped(&f.scope, &f.source(), &f.session.to_string(), pin)
.is_err()
);
f.runtime
.delete_session_public_source_scoped(
&f.scope,
&f.source(),
&f.session.to_string(),
SessionDeletionIntent::ErasePermanently,
)
.unwrap();
// Removing the remaining Worker registration/metadata still requires the
// caller's explicit erase intent; ordinary deletion may not infer it.
f.runtime
.delete_worker_with_intent_scoped(
&f.scope,
&f.worker,
SessionDeletionIntent::ErasePermanently,
)
.unwrap();
assert!(!f.root().parent().unwrap().exists());
assert!(matches!(
reader.read_content(&descriptor, 0, 64),
Err(session_store::public_material::PublicMaterialError::Erased)
));
}
#[test]
fn catalog_rejects_oversize_record_with_explicit_resource_limit() {
let f = Fixture::new();
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(
f.root()
.join("segments")
.join(format!("{}.jsonl", f.segment)),
)
.unwrap();
let start = file.seek(SeekFrom::End(0)).unwrap();
file.set_len(start + 64 * 1024 * 1024 + 1).unwrap();
file.write_all(b"\n").unwrap();
assert!(matches!(
f.runtime
.session_material_catalog_scoped(&f.scope, &f.request()),
Catalog::Unavailable {
reason: Reason::ResourceLimit,
..
}
));
}
#[test]
fn archive_scope_and_legacy_active_pointer_are_checked_and_erase_retry_never_restores_originals() {
let f = Fixture::new();
f.stop();
let request = f.retention_request(SessionDisposition::Archive);
f.runtime.execute_worker_retention(&request).unwrap();
let source = Source::Archived {
archive_id: "canonical-archive".into(),
};
let reader = f.reader(&source);
let descriptor = reader.descriptors().next().unwrap().clone();
let manifest_path = f
.temp
.path()
.join("archives/workers/canonical-archive/manifest.json");
let original: serde_json::Value =
serde_json::from_slice(&std::fs::read(&manifest_path).unwrap()).unwrap();
for field in [
"workspace_id",
"source_runtime_id",
"source_active_segment_id",
] {
let mut changed = original.clone();
if field == "source_active_segment_id" {
changed.as_object_mut().unwrap().remove(field);
} else {
changed[field] = serde_json::json!("other");
}
std::fs::write(&manifest_path, serde_json::to_vec(&changed).unwrap()).unwrap();
assert!(
f.runtime
.session_public_material_reader_scoped(&f.scope, &source, &f.session.to_string())
.is_err()
);
}
std::fs::write(&manifest_path, serde_json::to_vec(&original).unwrap()).unwrap();
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
f.runtime
.pin_session_public_source_scoped(&f.scope, &source, &f.session.to_string(), pin)
.unwrap();
let root = f
.temp
.path()
.join("archives/workers/canonical-archive/session");
let staged = f
.temp
.path()
.join("archives/workers/.interrupted-transfer/session");
std::fs::create_dir_all(&staged).unwrap();
std::fs::copy(
&manifest_path,
staged.parent().unwrap().join("manifest.json"),
)
.unwrap();
std::fs::write(
staged.join("partial-original.jsonl"),
"partially copied original",
)
.unwrap();
assert!(!staged.join("session.json").exists());
// Simulate the documented fence-before-physical-removal failure boundary.
let retention = runtime_session_retention(f.temp.path());
assert!(
retention
.delete(
f.session,
SessionDeletionIntent::ErasePermanently,
|| Err::<(), _>(session_store::StoreError::Io(std::io::Error::other(
"injected erase failure"
)))
)
.is_err()
);
assert!(root.exists());
assert!(matches!(
reader.read_content(&descriptor, 0, 64),
Err(session_store::public_material::PublicMaterialError::Erased)
));
f.runtime
.delete_session_public_source_scoped(
&f.scope,
&source,
&f.session.to_string(),
SessionDeletionIntent::ErasePermanently,
)
.unwrap();
f.runtime
.delete_session_public_source_scoped(
&f.scope,
&source,
&f.session.to_string(),
SessionDeletionIntent::ErasePermanently,
)
.unwrap();
assert!(!root.exists());
assert!(!staged.exists());
assert!(
f.runtime
.pin_session_public_source_scoped(&f.scope, &source, &f.session.to_string(), pin)
.is_err()
);
// An old removal receipt is evidence only and must not republish erased bytes.
f.runtime.execute_worker_retention(&request).unwrap();
assert!(!root.exists());
assert!(retention.pins(f.session).unwrap().is_empty());
}
#[test]
fn material_request_bounds_return_explicit_unavailable_without_previews() {
let f = Fixture::new();
for limit in [0, 101] {
let mut request = f.request();
request.limit = limit;
assert!(matches!(
f.runtime
.session_material_catalog_scoped(&f.scope, &request),
Catalog::Unavailable {
reason: Reason::ResourceLimit,
..
}
));
}
let descriptor = f.catalog().descriptors[0].clone();
for max_content_bytes in [0, 3, 16385] {
let request = runtime_api::SessionMaterialReadRequest {
workspace_id: f.scope.workspace_id.clone(),
source: f.source(),
expected_session_id: f.session.to_string(),
expected_descriptor: descriptor.clone(),
byte_offset: 0,
max_content_bytes,
};
assert!(matches!(
f.runtime.session_material_read_scoped(&f.scope, &request),
Read::Unavailable {
reason: Reason::ResourceLimit,
..
}
));
}
}
#[derive(Clone)]
struct SignedAuthorizer {
identity: RuntimeIdentityMaterial,
trust: WorkspaceIssuerTrustRecord,
}
impl runtime_api::client_support::RequestAuthorizer for SignedAuthorizer {
fn authorize(
&self,
request: runtime_api::client_support::AuthorizerRequest<'_>,
) -> Result<
runtime_api::client_support::framework::header::HeaderMap,
runtime_api::client_support::AuthorizationError,
> {
let claims = signed_claims(
&self.trust,
request.method.as_str(),
request.path_and_query,
request.body,
);
let token =
issue_workspace_capability_token(&self.identity.signing_key().unwrap(), &claims)
.unwrap();
let mut headers = runtime_api::client_support::framework::header::HeaderMap::new();
headers.insert("x-yoi-workspace-id", "workspace-a".parse().unwrap());
headers.insert(
header::AUTHORIZATION,
format!("Bearer {token}").parse().unwrap(),
);
Ok(headers)
}
}
struct ServerTask(tokio::task::JoinHandle<()>);
impl Drop for ServerTask {
fn drop(&mut self) {
self.0.abort();
}
}
#[tokio::test]
async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router() {
tokio::time::timeout(Duration::from_secs(30), async {
let f = Fixture::new();
let identity = RuntimeIdentityMaterial::generate("canonical-client-key").unwrap();
let trust = workspace_trust(&identity);
let app = runtime_http_router_with_workspace_auth(
f.runtime.clone(),
None,
workspace_test_auth(vec![trust.clone()]),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let _server = ServerTask(tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
}));
let client = runtime_api::RuntimeApiClient::builder(&format!("http://{address}"))
.unwrap()
.authorizer(SignedAuthorizer { identity, trust })
.build()
.unwrap();
let page = match client.session_material_catalog(f.request()).await.unwrap() {
Catalog::Page { page } => page,
other => panic!("{other:?}"),
};
let descriptor = page.descriptors[0].clone();
let read = client
.session_material_read(runtime_api::SessionMaterialReadRequest {
workspace_id: "workspace-a".into(),
source: f.source(),
expected_session_id: f.session.to_string(),
expected_descriptor: descriptor.clone(),
byte_offset: 0,
max_content_bytes: 16384,
})
.await
.unwrap();
let Read::Page { page } = read else {
panic!("{read:?}")
};
assert_eq!(page.descriptor, descriptor);
assert_eq!(page.next_offset, descriptor.total_bytes);
assert!(page.content.contains("公開の依頼"));
let pin = runtime_api::SessionMaterialPinRequest {
workspace_id: "workspace-a".into(),
source: f.source(),
expected_session_id: f.session.to_string(),
pin_id: uuid::Uuid::now_v7().to_string(),
};
assert_eq!(
client
.session_material_pin(pin.clone())
.await
.unwrap()
.pin_id,
pin.pin_id
);
assert_eq!(
runtime_session_retention(f.temp.path())
.pins(f.session)
.unwrap()
.len(),
1
);
client.session_material_release(pin).await.unwrap();
assert!(
runtime_session_retention(f.temp.path())
.pins(f.session)
.unwrap()
.is_empty()
);
})
.await
.expect("bounded signed client integration");
}
#[tokio::test]
async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
let f = Fixture::new();
let identity = RuntimeIdentityMaterial::generate("canonical-key").unwrap();
let trust = workspace_trust(&identity);
let app = runtime_http_router_with_workspace_auth(
f.runtime.clone(),
None,
workspace_test_auth(vec![trust.clone()]),
);
let path = "/v1/session-public/material/catalog";
let body = serde_json::to_vec(&f.request()).unwrap();
let claims = signed_claims(&trust, "POST", path, &body);
let first = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
assert_eq!(first.status(), StatusCode::OK);
let replay = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
assert_eq!(replay.status(), StatusCode::UNAUTHORIZED);
let claims = signed_claims(&trust, "POST", path, &body);
let changed = signed_request(
app.clone(),
&identity,
&claims,
b"{}".to_vec(),
"workspace-a",
)
.await;
assert_eq!(changed.status(), StatusCode::UNAUTHORIZED);
let mut wrong_scope = f.request();
wrong_scope.workspace_id = "other-workspace".into();
let wrong_body = serde_json::to_vec(&wrong_scope).unwrap();
let claims = signed_claims(&trust, "POST", path, &wrong_body);
let response = signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let unsigned = app
.oneshot(
Request::builder()
.method("POST")
.uri(path)
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
}
crates/worker-runtime/src/lib.rs
+3
View File
@@ -20,7 +20,10 @@ pub mod resource;
pub mod retention;
mod runtime;
#[cfg(feature = "fs-store")]
mod session_material;
#[cfg(feature = "fs-store")]
mod session_public;
mod session_retention;
pub mod ssh_host_key_probe;
pub mod worker_backend;
pub mod worker_source;
crates/worker-runtime/src/retention.rs
+243 -13
View File
@@ -133,6 +133,9 @@ pub struct WorkerSessionArchiveManifest {
pub source_runtime_id: String,
pub source_worker_id: WorkerId,
pub source_session_id: String,
/// Older archives have no attested active lineage and cannot serve canonical scans.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_active_segment_id: Option<session_store::SegmentId>,
pub segment_ids: Vec<String>,
pub source_created_at: String,
pub removed_at: String,
@@ -173,6 +176,154 @@ pub(crate) trait WorkerRetentionProvider: Send + Sync {
) -> Result<WorkerRetentionExecutionResult, RuntimeError>;
}
pub use crate::session_retention::runtime_session_retention;
pub(crate) fn aggregate_session_id(
worker_dir: &Path,
) -> Result<Option<session_store::SessionId>, RuntimeError> {
let session_path = worker_dir.join("session/session.json");
let saved = match fs::read(&session_path) {
Ok(bytes) => {
let manifest: CanonicalSessionManifest =
serde_json::from_slice(&bytes).map_err(|_| {
RuntimeError::InvalidRequest("Retained Session manifest is invalid".into())
})?;
Some(
manifest
.session_id
.parse::<session_store::SessionId>()
.map_err(|_| {
RuntimeError::InvalidRequest("Retained Session identity is invalid".into())
})?,
)
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
Err(source) => {
return Err(RuntimeError::StoreIo {
operation: "read retained Session identity",
path: session_path,
source,
});
}
};
let metadata_path = worker_dir.join("metadata.json");
let active = match fs::read(&metadata_path) {
Ok(bytes) => {
let metadata: session_store::WorkerMetadata =
serde_json::from_slice(&bytes).map_err(|_| {
RuntimeError::InvalidRequest("Retained Worker metadata is invalid".into())
})?;
metadata.active.map(|active| active.session_id)
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
Err(source) => {
return Err(RuntimeError::StoreIo {
operation: "read retained Session pointer",
path: metadata_path,
source,
});
}
};
if let (Some(saved), Some(active)) = (saved, active) {
if saved != active {
return Err(RuntimeError::InvalidRequest(
"Retained Session manifest and active pointer disagree".into(),
));
}
}
if saved.is_none() && active.is_none() {
let session_dir = worker_dir.join("session");
if session_dir.is_dir()
&& !collect_files(&session_dir, "inspect uninitialized Session")?.is_empty()
{
return Err(RuntimeError::InvalidRequest(
"Session material has no retained identity".into(),
));
}
}
Ok(saved.or(active))
}
/// Called only while the common Session deletion lock is held. Include staged
/// archives from interrupted transfers: erasure must not leave a second original.
pub(crate) fn remove_archived_session_copies(
runtime_root: &Path,
session: session_store::SessionId,
) -> Result<(), session_store::StoreError> {
let archives = runtime_root.join("archives/workers");
let entries = match fs::read_dir(archives) {
Ok(entries) => entries,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.into()),
};
for entry in entries {
let entry = entry?;
if !entry.file_type()?.is_dir() {
continue;
}
let session_root = entry.path().join("session");
let owner = match fs::read(entry.path().join("manifest.json")) {
Ok(bytes) => Some(
serde_json::from_slice::<WorkerSessionArchiveManifest>(&bytes)?.source_session_id,
),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
Err(error) => return Err(error.into()),
};
if let Some(owner) = owner {
if owner == session.to_string() {
match fs::remove_dir_all(&session_root) {
Ok(()) => (),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => (),
Err(error) => return Err(error.into()),
}
}
continue;
}
let bytes = match fs::read(session_root.join("session.json")) {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(error.into()),
};
let manifest: CanonicalSessionManifest = serde_json::from_slice(&bytes)?;
if manifest.session_id == session.to_string() {
fs::remove_dir_all(&session_root)?;
}
}
Ok(())
}
pub(crate) fn retention_error(error: session_store::StoreError) -> RuntimeError {
RuntimeError::InvalidRequest(format!("Session retention rejected operation: {error}"))
}
/// Direct Worker deletion cannot bypass pins. Purge policy is not erase intent.
pub(crate) fn delete_worker_aggregate(
runtime_root: &Path,
worker_dir: &Path,
intent: session_store::retention::SessionDeletionIntent,
) -> Result<(), RuntimeError> {
let session = aggregate_session_id(worker_dir)?;
let remove = || -> Result<(), session_store::StoreError> {
if intent == session_store::retention::SessionDeletionIntent::ErasePermanently {
if let Some(session) = session {
remove_archived_session_copies(runtime_root, session)?;
}
}
match fs::remove_dir_all(worker_dir) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error.into()),
}
};
if let Some(session) = session {
runtime_session_retention(runtime_root)
.delete(session, intent, remove)
.map_err(retention_error)
} else {
remove().map_err(retention_error)
}
}
/// Filesystem provider for the canonical Runtime Worker aggregate.
#[derive(Clone, Debug)]
pub(crate) struct FsWorkerRetentionProvider {
@@ -355,6 +506,16 @@ impl FsWorkerRetentionProvider {
pub(crate) fn session_archive(
&self,
archive_id: &str,
) -> Result<(WorkerSessionArchiveManifest, PathBuf), RuntimeError> {
let (manifest, session_root) = self.session_archive_identity(archive_id)?;
let archive_dir = self.archive_dir(archive_id)?;
let manifest = validate_existing_archive(&archive_dir, &manifest)?;
Ok((manifest, session_root))
}
pub(crate) fn session_archive_identity(
&self,
archive_id: &str,
) -> Result<(WorkerSessionArchiveManifest, PathBuf), RuntimeError> {
let archive_dir = self.archive_dir(archive_id)?;
let manifest: WorkerSessionArchiveManifest = read_json(
@@ -368,7 +529,6 @@ impl FsWorkerRetentionProvider {
message: "archive manifest identity mismatch".to_string(),
});
}
let manifest = validate_existing_archive(&archive_dir, &manifest)?;
Ok((manifest, archive_dir.join("session")))
}
@@ -503,12 +663,46 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
worker_id: request.worker_id,
});
}
// Transfer keeps the same Session pin/fence namespace. Hold its lock
// across verified archive publication and aggregate removal. A policy
// purge instead uses ordinary deletion and must reject outstanding pins.
let retention = runtime_session_retention(&self.runtime_root);
let transfer_guard = if request.session_disposition == SessionDisposition::Archive {
aggregate_session_id(&worker_dir)?
.map(|session| retention.lock_live(session))
.transpose()
.map_err(retention_error)?
} else {
None
};
let archive = match request.session_disposition {
SessionDisposition::Archive => {
Some(commit_session_archive(self, request, &worker_dir)?)
}
SessionDisposition::Purge => None,
};
if let Some(manifest) = &archive {
if let Some(active_segment) = manifest.source_active_segment_id {
// We already hold the Session transfer lock. Do not reacquire it
// through the external-retention constructor during verification.
let reader =
session_store::public_material::SessionPublicMaterialReader::open_worker(
&self.archive_dir(&manifest.archive_id)?.join("session"),
active_segment,
Default::default(),
)
.map_err(|error| {
RuntimeError::InvalidRequest(format!(
"Archived canonical source is unreadable: {error}"
))
})?;
if reader.session_id().to_string() != manifest.source_session_id {
return Err(RuntimeError::InvalidRequest(
"Archived canonical source identity mismatch".into(),
));
}
}
}
let diagnostics_retained = match request.diagnostics_disposition {
DiagnosticsDisposition::Purge => false,
DiagnosticsDisposition::Retain => {
@@ -540,11 +734,23 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
"commit pending Worker retention receipt",
)?;
fs::remove_dir_all(&worker_dir).map_err(|source| RuntimeError::StoreIo {
operation: "remove retained Worker aggregate",
path: worker_dir.clone(),
source,
})?;
match request.session_disposition {
SessionDisposition::Archive => {
// commit_session_archive already compared all original bytes.
// The transfer lock remains held until removal has completed.
fs::remove_dir_all(&worker_dir).map_err(|source| RuntimeError::StoreIo {
operation: "remove retained Worker aggregate",
path: worker_dir.clone(),
source,
})?;
}
SessionDisposition::Purge => delete_worker_aggregate(
&self.runtime_root,
&worker_dir,
session_store::retention::SessionDeletionIntent::Ordinary,
)?,
}
drop(transfer_guard);
sync_directory(
worker_dir.parent().unwrap_or(&self.runtime_root),
"remove retained Worker aggregate",
@@ -730,6 +936,25 @@ fn commit_session_archive(
workspace_id: request.workspace_id.clone(),
source_runtime_id: request.source_runtime_id.clone(),
source_worker_id: request.worker_id,
source_active_segment_id: if worker_dir.join("metadata.json").is_file() {
let metadata: session_store::WorkerMetadata = read_json(
&worker_dir.join("metadata.json"),
"archive active Session pointer",
)?;
let active = metadata.active.ok_or_else(|| {
RuntimeError::InvalidRequest("Worker has no active Session pointer".into())
})?;
if active.session_id.to_string() != session.session_id
|| metadata.workspace_id.as_deref() != Some(&request.workspace_id)
{
return Err(RuntimeError::InvalidRequest(
"Archive Session pointer scope mismatch".into(),
));
}
active.segment_id
} else {
None
},
source_session_id: session.session_id,
segment_ids: segment_ids.into_iter().collect(),
source_created_at: request.source_created_at.clone(),
@@ -770,6 +995,13 @@ fn commit_session_archive(
path: staging.clone(),
source,
})?;
// Persist source ownership before copying any original bytes. An
// interrupted staging tree remains attributable to explicit erasure.
atomic_write_json(
&staging.join("manifest.json"),
&manifest,
"archive Worker Session",
)?;
let target_session = staging.join("session");
fs::create_dir(&target_session).map_err(|source| RuntimeError::StoreIo {
operation: "archive Worker Session",
@@ -777,11 +1009,6 @@ fn commit_session_archive(
source,
})?;
copy_files(&source_files, &target_session, "archive Worker Session")?;
atomic_write_json(
&staging.join("manifest.json"),
&manifest,
"archive Worker Session",
)?;
sync_tree(&staging, "archive Worker Session")?;
fs::rename(&staging, &archive_dir).map_err(|source| RuntimeError::StoreIo {
operation: "archive Worker Session",
@@ -1264,7 +1491,7 @@ mod tests {
);
write_json(
&worker.join("session/session.json"),
&serde_json::json!({"schema_version": 1, "session_id": "session-a"}),
&serde_json::json!({"schema_version": 1, "session_id": "00000000-0000-4000-8000-000000000001"}),
);
fs::create_dir_all(worker.join("session/segments")).unwrap();
fs::write(worker.join("session/segments/segment-a.jsonl"), b"one\n").unwrap();
@@ -1310,7 +1537,10 @@ mod tests {
let first = provider.execute(&request).unwrap();
assert!(first.source_removed);
let archive = first.archive.as_ref().unwrap();
assert_eq!(archive.source_session_id, "session-a");
assert_eq!(
archive.source_session_id,
"00000000-0000-4000-8000-000000000001"
);
assert_eq!(archive.segment_ids, vec!["segment-a"]);
assert!(!temp.path().join(format!("workers/{worker_id}")).exists());
assert!(
crates/worker-runtime/src/runtime.rs
+302 -5
View File
@@ -3932,7 +3932,11 @@ impl Runtime {
scope: &RuntimeWorkspaceScope,
worker_ref: &WorkerRef,
) -> Result<WorkerDeleteResult, RuntimeError> {
self.delete_worker_with_scope(worker_ref, Some(scope))
self.delete_worker_with_scope(
worker_ref,
Some(scope),
session_store::retention::SessionDeletionIntent::Ordinary,
)
}
/// Delete a non-running Worker from Runtime state and persisted Worker storage.
@@ -3940,13 +3944,30 @@ impl Runtime {
&self,
worker_ref: &WorkerRef,
) -> Result<WorkerDeleteResult, RuntimeError> {
self.delete_worker_with_scope(worker_ref, None)
self.delete_worker_with_scope(
worker_ref,
None,
session_store::retention::SessionDeletionIntent::Ordinary,
)
}
/// A trusted Host must choose explicit erase intent separately from policy
/// Purge. This also permits finishing aggregate removal after source erasure.
#[cfg(feature = "fs-store")]
pub fn delete_worker_with_intent_scoped(
&self,
scope: &RuntimeWorkspaceScope,
worker_ref: &WorkerRef,
intent: session_store::retention::SessionDeletionIntent,
) -> Result<WorkerDeleteResult, RuntimeError> {
self.delete_worker_with_scope(worker_ref, Some(scope), intent)
}
fn delete_worker_with_scope(
&self,
worker_ref: &WorkerRef,
scope: Option<&RuntimeWorkspaceScope>,
intent: session_store::retention::SessionDeletionIntent,
) -> Result<WorkerDeleteResult, RuntimeError> {
let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?;
let _operation_guard = operation_lock
@@ -4002,7 +4023,17 @@ impl Runtime {
worker_ref.worker_id
)));
}
if state.delete_worker_record(&worker_ref.worker_id).is_err() {
#[cfg(feature = "fs-store")]
let deletion = match state.fs_store() {
Some(store) => store.delete_worker_record_with_intent(&worker_ref.worker_id, intent),
None => Ok(()),
};
#[cfg(not(feature = "fs-store"))]
let deletion = {
let _ = intent;
state.delete_worker_record(&worker_ref.worker_id)
};
if deletion.is_err() {
state.record_worker_delete_persistence_failure(worker_ref);
let _diagnostic_persistence = state.persist_runtime_snapshot();
return Err(RuntimeError::WorkerDeletePersistenceFailed {
@@ -4773,6 +4804,245 @@ impl Runtime {
)
}
/// Retain only an authenticated source whose saved Session identity matches.
/// The existence check and pin publication share the physical deletion lock.
#[cfg(feature = "fs-store")]
pub fn pin_session_public_source_scoped(
&self,
scope: &RuntimeWorkspaceScope,
source: &runtime_api::SessionPublicSource,
expected_session_id: &str,
pin: session_store::retention::SessionRetentionPin,
) -> Result<(), RuntimeError> {
let (root, _) = self.resolve_session_public_source(scope, source)?;
let session = expected_session_id
.parse::<session_store::SessionId>()
.map_err(|_| RuntimeError::InvalidRequest("Invalid Session identity".into()))?;
self.session_retention_store()?
.pin(session, pin, || {
let store = match session_store::WorkerSessionStore::open_read_only(&root) {
Ok(store) => store,
Err(session_store::StoreError::Io(error))
if error.kind() == std::io::ErrorKind::NotFound =>
{
return Ok(false);
}
Err(error) => return Err(error),
};
Ok(store.session_id()? == Some(session))
})
.map_err(crate::retention::retention_error)
}
#[cfg(feature = "fs-store")]
pub fn release_session_public_source_pin_scoped(
&self,
scope: &RuntimeWorkspaceScope,
source: &runtime_api::SessionPublicSource,
expected_session_id: &str,
pin: session_store::retention::SessionRetentionPin,
) -> Result<(), RuntimeError> {
let (root, _) = self.resolve_session_public_source(scope, source)?;
let session = expected_session_id
.parse::<session_store::SessionId>()
.map_err(|_| RuntimeError::InvalidRequest("Invalid Session identity".into()))?;
if session_store::WorkerSessionStore::open_read_only(&root)
.and_then(|store| store.session_id())
.map_err(crate::retention::retention_error)?
!= Some(session)
{
return Err(RuntimeError::InvalidRequest(
"Session source identity mismatch".into(),
));
}
self.session_retention_store()?
.release(session, pin)
.map_err(crate::retention::retention_error)
}
/// Physical source deletion is separate from Worker registration/lifecycle.
/// Only this explicit intent may invalidate pins and persist an erase fence;
/// a retention policy's Purge remains ordinary deletion.
#[cfg(feature = "fs-store")]
pub fn delete_session_public_source_scoped(
&self,
scope: &RuntimeWorkspaceScope,
source: &runtime_api::SessionPublicSource,
expected_session_id: &str,
intent: session_store::retention::SessionDeletionIntent,
) -> Result<(), RuntimeError> {
let (root, archive) =
self.resolve_session_public_source_with_validation(scope, source, false)?;
let owner = match source {
runtime_api::SessionPublicSource::Retained { worker_id } => *worker_id,
runtime_api::SessionPublicSource::Archived { .. } => {
archive
.as_ref()
.ok_or_else(|| {
RuntimeError::InvalidRequest("Archive identity unavailable".into())
})?
.source_worker_id
}
};
let operation_lock = self.worker_operation_lock(owner)?;
let _operation_guard = operation_lock
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
let state = self.lock()?;
if let Some(worker) = state.workers.get(&owner) {
if worker.workspace_id.as_deref() != Some(scope.workspace_id.as_str())
|| worker.status != WorkerStatus::Stopped
|| worker.pending_lifecycle_operation_name().is_some()
{
return Err(RuntimeError::InvalidRequest(
"Session deletion requires a stopped, settled source Worker".into(),
));
}
}
let store = state.fs_store().ok_or_else(|| {
RuntimeError::InvalidRequest("Session deletion requires filesystem storage".into())
})?;
let session = expected_session_id
.parse::<session_store::SessionId>()
.map_err(|_| RuntimeError::InvalidRequest("Invalid Session identity".into()))?;
let source_session = match archive.as_ref() {
Some(manifest) => manifest
.source_session_id
.parse::<session_store::SessionId>()
.ok(),
None => {
crate::retention::aggregate_session_id(root.parent().ok_or_else(|| {
RuntimeError::InvalidRequest("Source root unavailable".into())
})?)?
}
};
if source_session != Some(session) {
return Err(RuntimeError::InvalidRequest(
"Session source identity mismatch".into(),
));
}
crate::retention::runtime_session_retention(store.runtime_dir())
.delete(session, intent, || {
let saved = match archive.as_ref() {
Some(manifest) => manifest
.source_session_id
.parse::<session_store::SessionId>()
.ok(),
None => crate::retention::aggregate_session_id(
root.parent()
.ok_or(session_store::StoreError::SessionSourceMissing(session))?,
)
.map_err(|error| {
session_store::StoreError::Io(std::io::Error::other(error.to_string()))
})?,
};
if saved != Some(session) {
return Err(session_store::StoreError::SessionSourceMissing(session));
}
if intent == session_store::retention::SessionDeletionIntent::ErasePermanently {
crate::retention::remove_archived_session_copies(store.runtime_dir(), session)?;
let original = store
.runtime_dir()
.join("workers")
.join(owner.to_string())
.join("session");
if original != root
&& original.is_dir()
&& session_store::WorkerSessionStore::open_read_only(&original)?
.session_id()?
== Some(session)
{
std::fs::remove_dir_all(original)?;
}
}
match std::fs::remove_dir_all(&root) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error.into()),
}
})
.map_err(crate::retention::retention_error)
}
#[cfg(feature = "fs-store")]
pub fn session_material_catalog_scoped(
&self,
scope: &RuntimeWorkspaceScope,
request: &runtime_api::SessionMaterialCatalogRequest,
) -> runtime_api::SessionMaterialCatalogAvailability {
crate::session_material::catalog(self, scope, request)
}
#[cfg(feature = "fs-store")]
pub fn session_material_read_scoped(
&self,
scope: &RuntimeWorkspaceScope,
request: &runtime_api::SessionMaterialReadRequest,
) -> runtime_api::SessionMaterialReadAvailability {
crate::session_material::read(self, scope, request)
}
#[cfg(feature = "fs-store")]
fn session_retention_store(
&self,
) -> Result<session_store::retention::SessionRetentionStore, RuntimeError> {
let state = self.lock()?;
let store = state.fs_store().ok_or_else(|| {
RuntimeError::InvalidRequest("Session retention requires an fs-backed Runtime".into())
})?;
Ok(crate::retention::runtime_session_retention(
store.runtime_dir(),
))
}
/// Resolve the Backend-selected source without exposing local storage paths.
/// The saved active pointer, never a caller-provided Segment, selects lineage.
#[cfg(feature = "fs-store")]
pub fn session_public_material_reader_scoped(
&self,
scope: &RuntimeWorkspaceScope,
source: &runtime_api::SessionPublicSource,
expected_session_id: &str,
) -> Result<
session_store::public_material::SessionPublicMaterialReader,
crate::worker_backend::SessionMaterialSourceError,
> {
use crate::worker_backend::SessionMaterialSourceError::Source;
use session_store::public_material::{PublicMaterialLimits, SessionPublicMaterialReader};
let (session_root, archive) = self
.resolve_session_public_source(scope, source)
.map_err(|error| Source(error.to_string()))?;
let reader = match (source, archive) {
(runtime_api::SessionPublicSource::Retained { worker_id }, None) => {
crate::worker_backend::open_session_public_material_reader(
session_root.parent().ok_or("Session root is unavailable")?,
&scope.workspace_id,
&WorkerRef::new(*worker_id),
expected_session_id,
)?
}
(runtime_api::SessionPublicSource::Archived { .. }, Some(manifest)) => {
if manifest.source_session_id != expected_session_id {
return Err(Source("Archive Session identity mismatch".into()));
}
SessionPublicMaterialReader::open_worker_with_retention(
&session_root,
manifest
.source_active_segment_id
.ok_or("Archive has no canonical active Segment")?,
PublicMaterialLimits::default(),
self.session_retention_store()
.map_err(|error| Source(error.to_string()))?,
)?
}
_ => return Err(Source("Session source is unavailable".into())),
};
if reader.session_id().to_string() != expected_session_id {
return Err(Source("Session identity mismatch".into()));
}
Ok(reader)
}
/// Search the committed public projection of one Backend-selected retained
/// Session or verified archive without restoring a Worker.
#[cfg(feature = "fs-store")]
@@ -4844,6 +5114,22 @@ impl Runtime {
Option<crate::retention::WorkerSessionArchiveManifest>,
),
RuntimeError,
> {
self.resolve_session_public_source_with_validation(scope, source, true)
}
#[cfg(feature = "fs-store")]
fn resolve_session_public_source_with_validation(
&self,
scope: &RuntimeWorkspaceScope,
source: &runtime_api::SessionPublicSource,
verify_archive: bool,
) -> Result<
(
std::path::PathBuf,
Option<crate::retention::WorkerSessionArchiveManifest>,
),
RuntimeError,
> {
let state = self.lock()?;
let store = state.fs_store().ok_or_else(|| {
@@ -4876,7 +5162,11 @@ impl Runtime {
)
})?;
let provider = FsWorkerRetentionProvider::new(store.runtime_dir());
let (manifest, session_root) = provider.session_archive(archive_id)?;
let (manifest, session_root) = if verify_archive {
provider.session_archive(archive_id)?
} else {
provider.session_archive_identity(archive_id)?
};
if manifest.workspace_id != scope.workspace_id
|| manifest.source_runtime_id != runtime_id
{
@@ -4921,6 +5211,10 @@ impl Runtime {
&self,
request: &WorkerRetentionExecutionRequest,
) -> Result<WorkerRetentionExecutionResult, RuntimeError> {
let operation_lock = self.worker_operation_lock(request.worker_id)?;
let _operation_guard = operation_lock
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
let mut state = self.lock()?;
let runtime_id = state.runtime_identity.clone().ok_or_else(|| {
RuntimeError::InvalidRequest(
@@ -6451,7 +6745,8 @@ fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), R
.validate()
.map_err(|message| RuntimeError::InvalidRequest(message.into()))?;
if request.workspace_api.is_none()
|| (binding.subjektiv_consolidation && !request.subjektiv_attached)
|| ((binding.subjektiv_consolidation || binding.subjektiv_journal)
&& !request.subjektiv_attached)
{
return Err(RuntimeError::InvalidRequest("Backend Job capability requires Workspace authority and consolidation requires trusted subject attachment".into()));
}
@@ -7302,6 +7597,7 @@ mod tests {
attempt_id: "attempt-1".into(),
input_digest: Some(format!("sha256:{}", "a".repeat(64))),
subjektiv_consolidation: false,
subjektiv_journal: false,
});
let owner = scope("local", "server-job-test");
let created = runtime
@@ -7351,6 +7647,7 @@ mod tests {
attempt_id: "attempt-1".into(),
input_digest: None,
subjektiv_consolidation: true,
subjektiv_journal: false,
});
assert!(validate_create_worker_request(&request).is_err());
request.workspace_api = Some(WorkspaceApiRef {
crates/worker-runtime/src/session_material.rs
+154
View File
@@ -0,0 +1,154 @@
//! Bounded wire projection of canonical public material, never Memory previews.
use runtime_api::*;
use session_store::public_material::{
PublicMaterialDescriptor, PublicMaterialError, SessionPublicMaterialReader,
};
use sha2::{Digest, Sha256};
use crate::{Runtime, RuntimeWorkspaceScope, worker_backend::SessionMaterialSourceError};
fn descriptor(
value: &PublicMaterialDescriptor,
) -> Result<SessionMaterialDescriptor, PublicMaterialError> {
let result = SessionMaterialDescriptor {
segment_id: value.identity.segment_id.to_string(),
entry_ref: value.identity.entry_ref.clone(),
canonical_order: value.canonical_order,
origin_json: serde_json::to_string(&value.origin)?,
sha256: value.sha256.clone(),
total_bytes: value.total_bytes,
};
if serde_json::to_vec(&result)?.len() > 16 * 1024 {
return Err(PublicMaterialError::ResourceLimit);
}
Ok(result)
}
fn catalog_digest(reader: &SessionPublicMaterialReader) -> Result<String, PublicMaterialError> {
let mut digest = Sha256::new();
digest.update(reader.session_id().to_string());
digest.update(reader.active_segment_id().to_string());
for value in reader.descriptors() {
digest.update(serde_json::to_vec(&descriptor(value)?)?);
digest.update(b"\n");
}
Ok(digest
.finalize()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect())
}
fn unavailable(error: SessionMaterialSourceError) -> SessionPublicUnavailableReason {
use PublicMaterialError as E;
use SessionPublicUnavailableReason as R;
match error {
SessionMaterialSourceError::Material(E::ResourceLimit) => R::ResourceLimit,
SessionMaterialSourceError::Material(E::Changed | E::InvalidOffset) => R::InvalidCursor,
SessionMaterialSourceError::Material(E::Missing | E::Erased) => R::RetentionMissing,
SessionMaterialSourceError::Material(E::Inconsistent | E::Record(_)) => R::CorruptLog,
_ => R::StorageUnavailable,
}
}
pub(crate) fn catalog(
runtime: &Runtime,
scope: &RuntimeWorkspaceScope,
request: &SessionMaterialCatalogRequest,
) -> SessionMaterialCatalogAvailability {
let result = (|| -> Result<_, SessionMaterialSourceError> {
if scope.workspace_id != request.workspace_id {
return Err("Session Workspace scope mismatch".into());
}
if request.limit == 0 || request.limit > 100 {
return Err(PublicMaterialError::ResourceLimit.into());
}
let reader = runtime.session_public_material_reader_scoped(
scope,
&request.source,
&request.expected_session_id,
)?;
let digest = catalog_digest(&reader)?;
if request
.expected_catalog_sha256
.as_ref()
.is_some_and(|expected| expected != &digest)
{
return Err(PublicMaterialError::Changed.into());
}
let offset =
usize::try_from(request.offset).map_err(|_| PublicMaterialError::InvalidOffset)?;
let count = reader.descriptors().len();
if offset > count {
return Err(PublicMaterialError::InvalidOffset.into());
}
let descriptors = reader
.descriptors()
.skip(offset)
.take(request.limit)
.map(descriptor)
.collect::<Result<Vec<_>, _>>()?;
let end = offset + descriptors.len();
Ok(SessionMaterialCatalogPage {
session_id: reader.session_id().to_string(),
active_segment_id: reader.active_segment_id().to_string(),
catalog_sha256: digest,
descriptors,
next_offset: (end < count).then_some(end as u64),
})
})();
match result {
Ok(page) => SessionMaterialCatalogAvailability::Page { page },
Err(error) => SessionMaterialCatalogAvailability::Unavailable {
reason: unavailable(error),
message: "Canonical Session catalog unavailable".into(),
},
}
}
pub(crate) fn read(
runtime: &Runtime,
scope: &RuntimeWorkspaceScope,
request: &SessionMaterialReadRequest,
) -> SessionMaterialReadAvailability {
let result = (|| -> Result<_, SessionMaterialSourceError> {
if scope.workspace_id != request.workspace_id {
return Err("Session Workspace scope mismatch".into());
}
if !(4..=16 * 1024).contains(&request.max_content_bytes) {
return Err(PublicMaterialError::ResourceLimit.into());
}
let reader = runtime.session_public_material_reader_scoped(
scope,
&request.source,
&request.expected_session_id,
)?;
// Resolve saved identity/order/digest here; no caller-supplied descriptor is adopted.
// New tail entries may arrive during a Job without changing its frozen originals.
let selected = reader
.descriptors()
.find(|value| {
value.identity.segment_id.to_string() == request.expected_descriptor.segment_id
&& value.identity.entry_ref == request.expected_descriptor.entry_ref
})
.ok_or(PublicMaterialError::Missing)?;
if descriptor(selected)? != request.expected_descriptor {
return Err(PublicMaterialError::Changed.into());
}
let content =
reader.read_content(selected, request.byte_offset, request.max_content_bytes)?;
Ok(SessionMaterialReadPage {
descriptor: descriptor(selected)?,
content: content.content,
next_offset: content.next_offset,
})
})();
match result {
Ok(page) => SessionMaterialReadAvailability::Page { page },
Err(error) => SessionMaterialReadAvailability::Unavailable {
reason: unavailable(error),
message: "Canonical Session material unavailable".into(),
},
}
}
crates/worker-runtime/src/session_public.rs
+1
View File
@@ -364,6 +364,7 @@ fn api_archive_manifest(manifest: &DomainArchiveManifest) -> WorkerSessionArchiv
source_runtime_id: manifest.source_runtime_id.clone(),
source_worker_id: manifest.source_worker_id,
source_session_id: manifest.source_session_id.clone(),
source_active_segment_id: manifest.source_active_segment_id.map(|id| id.to_string()),
segment_ids: manifest.segment_ids.clone(),
source_created_at: manifest.source_created_at.clone(),
removed_at: manifest.removed_at.clone(),
crates/worker-runtime/src/session_retention.rs
+9
View File
@@ -0,0 +1,9 @@
//! One control namespace for live Worker sources and their verified archives.
//! It outlives aggregate removal and is also used by publishers and readers.
use session_store::retention::SessionRetentionStore;
use std::path::Path;
pub fn runtime_session_retention(runtime_root: &Path) -> SessionRetentionStore {
SessionRetentionStore::new(runtime_root.join("retention/session-sources"))
}
crates/worker-runtime/src/worker_backend.rs
+118 -18
View File
@@ -432,6 +432,66 @@ impl WorkspacePromptProjectionCache {
}
}
#[derive(Debug, thiserror::Error)]
pub enum SessionMaterialSourceError {
#[error("{0}")]
Source(String),
#[error(transparent)]
Material(#[from] session_store::public_material::PublicMaterialError),
}
impl From<String> for SessionMaterialSourceError {
fn from(message: String) -> Self {
Self::Source(message)
}
}
impl From<&str> for SessionMaterialSourceError {
fn from(message: &str) -> Self {
Self::Source(message.into())
}
}
pub(crate) fn open_session_public_material_reader(
aggregate: &Path,
workspace_id: &str,
worker_ref: &WorkerRef,
expected_session_id: &str,
) -> Result<session_store::public_material::SessionPublicMaterialReader, SessionMaterialSourceError>
{
use session_store::public_material::{PublicMaterialLimits, SessionPublicMaterialReader};
let metadata = WorkerAggregateStore::open_read_only(
aggregate,
ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(worker_ref),
)
.and_then(|store| store.read_read_only())
.map_err(|error| format!("Session metadata unavailable: {error}"))?
.ok_or("Session metadata missing")?;
if metadata.workspace_id.as_deref() != Some(workspace_id) {
return Err("Session Workspace does not match the authorized source".into());
}
let active = metadata.active.ok_or("Session active pointer is missing")?;
if active.session_id.to_string() != expected_session_id {
return Err("Session identity does not match the authorized current Session".into());
}
let reader = SessionPublicMaterialReader::open_worker_with_retention(
&aggregate.join("session"),
active
.segment_id
.ok_or("Session active Segment is missing")?,
PublicMaterialLimits::default(),
crate::session_retention::runtime_session_retention(
aggregate
.parent()
.and_then(Path::parent)
.ok_or("Worker aggregate root is unavailable")?,
),
)?;
if reader.session_id() != active.session_id {
return Err("Session storage identity does not match its active pointer".into());
}
Ok(reader)
}
#[derive(Clone)]
pub struct ProfileRuntimeWorkerFactory {
observation_hub: Arc<RuntimeWorkerObservationHub>,
@@ -536,6 +596,34 @@ impl ProfileRuntimeWorkerFactory {
})
}
/// Open only the saved current Session of this Workspace's Worker. Paths and
/// restoration state remain inside the Runtime adapter.
pub fn session_public_material_reader(
&self,
workspace_id: &str,
worker_ref: &WorkerRef,
expected_session_id: &str,
) -> Result<
session_store::public_material::SessionPublicMaterialReader,
SessionMaterialSourceError,
> {
open_session_public_material_reader(
&self.worker_aggregate_dir(worker_ref)?,
workspace_id,
worker_ref,
expected_session_id,
)
}
fn session_retention(&self) -> Result<session_store::retention::SessionRetentionStore, String> {
let root = self
.worker_aggregate_root
.as_ref()
.and_then(|root| root.parent())
.ok_or("Runtime Worker aggregate root is not configured")?;
Ok(crate::session_retention::runtime_session_retention(root))
}
fn worker_run_dir(&self, worker_ref: &WorkerRef) -> Result<PathBuf, String> {
Ok(self
.worker_aggregate_dir(worker_ref)?
@@ -1097,9 +1185,10 @@ fn validate_backend_job_profile(
if let Some(binding) = &request.backend_job {
let policy = &manifest.feature.subjektiv.profile;
if policy.consolidation_tools != binding.subjektiv_consolidation
|| (binding.subjektiv_consolidation && policy.extraction.enabled)
|| ((binding.subjektiv_consolidation || binding.subjektiv_journal)
&& policy.extraction.enabled)
{
return Err("Backend Job consolidation_tools must match its trusted consolidation grant and extraction must be disabled".into());
return Err("Backend Job tools must match their independent trusted grants and extraction must be disabled".into());
}
}
Ok(())
@@ -1392,12 +1481,14 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?;
let session_dir = worker_aggregate_dir.join("session");
let session_store = WorkerSessionStore::new(&session_dir).map_err(|err| {
format!(
"failed to initialize canonical Worker Session store at {}: {err}",
session_dir.display()
)
})?;
let session_store =
WorkerSessionStore::new_with_retention(&session_dir, self.session_retention()?)
.map_err(|err| {
format!(
"failed to initialize canonical Worker Session store at {}: {err}",
session_dir.display()
)
})?;
let worker_metadata_store =
WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()).map_err(
|err| {
@@ -1545,8 +1636,14 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
if let Some(active) = metadata.active.as_ref()
&& let Some(segment_id) = active.segment_id
{
let session_store = WorkerSessionStore::new(worker_aggregate_dir.join("session"))
.map_err(|error| format!("failed to read Worker Session store: {error}"))?;
// Preflight must not create lock files or migrate retained sources.
// Actual restore reacquires the live guard before any publication.
self.session_retention()?
.ensure_live(active.session_id)
.map_err(|error| format!("Worker Session source is unavailable: {error}"))?;
let session_store =
WorkerSessionStore::open_read_only(worker_aggregate_dir.join("session"))
.map_err(|error| format!("failed to read Worker Session store: {error}"))?;
if !session_store
.exists(active.session_id, segment_id)
.map_err(|error| format!("failed to inspect Worker Session segment: {error}"))?
@@ -1635,12 +1732,14 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?;
let session_dir = worker_aggregate_dir.join("session");
let session_store = WorkerSessionStore::new(&session_dir).map_err(|err| {
format!(
"failed to initialize canonical Worker Session store at {}: {err}",
session_dir.display()
)
})?;
let session_store =
WorkerSessionStore::new_with_retention(&session_dir, self.session_retention()?)
.map_err(|err| {
format!(
"failed to initialize canonical Worker Session store at {}: {err}",
session_dir.display()
)
})?;
let worker_metadata_store =
WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()).map_err(
|err| {
@@ -7270,18 +7369,19 @@ mod tests {
)
.unwrap();
let mut request = create_request("policy");
for granted in [false, true] {
for (granted, journal) in [(false, false), (true, false), (false, true)] {
request.backend_job = Some(crate::catalog::BackendJobExecutionBinding {
job_id: "job-1".into(),
attempt_id: "attempt-1".into(),
input_digest: None,
subjektiv_consolidation: granted,
subjektiv_journal: journal,
});
for tools in [false, true] {
for extraction in [false, true] {
manifest.feature.subjektiv.profile.consolidation_tools = tools;
manifest.feature.subjektiv.profile.extraction.enabled = extraction;
let expected = tools == granted && (!granted || !extraction);
let expected = tools == granted && (!(granted || journal) || !extraction);
let before = serde_json::to_value(&manifest).unwrap();
assert_eq!(
validate_backend_job_profile(&manifest, &request).is_ok(),