Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: fence Subject handoff on recorded Journal and seed new bodies once

Browse Source
This commit is contained in:
Hare
2026-10-16 20:51:08 +09:00
parent be1c030350
commit 04a4e59100
59 changed files with 7167 additions and 632 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/worker-runtime/Cargo.toml
+1
View File
@@ -23,6 +23,7 @@ http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
[dependencies]
agen.workspace = true
async-trait.workspace = true
base64.workspace = true
axum = { workspace = true, optional = true }
crates/worker-runtime/src/execution.rs
+15
View File
@@ -567,6 +567,14 @@ impl fmt::Debug for WorkerProtocolTransport {
pub trait WorkerExecutionBackend: Send + Sync + 'static {
fn backend_id(&self) -> &str;
/// Must reserve the actual Controller, not infer Idle from a catalog snapshot.
fn subject_release(
&self,
_request: &runtime_api::SubjectReleaseRequest,
) -> Result<runtime_api::SubjectReleaseResponse, String> {
Err("execution backend cannot reserve the actual Subject Controller".into())
}
fn fetch_workspace_config(
&self,
_request: WorkspaceConfigFetchRequest,
@@ -831,6 +839,13 @@ impl WorkerExecutionBackendRef {
self.backend.fetch_workspace_config(request)
}
pub(crate) fn subject_release(
&self,
request: &runtime_api::SubjectReleaseRequest,
) -> Result<runtime_api::SubjectReleaseResponse, String> {
self.backend.subject_release(request)
}
pub(crate) fn worker_session(
&self,
request: WorkerSessionObservationRequest,
crates/worker-runtime/src/fs_store.rs
+11
View File
@@ -479,6 +479,10 @@ pub(crate) struct OpenedFsRuntimeStore {
#[derive(Clone, Debug)]
pub(crate) struct PersistedRuntimeState {
pub(crate) subject_releases: Vec<(
runtime_api::SubjectReleaseRequest,
runtime_api::SubjectReleaseState,
)>,
pub(crate) display_name: Option<String>,
pub(crate) status: RuntimeStatus,
pub(crate) next_diagnostic_id: u64,
@@ -1915,6 +1919,11 @@ fn migrate_runtime_store_in_place(
#[derive(Clone, Debug, Serialize, Deserialize)]
struct RuntimeSnapshot {
#[serde(default)]
subject_releases: Vec<(
runtime_api::SubjectReleaseRequest,
runtime_api::SubjectReleaseState,
)>,
schema_version: u32,
display_name: Option<String>,
backend: RuntimeBackendKind,
@@ -1946,6 +1955,7 @@ fn record_worker_load_diagnostic(
impl RuntimeSnapshot {
fn from_persisted(state: &PersistedRuntimeState) -> Self {
Self {
subject_releases: state.subject_releases.clone(),
schema_version: SCHEMA_VERSION,
display_name: state.display_name.clone(),
backend: RuntimeBackendKind::FsStore,
@@ -1983,6 +1993,7 @@ impl RuntimeSnapshot {
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
) -> PersistedRuntimeState {
PersistedRuntimeState {
subject_releases: self.subject_releases,
display_name: self.display_name,
status: self.status,
next_diagnostic_id: self.next_diagnostic_id,
crates/worker-runtime/src/http_server.rs
+4 -1
View File
@@ -2102,7 +2102,10 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
{
return Some("workers:input");
}
if path.ends_with("/stop") || path.ends_with("/cancel") {
if path.ends_with("/stop")
|| path.ends_with("/cancel")
|| (path == "/v1/subject-release" && *method == Method::POST)
{
return Some("workers:stop");
}
if path == runtime_api::RUNTIME_ROUTE_PROTOCOL_WS {
crates/worker-runtime/src/http_server/runtime_management_api.rs
+12
View File
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
}
}
async fn subject_release(
&self,
request: runtime_api::SubjectReleaseRequest,
) -> Result<runtime_api::SubjectReleaseResponse, runtime_api::RuntimeApiError> {
let scope = material_scope(&self.state, &request.workspace_id)?;
self.state
.runtime
.subject_release_scoped(&scope, request)
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)
}
async fn session_material_pin(
&self,
request: runtime_api::SessionMaterialPinRequest,
crates/worker-runtime/src/runtime.rs
+257 -4
View File
@@ -356,6 +356,109 @@ impl Drop for WorkerOperationLease {
}
impl Runtime {
/// Reserves the actual Controller while the Host reconciles final saved Journal material.
/// A Runtime restart loses proof of a reservation, never reconstructs it from Worker status.
pub fn subject_release_scoped(
&self,
scope: &RuntimeWorkspaceScope,
request: runtime_api::SubjectReleaseRequest,
) -> Result<runtime_api::SubjectReleaseResponse, RuntimeError> {
use runtime_api::SubjectReleaseAction as Action;
if request.workspace_id != scope.workspace_id
|| request.binding_id.trim().is_empty()
|| request.operation_id.trim().is_empty()
|| request.binding_id.len() > 512
|| request.operation_id.len() > 512
{
return Err(RuntimeError::InvalidRequest(
"invalid Subject release identity".into(),
));
}
let worker_ref = WorkerRef::new(request.worker_id);
let lease = self.worker_operation_lock(request.worker_id)?;
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
self.ensure_worker_in_workspace(scope, &worker_ref)?;
let backend = {
let state = self.lock()?;
state.ensure_running()?;
let worker = state.worker(&worker_ref)?;
if worker.has_pending_lifecycle_operation() {
return Err(RuntimeError::InvalidRequest(
"Subject lifecycle is unresolved".into(),
));
}
if let Some((saved, _)) = state
.subject_release_operations
.get(&(request.worker_id, request.operation_id.clone()))
{
if saved.operation_id != request.operation_id
|| saved.binding_id != request.binding_id
|| saved.workspace_id != request.workspace_id
{
return Err(RuntimeError::InvalidRequest(
"different Subject release is retained".into(),
));
}
} else if request.action != Action::Reserve {
return Err(RuntimeError::InvalidRequest(
"Subject reservation is absent or lost; retain occupancy".into(),
));
} else {
state.ensure_subject_not_reserved(&worker_ref, false)?;
}
state.execution_backend.clone().ok_or_else(|| {
RuntimeError::InvalidRequest("actual Controller reservation is unavailable".into())
})?
};
let key = (request.worker_id, request.operation_id.clone());
{
let mut state = self.lock()?;
if let Some((_, saved)) = state.subject_release_operations.get(&key) {
if *saved == runtime_api::SubjectReleaseState::Cancelled {
if request.action != Action::Cancel {
return Err(RuntimeError::InvalidRequest(
"cancelled release cannot be rearmed; use a new operation".into(),
));
}
return Ok(runtime_api::SubjectReleaseResponse {
operation_id: request.operation_id,
binding_id: request.binding_id,
state: *saved,
});
}
} else {
// Persist admission exclusion before any Controller ACK, under the Worker op
// lock. Restart retains this fence without manufacturing disconnect proof.
state.subject_release_operations.insert(
key.clone(),
(request.clone(), runtime_api::SubjectReleaseState::Reserved),
);
}
state.persist_runtime_snapshot()?;
}
let response = backend
.subject_release(&request)
.map_err(RuntimeError::InvalidRequest)?;
if response.binding_id != request.binding_id
|| response.operation_id != request.operation_id
{
return Err(RuntimeError::InvalidRequest(
"Subject release acknowledgment mismatch".into(),
));
}
let mut state = self.lock()?;
let before = state
.subject_release_operations
.insert(key.clone(), (request.clone(), response.state));
if let Err(error) = state.persist_runtime_snapshot() {
if let Some(before) = before {
state.subject_release_operations.insert(key, before);
}
return Err(error);
}
Ok(response)
}
fn reserve_workdir_use<'a>(
&self,
ids: impl IntoIterator<Item = &'a str>,
@@ -1961,6 +2064,8 @@ impl Runtime {
let _operation_guard = operation_lock
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
self.lock()?
.ensure_subject_not_reserved(worker_ref, false)?;
if let Some(scope) = scope {
let mut state = self.lock()?;
if !state
@@ -2332,6 +2437,8 @@ impl Runtime {
mode: WorkerRestoreMode,
preparation_authorized: bool,
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
self.lock()?
.ensure_subject_not_reserved(worker_ref, false)?;
let owner = {
let state = self.lock()?;
let worker = state.worker(worker_ref)?;
@@ -2586,6 +2693,8 @@ impl Runtime {
worker_ref: &WorkerRef,
mode: WorkerRestoreMode,
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
self.lock()?
.ensure_subject_not_reserved(worker_ref, false)?;
let fresh_operation_id = {
let state = self.lock()?;
let worker = state.worker(worker_ref)?;
@@ -3007,6 +3116,7 @@ impl Runtime {
fn ensure_worker_operation_settled(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
let state = self.lock()?;
state.ensure_subject_not_reserved(worker_ref, false)?;
let worker = state.worker(worker_ref)?;
if let Some(operation) = worker.pending_lifecycle_operation_name() {
return Err(RuntimeError::WorkerExecutionUnavailable {
@@ -3803,6 +3913,7 @@ impl Runtime {
self.ensure_worker_in_workspace(scope, worker_ref)?;
}
self.lock()?.ensure_subject_not_reserved(worker_ref, true)?;
let (backend, pending_stop, pending_restore, status) = {
let state = self.lock()?;
state.ensure_running()?;
@@ -3980,6 +4091,7 @@ impl Runtime {
let state = self.lock()?;
state.ensure_running()?;
state.ensure_worker_ref(worker_ref)?;
state.ensure_subject_not_reserved(worker_ref, true)?;
let worker = state.worker(worker_ref)?;
if let Some(operation) = worker.pending_lifecycle_operation_name() {
return Err(RuntimeError::InvalidRequest(format!(
@@ -4270,7 +4382,10 @@ impl Runtime {
.workers
.values()
.filter(|worker| {
worker.restore_guard.active_request_id.is_none()
state
.ensure_subject_not_reserved(&worker.worker_ref, false)
.is_ok()
&& worker.restore_guard.active_request_id.is_none()
&& worker.pending_restore.is_none()
&& worker.pending_stop.is_none()
&& worker.execution_bound
@@ -4313,7 +4428,12 @@ impl Runtime {
state
.workers
.values()
.filter(|worker| worker.pending_stop.is_none())
.filter(|worker| {
worker.pending_stop.is_none()
&& state
.ensure_subject_not_reserved(&worker.worker_ref, false)
.is_ok()
})
.filter_map(|worker| {
worker
.pending_restore
@@ -5362,6 +5482,13 @@ struct RuntimeState {
#[cfg(feature = "fs-store")]
next_diagnostic_id: u64,
workers: BTreeMap<WorkerId, WorkerRecord>,
subject_release_operations: BTreeMap<
(WorkerId, String),
(
runtime_api::SubjectReleaseRequest,
runtime_api::SubjectReleaseState,
),
>,
workdir_operations: BTreeMap<String, WorkdirOperation>,
workspace_owners: BTreeMap<String, String>,
config_bundles: BTreeMap<String, ConfigBundle>,
@@ -5392,6 +5519,7 @@ impl RuntimeState {
#[cfg(feature = "fs-store")]
next_diagnostic_id: 1,
workers: BTreeMap::new(),
subject_release_operations: BTreeMap::new(),
workdir_operations: BTreeMap::new(),
workspace_owners: BTreeMap::new(),
config_bundles: BTreeMap::new(),
@@ -5423,6 +5551,7 @@ impl RuntimeState {
#[cfg(feature = "fs-store")]
next_diagnostic_id: 1,
workers: BTreeMap::new(),
subject_release_operations: BTreeMap::new(),
workdir_operations: BTreeMap::new(),
workspace_owners: BTreeMap::new(),
config_bundles: BTreeMap::new(),
@@ -5568,6 +5697,16 @@ impl RuntimeState {
workspace_backend_resource_clients: BTreeMap::new(),
next_diagnostic_id,
workers,
subject_release_operations: persisted
.subject_releases
.into_iter()
.map(|(request, state)| {
(
(request.worker_id, request.operation_id.clone()),
(request, state),
)
})
.collect(),
workdir_operations: BTreeMap::new(),
config_bundles: BTreeMap::new(),
workspace_config_latest: BTreeMap::new(),
@@ -5588,6 +5727,7 @@ impl RuntimeState {
#[cfg(feature = "fs-store")]
fn persisted_state(&self) -> PersistedRuntimeState {
PersistedRuntimeState {
subject_releases: self.subject_release_operations.values().cloned().collect(),
display_name: self.display_name.clone(),
status: self.status,
next_diagnostic_id: self.next_diagnostic_id,
@@ -5731,6 +5871,28 @@ impl RuntimeState {
Ok(())
}
fn ensure_subject_not_reserved(
&self,
worker: &WorkerRef,
allow_disconnected: bool,
) -> Result<(), RuntimeError> {
use runtime_api::SubjectReleaseState as State;
if self
.subject_release_operations
.iter()
.any(|((id, _), (_, state))| {
*id == worker.worker_id
&& (*state == State::Reserved
|| (!allow_disconnected && *state == State::Disconnected))
})
{
return Err(RuntimeError::InvalidRequest(
"Subject Controller is reserved for release or disconnected".into(),
));
}
Ok(())
}
fn ensure_worker_ref(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
if !self.workers.contains_key(&worker_ref.worker_id) {
return Err(RuntimeError::WorkerNotFound {
@@ -6777,8 +6939,7 @@ fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), R
.validate()
.map_err(|message| RuntimeError::InvalidRequest(message.into()))?;
if request.workspace_api.is_none()
|| ((binding.subjektiv_consolidation || binding.subjektiv_journal)
&& !request.subjektiv_attached)
|| (binding.subjektiv_consolidation && !request.subjektiv_attached)
{
return Err(RuntimeError::InvalidRequest("Backend Job capability requires Workspace authority and consolidation requires trusted subject attachment".into()));
}
@@ -8206,6 +8367,21 @@ mod tests {
}
impl WorkerExecutionBackend for TestExecutionBackend {
fn subject_release(
&self,
request: &runtime_api::SubjectReleaseRequest,
) -> Result<runtime_api::SubjectReleaseResponse, String> {
use runtime_api::{SubjectReleaseAction as A, SubjectReleaseState as S};
Ok(runtime_api::SubjectReleaseResponse {
operation_id: request.operation_id.clone(),
binding_id: request.binding_id.clone(),
state: match request.action {
A::Reserve => S::Reserved,
A::Cancel => S::Cancelled,
A::Disconnect => S::Disconnected,
},
})
}
fn cleanup_working_directory(
&self,
_id: &str,
@@ -13106,6 +13282,83 @@ mod tests {
let _ = std::fs::remove_dir_all(root);
}
// Persistence component test; real Controller reservation has separate coverage.
#[cfg(feature = "fs-store")]
#[test]
fn subject_release_fence_survives_reopen_without_automatic_or_explicit_restore() {
let root = tempfile::tempdir().unwrap();
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path())
.with_runtime_id("test-runtime");
let scoped = scope("workspace-a", "server-a");
let runtime = Runtime::with_fs_store_and_execution_backend(
options.clone(),
Arc::new(TestExecutionBackend::default()),
)
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker_scoped(
&scoped,
scoped_task_request("release durable", "workspace-a"),
)
.unwrap();
let mut request = runtime_api::SubjectReleaseRequest {
workspace_id: "workspace-a".into(),
worker_id: worker.worker_id,
binding_id: "writer-1".into(),
operation_id: "release-1".into(),
action: runtime_api::SubjectReleaseAction::Reserve,
};
runtime
.subject_release_scoped(&scoped, request.clone())
.unwrap();
drop(runtime);
let backend = Arc::new(TestExecutionBackend::default());
let reopened =
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
assert!(
reopened
.send_input(&worker.worker_ref, WorkerInput::user("must remain fenced"))
.is_err()
);
assert!(
reopened
.restore_worker(
&worker.worker_ref,
reopened.test_restore_request(&worker.worker_ref)
)
.is_err()
);
request.action = runtime_api::SubjectReleaseAction::Cancel;
assert_eq!(
reopened
.subject_release_scoped(&scoped, request.clone())
.unwrap()
.state,
runtime_api::SubjectReleaseState::Cancelled
);
request.action = runtime_api::SubjectReleaseAction::Reserve;
assert!(
reopened
.subject_release_scoped(&scoped, request.clone())
.is_err(),
"cancelled identity must never rearm"
);
drop(reopened);
let reopened = Runtime::with_fs_store_and_execution_backend(
options,
Arc::new(TestExecutionBackend::default()),
)
.unwrap();
assert!(
reopened
.subject_release_scoped(&scoped, request.clone())
.is_err(),
"terminal cancellation survives restart"
);
}
#[cfg(feature = "fs-store")]
#[test]
fn fs_store_restores_active_worker_executions() {
crates/worker-runtime/src/worker_backend.rs
+340 -15
View File
@@ -492,8 +492,12 @@ pub(crate) fn open_session_public_material_reader(
Ok(reader)
}
type SpawnModelClientFactory =
Arc<dyn Fn() -> Box<dyn agen::llm_client::client::LlmClient> + Send + Sync>;
#[derive(Clone)]
pub struct ProfileRuntimeWorkerFactory {
spawn_model_client_factory: Option<SpawnModelClientFactory>,
observation_hub: Arc<RuntimeWorkerObservationHub>,
profile_base_dir: PathBuf,
worker_aggregate_root: Option<PathBuf>,
@@ -521,6 +525,7 @@ impl ProfileRuntimeWorkerFactory {
pub fn new(profile_base_dir: impl Into<PathBuf>) -> Self {
let profile_base_dir = profile_base_dir.into();
Self {
spawn_model_client_factory: None,
observation_hub: Arc::new(RuntimeWorkerObservationHub::default()),
profile_base_dir,
worker_aggregate_root: None,
@@ -535,6 +540,17 @@ impl ProfileRuntimeWorkerFactory {
}
}
/// Supply the external model boundary for newly spawned Controllers. The factory is
/// process-owned, never persisted into a Profile or restored as execution authority.
/// Restores continue to use their saved model configuration.
pub fn with_spawn_model_client_factory(
mut self,
factory: impl Fn() -> Box<dyn agen::llm_client::client::LlmClient> + Send + Sync + 'static,
) -> Self {
self.spawn_model_client_factory = Some(Arc::new(factory));
self
}
pub fn with_runtime_id(mut self, runtime_id: impl Into<String>) -> Self {
self.runtime_id = Some(runtime_id.into());
self
@@ -1502,7 +1518,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
let run_dir = self.worker_run_dir(&request.worker_ref)?;
let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id);
let mut prepared = WorkerBootstrap::new(
let bootstrap = WorkerBootstrap::new(
manifest,
store,
loader,
@@ -1513,10 +1529,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
bash_output_dir,
},
self.controller_transport,
)
.prepare()
.await
.map_err(|error| match error {
);
let bootstrap = match &self.spawn_model_client_factory {
Some(factory) => bootstrap.with_model_client(factory()),
None => bootstrap,
};
let mut prepared = bootstrap.prepare().await.map_err(|error| match error {
WorkerBootstrapError::Worker(source) => {
format!("failed to create Worker from profile: {source}")
}
@@ -2331,6 +2349,12 @@ impl PendingRuntimeRestore {
}
}
struct SubjectControllerReservation {
request: runtime_api::SubjectReleaseRequest,
guard: Option<worker::SubjectReleaseReservation>,
state: runtime_api::SubjectReleaseState,
}
/// Worker execution backend backed by real worker crate Workers.
pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
backend_id: String,
@@ -2341,6 +2365,7 @@ pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
pending_restores: Mutex<HashMap<WorkerRef, Arc<PendingRuntimeRestore>>>,
worker_locks: Mutex<HashMap<WorkerRef, Arc<Mutex<()>>>>,
spawn_restore_timeout: Duration,
subject_reservations: Mutex<HashMap<(WorkerRef, String), SubjectControllerReservation>>,
shutdown_admission_timeout: Duration,
}
@@ -2363,6 +2388,7 @@ where
pending_restores: Mutex::new(HashMap::new()),
worker_locks: Mutex::new(HashMap::new()),
spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT,
subject_reservations: Mutex::new(HashMap::new()),
shutdown_admission_timeout: Duration::from_secs(5),
})
}
@@ -3343,6 +3369,119 @@ where
&self.backend_id
}
fn subject_release(
&self,
request: &runtime_api::SubjectReleaseRequest,
) -> Result<runtime_api::SubjectReleaseResponse, String> {
use runtime_api::{SubjectReleaseAction as Action, SubjectReleaseState as State};
let worker_ref = WorkerRef::new(request.worker_id);
let lock = self.worker_lock(&worker_ref)?;
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
let mut reservations = self
.subject_reservations
.lock()
.map_err(|_| "Subject reservation registry poisoned")?;
let key = (worker_ref.clone(), request.operation_id.clone());
if let Some(saved) = reservations.get(&key) {
if saved.request.operation_id != request.operation_id
|| saved.request.binding_id != request.binding_id
|| saved.request.workspace_id != request.workspace_id
{
return Err("Worker has another retained Subject release operation".into());
}
} else {
if request.action == Action::Cancel {
// Runtime has a durable admission intent, but no Controller reservation may
// have been acquired (or this is a new process). Under the same Worker op
// lock there is no late local Reserve left to publish. Retain a terminal
// cancellation receipt so an old Reserve can never rearm this identity.
reservations.insert(
key,
SubjectControllerReservation {
request: request.clone(),
guard: None,
state: State::Cancelled,
},
);
return Ok(runtime_api::SubjectReleaseResponse {
operation_id: request.operation_id.clone(),
binding_id: request.binding_id.clone(),
state: State::Cancelled,
});
}
if request.action != Action::Reserve {
return Err(
"Subject reservation is missing; no status-based recovery is allowed".into(),
);
}
if reservations.iter().any(|((worker, _), saved)| {
*worker == worker_ref && saved.state != State::Cancelled
}) {
return Err("Worker has another active Subject reservation".into());
}
if self
.pending_restores
.lock()
.map_err(|_| "restore registry poisoned")?
.contains_key(&worker_ref)
{
return Err("Subject Controller restore is unresolved".into());
}
let workers = self
.workers
.lock()
.map_err(|_| "worker registry poisoned")?;
let execution = workers
.get(&worker_ref)
.ok_or("Subject Controller is unavailable")?;
if execution.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|| execution.shutdown_requested.load(Ordering::Acquire)
|| execution.handle.protocol_is_closed()
{
return Err("Subject Controller is unavailable or Workspace mismatched".into());
}
let handle = execution.handle.clone();
drop(workers);
let guard =
self.run_on_adapter_runtime(async move { handle.reserve_subject_release().await })?;
reservations.insert(
key.clone(),
SubjectControllerReservation {
request: request.clone(),
guard: Some(guard),
state: State::Reserved,
},
);
}
let saved = reservations.get_mut(&key).expect("reservation inserted");
match (request.action, saved.state) {
(Action::Reserve, State::Reserved | State::Disconnected) => {}
(Action::Disconnect, State::Reserved) => {
saved
.guard
.as_mut()
.ok_or("Controller reservation missing")?
.commit_disconnect()?;
saved.state = State::Disconnected;
}
(Action::Disconnect, State::Disconnected) | (Action::Cancel, State::Cancelled) => {}
(Action::Cancel, State::Reserved) => {
saved.guard.take();
saved.state = State::Cancelled;
}
_ => {
return Err(
"Subject release operation already reached a different terminal state".into(),
);
}
}
Ok(runtime_api::SubjectReleaseResponse {
operation_id: request.operation_id.clone(),
binding_id: request.binding_id.clone(),
state: saved.state,
})
}
fn fetch_workspace_config(
&self,
request: WorkspaceConfigFetchRequest,
@@ -4932,7 +5071,7 @@ mod tests {
let aggregate_dir = root.join("workers").join(worker_ref.worker_id.to_string());
let session_store = WorkerSessionStore::new(aggregate_dir.join("session")).unwrap();
let store = WorkerAggregateStore::new(&aggregate_dir, &worker_name).unwrap();
let (saved, request) = saved_subjektiv_policy(root, &worker_ref);
let (saved, mut request) = saved_subjektiv_policy(root, &worker_ref);
let client = MockClient::new(Vec::new());
let mut engine =
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
@@ -4979,11 +5118,7 @@ mod tests {
Some(SAVED_SUBJEKTIV_SYSTEM_PROMPT)
);
} else {
assert!(matches!(
session_store.exists(session_id, segment_id),
Err(session_store::StoreError::Corrupt { message, .. })
if message.contains("no materialized Session")
));
assert!(!session_store.exists(session_id, segment_id).unwrap());
}
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
drop(worker);
@@ -5007,8 +5142,46 @@ mod tests {
metadata.workspace_id.as_deref(),
Some("workspace-saved-subjektiv")
);
// Attribution already committed before restart; the real restore needs
// no network fixture or replacement current Profile.
// Attribution history is already committed. Each new Controller must still
// obtain current publication authority, even for a Confirmed Session.
// This is a transport fixture for restore/stop component tests, not an
// assertion that a real Backend authorized a released Subject body.
#[cfg(feature = "http-server")]
{
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
request.workspace_api.as_mut().unwrap().base_url =
format!("http://{}", listener.local_addr().unwrap());
let app = axum::Router::new().route(
"/api/w/workspace-saved-subjektiv/subjektiv/sessions",
axum::routing::post(
move |axum::Json(input): axum::Json<
server_api::SubjektivRecordSessionRequest,
>| async move {
assert!(
!input.create_if_missing,
"publication must not rewrite attribution"
);
assert_eq!(input.session_id, session_id.to_string());
axum::Json(server_api::SubjektivRecordSessionResponse {
session_id: input.session_id,
subject_id: "saved-subject".into(),
})
},
),
);
let app = app.route(
"/api/w/workspace-saved-subjektiv/subjektiv/journal/startup",
axum::routing::get(move || async move {
assert!(!materialize_head, "normal restore must not refetch startup");
axum::Json(serde_json::json!({
"subject_id": "saved-subject", "journals": [], "omitted": false,
}))
}),
);
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
}
metadata.subjektiv_session_attribution =
Some(session_store::SubjektivSessionAttributionState::Confirmed {
session_id,
@@ -6077,7 +6250,7 @@ mod tests {
);
}
#[tokio::test]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(worker_allocation)]
async fn profile_factory_saved_subjektiv_manifest_survives_restart_and_explicit_restore() {
let root = tempfile::tempdir().unwrap();
@@ -6147,7 +6320,7 @@ mod tests {
assert!(!root.path().join("nonexistent-current-profiles").exists());
}
#[tokio::test]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(worker_allocation)]
async fn profile_factory_pending_subjektiv_restore_requires_operation_owned_launch_material() {
let root = tempfile::tempdir().unwrap();
@@ -9681,6 +9854,158 @@ mod tests {
assert!(error.to_string().contains("WorkerNotify"), "{error}");
}
#[test]
fn subject_release_reserves_real_controller_and_excludes_runtime_lifecycle() {
use runtime_api::{SubjectReleaseAction as Action, SubjectReleaseState as State};
let runtime_base = tempfile::tempdir().unwrap();
let cwd = tempfile::tempdir().unwrap();
let store = tempfile::tempdir().unwrap();
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(MockFactory {
client: MockClient::new(simple_text_events()),
runtime_base: runtime_base.path().to_path_buf(),
cwd: cwd.path().to_path_buf(),
store_dir: store.path().join("sessions"),
worker_metadata_dir: store.path().join("workers"),
observed_cwds: Arc::new(Mutex::new(Vec::new())),
observed_workspace_clients: Arc::new(Mutex::new(Vec::new())),
})
.unwrap(),
);
let runtime =
EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend.clone())
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let mut create = create_request("subject-release");
create.workspace_api = Some(crate::catalog::WorkspaceApiRef {
workspace_id: "ws-test".into(),
base_url: "http://127.0.0.1:3999".into(),
});
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "ws-test".into(),
language: "English".into(),
});
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "embedded-backend");
let detail = runtime.create_worker_scoped(&scope, create).unwrap();
let mut release = runtime_api::SubjectReleaseRequest {
workspace_id: "ws-test".into(),
worker_id: detail.worker_ref.worker_id,
binding_id: "original-binding".into(),
operation_id: "subject-release-1".into(),
action: Action::Reserve,
};
let reserved = runtime
.subject_release_scoped(&scope, release.clone())
.unwrap();
assert_eq!(reserved.state, State::Reserved);
assert_eq!(
runtime
.subject_release_scoped(&scope, release.clone())
.unwrap(),
reserved
);
let before = backend
.workers
.lock()
.unwrap()
.get(&detail.worker_ref)
.unwrap()
.handle
.committed_entries();
let blocked = runtime
.send_input(
&detail.worker_ref,
WorkerInput::user("must not enter history"),
)
.unwrap_err();
assert!(blocked.to_string().contains("Subject"));
assert!(
runtime
.stop_worker(&detail.worker_ref, None)
.unwrap_err()
.to_string()
.contains("Subject")
);
assert!(
runtime
.delete_worker(&detail.worker_ref)
.unwrap_err()
.to_string()
.contains("Subject")
);
assert!(
runtime
.restore_worker(
&detail.worker_ref,
runtime.test_restore_request(&detail.worker_ref)
)
.is_err()
);
let after = backend
.workers
.lock()
.unwrap()
.get(&detail.worker_ref)
.unwrap()
.handle
.committed_entries();
assert_eq!(
serde_json::to_value(after).unwrap(),
serde_json::to_value(before).unwrap()
);
let mut conflicting = release.clone();
conflicting.binding_id = "later-binding".into();
assert!(runtime.subject_release_scoped(&scope, conflicting).is_err());
release.action = Action::Cancel;
assert_eq!(
runtime
.subject_release_scoped(&scope, release.clone())
.unwrap()
.state,
State::Cancelled
);
release.operation_id = "subject-release-2".into();
release.action = Action::Reserve;
assert_eq!(
runtime
.subject_release_scoped(&scope, release.clone())
.unwrap()
.state,
State::Reserved
);
release.action = Action::Disconnect;
assert_eq!(
runtime
.subject_release_scoped(&scope, release.clone())
.unwrap()
.state,
State::Disconnected
);
assert_eq!(
runtime
.subject_release_scoped(&scope, release.clone())
.unwrap()
.state,
State::Disconnected
);
release.action = Action::Cancel;
assert!(runtime.subject_release_scoped(&scope, release).is_err());
assert!(
runtime
.send_input(&detail.worker_ref, WorkerInput::user("after disconnect"))
.is_err()
);
assert!(
runtime
.restore_worker(
&detail.worker_ref,
runtime.test_restore_request(&detail.worker_ref)
)
.is_err()
);
runtime.stop_worker(&detail.worker_ref, None).unwrap();
}
#[test]
fn stopped_runtime_worker_can_restore_and_accept_input() {
let client = MockClient::new(simple_text_events());