fix: reconcile Memory Journal links with handoff fencing

This commit is contained in:
2026-10-17 03:52:11 +09:00
62 changed files with 7238 additions and 652 deletions
+153 -5
View File
@@ -78,7 +78,45 @@ impl WorkerHandle {
self.method_tx.same_channel(&other.method_tx)
}
/// Ask the actual Controller to acknowledge quiescence. No catalog status,
/// profile, or model protocol message grants this Host lifecycle capability.
pub async fn reserve_subject_release(
&self,
) -> Result<crate::SubjectReleaseReservation, String> {
if self.protocol_is_closed() {
return Err("Controller is unavailable".into());
}
if self.shared_state.snapshot().state != WorkerState::Idle {
return Err("Controller is not Idle".into());
}
let control = self
.shared_state
.subject_release
.get()
.ok_or("Controller release authority unavailable")?;
tokio::select! {
result = control.request() => result,
_ = self.method_tx.closed() => Err("Controller closed before release acknowledgement".into()),
}
}
pub async fn send(&self, method: Method) -> Result<(), mpsc::error::SendError<Method>> {
// The lease covers enqueue itself. The Controller's exclusive barrier
// cannot overlook an accepted send between its queue check and ack.
let _admission = if release_fenced_method(&method) {
match self
.shared_state
.subject_release
.get()
.map(|control| control.authority.enter())
.transpose()
{
Ok(lease) => lease,
Err(_) => return Err(mpsc::error::SendError(method)),
}
} else {
None
};
// Reject known-busy sends before channel dispatch, including maintenance
// that cannot receive methods until it finishes. The controller checks
// again at admission to cover an Idle -> busy race after this snapshot.
@@ -290,6 +328,49 @@ impl WorkerHandle {
}
}
fn answer_idle_release<St: Store + Clone>(
control: &Arc<crate::subject_release::ReleaseControl>,
shared_state: &WorkerSharedState,
methods: &mpsc::Receiver<Method>,
staged: bool,
pending: &PendingSubmissionHandle<St>,
notifications: &NotifyBuffer,
attention: bool,
) {
control.respond(|| {
// Read ingress and durable pending state only after acquiring exclusivity.
// Actor-local staged/deferred/attention flags cannot change during this call.
if shared_state.snapshot().state != WorkerState::Idle
|| staged
|| attention
|| !methods.is_empty()
|| methods.is_closed()
|| !notifications.is_empty()
|| pending.has_unfinished_admission()
{
return Err("Controller has pending, staged, busy, or unavailable work".into());
}
Ok(())
});
}
fn release_fenced_method(method: &Method) -> bool {
!matches!(
method,
Method::Shutdown { .. }
| Method::ListWorkers
| Method::ListPendingSubmissions
| Method::ListCompletions { .. }
)
}
async fn release_requested(shared_state: &WorkerSharedState) {
match shared_state.subject_release.get() {
Some(control) => control.notified().await,
None => std::future::pending::<()>().await,
}
}
fn command_admission_disposition(
admission: WorkerCommandAdmission,
) -> Result<(), WorkerCommandDisposition> {
@@ -802,6 +883,10 @@ impl WorkerController {
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
worker
.validate_subjektiv_body_publication()
.map_err(std::io::Error::other)?;
// === 1. Initialization (channels / RuntimeDir / worker-immutable
// snapshots / SpawnedWorkerRegistry / alerter attach /
// bash-output scope) ===
@@ -911,13 +996,13 @@ impl WorkerController {
// the actual registered set instead of a hand-maintained mirror.
worker.engine().tool_server_handle().flush_pending();
// Runtime-owned Workers persist their current model-visible head before
// Fresh Workers persist their current model-visible head before
// controller exposure. Feature installation above must happen first so a
// fresh subject Worker captures the current resident surface in its durable
// prompt. Restored Workers defer generic Feature restore notification until
// the next model-visible context materialization; only an installed
// Feature may then request a typed durable history append.
if runtime_managed && worker.needs_initial_session_head_materialization() {
if worker.needs_initial_session_head_materialization() {
worker
.materialize_durable_session_head()
.await
@@ -958,6 +1043,13 @@ impl WorkerController {
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
let session_id = worker.session_id();
let pending_activations = worker.pending_activation_state();
let release_control =
crate::subject_release::ReleaseControl::new(worker.mutation_authority());
shared_state
.subject_release
.set(release_control)
.ok()
.expect("one Controller release authority");
let handle = WorkerHandle {
method_tx,
working_event_tx: working_event_tx.clone(),
@@ -1561,6 +1653,18 @@ where
{
feature_registry.add_module(subjektiv_sessions);
}
if ordinary_subjektiv_features_enabled
&& lifecycle_enabled
&& let Some(host) = subjektiv_host.as_ref()
&& host.host.journal_startup_granted()
{
worker.install_initial_context_source(Arc::new(
crate::feature::builtin::journal::JournalStartupContextSource::new(
host.clone(),
worker.prompts(),
),
));
}
if ordinary_subjektiv_features_enabled
&& lifecycle_enabled
&& let Some(host) = subjektiv_host.as_ref()
@@ -2232,6 +2336,12 @@ async fn controller_loop<C, St>(
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + 'static,
{
let release_control = shared_state
.subject_release
.get()
.expect("Controller release authority installed")
.clone();
let _release_lifetime = crate::subject_release::ControllerLifetime(release_control.clone());
// Hold an optional external attach server alive for the controller lifetime.
// In-process runtimes retain and drive the WorkerHandle directly.
let _socket_server = socket_server;
@@ -2452,6 +2562,13 @@ async fn controller_loop<C, St>(
break 'controller;
}
},
_ = release_control.notified() => {
answer_idle_release(&release_control, &shared_state, &method_rx,
pending.is_some() || !deferred_methods.is_empty(),
&pending_submissions, &notify_buffer,
worker.has_pending_compaction_cleanup() || worker.has_failed_segment_activation());
continue;
}
_ = notify_buffer.notified() => continue,
_ = tokio::time::sleep(remaining) => {
match prepare_pending_run(
@@ -2486,10 +2603,32 @@ async fn controller_loop<C, St>(
break 'controller;
}
},
_ = release_control.notified() => {
answer_idle_release(&release_control, &shared_state, &method_rx,
pending.is_some() || !deferred_methods.is_empty(),
&pending_submissions, &notify_buffer,
worker.has_pending_compaction_cleanup() || worker.has_failed_segment_activation());
continue;
}
_ = notify_buffer.notified() => continue,
}
};
let _method_admission = if release_fenced_method(&method) {
match release_control.authority.enter() {
Ok(lease) => Some(lease),
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::InvalidRequest,
message: error.to_string(),
});
continue;
}
}
} else {
None
};
if shared_state.snapshot().state != WorkerState::Idle
&& reject_idle_only_submission(&method, &working_event_tx)
{
@@ -2885,6 +3024,7 @@ async fn controller_loop<C, St>(
let mut compact = Box::pin(worker.manual_compact_with_cancel(cancel_rx));
loop {
tokio::select! {
_ = release_control.notified() => { release_control.reject_busy(); }
result = &mut compact => break result,
method = method_rx.recv() => {
match method {
@@ -3142,6 +3282,7 @@ async fn controller_loop<C, St>(
}
}
release_control.close();
// Close method admission before any fallible child/session cleanup. Existing
// senders then fail instead of accepting work that this execution can no
// longer process.
@@ -3286,6 +3427,9 @@ where
// Running only after processing the commit fence. This makes the
// Running snapshot contract deterministic even for immediate clients.
biased;
_ = release_requested(shared_state) => {
if let Some(control) = shared_state.subject_release.get() { control.reject_busy(); }
}
committed = async {
input_commit
.as_mut()
@@ -3906,6 +4050,10 @@ fn worker_error_code(e: &WorkerError) -> ErrorCode {
}
}
#[cfg(test)]
#[path = "controller_subject_release_tests.rs"]
mod subject_release_tests;
#[cfg(test)]
mod tests {
use super::*;
@@ -4251,7 +4399,7 @@ mod tests {
}
#[test]
fn runtime_session_head_is_materialized_after_feature_installation_before_exposure() {
fn fresh_session_head_is_materialized_after_feature_installation_before_exposure() {
let production = include_str!("controller.rs")
.split_once("#[cfg(test)]\nmod tests")
.map(|(production, _)| production)
@@ -4270,8 +4418,8 @@ mod tests {
.find("let handle = WorkerHandle")
.expect("controller handle exposure");
let materialize_gate = startup[..materialize]
.rfind("if runtime_managed && worker.needs_initial_session_head_materialization() {")
.expect("fresh runtime-managed materialization gate");
.rfind("if worker.needs_initial_session_head_materialization() {")
.expect("fresh materialization gate");
assert!(feature_install < materialize_gate);
assert!(materialize_gate < materialize);
@@ -0,0 +1,474 @@
//! Production Direct bootstrap, Controller admission, pending persistence and FsStore.
//! Only the provider is replaced by an explicitly released async boundary; no live Host,
//! Journal policy, socket or model continuation semantics are claimed.
use super::*;
use crate::{
PreparedWorker, WorkerBootstrap, WorkerBootstrapLayout, WorkerFilesystemAuthority,
WorkerManifest, WorkerWorkspaceContext,
};
use session_store::{CombinedStore, FsStore, FsWorkerStore};
use tempfile::TempDir;
#[derive(Clone)]
struct HeldProvider {
started: mpsc::UnboundedSender<()>,
finish: Arc<tokio::sync::Notify>,
}
#[async_trait::async_trait]
impl LlmClient for HeldProvider {
fn clone_boxed(&self) -> Box<dyn LlmClient> {
Box::new(self.clone())
}
async fn stream(
&self,
_: agen::llm_client::Request,
) -> Result<
std::pin::Pin<
Box<
dyn futures_util::Stream<
Item = Result<
agen::llm_client::event::Event,
agen::llm_client::ClientError,
>,
> + Send,
>,
>,
agen::llm_client::ClientError,
> {
self.started.send(()).unwrap();
self.finish.notified().await;
Ok(Box::pin(futures_util::stream::iter([Ok(
agen::llm_client::event::Event::Status(agen::llm_client::event::StatusEvent {
status: agen::llm_client::event::ResponseStatus::Completed,
}),
)])))
}
}
async fn prepared(
temp: &TempDir,
client: HeldProvider,
) -> PreparedWorker<Box<dyn LlmClient>, CombinedStore<FsStore, FsWorkerStore>> {
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='release-{}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]", uuid::Uuid::now_v7())).unwrap();
manifest.compaction = Some(manifest::CompactionConfig {
retained_tokens: 0,
..Default::default()
});
WorkerBootstrap::new(
manifest,
CombinedStore::new(
FsStore::new(temp.path().join("sessions")).unwrap(),
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
),
crate::PromptCatalogSource::builtins_only(),
WorkerWorkspaceContext::no_workspace(),
WorkerFilesystemAuthority::None,
WorkerBootstrapLayout::Direct {
runtime_base: temp.path().join("runtime"),
bash_output_dir: temp.path().join("bash"),
},
WorkerControllerTransport::InProcess,
)
.with_model_client(client)
.prepare()
.await
.unwrap()
}
async fn reserve(handle: &WorkerHandle) -> Result<crate::SubjectReleaseReservation, String> {
tokio::time::timeout(Duration::from_secs(5), handle.reserve_subject_release())
.await
.unwrap()
}
async fn shutdown(controller: crate::BootstrappedWorker) {
controller
.handle
.send(Method::Shutdown {
command: WorkerCommandEnvelope::new(1),
})
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
.await
.unwrap()
.unwrap();
controller.controller_task.await.unwrap();
}
fn extension() -> LogEntry {
LogEntry::Extension {
ts: 1,
domain: "release-test".into(),
payload: serde_json::json!({}),
}
}
#[tokio::test]
async fn reservation_fences_ingress_fallback_and_history_drop_recovers_commit_stays_fenced() {
let temp = TempDir::new().unwrap();
let _sandbox =
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
let (started, mut starts) = mpsc::unbounded_channel();
let prepared = prepared(
&temp,
HeldProvider {
started,
finish: Arc::default(),
},
)
.await;
let writer = prepared.worker().log_writer_handle();
let pending = prepared.worker().pending_submission_handle();
let buffer = prepared.worker().notify_buffer_handle();
let capture = prepared.worker().committed_session_capture_handle();
let extensions = prepared.worker().session_extension_handle();
let fallback = durable_parent_notification_target(pending.clone(), buffer.clone());
let controller = prepared.start().await.unwrap();
let before = controller.handle.committed_entries().len();
let expected = capture.capture().unwrap().location();
let guard = reserve(&controller.handle).await.unwrap();
assert!(
controller
.handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"blocked"
))
.await
.is_err()
);
assert!(
controller
.handle
.send(Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message: "blocked".into()
})
.await
.is_err()
);
assert!(
pending
.accept_notification("blocked".into(), "blocked".into())
.is_err()
);
fallback.notify("child-session".into(), "must not be admitted".into());
assert!(buffer.is_empty());
assert!(!pending.has_unfinished_admission());
assert!(writer.append_entry(extension()).is_err());
assert!(
extensions
.append_if_current(&expected, "release-test", serde_json::json!({}))
.is_err()
);
assert_eq!(controller.handle.committed_entries().len(), before);
assert!(starts.try_recv().is_err());
drop(guard);
writer.append_entry(extension()).unwrap();
let mut guard = reserve(&controller.handle).await.unwrap();
guard.commit_disconnect().unwrap();
guard.commit_disconnect().unwrap();
drop(guard);
assert!(writer.append_entry(extension()).is_err());
assert!(
pending
.accept_notification("after-commit".into(), "blocked".into())
.is_err()
);
assert!(
controller
.handle
.send(Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message: "blocked".into()
})
.await
.is_err()
);
assert!(reserve(&controller.handle).await.is_err());
let _ = controller.handle.snapshot_event();
controller
.handle
.send(Method::ListPendingSubmissions)
.await
.unwrap();
let handle = controller.handle.clone();
shutdown(controller).await;
assert!(handle.protocol_is_closed());
assert!(reserve(&handle).await.is_err());
}
#[tokio::test]
async fn accepted_input_cannot_be_overtaken_by_release_and_running_request_is_rejected() {
let temp = TempDir::new().unwrap();
let _sandbox =
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
let (started, mut starts) = mpsc::unbounded_channel();
let finish = Arc::new(tokio::sync::Notify::new());
let controller = prepared(
&temp,
HeldProvider {
started,
finish: finish.clone(),
},
)
.await
.start()
.await
.unwrap();
let mut events = controller.handle.subscribe();
// Enqueue succeeds before submitting the separate internal release request.
// Either the queue or the held provider is observed; neither is quiescence.
controller
.handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"accepted first",
))
.await
.unwrap();
assert!(reserve(&controller.handle).await.is_err());
tokio::time::timeout(Duration::from_secs(5), starts.recv())
.await
.unwrap()
.unwrap();
assert!(reserve(&controller.handle).await.is_err());
finish.notify_one();
tokio::time::timeout(Duration::from_secs(5), async {
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
}).await.unwrap();
let guard = reserve(&controller.handle).await.unwrap();
drop(guard);
shutdown(controller).await;
}
#[tokio::test]
async fn staged_notification_fallback_blocks_release_until_the_actual_turn_finishes() {
let temp = TempDir::new().unwrap();
let _sandbox =
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
let (started, mut starts) = mpsc::unbounded_channel();
let finish = Arc::new(tokio::sync::Notify::new());
let prepared = prepared(
&temp,
HeldProvider {
started,
finish: finish.clone(),
},
)
.await;
let pending = prepared.worker().pending_submission_handle();
let buffer = prepared.worker().notify_buffer_handle();
let fallback = durable_parent_notification_target(pending.clone(), buffer);
let controller = prepared.start().await.unwrap();
let mut events = controller.handle.subscribe();
fallback.notify("child".into(), "durable accepted notification".into());
assert!(reserve(&controller.handle).await.is_err());
tokio::time::timeout(Duration::from_secs(5), starts.recv())
.await
.unwrap()
.unwrap();
finish.notify_one();
tokio::time::timeout(Duration::from_secs(5), async {
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
}).await.unwrap();
let guard = reserve(&controller.handle).await.unwrap();
drop(guard);
shutdown(controller).await;
}
#[derive(Clone)]
struct HeldRewrite {
started: mpsc::UnboundedSender<()>,
finish: Arc<tokio::sync::Notify>,
}
impl crate::feature::FeatureModule for HeldRewrite {
fn descriptor(&self) -> crate::feature::FeatureDescriptor {
crate::feature::FeatureDescriptor::builtin("held-rewrite", "Held rewrite").with_hook(
crate::feature::HookDeclaration::new(
"hold",
crate::feature::FeatureHookPoint::BeforeSessionRewrite,
),
)
}
fn install(
&self,
context: &mut crate::feature::FeatureInstallContext<'_>,
) -> Result<(), crate::feature::FeatureInstallError> {
context.hooks().add_before_session_rewrite(
"hold",
crate::hook::HookExecutionPolicy::fail_closed(),
self.clone(),
)?;
Ok(())
}
}
#[async_trait::async_trait]
impl crate::hook::Hook<crate::hook::BeforeSessionRewrite> for HeldRewrite {
async fn call(
&self,
context: &crate::hook::BeforeSessionRewriteContext,
) -> Result<crate::hook::BeforeSessionRewriteAction, crate::hook::HookError> {
if context.kind == crate::hook::SessionRewriteKind::Compact {
self.started.send(()).unwrap();
self.finish.notified().await;
return Ok(crate::hook::BeforeSessionRewriteAction::Deny(
"stop before compactor boundary".into(),
));
}
Ok(crate::hook::BeforeSessionRewriteAction::Continue)
}
}
#[tokio::test]
async fn compact_maintenance_rejects_release_until_the_real_rewrite_boundary_exits() {
let temp = TempDir::new().unwrap();
let _sandbox =
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
let (started, mut starts) = mpsc::unbounded_channel();
let finish = Arc::new(tokio::sync::Notify::new());
let mut prepared = prepared(
&temp,
HeldProvider {
started,
finish: finish.clone(),
},
)
.await;
let (rewrite_started, mut rewrites) = mpsc::unbounded_channel();
let rewrite_finish = Arc::new(tokio::sync::Notify::new());
let mut registry = FeatureRegistryBuilder::new();
registry.add_module(HeldRewrite {
started: rewrite_started,
finish: rewrite_finish.clone(),
});
assert!(
!prepared
.worker_mut()
.install_features(registry)
.has_errors()
);
let controller = prepared.start().await.unwrap();
let mut events = controller.handle.subscribe();
controller
.handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"history to compact ".repeat(128),
))
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(5), starts.recv())
.await
.unwrap()
.unwrap();
finish.notify_one();
tokio::time::timeout(Duration::from_secs(5), async {
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
}).await.unwrap();
controller
.handle
.send(Method::Compact {
command: WorkerCommandEnvelope::new(1),
})
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(5), rewrites.recv())
.await
.unwrap()
.unwrap();
assert!(reserve(&controller.handle).await.is_err());
assert!(matches!(
controller.handle.shared_state.snapshot().state,
WorkerState::Busy(WorkerBusyState::Maintenance(_))
));
rewrite_finish.notify_one();
tokio::time::timeout(Duration::from_secs(5), async {
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
}).await.unwrap();
let guard = reserve(&controller.handle).await.unwrap();
drop(guard);
controller
.handle
.send(Method::Shutdown {
command: WorkerCommandEnvelope::new(2),
})
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
.await
.unwrap()
.unwrap();
controller.controller_task.await.unwrap();
}
#[tokio::test]
async fn queued_or_actor_staged_work_cannot_pass_the_exclusive_ack_boundary() {
let temp = TempDir::new().unwrap();
let _sandbox =
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
let (started, _) = mpsc::unbounded_channel();
let mut prepared = prepared(
&temp,
HeldProvider {
started,
finish: Arc::default(),
},
)
.await;
prepared
.worker_mut()
.materialize_durable_session_head()
.await
.unwrap();
let pending = prepared.worker().pending_submission_handle();
let buffer = prepared.worker().notify_buffer_handle();
let shared = WorkerSharedState::new(
"queue-boundary".into(),
prepared.worker().segment_id(),
String::new(),
build_greeting(prepared.worker()),
);
let control =
crate::subject_release::ReleaseControl::new(prepared.worker().mutation_authority());
for condition in [
"queued",
"staged",
"attention",
"closed",
"durable-notification",
] {
let (sender, receiver) = mpsc::channel(1);
if condition == "queued" {
sender
.send(Method::submit_text(
protocol::new_submission_request_id(),
"already enqueued",
))
.await
.unwrap();
}
if condition == "durable-notification" {
pending
.accept_notification("already-admitted".into(), "pending".into())
.unwrap();
}
if condition == "closed" {
drop(sender);
}
let request = tokio::spawn({
let control = control.clone();
async move { control.request().await }
});
control.notified().await;
answer_idle_release(
&control,
&shared,
&receiver,
condition == "staged",
&pending,
&buffer,
condition == "attention",
);
assert!(request.await.unwrap().is_err(), "{condition}");
// A failed request must not leave admission fenced for recovery.
assert!(prepared.worker().mutation_authority().enter().is_ok());
}
}
@@ -26,6 +26,59 @@ use subjektiv::journal::{
MAX_JOURNAL_RESPONSE_BYTES,
};
/// Subjektiv owns startup projection; Core only commits its typed initial items.
pub(crate) struct JournalStartupContextSource {
host: SubjektivHostConnection,
prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>,
}
impl JournalStartupContextSource {
pub(crate) fn new(
host: SubjektivHostConnection,
prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>,
) -> Self {
Self { host, prompts }
}
}
#[async_trait]
impl crate::worker::InitialContextSource for JournalStartupContextSource {
async fn load(&self) -> Result<Vec<session_store::SystemItem>, String> {
use sha2::{Digest, Sha256};
let snapshot = self
.host
.journal_startup_context()
.map_err(|e| e.to_string())?;
// Hash the same canonical JSON value retained by SystemItem, so replay
// can verify provenance without depending on domain struct field order.
let snapshot = serde_json::to_value(snapshot).map_err(|e| e.to_string())?;
let snapshot_json = serde_json::to_string(&snapshot).map_err(|e| e.to_string())?;
if snapshot_json.len() > subjektiv::journal::startup::STARTUP_MAX_BYTES {
return Err("Host Journal startup snapshot exceeds independent byte budget".into());
}
let prompts = self.prompts.load();
let body = prompts
.subjektiv_journal_startup(&snapshot_json)
.map_err(|e| e.to_string())?;
if body.len() > subjektiv::journal::startup::STARTUP_MAX_BYTES + 4096 {
return Err("Journal startup prompt exceeds rendered byte budget".into());
}
let projection = prompts.projection();
Ok(vec![session_store::SystemItem::JournalStartup {
body,
snapshot,
snapshot_sha256: Sha256::digest(snapshot_json.as_bytes())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect(),
prompt_provenance: session_store::PromptRenderProvenance {
workspace_id: self.host.host.workspace_id(),
source_digest: projection.source_digest.clone(),
projection_digest: projection.catalog_digest.clone(),
logical_name: "internal.subjektiv_journal_startup".into(),
},
}])
}
}
#[derive(Clone)]
pub struct JournalRecordingFeature {
host: SubjektivHostConnection,
+19
View File
@@ -76,9 +76,16 @@ impl PendingNotify {
pub struct NotifyBuffer {
inner: Arc<Mutex<VecDeque<PendingNotify>>>,
wake: Arc<tokio::sync::Notify>,
admission: Arc<std::sync::OnceLock<Arc<crate::subject_release::MutationAuthority>>>,
}
impl NotifyBuffer {
pub(crate) fn bind_mutation_authority(
&self,
authority: Arc<crate::subject_release::MutationAuthority>,
) {
let _ = self.admission.set(authority);
}
pub fn new() -> Self {
Self::default()
}
@@ -113,6 +120,18 @@ impl NotifyBuffer {
}
fn push_entry(&self, entry: PendingNotify) {
let _admission = match self
.admission
.get()
.map(|authority| authority.enter())
.transpose()
{
Ok(lease) => lease,
Err(error) => {
tracing::warn!(%error, "notification rejected by Subject release fence");
return;
}
};
self.inner
.lock()
.expect("notify buffer poisoned")
+2
View File
@@ -25,6 +25,8 @@ mod session_capture;
mod session_history;
pub mod shared_state;
mod shutdown_after_idle;
mod subject_release;
pub use subject_release::SubjectReleaseReservation;
pub mod skill;
pub mod spawn;
pub mod subjektiv;
+11
View File
@@ -102,6 +102,7 @@ pub enum WorkerPrompt {
ResidentMemorySummarySection,
ResidentMemoryRestoreSection,
SubjektivResidentContext,
SubjektivJournalStartup,
WorkerOrchestrationGuidanceSection,
SubWorkerSpawnToolDescription,
}
@@ -126,6 +127,7 @@ impl WorkerPrompt {
Self::AgentsMdSection => "internal.agents_md_section",
Self::ResidentMemorySummarySection => "internal.resident_memory_summary_section",
Self::ResidentMemoryRestoreSection => "internal.resident_memory_restore_section",
Self::SubjektivJournalStartup => "internal.subjektiv_journal_startup",
Self::SubjektivResidentContext => "internal.subjektiv_resident_context",
Self::WorkerOrchestrationGuidanceSection => {
"internal.worker_orchestration_guidance_section"
@@ -151,6 +153,7 @@ impl WorkerPrompt {
WorkerPrompt::ResidentMemorySummarySection,
WorkerPrompt::ResidentMemoryRestoreSection,
WorkerPrompt::SubjektivResidentContext,
WorkerPrompt::SubjektivJournalStartup,
WorkerPrompt::WorkerOrchestrationGuidanceSection,
WorkerPrompt::SubWorkerSpawnToolDescription,
];
@@ -540,6 +543,14 @@ impl PromptCatalog {
},
)
}
pub fn subjektiv_journal_startup(&self, snapshot_json: &str) -> Result<String, CatalogError> {
self.render(
WorkerPrompt::SubjektivJournalStartup,
minijinja::context! {
snapshot_json => snapshot_json,
},
)
}
pub fn subjektiv_resident_context(
&self,
behavior_md: &str,
+2
View File
@@ -50,6 +50,7 @@ pub struct WorkerSharedState {
fs_view: OnceLock<WorkerFsView>,
feature_invocations: RwLock<crate::feature::FeatureInvocationRegistry>,
flow_transition_enabled: AtomicBool,
pub(crate) subject_release: OnceLock<std::sync::Arc<crate::subject_release::ReleaseControl>>,
}
impl WorkerSharedState {
@@ -69,6 +70,7 @@ impl WorkerSharedState {
fs_view: OnceLock::new(),
feature_invocations: RwLock::new(crate::feature::FeatureInvocationRegistry::default()),
flow_transition_enabled: AtomicBool::new(false),
subject_release: OnceLock::new(),
}
}
+222
View File
@@ -0,0 +1,222 @@
//! Controller-owned quiescence with a structural persistence/admission fence.
//!
//! A reservation is acknowledged only by the actual idle Controller. Shared read
//! leases cover input enqueue, durable pending admission, and history writes; the
//! exclusive lease spans the Host's reconciliation and ownership release. No
//! Journal policy or catalog-status inference belongs here.
use session_store::StoreError;
use std::collections::VecDeque;
use std::sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
};
use tokio::sync::{Notify, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock, oneshot};
#[derive(Default)]
pub(crate) struct MutationAuthority {
barrier: Arc<RwLock<()>>,
disconnected: AtomicBool,
}
impl MutationAuthority {
pub(crate) fn enter(&self) -> Result<OwnedRwLockReadGuard<()>, StoreError> {
let lease = self
.barrier
.clone()
.try_read_owned()
.map_err(|_| denied())?;
if self.disconnected.load(Ordering::Acquire) {
return Err(denied());
}
Ok(lease)
}
}
fn denied() -> StoreError {
StoreError::Io(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"Subject release has fenced Worker admission and history mutation",
))
}
type Reply = oneshot::Sender<Result<SubjectReleaseReservation, String>>;
struct Requests {
alive: bool,
queue: VecDeque<Reply>,
}
pub(crate) struct ReleaseControl {
pub(crate) authority: Arc<MutationAuthority>,
requests: Mutex<Requests>,
changed: Notify,
}
impl ReleaseControl {
pub(crate) fn new(authority: Arc<MutationAuthority>) -> Arc<Self> {
Arc::new(Self {
authority,
requests: Mutex::new(Requests {
alive: true,
queue: VecDeque::new(),
}),
changed: Notify::new(),
})
}
pub(crate) async fn request(self: &Arc<Self>) -> Result<SubjectReleaseReservation, String> {
let (reply, receive) = oneshot::channel();
{
let mut requests = self
.requests
.lock()
.map_err(|_| "Controller release requests poisoned")?;
if !requests.alive {
return Err("Controller is unavailable".into());
}
requests.queue.push_back(reply);
}
self.changed.notify_one();
receive
.await
.map_err(|_| "Controller closed before release acknowledgement".to_string())?
}
pub(crate) async fn notified(&self) {
self.changed.notified().await;
}
pub(crate) fn respond(self: &Arc<Self>, check_quiescent: impl Fn() -> Result<(), String>) {
let requests = {
let mut state = self
.requests
.lock()
.expect("Controller release requests poisoned");
state.queue.drain(..).collect::<Vec<_>>()
};
for reply in requests {
if reply.is_closed() {
continue;
}
let result = self
.authority
.barrier
.clone()
.try_write_owned()
.map_err(|_| "Controller admission or history mutation is active".to_string())
.and_then(|exclusive| {
if self.authority.disconnected.load(Ordering::Acquire) {
return Err("Controller body is disconnected".into());
}
check_quiescent()?;
Ok(SubjectReleaseReservation {
control: self.clone(),
_exclusive: exclusive,
committed: false,
})
});
// If the waiter disappeared, dropping the unsent guard cancels the fence.
let _ = reply.send(result);
}
}
pub(crate) fn reject_busy(self: &Arc<Self>) {
self.respond(|| Err("Controller is not quiescent Idle".into()));
}
pub(crate) fn close(&self) {
let mut state = self
.requests
.lock()
.expect("Controller release requests poisoned");
state.alive = false;
for reply in state.queue.drain(..) {
let _ = reply.send(Err("Controller is unavailable".into()));
}
}
}
pub(crate) struct ControllerLifetime(pub(crate) Arc<ReleaseControl>);
impl Drop for ControllerLifetime {
fn drop(&mut self) {
self.0.close();
}
}
/// Exclusive permission to perform Host-side Subject handoff while the old body
/// cannot accept input or change source history. Dropping an uncommitted guard
/// cancels the reservation. Committing permanently revokes body admission; it
/// does not stop the Controller or disable read-only observation and shutdown.
pub struct SubjectReleaseReservation {
control: Arc<ReleaseControl>,
_exclusive: OwnedRwLockWriteGuard<()>,
committed: bool,
}
impl SubjectReleaseReservation {
pub fn commit_disconnect(&mut self) -> Result<(), String> {
if self.committed {
return Ok(());
}
let state = self
.control
.requests
.lock()
.map_err(|_| "Controller release requests poisoned")?;
if !state.alive {
return Err("Controller stopped before disconnect commit".into());
}
self.control
.authority
.disconnected
.store(true, Ordering::Release);
self.committed = true;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
// Structural lease test: the Controller's grant path is production code;
// the held reader represents an already-entered synchronous persistence call.
#[tokio::test]
async fn in_flight_mutation_rejects_release_without_waiting_and_retry_recovers() {
let authority = Arc::new(MutationAuthority::default());
let control = ReleaseControl::new(authority.clone());
let (entered, entry) = oneshot::channel();
let (finish, finished) = oneshot::channel();
let writer = tokio::spawn({
let authority = authority.clone();
async move {
let _lease = authority.enter().unwrap();
entered.send(()).unwrap();
finished.await.unwrap();
}
});
entry.await.unwrap();
let request = tokio::spawn({
let control = control.clone();
async move { control.request().await }
});
control.notified().await;
control.respond(|| Ok(()));
assert!(request.await.unwrap().is_err());
finish.send(()).unwrap();
writer.await.unwrap();
let request = tokio::spawn({
let control = control.clone();
async move { control.request().await }
});
control.notified().await;
control.respond(|| Ok(()));
let guard = request.await.unwrap().unwrap();
assert!(authority.enter().is_err());
drop(guard);
assert!(authority.enter().is_ok());
}
#[tokio::test]
async fn closed_controller_rejects_unanswered_request_and_guard_disconnect() {
let control = ReleaseControl::new(Arc::default());
let request = tokio::spawn({
let control = control.clone();
async move { control.request().await }
});
control.notified().await;
control.respond(|| Ok(()));
let mut guard = request.await.unwrap().unwrap();
control.close();
assert!(guard.commit_disconnect().is_err());
assert!(control.request().await.is_err());
}
}
+279 -1
View File
@@ -11,6 +11,7 @@ pub use crate::feature::builtin::journal::{
JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature, JournalRecordingFeature,
};
pub use subjektiv::journal::memory_links::{MemoryJournalRequest, MemoryJournalResponse};
use subjektiv::journal::startup::JournalStartupSnapshot;
pub use subjektiv::journal::transport;
use transport::{JournalRecordingReason, JournalRecordingRequest, JournalRecordingResponse};
@@ -67,6 +68,18 @@ impl From<SubjektivHostError> for ToolError {
/// identifier is not a grant. No method grants ambient filesystem access.
pub trait SubjektivHost: Send + Sync + 'static {
fn settings(&self) -> SubjektivHostSettings;
/// Startup reads are a body-only capability, independent of recording Jobs.
fn journal_startup_granted(&self) -> bool {
false
}
fn journal_startup_context(
&self,
_context: &SubjektivHostContext,
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
Err(SubjektivHostError::Unavailable(
"Journal startup not granted".into(),
))
}
/// Recording admission is separate from both Memory and Journal Job operation grants.
fn journal_recording_granted(&self) -> bool {
false
@@ -235,6 +248,25 @@ impl SubjektivHost for BackendSubjektivHost {
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?,
)
}
fn journal_startup_granted(&self) -> bool {
true
}
fn journal_startup_context(
&self,
_context: &SubjektivHostContext,
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
decode(
self.client
.execute_with_timeout(
WorkspaceRequest::get(format!(
"/api/w/{}/subjektiv/journal/startup",
self.workspace_id
)),
Duration::from_secs(5),
)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?,
)
}
fn journal_recording_granted(&self) -> bool {
// This adapter exists only for explicitly enabled, authenticated Host connections.
// The Backend still verifies the main Worker/Subject binding on every request.
@@ -448,6 +480,14 @@ pub struct SubjektivHostConnection {
}
impl SubjektivHostConnection {
pub fn journal_startup_context(&self) -> Result<JournalStartupSnapshot, SubjektivHostError> {
if !self.host.journal_startup_granted() {
return Err(SubjektivHostError::Unavailable(
"Journal startup not granted".into(),
));
}
self.host.journal_startup_context(&self.context)
}
pub fn new(
host: Arc<dyn SubjektivHost>,
context: SubjektivHostContext,
@@ -580,12 +620,33 @@ mod tests {
#[derive(Default)]
struct LocalHost {
startup: Option<JournalStartupSnapshot>,
startup_error: bool,
startup_calls: std::sync::atomic::AtomicUsize,
contexts: Mutex<Vec<SubjektivHostContext>>,
attributed: Mutex<Vec<(SubjektivHostContext, bool)>>,
journal_observer:
Option<Arc<dyn Fn(&SubjektivHostContext, JournalRecordingReason) + Send + Sync>>,
}
impl SubjektivHost for LocalHost {
fn journal_startup_granted(&self) -> bool {
self.startup.is_some() || self.startup_error
}
fn journal_startup_context(
&self,
_context: &SubjektivHostContext,
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
self.startup_calls
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if self.startup_error {
return Err(SubjektivHostError::Unavailable(
"startup read failed".into(),
));
}
self.startup
.clone()
.ok_or_else(|| SubjektivHostError::Unavailable("startup not granted".into()))
}
fn journal_recording_granted(&self) -> bool {
self.journal_observer.is_some()
}
@@ -953,6 +1014,223 @@ enabled = true
);
}
// Production Direct bootstrap, Feature selection/rendering, Core, FsStore,
// and model Request. The authenticated Host boundary and provider are fake;
// real Backend/standalone authority and live continuation are not claimed.
#[tokio::test]
async fn direct_startup_journal_is_durable_before_first_user_and_provider_exactly_once() {
use session_store::{LogEntry, LoggedSessionHistoryOrigin, Store, SystemItem};
use std::sync::atomic::Ordering;
for empty in [false, true] {
let temp = tempfile::tempdir().unwrap();
let _sandbox =
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
let name = format!("startup-{}", uuid::Uuid::now_v7());
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='{name}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]")).unwrap();
manifest.feature.subjektiv.profile.enabled = true;
manifest.feature.subjektiv.profile.extraction.enabled = false;
let snapshot = JournalStartupSnapshot {
subject_id: "subject".into(),
omitted: false,
journals: if empty {
vec![]
} else {
vec![subjektiv::journal::startup::StartupJournal {
journal_id: "journal".into(),
source_session_id: "previous-session".into(),
state_token: "consistent-token".into(),
process_order: 4,
process_byte: 0,
current_position: "B adopted after correcting A; implementation unfinished"
.into(),
entries: vec![],
omitted: true,
}]
},
};
let host = Arc::new(LocalHost {
startup: Some(snapshot.clone()),
..LocalHost::default()
});
let store = CombinedStore::new(
FsStore::new(temp.path().join("sessions")).unwrap(),
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
);
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let prepared = WorkerBootstrap::new(
manifest,
store.clone(),
crate::PromptCatalogSource::builtins_only(),
WorkerWorkspaceContext::local_filesystem(None),
WorkerFilesystemAuthority::None,
WorkerBootstrapLayout::Direct {
runtime_base: temp.path().join("runtime"),
bash_output_dir: temp.path().join("bash"),
},
WorkerControllerTransport::InProcess,
)
.with_model_client(ScriptedClient(tx))
.with_subjektiv_host(host.clone())
.prepare()
.await
.unwrap();
let (session, segment) = (
prepared.worker().session_id(),
prepared.worker().segment_id(),
);
let controller = prepared.start().await.unwrap();
let initial = store.read_all(session, segment).unwrap();
let journal = initial
.iter()
.find_map(|entry| match entry {
LogEntry::AnnotatedSystemItem { entry, .. }
if matches!(entry.item, SystemItem::JournalStartup { .. }) =>
{
Some(entry)
}
_ => None,
})
.unwrap();
assert_eq!(
journal.metadata.origin,
LoggedSessionHistoryOrigin::DerivedSummary
);
let SystemItem::JournalStartup {
body,
snapshot: saved,
snapshot_sha256,
prompt_provenance,
} = &journal.item
else {
unreachable!()
};
assert_eq!(*saved, serde_json::to_value(&snapshot).unwrap());
use sha2::{Digest, Sha256};
assert_eq!(
*snapshot_sha256,
Sha256::digest(serde_json::to_vec(saved).unwrap())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>()
);
assert_eq!(
prompt_provenance.logical_name,
"internal.subjektiv_journal_startup"
);
assert!(
!initial
.iter()
.any(|e| matches!(e, LogEntry::AnnotatedUserInput { .. }))
);
let mut events = controller.handle.subscribe();
for user in ["FIRST_USER_REQUEST", "SECOND_USER_REQUEST"] {
controller
.handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
user,
))
.await
.unwrap();
let request = tokio::time::timeout(Duration::from_secs(5), rx.recv())
.await
.unwrap()
.unwrap();
let index = request
.items
.iter()
.position(|item| item.as_text() == Some(body.as_str()))
.unwrap();
let user_index = request
.items
.iter()
.position(|item| item.as_text() == Some("FIRST_USER_REQUEST"))
.unwrap();
assert!(index < user_index);
assert_eq!(
request
.items
.iter()
.filter(|item| item.as_text() == Some(body.as_str()))
.count(),
1
);
tokio::time::timeout(Duration::from_secs(5), async {
loop {
if matches!(events.recv().await.unwrap(), protocol::Event::RunEnd { .. }) {
break;
}
}
})
.await
.unwrap();
}
assert_eq!(host.startup_calls.load(Ordering::SeqCst), 1);
let logs = store.read_all(session, segment).unwrap();
assert_eq!(logs.iter().filter(|entry| matches!(entry, LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, SystemItem::JournalStartup { .. }))).count(), 1);
controller
.handle
.send(Method::Shutdown {
command: protocol::WorkerCommandEnvelope::new(1),
})
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
.await
.unwrap()
.unwrap();
controller.controller_task.await.unwrap();
}
}
#[tokio::test]
async fn direct_startup_read_error_rejects_exposure_without_initialized_head() {
use session_store::Store;
let temp = tempfile::tempdir().unwrap();
let _sandbox =
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='startup-error-{}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]", uuid::Uuid::now_v7())).unwrap();
manifest.feature.subjektiv.profile.enabled = true;
manifest.feature.subjektiv.profile.extraction.enabled = false;
let store = CombinedStore::new(
FsStore::new(temp.path().join("sessions")).unwrap(),
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
);
let host = Arc::new(LocalHost {
startup_error: true,
..LocalHost::default()
});
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let prepared = WorkerBootstrap::new(
manifest,
store.clone(),
crate::PromptCatalogSource::builtins_only(),
WorkerWorkspaceContext::local_filesystem(None),
WorkerFilesystemAuthority::None,
WorkerBootstrapLayout::Direct {
runtime_base: temp.path().join("runtime"),
bash_output_dir: temp.path().join("bash"),
},
WorkerControllerTransport::InProcess,
)
.with_model_client(ScriptedClient(tx))
.with_subjektiv_host(host.clone())
.prepare()
.await
.unwrap();
let (session, segment) = (
prepared.worker().session_id(),
prepared.worker().segment_id(),
);
assert!(prepared.start().await.is_err());
assert!(!store.exists(session, segment).unwrap());
assert!(rx.try_recv().is_err());
assert_eq!(
host.startup_calls.load(std::sync::atomic::Ordering::SeqCst),
1
);
}
async fn launch_case(enabled: bool, attach: bool) {
let temp = tempfile::tempdir().unwrap();
let _runtime_sandbox =
@@ -1072,7 +1350,7 @@ enabled = true
if enabled && attach {
assert_eq!(
*host.attributed.lock().unwrap(),
vec![(expected.clone(), true)]
vec![(expected.clone(), true), (expected.clone(), false)]
);
assert!(
host.contexts
File diff suppressed because it is too large Load Diff