feat: fence Subject handoff on recorded Journal and seed new bodies once

This commit is contained in:
2026-10-16 20:51:08 +09:00
parent be1c030350
commit 04a4e59100
59 changed files with 7167 additions and 632 deletions
+150 -15
View File
@@ -57,6 +57,15 @@ pub struct StandaloneHost {
lease: Option<StandaloneWorkerLease>,
jobs: crate::jobs::StandaloneJobs,
subject: Option<crate::subjektiv::SubjectConnection>,
subject_release: Option<worker::SubjectReleaseReservation>,
}
#[derive(Debug, Clone, PartialEq, Eq, Error)]
pub enum StandaloneSubjectDisconnectError {
#[error("Subject release requires an actual Idle Controller without pending work: {0}")]
Controller(String),
#[error("Subject release remains incomplete; retained binding/lease must be reconciled: {0}")]
Incomplete(String),
}
#[derive(Debug, Clone, PartialEq, Eq, Error)]
@@ -89,6 +98,10 @@ pub enum StandaloneStartupError {
FeatureComposition,
#[error("the in-process Worker controller could not start")]
Controller,
#[error(
"mandatory initial context remains unconfirmed; restore retained Worker {worker_id} to retry this Session"
)]
InitialContextPending { worker_id: WorkerId },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
@@ -171,7 +184,7 @@ impl StandaloneHost {
let jobs = match crate::jobs::StandaloneJobs::open(&jobs_path, launch.cwd.clone()) {
Ok(jobs) => jobs,
Err(_) => {
let _ = store.abandon_allocation(allocation);
cancel_unpublished_start(&store, &mut subject, allocation)?;
return Err(StandaloneStartupError::StateStore);
}
};
@@ -179,8 +192,11 @@ impl StandaloneHost {
if let Some(client) = &model_client {
jobs.bind_model_client(client.clone_boxed());
}
if let Some(subject) = &subject {
subject.host.bind_jobs(jobs.clone())?;
if let Some(connection) = &subject
&& let Err(error) = connection.host.bind_jobs(jobs.clone())
{
cancel_unpublished_start(&store, &mut subject, allocation)?;
return Err(error);
}
// WorkerId is the stable identity. The current Worker store remains
// name-keyed, so keep its derived storage key separate from the
@@ -192,7 +208,7 @@ impl StandaloneHost {
let (backing_store, worker_store) = match backing_store(&store, worker_id) {
Ok(stores) => stores,
Err(error) => {
let _ = store.abandon_allocation(allocation);
cancel_unpublished_start(&store, &mut subject, allocation)?;
return Err(error);
}
};
@@ -220,12 +236,61 @@ impl StandaloneHost {
if let Some(model_client) = model_client {
bootstrap = bootstrap.with_model_client(model_client);
}
let prepared = match bootstrap.prepare().await {
Ok(prepared) => prepared,
Err(error) => {
cancel_unpublished_start(&store, &mut subject, allocation)?;
return Err(classify_startup_error(error));
}
};
// Reserve a recoverable identity before mandatory initial-context persistence.
// A failed/unknown seed must be reopened as this Session, never abandoned and
// replaced by a newly fetched context under an invisible ghost binding.
let pending_record = if subject.is_some() {
match store.commit_created_connected(
&allocation,
manifest.clone(),
storage_key.clone(),
prepared.worker().session_id(),
None,
subject.as_ref().map(|s| s.binding.clone()),
) {
Ok(record) => Some(record),
Err(_) => {
if let Some(subject) = &mut subject {
subject.retain();
}
allocation.into_lease().retain();
return Err(StandaloneStartupError::StateStore);
}
}
} else {
None
};
if let Some(subject) = &mut subject {
subject.arm();
}
let started = match bootstrap.start().await {
let started = match prepared.start().await {
Ok(started) => started,
Err(error) => {
if let Some(record) = &pending_record {
let cleanup = jobs.shutdown().await;
let saved = store.mark_stopped(
record,
record.active_session_id,
record.active_segment_id,
StandaloneShutdownReason::StartupFailed,
);
let closed = cleanup.is_ok()
&& saved.is_ok()
&& subject.as_mut().is_some_and(|s| s.close().is_ok());
if closed {
let _ = allocation.into_lease().release();
} else {
allocation.into_lease().retain();
}
return Err(StandaloneStartupError::InitialContextPending { worker_id });
}
let _ = store.abandon_allocation(allocation);
return Err(classify_startup_error(error));
}
@@ -234,22 +299,35 @@ impl StandaloneHost {
Ok(active) => active,
Err(error) => {
stop_started_worker(started).await;
let _ = store.abandon_allocation(allocation);
if pending_record.is_some() {
allocation.into_lease().retain();
} else {
let _ = store.abandon_allocation(allocation);
}
return Err(error);
}
};
let record = match store.commit_created_connected(
&allocation,
manifest,
storage_key,
active.session_id,
active.segment_id,
subject.as_ref().map(|s| s.binding.clone()),
) {
let committed = if let Some(record) = &pending_record {
store.update_active_pointer(record, active.session_id, active.segment_id)
} else {
store.commit_created_connected(
&allocation,
manifest,
storage_key,
active.session_id,
active.segment_id,
None,
)
};
let record = match committed {
Ok(record) => record,
Err(_) => {
stop_started_worker(started).await;
let _ = store.abandon_allocation(allocation);
if pending_record.is_some() {
allocation.into_lease().retain();
} else {
let _ = store.abandon_allocation(allocation);
}
return Err(StandaloneStartupError::StateStore);
}
};
@@ -427,6 +505,7 @@ impl StandaloneHost {
lease: Some(lease),
jobs,
subject,
subject_release: None,
}
}
@@ -457,6 +536,45 @@ impl StandaloneHost {
Client::new(socket)
}
/// Logical Subject disconnection, not Worker shutdown or completion of its represented work.
/// A successful release keeps the old Controller fenced; a fresh Worker/Session may then
/// acquire the Subject. Failures before durable revocation allow normal recovery and retry.
pub async fn disconnect_subject(&mut self) -> Result<(), StandaloneSubjectDisconnectError> {
let Some(subject) = &mut self.subject else {
return Ok(());
};
let mut reservation = match self.subject_release.take() {
Some(reservation) => reservation,
None => self
.handle
.reserve_subject_release()
.await
.map_err(StandaloneSubjectDisconnectError::Controller)?,
};
let release = match subject.release_for_handoff(&self.record, &mut reservation) {
Ok(()) => match self.jobs.shutdown().await {
Ok(()) => subject.close().map_err(|error| error.to_string()),
Err(error) => Err(error.to_string()),
},
Err(error) => Err(error),
};
match release {
Ok(()) => {
self.subject_release = Some(reservation);
self.subject = None;
Ok(())
}
Err(error) => {
// If revocation committed, keep the actual Controller reserved even if closing
// storage/unlocking failed. Dropping a reservation must not undo revocation.
if subject.writer_released_or_unknown() {
self.subject_release = Some(reservation);
}
Err(StandaloneSubjectDisconnectError::Incomplete(error))
}
}
}
pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
self.shutdown_timeout = shutdown_timeout;
self
@@ -695,6 +813,23 @@ async fn send_protocol_event(peer: &InProcessPeer, event: Event) -> bool {
peer.send(message).await.is_ok()
}
fn cancel_unpublished_start(
store: &StandaloneWorkerStore,
subject: &mut Option<crate::subjektiv::SubjectConnection>,
allocation: crate::store::StandaloneWorkerAllocation,
) -> Result<(), StandaloneStartupError> {
if let Some(connection) = subject {
if connection.abort_unpublished_start().is_err() {
connection.retain();
allocation.into_lease().retain();
return Err(StandaloneStartupError::StateStore);
}
}
store
.abandon_allocation(allocation)
.map_err(classify_store_startup_error)
}
fn backing_store(
store: &StandaloneWorkerStore,
worker_id: WorkerId,
+30
View File
@@ -66,6 +66,15 @@ struct Executions {
jobs: HashMap<String, Execution>,
}
pub(crate) struct QuiescentJobClosure<'a> {
executions: std::sync::MutexGuard<'a, Executions>,
}
impl QuiescentJobClosure<'_> {
pub(crate) fn commit(&mut self) {
self.executions.closed = true;
}
}
/// Cloneable Feature-facing access to a Host-owned service. Cloning this handle
/// neither extends the Host's authority nor permits opening/recovering its store.
#[derive(Clone)]
@@ -79,6 +88,27 @@ pub struct StandaloneJobs {
}
impl StandaloneJobs {
/// Excludes request/start/result acknowledgement while a Subject Host checks its final
/// saved obligations and releases ownership. Accepted results alone are insufficient:
/// every spawned execution must have actually finished successful cleanup.
pub(crate) fn reserve_quiescent_closure(
&self,
) -> Result<QuiescentJobClosure<'_>, JobServiceError> {
let executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
if executions.jobs.values().any(|execution| {
!execution.task.is_finished()
|| !matches!(execution.complete.borrow().as_ref(), Some(Ok(())))
}) {
return Err(JobServiceError::Cleanup(
"delegated Job execution/cleanup remains running or unconfirmed".into(),
));
}
Ok(QuiescentJobClosure { executions })
}
pub(crate) fn open(path: &Path, cwd: PathBuf) -> Result<Self, JobServiceError> {
Ok(Self {
store: Arc::new(Mutex::new(Some(JobStore::open(path)?))),
+4 -1
View File
@@ -8,7 +8,10 @@ pub mod store;
pub mod subjektiv;
mod subjektiv_sessions;
pub use host::{StandaloneHost, StandaloneShutdownError, StandaloneStartupError};
pub use host::{
StandaloneHost, StandaloneShutdownError, StandaloneStartupError,
StandaloneSubjectDisconnectError,
};
pub use launch::{ResolvedStandaloneLaunch, StandaloneLaunchConfig, StandaloneLaunchError};
pub use protocol::WorkerId;
pub use store::{
+2 -8
View File
@@ -396,14 +396,8 @@ impl StandaloneWorkerStore {
LeaseLiveness::Unknown => StandaloneStoreError::LeaseLivenessUnknown(id),
});
}
if let Some(retained) = crate::subjektiv::prepare_worker_deletion(&self.root, &record)? {
// Atomic relocation preserves the original Session, not a permanent duplicate.
// The Subject's durable consumer can read it after logical Worker removal.
fs::rename(self.worker_dir(id), &retained).map_err(StandaloneStoreError::Io)?;
sync_directory(retained.parent().expect("retained Worker parent"))?;
} else {
fs::remove_dir_all(self.worker_dir(id)).map_err(StandaloneStoreError::Io)?;
}
crate::subjektiv::prepare_worker_deletion(&self.root, &record)?;
fs::remove_dir_all(self.worker_dir(id)).map_err(StandaloneStoreError::Io)?;
sync_directory(&self.root)
}
+123 -5
View File
@@ -5,9 +5,11 @@
//! non-private nodes; delegates schema/migration to `subjektiv` and connection policy to `feature-storage`.
//!
//! The Host holds each selected Subject's exclusive kernel lease through body execution and
//! confirmed delegated Job cleanup. Process death releases kernel locks; uncertain in-process
//! cleanup retains leases. Restore validates saved Subject/storage bindings and reacquires the
//! same Subject. Launches without a selected Subject do not implicitly connect this store.
//! confirmed delegated Job cleanup. Kernel lock release never clears durable body ownership:
//! shutdown/process death permits only the same body to restore, not a replacement to start.
//! Explicit disconnection requires actual Controller quiescence and final Journal completion;
//! storage then revokes that writer lifetime before the lease is released. Uncertain cleanup
//! retains leases. Launches without a selected Subject do not implicitly connect this store.
//!
//! The selected Subject's Job store is shared across Workers; the generic Jobs service owns execution
//! and cleanup. Consolidation grants allow only immutable candidate batches and limited operations,
@@ -49,6 +51,8 @@ pub enum SubjectError {
#[error(transparent)]
Domain(#[from] subjektiv::SubjektivError),
#[error(transparent)]
Journal(#[from] subjektiv::journal::JournalError),
#[error(transparent)]
Storage(#[from] feature_storage::FeatureStorageError),
#[error("local Subject storage metadata: {0}")]
Json(#[from] serde_json::Error),
@@ -223,7 +227,49 @@ impl StandaloneSubjects {
scope_id: self.scope_id.clone(),
subject_id: subject_id.to_string(),
};
if self.store.journal_worker_was_released(
subject_id,
&self.scope_id,
&worker_id.to_string(),
)? {
return Err(SubjectError::InvalidScope("this body's Subject binding was released; start a new Worker/Session rather than restore its old authority".into()));
}
// Before writer lifetimes existed, a saved body record was the only durable
// attachment evidence. Never interpret its stopped status as a release on upgrade.
let workers = crate::store::StandaloneWorkerStore::open_existing(&self.state_root)
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
for entry in fs::read_dir(&self.state_root)? {
let entry = entry?;
let Ok(prior_id) = entry
.file_name()
.to_string_lossy()
.parse::<protocol::WorkerId>()
else {
continue;
};
if prior_id == worker_id {
continue;
}
let prior = match workers.load(prior_id) {
Ok(prior) => prior,
Err(crate::store::StandaloneStoreError::WorkerNotFound(_)) => continue,
Err(error) => return Err(SubjectError::InvalidScope(error.to_string())),
};
if prior.subject.as_ref() == Some(&binding)
&& !self.store.journal_worker_was_released(
subject_id,
&self.scope_id,
&prior_id.to_string(),
)?
{
return Err(SubjectError::InvalidScope("saved Subject body has no explicit release; restore it and complete the Idle/Journal handoff, rather than treating inactive storage as unoccupied".into()));
}
}
let writer =
self.store
.claim_journal_writer(subject_id, &self.scope_id, &worker_id.to_string())?;
let host = Arc::new_cyclic(|weak| LocalSubjectHost {
writer,
self_weak: weak.clone(),
journal_dispatch: Mutex::new(false),
language: self.memory_language.clone(),
@@ -252,6 +298,37 @@ pub(crate) struct SubjectConnection {
armed: bool,
}
impl SubjectConnection {
/// Construction failed before any Controller or mandatory startup context was published.
/// This is cancellation of an unpublished reservation, not Idle inferred from a stopped body.
pub(crate) fn abort_unpublished_start(&mut self) -> Result<(), SubjectError> {
self.host
.catalog
.store
.release_journal_writer(&self.host.writer)?;
self.close()
}
pub(crate) fn writer_released_or_unknown(&self) -> bool {
self.host
.catalog
.store
.journal_writer_active(&self.host.writer)
.map(|active| !active)
.unwrap_or(true)
}
pub(crate) fn release_for_handoff(
&mut self,
record: &crate::store::StandaloneWorkerRecord,
reservation: &mut worker::SubjectReleaseReservation,
) -> Result<(), String> {
if let Err(error) = self.host.release_for_handoff(record, reservation) {
self.host.wake_journals();
return Err(error.to_string());
}
reservation.commit_disconnect()
}
pub fn arm(&mut self) {
self.armed = true;
}
@@ -282,6 +359,7 @@ impl Drop for SubjectConnection {
}
pub(crate) struct LocalSubjectHost {
writer: subjektiv::journal::JournalWriterBinding,
self_weak: std::sync::Weak<Self>,
journal_dispatch: Mutex<bool>,
catalog: StandaloneSubjects,
@@ -300,6 +378,16 @@ impl LocalSubjectHost {
"local Subject Host is closed".into(),
));
}
if !self
.catalog
.store
.journal_writer_active(&self.writer)
.map_err(journal::journal_error)?
{
return Err(SubjektivHostError::Unavailable(
"Subject writer binding was released".into(),
));
}
Ok(())
}
fn body(&self, context: &SubjektivHostContext) -> Result<(), SubjektivHostError> {
@@ -339,6 +427,9 @@ impl LocalSubjectHost {
"Session does not belong to this local Subject/Worker scope".into(),
));
}
store
.bind_journal_session(&self.writer, record.clone())
.map_err(journal::journal_error)?;
return Ok(record);
}
if !create {
@@ -355,9 +446,13 @@ impl LocalSubjectHost {
&context.session_id,
)
.map_err(domain_error)?;
store
let record = store
.record_session_attribution(record)
.map_err(domain_error)
.map_err(domain_error)?;
store
.bind_journal_session(&self.writer, record.clone())
.map_err(journal::journal_error)?;
Ok(record)
}
}
@@ -698,6 +793,29 @@ impl LocalSubjectHost {
}
}
impl SubjektivHost for LocalSubjectHost {
fn journal_startup_granted(&self) -> bool {
self.live.load(Ordering::Acquire)
}
fn journal_startup_context(
&self,
context: &SubjektivHostContext,
) -> Result<subjektiv::journal::startup::JournalStartupSnapshot, SubjektivHostError> {
self.body(context)?;
let reader = subjektiv::journal::JournalHostContext::validated_reader(
&self.catalog.store,
&self.binding.subject_id,
)
.map_err(journal::journal_error)?;
let predecessor = self
.catalog
.store
.journal_predecessor_session(&self.writer)
.map_err(journal::journal_error)?;
self.catalog
.store
.journal_startup_snapshot(&reader, predecessor.as_deref())
.map_err(journal::journal_error)
}
fn journal_recording_granted(&self) -> bool {
self.live.load(Ordering::Acquire)
}
+257 -149
View File
@@ -21,6 +21,8 @@ const INPUT_BUDGET: usize = 48 * 1024;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct LocalJournalGrant {
#[serde(default, skip_serializing_if = "Option::is_none")]
writer_binding_id: Option<String>,
binding: StandaloneSubjectBinding,
session_id: String,
journal_id: String,
@@ -59,73 +61,72 @@ impl StandaloneSubjects {
"Session does not belong to the selected Subject/storage scope".into(),
)
})?;
let worker = attribution
.worker_id
.parse()
.map_err(|_| SubjectError::InvalidScope("invalid attributed Worker identity".into()))?;
let mut connection = self.connect(subject_id, worker)?;
let result = (|| -> Result<(), SubjektivHostError> {
let host = &connection.host;
let context = authority(host, session_id)?;
let path = crate::subjektiv_sessions::authorized_sessions_path(
&host.catalog.state_root,
&host.binding.scope_id,
subject_id,
&host.catalog.store,
session_id,
)?;
// Persist this even if source erasure subsequently fails. A retry resumes physical
// deletion with the same permanent fence; it must never generate from leftovers.
host.catalog
.store
.erase_journal_session(&context, request_id)
.map_err(journal_error)?;
let jobs = crate::jobs::StandaloneJobs::open(
&connection.jobs_path,
host.catalog.state_root.clone(),
let leases = self.root.join("leases");
private_dir(&leases, true)?;
let lease = private_file(&leases.join(format!("{subject_id}.lock")), true)?;
if !lease.try_lock_exclusive()? {
return Err(SubjectError::Active(subject_id.into()));
}
let context = JournalHostContext::validated_session(&self.store, attribution)?;
let path = crate::subjektiv_sessions::authorized_sessions_path(
&self.state_root,
&self.scope_id,
subject_id,
&self.store,
session_id,
)
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
// Erasure is not a logical reconnect. Never mint a new writer lifetime in order
// to delete a released bodys text; the permanent fence precedes physical deletion.
self.store.erase_journal_session(&context, request_id)?;
let jobs = crate::jobs::StandaloneJobs::open(
&self.root.join("jobs").join(subject_id).join("jobs.sqlite3"),
self.state_root.clone(),
)
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
for snapshot in jobs
.pending()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
.into_iter()
.chain(
jobs.unknown_granted()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
)
.map_err(service_error)?;
for snapshot in jobs
.pending()
.map_err(service_error)?
.into_iter()
.chain(jobs.unknown_granted().map_err(service_error)?)
{
if serde_json::from_value::<LocalJournalGrant>(
snapshot.domain_grant.unwrap_or_default(),
)
.is_ok_and(|grant| grant.binding == host.binding && grant.session_id == session_id)
{
jobs.cancel(&snapshot.request.job_id)
.map_err(service_error)?;
}
{
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
}) {
jobs.cancel(&snapshot.request.job_id)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
}
for snapshot in jobs.unacknowledged_results().map_err(service_error)? {
if serde_json::from_value::<LocalJournalGrant>(
snapshot.domain_grant.unwrap_or_default(),
)
.is_ok_and(|grant| grant.binding == host.binding && grant.session_id == session_id)
{
jobs.acknowledge(&snapshot.request.job_id)
.map_err(service_error)?;
}
}
for snapshot in jobs
.unacknowledged_results()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
{
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
}) {
jobs.acknowledge(&snapshot.request.job_id)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
}
session_store::FsStore::new_with_retention(
path,
crate::store::session_retention(&host.catalog.state_root),
)
.map_err(source_error)?
.delete_session_with_intent(
session_id
.parse()
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
session_store::retention::SessionDeletionIntent::ErasePermanently,
)
.map_err(source_error)?;
Ok(())
})();
connection.close()?;
result.map_err(|error| SubjectError::InvalidScope(error.to_string()))
}
session_store::FsStore::new_with_retention(
path,
crate::store::session_retention(&self.state_root),
)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
.delete_session_with_intent(
session_id
.parse()
.map_err(|_| SubjectError::InvalidScope("invalid Session".into()))?,
session_store::retention::SessionDeletionIntent::ErasePermanently,
)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
self.store.release_erased_journal_body(&context)?;
self.manager.shutdown()?;
lease.unlock()?;
Ok(())
}
}
@@ -133,17 +134,17 @@ pub(crate) fn retained_workers_root(state_root: &Path) -> PathBuf {
state_root.join("subjektiv").join("retained-workers")
}
/// Called while the generic Worker deletion mutation lease excludes restore/final commits.
/// Returning a destination means logical removal must move, not copy/delete, original material.
/// There is no model call, and failure leaves the original Worker and its sources in place.
/// Ordinary removal cannot turn a stopped/unknown body into an unoccupied Subject. The
/// live Host must first release it under the Controller Idle reservation. The saved
/// release fence also prevents newly appended old-body history from being recorded here.
pub(crate) fn prepare_worker_deletion(
state_root: &Path,
record: &crate::store::StandaloneWorkerRecord,
) -> Result<Option<PathBuf>, crate::store::StandaloneStoreError> {
) -> Result<(), crate::store::StandaloneStoreError> {
let Some(binding) = &record.subject else {
return Ok(None);
return Ok(());
};
let prepare = || -> Result<Option<PathBuf>, SubjektivHostError> {
let prepare = || -> Result<(), SubjektivHostError> {
let catalog = StandaloneSubjects::open_existing(state_root)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
if catalog.scope_id != binding.scope_id {
@@ -151,89 +152,68 @@ pub(crate) fn prepare_worker_deletion(
"deletion Subject scope mismatch".into(),
));
}
let mut connection = catalog
.connect(&binding.subject_id, record.worker_id)
let erased = catalog
.store
.session_attribution(&record.active_session_id.to_string())
.map_err(domain_error)?
.filter(|attr| {
attr.subject_id == binding.subject_id
&& attr.runtime_id == binding.scope_id
&& attr.worker_id == record.worker_id.to_string()
})
.map(|attr| JournalHostContext::validated_session(&catalog.store, attr))
.transpose()
.map_err(journal_error)?;
let destructively_released = match erased.as_ref() {
Some(context) => catalog
.store
.release_erased_journal_body(context)
.map_err(journal_error)?,
None => false,
};
if !destructively_released
&& !catalog
.store
.journal_worker_was_released(
&binding.subject_id,
&binding.scope_id,
&record.worker_id.to_string(),
)
.map_err(journal_error)?
{
return Err(SubjektivHostError::Conflict { code: "subject_occupied".into(), message: "Subject binding is retained; restore this body and explicitly disconnect while Idle after Journal activity, position and checkpoint complete".into() });
}
let workers = crate::store::StandaloneWorkerStore::open_existing(state_root)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
let result = (|| {
let host = &connection.host;
let jobs = crate::jobs::StandaloneJobs::open(
&connection.jobs_path,
record.cwd.canonical_path.clone(),
)
.map_err(service_error)?;
host.bind_jobs(jobs)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
let workers = crate::store::StandaloneWorkerStore::open_existing(state_root)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
let sessions = workers.sessions_dir(record.worker_id);
let mut pending = false;
// Check every original Session, including a save that preceded its notification.
// An unknown old Session attribution/lineage blocks removal instead of being erased.
if !sessions.is_dir() {
return Err(SubjektivHostError::Unavailable(
"original Session storage is missing".into(),
));
let source_store = session_store::FsStore::new_with_retention(
workers.sessions_dir(record.worker_id),
crate::store::session_retention(state_root),
)
.map_err(source_error)?;
for session in session_store::Store::list_sessions(&source_store).map_err(source_error)? {
if destructively_released {
source_store
.delete_session_with_intent(
session,
session_store::retention::SessionDeletionIntent::ErasePermanently,
)
.map_err(source_error)?;
} else {
source_store
.release_session_pin(
session,
session_store::retention::SessionRetentionPin(session),
)
.map_err(source_error)?;
}
let source_store = session_store::FsStore::new_with_retention(
&sessions,
crate::store::session_retention(state_root),
)
.map_err(source_error)?;
for session in
session_store::Store::list_sessions(&source_store).map_err(source_error)?
{
let context = SubjektivHostContext {
worker_id: record.worker_id.to_string(),
session_id: session.to_string(),
};
host.attribution(&context, session == record.active_session_id)?;
host.reconcile_journal_session(&context.session_id)?;
pending |= !host.session_fully_reflected(&context.session_id)?;
if source_store
.retention()
.pins(session)
.map_err(source_error)?
.iter()
.any(|pin| pin.0 != session)
{
return Err(SubjektivHostError::Unavailable(
"another consumer retains this Session; relocation is not authorized"
.into(),
));
}
}
if !pending {
for session in
session_store::Store::list_sessions(&source_store).map_err(source_error)?
{
host.prune_source_session(&source_store, session)?;
}
return Ok(None);
}
// Reservation is durable before sources leave the public Worker namespace. The
// same Subject's next Host consumes it with the retained attribution after reopen.
let _ = host.next_journal_job()?;
let archive = retained_workers_root(state_root);
private_dir(&archive, true)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
let destination = archive.join(record.worker_id.to_string());
if destination.exists() {
return Err(SubjektivHostError::Unavailable(
"retained Worker already exists; reconcile deletion outcome".into(),
));
}
Ok(Some(destination))
})();
connection
.close()
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
result
}
Ok(())
};
prepare()
.map_err(|error| crate::store::StandaloneStoreError::JournalRetention(error.to_string()))
}
fn journal_error(error: JournalError) -> SubjektivHostError {
pub(super) fn journal_error(error: JournalError) -> SubjektivHostError {
match error {
JournalError::Conflict(message) => SubjektivHostError::Conflict {
code: "journal_conflict".into(),
@@ -270,6 +250,89 @@ fn authority(
}
impl LocalSubjectHost {
/// The reservation is obtained by this Host from its actual Controller and kept through
/// final reconciliation and durable revocation. A stopped body cannot supply this proof.
pub(crate) fn release_for_handoff(
&self,
record: &crate::store::StandaloneWorkerRecord,
_reservation: &worker::SubjectReleaseReservation,
) -> Result<(), SubjektivHostError> {
if !self
.catalog
.store
.journal_writer_active(&self.writer)
.map_err(journal_error)?
{
return Ok(()); // reconcile an earlier committed revocation, never renew it
}
self.ensure_live()?;
let dispatch = self
.journal_dispatch
.lock()
.expect("Journal dispatcher poisoned");
if *dispatch {
return Err(SubjektivHostError::Conflict {
code: "journal_running".into(),
message: "Journal dispatcher is still running; binding retained".into(),
});
}
let workers = crate::store::StandaloneWorkerStore::open_existing(&self.catalog.state_root)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?;
let source_store = session_store::FsStore::new_with_retention(
workers.sessions_dir(self.worker_id),
crate::store::session_retention(&self.catalog.state_root),
)
.map_err(source_error)?;
for session in session_store::Store::list_sessions(&source_store).map_err(source_error)? {
let context = SubjektivHostContext {
worker_id: self.worker_id.to_string(),
session_id: session.to_string(),
};
self.attribution(&context, session == record.active_session_id)?;
self.reconcile_journal_session(&context.session_id)?;
if !self.session_fully_reflected(&context.session_id)? {
return Err(SubjektivHostError::Conflict {
code: "journal_unreflected".into(),
message:
"Final saved public history has not been fully recorded; binding retained"
.into(),
});
}
}
let jobs = self.jobs()?;
for result in jobs.unacknowledged_results().map_err(service_error)? {
if self.journal_grant(&result).is_ok() {
self.complete_journal_job(&jobs, &result)?;
}
}
let mut job_closure = jobs.reserve_quiescent_closure().map_err(service_error)?;
for job in jobs
.pending()
.map_err(service_error)?
.into_iter()
.chain(jobs.unknown_granted().map_err(service_error)?)
.chain(jobs.unacknowledged_results().map_err(service_error)?)
{
let _ = job;
return Err(SubjektivHostError::Conflict { code: "journal_outcome_unconfirmed".into(), message: "Subject has queued/running or Unknown/unacknowledged delegated work; binding retained".into() });
}
let released = self
.catalog
.store
.release_journal_writer(&self.writer)
.map_err(journal_error);
if released.is_ok()
|| !self
.catalog
.store
.journal_writer_active(&self.writer)
.unwrap_or(false)
{
job_closure.commit();
}
released
}
fn journal_session_erased(&self, session: &str) -> Result<bool, SubjektivHostError> {
let context = authority(self, session)?;
self.catalog
@@ -425,6 +488,11 @@ impl LocalSubjectHost {
// Old Workers' retained sources are reconciled as well; possession of the
// Subject lease alone does not bypass the original Worker binding checks.
if attribution.runtime_id == self.binding.scope_id
&& self
.catalog
.store
.journal_session_writer_matches(&self.writer, &attribution.session_id)
.map_err(journal_error)?
&& let Err(error) = self.reconcile_journal_session(&attribution.session_id)
{
eprintln!(
@@ -519,6 +587,7 @@ impl LocalSubjectHost {
fn next_journal_job(
&self,
) -> Result<Option<(crate::jobs::StandaloneJobs, String)>, SubjektivHostError> {
self.ensure_live()?;
let jobs = self.jobs()?;
for saved in jobs.unacknowledged_results().map_err(service_error)? {
if self.journal_grant(&saved).is_ok() {
@@ -578,6 +647,14 @@ impl LocalSubjectHost {
.list_journals(&reader, after.as_deref(), MAX_JOURNAL_PAGE)
.map_err(journal_error)?;
for journal in page.items {
if !self
.catalog
.store
.journal_session_writer_matches(&self.writer, &journal.source_session_id)
.map_err(journal_error)?
{
continue;
}
let context = authority(self, &journal.source_session_id)?;
if let Some(active) = self
.catalog
@@ -618,6 +695,7 @@ impl LocalSubjectHost {
.collect::<Vec<_>>();
let origins = self.journal_origins(&ranges)?;
let grant = LocalJournalGrant {
writer_binding_id: Some(self.writer.binding_id.clone()),
binding: self.binding.clone(),
session_id: journal.source_session_id,
journal_id: journal.id.clone(),
@@ -1012,6 +1090,10 @@ impl LocalSubjectHost {
})?)
.map_err(|error| SubjektivHostError::InvalidArgument(error.to_string()))?;
if grant.binding != self.binding
|| grant
.writer_binding_id
.as_ref()
.is_some_and(|id| id != &self.writer.binding_id)
|| snapshot.request.purpose != PURPOSE
|| snapshot.request.serialization_key.as_deref()
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
@@ -1021,6 +1103,24 @@ impl LocalSubjectHost {
"Job input/grant does not match this Journal capability".into(),
));
}
// Pre-binding persisted Jobs lack the new field. They may resume only after the
// Host has bound their ORIGINAL Session to this lifetime. Session binding is immutable;
// missing grant metadata never maps an old Session onto a later body/reconnection.
if !self
.catalog
.store
.journal_session_writer_matches(&self.writer, &grant.session_id)
.map_err(journal_error)?
|| !self
.catalog
.store
.journal_writer_active(&self.writer)
.map_err(journal_error)?
{
return Err(SubjektivHostError::InvalidArgument(
"Journal Job's original Session is not bound to this active body lifetime".into(),
));
}
let context = authority(self, &grant.session_id)?;
let journal = self
.catalog
@@ -1060,10 +1160,18 @@ impl LocalSubjectHost {
"Journal completion binding mismatch".into(),
));
}
self.catalog
.store
.close_journal_generation(&context, &generation.id, &generation.job)
let recovery = JournalJobContext::validated_recovery(
&self.catalog.store,
&context,
&generation.id,
generation.job.clone(),
)
.map_err(journal_error)?;
let saved = subjektiv::journal::service::finish(&self.catalog.store, &recovery)
.map_err(journal_error)?;
if saved != completion {
return Err(SubjektivHostError::Conflict { code: "journal_completion_mismatch".into(), message: "Job result differs from the saved Journal activity, position and checkpoint completion".into() });
}
jobs.acknowledge(&snapshot.request.job_id)
.map_err(service_error)?;
if let Err(error) = self.prune_retained_workers() {
+103 -54
View File
@@ -8,6 +8,7 @@ use session_store::{Store, WorkerMetadataStore};
use std::sync::atomic::AtomicBool;
mod compaction;
mod handoff;
struct Fixture {
_temp: tempfile::TempDir,
@@ -407,10 +408,19 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
JournalAction::ConfirmUnchanged,
state.selected_ranges.clone(),
);
let position = save(
&adapter,
"position",
&nochange.state_token,
JournalAction::UpdatePosition {
current_position: "First prefix checked; last input still pending".into(),
},
vec![],
);
save(
&adapter,
"checkpoint",
&nochange.state_token,
&position.state_token,
JournalAction::Checkpoint {
record_operation_ids: vec!["checked".into()],
},
@@ -675,6 +685,7 @@ impl agen::llm_client::LlmClient for JournalScript {
#[derive(Clone, Default)]
struct DialogAndJournal {
journal: JournalScript,
dialog_requests: Arc<std::sync::Mutex<Vec<agen::llm_client::Request>>>,
dialog_calls: Arc<std::sync::atomic::AtomicUsize>,
}
#[async_trait::async_trait]
@@ -730,6 +741,7 @@ impl agen::llm_client::LlmClient for DialogAndJournal {
};
return Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))));
}
self.dialog_requests.lock().unwrap().push(request.clone());
self.dialog_calls.fetch_add(1, Ordering::SeqCst);
Ok(Box::pin(futures::stream::iter(
[
@@ -1000,6 +1012,60 @@ async fn real_job_tools_save_and_drain_last_input_without_another_turn() {
jobs.shutdown().await.unwrap();
}
#[tokio::test]
async fn persisted_pre_binding_job_recovers_only_under_original_session_lifetime_and_never_after_release()
{
let mut fixture = Fixture::new("pre-upgrade pending Journal");
fixture.host().admit_journal(&fixture.context).unwrap();
let (jobs, modern_id) = fixture.host().next_journal_job().unwrap().unwrap();
let saved = jobs.get(&modern_id).unwrap();
jobs.cancel(&modern_id).unwrap();
let mut request = saved.request;
request.job_id = format!("legacy-journal:{}", uuid::Uuid::now_v7());
let mut legacy = saved.domain_grant.unwrap();
legacy.as_object_mut().unwrap().remove("writer_binding_id");
let pending = jobs.request_granted(request, legacy.clone()).unwrap();
assert_eq!(
jobs.get(&pending.request.job_id).unwrap().domain_grant,
Some(legacy)
);
jobs.start_with_model_client(&pending.request.job_id, JournalScript::default())
.await
.unwrap();
let completed = tokio::time::timeout(
std::time::Duration::from_secs(20),
jobs.wait(&pending.request.job_id),
)
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
fixture
.host()
.complete_journal_job(&jobs, &completed)
.unwrap();
let state = fixture.host().catalog.state_root.clone();
let subject = fixture.connection.binding.subject_id.clone();
fixture
.host()
.catalog
.store
.release_journal_writer(&fixture.host().writer)
.unwrap();
assert!(fixture.host().journal_grant(&completed).is_err());
jobs.shutdown().await.unwrap();
fixture.connection.close().unwrap();
let mut next = StandaloneSubjects::open_existing(&state)
.unwrap()
.connect(&subject, protocol::WorkerId::now_v7())
.unwrap();
assert!(
next.host.journal_grant(&completed).is_err(),
"missing legacy metadata cannot rebind its original Session to the next body"
);
next.close().unwrap();
}
#[tokio::test]
async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_create() {
let mut fixture = Fixture::new("proposed, corrected, partially performed, then interrupted");
@@ -1088,51 +1154,38 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
}
#[tokio::test]
async fn stopped_worker_removal_retains_originals_and_the_existing_durable_request() {
async fn stopped_worker_removal_and_replacement_require_explicit_idle_handoff_even_after_recording()
{
let mut fixture = Fixture::new("Unrecorded stopped Worker");
let state = fixture.host().catalog.state_root.clone();
let binding = fixture.connection.binding.clone();
let (worker, workers) = fixture.stop_worker();
workers.delete(worker).unwrap();
assert!(matches!(
workers.load(worker),
Err(crate::store::StandaloneStoreError::WorkerNotFound(_))
));
let retained = retained_workers_root(&state).join(worker.to_string());
assert!(workers.delete(worker).is_err());
assert!(workers.load(worker).is_ok());
assert!(
retained
.join("sessions")
workers
.sessions_dir(worker)
.join(&fixture.context.session_id)
.is_dir()
);
workers.delete(worker).unwrap();
assert!(
StandaloneSubjects::open_existing(&state)
.unwrap()
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
.is_err()
);
let mut reopened = StandaloneSubjects::open_existing(&state)
.unwrap()
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
.connect(&binding.subject_id, worker)
.unwrap();
let jobs = crate::jobs::StandaloneJobs::open(&reopened.jobs_path, fixture._temp.path().into())
.unwrap();
reopened.host.bind_jobs(jobs.clone()).unwrap();
assert_eq!(
jobs.pending().unwrap().len(),
1,
"repeated deletion must not multiply requests"
);
reopened
.host
.reconcile_journal_session(&fixture.context.session_id)
.unwrap();
let (_, id) = reopened.host.next_journal_job().unwrap().unwrap();
let snapshot = jobs.get(&id).unwrap();
assert!(
grant(
reopened.host.clone(),
&snapshot,
Arc::new(TestFence(AtomicBool::new(true)))
)
.is_ok(),
"retained originals remain readable through the real grant path"
);
assert!(
retained.is_dir(),
"unfinished generation cannot prune original material"
);
jobs.start_with_model_client(&id, JournalScript::default())
.await
.unwrap();
@@ -1146,31 +1199,23 @@ async fn stopped_worker_removal_retains_originals_and_the_existing_durable_reque
.complete_journal_job(&jobs, &completed)
.unwrap();
assert!(
!retained.exists(),
"only reflected original material may be pruned"
);
let context = authority(&reopened.host, &fixture.context.session_id).unwrap();
let journal = reopened
.host
.catalog
.store
.list_journals(&context, None, 1)
.unwrap()
.items
.pop()
.unwrap();
let (_, open) = reopened
.host
.catalog
.store
.journal_resume_state(&context, &journal.id)
.unwrap();
assert!(
open.unwrap().body.contains("Only part performed"),
"Journal remains readable without the removed Worker/Session"
reopened
.host
.session_fully_reflected(&fixture.context.session_id)
.unwrap()
);
jobs.shutdown().await.unwrap();
reopened.close().unwrap();
assert!(
workers.delete(worker).is_err(),
"recording completion is not actual Controller Idle/release"
);
assert!(
StandaloneSubjects::open_existing(&state)
.unwrap()
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
.is_err()
);
}
#[test]
@@ -1415,8 +1460,12 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
.unwrap();
let descriptor = old_reader.descriptors().next().unwrap().clone();
let (worker, workers) = fixture.stop_worker();
workers.delete(worker).unwrap();
assert!(workers.delete(worker).is_err());
// Reproduce a T-734 archive already persisted before durable body-release bindings.
// New ordinary deletion must not create this state, but its erasure contract remains.
let retained = retained_workers_root(&state).join(worker.to_string());
fs::create_dir_all(retained.parent().unwrap()).unwrap();
fs::rename(workers.root().join(worker.to_string()), &retained).unwrap();
let retained_sessions = retained.join("sessions");
let retained_store = session_store::FsStore::new_with_retention(
&retained_sessions,
@@ -183,7 +183,7 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
retained_tokens: 0,
..Default::default()
});
let host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
let mut host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
.await
.unwrap();
let session = host.record().active_session_id;
@@ -422,6 +422,13 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
)
.unwrap();
assert!(!position.text.is_empty());
loop {
match host.disconnect_subject().await {
Ok(()) => break,
Err(crate::StandaloneSubjectDisconnectError::Incomplete(message)) if message.contains("dispatcher") || message.contains("cleanup") => tokio::task::yield_now().await,
Err(error) => panic!("recorded compaction/cancellation lineage must remain releasable while work is unfinished: {error}"),
}
}
host.shutdown().await.unwrap();
})
.await
@@ -0,0 +1,228 @@
//! Real standalone Subject/Controller/Journal Job/initial input connection. Only external
//! model streams are scripted. The scenario checks persisted data and ordering, not whether
//! a real model understands the correction or can autonomously continue the work.
use super::*;
use crate::{StandaloneHost, StandaloneLaunchConfig, StandaloneSubjectDisconnectError};
use agen::llm_client::types::Item;
fn launch(root: &Path, state: &Path, subject: &str, name: &str) -> crate::ResolvedStandaloneLaunch {
StandaloneLaunchConfig::new(
root,
state,
manifest::ProfileSelector::source_named(
manifest::ProfileRegistrySource::Builtin,
"standalone-subjektiv",
),
name,
)
.with_subject(subject.into())
.resolve()
.unwrap()
}
#[tokio::test]
async fn recorded_idle_body_handoff_preserves_open_after_source_deletion_and_seeds_next_body_once()
{
tokio::time::timeout(std::time::Duration::from_secs(60), async {
let temp = tempfile::tempdir().unwrap();
let state = temp.path().join("state");
let catalog = StandaloneSubjects::open(&state).unwrap();
let subject = catalog.create("continuity", Some("BEHAVIOR_ORIGINAL_UNCHANGED")).unwrap();
let model = DialogAndJournal { journal: JournalScript { gate_first: true, ..Default::default() }, ..Default::default() };
let mut first = StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "first"), model.clone()).await.unwrap();
let old_worker = first.worker_id();
let old_session = first.record().active_session_id;
let old_writer = catalog.store.claim_journal_writer(&subject.id, &catalog.scope_id, &old_worker.to_string()).unwrap();
submit_dialog(&first, "A was proposed, user corrected its constraint; B selected because A is unusable. One step performed; tests not performed; external outcome Unknown. Work is unfinished.").await;
model.journal.entered.notified().await;
assert!(matches!(first.disconnect_subject().await, Err(StandaloneSubjectDisconnectError::Incomplete(_))));
assert!(catalog.store.journal_writer_active(&old_writer).unwrap());
assert!(StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "premature"), DialogAndJournal::default()).await.is_err());
let context = JournalHostContext::validated_session(&catalog.store, catalog.store.session_attribution(&old_session.to_string()).unwrap().unwrap()).unwrap();
let id = catalog.store.journal_for_session(&context).unwrap().unwrap();
let generation = catalog.store.active_journal_generation(&context, &id).unwrap().unwrap();
model.journal.release.notify_one();
first.jobs().wait(&generation.job.job_id).await.unwrap();
// Observe the authoritative gate after cleanup; yielding is cooperative scheduling,
// never elapsed time used as proof of recording completion.
loop {
match first.disconnect_subject().await {
Ok(()) => break,
Err(StandaloneSubjectDisconnectError::Incomplete(message)) if message.contains("dispatcher") || message.contains("cleanup") => tokio::task::yield_now().await,
Err(error) => panic!("unexpected handoff blocker: {error}"),
}
}
assert!(!catalog.store.journal_writer_active(&old_writer).unwrap());
assert!(catalog.store.accept_journal_sources(&context, "late old activity", &[generation.selected_ranges[0].clone()]).is_err());
first.shutdown().await.unwrap();
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
workers.delete(old_worker).unwrap();
assert!(!workers.sessions_dir(old_worker).exists());
let reader = JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap();
let historical = catalog.store.journal_startup_snapshot(&reader, Some(&old_session.to_string())).unwrap();
assert_eq!(historical.journals[0].entries[0].state, JournalEntryState::Open);
assert!(historical.journals[0].entries[0].ongoing.contains("unknown"));
assert!(catalog.store.list_memories(&subject.id).unwrap().is_empty());
let second_model = DialogAndJournal::default();
let second = StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "second"), second_model.clone()).await.unwrap();
submit_dialog(&second, "Continue B; describe what is still unperformed and unconfirmed.").await;
let requests = second_model.dialog_requests.lock().unwrap();
let initial = &requests[0];
let context_index = initial.items.iter().position(|item| item.as_text().is_some_and(|text| text.contains("Proposed A; user corrected it to B"))).expect("saved Journal visible to first model request");
let user_index = initial.items.iter().position(|item| matches!(item, Item::Message { role: agen::llm_client::types::Role::User, .. })).unwrap();
assert!(context_index < user_index);
assert_eq!(initial.items.iter().filter(|item| item.as_text().is_some_and(|text| text.contains("Proposed A; user corrected it to B"))).count(), 1);
drop(requests);
let session = second.record().active_session_id;
let sessions = session_store::FsStore::new_with_retention(workers.sessions_dir(second.worker_id()), crate::store::session_retention(&state)).unwrap();
let pointer = session_store::FsWorkerStore::new(workers.worker_metadata_dir(second.worker_id())).unwrap().read_by_name(&second.record().storage_key).unwrap().unwrap().active.unwrap();
let entries = sessions.read_all(session, pointer.segment_id.unwrap()).unwrap();
let journal_items = entries.iter().filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, session_store::SystemItem::JournalStartup { .. }))).count();
assert_eq!(journal_items, 1);
assert_eq!(catalog.store.subject(&subject.id).unwrap().unwrap().behavior_md, "BEHAVIOR_ORIGINAL_UNCHANGED");
second.shutdown().await.unwrap();
}).await.expect("bounded complete Subject handoff connection");
}
#[tokio::test]
async fn saved_public_history_without_admission_notification_blocks_release_until_actual_recording()
{
tokio::time::timeout(std::time::Duration::from_secs(60), async {
let temp = tempfile::tempdir().unwrap();
let state = temp.path().join("state");
let catalog = StandaloneSubjects::open(&state).unwrap();
let subject = catalog.create("saved-before-notified", None).unwrap();
let model = DialogAndJournal {
journal: JournalScript {
gate_first: true,
..Default::default()
},
..Default::default()
};
let mut host = StandaloneHost::start_with_model_client(
launch(temp.path(), &state, &subject.id, "gap"),
model.clone(),
)
.await
.unwrap();
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
let record = host.record().clone();
let writer = catalog
.store
.claim_journal_writer(
&subject.id,
&catalog.scope_id,
&host.worker_id().to_string(),
)
.unwrap();
let sessions = session_store::FsStore::new_with_retention(
workers.sessions_dir(host.worker_id()),
crate::store::session_retention(&state),
)
.unwrap();
// The production saved-history boundary, with its admission callback deliberately
// omitted. The real Host release must discover these bytes, not trust Job status.
sessions
.append(
record.active_session_id,
record.active_segment_id.unwrap(),
&session_store::LogEntry::AnnotatedUserInput {
ts: 3,
segments: vec![],
extensions: vec![],
history: vec![message(
"missed-admission",
"A corrected to B, partially performed, outcome Unknown",
)],
},
)
.unwrap();
assert!(host.disconnect_subject().await.is_err());
assert!(catalog.store.journal_writer_active(&writer).unwrap());
model.journal.entered.notified().await;
let context = JournalHostContext::validated_session(
&catalog.store,
catalog
.store
.session_attribution(&record.active_session_id.to_string())
.unwrap()
.unwrap(),
)
.unwrap();
let id = catalog
.store
.journal_for_session(&context)
.unwrap()
.unwrap();
let generation = catalog
.store
.active_journal_generation(&context, &id)
.unwrap()
.unwrap();
model.journal.release.notify_one();
host.jobs().wait(&generation.job.job_id).await.unwrap();
loop {
match host.disconnect_subject().await {
Ok(()) => break,
Err(StandaloneSubjectDisconnectError::Incomplete(message))
if message.contains("dispatcher") || message.contains("cleanup") =>
{
tokio::task::yield_now().await
}
Err(error) => {
panic!("unexpected release blocker after saved-history recording: {error}")
}
}
}
assert!(!catalog.store.journal_writer_active(&writer).unwrap());
let snapshot = catalog
.store
.journal_startup_snapshot(
&JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap(),
None,
)
.unwrap();
assert!(!snapshot.journals[0].current_position.is_empty());
assert_eq!(
snapshot.journals[0].entries[0].state,
JournalEntryState::Open
);
host.shutdown().await.unwrap();
})
.await
.expect("saved-history/notification-gap handoff is bounded");
}
#[tokio::test]
async fn shutdown_and_normal_deletion_cannot_release_subject_occupancy() {
let temp = tempfile::tempdir().unwrap();
let state = temp.path().join("state");
let catalog = StandaloneSubjects::open(&state).unwrap();
let subject = catalog.create("retained-owner", None).unwrap();
let host = StandaloneHost::start_with_model_client(
launch(temp.path(), &state, &subject.id, "retained"),
DialogAndJournal::default(),
)
.await
.unwrap();
let worker = host.worker_id();
host.shutdown().await.unwrap();
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
assert!(workers.delete(worker).is_err());
assert!(
StandaloneHost::start_with_model_client(
launch(temp.path(), &state, &subject.id, "not-an-owner"),
DialogAndJournal::default()
)
.await
.is_err()
);
let mut restored =
StandaloneHost::restore_with_model_client(state, worker, DialogAndJournal::default())
.await
.unwrap();
restored.disconnect_subject().await.unwrap();
restored.shutdown().await.unwrap();
}
+12 -6
View File
@@ -37,9 +37,8 @@ fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
let a = catalog.create("a", None).unwrap();
let b = catalog.create("b", None).unwrap();
let mut first = catalog
.connect(&a.id, protocol::WorkerId::now_v7())
.unwrap();
let first_id = protocol::WorkerId::now_v7();
let mut first = catalog.connect(&a.id, first_id).unwrap();
assert!(matches!(
StandaloneSubjects::open_existing(&temp.path().join("state"))
.unwrap()
@@ -51,11 +50,18 @@ fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
.connect(&b.id, protocol::WorkerId::now_v7())
.unwrap();
first.close().unwrap();
let mut replacement = StandaloneSubjects::open_existing(&temp.path().join("state"))
assert!(
StandaloneSubjects::open_existing(&temp.path().join("state"))
.unwrap()
.connect(&a.id, protocol::WorkerId::now_v7())
.is_err(),
"kernel lease release is not durable body release"
);
let mut resumed = StandaloneSubjects::open_existing(&temp.path().join("state"))
.unwrap()
.connect(&a.id, protocol::WorkerId::now_v7())
.connect(&a.id, first_id)
.unwrap();
replacement.close().unwrap();
resumed.close().unwrap();
second.close().unwrap();
}
+145 -12
View File
@@ -1,5 +1,7 @@
//! T-709: real standalone Hosts and test-owned processes, with scripted models.
//! These tests do not create a Workspace or contact a model provider.
//! Integration of real standalone Subject lifecycle, dialog, Journal and Memory services.
//! Only model streams are scripted; temporary storage and test-owned subprocesses isolate
//! persistence and kernel leases. Run with `cargo test -p standalone --test subjektiv`.
//! No Workspace or live model provider is used; this is not a continuity meaning evaluation.
use std::collections::{BTreeMap, VecDeque};
use std::io::{Read, Write};
@@ -354,17 +356,16 @@ impl LlmClient for ScriptedClient {
request: Request,
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
{
// This suite observes Memory and dialog requests. Journal's independent model
// boundary fails explicitly rather than consuming their script or using live transport;
// the Journal adapter suite exercises its successful sequential tool execution.
// Journal is a separate scripted model boundary. Successful production recording
// is required before these Memory scenarios may hand the Subject to a new body.
if request
.tools
.iter()
.any(|tool| tool.name == "ReadJournalState")
{
return Err(ClientError::Config(
"Journal model is outside this Memory suite".into(),
));
return Ok(Box::pin(stream::iter(
journal_response(&request).into_iter().map(Ok),
)));
}
// Lifecycle-triggered consolidation uses a separate Internal Worker. It
// must not consume the dialog script or silently become a live request.
@@ -423,6 +424,134 @@ impl LlmClient for ScriptedClient {
}
}
// The outer model script is not a continuity meaning evaluation. Persistence, complete
// source delivery and the handoff gate remain the production implementations.
fn journal_response(request: &Request) -> Vec<LlmEvent> {
let result = |id: &str| -> Option<Value> {
request.items.iter().find_map(|item| match item {
Item::ToolResult {
call_id,
content,
is_error,
summary,
..
} if call_id == id => {
assert!(!is_error, "{id}: {summary}; {content:?}");
Some(
content
.as_deref()
.map(|v| serde_json::from_str(v).unwrap())
.unwrap_or(Value::Null),
)
}
_ => None,
})
};
if result("journal-submit").is_some() {
return prose();
}
if result("journal-checkpoint").is_some() {
return tool("journal-submit", "SubmitJobResult", json!({"result":{}}));
}
let Some(state) = result("journal-state") else {
return tool("journal-state", "ReadJournalState", json!({}));
};
let state = &state["value"];
let generation = state["generation_id"].as_str().unwrap();
if let Some(position) = result("journal-position") {
return tool(
"journal-checkpoint",
"CheckpointJournal",
json!({
"request_id":format!("checkpoint:{generation}"), "expected_state":position["value"]["state_token"],
"action":{"action":"checkpoint","record_operation_ids":[format!("record:{generation}")]},
"covered_ranges":state["selected_ranges"]
}),
);
}
if let Some(record) = result("journal-record") {
return tool(
"journal-position",
"UpdateJournalPosition",
json!({
"request_id":format!("position:{generation}"), "expected_state":record["value"]["state_token"],
"action":{"action":"update_position","current_position":"Memory scenario recorded; represented work may remain unfinished"}, "covered_ranges":[]
}),
);
}
for (index, range) in state["selected_ranges"]
.as_array()
.unwrap()
.iter()
.enumerate()
{
let mut offset = range["bytes"]["start"].as_u64().unwrap();
let mut token = Value::Null;
loop {
let id = format!("journal-source-{index}-{offset}");
match result(&id) {
Some(page) => match page["value"]["next_byte"].as_u64() {
Some(next) => {
assert!(next > offset);
offset = next;
token = page["value"]["content_token"].clone();
}
None => break,
},
None => {
return tool(
&id,
"ReadJournalText",
json!({
"target":{"target":"source","pointer":range["source"]["pointer"]},
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
}),
);
}
}
}
}
let (name, action) = if state["current_open"].is_null() {
(
"CreateJournalEntry",
json!({"action":"create","title":"Memory conversation","body":"Conversation recorded by the scripted model fixture","ongoing":"Unfinished work is not declared complete","state":"open","source_end":null,"correction_refs":[]}),
)
} else {
(
"RewriteJournalEntry",
json!({"action":"rewrite","entry_id":state["current_open"]["id"],"title":"Memory conversation","body":"Further conversation recorded by the scripted model fixture","ongoing":"Unfinished work is not declared complete"}),
)
};
tool(
"journal-record",
name,
json!({
"request_id":format!("record:{generation}"), "expected_state":state["current_journal"]["state_token"],
"action":action,"covered_ranges":state["selected_ranges"]
}),
)
}
async fn disconnect_recorded(host: &mut StandaloneHost) {
let mut last_error = None;
let completed = tokio::time::timeout(Duration::from_secs(30), async {
loop {
match host.disconnect_subject().await {
Ok(()) => return,
Err(error) => {
last_error = Some(error.to_string());
tokio::task::yield_now().await;
}
}
}
})
.await;
assert!(
completed.is_ok(),
"actual Idle/Journal completion is required before Memory scenario handoff: {last_error:?}"
);
}
fn prose() -> Vec<LlmEvent> {
vec![
LlmEvent::text_block_start(0),
@@ -762,7 +891,7 @@ async fn explicit_remember_is_committed_staging_and_sessions_follow_subject_acro
tool("query-restored", "SubjektivMemoryQuery", json!({})),
prose(),
]);
let restored =
let mut restored =
StandaloneHost::restore_with_model_client(state.clone(), worker_id, restored_model.clone())
.await
.unwrap();
@@ -773,6 +902,7 @@ async fn explicit_remember_is_committed_staging_and_sessions_follow_subject_acro
assert!(results["recall"].to_string().contains(claim));
assert_eq!(results["query-restored"]["items"], json!([]));
assert!(format!("{:?}", restored_model.requests()[0].items).contains("Remember this lesson"));
disconnect_recorded(&mut restored).await;
restored.shutdown().await.unwrap();
let second_model = ScriptedClient::new(vec![
@@ -992,7 +1122,7 @@ async fn kernel_subject_lease_excludes_other_processes_and_recovers_after_abnorm
let independent = catalog.create("independent", None).unwrap();
drop(catalog);
let ready = temp.path().join("ready");
let parent = StandaloneHost::start_with_model_client(
let mut parent = StandaloneHost::start_with_model_client(
launch(&parent_cwd, &state, &subject.id, "parent-owner"),
ScriptedClient::new(Vec::new()),
)
@@ -1014,6 +1144,7 @@ async fn kernel_subject_lease_excludes_other_processes_and_recovers_after_abnorm
.await
.unwrap();
independent_host.shutdown().await.unwrap();
disconnect_recorded(&mut parent).await;
parent.shutdown().await.unwrap();
let mut owner = OwnedChild(
@@ -1082,7 +1213,7 @@ async fn automatic_generic_consolidation_applies_candidate_publishes_surface_and
drop(catalog);
let claim = "Validate the immutable result before reporting success";
let model = ScriptedClient::consolidating(remember_script(claim), false);
let host = StandaloneHost::start_with_model_client(
let mut host = StandaloneHost::start_with_model_client(
launch(temp.path(), &state, &subject.id, "consolidation-body"),
model.clone(),
)
@@ -1157,6 +1288,7 @@ async fn automatic_generic_consolidation_applies_candidate_publishes_surface_and
.exists(),
"selected Subject owns the shared generic Job ledger"
);
disconnect_recorded(&mut host).await;
host.shutdown().await.unwrap();
let memory = inspect_store(&state, |store| {
assert!(
@@ -1236,7 +1368,7 @@ async fn transport_loss_after_candidate_apply_preserves_memory_and_later_job_onl
drop(catalog);
let claim = "Candidate receipts survive transport failure";
let losing = ScriptedClient::consolidating(remember_script(claim), true);
let host = StandaloneHost::start_with_model_client(
let mut host = StandaloneHost::start_with_model_client(
launch(temp.path(), &state, &subject.id, "losing-body"),
losing.clone(),
)
@@ -1261,6 +1393,7 @@ async fn transport_loss_after_candidate_apply_preserves_memory_and_later_job_onl
failed.result.is_none(),
"partial apply is not an accepted Job result"
);
disconnect_recorded(&mut host).await;
host.shutdown().await.unwrap();
let original = inspect_store(&state, |store| {
assert!(