feat: fence Subject handoff on recorded Journal and seed new bodies once
This commit is contained in:
+150
-15
@@ -57,6 +57,15 @@ pub struct StandaloneHost {
|
||||
lease: Option<StandaloneWorkerLease>,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
subject: Option<crate::subjektiv::SubjectConnection>,
|
||||
subject_release: Option<worker::SubjectReleaseReservation>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum StandaloneSubjectDisconnectError {
|
||||
#[error("Subject release requires an actual Idle Controller without pending work: {0}")]
|
||||
Controller(String),
|
||||
#[error("Subject release remains incomplete; retained binding/lease must be reconciled: {0}")]
|
||||
Incomplete(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
@@ -89,6 +98,10 @@ pub enum StandaloneStartupError {
|
||||
FeatureComposition,
|
||||
#[error("the in-process Worker controller could not start")]
|
||||
Controller,
|
||||
#[error(
|
||||
"mandatory initial context remains unconfirmed; restore retained Worker {worker_id} to retry this Session"
|
||||
)]
|
||||
InitialContextPending { worker_id: WorkerId },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
@@ -171,7 +184,7 @@ impl StandaloneHost {
|
||||
let jobs = match crate::jobs::StandaloneJobs::open(&jobs_path, launch.cwd.clone()) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(_) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
};
|
||||
@@ -179,8 +192,11 @@ impl StandaloneHost {
|
||||
if let Some(client) = &model_client {
|
||||
jobs.bind_model_client(client.clone_boxed());
|
||||
}
|
||||
if let Some(subject) = &subject {
|
||||
subject.host.bind_jobs(jobs.clone())?;
|
||||
if let Some(connection) = &subject
|
||||
&& let Err(error) = connection.host.bind_jobs(jobs.clone())
|
||||
{
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(error);
|
||||
}
|
||||
// WorkerId is the stable identity. The current Worker store remains
|
||||
// name-keyed, so keep its derived storage key separate from the
|
||||
@@ -192,7 +208,7 @@ impl StandaloneHost {
|
||||
let (backing_store, worker_store) = match backing_store(&store, worker_id) {
|
||||
Ok(stores) => stores,
|
||||
Err(error) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
@@ -220,12 +236,61 @@ impl StandaloneHost {
|
||||
if let Some(model_client) = model_client {
|
||||
bootstrap = bootstrap.with_model_client(model_client);
|
||||
}
|
||||
let prepared = match bootstrap.prepare().await {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => {
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(classify_startup_error(error));
|
||||
}
|
||||
};
|
||||
// Reserve a recoverable identity before mandatory initial-context persistence.
|
||||
// A failed/unknown seed must be reopened as this Session, never abandoned and
|
||||
// replaced by a newly fetched context under an invisible ghost binding.
|
||||
let pending_record = if subject.is_some() {
|
||||
match store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest.clone(),
|
||||
storage_key.clone(),
|
||||
prepared.worker().session_id(),
|
||||
None,
|
||||
subject.as_ref().map(|s| s.binding.clone()),
|
||||
) {
|
||||
Ok(record) => Some(record),
|
||||
Err(_) => {
|
||||
if let Some(subject) = &mut subject {
|
||||
subject.retain();
|
||||
}
|
||||
allocation.into_lease().retain();
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(subject) = &mut subject {
|
||||
subject.arm();
|
||||
}
|
||||
let started = match bootstrap.start().await {
|
||||
let started = match prepared.start().await {
|
||||
Ok(started) => started,
|
||||
Err(error) => {
|
||||
if let Some(record) = &pending_record {
|
||||
let cleanup = jobs.shutdown().await;
|
||||
let saved = store.mark_stopped(
|
||||
record,
|
||||
record.active_session_id,
|
||||
record.active_segment_id,
|
||||
StandaloneShutdownReason::StartupFailed,
|
||||
);
|
||||
let closed = cleanup.is_ok()
|
||||
&& saved.is_ok()
|
||||
&& subject.as_mut().is_some_and(|s| s.close().is_ok());
|
||||
if closed {
|
||||
let _ = allocation.into_lease().release();
|
||||
} else {
|
||||
allocation.into_lease().retain();
|
||||
}
|
||||
return Err(StandaloneStartupError::InitialContextPending { worker_id });
|
||||
}
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
return Err(classify_startup_error(error));
|
||||
}
|
||||
@@ -234,22 +299,35 @@ impl StandaloneHost {
|
||||
Ok(active) => active,
|
||||
Err(error) => {
|
||||
stop_started_worker(started).await;
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
if pending_record.is_some() {
|
||||
allocation.into_lease().retain();
|
||||
} else {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let record = match store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest,
|
||||
storage_key,
|
||||
active.session_id,
|
||||
active.segment_id,
|
||||
subject.as_ref().map(|s| s.binding.clone()),
|
||||
) {
|
||||
let committed = if let Some(record) = &pending_record {
|
||||
store.update_active_pointer(record, active.session_id, active.segment_id)
|
||||
} else {
|
||||
store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest,
|
||||
storage_key,
|
||||
active.session_id,
|
||||
active.segment_id,
|
||||
None,
|
||||
)
|
||||
};
|
||||
let record = match committed {
|
||||
Ok(record) => record,
|
||||
Err(_) => {
|
||||
stop_started_worker(started).await;
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
if pending_record.is_some() {
|
||||
allocation.into_lease().retain();
|
||||
} else {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
}
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
};
|
||||
@@ -427,6 +505,7 @@ impl StandaloneHost {
|
||||
lease: Some(lease),
|
||||
jobs,
|
||||
subject,
|
||||
subject_release: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -457,6 +536,45 @@ impl StandaloneHost {
|
||||
Client::new(socket)
|
||||
}
|
||||
|
||||
/// Logical Subject disconnection, not Worker shutdown or completion of its represented work.
|
||||
/// A successful release keeps the old Controller fenced; a fresh Worker/Session may then
|
||||
/// acquire the Subject. Failures before durable revocation allow normal recovery and retry.
|
||||
pub async fn disconnect_subject(&mut self) -> Result<(), StandaloneSubjectDisconnectError> {
|
||||
let Some(subject) = &mut self.subject else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut reservation = match self.subject_release.take() {
|
||||
Some(reservation) => reservation,
|
||||
None => self
|
||||
.handle
|
||||
.reserve_subject_release()
|
||||
.await
|
||||
.map_err(StandaloneSubjectDisconnectError::Controller)?,
|
||||
};
|
||||
let release = match subject.release_for_handoff(&self.record, &mut reservation) {
|
||||
Ok(()) => match self.jobs.shutdown().await {
|
||||
Ok(()) => subject.close().map_err(|error| error.to_string()),
|
||||
Err(error) => Err(error.to_string()),
|
||||
},
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
match release {
|
||||
Ok(()) => {
|
||||
self.subject_release = Some(reservation);
|
||||
self.subject = None;
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => {
|
||||
// If revocation committed, keep the actual Controller reserved even if closing
|
||||
// storage/unlocking failed. Dropping a reservation must not undo revocation.
|
||||
if subject.writer_released_or_unknown() {
|
||||
self.subject_release = Some(reservation);
|
||||
}
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
|
||||
self.shutdown_timeout = shutdown_timeout;
|
||||
self
|
||||
@@ -695,6 +813,23 @@ async fn send_protocol_event(peer: &InProcessPeer, event: Event) -> bool {
|
||||
peer.send(message).await.is_ok()
|
||||
}
|
||||
|
||||
fn cancel_unpublished_start(
|
||||
store: &StandaloneWorkerStore,
|
||||
subject: &mut Option<crate::subjektiv::SubjectConnection>,
|
||||
allocation: crate::store::StandaloneWorkerAllocation,
|
||||
) -> Result<(), StandaloneStartupError> {
|
||||
if let Some(connection) = subject {
|
||||
if connection.abort_unpublished_start().is_err() {
|
||||
connection.retain();
|
||||
allocation.into_lease().retain();
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
}
|
||||
store
|
||||
.abandon_allocation(allocation)
|
||||
.map_err(classify_store_startup_error)
|
||||
}
|
||||
|
||||
fn backing_store(
|
||||
store: &StandaloneWorkerStore,
|
||||
worker_id: WorkerId,
|
||||
|
||||
@@ -66,6 +66,15 @@ struct Executions {
|
||||
jobs: HashMap<String, Execution>,
|
||||
}
|
||||
|
||||
pub(crate) struct QuiescentJobClosure<'a> {
|
||||
executions: std::sync::MutexGuard<'a, Executions>,
|
||||
}
|
||||
impl QuiescentJobClosure<'_> {
|
||||
pub(crate) fn commit(&mut self) {
|
||||
self.executions.closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Cloneable Feature-facing access to a Host-owned service. Cloning this handle
|
||||
/// neither extends the Host's authority nor permits opening/recovering its store.
|
||||
#[derive(Clone)]
|
||||
@@ -79,6 +88,27 @@ pub struct StandaloneJobs {
|
||||
}
|
||||
|
||||
impl StandaloneJobs {
|
||||
/// Excludes request/start/result acknowledgement while a Subject Host checks its final
|
||||
/// saved obligations and releases ownership. Accepted results alone are insufficient:
|
||||
/// every spawned execution must have actually finished successful cleanup.
|
||||
pub(crate) fn reserve_quiescent_closure(
|
||||
&self,
|
||||
) -> Result<QuiescentJobClosure<'_>, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
if executions.jobs.values().any(|execution| {
|
||||
!execution.task.is_finished()
|
||||
|| !matches!(execution.complete.borrow().as_ref(), Some(Ok(())))
|
||||
}) {
|
||||
return Err(JobServiceError::Cleanup(
|
||||
"delegated Job execution/cleanup remains running or unconfirmed".into(),
|
||||
));
|
||||
}
|
||||
Ok(QuiescentJobClosure { executions })
|
||||
}
|
||||
|
||||
pub(crate) fn open(path: &Path, cwd: PathBuf) -> Result<Self, JobServiceError> {
|
||||
Ok(Self {
|
||||
store: Arc::new(Mutex::new(Some(JobStore::open(path)?))),
|
||||
|
||||
@@ -8,7 +8,10 @@ pub mod store;
|
||||
pub mod subjektiv;
|
||||
mod subjektiv_sessions;
|
||||
|
||||
pub use host::{StandaloneHost, StandaloneShutdownError, StandaloneStartupError};
|
||||
pub use host::{
|
||||
StandaloneHost, StandaloneShutdownError, StandaloneStartupError,
|
||||
StandaloneSubjectDisconnectError,
|
||||
};
|
||||
pub use launch::{ResolvedStandaloneLaunch, StandaloneLaunchConfig, StandaloneLaunchError};
|
||||
pub use protocol::WorkerId;
|
||||
pub use store::{
|
||||
|
||||
@@ -396,14 +396,8 @@ impl StandaloneWorkerStore {
|
||||
LeaseLiveness::Unknown => StandaloneStoreError::LeaseLivenessUnknown(id),
|
||||
});
|
||||
}
|
||||
if let Some(retained) = crate::subjektiv::prepare_worker_deletion(&self.root, &record)? {
|
||||
// Atomic relocation preserves the original Session, not a permanent duplicate.
|
||||
// The Subject's durable consumer can read it after logical Worker removal.
|
||||
fs::rename(self.worker_dir(id), &retained).map_err(StandaloneStoreError::Io)?;
|
||||
sync_directory(retained.parent().expect("retained Worker parent"))?;
|
||||
} else {
|
||||
fs::remove_dir_all(self.worker_dir(id)).map_err(StandaloneStoreError::Io)?;
|
||||
}
|
||||
crate::subjektiv::prepare_worker_deletion(&self.root, &record)?;
|
||||
fs::remove_dir_all(self.worker_dir(id)).map_err(StandaloneStoreError::Io)?;
|
||||
sync_directory(&self.root)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,9 +5,11 @@
|
||||
//! non-private nodes; delegates schema/migration to `subjektiv` and connection policy to `feature-storage`.
|
||||
//!
|
||||
//! The Host holds each selected Subject's exclusive kernel lease through body execution and
|
||||
//! confirmed delegated Job cleanup. Process death releases kernel locks; uncertain in-process
|
||||
//! cleanup retains leases. Restore validates saved Subject/storage bindings and reacquires the
|
||||
//! same Subject. Launches without a selected Subject do not implicitly connect this store.
|
||||
//! confirmed delegated Job cleanup. Kernel lock release never clears durable body ownership:
|
||||
//! shutdown/process death permits only the same body to restore, not a replacement to start.
|
||||
//! Explicit disconnection requires actual Controller quiescence and final Journal completion;
|
||||
//! storage then revokes that writer lifetime before the lease is released. Uncertain cleanup
|
||||
//! retains leases. Launches without a selected Subject do not implicitly connect this store.
|
||||
//!
|
||||
//! The selected Subject's Job store is shared across Workers; the generic Jobs service owns execution
|
||||
//! and cleanup. Consolidation grants allow only immutable candidate batches and limited operations,
|
||||
@@ -49,6 +51,8 @@ pub enum SubjectError {
|
||||
#[error(transparent)]
|
||||
Domain(#[from] subjektiv::SubjektivError),
|
||||
#[error(transparent)]
|
||||
Journal(#[from] subjektiv::journal::JournalError),
|
||||
#[error(transparent)]
|
||||
Storage(#[from] feature_storage::FeatureStorageError),
|
||||
#[error("local Subject storage metadata: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
@@ -223,7 +227,49 @@ impl StandaloneSubjects {
|
||||
scope_id: self.scope_id.clone(),
|
||||
subject_id: subject_id.to_string(),
|
||||
};
|
||||
if self.store.journal_worker_was_released(
|
||||
subject_id,
|
||||
&self.scope_id,
|
||||
&worker_id.to_string(),
|
||||
)? {
|
||||
return Err(SubjectError::InvalidScope("this body's Subject binding was released; start a new Worker/Session rather than restore its old authority".into()));
|
||||
}
|
||||
// Before writer lifetimes existed, a saved body record was the only durable
|
||||
// attachment evidence. Never interpret its stopped status as a release on upgrade.
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(&self.state_root)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
for entry in fs::read_dir(&self.state_root)? {
|
||||
let entry = entry?;
|
||||
let Ok(prior_id) = entry
|
||||
.file_name()
|
||||
.to_string_lossy()
|
||||
.parse::<protocol::WorkerId>()
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if prior_id == worker_id {
|
||||
continue;
|
||||
}
|
||||
let prior = match workers.load(prior_id) {
|
||||
Ok(prior) => prior,
|
||||
Err(crate::store::StandaloneStoreError::WorkerNotFound(_)) => continue,
|
||||
Err(error) => return Err(SubjectError::InvalidScope(error.to_string())),
|
||||
};
|
||||
if prior.subject.as_ref() == Some(&binding)
|
||||
&& !self.store.journal_worker_was_released(
|
||||
subject_id,
|
||||
&self.scope_id,
|
||||
&prior_id.to_string(),
|
||||
)?
|
||||
{
|
||||
return Err(SubjectError::InvalidScope("saved Subject body has no explicit release; restore it and complete the Idle/Journal handoff, rather than treating inactive storage as unoccupied".into()));
|
||||
}
|
||||
}
|
||||
let writer =
|
||||
self.store
|
||||
.claim_journal_writer(subject_id, &self.scope_id, &worker_id.to_string())?;
|
||||
let host = Arc::new_cyclic(|weak| LocalSubjectHost {
|
||||
writer,
|
||||
self_weak: weak.clone(),
|
||||
journal_dispatch: Mutex::new(false),
|
||||
language: self.memory_language.clone(),
|
||||
@@ -252,6 +298,37 @@ pub(crate) struct SubjectConnection {
|
||||
armed: bool,
|
||||
}
|
||||
impl SubjectConnection {
|
||||
/// Construction failed before any Controller or mandatory startup context was published.
|
||||
/// This is cancellation of an unpublished reservation, not Idle inferred from a stopped body.
|
||||
pub(crate) fn abort_unpublished_start(&mut self) -> Result<(), SubjectError> {
|
||||
self.host
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&self.host.writer)?;
|
||||
self.close()
|
||||
}
|
||||
|
||||
pub(crate) fn writer_released_or_unknown(&self) -> bool {
|
||||
self.host
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.host.writer)
|
||||
.map(|active| !active)
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
pub(crate) fn release_for_handoff(
|
||||
&mut self,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
reservation: &mut worker::SubjectReleaseReservation,
|
||||
) -> Result<(), String> {
|
||||
if let Err(error) = self.host.release_for_handoff(record, reservation) {
|
||||
self.host.wake_journals();
|
||||
return Err(error.to_string());
|
||||
}
|
||||
reservation.commit_disconnect()
|
||||
}
|
||||
|
||||
pub fn arm(&mut self) {
|
||||
self.armed = true;
|
||||
}
|
||||
@@ -282,6 +359,7 @@ impl Drop for SubjectConnection {
|
||||
}
|
||||
|
||||
pub(crate) struct LocalSubjectHost {
|
||||
writer: subjektiv::journal::JournalWriterBinding,
|
||||
self_weak: std::sync::Weak<Self>,
|
||||
journal_dispatch: Mutex<bool>,
|
||||
catalog: StandaloneSubjects,
|
||||
@@ -300,6 +378,16 @@ impl LocalSubjectHost {
|
||||
"local Subject Host is closed".into(),
|
||||
));
|
||||
}
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal::journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"Subject writer binding was released".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn body(&self, context: &SubjektivHostContext) -> Result<(), SubjektivHostError> {
|
||||
@@ -339,6 +427,9 @@ impl LocalSubjectHost {
|
||||
"Session does not belong to this local Subject/Worker scope".into(),
|
||||
));
|
||||
}
|
||||
store
|
||||
.bind_journal_session(&self.writer, record.clone())
|
||||
.map_err(journal::journal_error)?;
|
||||
return Ok(record);
|
||||
}
|
||||
if !create {
|
||||
@@ -355,9 +446,13 @@ impl LocalSubjectHost {
|
||||
&context.session_id,
|
||||
)
|
||||
.map_err(domain_error)?;
|
||||
store
|
||||
let record = store
|
||||
.record_session_attribution(record)
|
||||
.map_err(domain_error)
|
||||
.map_err(domain_error)?;
|
||||
store
|
||||
.bind_journal_session(&self.writer, record.clone())
|
||||
.map_err(journal::journal_error)?;
|
||||
Ok(record)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -698,6 +793,29 @@ impl LocalSubjectHost {
|
||||
}
|
||||
}
|
||||
impl SubjektivHost for LocalSubjectHost {
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
self.live.load(Ordering::Acquire)
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
context: &SubjektivHostContext,
|
||||
) -> Result<subjektiv::journal::startup::JournalStartupSnapshot, SubjektivHostError> {
|
||||
self.body(context)?;
|
||||
let reader = subjektiv::journal::JournalHostContext::validated_reader(
|
||||
&self.catalog.store,
|
||||
&self.binding.subject_id,
|
||||
)
|
||||
.map_err(journal::journal_error)?;
|
||||
let predecessor = self
|
||||
.catalog
|
||||
.store
|
||||
.journal_predecessor_session(&self.writer)
|
||||
.map_err(journal::journal_error)?;
|
||||
self.catalog
|
||||
.store
|
||||
.journal_startup_snapshot(&reader, predecessor.as_deref())
|
||||
.map_err(journal::journal_error)
|
||||
}
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
self.live.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ const INPUT_BUDGET: usize = 48 * 1024;
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LocalJournalGrant {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
writer_binding_id: Option<String>,
|
||||
binding: StandaloneSubjectBinding,
|
||||
session_id: String,
|
||||
journal_id: String,
|
||||
@@ -59,73 +61,72 @@ impl StandaloneSubjects {
|
||||
"Session does not belong to the selected Subject/storage scope".into(),
|
||||
)
|
||||
})?;
|
||||
let worker = attribution
|
||||
.worker_id
|
||||
.parse()
|
||||
.map_err(|_| SubjectError::InvalidScope("invalid attributed Worker identity".into()))?;
|
||||
let mut connection = self.connect(subject_id, worker)?;
|
||||
let result = (|| -> Result<(), SubjektivHostError> {
|
||||
let host = &connection.host;
|
||||
let context = authority(host, session_id)?;
|
||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&host.catalog.state_root,
|
||||
&host.binding.scope_id,
|
||||
subject_id,
|
||||
&host.catalog.store,
|
||||
session_id,
|
||||
)?;
|
||||
// Persist this even if source erasure subsequently fails. A retry resumes physical
|
||||
// deletion with the same permanent fence; it must never generate from leftovers.
|
||||
host.catalog
|
||||
.store
|
||||
.erase_journal_session(&context, request_id)
|
||||
.map_err(journal_error)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&connection.jobs_path,
|
||||
host.catalog.state_root.clone(),
|
||||
let leases = self.root.join("leases");
|
||||
private_dir(&leases, true)?;
|
||||
let lease = private_file(&leases.join(format!("{subject_id}.lock")), true)?;
|
||||
if !lease.try_lock_exclusive()? {
|
||||
return Err(SubjectError::Active(subject_id.into()));
|
||||
}
|
||||
let context = JournalHostContext::validated_session(&self.store, attribution)?;
|
||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.state_root,
|
||||
&self.scope_id,
|
||||
subject_id,
|
||||
&self.store,
|
||||
session_id,
|
||||
)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
// Erasure is not a logical reconnect. Never mint a new writer lifetime in order
|
||||
// to delete a released bodys text; the permanent fence precedes physical deletion.
|
||||
self.store.erase_journal_session(&context, request_id)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&self.root.join("jobs").join(subject_id).join("jobs.sqlite3"),
|
||||
self.state_root.clone(),
|
||||
)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
for snapshot in jobs
|
||||
.pending()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
.into_iter()
|
||||
.chain(
|
||||
jobs.unknown_granted()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
for snapshot in jobs
|
||||
.pending()
|
||||
.map_err(service_error)?
|
||||
.into_iter()
|
||||
.chain(jobs.unknown_granted().map_err(service_error)?)
|
||||
{
|
||||
if serde_json::from_value::<LocalJournalGrant>(
|
||||
snapshot.domain_grant.unwrap_or_default(),
|
||||
)
|
||||
.is_ok_and(|grant| grant.binding == host.binding && grant.session_id == session_id)
|
||||
{
|
||||
jobs.cancel(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
}
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
jobs.cancel(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
for snapshot in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if serde_json::from_value::<LocalJournalGrant>(
|
||||
snapshot.domain_grant.unwrap_or_default(),
|
||||
)
|
||||
.is_ok_and(|grant| grant.binding == host.binding && grant.session_id == session_id)
|
||||
{
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
}
|
||||
}
|
||||
for snapshot in jobs
|
||||
.unacknowledged_results()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
session_store::FsStore::new_with_retention(
|
||||
path,
|
||||
crate::store::session_retention(&host.catalog.state_root),
|
||||
)
|
||||
.map_err(source_error)?
|
||||
.delete_session_with_intent(
|
||||
session_id
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
Ok(())
|
||||
})();
|
||||
connection.close()?;
|
||||
result.map_err(|error| SubjectError::InvalidScope(error.to_string()))
|
||||
}
|
||||
session_store::FsStore::new_with_retention(
|
||||
path,
|
||||
crate::store::session_retention(&self.state_root),
|
||||
)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
.delete_session_with_intent(
|
||||
session_id
|
||||
.parse()
|
||||
.map_err(|_| SubjectError::InvalidScope("invalid Session".into()))?,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
self.store.release_erased_journal_body(&context)?;
|
||||
self.manager.shutdown()?;
|
||||
lease.unlock()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,17 +134,17 @@ pub(crate) fn retained_workers_root(state_root: &Path) -> PathBuf {
|
||||
state_root.join("subjektiv").join("retained-workers")
|
||||
}
|
||||
|
||||
/// Called while the generic Worker deletion mutation lease excludes restore/final commits.
|
||||
/// Returning a destination means logical removal must move, not copy/delete, original material.
|
||||
/// There is no model call, and failure leaves the original Worker and its sources in place.
|
||||
/// Ordinary removal cannot turn a stopped/unknown body into an unoccupied Subject. The
|
||||
/// live Host must first release it under the Controller Idle reservation. The saved
|
||||
/// release fence also prevents newly appended old-body history from being recorded here.
|
||||
pub(crate) fn prepare_worker_deletion(
|
||||
state_root: &Path,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
) -> Result<Option<PathBuf>, crate::store::StandaloneStoreError> {
|
||||
) -> Result<(), crate::store::StandaloneStoreError> {
|
||||
let Some(binding) = &record.subject else {
|
||||
return Ok(None);
|
||||
return Ok(());
|
||||
};
|
||||
let prepare = || -> Result<Option<PathBuf>, SubjektivHostError> {
|
||||
let prepare = || -> Result<(), SubjektivHostError> {
|
||||
let catalog = StandaloneSubjects::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
if catalog.scope_id != binding.scope_id {
|
||||
@@ -151,89 +152,68 @@ pub(crate) fn prepare_worker_deletion(
|
||||
"deletion Subject scope mismatch".into(),
|
||||
));
|
||||
}
|
||||
let mut connection = catalog
|
||||
.connect(&binding.subject_id, record.worker_id)
|
||||
let erased = catalog
|
||||
.store
|
||||
.session_attribution(&record.active_session_id.to_string())
|
||||
.map_err(domain_error)?
|
||||
.filter(|attr| {
|
||||
attr.subject_id == binding.subject_id
|
||||
&& attr.runtime_id == binding.scope_id
|
||||
&& attr.worker_id == record.worker_id.to_string()
|
||||
})
|
||||
.map(|attr| JournalHostContext::validated_session(&catalog.store, attr))
|
||||
.transpose()
|
||||
.map_err(journal_error)?;
|
||||
let destructively_released = match erased.as_ref() {
|
||||
Some(context) => catalog
|
||||
.store
|
||||
.release_erased_journal_body(context)
|
||||
.map_err(journal_error)?,
|
||||
None => false,
|
||||
};
|
||||
if !destructively_released
|
||||
&& !catalog
|
||||
.store
|
||||
.journal_worker_was_released(
|
||||
&binding.subject_id,
|
||||
&binding.scope_id,
|
||||
&record.worker_id.to_string(),
|
||||
)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::Conflict { code: "subject_occupied".into(), message: "Subject binding is retained; restore this body and explicitly disconnect while Idle after Journal activity, position and checkpoint complete".into() });
|
||||
}
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let result = (|| {
|
||||
let host = &connection.host;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&connection.jobs_path,
|
||||
record.cwd.canonical_path.clone(),
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
host.bind_jobs(jobs)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let sessions = workers.sessions_dir(record.worker_id);
|
||||
let mut pending = false;
|
||||
// Check every original Session, including a save that preceded its notification.
|
||||
// An unknown old Session attribution/lineage blocks removal instead of being erased.
|
||||
if !sessions.is_dir() {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"original Session storage is missing".into(),
|
||||
));
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(record.worker_id),
|
||||
crate::store::session_retention(state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in session_store::Store::list_sessions(&source_store).map_err(source_error)? {
|
||||
if destructively_released {
|
||||
source_store
|
||||
.delete_session_with_intent(
|
||||
session,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
} else {
|
||||
source_store
|
||||
.release_session_pin(
|
||||
session,
|
||||
session_store::retention::SessionRetentionPin(session),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
}
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
&sessions,
|
||||
crate::store::session_retention(state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in
|
||||
session_store::Store::list_sessions(&source_store).map_err(source_error)?
|
||||
{
|
||||
let context = SubjektivHostContext {
|
||||
worker_id: record.worker_id.to_string(),
|
||||
session_id: session.to_string(),
|
||||
};
|
||||
host.attribution(&context, session == record.active_session_id)?;
|
||||
host.reconcile_journal_session(&context.session_id)?;
|
||||
pending |= !host.session_fully_reflected(&context.session_id)?;
|
||||
if source_store
|
||||
.retention()
|
||||
.pins(session)
|
||||
.map_err(source_error)?
|
||||
.iter()
|
||||
.any(|pin| pin.0 != session)
|
||||
{
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"another consumer retains this Session; relocation is not authorized"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if !pending {
|
||||
for session in
|
||||
session_store::Store::list_sessions(&source_store).map_err(source_error)?
|
||||
{
|
||||
host.prune_source_session(&source_store, session)?;
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
// Reservation is durable before sources leave the public Worker namespace. The
|
||||
// same Subject's next Host consumes it with the retained attribution after reopen.
|
||||
let _ = host.next_journal_job()?;
|
||||
let archive = retained_workers_root(state_root);
|
||||
private_dir(&archive, true)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let destination = archive.join(record.worker_id.to_string());
|
||||
if destination.exists() {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"retained Worker already exists; reconcile deletion outcome".into(),
|
||||
));
|
||||
}
|
||||
Ok(Some(destination))
|
||||
})();
|
||||
connection
|
||||
.close()
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
result
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
prepare()
|
||||
.map_err(|error| crate::store::StandaloneStoreError::JournalRetention(error.to_string()))
|
||||
}
|
||||
|
||||
fn journal_error(error: JournalError) -> SubjektivHostError {
|
||||
pub(super) fn journal_error(error: JournalError) -> SubjektivHostError {
|
||||
match error {
|
||||
JournalError::Conflict(message) => SubjektivHostError::Conflict {
|
||||
code: "journal_conflict".into(),
|
||||
@@ -270,6 +250,89 @@ fn authority(
|
||||
}
|
||||
|
||||
impl LocalSubjectHost {
|
||||
/// The reservation is obtained by this Host from its actual Controller and kept through
|
||||
/// final reconciliation and durable revocation. A stopped body cannot supply this proof.
|
||||
pub(crate) fn release_for_handoff(
|
||||
&self,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
_reservation: &worker::SubjectReleaseReservation,
|
||||
) -> Result<(), SubjektivHostError> {
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Ok(()); // reconcile an earlier committed revocation, never renew it
|
||||
}
|
||||
self.ensure_live()?;
|
||||
let dispatch = self
|
||||
.journal_dispatch
|
||||
.lock()
|
||||
.expect("Journal dispatcher poisoned");
|
||||
if *dispatch {
|
||||
return Err(SubjektivHostError::Conflict {
|
||||
code: "journal_running".into(),
|
||||
message: "Journal dispatcher is still running; binding retained".into(),
|
||||
});
|
||||
}
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(&self.catalog.state_root)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?;
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(self.worker_id),
|
||||
crate::store::session_retention(&self.catalog.state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in session_store::Store::list_sessions(&source_store).map_err(source_error)? {
|
||||
let context = SubjektivHostContext {
|
||||
worker_id: self.worker_id.to_string(),
|
||||
session_id: session.to_string(),
|
||||
};
|
||||
self.attribution(&context, session == record.active_session_id)?;
|
||||
self.reconcile_journal_session(&context.session_id)?;
|
||||
if !self.session_fully_reflected(&context.session_id)? {
|
||||
return Err(SubjektivHostError::Conflict {
|
||||
code: "journal_unreflected".into(),
|
||||
message:
|
||||
"Final saved public history has not been fully recorded; binding retained"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
let jobs = self.jobs()?;
|
||||
for result in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if self.journal_grant(&result).is_ok() {
|
||||
self.complete_journal_job(&jobs, &result)?;
|
||||
}
|
||||
}
|
||||
let mut job_closure = jobs.reserve_quiescent_closure().map_err(service_error)?;
|
||||
for job in jobs
|
||||
.pending()
|
||||
.map_err(service_error)?
|
||||
.into_iter()
|
||||
.chain(jobs.unknown_granted().map_err(service_error)?)
|
||||
.chain(jobs.unacknowledged_results().map_err(service_error)?)
|
||||
{
|
||||
let _ = job;
|
||||
return Err(SubjektivHostError::Conflict { code: "journal_outcome_unconfirmed".into(), message: "Subject has queued/running or Unknown/unacknowledged delegated work; binding retained".into() });
|
||||
}
|
||||
let released = self
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&self.writer)
|
||||
.map_err(journal_error);
|
||||
if released.is_ok()
|
||||
|| !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.unwrap_or(false)
|
||||
{
|
||||
job_closure.commit();
|
||||
}
|
||||
released
|
||||
}
|
||||
|
||||
fn journal_session_erased(&self, session: &str) -> Result<bool, SubjektivHostError> {
|
||||
let context = authority(self, session)?;
|
||||
self.catalog
|
||||
@@ -425,6 +488,11 @@ impl LocalSubjectHost {
|
||||
// Old Workers' retained sources are reconciled as well; possession of the
|
||||
// Subject lease alone does not bypass the original Worker binding checks.
|
||||
if attribution.runtime_id == self.binding.scope_id
|
||||
&& self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &attribution.session_id)
|
||||
.map_err(journal_error)?
|
||||
&& let Err(error) = self.reconcile_journal_session(&attribution.session_id)
|
||||
{
|
||||
eprintln!(
|
||||
@@ -519,6 +587,7 @@ impl LocalSubjectHost {
|
||||
fn next_journal_job(
|
||||
&self,
|
||||
) -> Result<Option<(crate::jobs::StandaloneJobs, String)>, SubjektivHostError> {
|
||||
self.ensure_live()?;
|
||||
let jobs = self.jobs()?;
|
||||
for saved in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if self.journal_grant(&saved).is_ok() {
|
||||
@@ -578,6 +647,14 @@ impl LocalSubjectHost {
|
||||
.list_journals(&reader, after.as_deref(), MAX_JOURNAL_PAGE)
|
||||
.map_err(journal_error)?;
|
||||
for journal in page.items {
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &journal.source_session_id)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let context = authority(self, &journal.source_session_id)?;
|
||||
if let Some(active) = self
|
||||
.catalog
|
||||
@@ -618,6 +695,7 @@ impl LocalSubjectHost {
|
||||
.collect::<Vec<_>>();
|
||||
let origins = self.journal_origins(&ranges)?;
|
||||
let grant = LocalJournalGrant {
|
||||
writer_binding_id: Some(self.writer.binding_id.clone()),
|
||||
binding: self.binding.clone(),
|
||||
session_id: journal.source_session_id,
|
||||
journal_id: journal.id.clone(),
|
||||
@@ -1012,6 +1090,10 @@ impl LocalSubjectHost {
|
||||
})?)
|
||||
.map_err(|error| SubjektivHostError::InvalidArgument(error.to_string()))?;
|
||||
if grant.binding != self.binding
|
||||
|| grant
|
||||
.writer_binding_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &self.writer.binding_id)
|
||||
|| snapshot.request.purpose != PURPOSE
|
||||
|| snapshot.request.serialization_key.as_deref()
|
||||
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|
||||
@@ -1021,6 +1103,24 @@ impl LocalSubjectHost {
|
||||
"Job input/grant does not match this Journal capability".into(),
|
||||
));
|
||||
}
|
||||
// Pre-binding persisted Jobs lack the new field. They may resume only after the
|
||||
// Host has bound their ORIGINAL Session to this lifetime. Session binding is immutable;
|
||||
// missing grant metadata never maps an old Session onto a later body/reconnection.
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &grant.session_id)
|
||||
.map_err(journal_error)?
|
||||
|| !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"Journal Job's original Session is not bound to this active body lifetime".into(),
|
||||
));
|
||||
}
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
let journal = self
|
||||
.catalog
|
||||
@@ -1060,10 +1160,18 @@ impl LocalSubjectHost {
|
||||
"Journal completion binding mismatch".into(),
|
||||
));
|
||||
}
|
||||
self.catalog
|
||||
.store
|
||||
.close_journal_generation(&context, &generation.id, &generation.job)
|
||||
let recovery = JournalJobContext::validated_recovery(
|
||||
&self.catalog.store,
|
||||
&context,
|
||||
&generation.id,
|
||||
generation.job.clone(),
|
||||
)
|
||||
.map_err(journal_error)?;
|
||||
let saved = subjektiv::journal::service::finish(&self.catalog.store, &recovery)
|
||||
.map_err(journal_error)?;
|
||||
if saved != completion {
|
||||
return Err(SubjektivHostError::Conflict { code: "journal_completion_mismatch".into(), message: "Job result differs from the saved Journal activity, position and checkpoint completion".into() });
|
||||
}
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
if let Err(error) = self.prune_retained_workers() {
|
||||
|
||||
@@ -8,6 +8,7 @@ use session_store::{Store, WorkerMetadataStore};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
mod compaction;
|
||||
mod handoff;
|
||||
|
||||
struct Fixture {
|
||||
_temp: tempfile::TempDir,
|
||||
@@ -407,10 +408,19 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
|
||||
JournalAction::ConfirmUnchanged,
|
||||
state.selected_ranges.clone(),
|
||||
);
|
||||
let position = save(
|
||||
&adapter,
|
||||
"position",
|
||||
&nochange.state_token,
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "First prefix checked; last input still pending".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
save(
|
||||
&adapter,
|
||||
"checkpoint",
|
||||
&nochange.state_token,
|
||||
&position.state_token,
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["checked".into()],
|
||||
},
|
||||
@@ -675,6 +685,7 @@ impl agen::llm_client::LlmClient for JournalScript {
|
||||
#[derive(Clone, Default)]
|
||||
struct DialogAndJournal {
|
||||
journal: JournalScript,
|
||||
dialog_requests: Arc<std::sync::Mutex<Vec<agen::llm_client::Request>>>,
|
||||
dialog_calls: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
@@ -730,6 +741,7 @@ impl agen::llm_client::LlmClient for DialogAndJournal {
|
||||
};
|
||||
return Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))));
|
||||
}
|
||||
self.dialog_requests.lock().unwrap().push(request.clone());
|
||||
self.dialog_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Box::pin(futures::stream::iter(
|
||||
[
|
||||
@@ -1000,6 +1012,60 @@ async fn real_job_tools_save_and_drain_last_input_without_another_turn() {
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_pre_binding_job_recovers_only_under_original_session_lifetime_and_never_after_release()
|
||||
{
|
||||
let mut fixture = Fixture::new("pre-upgrade pending Journal");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, modern_id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let saved = jobs.get(&modern_id).unwrap();
|
||||
jobs.cancel(&modern_id).unwrap();
|
||||
let mut request = saved.request;
|
||||
request.job_id = format!("legacy-journal:{}", uuid::Uuid::now_v7());
|
||||
let mut legacy = saved.domain_grant.unwrap();
|
||||
legacy.as_object_mut().unwrap().remove("writer_binding_id");
|
||||
let pending = jobs.request_granted(request, legacy.clone()).unwrap();
|
||||
assert_eq!(
|
||||
jobs.get(&pending.request.job_id).unwrap().domain_grant,
|
||||
Some(legacy)
|
||||
);
|
||||
jobs.start_with_model_client(&pending.request.job_id, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(20),
|
||||
jobs.wait(&pending.request.job_id),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let subject = fixture.connection.binding.subject_id.clone();
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&fixture.host().writer)
|
||||
.unwrap();
|
||||
assert!(fixture.host().journal_grant(&completed).is_err());
|
||||
jobs.shutdown().await.unwrap();
|
||||
fixture.connection.close().unwrap();
|
||||
let mut next = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&subject, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
assert!(
|
||||
next.host.journal_grant(&completed).is_err(),
|
||||
"missing legacy metadata cannot rebind its original Session to the next body"
|
||||
);
|
||||
next.close().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_create() {
|
||||
let mut fixture = Fixture::new("proposed, corrected, partially performed, then interrupted");
|
||||
@@ -1088,51 +1154,38 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopped_worker_removal_retains_originals_and_the_existing_durable_request() {
|
||||
async fn stopped_worker_removal_and_replacement_require_explicit_idle_handoff_even_after_recording()
|
||||
{
|
||||
let mut fixture = Fixture::new("Unrecorded stopped Worker");
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let binding = fixture.connection.binding.clone();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(matches!(
|
||||
workers.load(worker),
|
||||
Err(crate::store::StandaloneStoreError::WorkerNotFound(_))
|
||||
));
|
||||
let retained = retained_workers_root(&state).join(worker.to_string());
|
||||
assert!(workers.delete(worker).is_err());
|
||||
assert!(workers.load(worker).is_ok());
|
||||
assert!(
|
||||
retained
|
||||
.join("sessions")
|
||||
workers
|
||||
.sessions_dir(worker)
|
||||
.join(&fixture.context.session_id)
|
||||
.is_dir()
|
||||
);
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.is_err()
|
||||
);
|
||||
let mut reopened = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.connect(&binding.subject_id, worker)
|
||||
.unwrap();
|
||||
let jobs = crate::jobs::StandaloneJobs::open(&reopened.jobs_path, fixture._temp.path().into())
|
||||
.unwrap();
|
||||
reopened.host.bind_jobs(jobs.clone()).unwrap();
|
||||
assert_eq!(
|
||||
jobs.pending().unwrap().len(),
|
||||
1,
|
||||
"repeated deletion must not multiply requests"
|
||||
);
|
||||
reopened
|
||||
.host
|
||||
.reconcile_journal_session(&fixture.context.session_id)
|
||||
.unwrap();
|
||||
let (_, id) = reopened.host.next_journal_job().unwrap().unwrap();
|
||||
let snapshot = jobs.get(&id).unwrap();
|
||||
assert!(
|
||||
grant(
|
||||
reopened.host.clone(),
|
||||
&snapshot,
|
||||
Arc::new(TestFence(AtomicBool::new(true)))
|
||||
)
|
||||
.is_ok(),
|
||||
"retained originals remain readable through the real grant path"
|
||||
);
|
||||
assert!(
|
||||
retained.is_dir(),
|
||||
"unfinished generation cannot prune original material"
|
||||
);
|
||||
jobs.start_with_model_client(&id, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1146,31 +1199,23 @@ async fn stopped_worker_removal_retains_originals_and_the_existing_durable_reque
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!retained.exists(),
|
||||
"only reflected original material may be pruned"
|
||||
);
|
||||
let context = authority(&reopened.host, &fixture.context.session_id).unwrap();
|
||||
let journal = reopened
|
||||
.host
|
||||
.catalog
|
||||
.store
|
||||
.list_journals(&context, None, 1)
|
||||
.unwrap()
|
||||
.items
|
||||
.pop()
|
||||
.unwrap();
|
||||
let (_, open) = reopened
|
||||
.host
|
||||
.catalog
|
||||
.store
|
||||
.journal_resume_state(&context, &journal.id)
|
||||
.unwrap();
|
||||
assert!(
|
||||
open.unwrap().body.contains("Only part performed"),
|
||||
"Journal remains readable without the removed Worker/Session"
|
||||
reopened
|
||||
.host
|
||||
.session_fully_reflected(&fixture.context.session_id)
|
||||
.unwrap()
|
||||
);
|
||||
jobs.shutdown().await.unwrap();
|
||||
reopened.close().unwrap();
|
||||
assert!(
|
||||
workers.delete(worker).is_err(),
|
||||
"recording completion is not actual Controller Idle/release"
|
||||
);
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1415,8 +1460,12 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
|
||||
.unwrap();
|
||||
let descriptor = old_reader.descriptors().next().unwrap().clone();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(workers.delete(worker).is_err());
|
||||
// Reproduce a T-734 archive already persisted before durable body-release bindings.
|
||||
// New ordinary deletion must not create this state, but its erasure contract remains.
|
||||
let retained = retained_workers_root(&state).join(worker.to_string());
|
||||
fs::create_dir_all(retained.parent().unwrap()).unwrap();
|
||||
fs::rename(workers.root().join(worker.to_string()), &retained).unwrap();
|
||||
let retained_sessions = retained.join("sessions");
|
||||
let retained_store = session_store::FsStore::new_with_retention(
|
||||
&retained_sessions,
|
||||
|
||||
@@ -183,7 +183,7 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
|
||||
retained_tokens: 0,
|
||||
..Default::default()
|
||||
});
|
||||
let host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
|
||||
let mut host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let session = host.record().active_session_id;
|
||||
@@ -422,6 +422,13 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!position.text.is_empty());
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(crate::StandaloneSubjectDisconnectError::Incomplete(message)) if message.contains("dispatcher") || message.contains("cleanup") => tokio::task::yield_now().await,
|
||||
Err(error) => panic!("recorded compaction/cancellation lineage must remain releasable while work is unfinished: {error}"),
|
||||
}
|
||||
}
|
||||
host.shutdown().await.unwrap();
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
//! Real standalone Subject/Controller/Journal Job/initial input connection. Only external
|
||||
//! model streams are scripted. The scenario checks persisted data and ordering, not whether
|
||||
//! a real model understands the correction or can autonomously continue the work.
|
||||
|
||||
use super::*;
|
||||
use crate::{StandaloneHost, StandaloneLaunchConfig, StandaloneSubjectDisconnectError};
|
||||
use agen::llm_client::types::Item;
|
||||
|
||||
fn launch(root: &Path, state: &Path, subject: &str, name: &str) -> crate::ResolvedStandaloneLaunch {
|
||||
StandaloneLaunchConfig::new(
|
||||
root,
|
||||
state,
|
||||
manifest::ProfileSelector::source_named(
|
||||
manifest::ProfileRegistrySource::Builtin,
|
||||
"standalone-subjektiv",
|
||||
),
|
||||
name,
|
||||
)
|
||||
.with_subject(subject.into())
|
||||
.resolve()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recorded_idle_body_handoff_preserves_open_after_source_deletion_and_seeds_next_body_once()
|
||||
{
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("continuity", Some("BEHAVIOR_ORIGINAL_UNCHANGED")).unwrap();
|
||||
let model = DialogAndJournal { journal: JournalScript { gate_first: true, ..Default::default() }, ..Default::default() };
|
||||
let mut first = StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "first"), model.clone()).await.unwrap();
|
||||
let old_worker = first.worker_id();
|
||||
let old_session = first.record().active_session_id;
|
||||
let old_writer = catalog.store.claim_journal_writer(&subject.id, &catalog.scope_id, &old_worker.to_string()).unwrap();
|
||||
submit_dialog(&first, "A was proposed, user corrected its constraint; B selected because A is unusable. One step performed; tests not performed; external outcome Unknown. Work is unfinished.").await;
|
||||
model.journal.entered.notified().await;
|
||||
assert!(matches!(first.disconnect_subject().await, Err(StandaloneSubjectDisconnectError::Incomplete(_))));
|
||||
assert!(catalog.store.journal_writer_active(&old_writer).unwrap());
|
||||
assert!(StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "premature"), DialogAndJournal::default()).await.is_err());
|
||||
let context = JournalHostContext::validated_session(&catalog.store, catalog.store.session_attribution(&old_session.to_string()).unwrap().unwrap()).unwrap();
|
||||
let id = catalog.store.journal_for_session(&context).unwrap().unwrap();
|
||||
let generation = catalog.store.active_journal_generation(&context, &id).unwrap().unwrap();
|
||||
model.journal.release.notify_one();
|
||||
first.jobs().wait(&generation.job.job_id).await.unwrap();
|
||||
// Observe the authoritative gate after cleanup; yielding is cooperative scheduling,
|
||||
// never elapsed time used as proof of recording completion.
|
||||
loop {
|
||||
match first.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(message)) if message.contains("dispatcher") || message.contains("cleanup") => tokio::task::yield_now().await,
|
||||
Err(error) => panic!("unexpected handoff blocker: {error}"),
|
||||
}
|
||||
}
|
||||
assert!(!catalog.store.journal_writer_active(&old_writer).unwrap());
|
||||
assert!(catalog.store.accept_journal_sources(&context, "late old activity", &[generation.selected_ranges[0].clone()]).is_err());
|
||||
first.shutdown().await.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
workers.delete(old_worker).unwrap();
|
||||
assert!(!workers.sessions_dir(old_worker).exists());
|
||||
let reader = JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap();
|
||||
let historical = catalog.store.journal_startup_snapshot(&reader, Some(&old_session.to_string())).unwrap();
|
||||
assert_eq!(historical.journals[0].entries[0].state, JournalEntryState::Open);
|
||||
assert!(historical.journals[0].entries[0].ongoing.contains("unknown"));
|
||||
assert!(catalog.store.list_memories(&subject.id).unwrap().is_empty());
|
||||
|
||||
let second_model = DialogAndJournal::default();
|
||||
let second = StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "second"), second_model.clone()).await.unwrap();
|
||||
submit_dialog(&second, "Continue B; describe what is still unperformed and unconfirmed.").await;
|
||||
let requests = second_model.dialog_requests.lock().unwrap();
|
||||
let initial = &requests[0];
|
||||
let context_index = initial.items.iter().position(|item| item.as_text().is_some_and(|text| text.contains("Proposed A; user corrected it to B"))).expect("saved Journal visible to first model request");
|
||||
let user_index = initial.items.iter().position(|item| matches!(item, Item::Message { role: agen::llm_client::types::Role::User, .. })).unwrap();
|
||||
assert!(context_index < user_index);
|
||||
assert_eq!(initial.items.iter().filter(|item| item.as_text().is_some_and(|text| text.contains("Proposed A; user corrected it to B"))).count(), 1);
|
||||
drop(requests);
|
||||
let session = second.record().active_session_id;
|
||||
let sessions = session_store::FsStore::new_with_retention(workers.sessions_dir(second.worker_id()), crate::store::session_retention(&state)).unwrap();
|
||||
let pointer = session_store::FsWorkerStore::new(workers.worker_metadata_dir(second.worker_id())).unwrap().read_by_name(&second.record().storage_key).unwrap().unwrap().active.unwrap();
|
||||
let entries = sessions.read_all(session, pointer.segment_id.unwrap()).unwrap();
|
||||
let journal_items = entries.iter().filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, session_store::SystemItem::JournalStartup { .. }))).count();
|
||||
assert_eq!(journal_items, 1);
|
||||
assert_eq!(catalog.store.subject(&subject.id).unwrap().unwrap().behavior_md, "BEHAVIOR_ORIGINAL_UNCHANGED");
|
||||
second.shutdown().await.unwrap();
|
||||
}).await.expect("bounded complete Subject handoff connection");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saved_public_history_without_admission_notification_blocks_release_until_actual_recording()
|
||||
{
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("saved-before-notified", None).unwrap();
|
||||
let model = DialogAndJournal {
|
||||
journal: JournalScript {
|
||||
gate_first: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "gap"),
|
||||
model.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
let record = host.record().clone();
|
||||
let writer = catalog
|
||||
.store
|
||||
.claim_journal_writer(
|
||||
&subject.id,
|
||||
&catalog.scope_id,
|
||||
&host.worker_id().to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let sessions = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(host.worker_id()),
|
||||
crate::store::session_retention(&state),
|
||||
)
|
||||
.unwrap();
|
||||
// The production saved-history boundary, with its admission callback deliberately
|
||||
// omitted. The real Host release must discover these bytes, not trust Job status.
|
||||
sessions
|
||||
.append(
|
||||
record.active_session_id,
|
||||
record.active_segment_id.unwrap(),
|
||||
&session_store::LogEntry::AnnotatedUserInput {
|
||||
ts: 3,
|
||||
segments: vec![],
|
||||
extensions: vec![],
|
||||
history: vec![message(
|
||||
"missed-admission",
|
||||
"A corrected to B, partially performed, outcome Unknown",
|
||||
)],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(host.disconnect_subject().await.is_err());
|
||||
assert!(catalog.store.journal_writer_active(&writer).unwrap());
|
||||
model.journal.entered.notified().await;
|
||||
let context = JournalHostContext::validated_session(
|
||||
&catalog.store,
|
||||
catalog
|
||||
.store
|
||||
.session_attribution(&record.active_session_id.to_string())
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let id = catalog
|
||||
.store
|
||||
.journal_for_session(&context)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let generation = catalog
|
||||
.store
|
||||
.active_journal_generation(&context, &id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
model.journal.release.notify_one();
|
||||
host.jobs().wait(&generation.job.job_id).await.unwrap();
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(message))
|
||||
if message.contains("dispatcher") || message.contains("cleanup") =>
|
||||
{
|
||||
tokio::task::yield_now().await
|
||||
}
|
||||
Err(error) => {
|
||||
panic!("unexpected release blocker after saved-history recording: {error}")
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(!catalog.store.journal_writer_active(&writer).unwrap());
|
||||
let snapshot = catalog
|
||||
.store
|
||||
.journal_startup_snapshot(
|
||||
&JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!snapshot.journals[0].current_position.is_empty());
|
||||
assert_eq!(
|
||||
snapshot.journals[0].entries[0].state,
|
||||
JournalEntryState::Open
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
})
|
||||
.await
|
||||
.expect("saved-history/notification-gap handoff is bounded");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_and_normal_deletion_cannot_release_subject_occupancy() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("retained-owner", None).unwrap();
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "retained"),
|
||||
DialogAndJournal::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker = host.worker_id();
|
||||
host.shutdown().await.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
assert!(workers.delete(worker).is_err());
|
||||
assert!(
|
||||
StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "not-an-owner"),
|
||||
DialogAndJournal::default()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut restored =
|
||||
StandaloneHost::restore_with_model_client(state, worker, DialogAndJournal::default())
|
||||
.await
|
||||
.unwrap();
|
||||
restored.disconnect_subject().await.unwrap();
|
||||
restored.shutdown().await.unwrap();
|
||||
}
|
||||
@@ -37,9 +37,8 @@ fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
|
||||
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
|
||||
let a = catalog.create("a", None).unwrap();
|
||||
let b = catalog.create("b", None).unwrap();
|
||||
let mut first = catalog
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
let first_id = protocol::WorkerId::now_v7();
|
||||
let mut first = catalog.connect(&a.id, first_id).unwrap();
|
||||
assert!(matches!(
|
||||
StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
@@ -51,11 +50,18 @@ fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
|
||||
.connect(&b.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
first.close().unwrap();
|
||||
let mut replacement = StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.is_err(),
|
||||
"kernel lease release is not durable body release"
|
||||
);
|
||||
let mut resumed = StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.connect(&a.id, first_id)
|
||||
.unwrap();
|
||||
replacement.close().unwrap();
|
||||
resumed.close().unwrap();
|
||||
second.close().unwrap();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! T-709: real standalone Hosts and test-owned processes, with scripted models.
|
||||
//! These tests do not create a Workspace or contact a model provider.
|
||||
//! Integration of real standalone Subject lifecycle, dialog, Journal and Memory services.
|
||||
//! Only model streams are scripted; temporary storage and test-owned subprocesses isolate
|
||||
//! persistence and kernel leases. Run with `cargo test -p standalone --test subjektiv`.
|
||||
//! No Workspace or live model provider is used; this is not a continuity meaning evaluation.
|
||||
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::io::{Read, Write};
|
||||
@@ -354,17 +356,16 @@ impl LlmClient for ScriptedClient {
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
// This suite observes Memory and dialog requests. Journal's independent model
|
||||
// boundary fails explicitly rather than consuming their script or using live transport;
|
||||
// the Journal adapter suite exercises its successful sequential tool execution.
|
||||
// Journal is a separate scripted model boundary. Successful production recording
|
||||
// is required before these Memory scenarios may hand the Subject to a new body.
|
||||
if request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalState")
|
||||
{
|
||||
return Err(ClientError::Config(
|
||||
"Journal model is outside this Memory suite".into(),
|
||||
));
|
||||
return Ok(Box::pin(stream::iter(
|
||||
journal_response(&request).into_iter().map(Ok),
|
||||
)));
|
||||
}
|
||||
// Lifecycle-triggered consolidation uses a separate Internal Worker. It
|
||||
// must not consume the dialog script or silently become a live request.
|
||||
@@ -423,6 +424,134 @@ impl LlmClient for ScriptedClient {
|
||||
}
|
||||
}
|
||||
|
||||
// The outer model script is not a continuity meaning evaluation. Persistence, complete
|
||||
// source delivery and the handoff gate remain the production implementations.
|
||||
fn journal_response(request: &Request) -> Vec<LlmEvent> {
|
||||
let result = |id: &str| -> Option<Value> {
|
||||
request.items.iter().find_map(|item| match item {
|
||||
Item::ToolResult {
|
||||
call_id,
|
||||
content,
|
||||
is_error,
|
||||
summary,
|
||||
..
|
||||
} if call_id == id => {
|
||||
assert!(!is_error, "{id}: {summary}; {content:?}");
|
||||
Some(
|
||||
content
|
||||
.as_deref()
|
||||
.map(|v| serde_json::from_str(v).unwrap())
|
||||
.unwrap_or(Value::Null),
|
||||
)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
if result("journal-submit").is_some() {
|
||||
return prose();
|
||||
}
|
||||
if result("journal-checkpoint").is_some() {
|
||||
return tool("journal-submit", "SubmitJobResult", json!({"result":{}}));
|
||||
}
|
||||
let Some(state) = result("journal-state") else {
|
||||
return tool("journal-state", "ReadJournalState", json!({}));
|
||||
};
|
||||
let state = &state["value"];
|
||||
let generation = state["generation_id"].as_str().unwrap();
|
||||
if let Some(position) = result("journal-position") {
|
||||
return tool(
|
||||
"journal-checkpoint",
|
||||
"CheckpointJournal",
|
||||
json!({
|
||||
"request_id":format!("checkpoint:{generation}"), "expected_state":position["value"]["state_token"],
|
||||
"action":{"action":"checkpoint","record_operation_ids":[format!("record:{generation}")]},
|
||||
"covered_ranges":state["selected_ranges"]
|
||||
}),
|
||||
);
|
||||
}
|
||||
if let Some(record) = result("journal-record") {
|
||||
return tool(
|
||||
"journal-position",
|
||||
"UpdateJournalPosition",
|
||||
json!({
|
||||
"request_id":format!("position:{generation}"), "expected_state":record["value"]["state_token"],
|
||||
"action":{"action":"update_position","current_position":"Memory scenario recorded; represented work may remain unfinished"}, "covered_ranges":[]
|
||||
}),
|
||||
);
|
||||
}
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("journal-source-{index}-{offset}");
|
||||
match result(&id) {
|
||||
Some(page) => match page["value"]["next_byte"].as_u64() {
|
||||
Some(next) => {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["value"]["content_token"].clone();
|
||||
}
|
||||
None => break,
|
||||
},
|
||||
None => {
|
||||
return tool(
|
||||
&id,
|
||||
"ReadJournalText",
|
||||
json!({
|
||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
||||
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let (name, action) = if state["current_open"].is_null() {
|
||||
(
|
||||
"CreateJournalEntry",
|
||||
json!({"action":"create","title":"Memory conversation","body":"Conversation recorded by the scripted model fixture","ongoing":"Unfinished work is not declared complete","state":"open","source_end":null,"correction_refs":[]}),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"RewriteJournalEntry",
|
||||
json!({"action":"rewrite","entry_id":state["current_open"]["id"],"title":"Memory conversation","body":"Further conversation recorded by the scripted model fixture","ongoing":"Unfinished work is not declared complete"}),
|
||||
)
|
||||
};
|
||||
tool(
|
||||
"journal-record",
|
||||
name,
|
||||
json!({
|
||||
"request_id":format!("record:{generation}"), "expected_state":state["current_journal"]["state_token"],
|
||||
"action":action,"covered_ranges":state["selected_ranges"]
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
async fn disconnect_recorded(host: &mut StandaloneHost) {
|
||||
let mut last_error = None;
|
||||
let completed = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => return,
|
||||
Err(error) => {
|
||||
last_error = Some(error.to_string());
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
completed.is_ok(),
|
||||
"actual Idle/Journal completion is required before Memory scenario handoff: {last_error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
fn prose() -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
@@ -762,7 +891,7 @@ async fn explicit_remember_is_committed_staging_and_sessions_follow_subject_acro
|
||||
tool("query-restored", "SubjektivMemoryQuery", json!({})),
|
||||
prose(),
|
||||
]);
|
||||
let restored =
|
||||
let mut restored =
|
||||
StandaloneHost::restore_with_model_client(state.clone(), worker_id, restored_model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -773,6 +902,7 @@ async fn explicit_remember_is_committed_staging_and_sessions_follow_subject_acro
|
||||
assert!(results["recall"].to_string().contains(claim));
|
||||
assert_eq!(results["query-restored"]["items"], json!([]));
|
||||
assert!(format!("{:?}", restored_model.requests()[0].items).contains("Remember this lesson"));
|
||||
disconnect_recorded(&mut restored).await;
|
||||
restored.shutdown().await.unwrap();
|
||||
|
||||
let second_model = ScriptedClient::new(vec![
|
||||
@@ -992,7 +1122,7 @@ async fn kernel_subject_lease_excludes_other_processes_and_recovers_after_abnorm
|
||||
let independent = catalog.create("independent", None).unwrap();
|
||||
drop(catalog);
|
||||
let ready = temp.path().join("ready");
|
||||
let parent = StandaloneHost::start_with_model_client(
|
||||
let mut parent = StandaloneHost::start_with_model_client(
|
||||
launch(&parent_cwd, &state, &subject.id, "parent-owner"),
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
@@ -1014,6 +1144,7 @@ async fn kernel_subject_lease_excludes_other_processes_and_recovers_after_abnorm
|
||||
.await
|
||||
.unwrap();
|
||||
independent_host.shutdown().await.unwrap();
|
||||
disconnect_recorded(&mut parent).await;
|
||||
parent.shutdown().await.unwrap();
|
||||
|
||||
let mut owner = OwnedChild(
|
||||
@@ -1082,7 +1213,7 @@ async fn automatic_generic_consolidation_applies_candidate_publishes_surface_and
|
||||
drop(catalog);
|
||||
let claim = "Validate the immutable result before reporting success";
|
||||
let model = ScriptedClient::consolidating(remember_script(claim), false);
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "consolidation-body"),
|
||||
model.clone(),
|
||||
)
|
||||
@@ -1157,6 +1288,7 @@ async fn automatic_generic_consolidation_applies_candidate_publishes_surface_and
|
||||
.exists(),
|
||||
"selected Subject owns the shared generic Job ledger"
|
||||
);
|
||||
disconnect_recorded(&mut host).await;
|
||||
host.shutdown().await.unwrap();
|
||||
let memory = inspect_store(&state, |store| {
|
||||
assert!(
|
||||
@@ -1236,7 +1368,7 @@ async fn transport_loss_after_candidate_apply_preserves_memory_and_later_job_onl
|
||||
drop(catalog);
|
||||
let claim = "Candidate receipts survive transport failure";
|
||||
let losing = ScriptedClient::consolidating(remember_script(claim), true);
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "losing-body"),
|
||||
losing.clone(),
|
||||
)
|
||||
@@ -1261,6 +1393,7 @@ async fn transport_loss_after_candidate_apply_preserves_memory_and_later_job_onl
|
||||
failed.result.is_none(),
|
||||
"partial apply is not an accepted Job result"
|
||||
);
|
||||
disconnect_recorded(&mut host).await;
|
||||
host.shutdown().await.unwrap();
|
||||
let original = inspect_store(&state, |store| {
|
||||
assert!(
|
||||
|
||||
Reference in New Issue
Block a user