feat: fence Subject handoff on recorded Journal and seed new bodies once
This commit is contained in:
@@ -1104,8 +1104,50 @@ pub enum WorkerSessionAvailability {
|
||||
},
|
||||
}
|
||||
|
||||
/// Host-owned Subject release operation. A reservation has no timeout: uncertainty retains
|
||||
/// occupancy and mutation exclusion until the same operation is explicitly reconciled.
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjectReleaseRequest {
|
||||
pub workspace_id: String,
|
||||
pub worker_id: WorkerId,
|
||||
pub binding_id: String,
|
||||
pub operation_id: String,
|
||||
pub action: SubjectReleaseAction,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SubjectReleaseAction {
|
||||
Reserve,
|
||||
Disconnect,
|
||||
Cancel,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SubjectReleaseState {
|
||||
Reserved,
|
||||
Disconnected,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjectReleaseResponse {
|
||||
pub operation_id: String,
|
||||
pub binding_id: String,
|
||||
pub state: SubjectReleaseState,
|
||||
}
|
||||
|
||||
#[api(reqwest, axum)]
|
||||
pub trait RuntimeApi {
|
||||
#[post("/v1/subject-release", status = 200, error_status = 400)]
|
||||
async fn subject_release(
|
||||
&self,
|
||||
#[body] request: SubjectReleaseRequest,
|
||||
) -> Result<SubjectReleaseResponse, RuntimeApiError>;
|
||||
|
||||
#[get("/v1/ping", status = 200, error_status = 400)]
|
||||
async fn ping(
|
||||
&self,
|
||||
@@ -1633,6 +1675,13 @@ mod tests {
|
||||
) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
async fn subject_release(
|
||||
&self,
|
||||
_request: SubjectReleaseRequest,
|
||||
) -> Result<SubjectReleaseResponse, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
|
||||
async fn session_material_release(
|
||||
&self,
|
||||
_request: SessionMaterialPinRequest,
|
||||
|
||||
@@ -561,6 +561,28 @@ fn reasoning_proves_rollback_safety_but_private_and_derived_items_are_not_materi
|
||||
session,
|
||||
vec![derived, message("system", LoggedRole::System, "secret")],
|
||||
),
|
||||
LogEntry::AnnotatedSystemItem {
|
||||
ts: 1,
|
||||
entry: crate::LoggedSystemHistoryEntry {
|
||||
item: crate::SystemItem::JournalStartup {
|
||||
body: "previous Journal must not become new recording material".into(),
|
||||
snapshot: serde_json::json!({"journals": []}),
|
||||
snapshot_sha256: "retained-digest".into(),
|
||||
prompt_provenance: crate::PromptRenderProvenance {
|
||||
workspace_id: None,
|
||||
source_digest: "source".into(),
|
||||
projection_digest: "projection".into(),
|
||||
logical_name: "internal.subjektiv_journal_startup".into(),
|
||||
},
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
entry_id: LoggedSessionHistoryEntryId("startup-journal".into()),
|
||||
origin: LoggedSessionHistoryOrigin::DerivedSummary,
|
||||
derivation: None,
|
||||
},
|
||||
},
|
||||
extensions: vec![],
|
||||
},
|
||||
invoke(),
|
||||
input("u"),
|
||||
LogEntry::AnnotatedAssistantItem {
|
||||
|
||||
@@ -92,6 +92,14 @@ pub struct PromptRenderProvenance {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum SystemItem {
|
||||
/// One-time derived Journal reference context. The bounded Host snapshot is
|
||||
/// retained as JSON independently of its rendered body and prompt projection.
|
||||
JournalStartup {
|
||||
body: String,
|
||||
snapshot: serde_json::Value,
|
||||
snapshot_sha256: String,
|
||||
prompt_provenance: PromptRenderProvenance,
|
||||
},
|
||||
/// Free-form notification sent in by an external caller via
|
||||
/// `Method::Notify`. `message` is the raw caller-supplied text;
|
||||
/// `body` is the wrapped LLM-context form (Worker renders it via
|
||||
@@ -194,6 +202,7 @@ impl SystemItem {
|
||||
/// for this item. Returns the variant's stored `body` verbatim.
|
||||
pub fn history_text(&self) -> String {
|
||||
match self {
|
||||
SystemItem::JournalStartup { body, .. } => body.clone(),
|
||||
SystemItem::Notification { body, .. } => body.clone(),
|
||||
SystemItem::WorkerEvent { body, .. } => body.clone(),
|
||||
SystemItem::FeatureInvocationResult { body, .. } => body.clone(),
|
||||
@@ -220,6 +229,7 @@ impl SystemItem {
|
||||
/// wire — keep flexible.
|
||||
pub fn kind_label(&self) -> &'static str {
|
||||
match self {
|
||||
SystemItem::JournalStartup { .. } => "journal_startup",
|
||||
SystemItem::Notification { .. } => "notification",
|
||||
SystemItem::WorkerEvent { .. } => "worker_event",
|
||||
SystemItem::FeatureInvocationResult { .. } => "feature_invocation_result",
|
||||
|
||||
@@ -762,6 +762,11 @@ impl Store for WorkerSessionStore {
|
||||
}
|
||||
|
||||
fn list_segments(&self, session_id: SessionId) -> Result<Vec<SegmentId>, StoreError> {
|
||||
// A pending Worker can have metadata before its atomic initial seed.
|
||||
// Absence is queryable without materializing or binding this store.
|
||||
if self.session_id()?.is_none() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.ensure_session(session_id, false)?;
|
||||
let mut segments: Vec<SegmentId> = Vec::new();
|
||||
for entry in fs::read_dir(self.root.join(SEGMENTS_DIR))? {
|
||||
@@ -804,6 +809,9 @@ impl Store for WorkerSessionStore {
|
||||
}
|
||||
|
||||
fn exists(&self, session_id: SessionId, segment_id: SegmentId) -> Result<bool, StoreError> {
|
||||
if self.session_id()?.is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
self.ensure_session(session_id, false)?;
|
||||
Ok(self.log_path(segment_id).exists())
|
||||
}
|
||||
@@ -2103,6 +2111,24 @@ mod tests {
|
||||
assert!(matches!(error, StoreError::Corrupt { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_session_probes_are_empty_without_binding_and_foreign_materialized_session_is_rejected()
|
||||
{
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let session_id = new_session_id();
|
||||
let segment_id = new_segment_id();
|
||||
let store = WorkerSessionStore::new(root.path()).unwrap();
|
||||
assert!(!store.exists(session_id, segment_id).unwrap());
|
||||
assert!(store.list_segments(session_id).unwrap().is_empty());
|
||||
assert_eq!(store.session_id().unwrap(), None);
|
||||
let reopened = WorkerSessionStore::new(root.path()).unwrap();
|
||||
assert_eq!(reopened.session_id().unwrap(), None);
|
||||
store.create_segment(session_id, segment_id, &[]).unwrap();
|
||||
let foreign = new_session_id();
|
||||
assert!(store.exists(foreign, segment_id).is_err());
|
||||
assert!(store.list_segments(foreign).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reopen_preserves_session_and_segment_ids() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
|
||||
+150
-15
@@ -57,6 +57,15 @@ pub struct StandaloneHost {
|
||||
lease: Option<StandaloneWorkerLease>,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
subject: Option<crate::subjektiv::SubjectConnection>,
|
||||
subject_release: Option<worker::SubjectReleaseReservation>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum StandaloneSubjectDisconnectError {
|
||||
#[error("Subject release requires an actual Idle Controller without pending work: {0}")]
|
||||
Controller(String),
|
||||
#[error("Subject release remains incomplete; retained binding/lease must be reconciled: {0}")]
|
||||
Incomplete(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
@@ -89,6 +98,10 @@ pub enum StandaloneStartupError {
|
||||
FeatureComposition,
|
||||
#[error("the in-process Worker controller could not start")]
|
||||
Controller,
|
||||
#[error(
|
||||
"mandatory initial context remains unconfirmed; restore retained Worker {worker_id} to retry this Session"
|
||||
)]
|
||||
InitialContextPending { worker_id: WorkerId },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
@@ -171,7 +184,7 @@ impl StandaloneHost {
|
||||
let jobs = match crate::jobs::StandaloneJobs::open(&jobs_path, launch.cwd.clone()) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(_) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
};
|
||||
@@ -179,8 +192,11 @@ impl StandaloneHost {
|
||||
if let Some(client) = &model_client {
|
||||
jobs.bind_model_client(client.clone_boxed());
|
||||
}
|
||||
if let Some(subject) = &subject {
|
||||
subject.host.bind_jobs(jobs.clone())?;
|
||||
if let Some(connection) = &subject
|
||||
&& let Err(error) = connection.host.bind_jobs(jobs.clone())
|
||||
{
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(error);
|
||||
}
|
||||
// WorkerId is the stable identity. The current Worker store remains
|
||||
// name-keyed, so keep its derived storage key separate from the
|
||||
@@ -192,7 +208,7 @@ impl StandaloneHost {
|
||||
let (backing_store, worker_store) = match backing_store(&store, worker_id) {
|
||||
Ok(stores) => stores,
|
||||
Err(error) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
@@ -220,12 +236,61 @@ impl StandaloneHost {
|
||||
if let Some(model_client) = model_client {
|
||||
bootstrap = bootstrap.with_model_client(model_client);
|
||||
}
|
||||
let prepared = match bootstrap.prepare().await {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => {
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(classify_startup_error(error));
|
||||
}
|
||||
};
|
||||
// Reserve a recoverable identity before mandatory initial-context persistence.
|
||||
// A failed/unknown seed must be reopened as this Session, never abandoned and
|
||||
// replaced by a newly fetched context under an invisible ghost binding.
|
||||
let pending_record = if subject.is_some() {
|
||||
match store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest.clone(),
|
||||
storage_key.clone(),
|
||||
prepared.worker().session_id(),
|
||||
None,
|
||||
subject.as_ref().map(|s| s.binding.clone()),
|
||||
) {
|
||||
Ok(record) => Some(record),
|
||||
Err(_) => {
|
||||
if let Some(subject) = &mut subject {
|
||||
subject.retain();
|
||||
}
|
||||
allocation.into_lease().retain();
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(subject) = &mut subject {
|
||||
subject.arm();
|
||||
}
|
||||
let started = match bootstrap.start().await {
|
||||
let started = match prepared.start().await {
|
||||
Ok(started) => started,
|
||||
Err(error) => {
|
||||
if let Some(record) = &pending_record {
|
||||
let cleanup = jobs.shutdown().await;
|
||||
let saved = store.mark_stopped(
|
||||
record,
|
||||
record.active_session_id,
|
||||
record.active_segment_id,
|
||||
StandaloneShutdownReason::StartupFailed,
|
||||
);
|
||||
let closed = cleanup.is_ok()
|
||||
&& saved.is_ok()
|
||||
&& subject.as_mut().is_some_and(|s| s.close().is_ok());
|
||||
if closed {
|
||||
let _ = allocation.into_lease().release();
|
||||
} else {
|
||||
allocation.into_lease().retain();
|
||||
}
|
||||
return Err(StandaloneStartupError::InitialContextPending { worker_id });
|
||||
}
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
return Err(classify_startup_error(error));
|
||||
}
|
||||
@@ -234,22 +299,35 @@ impl StandaloneHost {
|
||||
Ok(active) => active,
|
||||
Err(error) => {
|
||||
stop_started_worker(started).await;
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
if pending_record.is_some() {
|
||||
allocation.into_lease().retain();
|
||||
} else {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let record = match store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest,
|
||||
storage_key,
|
||||
active.session_id,
|
||||
active.segment_id,
|
||||
subject.as_ref().map(|s| s.binding.clone()),
|
||||
) {
|
||||
let committed = if let Some(record) = &pending_record {
|
||||
store.update_active_pointer(record, active.session_id, active.segment_id)
|
||||
} else {
|
||||
store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest,
|
||||
storage_key,
|
||||
active.session_id,
|
||||
active.segment_id,
|
||||
None,
|
||||
)
|
||||
};
|
||||
let record = match committed {
|
||||
Ok(record) => record,
|
||||
Err(_) => {
|
||||
stop_started_worker(started).await;
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
if pending_record.is_some() {
|
||||
allocation.into_lease().retain();
|
||||
} else {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
}
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
};
|
||||
@@ -427,6 +505,7 @@ impl StandaloneHost {
|
||||
lease: Some(lease),
|
||||
jobs,
|
||||
subject,
|
||||
subject_release: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -457,6 +536,45 @@ impl StandaloneHost {
|
||||
Client::new(socket)
|
||||
}
|
||||
|
||||
/// Logical Subject disconnection, not Worker shutdown or completion of its represented work.
|
||||
/// A successful release keeps the old Controller fenced; a fresh Worker/Session may then
|
||||
/// acquire the Subject. Failures before durable revocation allow normal recovery and retry.
|
||||
pub async fn disconnect_subject(&mut self) -> Result<(), StandaloneSubjectDisconnectError> {
|
||||
let Some(subject) = &mut self.subject else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut reservation = match self.subject_release.take() {
|
||||
Some(reservation) => reservation,
|
||||
None => self
|
||||
.handle
|
||||
.reserve_subject_release()
|
||||
.await
|
||||
.map_err(StandaloneSubjectDisconnectError::Controller)?,
|
||||
};
|
||||
let release = match subject.release_for_handoff(&self.record, &mut reservation) {
|
||||
Ok(()) => match self.jobs.shutdown().await {
|
||||
Ok(()) => subject.close().map_err(|error| error.to_string()),
|
||||
Err(error) => Err(error.to_string()),
|
||||
},
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
match release {
|
||||
Ok(()) => {
|
||||
self.subject_release = Some(reservation);
|
||||
self.subject = None;
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => {
|
||||
// If revocation committed, keep the actual Controller reserved even if closing
|
||||
// storage/unlocking failed. Dropping a reservation must not undo revocation.
|
||||
if subject.writer_released_or_unknown() {
|
||||
self.subject_release = Some(reservation);
|
||||
}
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
|
||||
self.shutdown_timeout = shutdown_timeout;
|
||||
self
|
||||
@@ -695,6 +813,23 @@ async fn send_protocol_event(peer: &InProcessPeer, event: Event) -> bool {
|
||||
peer.send(message).await.is_ok()
|
||||
}
|
||||
|
||||
fn cancel_unpublished_start(
|
||||
store: &StandaloneWorkerStore,
|
||||
subject: &mut Option<crate::subjektiv::SubjectConnection>,
|
||||
allocation: crate::store::StandaloneWorkerAllocation,
|
||||
) -> Result<(), StandaloneStartupError> {
|
||||
if let Some(connection) = subject {
|
||||
if connection.abort_unpublished_start().is_err() {
|
||||
connection.retain();
|
||||
allocation.into_lease().retain();
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
}
|
||||
store
|
||||
.abandon_allocation(allocation)
|
||||
.map_err(classify_store_startup_error)
|
||||
}
|
||||
|
||||
fn backing_store(
|
||||
store: &StandaloneWorkerStore,
|
||||
worker_id: WorkerId,
|
||||
|
||||
@@ -66,6 +66,15 @@ struct Executions {
|
||||
jobs: HashMap<String, Execution>,
|
||||
}
|
||||
|
||||
pub(crate) struct QuiescentJobClosure<'a> {
|
||||
executions: std::sync::MutexGuard<'a, Executions>,
|
||||
}
|
||||
impl QuiescentJobClosure<'_> {
|
||||
pub(crate) fn commit(&mut self) {
|
||||
self.executions.closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Cloneable Feature-facing access to a Host-owned service. Cloning this handle
|
||||
/// neither extends the Host's authority nor permits opening/recovering its store.
|
||||
#[derive(Clone)]
|
||||
@@ -79,6 +88,27 @@ pub struct StandaloneJobs {
|
||||
}
|
||||
|
||||
impl StandaloneJobs {
|
||||
/// Excludes request/start/result acknowledgement while a Subject Host checks its final
|
||||
/// saved obligations and releases ownership. Accepted results alone are insufficient:
|
||||
/// every spawned execution must have actually finished successful cleanup.
|
||||
pub(crate) fn reserve_quiescent_closure(
|
||||
&self,
|
||||
) -> Result<QuiescentJobClosure<'_>, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
if executions.jobs.values().any(|execution| {
|
||||
!execution.task.is_finished()
|
||||
|| !matches!(execution.complete.borrow().as_ref(), Some(Ok(())))
|
||||
}) {
|
||||
return Err(JobServiceError::Cleanup(
|
||||
"delegated Job execution/cleanup remains running or unconfirmed".into(),
|
||||
));
|
||||
}
|
||||
Ok(QuiescentJobClosure { executions })
|
||||
}
|
||||
|
||||
pub(crate) fn open(path: &Path, cwd: PathBuf) -> Result<Self, JobServiceError> {
|
||||
Ok(Self {
|
||||
store: Arc::new(Mutex::new(Some(JobStore::open(path)?))),
|
||||
|
||||
@@ -8,7 +8,10 @@ pub mod store;
|
||||
pub mod subjektiv;
|
||||
mod subjektiv_sessions;
|
||||
|
||||
pub use host::{StandaloneHost, StandaloneShutdownError, StandaloneStartupError};
|
||||
pub use host::{
|
||||
StandaloneHost, StandaloneShutdownError, StandaloneStartupError,
|
||||
StandaloneSubjectDisconnectError,
|
||||
};
|
||||
pub use launch::{ResolvedStandaloneLaunch, StandaloneLaunchConfig, StandaloneLaunchError};
|
||||
pub use protocol::WorkerId;
|
||||
pub use store::{
|
||||
|
||||
@@ -396,14 +396,8 @@ impl StandaloneWorkerStore {
|
||||
LeaseLiveness::Unknown => StandaloneStoreError::LeaseLivenessUnknown(id),
|
||||
});
|
||||
}
|
||||
if let Some(retained) = crate::subjektiv::prepare_worker_deletion(&self.root, &record)? {
|
||||
// Atomic relocation preserves the original Session, not a permanent duplicate.
|
||||
// The Subject's durable consumer can read it after logical Worker removal.
|
||||
fs::rename(self.worker_dir(id), &retained).map_err(StandaloneStoreError::Io)?;
|
||||
sync_directory(retained.parent().expect("retained Worker parent"))?;
|
||||
} else {
|
||||
fs::remove_dir_all(self.worker_dir(id)).map_err(StandaloneStoreError::Io)?;
|
||||
}
|
||||
crate::subjektiv::prepare_worker_deletion(&self.root, &record)?;
|
||||
fs::remove_dir_all(self.worker_dir(id)).map_err(StandaloneStoreError::Io)?;
|
||||
sync_directory(&self.root)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,9 +5,11 @@
|
||||
//! non-private nodes; delegates schema/migration to `subjektiv` and connection policy to `feature-storage`.
|
||||
//!
|
||||
//! The Host holds each selected Subject's exclusive kernel lease through body execution and
|
||||
//! confirmed delegated Job cleanup. Process death releases kernel locks; uncertain in-process
|
||||
//! cleanup retains leases. Restore validates saved Subject/storage bindings and reacquires the
|
||||
//! same Subject. Launches without a selected Subject do not implicitly connect this store.
|
||||
//! confirmed delegated Job cleanup. Kernel lock release never clears durable body ownership:
|
||||
//! shutdown/process death permits only the same body to restore, not a replacement to start.
|
||||
//! Explicit disconnection requires actual Controller quiescence and final Journal completion;
|
||||
//! storage then revokes that writer lifetime before the lease is released. Uncertain cleanup
|
||||
//! retains leases. Launches without a selected Subject do not implicitly connect this store.
|
||||
//!
|
||||
//! The selected Subject's Job store is shared across Workers; the generic Jobs service owns execution
|
||||
//! and cleanup. Consolidation grants allow only immutable candidate batches and limited operations,
|
||||
@@ -49,6 +51,8 @@ pub enum SubjectError {
|
||||
#[error(transparent)]
|
||||
Domain(#[from] subjektiv::SubjektivError),
|
||||
#[error(transparent)]
|
||||
Journal(#[from] subjektiv::journal::JournalError),
|
||||
#[error(transparent)]
|
||||
Storage(#[from] feature_storage::FeatureStorageError),
|
||||
#[error("local Subject storage metadata: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
@@ -223,7 +227,49 @@ impl StandaloneSubjects {
|
||||
scope_id: self.scope_id.clone(),
|
||||
subject_id: subject_id.to_string(),
|
||||
};
|
||||
if self.store.journal_worker_was_released(
|
||||
subject_id,
|
||||
&self.scope_id,
|
||||
&worker_id.to_string(),
|
||||
)? {
|
||||
return Err(SubjectError::InvalidScope("this body's Subject binding was released; start a new Worker/Session rather than restore its old authority".into()));
|
||||
}
|
||||
// Before writer lifetimes existed, a saved body record was the only durable
|
||||
// attachment evidence. Never interpret its stopped status as a release on upgrade.
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(&self.state_root)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
for entry in fs::read_dir(&self.state_root)? {
|
||||
let entry = entry?;
|
||||
let Ok(prior_id) = entry
|
||||
.file_name()
|
||||
.to_string_lossy()
|
||||
.parse::<protocol::WorkerId>()
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if prior_id == worker_id {
|
||||
continue;
|
||||
}
|
||||
let prior = match workers.load(prior_id) {
|
||||
Ok(prior) => prior,
|
||||
Err(crate::store::StandaloneStoreError::WorkerNotFound(_)) => continue,
|
||||
Err(error) => return Err(SubjectError::InvalidScope(error.to_string())),
|
||||
};
|
||||
if prior.subject.as_ref() == Some(&binding)
|
||||
&& !self.store.journal_worker_was_released(
|
||||
subject_id,
|
||||
&self.scope_id,
|
||||
&prior_id.to_string(),
|
||||
)?
|
||||
{
|
||||
return Err(SubjectError::InvalidScope("saved Subject body has no explicit release; restore it and complete the Idle/Journal handoff, rather than treating inactive storage as unoccupied".into()));
|
||||
}
|
||||
}
|
||||
let writer =
|
||||
self.store
|
||||
.claim_journal_writer(subject_id, &self.scope_id, &worker_id.to_string())?;
|
||||
let host = Arc::new_cyclic(|weak| LocalSubjectHost {
|
||||
writer,
|
||||
self_weak: weak.clone(),
|
||||
journal_dispatch: Mutex::new(false),
|
||||
language: self.memory_language.clone(),
|
||||
@@ -252,6 +298,37 @@ pub(crate) struct SubjectConnection {
|
||||
armed: bool,
|
||||
}
|
||||
impl SubjectConnection {
|
||||
/// Construction failed before any Controller or mandatory startup context was published.
|
||||
/// This is cancellation of an unpublished reservation, not Idle inferred from a stopped body.
|
||||
pub(crate) fn abort_unpublished_start(&mut self) -> Result<(), SubjectError> {
|
||||
self.host
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&self.host.writer)?;
|
||||
self.close()
|
||||
}
|
||||
|
||||
pub(crate) fn writer_released_or_unknown(&self) -> bool {
|
||||
self.host
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.host.writer)
|
||||
.map(|active| !active)
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
pub(crate) fn release_for_handoff(
|
||||
&mut self,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
reservation: &mut worker::SubjectReleaseReservation,
|
||||
) -> Result<(), String> {
|
||||
if let Err(error) = self.host.release_for_handoff(record, reservation) {
|
||||
self.host.wake_journals();
|
||||
return Err(error.to_string());
|
||||
}
|
||||
reservation.commit_disconnect()
|
||||
}
|
||||
|
||||
pub fn arm(&mut self) {
|
||||
self.armed = true;
|
||||
}
|
||||
@@ -282,6 +359,7 @@ impl Drop for SubjectConnection {
|
||||
}
|
||||
|
||||
pub(crate) struct LocalSubjectHost {
|
||||
writer: subjektiv::journal::JournalWriterBinding,
|
||||
self_weak: std::sync::Weak<Self>,
|
||||
journal_dispatch: Mutex<bool>,
|
||||
catalog: StandaloneSubjects,
|
||||
@@ -300,6 +378,16 @@ impl LocalSubjectHost {
|
||||
"local Subject Host is closed".into(),
|
||||
));
|
||||
}
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal::journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"Subject writer binding was released".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn body(&self, context: &SubjektivHostContext) -> Result<(), SubjektivHostError> {
|
||||
@@ -339,6 +427,9 @@ impl LocalSubjectHost {
|
||||
"Session does not belong to this local Subject/Worker scope".into(),
|
||||
));
|
||||
}
|
||||
store
|
||||
.bind_journal_session(&self.writer, record.clone())
|
||||
.map_err(journal::journal_error)?;
|
||||
return Ok(record);
|
||||
}
|
||||
if !create {
|
||||
@@ -355,9 +446,13 @@ impl LocalSubjectHost {
|
||||
&context.session_id,
|
||||
)
|
||||
.map_err(domain_error)?;
|
||||
store
|
||||
let record = store
|
||||
.record_session_attribution(record)
|
||||
.map_err(domain_error)
|
||||
.map_err(domain_error)?;
|
||||
store
|
||||
.bind_journal_session(&self.writer, record.clone())
|
||||
.map_err(journal::journal_error)?;
|
||||
Ok(record)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -698,6 +793,29 @@ impl LocalSubjectHost {
|
||||
}
|
||||
}
|
||||
impl SubjektivHost for LocalSubjectHost {
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
self.live.load(Ordering::Acquire)
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
context: &SubjektivHostContext,
|
||||
) -> Result<subjektiv::journal::startup::JournalStartupSnapshot, SubjektivHostError> {
|
||||
self.body(context)?;
|
||||
let reader = subjektiv::journal::JournalHostContext::validated_reader(
|
||||
&self.catalog.store,
|
||||
&self.binding.subject_id,
|
||||
)
|
||||
.map_err(journal::journal_error)?;
|
||||
let predecessor = self
|
||||
.catalog
|
||||
.store
|
||||
.journal_predecessor_session(&self.writer)
|
||||
.map_err(journal::journal_error)?;
|
||||
self.catalog
|
||||
.store
|
||||
.journal_startup_snapshot(&reader, predecessor.as_deref())
|
||||
.map_err(journal::journal_error)
|
||||
}
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
self.live.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ const INPUT_BUDGET: usize = 48 * 1024;
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LocalJournalGrant {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
writer_binding_id: Option<String>,
|
||||
binding: StandaloneSubjectBinding,
|
||||
session_id: String,
|
||||
journal_id: String,
|
||||
@@ -59,73 +61,72 @@ impl StandaloneSubjects {
|
||||
"Session does not belong to the selected Subject/storage scope".into(),
|
||||
)
|
||||
})?;
|
||||
let worker = attribution
|
||||
.worker_id
|
||||
.parse()
|
||||
.map_err(|_| SubjectError::InvalidScope("invalid attributed Worker identity".into()))?;
|
||||
let mut connection = self.connect(subject_id, worker)?;
|
||||
let result = (|| -> Result<(), SubjektivHostError> {
|
||||
let host = &connection.host;
|
||||
let context = authority(host, session_id)?;
|
||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&host.catalog.state_root,
|
||||
&host.binding.scope_id,
|
||||
subject_id,
|
||||
&host.catalog.store,
|
||||
session_id,
|
||||
)?;
|
||||
// Persist this even if source erasure subsequently fails. A retry resumes physical
|
||||
// deletion with the same permanent fence; it must never generate from leftovers.
|
||||
host.catalog
|
||||
.store
|
||||
.erase_journal_session(&context, request_id)
|
||||
.map_err(journal_error)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&connection.jobs_path,
|
||||
host.catalog.state_root.clone(),
|
||||
let leases = self.root.join("leases");
|
||||
private_dir(&leases, true)?;
|
||||
let lease = private_file(&leases.join(format!("{subject_id}.lock")), true)?;
|
||||
if !lease.try_lock_exclusive()? {
|
||||
return Err(SubjectError::Active(subject_id.into()));
|
||||
}
|
||||
let context = JournalHostContext::validated_session(&self.store, attribution)?;
|
||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.state_root,
|
||||
&self.scope_id,
|
||||
subject_id,
|
||||
&self.store,
|
||||
session_id,
|
||||
)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
// Erasure is not a logical reconnect. Never mint a new writer lifetime in order
|
||||
// to delete a released bodys text; the permanent fence precedes physical deletion.
|
||||
self.store.erase_journal_session(&context, request_id)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&self.root.join("jobs").join(subject_id).join("jobs.sqlite3"),
|
||||
self.state_root.clone(),
|
||||
)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
for snapshot in jobs
|
||||
.pending()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
.into_iter()
|
||||
.chain(
|
||||
jobs.unknown_granted()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
for snapshot in jobs
|
||||
.pending()
|
||||
.map_err(service_error)?
|
||||
.into_iter()
|
||||
.chain(jobs.unknown_granted().map_err(service_error)?)
|
||||
{
|
||||
if serde_json::from_value::<LocalJournalGrant>(
|
||||
snapshot.domain_grant.unwrap_or_default(),
|
||||
)
|
||||
.is_ok_and(|grant| grant.binding == host.binding && grant.session_id == session_id)
|
||||
{
|
||||
jobs.cancel(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
}
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
jobs.cancel(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
for snapshot in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if serde_json::from_value::<LocalJournalGrant>(
|
||||
snapshot.domain_grant.unwrap_or_default(),
|
||||
)
|
||||
.is_ok_and(|grant| grant.binding == host.binding && grant.session_id == session_id)
|
||||
{
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
}
|
||||
}
|
||||
for snapshot in jobs
|
||||
.unacknowledged_results()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
session_store::FsStore::new_with_retention(
|
||||
path,
|
||||
crate::store::session_retention(&host.catalog.state_root),
|
||||
)
|
||||
.map_err(source_error)?
|
||||
.delete_session_with_intent(
|
||||
session_id
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
Ok(())
|
||||
})();
|
||||
connection.close()?;
|
||||
result.map_err(|error| SubjectError::InvalidScope(error.to_string()))
|
||||
}
|
||||
session_store::FsStore::new_with_retention(
|
||||
path,
|
||||
crate::store::session_retention(&self.state_root),
|
||||
)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
.delete_session_with_intent(
|
||||
session_id
|
||||
.parse()
|
||||
.map_err(|_| SubjectError::InvalidScope("invalid Session".into()))?,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
self.store.release_erased_journal_body(&context)?;
|
||||
self.manager.shutdown()?;
|
||||
lease.unlock()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,17 +134,17 @@ pub(crate) fn retained_workers_root(state_root: &Path) -> PathBuf {
|
||||
state_root.join("subjektiv").join("retained-workers")
|
||||
}
|
||||
|
||||
/// Called while the generic Worker deletion mutation lease excludes restore/final commits.
|
||||
/// Returning a destination means logical removal must move, not copy/delete, original material.
|
||||
/// There is no model call, and failure leaves the original Worker and its sources in place.
|
||||
/// Ordinary removal cannot turn a stopped/unknown body into an unoccupied Subject. The
|
||||
/// live Host must first release it under the Controller Idle reservation. The saved
|
||||
/// release fence also prevents newly appended old-body history from being recorded here.
|
||||
pub(crate) fn prepare_worker_deletion(
|
||||
state_root: &Path,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
) -> Result<Option<PathBuf>, crate::store::StandaloneStoreError> {
|
||||
) -> Result<(), crate::store::StandaloneStoreError> {
|
||||
let Some(binding) = &record.subject else {
|
||||
return Ok(None);
|
||||
return Ok(());
|
||||
};
|
||||
let prepare = || -> Result<Option<PathBuf>, SubjektivHostError> {
|
||||
let prepare = || -> Result<(), SubjektivHostError> {
|
||||
let catalog = StandaloneSubjects::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
if catalog.scope_id != binding.scope_id {
|
||||
@@ -151,89 +152,68 @@ pub(crate) fn prepare_worker_deletion(
|
||||
"deletion Subject scope mismatch".into(),
|
||||
));
|
||||
}
|
||||
let mut connection = catalog
|
||||
.connect(&binding.subject_id, record.worker_id)
|
||||
let erased = catalog
|
||||
.store
|
||||
.session_attribution(&record.active_session_id.to_string())
|
||||
.map_err(domain_error)?
|
||||
.filter(|attr| {
|
||||
attr.subject_id == binding.subject_id
|
||||
&& attr.runtime_id == binding.scope_id
|
||||
&& attr.worker_id == record.worker_id.to_string()
|
||||
})
|
||||
.map(|attr| JournalHostContext::validated_session(&catalog.store, attr))
|
||||
.transpose()
|
||||
.map_err(journal_error)?;
|
||||
let destructively_released = match erased.as_ref() {
|
||||
Some(context) => catalog
|
||||
.store
|
||||
.release_erased_journal_body(context)
|
||||
.map_err(journal_error)?,
|
||||
None => false,
|
||||
};
|
||||
if !destructively_released
|
||||
&& !catalog
|
||||
.store
|
||||
.journal_worker_was_released(
|
||||
&binding.subject_id,
|
||||
&binding.scope_id,
|
||||
&record.worker_id.to_string(),
|
||||
)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::Conflict { code: "subject_occupied".into(), message: "Subject binding is retained; restore this body and explicitly disconnect while Idle after Journal activity, position and checkpoint complete".into() });
|
||||
}
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let result = (|| {
|
||||
let host = &connection.host;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&connection.jobs_path,
|
||||
record.cwd.canonical_path.clone(),
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
host.bind_jobs(jobs)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let sessions = workers.sessions_dir(record.worker_id);
|
||||
let mut pending = false;
|
||||
// Check every original Session, including a save that preceded its notification.
|
||||
// An unknown old Session attribution/lineage blocks removal instead of being erased.
|
||||
if !sessions.is_dir() {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"original Session storage is missing".into(),
|
||||
));
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(record.worker_id),
|
||||
crate::store::session_retention(state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in session_store::Store::list_sessions(&source_store).map_err(source_error)? {
|
||||
if destructively_released {
|
||||
source_store
|
||||
.delete_session_with_intent(
|
||||
session,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
} else {
|
||||
source_store
|
||||
.release_session_pin(
|
||||
session,
|
||||
session_store::retention::SessionRetentionPin(session),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
}
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
&sessions,
|
||||
crate::store::session_retention(state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in
|
||||
session_store::Store::list_sessions(&source_store).map_err(source_error)?
|
||||
{
|
||||
let context = SubjektivHostContext {
|
||||
worker_id: record.worker_id.to_string(),
|
||||
session_id: session.to_string(),
|
||||
};
|
||||
host.attribution(&context, session == record.active_session_id)?;
|
||||
host.reconcile_journal_session(&context.session_id)?;
|
||||
pending |= !host.session_fully_reflected(&context.session_id)?;
|
||||
if source_store
|
||||
.retention()
|
||||
.pins(session)
|
||||
.map_err(source_error)?
|
||||
.iter()
|
||||
.any(|pin| pin.0 != session)
|
||||
{
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"another consumer retains this Session; relocation is not authorized"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if !pending {
|
||||
for session in
|
||||
session_store::Store::list_sessions(&source_store).map_err(source_error)?
|
||||
{
|
||||
host.prune_source_session(&source_store, session)?;
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
// Reservation is durable before sources leave the public Worker namespace. The
|
||||
// same Subject's next Host consumes it with the retained attribution after reopen.
|
||||
let _ = host.next_journal_job()?;
|
||||
let archive = retained_workers_root(state_root);
|
||||
private_dir(&archive, true)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let destination = archive.join(record.worker_id.to_string());
|
||||
if destination.exists() {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"retained Worker already exists; reconcile deletion outcome".into(),
|
||||
));
|
||||
}
|
||||
Ok(Some(destination))
|
||||
})();
|
||||
connection
|
||||
.close()
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
result
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
prepare()
|
||||
.map_err(|error| crate::store::StandaloneStoreError::JournalRetention(error.to_string()))
|
||||
}
|
||||
|
||||
fn journal_error(error: JournalError) -> SubjektivHostError {
|
||||
pub(super) fn journal_error(error: JournalError) -> SubjektivHostError {
|
||||
match error {
|
||||
JournalError::Conflict(message) => SubjektivHostError::Conflict {
|
||||
code: "journal_conflict".into(),
|
||||
@@ -270,6 +250,89 @@ fn authority(
|
||||
}
|
||||
|
||||
impl LocalSubjectHost {
|
||||
/// The reservation is obtained by this Host from its actual Controller and kept through
|
||||
/// final reconciliation and durable revocation. A stopped body cannot supply this proof.
|
||||
pub(crate) fn release_for_handoff(
|
||||
&self,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
_reservation: &worker::SubjectReleaseReservation,
|
||||
) -> Result<(), SubjektivHostError> {
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Ok(()); // reconcile an earlier committed revocation, never renew it
|
||||
}
|
||||
self.ensure_live()?;
|
||||
let dispatch = self
|
||||
.journal_dispatch
|
||||
.lock()
|
||||
.expect("Journal dispatcher poisoned");
|
||||
if *dispatch {
|
||||
return Err(SubjektivHostError::Conflict {
|
||||
code: "journal_running".into(),
|
||||
message: "Journal dispatcher is still running; binding retained".into(),
|
||||
});
|
||||
}
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(&self.catalog.state_root)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?;
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(self.worker_id),
|
||||
crate::store::session_retention(&self.catalog.state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in session_store::Store::list_sessions(&source_store).map_err(source_error)? {
|
||||
let context = SubjektivHostContext {
|
||||
worker_id: self.worker_id.to_string(),
|
||||
session_id: session.to_string(),
|
||||
};
|
||||
self.attribution(&context, session == record.active_session_id)?;
|
||||
self.reconcile_journal_session(&context.session_id)?;
|
||||
if !self.session_fully_reflected(&context.session_id)? {
|
||||
return Err(SubjektivHostError::Conflict {
|
||||
code: "journal_unreflected".into(),
|
||||
message:
|
||||
"Final saved public history has not been fully recorded; binding retained"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
let jobs = self.jobs()?;
|
||||
for result in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if self.journal_grant(&result).is_ok() {
|
||||
self.complete_journal_job(&jobs, &result)?;
|
||||
}
|
||||
}
|
||||
let mut job_closure = jobs.reserve_quiescent_closure().map_err(service_error)?;
|
||||
for job in jobs
|
||||
.pending()
|
||||
.map_err(service_error)?
|
||||
.into_iter()
|
||||
.chain(jobs.unknown_granted().map_err(service_error)?)
|
||||
.chain(jobs.unacknowledged_results().map_err(service_error)?)
|
||||
{
|
||||
let _ = job;
|
||||
return Err(SubjektivHostError::Conflict { code: "journal_outcome_unconfirmed".into(), message: "Subject has queued/running or Unknown/unacknowledged delegated work; binding retained".into() });
|
||||
}
|
||||
let released = self
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&self.writer)
|
||||
.map_err(journal_error);
|
||||
if released.is_ok()
|
||||
|| !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.unwrap_or(false)
|
||||
{
|
||||
job_closure.commit();
|
||||
}
|
||||
released
|
||||
}
|
||||
|
||||
fn journal_session_erased(&self, session: &str) -> Result<bool, SubjektivHostError> {
|
||||
let context = authority(self, session)?;
|
||||
self.catalog
|
||||
@@ -425,6 +488,11 @@ impl LocalSubjectHost {
|
||||
// Old Workers' retained sources are reconciled as well; possession of the
|
||||
// Subject lease alone does not bypass the original Worker binding checks.
|
||||
if attribution.runtime_id == self.binding.scope_id
|
||||
&& self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &attribution.session_id)
|
||||
.map_err(journal_error)?
|
||||
&& let Err(error) = self.reconcile_journal_session(&attribution.session_id)
|
||||
{
|
||||
eprintln!(
|
||||
@@ -519,6 +587,7 @@ impl LocalSubjectHost {
|
||||
fn next_journal_job(
|
||||
&self,
|
||||
) -> Result<Option<(crate::jobs::StandaloneJobs, String)>, SubjektivHostError> {
|
||||
self.ensure_live()?;
|
||||
let jobs = self.jobs()?;
|
||||
for saved in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if self.journal_grant(&saved).is_ok() {
|
||||
@@ -578,6 +647,14 @@ impl LocalSubjectHost {
|
||||
.list_journals(&reader, after.as_deref(), MAX_JOURNAL_PAGE)
|
||||
.map_err(journal_error)?;
|
||||
for journal in page.items {
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &journal.source_session_id)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let context = authority(self, &journal.source_session_id)?;
|
||||
if let Some(active) = self
|
||||
.catalog
|
||||
@@ -618,6 +695,7 @@ impl LocalSubjectHost {
|
||||
.collect::<Vec<_>>();
|
||||
let origins = self.journal_origins(&ranges)?;
|
||||
let grant = LocalJournalGrant {
|
||||
writer_binding_id: Some(self.writer.binding_id.clone()),
|
||||
binding: self.binding.clone(),
|
||||
session_id: journal.source_session_id,
|
||||
journal_id: journal.id.clone(),
|
||||
@@ -1012,6 +1090,10 @@ impl LocalSubjectHost {
|
||||
})?)
|
||||
.map_err(|error| SubjektivHostError::InvalidArgument(error.to_string()))?;
|
||||
if grant.binding != self.binding
|
||||
|| grant
|
||||
.writer_binding_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &self.writer.binding_id)
|
||||
|| snapshot.request.purpose != PURPOSE
|
||||
|| snapshot.request.serialization_key.as_deref()
|
||||
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|
||||
@@ -1021,6 +1103,24 @@ impl LocalSubjectHost {
|
||||
"Job input/grant does not match this Journal capability".into(),
|
||||
));
|
||||
}
|
||||
// Pre-binding persisted Jobs lack the new field. They may resume only after the
|
||||
// Host has bound their ORIGINAL Session to this lifetime. Session binding is immutable;
|
||||
// missing grant metadata never maps an old Session onto a later body/reconnection.
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &grant.session_id)
|
||||
.map_err(journal_error)?
|
||||
|| !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"Journal Job's original Session is not bound to this active body lifetime".into(),
|
||||
));
|
||||
}
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
let journal = self
|
||||
.catalog
|
||||
@@ -1060,10 +1160,18 @@ impl LocalSubjectHost {
|
||||
"Journal completion binding mismatch".into(),
|
||||
));
|
||||
}
|
||||
self.catalog
|
||||
.store
|
||||
.close_journal_generation(&context, &generation.id, &generation.job)
|
||||
let recovery = JournalJobContext::validated_recovery(
|
||||
&self.catalog.store,
|
||||
&context,
|
||||
&generation.id,
|
||||
generation.job.clone(),
|
||||
)
|
||||
.map_err(journal_error)?;
|
||||
let saved = subjektiv::journal::service::finish(&self.catalog.store, &recovery)
|
||||
.map_err(journal_error)?;
|
||||
if saved != completion {
|
||||
return Err(SubjektivHostError::Conflict { code: "journal_completion_mismatch".into(), message: "Job result differs from the saved Journal activity, position and checkpoint completion".into() });
|
||||
}
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
if let Err(error) = self.prune_retained_workers() {
|
||||
|
||||
@@ -8,6 +8,7 @@ use session_store::{Store, WorkerMetadataStore};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
mod compaction;
|
||||
mod handoff;
|
||||
|
||||
struct Fixture {
|
||||
_temp: tempfile::TempDir,
|
||||
@@ -407,10 +408,19 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
|
||||
JournalAction::ConfirmUnchanged,
|
||||
state.selected_ranges.clone(),
|
||||
);
|
||||
let position = save(
|
||||
&adapter,
|
||||
"position",
|
||||
&nochange.state_token,
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "First prefix checked; last input still pending".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
save(
|
||||
&adapter,
|
||||
"checkpoint",
|
||||
&nochange.state_token,
|
||||
&position.state_token,
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["checked".into()],
|
||||
},
|
||||
@@ -675,6 +685,7 @@ impl agen::llm_client::LlmClient for JournalScript {
|
||||
#[derive(Clone, Default)]
|
||||
struct DialogAndJournal {
|
||||
journal: JournalScript,
|
||||
dialog_requests: Arc<std::sync::Mutex<Vec<agen::llm_client::Request>>>,
|
||||
dialog_calls: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
@@ -730,6 +741,7 @@ impl agen::llm_client::LlmClient for DialogAndJournal {
|
||||
};
|
||||
return Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))));
|
||||
}
|
||||
self.dialog_requests.lock().unwrap().push(request.clone());
|
||||
self.dialog_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Box::pin(futures::stream::iter(
|
||||
[
|
||||
@@ -1000,6 +1012,60 @@ async fn real_job_tools_save_and_drain_last_input_without_another_turn() {
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_pre_binding_job_recovers_only_under_original_session_lifetime_and_never_after_release()
|
||||
{
|
||||
let mut fixture = Fixture::new("pre-upgrade pending Journal");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, modern_id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let saved = jobs.get(&modern_id).unwrap();
|
||||
jobs.cancel(&modern_id).unwrap();
|
||||
let mut request = saved.request;
|
||||
request.job_id = format!("legacy-journal:{}", uuid::Uuid::now_v7());
|
||||
let mut legacy = saved.domain_grant.unwrap();
|
||||
legacy.as_object_mut().unwrap().remove("writer_binding_id");
|
||||
let pending = jobs.request_granted(request, legacy.clone()).unwrap();
|
||||
assert_eq!(
|
||||
jobs.get(&pending.request.job_id).unwrap().domain_grant,
|
||||
Some(legacy)
|
||||
);
|
||||
jobs.start_with_model_client(&pending.request.job_id, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(20),
|
||||
jobs.wait(&pending.request.job_id),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let subject = fixture.connection.binding.subject_id.clone();
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&fixture.host().writer)
|
||||
.unwrap();
|
||||
assert!(fixture.host().journal_grant(&completed).is_err());
|
||||
jobs.shutdown().await.unwrap();
|
||||
fixture.connection.close().unwrap();
|
||||
let mut next = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&subject, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
assert!(
|
||||
next.host.journal_grant(&completed).is_err(),
|
||||
"missing legacy metadata cannot rebind its original Session to the next body"
|
||||
);
|
||||
next.close().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_create() {
|
||||
let mut fixture = Fixture::new("proposed, corrected, partially performed, then interrupted");
|
||||
@@ -1088,51 +1154,38 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopped_worker_removal_retains_originals_and_the_existing_durable_request() {
|
||||
async fn stopped_worker_removal_and_replacement_require_explicit_idle_handoff_even_after_recording()
|
||||
{
|
||||
let mut fixture = Fixture::new("Unrecorded stopped Worker");
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let binding = fixture.connection.binding.clone();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(matches!(
|
||||
workers.load(worker),
|
||||
Err(crate::store::StandaloneStoreError::WorkerNotFound(_))
|
||||
));
|
||||
let retained = retained_workers_root(&state).join(worker.to_string());
|
||||
assert!(workers.delete(worker).is_err());
|
||||
assert!(workers.load(worker).is_ok());
|
||||
assert!(
|
||||
retained
|
||||
.join("sessions")
|
||||
workers
|
||||
.sessions_dir(worker)
|
||||
.join(&fixture.context.session_id)
|
||||
.is_dir()
|
||||
);
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.is_err()
|
||||
);
|
||||
let mut reopened = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.connect(&binding.subject_id, worker)
|
||||
.unwrap();
|
||||
let jobs = crate::jobs::StandaloneJobs::open(&reopened.jobs_path, fixture._temp.path().into())
|
||||
.unwrap();
|
||||
reopened.host.bind_jobs(jobs.clone()).unwrap();
|
||||
assert_eq!(
|
||||
jobs.pending().unwrap().len(),
|
||||
1,
|
||||
"repeated deletion must not multiply requests"
|
||||
);
|
||||
reopened
|
||||
.host
|
||||
.reconcile_journal_session(&fixture.context.session_id)
|
||||
.unwrap();
|
||||
let (_, id) = reopened.host.next_journal_job().unwrap().unwrap();
|
||||
let snapshot = jobs.get(&id).unwrap();
|
||||
assert!(
|
||||
grant(
|
||||
reopened.host.clone(),
|
||||
&snapshot,
|
||||
Arc::new(TestFence(AtomicBool::new(true)))
|
||||
)
|
||||
.is_ok(),
|
||||
"retained originals remain readable through the real grant path"
|
||||
);
|
||||
assert!(
|
||||
retained.is_dir(),
|
||||
"unfinished generation cannot prune original material"
|
||||
);
|
||||
jobs.start_with_model_client(&id, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1146,31 +1199,23 @@ async fn stopped_worker_removal_retains_originals_and_the_existing_durable_reque
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!retained.exists(),
|
||||
"only reflected original material may be pruned"
|
||||
);
|
||||
let context = authority(&reopened.host, &fixture.context.session_id).unwrap();
|
||||
let journal = reopened
|
||||
.host
|
||||
.catalog
|
||||
.store
|
||||
.list_journals(&context, None, 1)
|
||||
.unwrap()
|
||||
.items
|
||||
.pop()
|
||||
.unwrap();
|
||||
let (_, open) = reopened
|
||||
.host
|
||||
.catalog
|
||||
.store
|
||||
.journal_resume_state(&context, &journal.id)
|
||||
.unwrap();
|
||||
assert!(
|
||||
open.unwrap().body.contains("Only part performed"),
|
||||
"Journal remains readable without the removed Worker/Session"
|
||||
reopened
|
||||
.host
|
||||
.session_fully_reflected(&fixture.context.session_id)
|
||||
.unwrap()
|
||||
);
|
||||
jobs.shutdown().await.unwrap();
|
||||
reopened.close().unwrap();
|
||||
assert!(
|
||||
workers.delete(worker).is_err(),
|
||||
"recording completion is not actual Controller Idle/release"
|
||||
);
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1415,8 +1460,12 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
|
||||
.unwrap();
|
||||
let descriptor = old_reader.descriptors().next().unwrap().clone();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(workers.delete(worker).is_err());
|
||||
// Reproduce a T-734 archive already persisted before durable body-release bindings.
|
||||
// New ordinary deletion must not create this state, but its erasure contract remains.
|
||||
let retained = retained_workers_root(&state).join(worker.to_string());
|
||||
fs::create_dir_all(retained.parent().unwrap()).unwrap();
|
||||
fs::rename(workers.root().join(worker.to_string()), &retained).unwrap();
|
||||
let retained_sessions = retained.join("sessions");
|
||||
let retained_store = session_store::FsStore::new_with_retention(
|
||||
&retained_sessions,
|
||||
|
||||
@@ -183,7 +183,7 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
|
||||
retained_tokens: 0,
|
||||
..Default::default()
|
||||
});
|
||||
let host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
|
||||
let mut host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let session = host.record().active_session_id;
|
||||
@@ -422,6 +422,13 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!position.text.is_empty());
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(crate::StandaloneSubjectDisconnectError::Incomplete(message)) if message.contains("dispatcher") || message.contains("cleanup") => tokio::task::yield_now().await,
|
||||
Err(error) => panic!("recorded compaction/cancellation lineage must remain releasable while work is unfinished: {error}"),
|
||||
}
|
||||
}
|
||||
host.shutdown().await.unwrap();
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
//! Real standalone Subject/Controller/Journal Job/initial input connection. Only external
|
||||
//! model streams are scripted. The scenario checks persisted data and ordering, not whether
|
||||
//! a real model understands the correction or can autonomously continue the work.
|
||||
|
||||
use super::*;
|
||||
use crate::{StandaloneHost, StandaloneLaunchConfig, StandaloneSubjectDisconnectError};
|
||||
use agen::llm_client::types::Item;
|
||||
|
||||
fn launch(root: &Path, state: &Path, subject: &str, name: &str) -> crate::ResolvedStandaloneLaunch {
|
||||
StandaloneLaunchConfig::new(
|
||||
root,
|
||||
state,
|
||||
manifest::ProfileSelector::source_named(
|
||||
manifest::ProfileRegistrySource::Builtin,
|
||||
"standalone-subjektiv",
|
||||
),
|
||||
name,
|
||||
)
|
||||
.with_subject(subject.into())
|
||||
.resolve()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recorded_idle_body_handoff_preserves_open_after_source_deletion_and_seeds_next_body_once()
|
||||
{
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("continuity", Some("BEHAVIOR_ORIGINAL_UNCHANGED")).unwrap();
|
||||
let model = DialogAndJournal { journal: JournalScript { gate_first: true, ..Default::default() }, ..Default::default() };
|
||||
let mut first = StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "first"), model.clone()).await.unwrap();
|
||||
let old_worker = first.worker_id();
|
||||
let old_session = first.record().active_session_id;
|
||||
let old_writer = catalog.store.claim_journal_writer(&subject.id, &catalog.scope_id, &old_worker.to_string()).unwrap();
|
||||
submit_dialog(&first, "A was proposed, user corrected its constraint; B selected because A is unusable. One step performed; tests not performed; external outcome Unknown. Work is unfinished.").await;
|
||||
model.journal.entered.notified().await;
|
||||
assert!(matches!(first.disconnect_subject().await, Err(StandaloneSubjectDisconnectError::Incomplete(_))));
|
||||
assert!(catalog.store.journal_writer_active(&old_writer).unwrap());
|
||||
assert!(StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "premature"), DialogAndJournal::default()).await.is_err());
|
||||
let context = JournalHostContext::validated_session(&catalog.store, catalog.store.session_attribution(&old_session.to_string()).unwrap().unwrap()).unwrap();
|
||||
let id = catalog.store.journal_for_session(&context).unwrap().unwrap();
|
||||
let generation = catalog.store.active_journal_generation(&context, &id).unwrap().unwrap();
|
||||
model.journal.release.notify_one();
|
||||
first.jobs().wait(&generation.job.job_id).await.unwrap();
|
||||
// Observe the authoritative gate after cleanup; yielding is cooperative scheduling,
|
||||
// never elapsed time used as proof of recording completion.
|
||||
loop {
|
||||
match first.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(message)) if message.contains("dispatcher") || message.contains("cleanup") => tokio::task::yield_now().await,
|
||||
Err(error) => panic!("unexpected handoff blocker: {error}"),
|
||||
}
|
||||
}
|
||||
assert!(!catalog.store.journal_writer_active(&old_writer).unwrap());
|
||||
assert!(catalog.store.accept_journal_sources(&context, "late old activity", &[generation.selected_ranges[0].clone()]).is_err());
|
||||
first.shutdown().await.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
workers.delete(old_worker).unwrap();
|
||||
assert!(!workers.sessions_dir(old_worker).exists());
|
||||
let reader = JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap();
|
||||
let historical = catalog.store.journal_startup_snapshot(&reader, Some(&old_session.to_string())).unwrap();
|
||||
assert_eq!(historical.journals[0].entries[0].state, JournalEntryState::Open);
|
||||
assert!(historical.journals[0].entries[0].ongoing.contains("unknown"));
|
||||
assert!(catalog.store.list_memories(&subject.id).unwrap().is_empty());
|
||||
|
||||
let second_model = DialogAndJournal::default();
|
||||
let second = StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "second"), second_model.clone()).await.unwrap();
|
||||
submit_dialog(&second, "Continue B; describe what is still unperformed and unconfirmed.").await;
|
||||
let requests = second_model.dialog_requests.lock().unwrap();
|
||||
let initial = &requests[0];
|
||||
let context_index = initial.items.iter().position(|item| item.as_text().is_some_and(|text| text.contains("Proposed A; user corrected it to B"))).expect("saved Journal visible to first model request");
|
||||
let user_index = initial.items.iter().position(|item| matches!(item, Item::Message { role: agen::llm_client::types::Role::User, .. })).unwrap();
|
||||
assert!(context_index < user_index);
|
||||
assert_eq!(initial.items.iter().filter(|item| item.as_text().is_some_and(|text| text.contains("Proposed A; user corrected it to B"))).count(), 1);
|
||||
drop(requests);
|
||||
let session = second.record().active_session_id;
|
||||
let sessions = session_store::FsStore::new_with_retention(workers.sessions_dir(second.worker_id()), crate::store::session_retention(&state)).unwrap();
|
||||
let pointer = session_store::FsWorkerStore::new(workers.worker_metadata_dir(second.worker_id())).unwrap().read_by_name(&second.record().storage_key).unwrap().unwrap().active.unwrap();
|
||||
let entries = sessions.read_all(session, pointer.segment_id.unwrap()).unwrap();
|
||||
let journal_items = entries.iter().filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, session_store::SystemItem::JournalStartup { .. }))).count();
|
||||
assert_eq!(journal_items, 1);
|
||||
assert_eq!(catalog.store.subject(&subject.id).unwrap().unwrap().behavior_md, "BEHAVIOR_ORIGINAL_UNCHANGED");
|
||||
second.shutdown().await.unwrap();
|
||||
}).await.expect("bounded complete Subject handoff connection");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saved_public_history_without_admission_notification_blocks_release_until_actual_recording()
|
||||
{
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("saved-before-notified", None).unwrap();
|
||||
let model = DialogAndJournal {
|
||||
journal: JournalScript {
|
||||
gate_first: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "gap"),
|
||||
model.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
let record = host.record().clone();
|
||||
let writer = catalog
|
||||
.store
|
||||
.claim_journal_writer(
|
||||
&subject.id,
|
||||
&catalog.scope_id,
|
||||
&host.worker_id().to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let sessions = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(host.worker_id()),
|
||||
crate::store::session_retention(&state),
|
||||
)
|
||||
.unwrap();
|
||||
// The production saved-history boundary, with its admission callback deliberately
|
||||
// omitted. The real Host release must discover these bytes, not trust Job status.
|
||||
sessions
|
||||
.append(
|
||||
record.active_session_id,
|
||||
record.active_segment_id.unwrap(),
|
||||
&session_store::LogEntry::AnnotatedUserInput {
|
||||
ts: 3,
|
||||
segments: vec![],
|
||||
extensions: vec![],
|
||||
history: vec![message(
|
||||
"missed-admission",
|
||||
"A corrected to B, partially performed, outcome Unknown",
|
||||
)],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(host.disconnect_subject().await.is_err());
|
||||
assert!(catalog.store.journal_writer_active(&writer).unwrap());
|
||||
model.journal.entered.notified().await;
|
||||
let context = JournalHostContext::validated_session(
|
||||
&catalog.store,
|
||||
catalog
|
||||
.store
|
||||
.session_attribution(&record.active_session_id.to_string())
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let id = catalog
|
||||
.store
|
||||
.journal_for_session(&context)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let generation = catalog
|
||||
.store
|
||||
.active_journal_generation(&context, &id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
model.journal.release.notify_one();
|
||||
host.jobs().wait(&generation.job.job_id).await.unwrap();
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(message))
|
||||
if message.contains("dispatcher") || message.contains("cleanup") =>
|
||||
{
|
||||
tokio::task::yield_now().await
|
||||
}
|
||||
Err(error) => {
|
||||
panic!("unexpected release blocker after saved-history recording: {error}")
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(!catalog.store.journal_writer_active(&writer).unwrap());
|
||||
let snapshot = catalog
|
||||
.store
|
||||
.journal_startup_snapshot(
|
||||
&JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!snapshot.journals[0].current_position.is_empty());
|
||||
assert_eq!(
|
||||
snapshot.journals[0].entries[0].state,
|
||||
JournalEntryState::Open
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
})
|
||||
.await
|
||||
.expect("saved-history/notification-gap handoff is bounded");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_and_normal_deletion_cannot_release_subject_occupancy() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("retained-owner", None).unwrap();
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "retained"),
|
||||
DialogAndJournal::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker = host.worker_id();
|
||||
host.shutdown().await.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
assert!(workers.delete(worker).is_err());
|
||||
assert!(
|
||||
StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "not-an-owner"),
|
||||
DialogAndJournal::default()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut restored =
|
||||
StandaloneHost::restore_with_model_client(state, worker, DialogAndJournal::default())
|
||||
.await
|
||||
.unwrap();
|
||||
restored.disconnect_subject().await.unwrap();
|
||||
restored.shutdown().await.unwrap();
|
||||
}
|
||||
@@ -37,9 +37,8 @@ fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
|
||||
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
|
||||
let a = catalog.create("a", None).unwrap();
|
||||
let b = catalog.create("b", None).unwrap();
|
||||
let mut first = catalog
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
let first_id = protocol::WorkerId::now_v7();
|
||||
let mut first = catalog.connect(&a.id, first_id).unwrap();
|
||||
assert!(matches!(
|
||||
StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
@@ -51,11 +50,18 @@ fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
|
||||
.connect(&b.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
first.close().unwrap();
|
||||
let mut replacement = StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.is_err(),
|
||||
"kernel lease release is not durable body release"
|
||||
);
|
||||
let mut resumed = StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.connect(&a.id, first_id)
|
||||
.unwrap();
|
||||
replacement.close().unwrap();
|
||||
resumed.close().unwrap();
|
||||
second.close().unwrap();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! T-709: real standalone Hosts and test-owned processes, with scripted models.
|
||||
//! These tests do not create a Workspace or contact a model provider.
|
||||
//! Integration of real standalone Subject lifecycle, dialog, Journal and Memory services.
|
||||
//! Only model streams are scripted; temporary storage and test-owned subprocesses isolate
|
||||
//! persistence and kernel leases. Run with `cargo test -p standalone --test subjektiv`.
|
||||
//! No Workspace or live model provider is used; this is not a continuity meaning evaluation.
|
||||
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::io::{Read, Write};
|
||||
@@ -354,17 +356,16 @@ impl LlmClient for ScriptedClient {
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
// This suite observes Memory and dialog requests. Journal's independent model
|
||||
// boundary fails explicitly rather than consuming their script or using live transport;
|
||||
// the Journal adapter suite exercises its successful sequential tool execution.
|
||||
// Journal is a separate scripted model boundary. Successful production recording
|
||||
// is required before these Memory scenarios may hand the Subject to a new body.
|
||||
if request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalState")
|
||||
{
|
||||
return Err(ClientError::Config(
|
||||
"Journal model is outside this Memory suite".into(),
|
||||
));
|
||||
return Ok(Box::pin(stream::iter(
|
||||
journal_response(&request).into_iter().map(Ok),
|
||||
)));
|
||||
}
|
||||
// Lifecycle-triggered consolidation uses a separate Internal Worker. It
|
||||
// must not consume the dialog script or silently become a live request.
|
||||
@@ -423,6 +424,134 @@ impl LlmClient for ScriptedClient {
|
||||
}
|
||||
}
|
||||
|
||||
// The outer model script is not a continuity meaning evaluation. Persistence, complete
|
||||
// source delivery and the handoff gate remain the production implementations.
|
||||
fn journal_response(request: &Request) -> Vec<LlmEvent> {
|
||||
let result = |id: &str| -> Option<Value> {
|
||||
request.items.iter().find_map(|item| match item {
|
||||
Item::ToolResult {
|
||||
call_id,
|
||||
content,
|
||||
is_error,
|
||||
summary,
|
||||
..
|
||||
} if call_id == id => {
|
||||
assert!(!is_error, "{id}: {summary}; {content:?}");
|
||||
Some(
|
||||
content
|
||||
.as_deref()
|
||||
.map(|v| serde_json::from_str(v).unwrap())
|
||||
.unwrap_or(Value::Null),
|
||||
)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
if result("journal-submit").is_some() {
|
||||
return prose();
|
||||
}
|
||||
if result("journal-checkpoint").is_some() {
|
||||
return tool("journal-submit", "SubmitJobResult", json!({"result":{}}));
|
||||
}
|
||||
let Some(state) = result("journal-state") else {
|
||||
return tool("journal-state", "ReadJournalState", json!({}));
|
||||
};
|
||||
let state = &state["value"];
|
||||
let generation = state["generation_id"].as_str().unwrap();
|
||||
if let Some(position) = result("journal-position") {
|
||||
return tool(
|
||||
"journal-checkpoint",
|
||||
"CheckpointJournal",
|
||||
json!({
|
||||
"request_id":format!("checkpoint:{generation}"), "expected_state":position["value"]["state_token"],
|
||||
"action":{"action":"checkpoint","record_operation_ids":[format!("record:{generation}")]},
|
||||
"covered_ranges":state["selected_ranges"]
|
||||
}),
|
||||
);
|
||||
}
|
||||
if let Some(record) = result("journal-record") {
|
||||
return tool(
|
||||
"journal-position",
|
||||
"UpdateJournalPosition",
|
||||
json!({
|
||||
"request_id":format!("position:{generation}"), "expected_state":record["value"]["state_token"],
|
||||
"action":{"action":"update_position","current_position":"Memory scenario recorded; represented work may remain unfinished"}, "covered_ranges":[]
|
||||
}),
|
||||
);
|
||||
}
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("journal-source-{index}-{offset}");
|
||||
match result(&id) {
|
||||
Some(page) => match page["value"]["next_byte"].as_u64() {
|
||||
Some(next) => {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["value"]["content_token"].clone();
|
||||
}
|
||||
None => break,
|
||||
},
|
||||
None => {
|
||||
return tool(
|
||||
&id,
|
||||
"ReadJournalText",
|
||||
json!({
|
||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
||||
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let (name, action) = if state["current_open"].is_null() {
|
||||
(
|
||||
"CreateJournalEntry",
|
||||
json!({"action":"create","title":"Memory conversation","body":"Conversation recorded by the scripted model fixture","ongoing":"Unfinished work is not declared complete","state":"open","source_end":null,"correction_refs":[]}),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"RewriteJournalEntry",
|
||||
json!({"action":"rewrite","entry_id":state["current_open"]["id"],"title":"Memory conversation","body":"Further conversation recorded by the scripted model fixture","ongoing":"Unfinished work is not declared complete"}),
|
||||
)
|
||||
};
|
||||
tool(
|
||||
"journal-record",
|
||||
name,
|
||||
json!({
|
||||
"request_id":format!("record:{generation}"), "expected_state":state["current_journal"]["state_token"],
|
||||
"action":action,"covered_ranges":state["selected_ranges"]
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
async fn disconnect_recorded(host: &mut StandaloneHost) {
|
||||
let mut last_error = None;
|
||||
let completed = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => return,
|
||||
Err(error) => {
|
||||
last_error = Some(error.to_string());
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
completed.is_ok(),
|
||||
"actual Idle/Journal completion is required before Memory scenario handoff: {last_error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
fn prose() -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
@@ -762,7 +891,7 @@ async fn explicit_remember_is_committed_staging_and_sessions_follow_subject_acro
|
||||
tool("query-restored", "SubjektivMemoryQuery", json!({})),
|
||||
prose(),
|
||||
]);
|
||||
let restored =
|
||||
let mut restored =
|
||||
StandaloneHost::restore_with_model_client(state.clone(), worker_id, restored_model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -773,6 +902,7 @@ async fn explicit_remember_is_committed_staging_and_sessions_follow_subject_acro
|
||||
assert!(results["recall"].to_string().contains(claim));
|
||||
assert_eq!(results["query-restored"]["items"], json!([]));
|
||||
assert!(format!("{:?}", restored_model.requests()[0].items).contains("Remember this lesson"));
|
||||
disconnect_recorded(&mut restored).await;
|
||||
restored.shutdown().await.unwrap();
|
||||
|
||||
let second_model = ScriptedClient::new(vec![
|
||||
@@ -992,7 +1122,7 @@ async fn kernel_subject_lease_excludes_other_processes_and_recovers_after_abnorm
|
||||
let independent = catalog.create("independent", None).unwrap();
|
||||
drop(catalog);
|
||||
let ready = temp.path().join("ready");
|
||||
let parent = StandaloneHost::start_with_model_client(
|
||||
let mut parent = StandaloneHost::start_with_model_client(
|
||||
launch(&parent_cwd, &state, &subject.id, "parent-owner"),
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
@@ -1014,6 +1144,7 @@ async fn kernel_subject_lease_excludes_other_processes_and_recovers_after_abnorm
|
||||
.await
|
||||
.unwrap();
|
||||
independent_host.shutdown().await.unwrap();
|
||||
disconnect_recorded(&mut parent).await;
|
||||
parent.shutdown().await.unwrap();
|
||||
|
||||
let mut owner = OwnedChild(
|
||||
@@ -1082,7 +1213,7 @@ async fn automatic_generic_consolidation_applies_candidate_publishes_surface_and
|
||||
drop(catalog);
|
||||
let claim = "Validate the immutable result before reporting success";
|
||||
let model = ScriptedClient::consolidating(remember_script(claim), false);
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "consolidation-body"),
|
||||
model.clone(),
|
||||
)
|
||||
@@ -1157,6 +1288,7 @@ async fn automatic_generic_consolidation_applies_candidate_publishes_surface_and
|
||||
.exists(),
|
||||
"selected Subject owns the shared generic Job ledger"
|
||||
);
|
||||
disconnect_recorded(&mut host).await;
|
||||
host.shutdown().await.unwrap();
|
||||
let memory = inspect_store(&state, |store| {
|
||||
assert!(
|
||||
@@ -1236,7 +1368,7 @@ async fn transport_loss_after_candidate_apply_preserves_memory_and_later_job_onl
|
||||
drop(catalog);
|
||||
let claim = "Candidate receipts survive transport failure";
|
||||
let losing = ScriptedClient::consolidating(remember_script(claim), true);
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "losing-body"),
|
||||
losing.clone(),
|
||||
)
|
||||
@@ -1261,6 +1393,7 @@ async fn transport_loss_after_candidate_apply_preserves_memory_and_later_job_onl
|
||||
failed.result.is_none(),
|
||||
"partial apply is not an accepted Job result"
|
||||
);
|
||||
disconnect_recorded(&mut host).await;
|
||||
host.shutdown().await.unwrap();
|
||||
let original = inspect_store(&state, |store| {
|
||||
assert!(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
Shared Host domain implementation for per-Subject Memory, change history, candidate dispositions, derived surfaces, and Session activity Journals.
|
||||
`feature-storage` manages physical SQLite storage; the calling Host handles authentication and execution authority.
|
||||
Journal storage does not generate prose, launch Jobs, or fetch Session history.
|
||||
Hosts retain body ownership across inactivity, reserve actual execution quiescence, and reconcile final committed history before releasing a Journal writer lifetime. [`journal::binding`] fences new mutations inside storage transactions; historical attribution and exact receipt recovery cannot renew a released lifetime. [`journal::startup`] selects bounded reference context independently of source retention and long-term Memory.
|
||||
|
||||
## Main entry points
|
||||
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
//! Durable Journal writer lifetime, issued only from the Host's body binding. Kernel leases
|
||||
//! and Runtime liveness do not end ownership. The immutable binding ID fences retained contexts
|
||||
//! even when the same Worker later reconnects. Hosts serialize release with body lifecycle and
|
||||
//! reconcile final committed history before calling release; this store checks saved effects.
|
||||
|
||||
use super::*;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JournalWriterBinding {
|
||||
pub subject_id: String,
|
||||
pub runtime_id: String,
|
||||
pub worker_id: String,
|
||||
pub binding_id: String,
|
||||
}
|
||||
|
||||
pub(crate) fn migrate(tx: &Transaction<'_>) -> Result<(), FeatureStorageError> {
|
||||
tx.execute_batch("CREATE TABLE journal_writer_bindings (
|
||||
binding_id TEXT PRIMARY KEY,
|
||||
subject_id TEXT NOT NULL REFERENCES subjects(subject_id) ON DELETE RESTRICT,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
active INTEGER NOT NULL CHECK(active IN (0,1))
|
||||
);
|
||||
CREATE UNIQUE INDEX journal_one_body_binding ON journal_writer_bindings(subject_id) WHERE active=1;
|
||||
CREATE TABLE journal_session_bindings (
|
||||
session_id TEXT PRIMARY KEY REFERENCES subject_session_attributions(session_id) ON DELETE RESTRICT,
|
||||
binding_id TEXT NOT NULL REFERENCES journal_writer_bindings(binding_id) ON DELETE RESTRICT
|
||||
);
|
||||
CREATE INDEX journal_body_history ON journal_writer_bindings(subject_id,active);
|
||||
CREATE INDEX journal_body_sessions ON journal_session_bindings(binding_id);
|
||||
CREATE INDEX journal_recent_subject ON journals(subject_id,updated_at DESC,id DESC);")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn denied() -> JournalError {
|
||||
JournalError::PermissionDenied("Journal writer binding is absent, released or replaced".into())
|
||||
}
|
||||
|
||||
pub(super) fn check_writer(tx: &Connection, host: &JournalHostContext) -> JournalResult<()> {
|
||||
let attr = host.attribution.as_ref().ok_or_else(denied)?;
|
||||
let binding = host.writer_binding_id.as_ref().ok_or_else(denied)?;
|
||||
if !tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings b JOIN journal_session_bindings s ON s.binding_id=b.binding_id WHERE b.binding_id=?1 AND b.subject_id=?2 AND b.runtime_id=?3 AND b.worker_id=?4 AND b.active=1 AND s.session_id=?5)", params![binding,host.subject_id,attr.runtime_id,attr.worker_id,attr.session_id], |r| r.get::<_,bool>(0))? {
|
||||
return Err(denied());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn require_saved_position(tx: &Connection, generation: &str) -> JournalResult<()> {
|
||||
let saved: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_operations p WHERE p.generation_id=?1 AND json_extract(p.request_json,'$.action.action')='update_position' AND p.rowid > COALESCE((SELECT MAX(r.rowid) FROM journal_operations r WHERE r.generation_id=?1 AND json_extract(r.request_json,'$.action.action') IN ('create','rewrite','fix','confirm_unchanged')),0))",[generation],|r|r.get(0))?;
|
||||
if !saved {
|
||||
return Err(JournalError::Conflict("Journal current position must be saved after the generation's activity records before completion".into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl SubjektivStore {
|
||||
/// The last released body's most recently updated Journal Session, independent of source
|
||||
/// retention. Session attachment order may differ from the order of recorded activity.
|
||||
pub fn journal_predecessor_session(
|
||||
&self,
|
||||
binding: &JournalWriterBinding,
|
||||
) -> JournalResult<Option<String>> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.query_row("SELECT j.source_session_id FROM journals j JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE j.subject_id=?1 AND s.binding_id=(SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=0 ORDER BY rowid DESC LIMIT 1) ORDER BY j.updated_at DESC,j.id DESC LIMIT 1", [&binding.subject_id], |r| r.get(0)).optional()?)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn journal_worker_was_released(
|
||||
&self,
|
||||
subject: &str,
|
||||
runtime: &str,
|
||||
worker: &str,
|
||||
) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 AND active=0)", params![subject,runtime,worker], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn journal_writer_active(&self, binding: &JournalWriterBinding) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4 AND active=1)", params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn journal_session_writer_matches(
|
||||
&self,
|
||||
binding: &JournalWriterBinding,
|
||||
session: &str,
|
||||
) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_session_bindings WHERE binding_id=?1 AND session_id=?2)", params![binding.binding_id,session], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
|
||||
/// Called under the existing Host Subject lifecycle authority. A stopped or unavailable
|
||||
/// owner remains the owner; only the exact same body may resume an unreleased binding.
|
||||
pub fn claim_journal_writer(
|
||||
&self,
|
||||
subject: &str,
|
||||
runtime: &str,
|
||||
worker: &str,
|
||||
) -> JournalResult<JournalWriterBinding> {
|
||||
for identifier in [subject, runtime, worker] {
|
||||
if identifier.is_empty()
|
||||
|| identifier.len() > 128
|
||||
|| identifier.trim() != identifier
|
||||
|| identifier.chars().any(char::is_control)
|
||||
{
|
||||
return Err(JournalError::InvalidInput(
|
||||
"invalid Journal binding identity".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
self.database.try_transaction(|tx| {
|
||||
let state: Option<String> = tx.query_row("SELECT state FROM subjects WHERE subject_id=?1", [subject], |r| r.get(0)).optional()?;
|
||||
if state.as_deref() != Some("active") { return Err(JournalError::PermissionDenied("Subject is absent or retired".into())); }
|
||||
let current: Option<JournalWriterBinding> = tx.query_row("SELECT subject_id,runtime_id,worker_id,binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=1", [subject], |r| Ok(JournalWriterBinding { subject_id:r.get(0)?,runtime_id:r.get(1)?,worker_id:r.get(2)?,binding_id:r.get(3)? })).optional()?;
|
||||
if let Some(current) = current {
|
||||
if current.runtime_id != runtime || current.worker_id != worker {
|
||||
return Err(JournalError::Conflict("Subject is still occupied; explicit Idle and Journal-complete release required".into()));
|
||||
}
|
||||
return Ok(current);
|
||||
}
|
||||
// Upgraded T-734 data can contain a removed body's durable recording handoff.
|
||||
// Retention is not permission for a different body to occupy that Subject.
|
||||
let legacy_busy: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE j.subject_id=?1 AND (a.runtime_id<>?2 OR a.worker_id<>?3) AND NOT EXISTS(SELECT 1 FROM journal_session_bindings b WHERE b.session_id=j.source_session_id) AND (EXISTS(SELECT 1 FROM journal_generations g WHERE g.journal_id=j.id AND g.active=1) OR EXISTS(SELECT 1 FROM journal_sources s,json_each(s.accepted_json) r WHERE s.journal_id=j.id AND json_extract(r.value,'$.end')>s.reflected_bytes)))", params![subject,runtime,worker], |r| r.get(0))?;
|
||||
if legacy_busy { return Err(JournalError::Conflict("previous body's retained Journal handoff is not complete".into())); }
|
||||
let binding = JournalWriterBinding { subject_id:subject.into(),runtime_id:runtime.into(),worker_id:worker.into(),binding_id:crate::issued_id("journal-binding") };
|
||||
tx.execute("INSERT INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,1)", params![binding.binding_id,subject,runtime,worker])?;
|
||||
Ok(binding)
|
||||
})
|
||||
}
|
||||
|
||||
/// Binds a new Session once. Reconnecting a Worker under a later binding cannot acquire
|
||||
/// mutation authority for its old Session. Attribution and binding are Host-only input.
|
||||
pub fn bind_journal_session(
|
||||
&self,
|
||||
binding: &JournalWriterBinding,
|
||||
attribution: SubjectSessionAttribution,
|
||||
) -> JournalResult<JournalHostContext> {
|
||||
let mut host = JournalHostContext::validated_session(self, attribution)?;
|
||||
self.database.try_transaction(|tx| {
|
||||
let attr = host.attribution.as_ref().expect("validated Session");
|
||||
if binding.subject_id != attr.subject_id || binding.runtime_id != attr.runtime_id || binding.worker_id != attr.worker_id {
|
||||
return Err(denied());
|
||||
}
|
||||
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT ?1,binding_id FROM journal_writer_bindings WHERE binding_id=?2 AND subject_id=?3 AND runtime_id=?4 AND worker_id=?5 AND active=1",params![attr.session_id,binding.binding_id,attr.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
host.writer_binding_id = Some(binding.binding_id.clone());
|
||||
check_writer(tx,&host)?;
|
||||
Ok(host.clone())
|
||||
})
|
||||
}
|
||||
|
||||
/// Destructive release is distinct from a continuity handoff: every attributed Session
|
||||
/// of this body must already have a permanent erasure tombstone. This also records a
|
||||
/// released lifetime for pre-binding bodies, without granting them new mutation authority.
|
||||
pub fn release_erased_journal_body(&self, host: &JournalHostContext) -> JournalResult<bool> {
|
||||
self.database.try_transaction(|tx| {
|
||||
super::store::check_session_host(self, tx, host)?;
|
||||
let attr = host.attribution.as_ref().expect("checked Session authority");
|
||||
let remaining: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_session_attributions a WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3 AND NOT EXISTS(SELECT 1 FROM journal_erased_sessions e WHERE e.session_id=a.session_id AND e.subject_id=a.subject_id))", params![host.subject_id,attr.runtime_id,attr.worker_id], |r| r.get(0))?;
|
||||
if remaining {
|
||||
return Ok(false);
|
||||
}
|
||||
let existing: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3)", params![host.subject_id,attr.runtime_id,attr.worker_id], |r| r.get(0))?;
|
||||
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
if !existing {
|
||||
tx.execute("INSERT INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)", params![crate::issued_id("journal-erased-body"),host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
}
|
||||
Ok(true)
|
||||
})
|
||||
}
|
||||
|
||||
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
||||
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
||||
/// This transaction rejects retained writers, pending source bytes and active generations.
|
||||
/// It never fixes an open entry or declares the represented work complete.
|
||||
pub fn release_journal_writer(&self, binding: &JournalWriterBinding) -> JournalResult<()> {
|
||||
self.database.try_transaction(|tx| {
|
||||
let active: Option<bool> = tx.query_row("SELECT active FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0)).optional()?;
|
||||
let Some(active) = active else { return Err(denied()); };
|
||||
if !active { return Ok(()); }
|
||||
let unbound: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_session_attributions a WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3 AND NOT EXISTS(SELECT 1 FROM journal_session_bindings s WHERE s.session_id=a.session_id) AND NOT EXISTS(SELECT 1 FROM journal_erased_sessions e WHERE e.session_id=a.session_id AND e.subject_id=a.subject_id))",params![binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0))?;
|
||||
if unbound { return Err(JournalError::Conflict("body has legacy Sessions whose committed history has not been reconciled under its writer binding".into())); }
|
||||
let busy: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_generations g JOIN journals j ON j.id=g.journal_id JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE s.binding_id=?1 AND g.active=1)",[&binding.binding_id],|r|r.get(0))?;
|
||||
if busy { return Err(JournalError::Conflict("Journal generation remains active or its result is unconfirmed".into())); }
|
||||
let mut stmt = tx.prepare("SELECT s.accepted_json,s.reflected_bytes,s.unavailable_json FROM journal_sources s JOIN journals j ON j.id=s.journal_id JOIN journal_session_bindings b ON b.session_id=j.source_session_id WHERE b.binding_id=?1")?;
|
||||
let mut rows = stmt.query([&binding.binding_id])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let accepted: Vec<JournalByteRange> = serde_json::from_str(&row.get::<_,String>(0)?)?;
|
||||
let reflected: u64 = row.get(1)?;
|
||||
if accepted.iter().any(|r| r.end > reflected) {
|
||||
return Err(JournalError::Conflict("Journal activity/position/checkpoint has unreflected source bytes".into()));
|
||||
}
|
||||
}
|
||||
let mut checkpoints = tx.prepare("SELECT p.generation_id FROM journal_operations p JOIN journals j ON j.id=p.journal_id JOIN journal_session_bindings b ON b.session_id=j.source_session_id WHERE b.binding_id=?1 AND p.rowid=(SELECT MAX(c.rowid) FROM journal_operations c WHERE c.journal_id=j.id AND json_extract(c.request_json,'$.action.action')='checkpoint')")?;
|
||||
let generations = checkpoints.query_map([&binding.binding_id], |r| r.get::<_,String>(0))?;
|
||||
for generation in generations { require_saved_position(tx, &generation?)?; }
|
||||
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE binding_id=?1 AND active=1",[&binding.binding_id])?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -37,10 +37,14 @@
|
||||
//! Hosts resuming at the processing cursor can use a bounded source point lookup instead of
|
||||
//! traversing already reflected pages. Receipt/entry keysets remain append-stable.
|
||||
|
||||
pub mod binding;
|
||||
pub use binding::JournalWriterBinding;
|
||||
pub(crate) use binding::migrate as migrate_binding;
|
||||
mod lifecycle;
|
||||
pub(crate) use lifecycle::migrate as migrate_lifecycle;
|
||||
pub mod recording;
|
||||
pub mod service;
|
||||
pub mod startup;
|
||||
mod store;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -110,8 +114,14 @@ pub struct JournalHostContext {
|
||||
scope_id: String,
|
||||
subject_id: String,
|
||||
attribution: Option<SubjectSessionAttribution>,
|
||||
writer_binding_id: Option<String>,
|
||||
}
|
||||
impl JournalHostContext {
|
||||
/// Evidence for a retained Job grant, not permission to mint or renew a binding.
|
||||
pub fn writer_binding_id(&self) -> Option<&str> {
|
||||
self.writer_binding_id.as_deref()
|
||||
}
|
||||
|
||||
pub fn validated_reader(store: &SubjektivStore, subject_id: &str) -> JournalResult<Self> {
|
||||
store
|
||||
.subject(subject_id)?
|
||||
@@ -120,6 +130,7 @@ impl JournalHostContext {
|
||||
scope_id: store.scope_id().into(),
|
||||
subject_id: subject_id.into(),
|
||||
attribution: None,
|
||||
writer_binding_id: None,
|
||||
})
|
||||
}
|
||||
/// External authentication, public-history validation and execution authority are Host duties.
|
||||
@@ -133,6 +144,17 @@ impl JournalHostContext {
|
||||
));
|
||||
}
|
||||
let mut context = Self::validated_reader(store, &attribution.subject_id)?;
|
||||
use rusqlite::OptionalExtension;
|
||||
context.writer_binding_id = store.database.try_with_connection(|tx| {
|
||||
Ok::<_, JournalError>(
|
||||
tx.query_row(
|
||||
"SELECT binding_id FROM journal_session_bindings WHERE session_id=?1",
|
||||
[&attribution.session_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?,
|
||||
)
|
||||
})?;
|
||||
context.attribution = Some(attribution);
|
||||
Ok(context)
|
||||
}
|
||||
|
||||
@@ -386,6 +386,7 @@ pub fn finish(
|
||||
return Ok(serde_json::from_str(&saved)?);
|
||||
}
|
||||
super::store::job(store, tx, context, true)?;
|
||||
super::binding::require_saved_position(tx, &generation.id)?;
|
||||
let last = generation
|
||||
.selected_ranges
|
||||
.last()
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
//! Read-only, bounded Journal context for a new body Session. Selection and text projection
|
||||
//! share one storage transaction; no source history, Job, admission or checkpoint is consulted.
|
||||
//! This is derived reference material, never a new observation or an execution capability.
|
||||
|
||||
use super::*;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
pub const STARTUP_MAX_BYTES: usize = 32 * 1024;
|
||||
const JOURNALS: usize = 3;
|
||||
const ENTRIES: usize = 3;
|
||||
const TEXT_BYTES: usize = 2048;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JournalStartupSnapshot {
|
||||
pub subject_id: String,
|
||||
pub journals: Vec<StartupJournal>,
|
||||
pub omitted: bool,
|
||||
}
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StartupJournal {
|
||||
pub journal_id: String,
|
||||
pub source_session_id: String,
|
||||
pub state_token: String,
|
||||
pub process_order: u64,
|
||||
pub process_byte: u64,
|
||||
pub current_position: String,
|
||||
pub entries: Vec<StartupEntry>,
|
||||
pub omitted: bool,
|
||||
}
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StartupEntry {
|
||||
pub entry_id: String,
|
||||
pub position: u64,
|
||||
pub state: JournalEntryState,
|
||||
pub title: String,
|
||||
pub body: String,
|
||||
pub ongoing: String,
|
||||
pub source_end: Option<JournalSourcePointer>,
|
||||
pub correction_refs: Vec<String>,
|
||||
pub omitted: bool,
|
||||
}
|
||||
|
||||
// Read a bounded UTF-8 prefix at SQLite, not a full megabyte field followed by truncation.
|
||||
fn text(tx: &Connection, table: &str, id: &str, field: &str) -> JournalResult<(String, bool)> {
|
||||
let (mut bytes, length): (Vec<u8>, usize) = tx.query_row(
|
||||
&format!("SELECT COALESCE(substr(CAST({field} AS BLOB),1,?2),X''),length(CAST({field} AS BLOB)) FROM {table} WHERE id=?1"),
|
||||
params![id, TEXT_BYTES],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
let length_read = bytes.len();
|
||||
match std::str::from_utf8(&bytes) {
|
||||
Ok(_) => {}
|
||||
Err(error) if error.error_len().is_none() => bytes.truncate(error.valid_up_to()),
|
||||
Err(_) => {
|
||||
return Err(JournalError::Conflict(
|
||||
"stored Journal text is not UTF-8".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let omitted = bytes.len() < length || length_read < length;
|
||||
Ok((String::from_utf8(bytes).expect("validated UTF-8"), omitted))
|
||||
}
|
||||
|
||||
impl SubjektivStore {
|
||||
/// `previous_session` comes from the Host's retained predecessor binding, not model input.
|
||||
/// Recent Subject-local Journals supplement that predecessor under independent fixed budgets.
|
||||
pub fn journal_startup_snapshot(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
previous_session: Option<&str>,
|
||||
) -> JournalResult<JournalStartupSnapshot> {
|
||||
self.database.try_transaction(|tx| {
|
||||
super::store::check_host(self, tx, host, false)?;
|
||||
let mut ids = Vec::new();
|
||||
if let Some(session) = previous_session {
|
||||
let previous: Option<(String, String)> = tx.query_row(
|
||||
"SELECT id,subject_id FROM journals WHERE source_session_id=?1",
|
||||
[session], |r| Ok((r.get(0)?, r.get(1)?)),
|
||||
).optional()?;
|
||||
if let Some((id, subject)) = previous {
|
||||
if subject != host.subject_id { return Err(JournalError::PermissionDenied("predecessor Journal belongs to another Subject".into())); }
|
||||
ids.push(id);
|
||||
}
|
||||
}
|
||||
let mut query = tx.prepare("SELECT id FROM journals WHERE subject_id=?1 ORDER BY updated_at DESC,id DESC LIMIT ?2")?;
|
||||
let recent = query.query_map(params![host.subject_id, JOURNALS + 1], |r| r.get::<_, String>(0))?;
|
||||
for id in recent { let id = id?; if !ids.contains(&id) { ids.push(id); } }
|
||||
let omitted = ids.len() > JOURNALS;
|
||||
ids.truncate(JOURNALS);
|
||||
let mut snapshot = JournalStartupSnapshot { subject_id: host.subject_id.clone(), journals: Vec::new(), omitted };
|
||||
for id in ids {
|
||||
let (session, token, order, byte) = tx.query_row(
|
||||
"SELECT source_session_id,state_token,process_order,process_byte FROM journals WHERE id=?1 AND subject_id=?2",
|
||||
params![id, host.subject_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?)),
|
||||
)?;
|
||||
let (current_position, mut omitted) = text(tx, "journals", &id, "current_position")?;
|
||||
let mut stmt = tx.prepare("SELECT id,position,state,source_end_json FROM journal_entries WHERE journal_id=?1 ORDER BY position DESC LIMIT ?2")?;
|
||||
let rows = stmt.query_map(params![id, ENTRIES + 1], |r| Ok((r.get::<_,String>(0)?,r.get::<_,u64>(1)?,r.get::<_,String>(2)?,r.get::<_,Option<String>>(3)?)))?.collect::<Result<Vec<_>,_>>()?;
|
||||
omitted |= rows.len() > ENTRIES;
|
||||
let mut entries = Vec::new();
|
||||
for (entry_id, position, state, source_end) in rows.into_iter().take(ENTRIES) {
|
||||
let (title, title_cut) = text(tx, "journal_entries", &entry_id, "title")?;
|
||||
let (body, body_cut) = text(tx, "journal_entries", &entry_id, "body")?;
|
||||
let (ongoing, ongoing_cut) = text(tx, "journal_entries", &entry_id, "ongoing")?;
|
||||
let mut refs = tx.prepare("SELECT target_id FROM journal_corrections WHERE entry_id=?1 ORDER BY target_id LIMIT 17")?;
|
||||
let mut correction_refs = refs.query_map([&entry_id], |r| r.get::<_,String>(0))?.collect::<Result<Vec<_>,_>>()?;
|
||||
let omitted = title_cut || body_cut || ongoing_cut || correction_refs.len() > 16;
|
||||
correction_refs.truncate(16);
|
||||
entries.push(StartupEntry { entry_id, position, state: match state.as_str() { "open" => JournalEntryState::Open, "fixed" => JournalEntryState::Fixed, _ => return Err(JournalError::Conflict("invalid stored Journal entry state".into())) }, title, body, ongoing, source_end: source_end.map(|s| serde_json::from_str(&s)).transpose()?, correction_refs, omitted });
|
||||
}
|
||||
snapshot.journals.push(StartupJournal { journal_id: id, source_session_id: session, state_token: token, process_order: order, process_byte: byte, current_position, entries, omitted });
|
||||
}
|
||||
// JSON escaping also consumes the independent output budget. Drop least-priority
|
||||
// entries/Journals rather than returning an over-budget or silently partial document.
|
||||
while serde_json::to_vec(&snapshot)?.len() > STARTUP_MAX_BYTES {
|
||||
snapshot.omitted = true;
|
||||
let journal = snapshot.journals.last_mut().ok_or_else(|| JournalError::InvalidInput("startup metadata exceeds budget".into()))?;
|
||||
if journal.entries.pop().is_some() { journal.omitted = true; } else { snapshot.journals.pop(); }
|
||||
}
|
||||
Ok(snapshot)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -141,6 +141,7 @@ pub(super) fn check_host(
|
||||
return Err(JournalError::PermissionDenied("Subject is retired".into()));
|
||||
}
|
||||
check_session_host(store, tx, host)?;
|
||||
super::binding::check_writer(tx, host)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -373,7 +374,10 @@ pub(super) fn job(
|
||||
ctx: &JournalJobContext,
|
||||
active: bool,
|
||||
) -> JournalResult<JournalGeneration> {
|
||||
check_host(store, tx, &ctx.host, true)?;
|
||||
// Recovery can inspect the original generation and exact saved receipts after release;
|
||||
// only a new mutation requires the still-current writer lifetime.
|
||||
check_host(store, tx, &ctx.host, active)?;
|
||||
check_session_host(store, tx, &ctx.host)?;
|
||||
let (g, is_active) = generation(tx, &ctx.host, &ctx.generation_id)?;
|
||||
if g.job != ctx.binding {
|
||||
return Err(JournalError::PermissionDenied(
|
||||
@@ -629,6 +633,13 @@ impl SubjektivStore {
|
||||
if operation.expected_state!=j.state_token {return Err(conflict("stored Journal content changed; reread before applying"));}
|
||||
in_snapshot(&g,&operation.covered_ranges)?;
|
||||
if !operation.covered_ranges.is_empty() {check_ranges(tx,&j,&operation.covered_ranges,true)?;}
|
||||
// A fully checkpointed selection has no pending source to recover another
|
||||
// activity write if the Job subsequently fails. Receipts and position repair
|
||||
// remain available, but new activity belongs to the next selected prefix.
|
||||
if matches!(operation.action, JournalAction::Create {..}|JournalAction::Rewrite {..}|JournalAction::Fix {..}|JournalAction::ConfirmUnchanged)
|
||||
&& (j.process_order,j.process_byte)==prefix(&g.initial_journal,&g.selected_ranges)? {
|
||||
return Err(conflict("generation activity is already fully checkpointed; finish this Job before recording new input"));
|
||||
}
|
||||
let mut entry_id=None;
|
||||
match &operation.action {
|
||||
JournalAction::Create {title,body,ongoing,state,source_end,correction_refs}=> {
|
||||
@@ -680,6 +691,9 @@ impl SubjektivStore {
|
||||
if !contains(&merged,r.bytes) {return Err(invalid("checkpoint has input with no durable record correspondence"));}
|
||||
tx.execute("UPDATE journal_sources SET reflected_bytes=?3 WHERE journal_id=?1 AND source_order=?2",params![j.id,r.source.order,r.bytes.end])?;
|
||||
}
|
||||
// Advancing reflection without position would strand a terminal failed
|
||||
// Job: source selection would see no pending bytes to repair that memo.
|
||||
super::binding::require_saved_position(tx,&g.id)?;
|
||||
j.process_order=order;j.process_byte=byte;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,450 @@ use sha2::{Digest, Sha256};
|
||||
use std::sync::{Arc, Barrier};
|
||||
use std::thread;
|
||||
|
||||
#[test]
|
||||
fn explicit_erasure_releases_only_after_all_body_sessions_including_unadmitted_history_are_erased()
|
||||
{
|
||||
let f = Fixture::new();
|
||||
let subject = f.host.subject_id.clone();
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&subject, "runtime", "worker")
|
||||
.unwrap();
|
||||
let attr = f
|
||||
.store()
|
||||
.record_session_attribution(
|
||||
SubjectSessionAttribution::new(&subject, "runtime", "worker", "saved-before-admission")
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let unadmitted = f.store().bind_journal_session(&binding, attr).unwrap();
|
||||
assert!(!f.store().release_erased_journal_body(&f.host).unwrap());
|
||||
f.store()
|
||||
.erase_journal_session(&f.host, "erase-first")
|
||||
.unwrap();
|
||||
assert!(!f.store().release_erased_journal_body(&f.host).unwrap());
|
||||
assert!(f.store().journal_writer_active(&binding).unwrap());
|
||||
f.store()
|
||||
.erase_journal_session(&unadmitted, "erase-last")
|
||||
.unwrap();
|
||||
assert!(f.store().release_erased_journal_body(&unadmitted).unwrap());
|
||||
assert!(f.store().release_erased_journal_body(&unadmitted).unwrap());
|
||||
assert!(!f.store().journal_writer_active(&binding).unwrap());
|
||||
assert!(
|
||||
f.store()
|
||||
.journal_worker_was_released(&subject, "runtime", "worker")
|
||||
.unwrap()
|
||||
);
|
||||
let next = f
|
||||
.store()
|
||||
.claim_journal_writer(&subject, "runtime", "replacement")
|
||||
.unwrap();
|
||||
assert!(f.store().release_erased_journal_body(&f.host).unwrap());
|
||||
assert!(
|
||||
f.store().journal_writer_active(&next).unwrap(),
|
||||
"late erase retry cannot revoke another body"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn released_binding_preserves_open_and_receipts_but_never_revives_old_session_writes() {
|
||||
let mut f = Fixture::new();
|
||||
let subject = f.host.subject_id.clone();
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&subject, "runtime", "worker")
|
||||
.unwrap();
|
||||
assert!(
|
||||
f.store()
|
||||
.claim_journal_writer(&subject, "runtime", "other-worker")
|
||||
.is_err()
|
||||
);
|
||||
assert!(f.store().release_journal_writer(&binding).is_err());
|
||||
let (generation, context) = f.generation("first-body", f.ranges.clone());
|
||||
let create = f.operation(
|
||||
"create",
|
||||
open_action("A was proposed; user corrected a constraint; B selected and partly executed"),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let receipt = f.store().execute_journal(&context, create.clone()).unwrap();
|
||||
f.execute(
|
||||
&context,
|
||||
"position",
|
||||
JournalAction::UpdatePosition {
|
||||
current_position:
|
||||
"B: configuration changed; tests not executed; external result Unknown".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
assert!(f.store().release_journal_writer(&binding).is_err());
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["create".into()],
|
||||
},
|
||||
f.ranges.clone(),
|
||||
);
|
||||
// A saved checkpoint is not proof that the generation has stopped executing.
|
||||
assert!(f.store().release_journal_writer(&binding).is_err());
|
||||
f.close(&generation);
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
f.reopen();
|
||||
let old = f
|
||||
.store()
|
||||
.query_journal_operation(&f.host, &f.journal_id, "create")
|
||||
.unwrap();
|
||||
assert_eq!(old, Some(receipt.clone()));
|
||||
assert_eq!(
|
||||
f.store().execute_journal(&context, create).unwrap(),
|
||||
receipt
|
||||
);
|
||||
let reader = JournalHostContext::validated_reader(f.store(), &subject).unwrap();
|
||||
let snapshot = f
|
||||
.store()
|
||||
.journal_startup_snapshot(&reader, Some("source-session"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
snapshot.journals[0].entries[0].state,
|
||||
JournalEntryState::Open
|
||||
);
|
||||
assert!(snapshot.journals[0].current_position.contains("Unknown"));
|
||||
let new = f
|
||||
.store()
|
||||
.claim_journal_writer(&subject, "runtime", "worker")
|
||||
.unwrap();
|
||||
assert_ne!(new.binding_id, binding.binding_id);
|
||||
assert!(
|
||||
f.store()
|
||||
.bind_journal_session(&new, f.host.attribution.clone().unwrap())
|
||||
.is_err()
|
||||
);
|
||||
let late = source_range("source-session", 2, 64);
|
||||
assert!(matches!(
|
||||
f.store().accept_journal_sources(&f.host, "late", &[late]),
|
||||
Err(JournalError::PermissionDenied(_))
|
||||
));
|
||||
assert!(
|
||||
f.store()
|
||||
.prepare_journal_generation(
|
||||
&f.host,
|
||||
&f.journal_id,
|
||||
JobAttemptBinding {
|
||||
job_id: "late".into(),
|
||||
attempt_id: "late".into()
|
||||
},
|
||||
f.ranges.clone()
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let operation = JournalOperation {
|
||||
request_id: "late-position".into(),
|
||||
expected_state: snapshot.journals[0].state_token.clone(),
|
||||
action: JournalAction::UpdatePosition {
|
||||
current_position: "forged".into(),
|
||||
},
|
||||
covered_ranges: vec![],
|
||||
};
|
||||
assert!(f.store().execute_journal(&context, operation).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_rejects_unbound_legacy_body_sessions_even_without_admitted_sources() {
|
||||
let f = Fixture::new();
|
||||
let (generation, context) = f.generation("recorded-body", f.ranges.clone());
|
||||
f.execute(
|
||||
&context,
|
||||
"record",
|
||||
JournalAction::ConfirmUnchanged,
|
||||
f.ranges.clone(),
|
||||
);
|
||||
save_position(f.store(), &context, "position");
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
checkpoint(&["record"]),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.close(&generation);
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
let legacy = f
|
||||
.store()
|
||||
.record_session_attribution(
|
||||
SubjectSessionAttribution::new(
|
||||
&f.host.subject_id,
|
||||
"runtime",
|
||||
"worker",
|
||||
"legacy-not-admitted",
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
f.store().release_journal_writer(&binding),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
assert!(f.store().journal_writer_active(&binding).unwrap());
|
||||
// The Host must bind and reconcile this original Session, even if its canonical
|
||||
// history proves empty and therefore creates no Journal.
|
||||
f.store().bind_journal_session(&binding, legacy).unwrap();
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn predecessor_prefers_latest_journal_activity_not_session_attachment_order() {
|
||||
let f = Fixture::new();
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
let second = session_host(f.store(), &f.host.subject_id, "later-attached-session");
|
||||
let second_ranges = vec![source_range("later-attached-session", 0, 8)];
|
||||
let second_id = f
|
||||
.store()
|
||||
.accept_journal_sources(&second, "Earlier activity", &second_ranges)
|
||||
.unwrap();
|
||||
for (host, journal_id, ranges) in [
|
||||
(&f.host, &f.journal_id, &f.ranges),
|
||||
(&second, &second_id, &second_ranges),
|
||||
] {
|
||||
let job = JobAttemptBinding {
|
||||
job_id: journal_id.clone(),
|
||||
attempt_id: "recording".into(),
|
||||
};
|
||||
let generation = f
|
||||
.store()
|
||||
.prepare_journal_generation(host, journal_id, job.clone(), ranges.clone())
|
||||
.unwrap();
|
||||
let context = JournalJobContext::validated(f.store(), host, &generation.id, job).unwrap();
|
||||
let state = f
|
||||
.store()
|
||||
.read_journal(host, journal_id)
|
||||
.unwrap()
|
||||
.state_token;
|
||||
f.store()
|
||||
.execute_journal(
|
||||
&context,
|
||||
JournalOperation {
|
||||
request_id: "record".into(),
|
||||
expected_state: state,
|
||||
action: open_action("Unfinished activity"),
|
||||
covered_ranges: ranges.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let position = save_position(f.store(), &context, "position");
|
||||
f.store()
|
||||
.checkpoint_journal(
|
||||
&context,
|
||||
JournalOperation {
|
||||
request_id: "checkpoint".into(),
|
||||
expected_state: position.state_token,
|
||||
action: checkpoint(&["record"]),
|
||||
covered_ranges: ranges.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
f.store()
|
||||
.close_journal_generation(host, &generation.id, &generation.job)
|
||||
.unwrap();
|
||||
}
|
||||
// Stored timestamps model a later update to the earlier-attached Session without
|
||||
// using elapsed wall time as a synchronization or ordering assertion.
|
||||
f.store()
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
tx.execute(
|
||||
"UPDATE journals SET updated_at='2001-01-01T00:00:00Z' WHERE id=?1",
|
||||
[&f.journal_id],
|
||||
)?;
|
||||
tx.execute(
|
||||
"UPDATE journals SET updated_at='2000-01-01T00:00:00Z' WHERE id=?1",
|
||||
[&second_id],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let _empty = session_host(
|
||||
f.store(),
|
||||
&f.host.subject_id,
|
||||
"latest-attachment-no-journal",
|
||||
);
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
let replacement = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "replacement")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
f.store()
|
||||
.journal_predecessor_session(&replacement)
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("source-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_body_claims_from_independent_connections_bind_only_one_worker() {
|
||||
let f = Fixture::new();
|
||||
let subject = f
|
||||
.store()
|
||||
.create_subject(SubjectRole::new("claim-race").unwrap())
|
||||
.unwrap();
|
||||
let manager = FeatureStorage::new(f.temp.path().join("storage"));
|
||||
let workspace = manager.workspace("journal-tests").unwrap();
|
||||
let registration = SubjektivStore::register(&workspace).unwrap();
|
||||
let second = SubjektivStore::open(&workspace, ®istration).unwrap();
|
||||
let barrier = Arc::new(Barrier::new(3));
|
||||
let spawn = |store: SubjektivStore, worker: &'static str| {
|
||||
let barrier = barrier.clone();
|
||||
let subject = subject.id.clone();
|
||||
thread::spawn(move || {
|
||||
barrier.wait();
|
||||
store.claim_journal_writer(&subject, "runtime", worker)
|
||||
})
|
||||
};
|
||||
let first = spawn(f.store().clone(), "first");
|
||||
let second = spawn(second, "second");
|
||||
barrier.wait();
|
||||
let outcomes = [first.join().unwrap(), second.join().unwrap()];
|
||||
assert_eq!(outcomes.iter().filter(|r| r.is_ok()).count(), 1);
|
||||
let winner = outcomes.into_iter().find_map(Result::ok).unwrap();
|
||||
assert!(f.store().journal_writer_active(&winner).unwrap());
|
||||
let loser = if winner.worker_id == "first" {
|
||||
"second"
|
||||
} else {
|
||||
"first"
|
||||
};
|
||||
assert!(
|
||||
f.store()
|
||||
.claim_journal_writer(&subject.id, "runtime", loser)
|
||||
.is_err()
|
||||
);
|
||||
drop(workspace);
|
||||
manager.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_and_late_admission_are_serialized_by_the_persisted_binding_check() {
|
||||
let f = Fixture::new();
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
let (generation, context) = f.generation("release-race", f.ranges.clone());
|
||||
f.execute(
|
||||
&context,
|
||||
"record",
|
||||
open_action("unfinished"),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.execute(
|
||||
&context,
|
||||
"position",
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "remaining work".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["record".into()],
|
||||
},
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.close(&generation);
|
||||
let manager = FeatureStorage::new(f.temp.path().join("storage"));
|
||||
let workspace = manager.workspace("journal-tests").unwrap();
|
||||
let registration = SubjektivStore::register(&workspace).unwrap();
|
||||
let second = SubjektivStore::open(&workspace, ®istration).unwrap();
|
||||
let barrier = Arc::new(Barrier::new(3));
|
||||
let b = barrier.clone();
|
||||
let store = f.store().clone();
|
||||
let released = binding.clone();
|
||||
let release = thread::spawn(move || {
|
||||
b.wait();
|
||||
store.release_journal_writer(&released)
|
||||
});
|
||||
let b = barrier.clone();
|
||||
let host = f.host.clone();
|
||||
let late = source_range("source-session", 2, 16);
|
||||
let admission = thread::spawn(move || {
|
||||
b.wait();
|
||||
second.accept_journal_sources(&host, "late", &[late])
|
||||
});
|
||||
barrier.wait();
|
||||
let release = release.join().unwrap();
|
||||
let admission = admission.join().unwrap();
|
||||
assert_ne!(
|
||||
release.is_ok(),
|
||||
admission.is_ok(),
|
||||
"exactly one ordered effect may commit"
|
||||
);
|
||||
assert_eq!(
|
||||
f.store().journal_writer_active(&binding).unwrap(),
|
||||
admission.is_ok()
|
||||
);
|
||||
assert_eq!(
|
||||
f.store()
|
||||
.read_journal_source(&f.host, &f.journal_id, 2)
|
||||
.is_ok(),
|
||||
admission.is_ok()
|
||||
);
|
||||
drop(workspace);
|
||||
manager.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_snapshot_distinguishes_none_is_subject_local_and_bounds_utf8_at_storage() {
|
||||
let f = Fixture::new();
|
||||
let (_, context) = f.generation("snapshot", f.ranges.clone());
|
||||
f.execute(
|
||||
&context,
|
||||
"large-open",
|
||||
open_action(&"日本語\\\"\\n".repeat(6000)),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let reader = JournalHostContext::validated_reader(f.store(), &f.host.subject_id).unwrap();
|
||||
let snapshot = f
|
||||
.store()
|
||||
.journal_startup_snapshot(&reader, Some("source-session"))
|
||||
.unwrap();
|
||||
let encoded = serde_json::to_vec(&snapshot).unwrap();
|
||||
assert!(encoded.len() <= startup::STARTUP_MAX_BYTES);
|
||||
assert!(snapshot.journals[0].entries[0].omitted);
|
||||
assert!(snapshot.journals[0].entries[0].body.len() <= 2048);
|
||||
assert_eq!(snapshot.journals[0].source_session_id, "source-session");
|
||||
let other = f
|
||||
.store()
|
||||
.create_subject_with_behavior(
|
||||
SubjectRole::new("other").unwrap(),
|
||||
"behavior unchanged".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let other_reader = JournalHostContext::validated_reader(f.store(), &other.id).unwrap();
|
||||
assert!(matches!(
|
||||
f.store()
|
||||
.journal_startup_snapshot(&other_reader, Some("source-session")),
|
||||
Err(JournalError::PermissionDenied(_))
|
||||
));
|
||||
let none = f
|
||||
.store()
|
||||
.journal_startup_snapshot(&other_reader, None)
|
||||
.unwrap();
|
||||
assert!(none.journals.is_empty());
|
||||
assert!(!none.omitted);
|
||||
assert_eq!(
|
||||
f.store().subject(&other.id).unwrap().unwrap().behavior_md,
|
||||
"behavior unchanged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stable_legacy_run_references_are_accepted_without_relaxing_character_or_size_limits() {
|
||||
let f = Fixture::new();
|
||||
@@ -430,6 +874,111 @@ fn erasure_failure_rolls_back_tombstone_and_content_then_exact_retry_succeeds()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpointed_generation_rejects_late_activity_but_retains_saved_receipts() {
|
||||
let f = Fixture::new();
|
||||
let (generation, context) = f.generation("checkpointed-job", f.ranges.clone());
|
||||
let operation = f.operation("record", open_action("B partly executed"), f.ranges.clone());
|
||||
let receipt = f
|
||||
.store()
|
||||
.execute_journal(&context, operation.clone())
|
||||
.unwrap();
|
||||
let entry_id = receipt.entry_id.clone().unwrap();
|
||||
save_position(f.store(), &context, "position");
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
checkpoint(&["record"]),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let before = f.state();
|
||||
for (index, action) in [
|
||||
open_action("late create"),
|
||||
JournalAction::Rewrite {
|
||||
entry_id: entry_id.clone(),
|
||||
title: "late rewrite".into(),
|
||||
body: "late".into(),
|
||||
ongoing: "late".into(),
|
||||
},
|
||||
JournalAction::Fix {
|
||||
entry_id,
|
||||
source_end: f.ranges.last().unwrap().source.pointer.clone(),
|
||||
},
|
||||
JournalAction::ConfirmUnchanged,
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let late = f.operation(&format!("late-{index}"), action, f.ranges.clone());
|
||||
assert!(matches!(
|
||||
f.store().execute_journal(&context, late),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
assert_eq!(f.state(), before);
|
||||
}
|
||||
assert_eq!(
|
||||
f.store().execute_journal(&context, operation).unwrap(),
|
||||
receipt
|
||||
);
|
||||
f.close(&generation);
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_without_current_position_leaves_failed_job_input_recoverable_after_reopen() {
|
||||
let mut f = Fixture::new();
|
||||
let (failed, context) = f.generation("incomplete-position", f.ranges.clone());
|
||||
save_position(f.store(), &context, "outdated-position");
|
||||
f.execute(
|
||||
&context,
|
||||
"record",
|
||||
open_action("B selected; tests not run"),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let checkpoint_operation = f.operation("checkpoint", checkpoint(&["record"]), f.ranges.clone());
|
||||
let before = f.state();
|
||||
assert!(matches!(
|
||||
f.store().execute_journal(&context, checkpoint_operation),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
assert_eq!(f.state(), before);
|
||||
assert!(f.sources().iter().all(|source| source.reflected_bytes == 0));
|
||||
assert!(f.receipt("checkpoint").is_none());
|
||||
f.close(&failed);
|
||||
f.reopen();
|
||||
let (replacement, context) = f.generation("recover-position", f.ranges.clone());
|
||||
f.execute(
|
||||
&context,
|
||||
"recovered-position",
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "B partly performed; tests unperformed; external result Unknown"
|
||||
.into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
// Use the saved record receipt rather than duplicating the old activity entry.
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
checkpoint(&["record"]),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.close(&replacement);
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
assert_eq!(f.entries().len(), 1);
|
||||
assert_eq!(f.entries()[0].state, JournalEntryState::Open);
|
||||
assert!(f.state().0.current_position.contains("Unknown"));
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
assert!(!f.store().journal_writer_active(&binding).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_service_completion_requires_checkpoint_and_replays_saved_completion_after_new_generation() {
|
||||
let mut f = Fixture::new();
|
||||
@@ -476,22 +1025,43 @@ fn job_service_completion_requires_checkpoint_and_replays_saved_completion_after
|
||||
service::finish(f.store(), &context),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
service::execute(
|
||||
f.store(),
|
||||
&context,
|
||||
transport::JournalJobRequest::Execute {
|
||||
operation: JournalOperation {
|
||||
request_id: "service-checkpoint".into(),
|
||||
expected_state: receipt.state_token,
|
||||
action: JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["service-create".into()],
|
||||
},
|
||||
covered_ranges: f.ranges.clone(),
|
||||
},
|
||||
let checkpoint_operation = JournalOperation {
|
||||
request_id: "service-checkpoint".into(),
|
||||
expected_state: receipt.state_token,
|
||||
action: JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["service-create".into()],
|
||||
},
|
||||
|_| unreachable!(),
|
||||
)
|
||||
.unwrap();
|
||||
covered_ranges: f.ranges.clone(),
|
||||
};
|
||||
// Reject the checkpoint atomically: failed/terminal Jobs must leave pending input
|
||||
// recoverable rather than stranding a fully reflected prefix without a position memo.
|
||||
assert!(matches!(
|
||||
service::execute(
|
||||
f.store(),
|
||||
&context,
|
||||
transport::JournalJobRequest::Execute {
|
||||
operation: checkpoint_operation
|
||||
},
|
||||
|_| unreachable!(),
|
||||
),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
assert!(f.sources().iter().all(|source| source.reflected_bytes == 0));
|
||||
assert!(f.receipt("service-checkpoint").is_none());
|
||||
f.execute(
|
||||
&context,
|
||||
"service-position",
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "ongoing; work remains".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
f.execute(
|
||||
&context,
|
||||
"service-checkpoint",
|
||||
checkpoint(&["service-create"]),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let completion = service::finish(f.store(), &context).unwrap();
|
||||
assert_eq!(completion.generation_id, generation.id);
|
||||
assert_eq!((completion.process_order, completion.process_byte), (2, 0));
|
||||
@@ -685,7 +1255,7 @@ fn retention_release_requires_checkpoint_not_saved_entry_or_unavailability() {
|
||||
let f = Fixture::new();
|
||||
let (_, job) = f.generation("retained-job", f.ranges.clone());
|
||||
let operation = f.operation("record", open_action("ongoing activity"), f.ranges.clone());
|
||||
let receipt = f.store().execute_journal(&job, operation).unwrap();
|
||||
f.store().execute_journal(&job, operation).unwrap();
|
||||
assert!(f.store().journal_retention_required(&f.host).unwrap());
|
||||
f.store()
|
||||
.mark_journal_sources_unavailable(&f.host, &f.journal_id, &f.ranges)
|
||||
@@ -694,6 +1264,7 @@ fn retention_release_requires_checkpoint_not_saved_entry_or_unavailability() {
|
||||
f.store()
|
||||
.mark_journal_sources_available(&f.host, &f.journal_id, &f.ranges)
|
||||
.unwrap();
|
||||
let receipt = save_position(f.store(), &job, "retained-position");
|
||||
f.store()
|
||||
.checkpoint_journal(
|
||||
&job,
|
||||
@@ -805,8 +1376,7 @@ fn recording_materials_resume_exact_saved_byte_prefix_after_reopen() {
|
||||
.unwrap();
|
||||
let job = JournalJobContext::validated(f.store(), &host, &generation.id, binding).unwrap();
|
||||
let state = f.store().read_journal(&host, &id).unwrap().state_token;
|
||||
let receipt = f
|
||||
.store()
|
||||
f.store()
|
||||
.execute_journal(
|
||||
&job,
|
||||
JournalOperation {
|
||||
@@ -817,6 +1387,7 @@ fn recording_materials_resume_exact_saved_byte_prefix_after_reopen() {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let receipt = save_position(f.store(), &job, "recording-position");
|
||||
f.store()
|
||||
.checkpoint_journal(
|
||||
&job,
|
||||
@@ -891,7 +1462,12 @@ impl Fixture {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let host = session_host(&store, &subject.id, "source-session");
|
||||
let attribution =
|
||||
SubjectSessionAttribution::new(&subject.id, "runtime", "worker", "source-session")
|
||||
.unwrap();
|
||||
store
|
||||
.record_session_attribution(attribution.clone())
|
||||
.unwrap();
|
||||
let (manager, workspace, store) = if upgrade {
|
||||
let has_journals: bool = store.database.try_with_connection::<_, JournalError>(|connection| {
|
||||
Ok(connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='journals')", [], |row| row.get(0))?)
|
||||
@@ -905,15 +1481,16 @@ impl Fixture {
|
||||
let workspace = manager.workspace("journal-tests").unwrap();
|
||||
let registration = SubjektivStore::register(&workspace).unwrap();
|
||||
let store = SubjektivStore::open(&workspace, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject));
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
assert_eq!(
|
||||
store.session_attribution("source-session").unwrap(),
|
||||
host.attribution
|
||||
Some(attribution)
|
||||
);
|
||||
(manager, workspace, store)
|
||||
} else {
|
||||
(manager, workspace, store)
|
||||
};
|
||||
let host = session_host(&store, &subject.id, "source-session");
|
||||
let ranges = sizes
|
||||
.iter()
|
||||
.enumerate()
|
||||
@@ -1070,10 +1647,11 @@ impl Drop for Fixture {
|
||||
fn session_host(store: &SubjektivStore, subject: &str, session: &str) -> JournalHostContext {
|
||||
let attribution =
|
||||
SubjectSessionAttribution::new(subject, "runtime", "worker", session).unwrap();
|
||||
store
|
||||
.record_session_attribution(attribution.clone())
|
||||
let attribution = store.record_session_attribution(attribution).unwrap();
|
||||
let binding = store
|
||||
.claim_journal_writer(subject, "runtime", "worker")
|
||||
.unwrap();
|
||||
JournalHostContext::validated_session(store, attribution).unwrap()
|
||||
store.bind_journal_session(&binding, attribution).unwrap()
|
||||
}
|
||||
|
||||
fn source_range(session: &str, order: u64, total_bytes: u64) -> JournalSourceRange {
|
||||
@@ -1127,6 +1705,32 @@ fn fixed_action(body: &str, range: &JournalSourceRange, corrections: Vec<String>
|
||||
}
|
||||
}
|
||||
|
||||
fn save_position(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
request_id: &str,
|
||||
) -> JournalReceipt {
|
||||
let generation = store
|
||||
.read_journal_generation(&context.host, &context.generation_id)
|
||||
.unwrap();
|
||||
let (journal, _) = store
|
||||
.journal_resume_state(&context.host, &generation.journal_id)
|
||||
.unwrap();
|
||||
store
|
||||
.execute_journal(
|
||||
context,
|
||||
JournalOperation {
|
||||
request_id: request_id.into(),
|
||||
expected_state: journal.state_token,
|
||||
action: JournalAction::UpdatePosition {
|
||||
current_position: journal.current_position,
|
||||
},
|
||||
covered_ranges: vec![],
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn checkpoint(ids: &[&str]) -> JournalAction {
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: ids.iter().map(|id| (*id).into()).collect(),
|
||||
@@ -1249,6 +1853,7 @@ fn sequential_open_rewrite_fix_and_correction_preserve_fixed_history() {
|
||||
assert_eq!(entries[1].correction_refs, vec![first_id]);
|
||||
assert_eq!(entries[1].previous_fixed_end, fixed.source_end);
|
||||
assert_eq!(fixture.state().0.process_order, 0);
|
||||
save_position(fixture.store(), &ctx, "final-position");
|
||||
let cp = fixture.execute(
|
||||
&ctx,
|
||||
"checkpoint",
|
||||
@@ -1966,6 +2571,7 @@ fn checkpoint_requires_saved_record_correspondence_and_never_position_only_evide
|
||||
JournalAction::ConfirmUnchanged,
|
||||
vec![fixture.ranges[1].clone()],
|
||||
);
|
||||
save_position(fixture.store(), &ctx, "checked-position");
|
||||
let receipt = fixture.execute(
|
||||
&ctx,
|
||||
"valid-checkpoint",
|
||||
@@ -2017,6 +2623,7 @@ fn huge_source_partial_generations_checkpoint_only_read_prefix_and_fix_only_at_e
|
||||
fixture.store().execute_journal(&first_ctx, unread_end),
|
||||
Err(JournalError::InvalidInput(_))
|
||||
));
|
||||
save_position(fixture.store(), &first_ctx, "partial-position");
|
||||
let cp = fixture.execute(
|
||||
&first_ctx,
|
||||
"partial-checkpoint",
|
||||
@@ -2082,6 +2689,7 @@ fn huge_source_partial_generations_checkpoint_only_read_prefix_and_fix_only_at_e
|
||||
vec![],
|
||||
);
|
||||
assert_eq!(fixture.state().0.process_byte, head.bytes.end);
|
||||
save_position(fixture.store(), &next_ctx, "tail-position");
|
||||
let cp = fixture.execute(
|
||||
&next_ctx,
|
||||
"tail-checkpoint",
|
||||
@@ -2913,6 +3521,7 @@ fn failure_before_receipt_rolls_back_record_and_checkpoint_then_exact_retry_succ
|
||||
fixture.store().execute_journal(&ctx, record).unwrap(),
|
||||
saved
|
||||
);
|
||||
save_position(fixture.store(), &ctx, "fault-position");
|
||||
let cp = fixture.operation(
|
||||
"fault-checkpoint",
|
||||
checkpoint(&["fault-record"]),
|
||||
@@ -2994,6 +3603,7 @@ fn per_operation_reopen_recovers_a_realistic_investigation_and_later_correction_
|
||||
assert!(fixture.state().1.is_none());
|
||||
assert_eq!(fixture.state().0.process_order, 0);
|
||||
let fixed_before_correction = fixture.entries()[0].clone();
|
||||
save_position(fixture.store(), &first_ctx, "investigation-fixed-position");
|
||||
fixture.execute_and_reopen(
|
||||
&first_ctx,
|
||||
"investigation-checkpoint",
|
||||
@@ -3117,6 +3727,7 @@ fn split_input_and_replacement_job_reconcile_old_record_coverage_without_duplica
|
||||
.unwrap();
|
||||
assert_eq!(coverage.items, vec![whole.clone()]);
|
||||
let (_, replacement) = fixture.generation("replacement-job", vec![whole.clone()]);
|
||||
save_position(fixture.store(), &replacement, "replacement-position");
|
||||
let cp = fixture.operation("recover-reflection", checkpoint(&["record"]), vec![whole]);
|
||||
let result = fixture
|
||||
.store()
|
||||
@@ -3200,86 +3811,95 @@ fn serialized_metadata_and_text_budgets_preserve_continuation_and_availability_r
|
||||
|
||||
#[test]
|
||||
fn journal_migration_failure_rolls_back_ddl_and_retries_without_changing_existing_subject() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("storage");
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = scope
|
||||
.register(feature_storage::FeatureRegistration::new(
|
||||
crate::SUBJEKTIV_FEATURE_ID,
|
||||
&crate::MIGRATIONS[..7],
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
SubjectRole::new("preserved").unwrap(),
|
||||
"Original behavior".into(),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
tx.execute_batch("CREATE TABLE journal_entries(injected INTEGER)")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = SubjektivStore::register(&scope).unwrap();
|
||||
assert!(matches!(
|
||||
SubjektivStore::open(&scope, ®istration),
|
||||
Err(crate::SubjektivError::Storage(
|
||||
feature_storage::FeatureStorageError::MigrationFailed { .. }
|
||||
))
|
||||
));
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = scope
|
||||
.register(feature_storage::FeatureRegistration::new(
|
||||
crate::SUBJEKTIV_FEATURE_ID,
|
||||
&crate::MIGRATIONS[..7],
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
assert!(!tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='journals')",
|
||||
[],
|
||||
|r| r.get::<_, bool>(0)
|
||||
)?);
|
||||
tx.execute_batch("DROP TABLE journal_entries")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = SubjektivStore::register(&scope).unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
let host = session_host(&store, &subject.id, "post-migration-session");
|
||||
let range = source_range("post-migration-session", 0, 8);
|
||||
let id = store
|
||||
.accept_journal_sources(&host, "Recovered migration", &[range])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.read_journal(&host, &id).unwrap().source_session_id,
|
||||
"post-migration-session"
|
||||
);
|
||||
for (baseline, collision, rolled_back) in [
|
||||
(&crate::MIGRATIONS[..7], "journal_entries", "journals"),
|
||||
(
|
||||
&crate::MIGRATIONS[..9],
|
||||
"journal_session_bindings",
|
||||
"journal_writer_bindings",
|
||||
),
|
||||
] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("storage");
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = scope
|
||||
.register(feature_storage::FeatureRegistration::new(
|
||||
crate::SUBJEKTIV_FEATURE_ID,
|
||||
baseline,
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
SubjectRole::new("preserved").unwrap(),
|
||||
"Original behavior".into(),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
tx.execute_batch(&format!("CREATE TABLE {collision}(injected INTEGER)"))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = SubjektivStore::register(&scope).unwrap();
|
||||
assert!(matches!(
|
||||
SubjektivStore::open(&scope, ®istration),
|
||||
Err(crate::SubjektivError::Storage(
|
||||
feature_storage::FeatureStorageError::MigrationFailed { .. }
|
||||
))
|
||||
));
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = scope
|
||||
.register(feature_storage::FeatureRegistration::new(
|
||||
crate::SUBJEKTIV_FEATURE_ID,
|
||||
baseline,
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
assert!(!tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name=?1)",
|
||||
[rolled_back],
|
||||
|r| r.get::<_, bool>(0)
|
||||
)?);
|
||||
tx.execute_batch(&format!("DROP TABLE {collision}"))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = SubjektivStore::register(&scope).unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
let host = session_host(&store, &subject.id, "post-migration-session");
|
||||
let range = source_range("post-migration-session", 0, 8);
|
||||
let id = store
|
||||
.accept_journal_sources(&host, "Recovered migration", &[range])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.read_journal(&host, &id).unwrap().source_session_id,
|
||||
"post-migration-session"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3365,6 +3985,7 @@ fn late_gap_admission_and_partial_updates_reject_stale_source_page_cursors() {
|
||||
.list_journal_sources(&fixture.host, &fixture.journal_id, fresh.next.as_deref(), 2)
|
||||
.is_ok()
|
||||
);
|
||||
save_position(fixture.store(), &job, "listing-position");
|
||||
fixture.execute(
|
||||
&job,
|
||||
"checkpoint",
|
||||
|
||||
@@ -953,11 +953,12 @@ mod tests {
|
||||
store.session_attribution("session-1").unwrap().unwrap(),
|
||||
stored_attribution
|
||||
);
|
||||
let host = crate::journal::JournalHostContext::validated_session(
|
||||
&store,
|
||||
stored_attribution.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let binding = store
|
||||
.claim_journal_writer(&subject_id, "runtime-1", "worker-1")
|
||||
.unwrap();
|
||||
let host = store
|
||||
.bind_journal_session(&binding, stored_attribution.clone())
|
||||
.unwrap();
|
||||
let range = crate::journal::JournalSourceRange {
|
||||
source: crate::journal::JournalSource {
|
||||
session_id: stored_attribution.session_id.clone(),
|
||||
|
||||
@@ -613,6 +613,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
|
||||
"fence erased journal Sessions and permit explicit content erasure",
|
||||
journal::migrate_lifecycle,
|
||||
),
|
||||
FeatureMigration::new(
|
||||
10,
|
||||
"fence journal writers by explicit body binding lifetime",
|
||||
journal::migrate_binding,
|
||||
),
|
||||
];
|
||||
|
||||
pub const REGISTRATION: FeatureRegistration =
|
||||
|
||||
@@ -3034,6 +3034,7 @@ impl App {
|
||||
session_store::SystemItem::FeatureInvocationResult { body, .. }
|
||||
| session_store::SystemItem::FileAttachment { body, .. }
|
||||
| session_store::SystemItem::SkillActivation { body, .. }
|
||||
| session_store::SystemItem::JournalStartup { body, .. }
|
||||
| session_store::SystemItem::ResidentSummaryRefresh { body, .. }
|
||||
| session_store::SystemItem::SubjectBehaviorRefresh { body, .. }
|
||||
| session_store::SystemItem::Interrupt { body, .. } => {
|
||||
|
||||
@@ -23,6 +23,7 @@ http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
||||
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
|
||||
|
||||
[dependencies]
|
||||
agen.workspace = true
|
||||
async-trait.workspace = true
|
||||
base64.workspace = true
|
||||
axum = { workspace = true, optional = true }
|
||||
|
||||
@@ -567,6 +567,14 @@ impl fmt::Debug for WorkerProtocolTransport {
|
||||
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||
fn backend_id(&self) -> &str;
|
||||
|
||||
/// Must reserve the actual Controller, not infer Idle from a catalog snapshot.
|
||||
fn subject_release(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
Err("execution backend cannot reserve the actual Subject Controller".into())
|
||||
}
|
||||
|
||||
fn fetch_workspace_config(
|
||||
&self,
|
||||
_request: WorkspaceConfigFetchRequest,
|
||||
@@ -831,6 +839,13 @@ impl WorkerExecutionBackendRef {
|
||||
self.backend.fetch_workspace_config(request)
|
||||
}
|
||||
|
||||
pub(crate) fn subject_release(
|
||||
&self,
|
||||
request: &runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
self.backend.subject_release(request)
|
||||
}
|
||||
|
||||
pub(crate) fn worker_session(
|
||||
&self,
|
||||
request: WorkerSessionObservationRequest,
|
||||
|
||||
@@ -479,6 +479,10 @@ pub(crate) struct OpenedFsRuntimeStore {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct PersistedRuntimeState {
|
||||
pub(crate) subject_releases: Vec<(
|
||||
runtime_api::SubjectReleaseRequest,
|
||||
runtime_api::SubjectReleaseState,
|
||||
)>,
|
||||
pub(crate) display_name: Option<String>,
|
||||
pub(crate) status: RuntimeStatus,
|
||||
pub(crate) next_diagnostic_id: u64,
|
||||
@@ -1915,6 +1919,11 @@ fn migrate_runtime_store_in_place(
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
struct RuntimeSnapshot {
|
||||
#[serde(default)]
|
||||
subject_releases: Vec<(
|
||||
runtime_api::SubjectReleaseRequest,
|
||||
runtime_api::SubjectReleaseState,
|
||||
)>,
|
||||
schema_version: u32,
|
||||
display_name: Option<String>,
|
||||
backend: RuntimeBackendKind,
|
||||
@@ -1946,6 +1955,7 @@ fn record_worker_load_diagnostic(
|
||||
impl RuntimeSnapshot {
|
||||
fn from_persisted(state: &PersistedRuntimeState) -> Self {
|
||||
Self {
|
||||
subject_releases: state.subject_releases.clone(),
|
||||
schema_version: SCHEMA_VERSION,
|
||||
display_name: state.display_name.clone(),
|
||||
backend: RuntimeBackendKind::FsStore,
|
||||
@@ -1983,6 +1993,7 @@ impl RuntimeSnapshot {
|
||||
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
|
||||
) -> PersistedRuntimeState {
|
||||
PersistedRuntimeState {
|
||||
subject_releases: self.subject_releases,
|
||||
display_name: self.display_name,
|
||||
status: self.status,
|
||||
next_diagnostic_id: self.next_diagnostic_id,
|
||||
|
||||
@@ -2102,7 +2102,10 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
||||
{
|
||||
return Some("workers:input");
|
||||
}
|
||||
if path.ends_with("/stop") || path.ends_with("/cancel") {
|
||||
if path.ends_with("/stop")
|
||||
|| path.ends_with("/cancel")
|
||||
|| (path == "/v1/subject-release" && *method == Method::POST)
|
||||
{
|
||||
return Some("workers:stop");
|
||||
}
|
||||
if path == runtime_api::RUNTIME_ROUTE_PROTOCOL_WS {
|
||||
|
||||
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
}
|
||||
}
|
||||
|
||||
async fn subject_release(
|
||||
&self,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, runtime_api::RuntimeApiError> {
|
||||
let scope = material_scope(&self.state, &request.workspace_id)?;
|
||||
self.state
|
||||
.runtime
|
||||
.subject_release_scoped(&scope, request)
|
||||
.map_err(RuntimeHttpRestError::runtime)
|
||||
.map_err(api_error)
|
||||
}
|
||||
|
||||
async fn session_material_pin(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialPinRequest,
|
||||
|
||||
@@ -356,6 +356,109 @@ impl Drop for WorkerOperationLease {
|
||||
}
|
||||
|
||||
impl Runtime {
|
||||
/// Reserves the actual Controller while the Host reconciles final saved Journal material.
|
||||
/// A Runtime restart loses proof of a reservation, never reconstructs it from Worker status.
|
||||
pub fn subject_release_scoped(
|
||||
&self,
|
||||
scope: &RuntimeWorkspaceScope,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, RuntimeError> {
|
||||
use runtime_api::SubjectReleaseAction as Action;
|
||||
if request.workspace_id != scope.workspace_id
|
||||
|| request.binding_id.trim().is_empty()
|
||||
|| request.operation_id.trim().is_empty()
|
||||
|| request.binding_id.len() > 512
|
||||
|| request.operation_id.len() > 512
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"invalid Subject release identity".into(),
|
||||
));
|
||||
}
|
||||
let worker_ref = WorkerRef::new(request.worker_id);
|
||||
let lease = self.worker_operation_lock(request.worker_id)?;
|
||||
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
self.ensure_worker_in_workspace(scope, &worker_ref)?;
|
||||
let backend = {
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
let worker = state.worker(&worker_ref)?;
|
||||
if worker.has_pending_lifecycle_operation() {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject lifecycle is unresolved".into(),
|
||||
));
|
||||
}
|
||||
if let Some((saved, _)) = state
|
||||
.subject_release_operations
|
||||
.get(&(request.worker_id, request.operation_id.clone()))
|
||||
{
|
||||
if saved.operation_id != request.operation_id
|
||||
|| saved.binding_id != request.binding_id
|
||||
|| saved.workspace_id != request.workspace_id
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"different Subject release is retained".into(),
|
||||
));
|
||||
}
|
||||
} else if request.action != Action::Reserve {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject reservation is absent or lost; retain occupancy".into(),
|
||||
));
|
||||
} else {
|
||||
state.ensure_subject_not_reserved(&worker_ref, false)?;
|
||||
}
|
||||
state.execution_backend.clone().ok_or_else(|| {
|
||||
RuntimeError::InvalidRequest("actual Controller reservation is unavailable".into())
|
||||
})?
|
||||
};
|
||||
let key = (request.worker_id, request.operation_id.clone());
|
||||
{
|
||||
let mut state = self.lock()?;
|
||||
if let Some((_, saved)) = state.subject_release_operations.get(&key) {
|
||||
if *saved == runtime_api::SubjectReleaseState::Cancelled {
|
||||
if request.action != Action::Cancel {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"cancelled release cannot be rearmed; use a new operation".into(),
|
||||
));
|
||||
}
|
||||
return Ok(runtime_api::SubjectReleaseResponse {
|
||||
operation_id: request.operation_id,
|
||||
binding_id: request.binding_id,
|
||||
state: *saved,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Persist admission exclusion before any Controller ACK, under the Worker op
|
||||
// lock. Restart retains this fence without manufacturing disconnect proof.
|
||||
state.subject_release_operations.insert(
|
||||
key.clone(),
|
||||
(request.clone(), runtime_api::SubjectReleaseState::Reserved),
|
||||
);
|
||||
}
|
||||
state.persist_runtime_snapshot()?;
|
||||
}
|
||||
let response = backend
|
||||
.subject_release(&request)
|
||||
.map_err(RuntimeError::InvalidRequest)?;
|
||||
if response.binding_id != request.binding_id
|
||||
|| response.operation_id != request.operation_id
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject release acknowledgment mismatch".into(),
|
||||
));
|
||||
}
|
||||
let mut state = self.lock()?;
|
||||
let before = state
|
||||
.subject_release_operations
|
||||
.insert(key.clone(), (request.clone(), response.state));
|
||||
if let Err(error) = state.persist_runtime_snapshot() {
|
||||
if let Some(before) = before {
|
||||
state.subject_release_operations.insert(key, before);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn reserve_workdir_use<'a>(
|
||||
&self,
|
||||
ids: impl IntoIterator<Item = &'a str>,
|
||||
@@ -1961,6 +2064,8 @@ impl Runtime {
|
||||
let _operation_guard = operation_lock
|
||||
.lock()
|
||||
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
self.lock()?
|
||||
.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
if let Some(scope) = scope {
|
||||
let mut state = self.lock()?;
|
||||
if !state
|
||||
@@ -2332,6 +2437,8 @@ impl Runtime {
|
||||
mode: WorkerRestoreMode,
|
||||
preparation_authorized: bool,
|
||||
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
|
||||
self.lock()?
|
||||
.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
let owner = {
|
||||
let state = self.lock()?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
@@ -2586,6 +2693,8 @@ impl Runtime {
|
||||
worker_ref: &WorkerRef,
|
||||
mode: WorkerRestoreMode,
|
||||
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
|
||||
self.lock()?
|
||||
.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
let fresh_operation_id = {
|
||||
let state = self.lock()?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
@@ -3007,6 +3116,7 @@ impl Runtime {
|
||||
|
||||
fn ensure_worker_operation_settled(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
|
||||
let state = self.lock()?;
|
||||
state.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
if let Some(operation) = worker.pending_lifecycle_operation_name() {
|
||||
return Err(RuntimeError::WorkerExecutionUnavailable {
|
||||
@@ -3803,6 +3913,7 @@ impl Runtime {
|
||||
self.ensure_worker_in_workspace(scope, worker_ref)?;
|
||||
}
|
||||
|
||||
self.lock()?.ensure_subject_not_reserved(worker_ref, true)?;
|
||||
let (backend, pending_stop, pending_restore, status) = {
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
@@ -3980,6 +4091,7 @@ impl Runtime {
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
state.ensure_worker_ref(worker_ref)?;
|
||||
state.ensure_subject_not_reserved(worker_ref, true)?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
if let Some(operation) = worker.pending_lifecycle_operation_name() {
|
||||
return Err(RuntimeError::InvalidRequest(format!(
|
||||
@@ -4270,7 +4382,10 @@ impl Runtime {
|
||||
.workers
|
||||
.values()
|
||||
.filter(|worker| {
|
||||
worker.restore_guard.active_request_id.is_none()
|
||||
state
|
||||
.ensure_subject_not_reserved(&worker.worker_ref, false)
|
||||
.is_ok()
|
||||
&& worker.restore_guard.active_request_id.is_none()
|
||||
&& worker.pending_restore.is_none()
|
||||
&& worker.pending_stop.is_none()
|
||||
&& worker.execution_bound
|
||||
@@ -4313,7 +4428,12 @@ impl Runtime {
|
||||
state
|
||||
.workers
|
||||
.values()
|
||||
.filter(|worker| worker.pending_stop.is_none())
|
||||
.filter(|worker| {
|
||||
worker.pending_stop.is_none()
|
||||
&& state
|
||||
.ensure_subject_not_reserved(&worker.worker_ref, false)
|
||||
.is_ok()
|
||||
})
|
||||
.filter_map(|worker| {
|
||||
worker
|
||||
.pending_restore
|
||||
@@ -5362,6 +5482,13 @@ struct RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
next_diagnostic_id: u64,
|
||||
workers: BTreeMap<WorkerId, WorkerRecord>,
|
||||
subject_release_operations: BTreeMap<
|
||||
(WorkerId, String),
|
||||
(
|
||||
runtime_api::SubjectReleaseRequest,
|
||||
runtime_api::SubjectReleaseState,
|
||||
),
|
||||
>,
|
||||
workdir_operations: BTreeMap<String, WorkdirOperation>,
|
||||
workspace_owners: BTreeMap<String, String>,
|
||||
config_bundles: BTreeMap<String, ConfigBundle>,
|
||||
@@ -5392,6 +5519,7 @@ impl RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
next_diagnostic_id: 1,
|
||||
workers: BTreeMap::new(),
|
||||
subject_release_operations: BTreeMap::new(),
|
||||
workdir_operations: BTreeMap::new(),
|
||||
workspace_owners: BTreeMap::new(),
|
||||
config_bundles: BTreeMap::new(),
|
||||
@@ -5423,6 +5551,7 @@ impl RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
next_diagnostic_id: 1,
|
||||
workers: BTreeMap::new(),
|
||||
subject_release_operations: BTreeMap::new(),
|
||||
workdir_operations: BTreeMap::new(),
|
||||
workspace_owners: BTreeMap::new(),
|
||||
config_bundles: BTreeMap::new(),
|
||||
@@ -5568,6 +5697,16 @@ impl RuntimeState {
|
||||
workspace_backend_resource_clients: BTreeMap::new(),
|
||||
next_diagnostic_id,
|
||||
workers,
|
||||
subject_release_operations: persisted
|
||||
.subject_releases
|
||||
.into_iter()
|
||||
.map(|(request, state)| {
|
||||
(
|
||||
(request.worker_id, request.operation_id.clone()),
|
||||
(request, state),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
workdir_operations: BTreeMap::new(),
|
||||
config_bundles: BTreeMap::new(),
|
||||
workspace_config_latest: BTreeMap::new(),
|
||||
@@ -5588,6 +5727,7 @@ impl RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
fn persisted_state(&self) -> PersistedRuntimeState {
|
||||
PersistedRuntimeState {
|
||||
subject_releases: self.subject_release_operations.values().cloned().collect(),
|
||||
display_name: self.display_name.clone(),
|
||||
status: self.status,
|
||||
next_diagnostic_id: self.next_diagnostic_id,
|
||||
@@ -5731,6 +5871,28 @@ impl RuntimeState {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_subject_not_reserved(
|
||||
&self,
|
||||
worker: &WorkerRef,
|
||||
allow_disconnected: bool,
|
||||
) -> Result<(), RuntimeError> {
|
||||
use runtime_api::SubjectReleaseState as State;
|
||||
if self
|
||||
.subject_release_operations
|
||||
.iter()
|
||||
.any(|((id, _), (_, state))| {
|
||||
*id == worker.worker_id
|
||||
&& (*state == State::Reserved
|
||||
|| (!allow_disconnected && *state == State::Disconnected))
|
||||
})
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject Controller is reserved for release or disconnected".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_worker_ref(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
|
||||
if !self.workers.contains_key(&worker_ref.worker_id) {
|
||||
return Err(RuntimeError::WorkerNotFound {
|
||||
@@ -6777,8 +6939,7 @@ fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), R
|
||||
.validate()
|
||||
.map_err(|message| RuntimeError::InvalidRequest(message.into()))?;
|
||||
if request.workspace_api.is_none()
|
||||
|| ((binding.subjektiv_consolidation || binding.subjektiv_journal)
|
||||
&& !request.subjektiv_attached)
|
||||
|| (binding.subjektiv_consolidation && !request.subjektiv_attached)
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest("Backend Job capability requires Workspace authority and consolidation requires trusted subject attachment".into()));
|
||||
}
|
||||
@@ -8206,6 +8367,21 @@ mod tests {
|
||||
}
|
||||
|
||||
impl WorkerExecutionBackend for TestExecutionBackend {
|
||||
fn subject_release(
|
||||
&self,
|
||||
request: &runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
use runtime_api::{SubjectReleaseAction as A, SubjectReleaseState as S};
|
||||
Ok(runtime_api::SubjectReleaseResponse {
|
||||
operation_id: request.operation_id.clone(),
|
||||
binding_id: request.binding_id.clone(),
|
||||
state: match request.action {
|
||||
A::Reserve => S::Reserved,
|
||||
A::Cancel => S::Cancelled,
|
||||
A::Disconnect => S::Disconnected,
|
||||
},
|
||||
})
|
||||
}
|
||||
fn cleanup_working_directory(
|
||||
&self,
|
||||
_id: &str,
|
||||
@@ -13106,6 +13282,83 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
// Persistence component test; real Controller reservation has separate coverage.
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn subject_release_fence_survives_reopen_without_automatic_or_explicit_restore() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path())
|
||||
.with_runtime_id("test-runtime");
|
||||
let scoped = scope("workspace-a", "server-a");
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||
options.clone(),
|
||||
Arc::new(TestExecutionBackend::default()),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let worker = runtime
|
||||
.create_worker_scoped(
|
||||
&scoped,
|
||||
scoped_task_request("release durable", "workspace-a"),
|
||||
)
|
||||
.unwrap();
|
||||
let mut request = runtime_api::SubjectReleaseRequest {
|
||||
workspace_id: "workspace-a".into(),
|
||||
worker_id: worker.worker_id,
|
||||
binding_id: "writer-1".into(),
|
||||
operation_id: "release-1".into(),
|
||||
action: runtime_api::SubjectReleaseAction::Reserve,
|
||||
};
|
||||
runtime
|
||||
.subject_release_scoped(&scoped, request.clone())
|
||||
.unwrap();
|
||||
drop(runtime);
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
let reopened =
|
||||
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
|
||||
assert!(
|
||||
reopened
|
||||
.send_input(&worker.worker_ref, WorkerInput::user("must remain fenced"))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.restore_worker(
|
||||
&worker.worker_ref,
|
||||
reopened.test_restore_request(&worker.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
request.action = runtime_api::SubjectReleaseAction::Cancel;
|
||||
assert_eq!(
|
||||
reopened
|
||||
.subject_release_scoped(&scoped, request.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
runtime_api::SubjectReleaseState::Cancelled
|
||||
);
|
||||
request.action = runtime_api::SubjectReleaseAction::Reserve;
|
||||
assert!(
|
||||
reopened
|
||||
.subject_release_scoped(&scoped, request.clone())
|
||||
.is_err(),
|
||||
"cancelled identity must never rearm"
|
||||
);
|
||||
drop(reopened);
|
||||
let reopened = Runtime::with_fs_store_and_execution_backend(
|
||||
options,
|
||||
Arc::new(TestExecutionBackend::default()),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
reopened
|
||||
.subject_release_scoped(&scoped, request.clone())
|
||||
.is_err(),
|
||||
"terminal cancellation survives restart"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn fs_store_restores_active_worker_executions() {
|
||||
|
||||
@@ -492,8 +492,12 @@ pub(crate) fn open_session_public_material_reader(
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
type SpawnModelClientFactory =
|
||||
Arc<dyn Fn() -> Box<dyn agen::llm_client::client::LlmClient> + Send + Sync>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ProfileRuntimeWorkerFactory {
|
||||
spawn_model_client_factory: Option<SpawnModelClientFactory>,
|
||||
observation_hub: Arc<RuntimeWorkerObservationHub>,
|
||||
profile_base_dir: PathBuf,
|
||||
worker_aggregate_root: Option<PathBuf>,
|
||||
@@ -521,6 +525,7 @@ impl ProfileRuntimeWorkerFactory {
|
||||
pub fn new(profile_base_dir: impl Into<PathBuf>) -> Self {
|
||||
let profile_base_dir = profile_base_dir.into();
|
||||
Self {
|
||||
spawn_model_client_factory: None,
|
||||
observation_hub: Arc::new(RuntimeWorkerObservationHub::default()),
|
||||
profile_base_dir,
|
||||
worker_aggregate_root: None,
|
||||
@@ -535,6 +540,17 @@ impl ProfileRuntimeWorkerFactory {
|
||||
}
|
||||
}
|
||||
|
||||
/// Supply the external model boundary for newly spawned Controllers. The factory is
|
||||
/// process-owned, never persisted into a Profile or restored as execution authority.
|
||||
/// Restores continue to use their saved model configuration.
|
||||
pub fn with_spawn_model_client_factory(
|
||||
mut self,
|
||||
factory: impl Fn() -> Box<dyn agen::llm_client::client::LlmClient> + Send + Sync + 'static,
|
||||
) -> Self {
|
||||
self.spawn_model_client_factory = Some(Arc::new(factory));
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_runtime_id(mut self, runtime_id: impl Into<String>) -> Self {
|
||||
self.runtime_id = Some(runtime_id.into());
|
||||
self
|
||||
@@ -1502,7 +1518,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
|
||||
let run_dir = self.worker_run_dir(&request.worker_ref)?;
|
||||
let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id);
|
||||
let mut prepared = WorkerBootstrap::new(
|
||||
let bootstrap = WorkerBootstrap::new(
|
||||
manifest,
|
||||
store,
|
||||
loader,
|
||||
@@ -1513,10 +1529,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
bash_output_dir,
|
||||
},
|
||||
self.controller_transport,
|
||||
)
|
||||
.prepare()
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
);
|
||||
let bootstrap = match &self.spawn_model_client_factory {
|
||||
Some(factory) => bootstrap.with_model_client(factory()),
|
||||
None => bootstrap,
|
||||
};
|
||||
let mut prepared = bootstrap.prepare().await.map_err(|error| match error {
|
||||
WorkerBootstrapError::Worker(source) => {
|
||||
format!("failed to create Worker from profile: {source}")
|
||||
}
|
||||
@@ -2331,6 +2349,12 @@ impl PendingRuntimeRestore {
|
||||
}
|
||||
}
|
||||
|
||||
struct SubjectControllerReservation {
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
guard: Option<worker::SubjectReleaseReservation>,
|
||||
state: runtime_api::SubjectReleaseState,
|
||||
}
|
||||
|
||||
/// Worker execution backend backed by real worker crate Workers.
|
||||
pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
||||
backend_id: String,
|
||||
@@ -2341,6 +2365,7 @@ pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
||||
pending_restores: Mutex<HashMap<WorkerRef, Arc<PendingRuntimeRestore>>>,
|
||||
worker_locks: Mutex<HashMap<WorkerRef, Arc<Mutex<()>>>>,
|
||||
spawn_restore_timeout: Duration,
|
||||
subject_reservations: Mutex<HashMap<(WorkerRef, String), SubjectControllerReservation>>,
|
||||
shutdown_admission_timeout: Duration,
|
||||
}
|
||||
|
||||
@@ -2363,6 +2388,7 @@ where
|
||||
pending_restores: Mutex::new(HashMap::new()),
|
||||
worker_locks: Mutex::new(HashMap::new()),
|
||||
spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT,
|
||||
subject_reservations: Mutex::new(HashMap::new()),
|
||||
shutdown_admission_timeout: Duration::from_secs(5),
|
||||
})
|
||||
}
|
||||
@@ -3343,6 +3369,119 @@ where
|
||||
&self.backend_id
|
||||
}
|
||||
|
||||
fn subject_release(
|
||||
&self,
|
||||
request: &runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
use runtime_api::{SubjectReleaseAction as Action, SubjectReleaseState as State};
|
||||
let worker_ref = WorkerRef::new(request.worker_id);
|
||||
let lock = self.worker_lock(&worker_ref)?;
|
||||
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
|
||||
let mut reservations = self
|
||||
.subject_reservations
|
||||
.lock()
|
||||
.map_err(|_| "Subject reservation registry poisoned")?;
|
||||
let key = (worker_ref.clone(), request.operation_id.clone());
|
||||
if let Some(saved) = reservations.get(&key) {
|
||||
if saved.request.operation_id != request.operation_id
|
||||
|| saved.request.binding_id != request.binding_id
|
||||
|| saved.request.workspace_id != request.workspace_id
|
||||
{
|
||||
return Err("Worker has another retained Subject release operation".into());
|
||||
}
|
||||
} else {
|
||||
if request.action == Action::Cancel {
|
||||
// Runtime has a durable admission intent, but no Controller reservation may
|
||||
// have been acquired (or this is a new process). Under the same Worker op
|
||||
// lock there is no late local Reserve left to publish. Retain a terminal
|
||||
// cancellation receipt so an old Reserve can never rearm this identity.
|
||||
reservations.insert(
|
||||
key,
|
||||
SubjectControllerReservation {
|
||||
request: request.clone(),
|
||||
guard: None,
|
||||
state: State::Cancelled,
|
||||
},
|
||||
);
|
||||
return Ok(runtime_api::SubjectReleaseResponse {
|
||||
operation_id: request.operation_id.clone(),
|
||||
binding_id: request.binding_id.clone(),
|
||||
state: State::Cancelled,
|
||||
});
|
||||
}
|
||||
if request.action != Action::Reserve {
|
||||
return Err(
|
||||
"Subject reservation is missing; no status-based recovery is allowed".into(),
|
||||
);
|
||||
}
|
||||
if reservations.iter().any(|((worker, _), saved)| {
|
||||
*worker == worker_ref && saved.state != State::Cancelled
|
||||
}) {
|
||||
return Err("Worker has another active Subject reservation".into());
|
||||
}
|
||||
if self
|
||||
.pending_restores
|
||||
.lock()
|
||||
.map_err(|_| "restore registry poisoned")?
|
||||
.contains_key(&worker_ref)
|
||||
{
|
||||
return Err("Subject Controller restore is unresolved".into());
|
||||
}
|
||||
let workers = self
|
||||
.workers
|
||||
.lock()
|
||||
.map_err(|_| "worker registry poisoned")?;
|
||||
let execution = workers
|
||||
.get(&worker_ref)
|
||||
.ok_or("Subject Controller is unavailable")?;
|
||||
if execution.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|
||||
|| execution.shutdown_requested.load(Ordering::Acquire)
|
||||
|| execution.handle.protocol_is_closed()
|
||||
{
|
||||
return Err("Subject Controller is unavailable or Workspace mismatched".into());
|
||||
}
|
||||
let handle = execution.handle.clone();
|
||||
drop(workers);
|
||||
let guard =
|
||||
self.run_on_adapter_runtime(async move { handle.reserve_subject_release().await })?;
|
||||
reservations.insert(
|
||||
key.clone(),
|
||||
SubjectControllerReservation {
|
||||
request: request.clone(),
|
||||
guard: Some(guard),
|
||||
state: State::Reserved,
|
||||
},
|
||||
);
|
||||
}
|
||||
let saved = reservations.get_mut(&key).expect("reservation inserted");
|
||||
match (request.action, saved.state) {
|
||||
(Action::Reserve, State::Reserved | State::Disconnected) => {}
|
||||
(Action::Disconnect, State::Reserved) => {
|
||||
saved
|
||||
.guard
|
||||
.as_mut()
|
||||
.ok_or("Controller reservation missing")?
|
||||
.commit_disconnect()?;
|
||||
saved.state = State::Disconnected;
|
||||
}
|
||||
(Action::Disconnect, State::Disconnected) | (Action::Cancel, State::Cancelled) => {}
|
||||
(Action::Cancel, State::Reserved) => {
|
||||
saved.guard.take();
|
||||
saved.state = State::Cancelled;
|
||||
}
|
||||
_ => {
|
||||
return Err(
|
||||
"Subject release operation already reached a different terminal state".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(runtime_api::SubjectReleaseResponse {
|
||||
operation_id: request.operation_id.clone(),
|
||||
binding_id: request.binding_id.clone(),
|
||||
state: saved.state,
|
||||
})
|
||||
}
|
||||
|
||||
fn fetch_workspace_config(
|
||||
&self,
|
||||
request: WorkspaceConfigFetchRequest,
|
||||
@@ -4932,7 +5071,7 @@ mod tests {
|
||||
let aggregate_dir = root.join("workers").join(worker_ref.worker_id.to_string());
|
||||
let session_store = WorkerSessionStore::new(aggregate_dir.join("session")).unwrap();
|
||||
let store = WorkerAggregateStore::new(&aggregate_dir, &worker_name).unwrap();
|
||||
let (saved, request) = saved_subjektiv_policy(root, &worker_ref);
|
||||
let (saved, mut request) = saved_subjektiv_policy(root, &worker_ref);
|
||||
let client = MockClient::new(Vec::new());
|
||||
let mut engine =
|
||||
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||
@@ -4979,11 +5118,7 @@ mod tests {
|
||||
Some(SAVED_SUBJEKTIV_SYSTEM_PROMPT)
|
||||
);
|
||||
} else {
|
||||
assert!(matches!(
|
||||
session_store.exists(session_id, segment_id),
|
||||
Err(session_store::StoreError::Corrupt { message, .. })
|
||||
if message.contains("no materialized Session")
|
||||
));
|
||||
assert!(!session_store.exists(session_id, segment_id).unwrap());
|
||||
}
|
||||
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
|
||||
drop(worker);
|
||||
@@ -5007,8 +5142,46 @@ mod tests {
|
||||
metadata.workspace_id.as_deref(),
|
||||
Some("workspace-saved-subjektiv")
|
||||
);
|
||||
// Attribution already committed before restart; the real restore needs
|
||||
// no network fixture or replacement current Profile.
|
||||
// Attribution history is already committed. Each new Controller must still
|
||||
// obtain current publication authority, even for a Confirmed Session.
|
||||
// This is a transport fixture for restore/stop component tests, not an
|
||||
// assertion that a real Backend authorized a released Subject body.
|
||||
#[cfg(feature = "http-server")]
|
||||
{
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
request.workspace_api.as_mut().unwrap().base_url =
|
||||
format!("http://{}", listener.local_addr().unwrap());
|
||||
let app = axum::Router::new().route(
|
||||
"/api/w/workspace-saved-subjektiv/subjektiv/sessions",
|
||||
axum::routing::post(
|
||||
move |axum::Json(input): axum::Json<
|
||||
server_api::SubjektivRecordSessionRequest,
|
||||
>| async move {
|
||||
assert!(
|
||||
!input.create_if_missing,
|
||||
"publication must not rewrite attribution"
|
||||
);
|
||||
assert_eq!(input.session_id, session_id.to_string());
|
||||
axum::Json(server_api::SubjektivRecordSessionResponse {
|
||||
session_id: input.session_id,
|
||||
subject_id: "saved-subject".into(),
|
||||
})
|
||||
},
|
||||
),
|
||||
);
|
||||
let app = app.route(
|
||||
"/api/w/workspace-saved-subjektiv/subjektiv/journal/startup",
|
||||
axum::routing::get(move || async move {
|
||||
assert!(!materialize_head, "normal restore must not refetch startup");
|
||||
axum::Json(serde_json::json!({
|
||||
"subject_id": "saved-subject", "journals": [], "omitted": false,
|
||||
}))
|
||||
}),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
}
|
||||
metadata.subjektiv_session_attribution =
|
||||
Some(session_store::SubjektivSessionAttributionState::Confirmed {
|
||||
session_id,
|
||||
@@ -6077,7 +6250,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
async fn profile_factory_saved_subjektiv_manifest_survives_restart_and_explicit_restore() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
@@ -6147,7 +6320,7 @@ mod tests {
|
||||
assert!(!root.path().join("nonexistent-current-profiles").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
async fn profile_factory_pending_subjektiv_restore_requires_operation_owned_launch_material() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
@@ -9681,6 +9854,158 @@ mod tests {
|
||||
assert!(error.to_string().contains("WorkerNotify"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_release_reserves_real_controller_and_excludes_runtime_lifecycle() {
|
||||
use runtime_api::{SubjectReleaseAction as Action, SubjectReleaseState as State};
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
let cwd = tempfile::tempdir().unwrap();
|
||||
let store = tempfile::tempdir().unwrap();
|
||||
let backend = Arc::new(
|
||||
WorkerRuntimeExecutionBackend::new(MockFactory {
|
||||
client: MockClient::new(simple_text_events()),
|
||||
runtime_base: runtime_base.path().to_path_buf(),
|
||||
cwd: cwd.path().to_path_buf(),
|
||||
store_dir: store.path().join("sessions"),
|
||||
worker_metadata_dir: store.path().join("workers"),
|
||||
observed_cwds: Arc::new(Mutex::new(Vec::new())),
|
||||
observed_workspace_clients: Arc::new(Mutex::new(Vec::new())),
|
||||
})
|
||||
.unwrap(),
|
||||
);
|
||||
let runtime =
|
||||
EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend.clone())
|
||||
.unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let mut create = create_request("subject-release");
|
||||
create.workspace_api = Some(crate::catalog::WorkspaceApiRef {
|
||||
workspace_id: "ws-test".into(),
|
||||
base_url: "http://127.0.0.1:3999".into(),
|
||||
});
|
||||
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||
workspace_id: "ws-test".into(),
|
||||
language: "English".into(),
|
||||
});
|
||||
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "embedded-backend");
|
||||
let detail = runtime.create_worker_scoped(&scope, create).unwrap();
|
||||
let mut release = runtime_api::SubjectReleaseRequest {
|
||||
workspace_id: "ws-test".into(),
|
||||
worker_id: detail.worker_ref.worker_id,
|
||||
binding_id: "original-binding".into(),
|
||||
operation_id: "subject-release-1".into(),
|
||||
action: Action::Reserve,
|
||||
};
|
||||
let reserved = runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap();
|
||||
assert_eq!(reserved.state, State::Reserved);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap(),
|
||||
reserved
|
||||
);
|
||||
let before = backend
|
||||
.workers
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&detail.worker_ref)
|
||||
.unwrap()
|
||||
.handle
|
||||
.committed_entries();
|
||||
let blocked = runtime
|
||||
.send_input(
|
||||
&detail.worker_ref,
|
||||
WorkerInput::user("must not enter history"),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(blocked.to_string().contains("Subject"));
|
||||
assert!(
|
||||
runtime
|
||||
.stop_worker(&detail.worker_ref, None)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("Subject")
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.delete_worker(&detail.worker_ref)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("Subject")
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.restore_worker(
|
||||
&detail.worker_ref,
|
||||
runtime.test_restore_request(&detail.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let after = backend
|
||||
.workers
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&detail.worker_ref)
|
||||
.unwrap()
|
||||
.handle
|
||||
.committed_entries();
|
||||
assert_eq!(
|
||||
serde_json::to_value(after).unwrap(),
|
||||
serde_json::to_value(before).unwrap()
|
||||
);
|
||||
let mut conflicting = release.clone();
|
||||
conflicting.binding_id = "later-binding".into();
|
||||
assert!(runtime.subject_release_scoped(&scope, conflicting).is_err());
|
||||
release.action = Action::Cancel;
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
State::Cancelled
|
||||
);
|
||||
release.operation_id = "subject-release-2".into();
|
||||
release.action = Action::Reserve;
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
State::Reserved
|
||||
);
|
||||
release.action = Action::Disconnect;
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
State::Disconnected
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
State::Disconnected
|
||||
);
|
||||
release.action = Action::Cancel;
|
||||
assert!(runtime.subject_release_scoped(&scope, release).is_err());
|
||||
assert!(
|
||||
runtime
|
||||
.send_input(&detail.worker_ref, WorkerInput::user("after disconnect"))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.restore_worker(
|
||||
&detail.worker_ref,
|
||||
runtime.test_restore_request(&detail.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
runtime.stop_worker(&detail.worker_ref, None).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stopped_runtime_worker_can_restore_and_accept_input() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
|
||||
@@ -78,7 +78,45 @@ impl WorkerHandle {
|
||||
self.method_tx.same_channel(&other.method_tx)
|
||||
}
|
||||
|
||||
/// Ask the actual Controller to acknowledge quiescence. No catalog status,
|
||||
/// profile, or model protocol message grants this Host lifecycle capability.
|
||||
pub async fn reserve_subject_release(
|
||||
&self,
|
||||
) -> Result<crate::SubjectReleaseReservation, String> {
|
||||
if self.protocol_is_closed() {
|
||||
return Err("Controller is unavailable".into());
|
||||
}
|
||||
if self.shared_state.snapshot().state != WorkerState::Idle {
|
||||
return Err("Controller is not Idle".into());
|
||||
}
|
||||
let control = self
|
||||
.shared_state
|
||||
.subject_release
|
||||
.get()
|
||||
.ok_or("Controller release authority unavailable")?;
|
||||
tokio::select! {
|
||||
result = control.request() => result,
|
||||
_ = self.method_tx.closed() => Err("Controller closed before release acknowledgement".into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn send(&self, method: Method) -> Result<(), mpsc::error::SendError<Method>> {
|
||||
// The lease covers enqueue itself. The Controller's exclusive barrier
|
||||
// cannot overlook an accepted send between its queue check and ack.
|
||||
let _admission = if release_fenced_method(&method) {
|
||||
match self
|
||||
.shared_state
|
||||
.subject_release
|
||||
.get()
|
||||
.map(|control| control.authority.enter())
|
||||
.transpose()
|
||||
{
|
||||
Ok(lease) => lease,
|
||||
Err(_) => return Err(mpsc::error::SendError(method)),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Reject known-busy sends before channel dispatch, including maintenance
|
||||
// that cannot receive methods until it finishes. The controller checks
|
||||
// again at admission to cover an Idle -> busy race after this snapshot.
|
||||
@@ -290,6 +328,49 @@ impl WorkerHandle {
|
||||
}
|
||||
}
|
||||
|
||||
fn answer_idle_release<St: Store + Clone>(
|
||||
control: &Arc<crate::subject_release::ReleaseControl>,
|
||||
shared_state: &WorkerSharedState,
|
||||
methods: &mpsc::Receiver<Method>,
|
||||
staged: bool,
|
||||
pending: &PendingSubmissionHandle<St>,
|
||||
notifications: &NotifyBuffer,
|
||||
attention: bool,
|
||||
) {
|
||||
control.respond(|| {
|
||||
// Read ingress and durable pending state only after acquiring exclusivity.
|
||||
// Actor-local staged/deferred/attention flags cannot change during this call.
|
||||
if shared_state.snapshot().state != WorkerState::Idle
|
||||
|| staged
|
||||
|| attention
|
||||
|| !methods.is_empty()
|
||||
|| methods.is_closed()
|
||||
|| !notifications.is_empty()
|
||||
|| pending.has_unfinished_admission()
|
||||
{
|
||||
return Err("Controller has pending, staged, busy, or unavailable work".into());
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
|
||||
fn release_fenced_method(method: &Method) -> bool {
|
||||
!matches!(
|
||||
method,
|
||||
Method::Shutdown { .. }
|
||||
| Method::ListWorkers
|
||||
| Method::ListPendingSubmissions
|
||||
| Method::ListCompletions { .. }
|
||||
)
|
||||
}
|
||||
|
||||
async fn release_requested(shared_state: &WorkerSharedState) {
|
||||
match shared_state.subject_release.get() {
|
||||
Some(control) => control.notified().await,
|
||||
None => std::future::pending::<()>().await,
|
||||
}
|
||||
}
|
||||
|
||||
fn command_admission_disposition(
|
||||
admission: WorkerCommandAdmission,
|
||||
) -> Result<(), WorkerCommandDisposition> {
|
||||
@@ -802,6 +883,10 @@ impl WorkerController {
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
worker
|
||||
.validate_subjektiv_body_publication()
|
||||
.map_err(std::io::Error::other)?;
|
||||
|
||||
// === 1. Initialization (channels / RuntimeDir / worker-immutable
|
||||
// snapshots / SpawnedWorkerRegistry / alerter attach /
|
||||
// bash-output scope) ===
|
||||
@@ -911,13 +996,13 @@ impl WorkerController {
|
||||
// the actual registered set instead of a hand-maintained mirror.
|
||||
worker.engine().tool_server_handle().flush_pending();
|
||||
|
||||
// Runtime-owned Workers persist their current model-visible head before
|
||||
// Fresh Workers persist their current model-visible head before
|
||||
// controller exposure. Feature installation above must happen first so a
|
||||
// fresh subject Worker captures the current resident surface in its durable
|
||||
// prompt. Restored Workers defer generic Feature restore notification until
|
||||
// the next model-visible context materialization; only an installed
|
||||
// Feature may then request a typed durable history append.
|
||||
if runtime_managed && worker.needs_initial_session_head_materialization() {
|
||||
if worker.needs_initial_session_head_materialization() {
|
||||
worker
|
||||
.materialize_durable_session_head()
|
||||
.await
|
||||
@@ -958,6 +1043,13 @@ impl WorkerController {
|
||||
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
|
||||
let session_id = worker.session_id();
|
||||
let pending_activations = worker.pending_activation_state();
|
||||
let release_control =
|
||||
crate::subject_release::ReleaseControl::new(worker.mutation_authority());
|
||||
shared_state
|
||||
.subject_release
|
||||
.set(release_control)
|
||||
.ok()
|
||||
.expect("one Controller release authority");
|
||||
let handle = WorkerHandle {
|
||||
method_tx,
|
||||
working_event_tx: working_event_tx.clone(),
|
||||
@@ -1561,6 +1653,18 @@ where
|
||||
{
|
||||
feature_registry.add_module(subjektiv_sessions);
|
||||
}
|
||||
if ordinary_subjektiv_features_enabled
|
||||
&& lifecycle_enabled
|
||||
&& let Some(host) = subjektiv_host.as_ref()
|
||||
&& host.host.journal_startup_granted()
|
||||
{
|
||||
worker.install_initial_context_source(Arc::new(
|
||||
crate::feature::builtin::journal::JournalStartupContextSource::new(
|
||||
host.clone(),
|
||||
worker.prompts(),
|
||||
),
|
||||
));
|
||||
}
|
||||
if ordinary_subjektiv_features_enabled
|
||||
&& lifecycle_enabled
|
||||
&& let Some(host) = subjektiv_host.as_ref()
|
||||
@@ -2232,6 +2336,12 @@ async fn controller_loop<C, St>(
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + 'static,
|
||||
{
|
||||
let release_control = shared_state
|
||||
.subject_release
|
||||
.get()
|
||||
.expect("Controller release authority installed")
|
||||
.clone();
|
||||
let _release_lifetime = crate::subject_release::ControllerLifetime(release_control.clone());
|
||||
// Hold an optional external attach server alive for the controller lifetime.
|
||||
// In-process runtimes retain and drive the WorkerHandle directly.
|
||||
let _socket_server = socket_server;
|
||||
@@ -2452,6 +2562,13 @@ async fn controller_loop<C, St>(
|
||||
break 'controller;
|
||||
}
|
||||
},
|
||||
_ = release_control.notified() => {
|
||||
answer_idle_release(&release_control, &shared_state, &method_rx,
|
||||
pending.is_some() || !deferred_methods.is_empty(),
|
||||
&pending_submissions, ¬ify_buffer,
|
||||
worker.has_pending_compaction_cleanup() || worker.has_failed_segment_activation());
|
||||
continue;
|
||||
}
|
||||
_ = notify_buffer.notified() => continue,
|
||||
_ = tokio::time::sleep(remaining) => {
|
||||
match prepare_pending_run(
|
||||
@@ -2486,10 +2603,32 @@ async fn controller_loop<C, St>(
|
||||
break 'controller;
|
||||
}
|
||||
},
|
||||
_ = release_control.notified() => {
|
||||
answer_idle_release(&release_control, &shared_state, &method_rx,
|
||||
pending.is_some() || !deferred_methods.is_empty(),
|
||||
&pending_submissions, ¬ify_buffer,
|
||||
worker.has_pending_compaction_cleanup() || worker.has_failed_segment_activation());
|
||||
continue;
|
||||
}
|
||||
_ = notify_buffer.notified() => continue,
|
||||
}
|
||||
};
|
||||
|
||||
let _method_admission = if release_fenced_method(&method) {
|
||||
match release_control.authority.enter() {
|
||||
Ok(lease) => Some(lease),
|
||||
Err(error) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: error.to_string(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if shared_state.snapshot().state != WorkerState::Idle
|
||||
&& reject_idle_only_submission(&method, &working_event_tx)
|
||||
{
|
||||
@@ -2885,6 +3024,7 @@ async fn controller_loop<C, St>(
|
||||
let mut compact = Box::pin(worker.manual_compact_with_cancel(cancel_rx));
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = release_control.notified() => { release_control.reject_busy(); }
|
||||
result = &mut compact => break result,
|
||||
method = method_rx.recv() => {
|
||||
match method {
|
||||
@@ -3142,6 +3282,7 @@ async fn controller_loop<C, St>(
|
||||
}
|
||||
}
|
||||
|
||||
release_control.close();
|
||||
// Close method admission before any fallible child/session cleanup. Existing
|
||||
// senders then fail instead of accepting work that this execution can no
|
||||
// longer process.
|
||||
@@ -3286,6 +3427,9 @@ where
|
||||
// Running only after processing the commit fence. This makes the
|
||||
// Running snapshot contract deterministic even for immediate clients.
|
||||
biased;
|
||||
_ = release_requested(shared_state) => {
|
||||
if let Some(control) = shared_state.subject_release.get() { control.reject_busy(); }
|
||||
}
|
||||
committed = async {
|
||||
input_commit
|
||||
.as_mut()
|
||||
@@ -3906,6 +4050,10 @@ fn worker_error_code(e: &WorkerError) -> ErrorCode {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "controller_subject_release_tests.rs"]
|
||||
mod subject_release_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -4251,7 +4399,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_session_head_is_materialized_after_feature_installation_before_exposure() {
|
||||
fn fresh_session_head_is_materialized_after_feature_installation_before_exposure() {
|
||||
let production = include_str!("controller.rs")
|
||||
.split_once("#[cfg(test)]\nmod tests")
|
||||
.map(|(production, _)| production)
|
||||
@@ -4270,8 +4418,8 @@ mod tests {
|
||||
.find("let handle = WorkerHandle")
|
||||
.expect("controller handle exposure");
|
||||
let materialize_gate = startup[..materialize]
|
||||
.rfind("if runtime_managed && worker.needs_initial_session_head_materialization() {")
|
||||
.expect("fresh runtime-managed materialization gate");
|
||||
.rfind("if worker.needs_initial_session_head_materialization() {")
|
||||
.expect("fresh materialization gate");
|
||||
|
||||
assert!(feature_install < materialize_gate);
|
||||
assert!(materialize_gate < materialize);
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
//! Production Direct bootstrap, Controller admission, pending persistence and FsStore.
|
||||
//! Only the provider is replaced by an explicitly released async boundary; no live Host,
|
||||
//! Journal policy, socket or model continuation semantics are claimed.
|
||||
use super::*;
|
||||
use crate::{
|
||||
PreparedWorker, WorkerBootstrap, WorkerBootstrapLayout, WorkerFilesystemAuthority,
|
||||
WorkerManifest, WorkerWorkspaceContext,
|
||||
};
|
||||
use session_store::{CombinedStore, FsStore, FsWorkerStore};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct HeldProvider {
|
||||
started: mpsc::UnboundedSender<()>,
|
||||
finish: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl LlmClient for HeldProvider {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
async fn stream(
|
||||
&self,
|
||||
_: agen::llm_client::Request,
|
||||
) -> Result<
|
||||
std::pin::Pin<
|
||||
Box<
|
||||
dyn futures_util::Stream<
|
||||
Item = Result<
|
||||
agen::llm_client::event::Event,
|
||||
agen::llm_client::ClientError,
|
||||
>,
|
||||
> + Send,
|
||||
>,
|
||||
>,
|
||||
agen::llm_client::ClientError,
|
||||
> {
|
||||
self.started.send(()).unwrap();
|
||||
self.finish.notified().await;
|
||||
Ok(Box::pin(futures_util::stream::iter([Ok(
|
||||
agen::llm_client::event::Event::Status(agen::llm_client::event::StatusEvent {
|
||||
status: agen::llm_client::event::ResponseStatus::Completed,
|
||||
}),
|
||||
)])))
|
||||
}
|
||||
}
|
||||
async fn prepared(
|
||||
temp: &TempDir,
|
||||
client: HeldProvider,
|
||||
) -> PreparedWorker<Box<dyn LlmClient>, CombinedStore<FsStore, FsWorkerStore>> {
|
||||
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='release-{}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]", uuid::Uuid::now_v7())).unwrap();
|
||||
manifest.compaction = Some(manifest::CompactionConfig {
|
||||
retained_tokens: 0,
|
||||
..Default::default()
|
||||
});
|
||||
WorkerBootstrap::new(
|
||||
manifest,
|
||||
CombinedStore::new(
|
||||
FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||
),
|
||||
crate::PromptCatalogSource::builtins_only(),
|
||||
WorkerWorkspaceContext::no_workspace(),
|
||||
WorkerFilesystemAuthority::None,
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
bash_output_dir: temp.path().join("bash"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(client)
|
||||
.prepare()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
async fn reserve(handle: &WorkerHandle) -> Result<crate::SubjectReleaseReservation, String> {
|
||||
tokio::time::timeout(Duration::from_secs(5), handle.reserve_subject_release())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
async fn shutdown(controller: crate::BootstrappedWorker) {
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Shutdown {
|
||||
command: WorkerCommandEnvelope::new(1),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
controller.controller_task.await.unwrap();
|
||||
}
|
||||
fn extension() -> LogEntry {
|
||||
LogEntry::Extension {
|
||||
ts: 1,
|
||||
domain: "release-test".into(),
|
||||
payload: serde_json::json!({}),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reservation_fences_ingress_fallback_and_history_drop_recovers_commit_stays_fenced() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, mut starts) = mpsc::unbounded_channel();
|
||||
let prepared = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: Arc::default(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let writer = prepared.worker().log_writer_handle();
|
||||
let pending = prepared.worker().pending_submission_handle();
|
||||
let buffer = prepared.worker().notify_buffer_handle();
|
||||
let capture = prepared.worker().committed_session_capture_handle();
|
||||
let extensions = prepared.worker().session_extension_handle();
|
||||
let fallback = durable_parent_notification_target(pending.clone(), buffer.clone());
|
||||
let controller = prepared.start().await.unwrap();
|
||||
let before = controller.handle.committed_entries().len();
|
||||
let expected = capture.capture().unwrap().location();
|
||||
let guard = reserve(&controller.handle).await.unwrap();
|
||||
assert!(
|
||||
controller
|
||||
.handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"blocked"
|
||||
))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "blocked".into()
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
pending
|
||||
.accept_notification("blocked".into(), "blocked".into())
|
||||
.is_err()
|
||||
);
|
||||
fallback.notify("child-session".into(), "must not be admitted".into());
|
||||
assert!(buffer.is_empty());
|
||||
assert!(!pending.has_unfinished_admission());
|
||||
assert!(writer.append_entry(extension()).is_err());
|
||||
assert!(
|
||||
extensions
|
||||
.append_if_current(&expected, "release-test", serde_json::json!({}))
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(controller.handle.committed_entries().len(), before);
|
||||
assert!(starts.try_recv().is_err());
|
||||
drop(guard);
|
||||
writer.append_entry(extension()).unwrap();
|
||||
let mut guard = reserve(&controller.handle).await.unwrap();
|
||||
guard.commit_disconnect().unwrap();
|
||||
guard.commit_disconnect().unwrap();
|
||||
drop(guard);
|
||||
assert!(writer.append_entry(extension()).is_err());
|
||||
assert!(
|
||||
pending
|
||||
.accept_notification("after-commit".into(), "blocked".into())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "blocked".into()
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
let _ = controller.handle.snapshot_event();
|
||||
controller
|
||||
.handle
|
||||
.send(Method::ListPendingSubmissions)
|
||||
.await
|
||||
.unwrap();
|
||||
let handle = controller.handle.clone();
|
||||
shutdown(controller).await;
|
||||
assert!(handle.protocol_is_closed());
|
||||
assert!(reserve(&handle).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_input_cannot_be_overtaken_by_release_and_running_request_is_rejected() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, mut starts) = mpsc::unbounded_channel();
|
||||
let finish = Arc::new(tokio::sync::Notify::new());
|
||||
let controller = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: finish.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.start()
|
||||
.await
|
||||
.unwrap();
|
||||
let mut events = controller.handle.subscribe();
|
||||
// Enqueue succeeds before submitting the separate internal release request.
|
||||
// Either the queue or the held provider is observed; neither is quiescence.
|
||||
controller
|
||||
.handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"accepted first",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
tokio::time::timeout(Duration::from_secs(5), starts.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
|
||||
}).await.unwrap();
|
||||
let guard = reserve(&controller.handle).await.unwrap();
|
||||
drop(guard);
|
||||
shutdown(controller).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_notification_fallback_blocks_release_until_the_actual_turn_finishes() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, mut starts) = mpsc::unbounded_channel();
|
||||
let finish = Arc::new(tokio::sync::Notify::new());
|
||||
let prepared = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: finish.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let pending = prepared.worker().pending_submission_handle();
|
||||
let buffer = prepared.worker().notify_buffer_handle();
|
||||
let fallback = durable_parent_notification_target(pending.clone(), buffer);
|
||||
let controller = prepared.start().await.unwrap();
|
||||
let mut events = controller.handle.subscribe();
|
||||
fallback.notify("child".into(), "durable accepted notification".into());
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
tokio::time::timeout(Duration::from_secs(5), starts.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
|
||||
}).await.unwrap();
|
||||
let guard = reserve(&controller.handle).await.unwrap();
|
||||
drop(guard);
|
||||
shutdown(controller).await;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct HeldRewrite {
|
||||
started: mpsc::UnboundedSender<()>,
|
||||
finish: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
impl crate::feature::FeatureModule for HeldRewrite {
|
||||
fn descriptor(&self) -> crate::feature::FeatureDescriptor {
|
||||
crate::feature::FeatureDescriptor::builtin("held-rewrite", "Held rewrite").with_hook(
|
||||
crate::feature::HookDeclaration::new(
|
||||
"hold",
|
||||
crate::feature::FeatureHookPoint::BeforeSessionRewrite,
|
||||
),
|
||||
)
|
||||
}
|
||||
fn install(
|
||||
&self,
|
||||
context: &mut crate::feature::FeatureInstallContext<'_>,
|
||||
) -> Result<(), crate::feature::FeatureInstallError> {
|
||||
context.hooks().add_before_session_rewrite(
|
||||
"hold",
|
||||
crate::hook::HookExecutionPolicy::fail_closed(),
|
||||
self.clone(),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl crate::hook::Hook<crate::hook::BeforeSessionRewrite> for HeldRewrite {
|
||||
async fn call(
|
||||
&self,
|
||||
context: &crate::hook::BeforeSessionRewriteContext,
|
||||
) -> Result<crate::hook::BeforeSessionRewriteAction, crate::hook::HookError> {
|
||||
if context.kind == crate::hook::SessionRewriteKind::Compact {
|
||||
self.started.send(()).unwrap();
|
||||
self.finish.notified().await;
|
||||
return Ok(crate::hook::BeforeSessionRewriteAction::Deny(
|
||||
"stop before compactor boundary".into(),
|
||||
));
|
||||
}
|
||||
Ok(crate::hook::BeforeSessionRewriteAction::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_maintenance_rejects_release_until_the_real_rewrite_boundary_exits() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, mut starts) = mpsc::unbounded_channel();
|
||||
let finish = Arc::new(tokio::sync::Notify::new());
|
||||
let mut prepared = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: finish.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let (rewrite_started, mut rewrites) = mpsc::unbounded_channel();
|
||||
let rewrite_finish = Arc::new(tokio::sync::Notify::new());
|
||||
let mut registry = FeatureRegistryBuilder::new();
|
||||
registry.add_module(HeldRewrite {
|
||||
started: rewrite_started,
|
||||
finish: rewrite_finish.clone(),
|
||||
});
|
||||
assert!(
|
||||
!prepared
|
||||
.worker_mut()
|
||||
.install_features(registry)
|
||||
.has_errors()
|
||||
);
|
||||
let controller = prepared.start().await.unwrap();
|
||||
let mut events = controller.handle.subscribe();
|
||||
controller
|
||||
.handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"history to compact ".repeat(128),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), starts.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
|
||||
}).await.unwrap();
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Compact {
|
||||
command: WorkerCommandEnvelope::new(1),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), rewrites.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
assert!(matches!(
|
||||
controller.handle.shared_state.snapshot().state,
|
||||
WorkerState::Busy(WorkerBusyState::Maintenance(_))
|
||||
));
|
||||
rewrite_finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
|
||||
}).await.unwrap();
|
||||
let guard = reserve(&controller.handle).await.unwrap();
|
||||
drop(guard);
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Shutdown {
|
||||
command: WorkerCommandEnvelope::new(2),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
controller.controller_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_or_actor_staged_work_cannot_pass_the_exclusive_ack_boundary() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, _) = mpsc::unbounded_channel();
|
||||
let mut prepared = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: Arc::default(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
prepared
|
||||
.worker_mut()
|
||||
.materialize_durable_session_head()
|
||||
.await
|
||||
.unwrap();
|
||||
let pending = prepared.worker().pending_submission_handle();
|
||||
let buffer = prepared.worker().notify_buffer_handle();
|
||||
let shared = WorkerSharedState::new(
|
||||
"queue-boundary".into(),
|
||||
prepared.worker().segment_id(),
|
||||
String::new(),
|
||||
build_greeting(prepared.worker()),
|
||||
);
|
||||
let control =
|
||||
crate::subject_release::ReleaseControl::new(prepared.worker().mutation_authority());
|
||||
for condition in [
|
||||
"queued",
|
||||
"staged",
|
||||
"attention",
|
||||
"closed",
|
||||
"durable-notification",
|
||||
] {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
if condition == "queued" {
|
||||
sender
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"already enqueued",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
if condition == "durable-notification" {
|
||||
pending
|
||||
.accept_notification("already-admitted".into(), "pending".into())
|
||||
.unwrap();
|
||||
}
|
||||
if condition == "closed" {
|
||||
drop(sender);
|
||||
}
|
||||
let request = tokio::spawn({
|
||||
let control = control.clone();
|
||||
async move { control.request().await }
|
||||
});
|
||||
control.notified().await;
|
||||
answer_idle_release(
|
||||
&control,
|
||||
&shared,
|
||||
&receiver,
|
||||
condition == "staged",
|
||||
&pending,
|
||||
&buffer,
|
||||
condition == "attention",
|
||||
);
|
||||
assert!(request.await.unwrap().is_err(), "{condition}");
|
||||
// A failed request must not leave admission fenced for recovery.
|
||||
assert!(prepared.worker().mutation_authority().enter().is_ok());
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,59 @@ use subjektiv::journal::{
|
||||
MAX_JOURNAL_RESPONSE_BYTES,
|
||||
};
|
||||
|
||||
/// Subjektiv owns startup projection; Core only commits its typed initial items.
|
||||
pub(crate) struct JournalStartupContextSource {
|
||||
host: SubjektivHostConnection,
|
||||
prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>,
|
||||
}
|
||||
impl JournalStartupContextSource {
|
||||
pub(crate) fn new(
|
||||
host: SubjektivHostConnection,
|
||||
prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>,
|
||||
) -> Self {
|
||||
Self { host, prompts }
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl crate::worker::InitialContextSource for JournalStartupContextSource {
|
||||
async fn load(&self) -> Result<Vec<session_store::SystemItem>, String> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let snapshot = self
|
||||
.host
|
||||
.journal_startup_context()
|
||||
.map_err(|e| e.to_string())?;
|
||||
// Hash the same canonical JSON value retained by SystemItem, so replay
|
||||
// can verify provenance without depending on domain struct field order.
|
||||
let snapshot = serde_json::to_value(snapshot).map_err(|e| e.to_string())?;
|
||||
let snapshot_json = serde_json::to_string(&snapshot).map_err(|e| e.to_string())?;
|
||||
if snapshot_json.len() > subjektiv::journal::startup::STARTUP_MAX_BYTES {
|
||||
return Err("Host Journal startup snapshot exceeds independent byte budget".into());
|
||||
}
|
||||
let prompts = self.prompts.load();
|
||||
let body = prompts
|
||||
.subjektiv_journal_startup(&snapshot_json)
|
||||
.map_err(|e| e.to_string())?;
|
||||
if body.len() > subjektiv::journal::startup::STARTUP_MAX_BYTES + 4096 {
|
||||
return Err("Journal startup prompt exceeds rendered byte budget".into());
|
||||
}
|
||||
let projection = prompts.projection();
|
||||
Ok(vec![session_store::SystemItem::JournalStartup {
|
||||
body,
|
||||
snapshot,
|
||||
snapshot_sha256: Sha256::digest(snapshot_json.as_bytes())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect(),
|
||||
prompt_provenance: session_store::PromptRenderProvenance {
|
||||
workspace_id: self.host.host.workspace_id(),
|
||||
source_digest: projection.source_digest.clone(),
|
||||
projection_digest: projection.catalog_digest.clone(),
|
||||
logical_name: "internal.subjektiv_journal_startup".into(),
|
||||
},
|
||||
}])
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct JournalRecordingFeature {
|
||||
host: SubjektivHostConnection,
|
||||
|
||||
@@ -76,9 +76,16 @@ impl PendingNotify {
|
||||
pub struct NotifyBuffer {
|
||||
inner: Arc<Mutex<VecDeque<PendingNotify>>>,
|
||||
wake: Arc<tokio::sync::Notify>,
|
||||
admission: Arc<std::sync::OnceLock<Arc<crate::subject_release::MutationAuthority>>>,
|
||||
}
|
||||
|
||||
impl NotifyBuffer {
|
||||
pub(crate) fn bind_mutation_authority(
|
||||
&self,
|
||||
authority: Arc<crate::subject_release::MutationAuthority>,
|
||||
) {
|
||||
let _ = self.admission.set(authority);
|
||||
}
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
@@ -113,6 +120,18 @@ impl NotifyBuffer {
|
||||
}
|
||||
|
||||
fn push_entry(&self, entry: PendingNotify) {
|
||||
let _admission = match self
|
||||
.admission
|
||||
.get()
|
||||
.map(|authority| authority.enter())
|
||||
.transpose()
|
||||
{
|
||||
Ok(lease) => lease,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "notification rejected by Subject release fence");
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.inner
|
||||
.lock()
|
||||
.expect("notify buffer poisoned")
|
||||
|
||||
@@ -25,6 +25,8 @@ mod session_capture;
|
||||
mod session_history;
|
||||
pub mod shared_state;
|
||||
mod shutdown_after_idle;
|
||||
mod subject_release;
|
||||
pub use subject_release::SubjectReleaseReservation;
|
||||
pub mod skill;
|
||||
pub mod spawn;
|
||||
pub mod subjektiv;
|
||||
|
||||
@@ -102,6 +102,7 @@ pub enum WorkerPrompt {
|
||||
ResidentMemorySummarySection,
|
||||
ResidentMemoryRestoreSection,
|
||||
SubjektivResidentContext,
|
||||
SubjektivJournalStartup,
|
||||
WorkerOrchestrationGuidanceSection,
|
||||
SubWorkerSpawnToolDescription,
|
||||
}
|
||||
@@ -126,6 +127,7 @@ impl WorkerPrompt {
|
||||
Self::AgentsMdSection => "internal.agents_md_section",
|
||||
Self::ResidentMemorySummarySection => "internal.resident_memory_summary_section",
|
||||
Self::ResidentMemoryRestoreSection => "internal.resident_memory_restore_section",
|
||||
Self::SubjektivJournalStartup => "internal.subjektiv_journal_startup",
|
||||
Self::SubjektivResidentContext => "internal.subjektiv_resident_context",
|
||||
Self::WorkerOrchestrationGuidanceSection => {
|
||||
"internal.worker_orchestration_guidance_section"
|
||||
@@ -151,6 +153,7 @@ impl WorkerPrompt {
|
||||
WorkerPrompt::ResidentMemorySummarySection,
|
||||
WorkerPrompt::ResidentMemoryRestoreSection,
|
||||
WorkerPrompt::SubjektivResidentContext,
|
||||
WorkerPrompt::SubjektivJournalStartup,
|
||||
WorkerPrompt::WorkerOrchestrationGuidanceSection,
|
||||
WorkerPrompt::SubWorkerSpawnToolDescription,
|
||||
];
|
||||
@@ -540,6 +543,14 @@ impl PromptCatalog {
|
||||
},
|
||||
)
|
||||
}
|
||||
pub fn subjektiv_journal_startup(&self, snapshot_json: &str) -> Result<String, CatalogError> {
|
||||
self.render(
|
||||
WorkerPrompt::SubjektivJournalStartup,
|
||||
minijinja::context! {
|
||||
snapshot_json => snapshot_json,
|
||||
},
|
||||
)
|
||||
}
|
||||
pub fn subjektiv_resident_context(
|
||||
&self,
|
||||
behavior_md: &str,
|
||||
|
||||
@@ -50,6 +50,7 @@ pub struct WorkerSharedState {
|
||||
fs_view: OnceLock<WorkerFsView>,
|
||||
feature_invocations: RwLock<crate::feature::FeatureInvocationRegistry>,
|
||||
flow_transition_enabled: AtomicBool,
|
||||
pub(crate) subject_release: OnceLock<std::sync::Arc<crate::subject_release::ReleaseControl>>,
|
||||
}
|
||||
|
||||
impl WorkerSharedState {
|
||||
@@ -69,6 +70,7 @@ impl WorkerSharedState {
|
||||
fs_view: OnceLock::new(),
|
||||
feature_invocations: RwLock::new(crate::feature::FeatureInvocationRegistry::default()),
|
||||
flow_transition_enabled: AtomicBool::new(false),
|
||||
subject_release: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Controller-owned quiescence with a structural persistence/admission fence.
|
||||
//!
|
||||
//! A reservation is acknowledged only by the actual idle Controller. Shared read
|
||||
//! leases cover input enqueue, durable pending admission, and history writes; the
|
||||
//! exclusive lease spans the Host's reconciliation and ownership release. No
|
||||
//! Journal policy or catalog-status inference belongs here.
|
||||
|
||||
use session_store::StoreError;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use tokio::sync::{Notify, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock, oneshot};
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct MutationAuthority {
|
||||
barrier: Arc<RwLock<()>>,
|
||||
disconnected: AtomicBool,
|
||||
}
|
||||
impl MutationAuthority {
|
||||
pub(crate) fn enter(&self) -> Result<OwnedRwLockReadGuard<()>, StoreError> {
|
||||
let lease = self
|
||||
.barrier
|
||||
.clone()
|
||||
.try_read_owned()
|
||||
.map_err(|_| denied())?;
|
||||
if self.disconnected.load(Ordering::Acquire) {
|
||||
return Err(denied());
|
||||
}
|
||||
Ok(lease)
|
||||
}
|
||||
}
|
||||
fn denied() -> StoreError {
|
||||
StoreError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
"Subject release has fenced Worker admission and history mutation",
|
||||
))
|
||||
}
|
||||
|
||||
type Reply = oneshot::Sender<Result<SubjectReleaseReservation, String>>;
|
||||
struct Requests {
|
||||
alive: bool,
|
||||
queue: VecDeque<Reply>,
|
||||
}
|
||||
pub(crate) struct ReleaseControl {
|
||||
pub(crate) authority: Arc<MutationAuthority>,
|
||||
requests: Mutex<Requests>,
|
||||
changed: Notify,
|
||||
}
|
||||
impl ReleaseControl {
|
||||
pub(crate) fn new(authority: Arc<MutationAuthority>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
authority,
|
||||
requests: Mutex::new(Requests {
|
||||
alive: true,
|
||||
queue: VecDeque::new(),
|
||||
}),
|
||||
changed: Notify::new(),
|
||||
})
|
||||
}
|
||||
pub(crate) async fn request(self: &Arc<Self>) -> Result<SubjectReleaseReservation, String> {
|
||||
let (reply, receive) = oneshot::channel();
|
||||
{
|
||||
let mut requests = self
|
||||
.requests
|
||||
.lock()
|
||||
.map_err(|_| "Controller release requests poisoned")?;
|
||||
if !requests.alive {
|
||||
return Err("Controller is unavailable".into());
|
||||
}
|
||||
requests.queue.push_back(reply);
|
||||
}
|
||||
self.changed.notify_one();
|
||||
receive
|
||||
.await
|
||||
.map_err(|_| "Controller closed before release acknowledgement".to_string())?
|
||||
}
|
||||
pub(crate) async fn notified(&self) {
|
||||
self.changed.notified().await;
|
||||
}
|
||||
pub(crate) fn respond(self: &Arc<Self>, check_quiescent: impl Fn() -> Result<(), String>) {
|
||||
let requests = {
|
||||
let mut state = self
|
||||
.requests
|
||||
.lock()
|
||||
.expect("Controller release requests poisoned");
|
||||
state.queue.drain(..).collect::<Vec<_>>()
|
||||
};
|
||||
for reply in requests {
|
||||
if reply.is_closed() {
|
||||
continue;
|
||||
}
|
||||
let result = self
|
||||
.authority
|
||||
.barrier
|
||||
.clone()
|
||||
.try_write_owned()
|
||||
.map_err(|_| "Controller admission or history mutation is active".to_string())
|
||||
.and_then(|exclusive| {
|
||||
if self.authority.disconnected.load(Ordering::Acquire) {
|
||||
return Err("Controller body is disconnected".into());
|
||||
}
|
||||
check_quiescent()?;
|
||||
Ok(SubjectReleaseReservation {
|
||||
control: self.clone(),
|
||||
_exclusive: exclusive,
|
||||
committed: false,
|
||||
})
|
||||
});
|
||||
// If the waiter disappeared, dropping the unsent guard cancels the fence.
|
||||
let _ = reply.send(result);
|
||||
}
|
||||
}
|
||||
pub(crate) fn reject_busy(self: &Arc<Self>) {
|
||||
self.respond(|| Err("Controller is not quiescent Idle".into()));
|
||||
}
|
||||
pub(crate) fn close(&self) {
|
||||
let mut state = self
|
||||
.requests
|
||||
.lock()
|
||||
.expect("Controller release requests poisoned");
|
||||
state.alive = false;
|
||||
for reply in state.queue.drain(..) {
|
||||
let _ = reply.send(Err("Controller is unavailable".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ControllerLifetime(pub(crate) Arc<ReleaseControl>);
|
||||
impl Drop for ControllerLifetime {
|
||||
fn drop(&mut self) {
|
||||
self.0.close();
|
||||
}
|
||||
}
|
||||
|
||||
/// Exclusive permission to perform Host-side Subject handoff while the old body
|
||||
/// cannot accept input or change source history. Dropping an uncommitted guard
|
||||
/// cancels the reservation. Committing permanently revokes body admission; it
|
||||
/// does not stop the Controller or disable read-only observation and shutdown.
|
||||
pub struct SubjectReleaseReservation {
|
||||
control: Arc<ReleaseControl>,
|
||||
_exclusive: OwnedRwLockWriteGuard<()>,
|
||||
committed: bool,
|
||||
}
|
||||
impl SubjectReleaseReservation {
|
||||
pub fn commit_disconnect(&mut self) -> Result<(), String> {
|
||||
if self.committed {
|
||||
return Ok(());
|
||||
}
|
||||
let state = self
|
||||
.control
|
||||
.requests
|
||||
.lock()
|
||||
.map_err(|_| "Controller release requests poisoned")?;
|
||||
if !state.alive {
|
||||
return Err("Controller stopped before disconnect commit".into());
|
||||
}
|
||||
self.control
|
||||
.authority
|
||||
.disconnected
|
||||
.store(true, Ordering::Release);
|
||||
self.committed = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
// Structural lease test: the Controller's grant path is production code;
|
||||
// the held reader represents an already-entered synchronous persistence call.
|
||||
#[tokio::test]
|
||||
async fn in_flight_mutation_rejects_release_without_waiting_and_retry_recovers() {
|
||||
let authority = Arc::new(MutationAuthority::default());
|
||||
let control = ReleaseControl::new(authority.clone());
|
||||
let (entered, entry) = oneshot::channel();
|
||||
let (finish, finished) = oneshot::channel();
|
||||
let writer = tokio::spawn({
|
||||
let authority = authority.clone();
|
||||
async move {
|
||||
let _lease = authority.enter().unwrap();
|
||||
entered.send(()).unwrap();
|
||||
finished.await.unwrap();
|
||||
}
|
||||
});
|
||||
entry.await.unwrap();
|
||||
let request = tokio::spawn({
|
||||
let control = control.clone();
|
||||
async move { control.request().await }
|
||||
});
|
||||
control.notified().await;
|
||||
control.respond(|| Ok(()));
|
||||
assert!(request.await.unwrap().is_err());
|
||||
finish.send(()).unwrap();
|
||||
writer.await.unwrap();
|
||||
let request = tokio::spawn({
|
||||
let control = control.clone();
|
||||
async move { control.request().await }
|
||||
});
|
||||
control.notified().await;
|
||||
control.respond(|| Ok(()));
|
||||
let guard = request.await.unwrap().unwrap();
|
||||
assert!(authority.enter().is_err());
|
||||
drop(guard);
|
||||
assert!(authority.enter().is_ok());
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn closed_controller_rejects_unanswered_request_and_guard_disconnect() {
|
||||
let control = ReleaseControl::new(Arc::default());
|
||||
let request = tokio::spawn({
|
||||
let control = control.clone();
|
||||
async move { control.request().await }
|
||||
});
|
||||
control.notified().await;
|
||||
control.respond(|| Ok(()));
|
||||
let mut guard = request.await.unwrap().unwrap();
|
||||
control.close();
|
||||
assert!(guard.commit_disconnect().is_err());
|
||||
assert!(control.request().await.is_err());
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ pub use crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifec
|
||||
pub use crate::feature::builtin::journal::{
|
||||
JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature, JournalRecordingFeature,
|
||||
};
|
||||
use subjektiv::journal::startup::JournalStartupSnapshot;
|
||||
pub use subjektiv::journal::transport;
|
||||
use transport::{JournalRecordingReason, JournalRecordingRequest, JournalRecordingResponse};
|
||||
|
||||
@@ -66,6 +67,18 @@ impl From<SubjektivHostError> for ToolError {
|
||||
/// identifier is not a grant. No method grants ambient filesystem access.
|
||||
pub trait SubjektivHost: Send + Sync + 'static {
|
||||
fn settings(&self) -> SubjektivHostSettings;
|
||||
/// Startup reads are a body-only capability, independent of recording Jobs.
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
_context: &SubjektivHostContext,
|
||||
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
|
||||
Err(SubjektivHostError::Unavailable(
|
||||
"Journal startup not granted".into(),
|
||||
))
|
||||
}
|
||||
/// Recording admission is separate from both Memory and Journal Job operation grants.
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
false
|
||||
@@ -207,6 +220,25 @@ fn decode<T: serde::de::DeserializeOwned>(
|
||||
}
|
||||
|
||||
impl SubjektivHost for BackendSubjektivHost {
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
_context: &SubjektivHostContext,
|
||||
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
|
||||
decode(
|
||||
self.client
|
||||
.execute_with_timeout(
|
||||
WorkspaceRequest::get(format!(
|
||||
"/api/w/{}/subjektiv/journal/startup",
|
||||
self.workspace_id
|
||||
)),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?,
|
||||
)
|
||||
}
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
// This adapter exists only for explicitly enabled, authenticated Host connections.
|
||||
// The Backend still verifies the main Worker/Subject binding on every request.
|
||||
@@ -420,6 +452,14 @@ pub struct SubjektivHostConnection {
|
||||
}
|
||||
|
||||
impl SubjektivHostConnection {
|
||||
pub fn journal_startup_context(&self) -> Result<JournalStartupSnapshot, SubjektivHostError> {
|
||||
if !self.host.journal_startup_granted() {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"Journal startup not granted".into(),
|
||||
));
|
||||
}
|
||||
self.host.journal_startup_context(&self.context)
|
||||
}
|
||||
pub fn new(
|
||||
host: Arc<dyn SubjektivHost>,
|
||||
context: SubjektivHostContext,
|
||||
@@ -546,12 +586,33 @@ mod tests {
|
||||
|
||||
#[derive(Default)]
|
||||
struct LocalHost {
|
||||
startup: Option<JournalStartupSnapshot>,
|
||||
startup_error: bool,
|
||||
startup_calls: std::sync::atomic::AtomicUsize,
|
||||
contexts: Mutex<Vec<SubjektivHostContext>>,
|
||||
attributed: Mutex<Vec<(SubjektivHostContext, bool)>>,
|
||||
journal_observer:
|
||||
Option<Arc<dyn Fn(&SubjektivHostContext, JournalRecordingReason) + Send + Sync>>,
|
||||
}
|
||||
impl SubjektivHost for LocalHost {
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
self.startup.is_some() || self.startup_error
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
_context: &SubjektivHostContext,
|
||||
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
|
||||
self.startup_calls
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
if self.startup_error {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"startup read failed".into(),
|
||||
));
|
||||
}
|
||||
self.startup
|
||||
.clone()
|
||||
.ok_or_else(|| SubjektivHostError::Unavailable("startup not granted".into()))
|
||||
}
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
self.journal_observer.is_some()
|
||||
}
|
||||
@@ -919,6 +980,223 @@ enabled = true
|
||||
);
|
||||
}
|
||||
|
||||
// Production Direct bootstrap, Feature selection/rendering, Core, FsStore,
|
||||
// and model Request. The authenticated Host boundary and provider are fake;
|
||||
// real Backend/standalone authority and live continuation are not claimed.
|
||||
#[tokio::test]
|
||||
async fn direct_startup_journal_is_durable_before_first_user_and_provider_exactly_once() {
|
||||
use session_store::{LogEntry, LoggedSessionHistoryOrigin, Store, SystemItem};
|
||||
use std::sync::atomic::Ordering;
|
||||
for empty in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let name = format!("startup-{}", uuid::Uuid::now_v7());
|
||||
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='{name}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]")).unwrap();
|
||||
manifest.feature.subjektiv.profile.enabled = true;
|
||||
manifest.feature.subjektiv.profile.extraction.enabled = false;
|
||||
let snapshot = JournalStartupSnapshot {
|
||||
subject_id: "subject".into(),
|
||||
omitted: false,
|
||||
journals: if empty {
|
||||
vec![]
|
||||
} else {
|
||||
vec![subjektiv::journal::startup::StartupJournal {
|
||||
journal_id: "journal".into(),
|
||||
source_session_id: "previous-session".into(),
|
||||
state_token: "consistent-token".into(),
|
||||
process_order: 4,
|
||||
process_byte: 0,
|
||||
current_position: "B adopted after correcting A; implementation unfinished"
|
||||
.into(),
|
||||
entries: vec![],
|
||||
omitted: true,
|
||||
}]
|
||||
},
|
||||
};
|
||||
let host = Arc::new(LocalHost {
|
||||
startup: Some(snapshot.clone()),
|
||||
..LocalHost::default()
|
||||
});
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||
);
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let prepared = WorkerBootstrap::new(
|
||||
manifest,
|
||||
store.clone(),
|
||||
crate::PromptCatalogSource::builtins_only(),
|
||||
WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::None,
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
bash_output_dir: temp.path().join("bash"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx))
|
||||
.with_subjektiv_host(host.clone())
|
||||
.prepare()
|
||||
.await
|
||||
.unwrap();
|
||||
let (session, segment) = (
|
||||
prepared.worker().session_id(),
|
||||
prepared.worker().segment_id(),
|
||||
);
|
||||
let controller = prepared.start().await.unwrap();
|
||||
let initial = store.read_all(session, segment).unwrap();
|
||||
let journal = initial
|
||||
.iter()
|
||||
.find_map(|entry| match entry {
|
||||
LogEntry::AnnotatedSystemItem { entry, .. }
|
||||
if matches!(entry.item, SystemItem::JournalStartup { .. }) =>
|
||||
{
|
||||
Some(entry)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
journal.metadata.origin,
|
||||
LoggedSessionHistoryOrigin::DerivedSummary
|
||||
);
|
||||
let SystemItem::JournalStartup {
|
||||
body,
|
||||
snapshot: saved,
|
||||
snapshot_sha256,
|
||||
prompt_provenance,
|
||||
} = &journal.item
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(*saved, serde_json::to_value(&snapshot).unwrap());
|
||||
use sha2::{Digest, Sha256};
|
||||
assert_eq!(
|
||||
*snapshot_sha256,
|
||||
Sha256::digest(serde_json::to_vec(saved).unwrap())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>()
|
||||
);
|
||||
assert_eq!(
|
||||
prompt_provenance.logical_name,
|
||||
"internal.subjektiv_journal_startup"
|
||||
);
|
||||
assert!(
|
||||
!initial
|
||||
.iter()
|
||||
.any(|e| matches!(e, LogEntry::AnnotatedUserInput { .. }))
|
||||
);
|
||||
let mut events = controller.handle.subscribe();
|
||||
for user in ["FIRST_USER_REQUEST", "SECOND_USER_REQUEST"] {
|
||||
controller
|
||||
.handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
user,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let request = tokio::time::timeout(Duration::from_secs(5), rx.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let index = request
|
||||
.items
|
||||
.iter()
|
||||
.position(|item| item.as_text() == Some(body.as_str()))
|
||||
.unwrap();
|
||||
let user_index = request
|
||||
.items
|
||||
.iter()
|
||||
.position(|item| item.as_text() == Some("FIRST_USER_REQUEST"))
|
||||
.unwrap();
|
||||
assert!(index < user_index);
|
||||
assert_eq!(
|
||||
request
|
||||
.items
|
||||
.iter()
|
||||
.filter(|item| item.as_text() == Some(body.as_str()))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if matches!(events.recv().await.unwrap(), protocol::Event::RunEnd { .. }) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(host.startup_calls.load(Ordering::SeqCst), 1);
|
||||
let logs = store.read_all(session, segment).unwrap();
|
||||
assert_eq!(logs.iter().filter(|entry| matches!(entry, LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, SystemItem::JournalStartup { .. }))).count(), 1);
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(1),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
controller.controller_task.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_startup_read_error_rejects_exposure_without_initialized_head() {
|
||||
use session_store::Store;
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='startup-error-{}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]", uuid::Uuid::now_v7())).unwrap();
|
||||
manifest.feature.subjektiv.profile.enabled = true;
|
||||
manifest.feature.subjektiv.profile.extraction.enabled = false;
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||
);
|
||||
let host = Arc::new(LocalHost {
|
||||
startup_error: true,
|
||||
..LocalHost::default()
|
||||
});
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let prepared = WorkerBootstrap::new(
|
||||
manifest,
|
||||
store.clone(),
|
||||
crate::PromptCatalogSource::builtins_only(),
|
||||
WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::None,
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
bash_output_dir: temp.path().join("bash"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx))
|
||||
.with_subjektiv_host(host.clone())
|
||||
.prepare()
|
||||
.await
|
||||
.unwrap();
|
||||
let (session, segment) = (
|
||||
prepared.worker().session_id(),
|
||||
prepared.worker().segment_id(),
|
||||
);
|
||||
assert!(prepared.start().await.is_err());
|
||||
assert!(!store.exists(session, segment).unwrap());
|
||||
assert!(rx.try_recv().is_err());
|
||||
assert_eq!(
|
||||
host.startup_calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
async fn launch_case(enabled: bool, attach: bool) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let _runtime_sandbox =
|
||||
@@ -1038,7 +1316,7 @@ enabled = true
|
||||
if enabled && attach {
|
||||
assert_eq!(
|
||||
*host.attributed.lock().unwrap(),
|
||||
vec![(expected.clone(), true)]
|
||||
vec![(expected.clone(), true), (expected.clone(), false)]
|
||||
);
|
||||
assert!(
|
||||
host.contexts
|
||||
|
||||
+878
-46
File diff suppressed because it is too large
Load Diff
@@ -529,7 +529,7 @@ fn activation_oldest_and_previous_schema_migrate_with_same_constraints_and_prese
|
||||
assert!(!store.is_user_pending_activation("old-user").unwrap());
|
||||
store.with_conn(|conn| {
|
||||
assert_eq!(schema(conn), expected);
|
||||
assert_eq!(current_schema_version(conn)?, 100);
|
||||
assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION);
|
||||
assert!(conn.execute("INSERT INTO user_activations VALUES('missing','issue','hash','future',0)", []).is_err());
|
||||
for attempts in [-1, 6] { assert!(conn.execute("INSERT INTO user_activations VALUES('old-user','issue','hash','future',?1)", [attempts]).is_err()); }
|
||||
Ok(())
|
||||
|
||||
@@ -474,7 +474,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
(97, "MR notifications and authenticated implementation results".into()),
|
||||
(98, "Assignment-independent implementation reports".into()),
|
||||
(99, "Explicit hosting administrators and self-registration policy".into()),
|
||||
(100, "Provisional users and scoped passkey activation".into())
|
||||
(100, "Provisional users and scoped passkey activation".into()),
|
||||
(101, "Durable explicit Subject body release".into())
|
||||
]
|
||||
);
|
||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||
|
||||
@@ -282,6 +282,10 @@ pub struct SubjektivConsolidationGrant {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjektivJournalGrant {
|
||||
/// Frozen writer generation. Legacy absence preserves its original canonical wire/digest;
|
||||
/// recovery must use only the immutable original Session and still-current body.
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub binding_id: String,
|
||||
pub subject_id: String,
|
||||
pub session_id: String,
|
||||
pub journal_id: String,
|
||||
|
||||
@@ -1288,6 +1288,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_release(
|
||||
&self,
|
||||
_request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
Err("actual Subject Controller reservation is unavailable on this Runtime".into())
|
||||
}
|
||||
|
||||
fn session_material_pin(
|
||||
&self,
|
||||
_request: runtime_api::SessionMaterialPinRequest,
|
||||
@@ -2244,6 +2251,21 @@ impl RuntimeRegistry {
|
||||
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
|
||||
}
|
||||
|
||||
pub fn subject_release(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, RuntimeRegistryError> {
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
self.runtime(runtime_id)?
|
||||
.subject_release(request)
|
||||
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: runtime_id.into(),
|
||||
code: "subject_release_unproven".into(),
|
||||
message,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn session_material_pin(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -3297,6 +3319,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_release(
|
||||
&self,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
if request.workspace_id != self.workspace_id {
|
||||
return Err("Subject release Workspace mismatch".into());
|
||||
}
|
||||
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
|
||||
self.runtime
|
||||
.subject_release_scoped(&scope, request)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn session_material_pin(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialPinRequest,
|
||||
@@ -4114,7 +4149,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
|
||||
{
|
||||
return "workers:input";
|
||||
}
|
||||
if path.ends_with("/stop") || path.ends_with("/cancel") {
|
||||
if path.ends_with("/stop")
|
||||
|| path.ends_with("/cancel")
|
||||
|| (path == "/v1/subject-release" && method == "POST")
|
||||
{
|
||||
return "workers:stop";
|
||||
}
|
||||
if path == "/v1/protocol/ws" {
|
||||
@@ -5523,6 +5561,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_release(
|
||||
&self,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
self.run_runtime_api(
|
||||
self.request_timeout,
|
||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||
move |client| async move { client.subject_release(request).await },
|
||||
)
|
||||
.map_err(|diagnostic| diagnostic.message)
|
||||
}
|
||||
|
||||
fn session_material_pin(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialPinRequest,
|
||||
|
||||
@@ -381,6 +381,8 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
|
||||
DROP TABLE ticket_implementation_results;
|
||||
DROP TABLE ticket_implementation_delegations;
|
||||
DROP TABLE merge_request_source_observations;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
@@ -513,6 +515,8 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
|
||||
// Reconstruct the prior composition before its new business capture existed.
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
@@ -671,7 +675,9 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
|
||||
if historical {
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TABLE user_activations;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||
|
||||
@@ -699,6 +699,22 @@ CREATE TABLE worker_create_reservations (
|
||||
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
|
||||
CHECK ((singleton_key IS NULL) = (singleton_generation IS NULL))
|
||||
);
|
||||
CREATE TABLE subject_worker_releases (
|
||||
workspace_id TEXT NOT NULL,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
singleton_key TEXT NOT NULL,
|
||||
singleton_generation INTEGER NOT NULL CHECK (singleton_generation > 0),
|
||||
binding_id TEXT NOT NULL,
|
||||
operation_id TEXT NOT NULL,
|
||||
proof TEXT CHECK (proof IN ('runtime_disconnected','erased')),
|
||||
completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0,1)),
|
||||
cancelled INTEGER NOT NULL DEFAULT 0 CHECK (cancelled IN (0,1)),
|
||||
PRIMARY KEY (workspace_id, operation_id),
|
||||
CHECK (cancelled = 0 OR (completed = 0 AND proof IS NULL)),
|
||||
FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_create_reservations(workspace_id, worker_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE UNIQUE INDEX subject_worker_release_active ON subject_worker_releases(workspace_id,worker_id) WHERE cancelled=0;
|
||||
CREATE TRIGGER worker_create_reservation_singleton_lease_insert
|
||||
BEFORE INSERT ON worker_create_reservations
|
||||
WHEN (NEW.singleton_key IS NULL AND NEW.singleton_generation IS NOT NULL)
|
||||
|
||||
@@ -57,6 +57,7 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
||||
"workspace_config_tree_history",
|
||||
"workspace_memory_settings",
|
||||
"worker_create_reservations",
|
||||
"subject_worker_releases",
|
||||
"backend_jobs",
|
||||
"backend_job_attempts",
|
||||
"flow_sources",
|
||||
|
||||
@@ -1654,6 +1654,7 @@ pub struct WorkspaceApi {
|
||||
Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<Mutex<OrchestratorAttentionState>>>>>,
|
||||
notification_delivery_lifecycle: Arc<notification_delivery::NotificationDeliveryLifecycle>,
|
||||
backend_job_drain_requests: Arc<AtomicU64>,
|
||||
journal_lifecycle: Arc<Mutex<HashMap<String, Arc<Mutex<()>>>>>,
|
||||
journal_dispatch_owner: Option<Arc<WorkspaceApi>>,
|
||||
observation_proxy: BackendObservationProxy,
|
||||
runtime_subscription_broker: RuntimeSubscriptionBroker,
|
||||
@@ -3379,6 +3380,7 @@ fn workspace_request_is_callback(method: &Method, path: &str, workspace_id: &str
|
||||
|| path == format!("/api/runtime/v1/workspaces/{workspace_id}/resources/fetch")))
|
||||
|| (*method == Method::GET
|
||||
&& (path == format!("/api/w/{workspace_id}/runtime-config")
|
||||
|| path == format!("/api/w/{workspace_id}/subjektiv/journal/startup")
|
||||
|| path == format!("/api/w/{workspace_id}/worker-discovery/workers")))
|
||||
}
|
||||
|
||||
@@ -4160,6 +4162,7 @@ impl WorkspaceApi {
|
||||
notification_delivery_lifecycle: Arc::new(Default::default()),
|
||||
orchestrator_attention: Arc::new(Mutex::new(HashMap::new())),
|
||||
backend_job_drain_requests: Arc::new(AtomicU64::new(0)),
|
||||
journal_lifecycle: Arc::new(Mutex::new(HashMap::new())),
|
||||
journal_dispatch_owner: None,
|
||||
observation_proxy,
|
||||
runtime_subscription_broker,
|
||||
@@ -4362,8 +4365,9 @@ impl WorkspaceApi {
|
||||
resolved_worker_observation_grants: Vec::new(),
|
||||
resolved_workspace_api: None,
|
||||
resolved_memory_settings: None,
|
||||
resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some()
|
||||
|| request.grants.subjektiv_journal.is_some(),
|
||||
// Journal Jobs use their independent frozen Job Host grant, not the
|
||||
// main-Subject/Memory attachment or its resident/extraction policy.
|
||||
resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some(),
|
||||
resolved_control_operation: Some(control_operation),
|
||||
},
|
||||
) {
|
||||
@@ -5468,6 +5472,22 @@ impl WorkspaceApi {
|
||||
.map(|assignment| assignment.operation_id.clone())
|
||||
})
|
||||
.unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7()));
|
||||
let subject = singleton_key
|
||||
.as_deref()
|
||||
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
|
||||
let subject_gate = subject
|
||||
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
|
||||
.transpose()?;
|
||||
let subject_guard = subject_gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subject {
|
||||
subjektiv_journal::check_start_allowed(self, subject)?;
|
||||
}
|
||||
let reservation = match self.config_store.reserve_worker_create(
|
||||
&self.config.workspace_id,
|
||||
runtime_id,
|
||||
@@ -5490,6 +5510,7 @@ impl WorkspaceApi {
|
||||
return Err(ApiError::with_diagnostics(error, diagnostics));
|
||||
}
|
||||
};
|
||||
drop(subject_guard);
|
||||
let worker_id = reservation.worker_id;
|
||||
if let Some(assignment) = request.ticket_assignment.as_ref() {
|
||||
self.store.bind_ticket_assignment_operation_worker(
|
||||
@@ -5803,6 +5824,9 @@ impl WorkspaceApi {
|
||||
diagnostics,
|
||||
));
|
||||
}
|
||||
if let Err(error) = subjektiv_journal::reconcile_created_worker(self, &worker_ref) {
|
||||
tracing::warn!(%error, worker_id=%worker_ref.worker_id, "Journal first-run reconciliation remains recoverable; saved history retained");
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -13122,6 +13146,8 @@ fn build_inner_router(api: WorkspaceApi) -> Router {
|
||||
"/api/w/{workspace_id}/workers/self/backend-job-result",
|
||||
post(scoped_submit_backend_job_result),
|
||||
)
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/release", post(subjektiv_journal::release_route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/startup", get(subjektiv_journal::startup_route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/recording", post(subjektiv_journal::recording_route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/job", post(subjektiv_journal::job_route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/job/finish", post(subjektiv_journal::finish_route))
|
||||
@@ -22367,6 +22393,7 @@ async fn scoped_record_subjektiv_session(
|
||||
&request.session_id,
|
||||
)?
|
||||
};
|
||||
subjektiv_journal::bind_host_session(&api, attribution.clone())?;
|
||||
Ok(Json(server_api::SubjektivRecordSessionResponse {
|
||||
subject_id: attribution.subject_id,
|
||||
session_id: attribution.session_id,
|
||||
@@ -46478,8 +46505,8 @@ mod tests {
|
||||
let runtime = WorkdirlessFixtureRuntime::default();
|
||||
api.runtime.register_or_replace(runtime.clone());
|
||||
let request = |operation_id: &str| WorkerSpawnRequest {
|
||||
requested_worker_name: Some("Subject Worker".to_string()),
|
||||
singleton_key: Some("subjektiv:subject-retry".to_string()),
|
||||
requested_worker_name: Some("Generic singleton Worker".to_string()),
|
||||
singleton_key: Some("fixture:subject-retry".to_string()),
|
||||
intent: WorkerSpawnIntent::WorkspaceCompanion,
|
||||
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
|
||||
expected_segments: 0,
|
||||
@@ -46516,7 +46543,7 @@ mod tests {
|
||||
// Runtime dispatch. The restarted request receives a fresh manual allocation key, yet its
|
||||
// identical singleton intent replays the reserved Worker and dispatches that generation.
|
||||
let mut restart_request = request("ignored-manual-operation");
|
||||
restart_request.singleton_key = Some("subjektiv:restart-retry".to_string());
|
||||
restart_request.singleton_key = Some("fixture:restart-retry".to_string());
|
||||
restart_request.resolved_control_operation = None;
|
||||
let restart_fingerprint = worker_spawn_create_fingerprint(&restart_request).unwrap();
|
||||
let memory = api
|
||||
@@ -46563,7 +46590,7 @@ mod tests {
|
||||
assert_eq!(rejected.state, InternalWorkerOperationState::Rejected);
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "subjektiv:subject-retry")
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "fixture:subject-retry")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
@@ -46577,14 +46604,14 @@ mod tests {
|
||||
let worker = created.worker.unwrap();
|
||||
let current_owner = api
|
||||
.store
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "subjektiv:subject-retry")
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "fixture:subject-retry")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(current_owner.worker, worker.worker);
|
||||
assert_eq!(current_owner.generation, 3);
|
||||
assert_eq!(
|
||||
worker.singleton_key,
|
||||
Some("subjektiv:subject-retry".to_string())
|
||||
Some("fixture:subject-retry".to_string())
|
||||
);
|
||||
|
||||
// Idle and paused Runtime metadata never releases Backend ownership, and a conflict is
|
||||
@@ -46614,7 +46641,7 @@ mod tests {
|
||||
assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted);
|
||||
assert_eq!(
|
||||
restored.worker.unwrap().singleton_key.as_deref(),
|
||||
Some("subjektiv:subject-retry")
|
||||
Some("fixture:subject-retry")
|
||||
);
|
||||
|
||||
// Replacing the connected Runtime with an empty instance models an unavailable/restarted
|
||||
@@ -46634,10 +46661,7 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(
|
||||
&api.config.workspace_id,
|
||||
"subjektiv:subject-retry",
|
||||
)
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "fixture:subject-retry",)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.worker,
|
||||
@@ -46714,10 +46738,7 @@ mod tests {
|
||||
));
|
||||
assert_eq!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(
|
||||
&api.config.workspace_id,
|
||||
"subjektiv:subject-retry",
|
||||
)
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "fixture:subject-retry",)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.worker,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -110,7 +110,7 @@ async fn subject_callbacks_finish_before_waiting_host_parents_release_capacity()
|
||||
));
|
||||
}
|
||||
// Resident memory is not an allowed deletion-drain callback. Preserve that
|
||||
// admission fence; only the exact Session attribution route reenters below.
|
||||
// admission fence; only exact Session attribution and startup read callbacks reenter below.
|
||||
let mut deletion = Box::pin(gate.lock_deletion());
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
for (subject_id, worker_id, session_id) in &callbacks {
|
||||
@@ -139,6 +139,27 @@ async fn subject_callbacks_finish_before_waiting_host_parents_release_capacity()
|
||||
let stored = subjektiv.session_attribution(session_id).unwrap().unwrap();
|
||||
assert_eq!(stored.worker_id, *worker_id);
|
||||
assert_eq!(stored.subject_id, *subject_id);
|
||||
let startup_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/journal/startup");
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
app.clone().oneshot(runtime_source_request(
|
||||
&identity,
|
||||
Some(worker_id),
|
||||
"GET",
|
||||
&startup_path,
|
||||
Vec::new(),
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("startup snapshot must not wait for Runtime's occupied Host parents")
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let snapshot: subjektiv::journal::startup::JournalStartupSnapshot =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(snapshot.subject_id, *subject_id);
|
||||
assert!(snapshot.journals.is_empty());
|
||||
assert!(!snapshot.omitted);
|
||||
}
|
||||
// Neither actual parent nor its admission lease was released to make these
|
||||
// callbacks succeed. Only durable attribution completion releases them.
|
||||
@@ -297,7 +318,7 @@ async fn only_exact_authenticated_callbacks_can_reenter_deletion_drain() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_create_releases_reservation_and_key_without_cleanup_errors() {
|
||||
async fn rejected_create_releases_key_but_retains_subject_occupancy_without_release_proof() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let api = test_api(workspace.path()).await;
|
||||
let memory = api
|
||||
@@ -349,11 +370,13 @@ async fn rejected_create_releases_reservation_and_key_without_cleanup_errors() {
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
assert_eq!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, "subjektiv:rejected-create")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
.expect("failed creation is not explicit Subject release authority")
|
||||
.worker,
|
||||
worker,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,623 @@
|
||||
//! Connected Backend → production Runtime factory → real Controller → Journal → signed
|
||||
//! Host startup reader → persisted first input. Only the external model stream is scripted.
|
||||
//! Browser authentication is outside this regression: creation enters the real scoped handler.
|
||||
use super::*;
|
||||
use agen::llm_client::{
|
||||
LlmClient, Request,
|
||||
error::ClientError,
|
||||
event::{Event, StopReason},
|
||||
types::Item,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ConnectedJournalModel {
|
||||
dialogs: Arc<Mutex<Vec<Request>>>,
|
||||
}
|
||||
|
||||
impl ConnectedJournalModel {
|
||||
fn journal_response(request: &Request) -> Vec<Event> {
|
||||
// Runtime Controllers deliberately retain alias-routed core schemas for their
|
||||
// lifetime even with zero attachments. Verify invocation authority, not schema
|
||||
// absence (standalone Internal Jobs use a different composition).
|
||||
for (id, tool, input) in [
|
||||
(
|
||||
"denied-bash",
|
||||
"Bash",
|
||||
json!({"command":"printf JOURNAL_MUST_NOT_EXECUTE","cwd":"."}),
|
||||
),
|
||||
(
|
||||
"denied-read",
|
||||
"Read",
|
||||
json!({"file_path":"journal-must-not-be-readable"}),
|
||||
),
|
||||
] {
|
||||
if let Some(error) = request.items.iter().find_map(|item| match item {
|
||||
Item::ToolResult {
|
||||
call_id, is_error, ..
|
||||
} if call_id == id => Some(*is_error),
|
||||
_ => None,
|
||||
}) {
|
||||
assert!(
|
||||
error,
|
||||
"Journal Job unexpectedly gained {tool} Workdir authority"
|
||||
);
|
||||
} else {
|
||||
return vec![
|
||||
Event::tool_use_start(0, id, tool),
|
||||
Event::tool_input_delta(0, input.to_string()),
|
||||
Event::tool_use_stop(0),
|
||||
];
|
||||
}
|
||||
}
|
||||
let result = |id: &str| -> Option<Value> {
|
||||
request.items.iter().find_map(|item| match item {
|
||||
Item::ToolResult {
|
||||
call_id,
|
||||
content,
|
||||
is_error,
|
||||
summary,
|
||||
..
|
||||
} if call_id == id => {
|
||||
assert!(!is_error, "{id}: {summary}; {content:?}");
|
||||
Some(
|
||||
content
|
||||
.as_deref()
|
||||
.map(|text| serde_json::from_str(text).unwrap())
|
||||
.unwrap_or(Value::Null),
|
||||
)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
if result("submit").is_some() {
|
||||
return vec![
|
||||
Event::text_block_start(0),
|
||||
Event::text_delta(0, "saved"),
|
||||
Event::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||
];
|
||||
}
|
||||
let (tool, id, input) = if result("checkpoint").is_some() {
|
||||
("SubmitBackendJobResult", "submit", json!({"result":{}}))
|
||||
} else if let Some(position) = result("position") {
|
||||
let state = result("state").unwrap();
|
||||
(
|
||||
"CheckpointJournal",
|
||||
"checkpoint",
|
||||
json!({
|
||||
"request_id":format!("checkpoint:{}",state["value"]["generation_id"].as_str().unwrap()),
|
||||
"expected_state":position["value"]["state_token"],
|
||||
"action":{"action":"checkpoint","record_operation_ids":[format!("record:{}",state["value"]["generation_id"].as_str().unwrap())]},
|
||||
"covered_ranges":state["value"]["selected_ranges"],
|
||||
}),
|
||||
)
|
||||
} else if let Some(record) = result("record") {
|
||||
let state = result("state").unwrap();
|
||||
(
|
||||
"UpdateJournalPosition",
|
||||
"position",
|
||||
json!({
|
||||
"request_id":format!("position:{}",state["value"]["generation_id"].as_str().unwrap()),
|
||||
"expected_state":record["value"]["state_token"],
|
||||
"action":{"action":"update_position","current_position":"CONNECTED_HANDOFF: partially implemented; outcome unconfirmed"},
|
||||
"covered_ranges":[],
|
||||
}),
|
||||
)
|
||||
} else if let Some(state) = result("state") {
|
||||
let state = &state["value"];
|
||||
// Deliver the actual frozen Runtime source through authenticated Job tools, rather
|
||||
// than injecting a fixture Journal or claiming unseen source was processed.
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("source-{index}-{offset}");
|
||||
if let Some(page) = result(&id) {
|
||||
let page = &page["value"];
|
||||
if let Some(next) = page["next_byte"].as_u64() {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["content_token"].clone();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
assert!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalText")
|
||||
);
|
||||
return vec![Event::tool_use_start(0, &id, "ReadJournalText"), Event::tool_input_delta(0, json!({
|
||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
||||
"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token,
|
||||
}).to_string()), Event::tool_use_stop(0)];
|
||||
}
|
||||
}
|
||||
}
|
||||
let (tool, action) = if state["current_open"].is_null() {
|
||||
(
|
||||
"CreateJournalEntry",
|
||||
json!({"action":"create","title":"Connected handoff","body":"CONNECTED_HANDOFF: proposed A, user chose B; only partial work performed.","ongoing":"Remaining work; do not claim success","state":"open","source_end":null,"correction_refs":[]}),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"RewriteJournalEntry",
|
||||
json!({"action":"rewrite","entry_id":state["current_open"]["id"],"title":"Connected handoff","body":"CONNECTED_HANDOFF: B selected; partial work remains.","ongoing":"Remaining work; result unknown"}),
|
||||
)
|
||||
};
|
||||
(
|
||||
tool,
|
||||
"record",
|
||||
json!({
|
||||
"request_id":format!("record:{}",state["generation_id"].as_str().unwrap()),
|
||||
"expected_state":state["current_journal"]["state_token"],
|
||||
"action":action,"covered_ranges":state["selected_ranges"],
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
("ReadJournalState", "state", json!({}))
|
||||
};
|
||||
assert!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|definition| definition.name == tool),
|
||||
"missing {tool}"
|
||||
);
|
||||
for forbidden in ["MemoryApplyCandidate", "SubjektivSessionRead"] {
|
||||
assert!(
|
||||
!request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|definition| definition.name == forbidden),
|
||||
"Journal exposed forbidden {forbidden}: {:?}",
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| &tool.name)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
vec![
|
||||
Event::tool_use_start(0, id, tool),
|
||||
Event::tool_input_delta(0, input.to_string()),
|
||||
Event::tool_use_stop(0),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LlmClient for ConnectedJournalModel {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> std::result::Result<
|
||||
std::pin::Pin<
|
||||
Box<dyn futures::Stream<Item = std::result::Result<Event, ClientError>> + Send>,
|
||||
>,
|
||||
ClientError,
|
||||
> {
|
||||
let events = if request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalState")
|
||||
{
|
||||
Self::journal_response(&request)
|
||||
} else if request.tools.iter().any(|tool| {
|
||||
matches!(
|
||||
tool.name.as_str(),
|
||||
"SubmitBackendJobResult" | "SubmitMemorySurface"
|
||||
)
|
||||
}) {
|
||||
// The real companion also requests an empty Memory consolidation Job. Keep that
|
||||
// lifecycle real without counting its model requests as main-body dialog requests.
|
||||
if request.items.iter().any(|item| matches!(item, Item::ToolResult { call_id, is_error, .. } if call_id == "empty-memory" && !is_error)) {
|
||||
vec![Event::text_block_start(0), Event::text_delta(0, "empty Memory acknowledged"), Event::text_block_stop(0, Some(StopReason::EndTurn))]
|
||||
} else {
|
||||
let (tool, input) = if request.tools.iter().any(|tool| tool.name == "SubmitMemorySurface") {
|
||||
("SubmitMemorySurface", json!({"points":[]}))
|
||||
} else {
|
||||
let input: Value = request.items.iter().filter_map(Item::as_text).find_map(|text| text.lines().find_map(|line| line.strip_prefix("input_json: "))).map(|input| serde_json::from_str(input).unwrap()).expect("Backend Memory Job envelope");
|
||||
assert_eq!(input["candidate_ids"], json!([]));
|
||||
("SubmitBackendJobResult", json!({"result":{"subject_id":input["subject_id"],"candidate_ids":[]}}))
|
||||
};
|
||||
vec![Event::tool_use_start(0, "empty-memory", tool), Event::tool_input_delta(0, input.to_string()), Event::tool_use_stop(0)]
|
||||
}
|
||||
} else {
|
||||
self.dialogs.lock().unwrap().push(request);
|
||||
vec![
|
||||
Event::text_block_start(0),
|
||||
Event::text_delta(
|
||||
0,
|
||||
"Partial B implementation; remaining work is not confirmed.",
|
||||
),
|
||||
Event::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||
]
|
||||
};
|
||||
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
async fn connected_create(api: &WorkspaceApi, subject: &str, name: &str) -> RuntimeWorkerRef {
|
||||
let Json(created) = scoped_create_workspace_worker(
|
||||
State(api.clone()),
|
||||
AxumPath(ScopedWorkspacePath { workspace_id: TEST_WORKSPACE_ID.into() }),
|
||||
HeaderMap::new(),
|
||||
Json(CreateWorkspaceWorkerRequest {
|
||||
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), display_name: name.into(),
|
||||
singleton_key: None, profile: Some("builtin:companion".into()), ticket_assignment: None,
|
||||
initial_submit: vec![protocol::Segment::Text { content: "User chose B rather than A. Continue the partial work without assuming success.".into() }],
|
||||
workdir_attachments: vec![],
|
||||
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
|
||||
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest { subject_id: subject.into() }),
|
||||
}, control_operation_id: None,
|
||||
}),
|
||||
).await.unwrap();
|
||||
RuntimeWorkerRef::new(created.runtime_id, created.worker_id)
|
||||
}
|
||||
|
||||
async fn connected_wait_for_recording(
|
||||
api: &WorkspaceApi,
|
||||
store: &crate::subjektiv::SubjektivStore,
|
||||
attr: &crate::subjektiv::SubjectSessionAttribution,
|
||||
) {
|
||||
let host = JournalHostContext::validated_session(store, attr.clone()).unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
let mut tick = tokio::time::interval(std::time::Duration::from_millis(25));
|
||||
loop {
|
||||
tick.tick().await;
|
||||
let jobs = api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &attr.session_id, None, 100)
|
||||
.unwrap();
|
||||
let generation_closed =
|
||||
store
|
||||
.journal_for_session(&host)
|
||||
.unwrap()
|
||||
.is_some_and(|journal| {
|
||||
store
|
||||
.active_journal_generation(&host, &journal)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
});
|
||||
if !jobs.is_empty()
|
||||
&& generation_closed
|
||||
&& jobs
|
||||
.iter()
|
||||
.all(|job| job.state == BackendJobState::Completed)
|
||||
&& !store.journal_retention_required(&host).unwrap()
|
||||
&& api
|
||||
.runtime
|
||||
.worker(&RuntimeWorkerRef::new(&attr.runtime_id, &attr.worker_id))
|
||||
.unwrap()
|
||||
.worker_state
|
||||
.is_some_and(|state| state.catalog_status() == protocol::WorkerStatus::Idle)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!(
|
||||
"real Controller recording did not complete: {:?}",
|
||||
api.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &attr.session_id, None, 100)
|
||||
.unwrap()
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
|
||||
async fn backend_real_runtime_journal_release_and_next_body_persisted_startup() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(dir.path());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let mut config = test_server_config(dir.path());
|
||||
config.backend_base_url = Some(format!("http://{}", listener.local_addr().unwrap()));
|
||||
let control = Arc::new(test_control_store(&config));
|
||||
seed_test_registered_workspace(control.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
let broker = BackendResourceBroker::default();
|
||||
let dispatcher = Arc::new(
|
||||
crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
|
||||
config.clone(),
|
||||
control.clone(),
|
||||
),
|
||||
);
|
||||
let model = ConnectedJournalModel::default();
|
||||
let model_factory = model.clone();
|
||||
let audience = embedded_runtime_request_audience(&config).unwrap();
|
||||
let execution = worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new(
|
||||
worker_runtime::worker_backend::ProfileRuntimeWorkerFactory::new(
|
||||
config.workspace_execution_root.clone(),
|
||||
)
|
||||
.with_embedded_worker_mutation_dispatcher(EMBEDDED_WORKER_RUNTIME_ID, dispatcher.clone())
|
||||
.with_workspace_request_client(
|
||||
worker_runtime::workspace_request::RuntimeWorkspaceRequestClient::new(
|
||||
TEST_WORKSPACE_ID,
|
||||
audience.clone(),
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
)
|
||||
.with_runtime_request_source(&EMBEDDED_RUNTIME_REQUEST_IDENTITY, audience),
|
||||
)
|
||||
.with_runtime_store_dir(config.embedded_runtime_store_root.clone())
|
||||
.with_resource_client(Arc::new(broker.clone()))
|
||||
.with_spawn_model_client_factory(move || Box::new(model_factory.clone())),
|
||||
)
|
||||
.unwrap();
|
||||
let api = WorkspaceApi::new_with_execution_backend_and_broker(
|
||||
config,
|
||||
control,
|
||||
Arc::new(execution),
|
||||
broker,
|
||||
HostExecutors::default(),
|
||||
Some(dispatcher),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
seed_test_delivery_binding(&api);
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
let observed = requests.clone();
|
||||
let app = build_router(api.clone()).layer(axum::middleware::from_fn(
|
||||
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
||||
let observed = observed.clone();
|
||||
async move {
|
||||
let path = request.uri().path().to_string();
|
||||
let response = next.run(request).await;
|
||||
observed.lock().unwrap().push((path, response.status()));
|
||||
response
|
||||
}
|
||||
},
|
||||
));
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("connected-handoff").unwrap())
|
||||
.unwrap();
|
||||
let old = connected_create(&api, &subject.id, "connected-old-body").await;
|
||||
let old_attr = store
|
||||
.subject_session_attributions(&subject.id)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|attr| attr.worker_id == old.worker_id)
|
||||
.unwrap();
|
||||
connected_wait_for_recording(&api, &store, &old_attr).await;
|
||||
let old_host = JournalHostContext::validated_session(&store, old_attr.clone()).unwrap();
|
||||
let old_binding = JournalWriterBinding {
|
||||
subject_id: subject.id.clone(),
|
||||
runtime_id: old.runtime_id.clone(),
|
||||
worker_id: old.worker_id.clone(),
|
||||
binding_id: old_host.writer_binding_id().unwrap().into(),
|
||||
};
|
||||
// Inject a durable result-acceptance gap after real Journal completion. This must
|
||||
// reject before writer revocation and confirm cancellation of the actual Controller
|
||||
// guard; merely returning Err would strand the body and its execution-admission fence.
|
||||
let recorded = api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &old_attr.session_id, None, 100)
|
||||
.unwrap();
|
||||
let uncertain_job = &recorded[0].request.job_id;
|
||||
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
|
||||
let saved_result: (String, String) = conn.query_row(
|
||||
"SELECT result_json,result_digest FROM backend_jobs WHERE workspace_id=?1 AND job_id=?2",
|
||||
rusqlite::params![TEST_WORKSPACE_ID, uncertain_job], |r| Ok((r.get(0)?, r.get(1)?)),
|
||||
).unwrap();
|
||||
conn.execute(
|
||||
"UPDATE backend_jobs SET state='unknown',result_json=NULL,result_digest=NULL WHERE workspace_id=?1 AND job_id=?2",
|
||||
rusqlite::params![TEST_WORKSPACE_ID, uncertain_job],
|
||||
)
|
||||
.unwrap();
|
||||
let incomplete = json!({
|
||||
"worker":old,"binding_id":old_binding.binding_id,"operation_id":"connected-incomplete-release",
|
||||
});
|
||||
let error = subjektiv_journal::release_subject(
|
||||
&api,
|
||||
serde_json::from_value(incomplete.clone()).unwrap(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("Unknown"), "{error}");
|
||||
assert!(store.journal_writer_active(&old_binding).unwrap());
|
||||
assert!(
|
||||
!api.config_store
|
||||
.subject_release_pending(TEST_WORKSPACE_ID, &old)
|
||||
.unwrap()
|
||||
);
|
||||
let cancelled: bool = conn.query_row("SELECT cancelled FROM subject_worker_releases WHERE workspace_id=?1 AND operation_id='connected-incomplete-release'", [TEST_WORKSPACE_ID], |r| r.get(0)).unwrap();
|
||||
assert!(
|
||||
cancelled,
|
||||
"checked pre-revoke refusal needs a terminal cancellation receipt"
|
||||
);
|
||||
assert!(
|
||||
subjektiv_journal::release_subject(&api, serde_json::from_value(incomplete).unwrap())
|
||||
.is_err(),
|
||||
"cancelled release identity must not rearm"
|
||||
);
|
||||
conn.execute(
|
||||
"UPDATE backend_jobs SET state='completed',result_json=?3,result_digest=?4 WHERE workspace_id=?1 AND job_id=?2",
|
||||
rusqlite::params![TEST_WORKSPACE_ID, uncertain_job, saved_result.0, saved_result.1],
|
||||
)
|
||||
.unwrap();
|
||||
// The new release below must reacquire the genuine Controller guard. It cannot
|
||||
// succeed if the failed pre-revoke operation left the body frozen.
|
||||
let release = subjektiv_journal::release_subject(&api, serde_json::from_value(json!({
|
||||
"worker":old,"binding_id":old_binding.binding_id,"operation_id":"connected-normal-release",
|
||||
})).unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
release.state,
|
||||
runtime_api::SubjectReleaseState::Disconnected
|
||||
);
|
||||
assert!(!store.journal_writer_active(&old_binding).unwrap());
|
||||
let key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap();
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(&old, WorkerLifecycleRequest::default())
|
||||
.unwrap();
|
||||
assert_eq!(stopped.state, InternalWorkerOperationState::Accepted);
|
||||
let removed = WorkerRemovalService::new(&api)
|
||||
.execute_target_removal(
|
||||
api.runtime.as_ref(),
|
||||
&old,
|
||||
"connected regression removes prior body",
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(removed.status, StatusCode::OK.as_u16(), "{}", removed.body);
|
||||
assert!(api.runtime.worker(&old).is_err());
|
||||
let original_root = api
|
||||
.config
|
||||
.embedded_runtime_store_root
|
||||
.join("workers")
|
||||
.join(&old.worker_id)
|
||||
.join("session");
|
||||
assert!(
|
||||
!original_root.exists(),
|
||||
"prior Worker original Session must actually be removed before startup"
|
||||
);
|
||||
// Purge retained original archive through Runtime, not fixture filesystem deletion.
|
||||
// Journal text is independent and remains readable for new-body startup.
|
||||
if let Some(archive) = api
|
||||
.config_store
|
||||
.worker_tombstone(TEST_WORKSPACE_ID, &old)
|
||||
.unwrap()
|
||||
.and_then(|t| t.archive_id)
|
||||
{
|
||||
api.runtime
|
||||
.session_material_erase(
|
||||
&old.runtime_id,
|
||||
runtime_api::SessionMaterialEraseRequest {
|
||||
workspace_id: TEST_WORKSPACE_ID.into(),
|
||||
source: runtime_api::SessionPublicSource::Archived {
|
||||
archive_id: archive.clone(),
|
||||
},
|
||||
expected_session_id: old_attr.session_id.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!api.config
|
||||
.embedded_runtime_store_root
|
||||
.join("archives/workers")
|
||||
.join(archive)
|
||||
.join("session")
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
let next = connected_create(&api, &subject.id, "connected-next-body").await;
|
||||
assert_ne!(next, old);
|
||||
let next_attr = store
|
||||
.subject_session_attributions(&subject.id)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|attr| attr.worker_id == next.worker_id)
|
||||
.unwrap();
|
||||
assert_ne!(next_attr.session_id, old_attr.session_id);
|
||||
connected_wait_for_recording(&api, &store, &next_attr).await;
|
||||
|
||||
// Read actual Session files produced by the real Controller; no fixture Session append,
|
||||
// startup-item injection, fake disconnect or store-only simulation is used in this test.
|
||||
use session_store::Store as _;
|
||||
let session = session_store::WorkerSessionStore::open_read_only(
|
||||
api.config
|
||||
.embedded_runtime_store_root
|
||||
.join("workers")
|
||||
.join(&next.worker_id)
|
||||
.join("session"),
|
||||
)
|
||||
.unwrap();
|
||||
let session_id = next_attr.session_id.parse().unwrap();
|
||||
let segments = session.list_segments(session_id).unwrap();
|
||||
assert_eq!(segments.len(), 1);
|
||||
let entries = session.read_all_read_only(session_id, segments[0]).unwrap();
|
||||
let startup: Vec<_> = entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, log)| match log {
|
||||
session_store::LogEntry::AnnotatedSystemItem { entry, .. } => match &entry.item {
|
||||
session_store::SystemItem::JournalStartup {
|
||||
snapshot,
|
||||
snapshot_sha256,
|
||||
body,
|
||||
prompt_provenance,
|
||||
} => Some((index, snapshot, snapshot_sha256, body, prompt_provenance)),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(startup.len(), 1, "startup must be durably committed once");
|
||||
let (startup_index, snapshot, digest, body, provenance) = startup[0];
|
||||
let first_input = entries
|
||||
.iter()
|
||||
.position(|entry| matches!(entry, session_store::LogEntry::AnnotatedUserInput { .. }))
|
||||
.expect("new body persisted its initial human input");
|
||||
assert!(
|
||||
startup_index < first_input,
|
||||
"startup reference must be persisted before the first human input/model request"
|
||||
);
|
||||
assert_eq!(snapshot["subject_id"], subject.id);
|
||||
assert_eq!(
|
||||
snapshot["journals"][0]["source_session_id"], old_attr.session_id,
|
||||
"Host-selected writer predecessor must lead the snapshot"
|
||||
);
|
||||
assert!(body.contains("CONNECTED_HANDOFF"));
|
||||
assert_eq!(
|
||||
provenance.logical_name,
|
||||
"internal.subjektiv_journal_startup"
|
||||
);
|
||||
assert_eq!(
|
||||
*digest,
|
||||
Sha256::digest(serde_json::to_vec(snapshot).unwrap())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>()
|
||||
);
|
||||
let dialogs = model.dialogs.lock().unwrap();
|
||||
assert!(dialogs.len() >= 2);
|
||||
assert!(
|
||||
dialogs[1]
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(Item::as_text)
|
||||
.any(|text| text.contains(body)),
|
||||
"first new-body model request must contain the same persisted startup body"
|
||||
);
|
||||
drop(dialogs);
|
||||
let requests = requests.lock().unwrap();
|
||||
assert_eq!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(
|
||||
|(path, status)| path.ends_with("/subjektiv/journal/startup")
|
||||
&& *status == StatusCode::OK
|
||||
)
|
||||
.count(),
|
||||
2,
|
||||
"both bodies must use the authenticated HTTP Host startup reader"
|
||||
);
|
||||
assert!(
|
||||
requests.iter().any(
|
||||
|(path, status)| path.ends_with("/subjektiv/sessions") && *status == StatusCode::OK
|
||||
)
|
||||
);
|
||||
drop(requests);
|
||||
server.abort();
|
||||
}
|
||||
@@ -8,6 +8,19 @@ use sha2::{Digest, Sha256};
|
||||
use subjektiv::journal::transport::*;
|
||||
use subjektiv::journal::*;
|
||||
|
||||
#[path = "subjektiv_journal_connected_tests.rs"]
|
||||
mod connected;
|
||||
|
||||
fn bind_fixture_session(
|
||||
store: &crate::subjektiv::SubjektivStore,
|
||||
attr: crate::subjektiv::SubjectSessionAttribution,
|
||||
) -> JournalHostContext {
|
||||
let binding = store
|
||||
.claim_journal_writer(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||
.unwrap();
|
||||
store.bind_journal_session(&binding, attr).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_job_reserves_before_binding_and_retains_partial_receipts_on_cancel() {
|
||||
let Fixture {
|
||||
@@ -195,7 +208,7 @@ impl Fixture {
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap();
|
||||
let host = bind_fixture_session(&store, attr.clone());
|
||||
let text = "original public input";
|
||||
let range = JournalSourceRange {
|
||||
source: JournalSource {
|
||||
@@ -228,6 +241,7 @@ impl Fixture {
|
||||
profile: "builtin:subjektiv-journal-recording".into(),
|
||||
grants: BackendJobGrants {
|
||||
subjektiv_journal: Some(SubjektivJournalGrant {
|
||||
binding_id: host.writer_binding_id().unwrap().into(),
|
||||
subject_id: subject.id,
|
||||
session_id: attr.session_id,
|
||||
journal_id: id.clone(),
|
||||
@@ -288,13 +302,14 @@ async fn journal_completion_replays_only_saved_checkpoint_and_fences_late_operat
|
||||
panic!("receipt required")
|
||||
};
|
||||
assert!(subjektiv_journal::finish(&f.api, worker).is_err());
|
||||
let position = save_fixture_position(&f.api, worker, receipt.state_token);
|
||||
subjektiv_journal::execute(
|
||||
&f.api,
|
||||
worker,
|
||||
JournalJobRequest::Execute {
|
||||
operation: JournalOperation {
|
||||
request_id: "checkpoint".into(),
|
||||
expected_state: receipt.state_token,
|
||||
expected_state: position,
|
||||
action: JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["no-change".into()],
|
||||
},
|
||||
@@ -332,6 +347,69 @@ async fn journal_completion_replays_only_saved_checkpoint_and_fences_late_operat
|
||||
);
|
||||
assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err());
|
||||
assert!(!f.store.journal_retention_required(&f.host).unwrap());
|
||||
// Store authority boundary: Host Job-state gating is separate from revocation itself.
|
||||
let grant = f.request.grants.subjektiv_journal.as_ref().unwrap();
|
||||
let attr = f
|
||||
.store
|
||||
.session_attribution(&grant.session_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let binding = JournalWriterBinding {
|
||||
subject_id: attr.subject_id.clone(),
|
||||
runtime_id: attr.runtime_id.clone(),
|
||||
worker_id: attr.worker_id.clone(),
|
||||
binding_id: grant.binding_id.clone(),
|
||||
};
|
||||
f.store.release_journal_writer(&binding).unwrap();
|
||||
let next = f
|
||||
.store
|
||||
.claim_journal_writer(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||
.unwrap();
|
||||
assert_ne!(next.binding_id, binding.binding_id);
|
||||
assert!(f.store.bind_journal_session(&next, attr).is_err());
|
||||
let mut stale = dispatched.clone();
|
||||
stale
|
||||
.job
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.binding_id = next.binding_id;
|
||||
assert!(subjektiv_journal::prepare(&f.api, &stale).is_err());
|
||||
assert!(
|
||||
f.store
|
||||
.query_journal_operation(&f.host, &f.id, "no-change")
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
// Exact saved completion is still recoverable; it is not a new write grant.
|
||||
subjektiv_journal::validate_result(&f.api, &f.request, &dispatched.attempt, &value).unwrap();
|
||||
}
|
||||
|
||||
fn save_fixture_position(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
expected_state: String,
|
||||
) -> String {
|
||||
let JournalJobResponse::Saved(receipt) = subjektiv_journal::execute(
|
||||
api,
|
||||
worker,
|
||||
JournalJobRequest::Execute {
|
||||
operation: JournalOperation {
|
||||
request_id: "position".into(),
|
||||
expected_state,
|
||||
action: JournalAction::UpdatePosition {
|
||||
current_position: "No activity change; work remains pending.".into(),
|
||||
},
|
||||
covered_ranges: vec![],
|
||||
},
|
||||
},
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("saved position receipt required")
|
||||
};
|
||||
receipt.state_token
|
||||
}
|
||||
|
||||
fn deliver_fixture_source(
|
||||
@@ -483,7 +561,7 @@ impl RuntimeFixture {
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap();
|
||||
let host = bind_fixture_session(&store, attr.clone());
|
||||
let src = runtime_api::SessionPublicSource::Retained {
|
||||
worker_id: worker_runtime::identity::WorkerId::parse(&source.worker_id).unwrap(),
|
||||
};
|
||||
@@ -543,13 +621,15 @@ impl RuntimeFixture {
|
||||
.accept_journal_sources(&host, "source", &[range.clone()])
|
||||
.unwrap();
|
||||
let request = subjektiv_journal::recording_request(
|
||||
&host,
|
||||
attr,
|
||||
&id,
|
||||
vec![subjektiv::journal::recording::JournalMaterial {
|
||||
range: range.clone(),
|
||||
text: SOURCE_SENTINEL.into(),
|
||||
}],
|
||||
);
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!serde_json::to_string(&request)
|
||||
.unwrap()
|
||||
@@ -650,13 +730,14 @@ impl RuntimeFixture {
|
||||
.unwrap() else {
|
||||
panic!("receipt")
|
||||
};
|
||||
let position = save_fixture_position(&self.api, worker, receipt.state_token);
|
||||
subjektiv_journal::execute(
|
||||
&self.api,
|
||||
worker,
|
||||
JournalJobRequest::Execute {
|
||||
operation: JournalOperation {
|
||||
request_id: "checkpoint".into(),
|
||||
expected_state: receipt.state_token,
|
||||
expected_state: position,
|
||||
action: JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["record".into()],
|
||||
},
|
||||
@@ -1661,7 +1742,7 @@ impl SubjectRemovalFixture {
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap();
|
||||
let host = bind_fixture_session(&store, attr.clone());
|
||||
seed_worker_control_grant(&api, &controller, &target, "subject-removal-fixture");
|
||||
api.runtime
|
||||
.stop_worker(
|
||||
@@ -1696,6 +1777,43 @@ impl SubjectRemovalFixture {
|
||||
registry.profile = Some("builtin:default".into());
|
||||
self.api.store.upsert_worker_registry(®istry).unwrap();
|
||||
}
|
||||
/// Seed the historical release boundary for retention-recovery component tests.
|
||||
/// This deterministic stopped-Worker fixture has no Controller and does NOT
|
||||
/// prove disconnect. The connected test below exercises actual release proof.
|
||||
fn seed_completed_release_for_archive_recovery(&self) {
|
||||
let lease = self
|
||||
.api
|
||||
.store
|
||||
.require_current_worker_singleton_owner(TEST_WORKSPACE_ID, &self.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let binding = JournalWriterBinding {
|
||||
subject_id: self.attr.subject_id.clone(),
|
||||
runtime_id: self.target.runtime_id.clone(),
|
||||
worker_id: self.target.worker_id.clone(),
|
||||
binding_id: self.host.writer_binding_id().unwrap().into(),
|
||||
};
|
||||
let operation = "historical-release-retention-fixture";
|
||||
self.api
|
||||
.config_store
|
||||
.reserve_subject_release(TEST_WORKSPACE_ID, &lease, &binding.binding_id, operation)
|
||||
.unwrap();
|
||||
self.store.release_journal_writer(&binding).unwrap();
|
||||
self.api
|
||||
.config_store
|
||||
.confirm_subject_release_proof(
|
||||
TEST_WORKSPACE_ID,
|
||||
&lease,
|
||||
&binding.binding_id,
|
||||
operation,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
self.api
|
||||
.config_store
|
||||
.release_subject_singleton(TEST_WORKSPACE_ID, &lease, &binding.binding_id, operation)
|
||||
.unwrap();
|
||||
}
|
||||
fn remove(&self) -> worker::WorkspaceResponse {
|
||||
let key = self
|
||||
.api
|
||||
@@ -1714,6 +1832,37 @@ impl SubjectRemovalFixture {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subject_release_never_substitutes_catalog_idle_or_stopped_for_controller_reservation() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
let input = serde_json::from_value(serde_json::json!({
|
||||
"worker":f.target,"binding_id":f.host.writer_binding_id().unwrap(),"operation_id":"actual-controller-required",
|
||||
})).unwrap();
|
||||
let error = subjektiv_journal::release_subject(&f.api, input).unwrap_err();
|
||||
assert!(
|
||||
error.to_string().contains("actual") || error.to_string().contains("Controller"),
|
||||
"{error}"
|
||||
);
|
||||
let key = crate::subjektiv::subject_worker_singleton_key(&f.attr.subject_id).unwrap();
|
||||
assert_eq!(
|
||||
f.api
|
||||
.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.worker,
|
||||
f.target
|
||||
);
|
||||
let binding = JournalWriterBinding {
|
||||
subject_id: f.attr.subject_id.clone(),
|
||||
runtime_id: f.target.runtime_id.clone(),
|
||||
worker_id: f.target.worker_id.clone(),
|
||||
binding_id: f.host.writer_binding_id().unwrap().into(),
|
||||
};
|
||||
assert!(f.store.journal_writer_active(&binding).unwrap());
|
||||
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_subject_removal_does_not_admit_new_public_history_or_reserve_a_job() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
@@ -1721,7 +1870,7 @@ async fn disabled_subject_removal_does_not_admit_new_public_history_or_reserve_a
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
StatusCode::CONFLICT.as_u16(),
|
||||
"{}",
|
||||
response.body
|
||||
);
|
||||
@@ -1730,7 +1879,7 @@ async fn disabled_subject_removal_does_not_admit_new_public_history_or_reserve_a
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
.is_some()
|
||||
);
|
||||
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
|
||||
assert!(
|
||||
@@ -1795,7 +1944,7 @@ async fn disabled_subject_removal_retains_only_the_already_admitted_prefix() {
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
StatusCode::CONFLICT.as_u16(),
|
||||
"{}",
|
||||
response.body
|
||||
);
|
||||
@@ -1804,62 +1953,25 @@ async fn disabled_subject_removal_retains_only_the_already_admitted_prefix() {
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
.is_some()
|
||||
);
|
||||
let jobs = f
|
||||
.api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.len(), 1);
|
||||
assert_eq!(
|
||||
jobs[0]
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
|
||||
.unwrap()
|
||||
.selected_ranges,
|
||||
ranges[..1]
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
f.store.journal_for_session(&f.host).unwrap().as_deref(),
|
||||
Some(id.as_str())
|
||||
);
|
||||
assert!(f.store.journal_retention_required(&f.host).unwrap());
|
||||
let archive = f
|
||||
.api
|
||||
.config_store
|
||||
.worker_session_archive(TEST_WORKSPACE_ID, &f.target, &f.attr.session_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(!session_root(f.dir.path(), &f.target).exists());
|
||||
let source = runtime_api::SessionPublicSource::Archived {
|
||||
archive_id: archive.archive_id,
|
||||
};
|
||||
let response = f
|
||||
.api
|
||||
.runtime
|
||||
.session_material_catalog(
|
||||
&f.target.runtime_id,
|
||||
runtime_api::SessionMaterialCatalogRequest {
|
||||
workspace_id: TEST_WORKSPACE_ID.into(),
|
||||
source,
|
||||
expected_session_id: f.attr.session_id.clone(),
|
||||
expected_catalog_sha256: None,
|
||||
offset: 0,
|
||||
limit: 100,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
response,
|
||||
runtime_api::SessionMaterialCatalogAvailability::Page { .. }
|
||||
));
|
||||
assert!(session_root(f.dir.path(), &f.target).exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attributed_unmaterialized_subject_removal_needs_no_journal_job() {
|
||||
async fn attributed_unmaterialized_subject_still_requires_actual_idle_release() {
|
||||
let f = SubjectRemovalFixture::with_materialized_source(false).await;
|
||||
let retention = source_retention(f.dir.path());
|
||||
let session = f.attr.session_id.parse().unwrap();
|
||||
@@ -1882,28 +1994,21 @@ async fn attributed_unmaterialized_subject_removal_needs_no_journal_job() {
|
||||
.unwrap();
|
||||
assert_eq!(retention.pins(session).unwrap().len(), 1);
|
||||
let response = f.remove();
|
||||
assert!(retention.pins(session).unwrap().is_empty());
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
StatusCode::CONFLICT.as_u16(),
|
||||
"{}",
|
||||
response.body
|
||||
);
|
||||
assert_eq!(retention.pins(session).unwrap().len(), 1);
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
.is_some()
|
||||
);
|
||||
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2237,6 +2342,7 @@ async fn assert_erased_archive_plan_replans_without_republishing_sources(
|
||||
) {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
f.seed_completed_release_for_archive_recovery();
|
||||
let policy = f
|
||||
.api
|
||||
.config_store
|
||||
@@ -2331,6 +2437,7 @@ journal_recovery_cases! {
|
||||
async fn assert_erased_completed_archive_receipt_is_audited_without_republishing(restart: bool) {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
f.seed_completed_release_for_archive_recovery();
|
||||
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
// Runtime has committed source_removed, but Backend commit/response is lost.
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected metadata commit interruption'); END;").unwrap();
|
||||
@@ -2406,6 +2513,7 @@ journal_recovery_cases! {
|
||||
async fn erased_archive_without_receipt_or_inventory_remains_unknown_without_replanning() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
f.seed_completed_release_for_archive_recovery();
|
||||
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE UPDATE OF state ON worker_removal_operations WHEN NEW.state='executing' BEGIN SELECT RAISE(ABORT,'injected interruption'); END;").unwrap();
|
||||
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||
@@ -2465,6 +2573,7 @@ async fn assert_erased_archive_retained_diagnostics_receipt_finalizes_only_after
|
||||
) {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
f.seed_completed_release_for_archive_recovery();
|
||||
let policy = f
|
||||
.api
|
||||
.config_store
|
||||
@@ -2662,3 +2771,36 @@ journal_recovery_cases! {
|
||||
erased_archive_retained_diagnostics_reopen_finalizes_only_after_physical_ack => (true, false),
|
||||
erased_archive_retained_diagnostics_interrupted_reopen_finalizes_only_after_physical_ack => (true, true),
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_journal_grant_preserves_old_json_and_only_recovers_original_current_body() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
let old = json!({"subject_id":f.attr.subject_id,"session_id":f.attr.session_id,"journal_id":"legacy-journal","selected_ranges":[]});
|
||||
let wire = serde_json::to_string(&old).unwrap();
|
||||
let grant: SubjektivJournalGrant = serde_json::from_str(&wire).unwrap();
|
||||
assert!(grant.binding_id.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::to_value(&grant).unwrap(),
|
||||
old,
|
||||
"old canonical request/digest must not acquire a new empty binding_id field"
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&serde_json::to_value(&grant).unwrap()).unwrap(),
|
||||
wire
|
||||
);
|
||||
assert!(subjektiv_journal::host(&f.api, &f.store, &grant).is_ok());
|
||||
let writer = JournalWriterBinding {
|
||||
subject_id: f.attr.subject_id.clone(),
|
||||
runtime_id: f.attr.runtime_id.clone(),
|
||||
worker_id: f.attr.worker_id.clone(),
|
||||
binding_id: f.host.writer_binding_id().unwrap().into(),
|
||||
};
|
||||
f.store.release_journal_writer(&writer).unwrap();
|
||||
f.store
|
||||
.claim_journal_writer(&f.attr.subject_id, &f.attr.runtime_id, "different-body")
|
||||
.unwrap();
|
||||
assert!(
|
||||
subjektiv_journal::host(&f.api, &f.store, &grant).is_err(),
|
||||
"legacy grant must never map to a later writer"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ pub use ticket::{
|
||||
};
|
||||
|
||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 100;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 101;
|
||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||
@@ -427,6 +427,11 @@ const MIGRATIONS: &[Migration] = &[
|
||||
name: "Provisional users and scoped passkey activation",
|
||||
apply: activation::migrate,
|
||||
},
|
||||
Migration {
|
||||
version: 101,
|
||||
name: "Durable explicit Subject body release",
|
||||
apply: migrate_subject_release_v100_to_v101,
|
||||
},
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -1954,6 +1959,15 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
||||
workspace_id: &str,
|
||||
singleton_key: &str,
|
||||
) -> Result<Option<WorkerSingletonLease>>;
|
||||
fn retained_worker_singleton_lease(
|
||||
&self,
|
||||
_workspace_id: &str,
|
||||
_worker: &RuntimeWorkerRef,
|
||||
) -> Result<Option<WorkerSingletonLease>> {
|
||||
Err(Error::Store(
|
||||
"retained singleton lookup is unavailable".into(),
|
||||
))
|
||||
}
|
||||
fn require_current_worker_singleton_owner(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -3164,6 +3178,156 @@ impl SqliteWorkspaceStore {
|
||||
})
|
||||
}
|
||||
|
||||
/// Retained identity for lifecycle/retention reconciliation only, never execution authority.
|
||||
pub(crate) fn retained_worker_singleton_lease(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> Result<Option<WorkerSingletonLease>> {
|
||||
self.with_conn(|conn| {
|
||||
let row: Option<(Option<String>, Option<i64>)> = conn.query_row(
|
||||
"SELECT singleton_key,singleton_generation FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
|
||||
params![workspace_id,worker.runtime_id,worker.worker_id], |r| Ok((r.get(0)?,r.get(1)?)),
|
||||
).optional()?;
|
||||
match row {
|
||||
Some((Some(key),Some(generation))) => Ok(Some(WorkerSingletonLease {
|
||||
key, generation: generation.try_into().map_err(|_| Error::Store("invalid singleton generation".into()))?, worker:worker.clone(),
|
||||
})),
|
||||
Some((None,None)) | None => Ok(None),
|
||||
_ => Err(Error::Store("incomplete singleton lease".into())),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn retained_subject_release_operation(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
) -> Result<Option<String>> {
|
||||
self.with_conn(|c| Ok(c.query_row("SELECT operation_id FROM subject_worker_releases WHERE workspace_id=?1 AND worker_id=?2 AND runtime_id=?3 AND singleton_generation=?4 AND binding_id=?5 AND cancelled=0",params![workspace,lease.worker.worker_id,lease.worker.runtime_id,lease.generation as i64,binding],|r|r.get(0)).optional()?))
|
||||
}
|
||||
|
||||
pub(crate) fn subject_release_pending(
|
||||
&self,
|
||||
workspace: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> Result<bool> {
|
||||
self.with_conn(|c| Ok(c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND cancelled=0 AND completed=0)",params![workspace,worker.runtime_id,worker.worker_id],|r|r.get(0))?))
|
||||
}
|
||||
|
||||
pub(crate) fn require_subject_release_intent(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<()> {
|
||||
self.with_conn(|c| {
|
||||
let active: bool = c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND binding_id=?6 AND operation_id=?7 AND cancelled=0)",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,binding,operation],|r|r.get(0))?;
|
||||
if !active {return Err(Error::RepositoryConflict("Subject release intent absent, cancelled or mismatched".into()));} Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn reserve_subject_release(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<()> {
|
||||
if binding.trim().is_empty()
|
||||
|| operation.trim().is_empty()
|
||||
|| binding.len() > 512
|
||||
|| operation.len() > 512
|
||||
{
|
||||
return Err(Error::InvalidInput(
|
||||
"invalid Subject release identity".into(),
|
||||
));
|
||||
}
|
||||
self.with_conn_mut(|c| {
|
||||
let tx=c.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let saved: Option<(String,String,String,u64,String,bool)> = tx.query_row("SELECT runtime_id,worker_id,singleton_key,singleton_generation,binding_id,cancelled FROM subject_worker_releases WHERE workspace_id=?1 AND operation_id=?2",params![workspace,operation],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?))).optional()?;
|
||||
if let Some(saved)=saved {
|
||||
if saved != (lease.worker.runtime_id.clone(),lease.worker.worker_id.clone(),lease.key.clone(),lease.generation,binding.into(),false) {return Err(Error::RepositoryConflict("release operation mismatched or terminally cancelled; use a new operation".into()));} return Ok(());
|
||||
}
|
||||
require_current_worker_singleton_lease(&tx,workspace,lease)?;
|
||||
let busy: bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND worker_id=?2 AND cancelled=0)",params![workspace,lease.worker.worker_id],|r|r.get(0))?;
|
||||
if busy {return Err(Error::RepositoryConflict("another Subject release intent is active".into()));}
|
||||
tx.execute("INSERT INTO subject_worker_releases(workspace_id,runtime_id,worker_id,singleton_key,singleton_generation,binding_id,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7)",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,binding,operation])?;
|
||||
tx.commit()?; Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn cancel_subject_release(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<()> {
|
||||
self.with_conn(|c| {
|
||||
let changed=c.execute("UPDATE subject_worker_releases SET cancelled=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_generation=?4 AND binding_id=?5 AND operation_id=?6 AND proof IS NULL AND completed=0",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.generation as i64,binding,operation])?;
|
||||
if changed!=1 {return Err(Error::RepositoryConflict("cannot confirm cancellation of different or committed Subject release".into()));} Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn subject_release_proven(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<bool> {
|
||||
self.with_conn(|c|Ok(c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND binding_id=?6 AND operation_id=?7 AND proof IS NOT NULL AND cancelled=0)",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,binding,operation],|r|r.get(0))?))
|
||||
}
|
||||
|
||||
pub(crate) fn confirm_subject_release_proof(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
erased: bool,
|
||||
) -> Result<()> {
|
||||
let proof = if erased {
|
||||
"erased"
|
||||
} else {
|
||||
"runtime_disconnected"
|
||||
};
|
||||
self.with_conn(|c| {
|
||||
let changed=c.execute("UPDATE subject_worker_releases SET proof=?8 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND binding_id=?6 AND operation_id=?7 AND cancelled=0 AND (proof IS NULL OR proof=?8)",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,binding,operation,proof])?;
|
||||
if changed!=1 {return Err(Error::RepositoryConflict("Subject release proof does not match active intent".into()));} Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn release_subject_singleton(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<()> {
|
||||
self.require_subject_release_intent(workspace, lease, binding, operation)?;
|
||||
if !lease
|
||||
.key
|
||||
.starts_with(crate::subjektiv::SUBJECT_WORKER_SINGLETON_PREFIX)
|
||||
{
|
||||
return Err(Error::InvalidInput("not a Subject singleton".into()));
|
||||
}
|
||||
self.with_conn_mut(|c| {
|
||||
let tx=c.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let (proof,completed,cancelled):(Option<String>,bool,bool)=tx.query_row("SELECT proof,completed,cancelled FROM subject_worker_releases WHERE workspace_id=?1 AND operation_id=?2",params![workspace,operation],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?;
|
||||
if cancelled || proof.is_none() {return Err(Error::RepositoryConflict("Subject release lacks confirmed proof; retain occupancy".into()));}
|
||||
if completed {return Ok(());}
|
||||
require_current_worker_singleton_lease(&tx,workspace,lease)?;
|
||||
tx.execute("UPDATE subject_worker_releases SET completed=1 WHERE workspace_id=?1 AND operation_id=?2 AND cancelled=0",params![workspace,operation])?;
|
||||
let changed=tx.execute("UPDATE worker_create_reservations SET state='removed',updated_at=?6 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND state IN ('reserved','created','removed')",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,chrono::Utc::now().to_rfc3339()])?;
|
||||
if changed!=1 {return Err(Error::RepositoryConflict("Subject owner changed before release".into()));}
|
||||
tx.commit()?; Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn current_worker_singleton_owner(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -7931,6 +8095,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
SqliteWorkspaceStore::current_worker_singleton_owner(self, workspace_id, singleton_key)
|
||||
}
|
||||
|
||||
fn retained_worker_singleton_lease(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> Result<Option<WorkerSingletonLease>> {
|
||||
SqliteWorkspaceStore::retained_worker_singleton_lease(self, workspace_id, worker)
|
||||
}
|
||||
|
||||
fn require_current_worker_singleton_owner(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -14840,6 +15012,22 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_subject_release_v100_to_v101(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? != 100 {
|
||||
return Err(Error::Store(
|
||||
"Subject release migration requires schema 100".into(),
|
||||
));
|
||||
}
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
tx.execute_batch(include_str!("subject_release_schema.sql"))?;
|
||||
tx.execute(
|
||||
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(101,?1)",
|
||||
["Durable explicit Subject body release"],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_hosting_policy_v98_to_v99(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? != 98 {
|
||||
return Err(Error::Store(
|
||||
@@ -15521,7 +15709,10 @@ fn current_worker_singleton_owner(
|
||||
) -> Result<Option<(WorkerSingletonLease, Option<String>)>> {
|
||||
let owner = conn
|
||||
.query_row(
|
||||
"SELECT owner.runtime_id, owner.worker_id, owner.generation, reservation.state \
|
||||
"SELECT owner.runtime_id, owner.worker_id, owner.generation, \
|
||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||
THEN 'created' ELSE reservation.state END \
|
||||
FROM worker_singleton_owners owner \
|
||||
LEFT JOIN worker_create_reservations reservation \
|
||||
ON reservation.workspace_id = owner.workspace_id \
|
||||
@@ -18547,6 +18738,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
WorkspaceSchemaMigrationStep {version:98, name:"Assignment-independent implementation reports".into()},
|
||||
WorkspaceSchemaMigrationStep {version:99, name:"Explicit hosting administrators and self-registration policy".into()},
|
||||
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
|
||||
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -18657,6 +18849,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(98, "Assignment-independent implementation reports".to_string()),
|
||||
(99, "Explicit hosting administrators and self-registration policy".to_string()),
|
||||
(100, "Provisional users and scoped passkey activation".to_string()),
|
||||
(101, "Durable explicit Subject body release".to_string()),
|
||||
]
|
||||
);
|
||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||
@@ -20734,7 +20927,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let key = "subjektiv:主体 id/?!";
|
||||
// Generic singleton lifecycle; Subject-prefixed keys have a separate explicit release contract.
|
||||
let key = "fixture:主体 id/?!";
|
||||
let memory_a = store.get_workspace_memory_settings("workspace-a").unwrap();
|
||||
let first = store
|
||||
.reserve_worker_create(
|
||||
@@ -20842,6 +21036,175 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_release_migration_rolls_back_schema_and_marker_then_reopens() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let database = dir.path().join("server.db");
|
||||
prepare_retained_schema(&database, 100);
|
||||
let conn = Connection::open(&database).unwrap();
|
||||
configure_sqlite(&conn).unwrap();
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_subject_release_migration BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=101 BEGIN SELECT RAISE(ABORT,'injected Subject release migration interruption'); END;").unwrap();
|
||||
assert!(migrate_subject_release_v100_to_v101(&conn).is_err());
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 100);
|
||||
assert!(!table_exists(&conn, "subject_worker_releases").unwrap());
|
||||
conn.execute_batch("DROP TRIGGER interrupt_subject_release_migration")
|
||||
.unwrap();
|
||||
migrate_subject_release_v100_to_v101(&conn).unwrap();
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 101);
|
||||
assert!(table_exists(&conn, "subject_worker_releases").unwrap());
|
||||
assert!(migrate_subject_release_v100_to_v101(&conn).is_err());
|
||||
drop(conn);
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(database).unwrap();
|
||||
reopened
|
||||
.with_conn(|conn| verify_schema_history(conn, 101))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subject_generic_removal_retains_occupancy_until_exact_explicit_release_receipt() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let database = dir.path().join("server.db");
|
||||
let store = SqliteWorkspaceStore::open_api_fixture(&database).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-a".into(),
|
||||
owner_account_id: "owner-account".into(),
|
||||
display_name: "Workspace A".into(),
|
||||
state: "active".into(),
|
||||
created_at: "2026-10-01T00:00:00Z".into(),
|
||||
updated_at: "2026-10-01T00:00:00Z".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let memory = store.get_workspace_memory_settings("workspace-a").unwrap();
|
||||
let key = "subjektiv:subject-a";
|
||||
let first = store
|
||||
.reserve_worker_create(
|
||||
"workspace-a",
|
||||
"runtime-a",
|
||||
"first",
|
||||
"sha256:first",
|
||||
Some(key),
|
||||
&memory,
|
||||
)
|
||||
.unwrap();
|
||||
let lease = first.singleton.unwrap();
|
||||
store
|
||||
.fail_worker_create_reservation(
|
||||
"workspace-a",
|
||||
"runtime-a",
|
||||
first.worker_id,
|
||||
&first.create_fingerprint,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.current_worker_singleton_owner("workspace-a", key)
|
||||
.unwrap(),
|
||||
Some(lease.clone())
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.reserve_worker_create(
|
||||
"workspace-a",
|
||||
"runtime-b",
|
||||
"early",
|
||||
"sha256:early",
|
||||
Some(key),
|
||||
&memory
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
store
|
||||
.reserve_subject_release("workspace-a", &lease, "binding-a", "cancelled-first")
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.subject_release_pending("workspace-a", &lease.worker)
|
||||
.unwrap()
|
||||
);
|
||||
store
|
||||
.cancel_subject_release("workspace-a", &lease, "binding-a", "cancelled-first")
|
||||
.unwrap();
|
||||
assert!(
|
||||
!store
|
||||
.subject_release_pending("workspace-a", &lease.worker)
|
||||
.unwrap()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.reserve_subject_release("workspace-a", &lease, "binding-a", "cancelled-first")
|
||||
.is_err()
|
||||
);
|
||||
store
|
||||
.reserve_subject_release("workspace-a", &lease, "binding-a", "release-a")
|
||||
.unwrap();
|
||||
drop(store);
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.current_worker_singleton_owner("workspace-a", key)
|
||||
.unwrap(),
|
||||
Some(lease.clone())
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.reserve_subject_release("workspace-a", &lease, "binding-later", "release-a")
|
||||
.is_err()
|
||||
);
|
||||
// SQL component boundary: the Host's actual Controller and durable writer-revoke proof
|
||||
// are tested at their integration boundaries, not certified by this private SQL helper.
|
||||
assert!(
|
||||
reopened
|
||||
.release_subject_singleton("workspace-a", &lease, "binding-a", "release-a")
|
||||
.is_err()
|
||||
);
|
||||
reopened
|
||||
.confirm_subject_release_proof("workspace-a", &lease, "binding-a", "release-a", false)
|
||||
.unwrap();
|
||||
reopened
|
||||
.release_subject_singleton("workspace-a", &lease, "binding-a", "release-a")
|
||||
.unwrap();
|
||||
assert!(
|
||||
reopened
|
||||
.current_worker_singleton_owner("workspace-a", key)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let replacement = reopened
|
||||
.reserve_worker_create(
|
||||
"workspace-a",
|
||||
"runtime-b",
|
||||
"next",
|
||||
"sha256:next",
|
||||
Some(key),
|
||||
&memory,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
replacement.singleton.unwrap().generation,
|
||||
lease.generation + 1
|
||||
);
|
||||
reopened
|
||||
.release_subject_singleton("workspace-a", &lease, "binding-a", "release-a")
|
||||
.unwrap();
|
||||
assert!(
|
||||
reopened
|
||||
.release_subject_singleton(
|
||||
"workspace-a",
|
||||
&lease,
|
||||
"binding-a",
|
||||
"different-operation"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.require_current_worker_singleton_owner("workspace-a", &lease.worker)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn concurrent_worker_singleton_reservations_have_one_winner() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -21272,7 +21635,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(97, "MR notifications and authenticated implementation results".into()),
|
||||
(98, "Assignment-independent implementation reports".into()),
|
||||
(99, "Explicit hosting administrators and self-registration policy".into()),
|
||||
(100, "Provisional users and scoped passkey activation".into())]);
|
||||
(100, "Provisional users and scoped passkey activation".into()),
|
||||
(101, "Durable explicit Subject body release".into())]);
|
||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE subject_worker_releases (
|
||||
workspace_id TEXT NOT NULL,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
singleton_key TEXT NOT NULL,
|
||||
singleton_generation INTEGER NOT NULL CHECK (singleton_generation > 0),
|
||||
binding_id TEXT NOT NULL,
|
||||
operation_id TEXT NOT NULL,
|
||||
proof TEXT CHECK (proof IN ('runtime_disconnected','erased')),
|
||||
completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0,1)),
|
||||
cancelled INTEGER NOT NULL DEFAULT 0 CHECK (cancelled IN (0,1)),
|
||||
PRIMARY KEY (workspace_id, operation_id),
|
||||
CHECK (cancelled = 0 OR (completed = 0 AND proof IS NULL)),
|
||||
FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_create_reservations(workspace_id, worker_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE UNIQUE INDEX subject_worker_release_active ON subject_worker_releases(workspace_id,worker_id) WHERE cancelled=0;
|
||||
@@ -94,6 +94,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
||||
"workdir_registry",
|
||||
"workdir_removal_operations",
|
||||
"worker_control_grants",
|
||||
"subject_worker_releases",
|
||||
"worker_create_reservations",
|
||||
"worker_diagnostics_archives",
|
||||
"worker_mutation_source_proof_jtis",
|
||||
|
||||
Reference in New Issue
Block a user