chore: merge develop into hare/develop
This commit is contained in:
@@ -1104,8 +1104,50 @@ pub enum WorkerSessionAvailability {
|
||||
},
|
||||
}
|
||||
|
||||
/// Host-owned Subject release operation. A reservation has no timeout: uncertainty retains
|
||||
/// occupancy and mutation exclusion until the same operation is explicitly reconciled.
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjectReleaseRequest {
|
||||
pub workspace_id: String,
|
||||
pub worker_id: WorkerId,
|
||||
pub binding_id: String,
|
||||
pub operation_id: String,
|
||||
pub action: SubjectReleaseAction,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SubjectReleaseAction {
|
||||
Reserve,
|
||||
Disconnect,
|
||||
Cancel,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SubjectReleaseState {
|
||||
Reserved,
|
||||
Disconnected,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjectReleaseResponse {
|
||||
pub operation_id: String,
|
||||
pub binding_id: String,
|
||||
pub state: SubjectReleaseState,
|
||||
}
|
||||
|
||||
#[api(reqwest, axum)]
|
||||
pub trait RuntimeApi {
|
||||
#[post("/v1/subject-release", status = 200, error_status = 400)]
|
||||
async fn subject_release(
|
||||
&self,
|
||||
#[body] request: SubjectReleaseRequest,
|
||||
) -> Result<SubjectReleaseResponse, RuntimeApiError>;
|
||||
|
||||
#[get("/v1/ping", status = 200, error_status = 400)]
|
||||
async fn ping(
|
||||
&self,
|
||||
@@ -1633,6 +1675,13 @@ mod tests {
|
||||
) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
async fn subject_release(
|
||||
&self,
|
||||
_request: SubjectReleaseRequest,
|
||||
) -> Result<SubjectReleaseResponse, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
|
||||
async fn session_material_release(
|
||||
&self,
|
||||
_request: SessionMaterialPinRequest,
|
||||
|
||||
@@ -4088,10 +4088,11 @@ pub struct PasskeyRegistrationCompleteRequest {
|
||||
pub credential: RegisterPublicKeyCredential,
|
||||
}
|
||||
|
||||
/// Identified-account login. Missing handles are rejected, not resolved to an implicit user.
|
||||
///
|
||||
/// The nullable wire field permits an explicit authentication error for old callers; it does not
|
||||
/// advertise a discoverable-login ceremony.
|
||||
/// Normal discoverable login omits the handle: the authenticator selects a credential.
|
||||
/// An explicit handle selects legacy identified-account login, for non-resident keys or
|
||||
/// keys registered with an old random WebAuthn user handle. This never adds a key or
|
||||
/// falls back to another identity. Authenticate first, then enroll an RK from Account.
|
||||
/// Existing challenge states are short-lived and are not migrated to the new ceremony.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
|
||||
@@ -561,6 +561,28 @@ fn reasoning_proves_rollback_safety_but_private_and_derived_items_are_not_materi
|
||||
session,
|
||||
vec![derived, message("system", LoggedRole::System, "secret")],
|
||||
),
|
||||
LogEntry::AnnotatedSystemItem {
|
||||
ts: 1,
|
||||
entry: crate::LoggedSystemHistoryEntry {
|
||||
item: crate::SystemItem::JournalStartup {
|
||||
body: "previous Journal must not become new recording material".into(),
|
||||
snapshot: serde_json::json!({"journals": []}),
|
||||
snapshot_sha256: "retained-digest".into(),
|
||||
prompt_provenance: crate::PromptRenderProvenance {
|
||||
workspace_id: None,
|
||||
source_digest: "source".into(),
|
||||
projection_digest: "projection".into(),
|
||||
logical_name: "internal.subjektiv_journal_startup".into(),
|
||||
},
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
entry_id: LoggedSessionHistoryEntryId("startup-journal".into()),
|
||||
origin: LoggedSessionHistoryOrigin::DerivedSummary,
|
||||
derivation: None,
|
||||
},
|
||||
},
|
||||
extensions: vec![],
|
||||
},
|
||||
invoke(),
|
||||
input("u"),
|
||||
LogEntry::AnnotatedAssistantItem {
|
||||
|
||||
@@ -92,6 +92,14 @@ pub struct PromptRenderProvenance {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum SystemItem {
|
||||
/// One-time derived Journal reference context. The bounded Host snapshot is
|
||||
/// retained as JSON independently of its rendered body and prompt projection.
|
||||
JournalStartup {
|
||||
body: String,
|
||||
snapshot: serde_json::Value,
|
||||
snapshot_sha256: String,
|
||||
prompt_provenance: PromptRenderProvenance,
|
||||
},
|
||||
/// Free-form notification sent in by an external caller via
|
||||
/// `Method::Notify`. `message` is the raw caller-supplied text;
|
||||
/// `body` is the wrapped LLM-context form (Worker renders it via
|
||||
@@ -194,6 +202,7 @@ impl SystemItem {
|
||||
/// for this item. Returns the variant's stored `body` verbatim.
|
||||
pub fn history_text(&self) -> String {
|
||||
match self {
|
||||
SystemItem::JournalStartup { body, .. } => body.clone(),
|
||||
SystemItem::Notification { body, .. } => body.clone(),
|
||||
SystemItem::WorkerEvent { body, .. } => body.clone(),
|
||||
SystemItem::FeatureInvocationResult { body, .. } => body.clone(),
|
||||
@@ -220,6 +229,7 @@ impl SystemItem {
|
||||
/// wire — keep flexible.
|
||||
pub fn kind_label(&self) -> &'static str {
|
||||
match self {
|
||||
SystemItem::JournalStartup { .. } => "journal_startup",
|
||||
SystemItem::Notification { .. } => "notification",
|
||||
SystemItem::WorkerEvent { .. } => "worker_event",
|
||||
SystemItem::FeatureInvocationResult { .. } => "feature_invocation_result",
|
||||
|
||||
@@ -762,6 +762,11 @@ impl Store for WorkerSessionStore {
|
||||
}
|
||||
|
||||
fn list_segments(&self, session_id: SessionId) -> Result<Vec<SegmentId>, StoreError> {
|
||||
// A pending Worker can have metadata before its atomic initial seed.
|
||||
// Absence is queryable without materializing or binding this store.
|
||||
if self.session_id()?.is_none() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.ensure_session(session_id, false)?;
|
||||
let mut segments: Vec<SegmentId> = Vec::new();
|
||||
for entry in fs::read_dir(self.root.join(SEGMENTS_DIR))? {
|
||||
@@ -804,6 +809,9 @@ impl Store for WorkerSessionStore {
|
||||
}
|
||||
|
||||
fn exists(&self, session_id: SessionId, segment_id: SegmentId) -> Result<bool, StoreError> {
|
||||
if self.session_id()?.is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
self.ensure_session(session_id, false)?;
|
||||
Ok(self.log_path(segment_id).exists())
|
||||
}
|
||||
@@ -2103,6 +2111,24 @@ mod tests {
|
||||
assert!(matches!(error, StoreError::Corrupt { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_session_probes_are_empty_without_binding_and_foreign_materialized_session_is_rejected()
|
||||
{
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let session_id = new_session_id();
|
||||
let segment_id = new_segment_id();
|
||||
let store = WorkerSessionStore::new(root.path()).unwrap();
|
||||
assert!(!store.exists(session_id, segment_id).unwrap());
|
||||
assert!(store.list_segments(session_id).unwrap().is_empty());
|
||||
assert_eq!(store.session_id().unwrap(), None);
|
||||
let reopened = WorkerSessionStore::new(root.path()).unwrap();
|
||||
assert_eq!(reopened.session_id().unwrap(), None);
|
||||
store.create_segment(session_id, segment_id, &[]).unwrap();
|
||||
let foreign = new_session_id();
|
||||
assert!(store.exists(foreign, segment_id).is_err());
|
||||
assert!(store.list_segments(foreign).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reopen_preserves_session_and_segment_ids() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
|
||||
+150
-15
@@ -57,6 +57,15 @@ pub struct StandaloneHost {
|
||||
lease: Option<StandaloneWorkerLease>,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
subject: Option<crate::subjektiv::SubjectConnection>,
|
||||
subject_release: Option<worker::SubjectReleaseReservation>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum StandaloneSubjectDisconnectError {
|
||||
#[error("Subject release requires an actual Idle Controller without pending work: {0}")]
|
||||
Controller(String),
|
||||
#[error("Subject release remains incomplete; retained binding/lease must be reconciled: {0}")]
|
||||
Incomplete(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
@@ -89,6 +98,10 @@ pub enum StandaloneStartupError {
|
||||
FeatureComposition,
|
||||
#[error("the in-process Worker controller could not start")]
|
||||
Controller,
|
||||
#[error(
|
||||
"mandatory initial context remains unconfirmed; restore retained Worker {worker_id} to retry this Session"
|
||||
)]
|
||||
InitialContextPending { worker_id: WorkerId },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
@@ -171,7 +184,7 @@ impl StandaloneHost {
|
||||
let jobs = match crate::jobs::StandaloneJobs::open(&jobs_path, launch.cwd.clone()) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(_) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
};
|
||||
@@ -179,8 +192,11 @@ impl StandaloneHost {
|
||||
if let Some(client) = &model_client {
|
||||
jobs.bind_model_client(client.clone_boxed());
|
||||
}
|
||||
if let Some(subject) = &subject {
|
||||
subject.host.bind_jobs(jobs.clone())?;
|
||||
if let Some(connection) = &subject
|
||||
&& let Err(error) = connection.host.bind_jobs(jobs.clone())
|
||||
{
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(error);
|
||||
}
|
||||
// WorkerId is the stable identity. The current Worker store remains
|
||||
// name-keyed, so keep its derived storage key separate from the
|
||||
@@ -192,7 +208,7 @@ impl StandaloneHost {
|
||||
let (backing_store, worker_store) = match backing_store(&store, worker_id) {
|
||||
Ok(stores) => stores,
|
||||
Err(error) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
@@ -220,12 +236,61 @@ impl StandaloneHost {
|
||||
if let Some(model_client) = model_client {
|
||||
bootstrap = bootstrap.with_model_client(model_client);
|
||||
}
|
||||
let prepared = match bootstrap.prepare().await {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => {
|
||||
cancel_unpublished_start(&store, &mut subject, allocation)?;
|
||||
return Err(classify_startup_error(error));
|
||||
}
|
||||
};
|
||||
// Reserve a recoverable identity before mandatory initial-context persistence.
|
||||
// A failed/unknown seed must be reopened as this Session, never abandoned and
|
||||
// replaced by a newly fetched context under an invisible ghost binding.
|
||||
let pending_record = if subject.is_some() {
|
||||
match store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest.clone(),
|
||||
storage_key.clone(),
|
||||
prepared.worker().session_id(),
|
||||
None,
|
||||
subject.as_ref().map(|s| s.binding.clone()),
|
||||
) {
|
||||
Ok(record) => Some(record),
|
||||
Err(_) => {
|
||||
if let Some(subject) = &mut subject {
|
||||
subject.retain();
|
||||
}
|
||||
allocation.into_lease().retain();
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(subject) = &mut subject {
|
||||
subject.arm();
|
||||
}
|
||||
let started = match bootstrap.start().await {
|
||||
let started = match prepared.start().await {
|
||||
Ok(started) => started,
|
||||
Err(error) => {
|
||||
if let Some(record) = &pending_record {
|
||||
let cleanup = jobs.shutdown().await;
|
||||
let saved = store.mark_stopped(
|
||||
record,
|
||||
record.active_session_id,
|
||||
record.active_segment_id,
|
||||
StandaloneShutdownReason::StartupFailed,
|
||||
);
|
||||
let closed = cleanup.is_ok()
|
||||
&& saved.is_ok()
|
||||
&& subject.as_mut().is_some_and(|s| s.close().is_ok());
|
||||
if closed {
|
||||
let _ = allocation.into_lease().release();
|
||||
} else {
|
||||
allocation.into_lease().retain();
|
||||
}
|
||||
return Err(StandaloneStartupError::InitialContextPending { worker_id });
|
||||
}
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
return Err(classify_startup_error(error));
|
||||
}
|
||||
@@ -234,22 +299,35 @@ impl StandaloneHost {
|
||||
Ok(active) => active,
|
||||
Err(error) => {
|
||||
stop_started_worker(started).await;
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
if pending_record.is_some() {
|
||||
allocation.into_lease().retain();
|
||||
} else {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let record = match store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest,
|
||||
storage_key,
|
||||
active.session_id,
|
||||
active.segment_id,
|
||||
subject.as_ref().map(|s| s.binding.clone()),
|
||||
) {
|
||||
let committed = if let Some(record) = &pending_record {
|
||||
store.update_active_pointer(record, active.session_id, active.segment_id)
|
||||
} else {
|
||||
store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest,
|
||||
storage_key,
|
||||
active.session_id,
|
||||
active.segment_id,
|
||||
None,
|
||||
)
|
||||
};
|
||||
let record = match committed {
|
||||
Ok(record) => record,
|
||||
Err(_) => {
|
||||
stop_started_worker(started).await;
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
if pending_record.is_some() {
|
||||
allocation.into_lease().retain();
|
||||
} else {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
}
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
};
|
||||
@@ -427,6 +505,7 @@ impl StandaloneHost {
|
||||
lease: Some(lease),
|
||||
jobs,
|
||||
subject,
|
||||
subject_release: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -457,6 +536,45 @@ impl StandaloneHost {
|
||||
Client::new(socket)
|
||||
}
|
||||
|
||||
/// Logical Subject disconnection, not Worker shutdown or completion of its represented work.
|
||||
/// A successful release keeps the old Controller fenced; a fresh Worker/Session may then
|
||||
/// acquire the Subject. Failures before durable revocation allow normal recovery and retry.
|
||||
pub async fn disconnect_subject(&mut self) -> Result<(), StandaloneSubjectDisconnectError> {
|
||||
let Some(subject) = &mut self.subject else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut reservation = match self.subject_release.take() {
|
||||
Some(reservation) => reservation,
|
||||
None => self
|
||||
.handle
|
||||
.reserve_subject_release()
|
||||
.await
|
||||
.map_err(StandaloneSubjectDisconnectError::Controller)?,
|
||||
};
|
||||
let release = match subject.release_for_handoff(&self.record, &mut reservation) {
|
||||
Ok(()) => match self.jobs.shutdown().await {
|
||||
Ok(()) => subject.close().map_err(|error| error.to_string()),
|
||||
Err(error) => Err(error.to_string()),
|
||||
},
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
match release {
|
||||
Ok(()) => {
|
||||
self.subject_release = Some(reservation);
|
||||
self.subject = None;
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => {
|
||||
// If revocation committed, keep the actual Controller reserved even if closing
|
||||
// storage/unlocking failed. Dropping a reservation must not undo revocation.
|
||||
if subject.writer_released_or_unknown() {
|
||||
self.subject_release = Some(reservation);
|
||||
}
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
|
||||
self.shutdown_timeout = shutdown_timeout;
|
||||
self
|
||||
@@ -695,6 +813,23 @@ async fn send_protocol_event(peer: &InProcessPeer, event: Event) -> bool {
|
||||
peer.send(message).await.is_ok()
|
||||
}
|
||||
|
||||
fn cancel_unpublished_start(
|
||||
store: &StandaloneWorkerStore,
|
||||
subject: &mut Option<crate::subjektiv::SubjectConnection>,
|
||||
allocation: crate::store::StandaloneWorkerAllocation,
|
||||
) -> Result<(), StandaloneStartupError> {
|
||||
if let Some(connection) = subject {
|
||||
if connection.abort_unpublished_start().is_err() {
|
||||
connection.retain();
|
||||
allocation.into_lease().retain();
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
}
|
||||
store
|
||||
.abandon_allocation(allocation)
|
||||
.map_err(classify_store_startup_error)
|
||||
}
|
||||
|
||||
fn backing_store(
|
||||
store: &StandaloneWorkerStore,
|
||||
worker_id: WorkerId,
|
||||
|
||||
@@ -66,6 +66,15 @@ struct Executions {
|
||||
jobs: HashMap<String, Execution>,
|
||||
}
|
||||
|
||||
pub(crate) struct QuiescentJobClosure<'a> {
|
||||
executions: std::sync::MutexGuard<'a, Executions>,
|
||||
}
|
||||
impl QuiescentJobClosure<'_> {
|
||||
pub(crate) fn commit(&mut self) {
|
||||
self.executions.closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Cloneable Feature-facing access to a Host-owned service. Cloning this handle
|
||||
/// neither extends the Host's authority nor permits opening/recovering its store.
|
||||
#[derive(Clone)]
|
||||
@@ -79,6 +88,27 @@ pub struct StandaloneJobs {
|
||||
}
|
||||
|
||||
impl StandaloneJobs {
|
||||
/// Excludes request/start/result acknowledgement while a Subject Host checks its final
|
||||
/// saved obligations and releases ownership. Accepted results alone are insufficient:
|
||||
/// every spawned execution must have actually finished successful cleanup.
|
||||
pub(crate) fn reserve_quiescent_closure(
|
||||
&self,
|
||||
) -> Result<QuiescentJobClosure<'_>, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
if executions.jobs.values().any(|execution| {
|
||||
!execution.task.is_finished()
|
||||
|| !matches!(execution.complete.borrow().as_ref(), Some(Ok(())))
|
||||
}) {
|
||||
return Err(JobServiceError::Cleanup(
|
||||
"delegated Job execution/cleanup remains running or unconfirmed".into(),
|
||||
));
|
||||
}
|
||||
Ok(QuiescentJobClosure { executions })
|
||||
}
|
||||
|
||||
pub(crate) fn open(path: &Path, cwd: PathBuf) -> Result<Self, JobServiceError> {
|
||||
Ok(Self {
|
||||
store: Arc::new(Mutex::new(Some(JobStore::open(path)?))),
|
||||
|
||||
@@ -8,7 +8,10 @@ pub mod store;
|
||||
pub mod subjektiv;
|
||||
mod subjektiv_sessions;
|
||||
|
||||
pub use host::{StandaloneHost, StandaloneShutdownError, StandaloneStartupError};
|
||||
pub use host::{
|
||||
StandaloneHost, StandaloneShutdownError, StandaloneStartupError,
|
||||
StandaloneSubjectDisconnectError,
|
||||
};
|
||||
pub use launch::{ResolvedStandaloneLaunch, StandaloneLaunchConfig, StandaloneLaunchError};
|
||||
pub use protocol::WorkerId;
|
||||
pub use store::{
|
||||
|
||||
@@ -396,14 +396,8 @@ impl StandaloneWorkerStore {
|
||||
LeaseLiveness::Unknown => StandaloneStoreError::LeaseLivenessUnknown(id),
|
||||
});
|
||||
}
|
||||
if let Some(retained) = crate::subjektiv::prepare_worker_deletion(&self.root, &record)? {
|
||||
// Atomic relocation preserves the original Session, not a permanent duplicate.
|
||||
// The Subject's durable consumer can read it after logical Worker removal.
|
||||
fs::rename(self.worker_dir(id), &retained).map_err(StandaloneStoreError::Io)?;
|
||||
sync_directory(retained.parent().expect("retained Worker parent"))?;
|
||||
} else {
|
||||
fs::remove_dir_all(self.worker_dir(id)).map_err(StandaloneStoreError::Io)?;
|
||||
}
|
||||
crate::subjektiv::prepare_worker_deletion(&self.root, &record)?;
|
||||
fs::remove_dir_all(self.worker_dir(id)).map_err(StandaloneStoreError::Io)?;
|
||||
sync_directory(&self.root)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,9 +5,11 @@
|
||||
//! non-private nodes; delegates schema/migration to `subjektiv` and connection policy to `feature-storage`.
|
||||
//!
|
||||
//! The Host holds each selected Subject's exclusive kernel lease through body execution and
|
||||
//! confirmed delegated Job cleanup. Process death releases kernel locks; uncertain in-process
|
||||
//! cleanup retains leases. Restore validates saved Subject/storage bindings and reacquires the
|
||||
//! same Subject. Launches without a selected Subject do not implicitly connect this store.
|
||||
//! confirmed delegated Job cleanup. Kernel lock release never clears durable body ownership:
|
||||
//! shutdown/process death permits only the same body to restore, not a replacement to start.
|
||||
//! Explicit disconnection requires actual Controller quiescence and final Journal completion;
|
||||
//! storage then revokes that writer lifetime before the lease is released. Uncertain cleanup
|
||||
//! retains leases. Launches without a selected Subject do not implicitly connect this store.
|
||||
//!
|
||||
//! The selected Subject's Job store is shared across Workers; the generic Jobs service owns execution
|
||||
//! and cleanup. Consolidation grants allow only immutable candidate batches and limited operations,
|
||||
@@ -49,6 +51,8 @@ pub enum SubjectError {
|
||||
#[error(transparent)]
|
||||
Domain(#[from] subjektiv::SubjektivError),
|
||||
#[error(transparent)]
|
||||
Journal(#[from] subjektiv::journal::JournalError),
|
||||
#[error(transparent)]
|
||||
Storage(#[from] feature_storage::FeatureStorageError),
|
||||
#[error("local Subject storage metadata: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
@@ -223,7 +227,49 @@ impl StandaloneSubjects {
|
||||
scope_id: self.scope_id.clone(),
|
||||
subject_id: subject_id.to_string(),
|
||||
};
|
||||
if self.store.journal_worker_was_released(
|
||||
subject_id,
|
||||
&self.scope_id,
|
||||
&worker_id.to_string(),
|
||||
)? {
|
||||
return Err(SubjectError::InvalidScope("this body's Subject binding was released; start a new Worker/Session rather than restore its old authority".into()));
|
||||
}
|
||||
// Before writer lifetimes existed, a saved body record was the only durable
|
||||
// attachment evidence. Never interpret its stopped status as a release on upgrade.
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(&self.state_root)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
for entry in fs::read_dir(&self.state_root)? {
|
||||
let entry = entry?;
|
||||
let Ok(prior_id) = entry
|
||||
.file_name()
|
||||
.to_string_lossy()
|
||||
.parse::<protocol::WorkerId>()
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if prior_id == worker_id {
|
||||
continue;
|
||||
}
|
||||
let prior = match workers.load(prior_id) {
|
||||
Ok(prior) => prior,
|
||||
Err(crate::store::StandaloneStoreError::WorkerNotFound(_)) => continue,
|
||||
Err(error) => return Err(SubjectError::InvalidScope(error.to_string())),
|
||||
};
|
||||
if prior.subject.as_ref() == Some(&binding)
|
||||
&& !self.store.journal_worker_was_released(
|
||||
subject_id,
|
||||
&self.scope_id,
|
||||
&prior_id.to_string(),
|
||||
)?
|
||||
{
|
||||
return Err(SubjectError::InvalidScope("saved Subject body has no explicit release; restore it and complete the Idle/Journal handoff, rather than treating inactive storage as unoccupied".into()));
|
||||
}
|
||||
}
|
||||
let writer =
|
||||
self.store
|
||||
.claim_journal_writer(subject_id, &self.scope_id, &worker_id.to_string())?;
|
||||
let host = Arc::new_cyclic(|weak| LocalSubjectHost {
|
||||
writer,
|
||||
self_weak: weak.clone(),
|
||||
journal_dispatch: Mutex::new(false),
|
||||
language: self.memory_language.clone(),
|
||||
@@ -252,6 +298,37 @@ pub(crate) struct SubjectConnection {
|
||||
armed: bool,
|
||||
}
|
||||
impl SubjectConnection {
|
||||
/// Construction failed before any Controller or mandatory startup context was published.
|
||||
/// This is cancellation of an unpublished reservation, not Idle inferred from a stopped body.
|
||||
pub(crate) fn abort_unpublished_start(&mut self) -> Result<(), SubjectError> {
|
||||
self.host
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&self.host.writer)?;
|
||||
self.close()
|
||||
}
|
||||
|
||||
pub(crate) fn writer_released_or_unknown(&self) -> bool {
|
||||
self.host
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.host.writer)
|
||||
.map(|active| !active)
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
pub(crate) fn release_for_handoff(
|
||||
&mut self,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
reservation: &mut worker::SubjectReleaseReservation,
|
||||
) -> Result<(), String> {
|
||||
if let Err(error) = self.host.release_for_handoff(record, reservation) {
|
||||
self.host.wake_journals();
|
||||
return Err(error.to_string());
|
||||
}
|
||||
reservation.commit_disconnect()
|
||||
}
|
||||
|
||||
pub fn arm(&mut self) {
|
||||
self.armed = true;
|
||||
}
|
||||
@@ -282,6 +359,7 @@ impl Drop for SubjectConnection {
|
||||
}
|
||||
|
||||
pub(crate) struct LocalSubjectHost {
|
||||
writer: subjektiv::journal::JournalWriterBinding,
|
||||
self_weak: std::sync::Weak<Self>,
|
||||
journal_dispatch: Mutex<bool>,
|
||||
catalog: StandaloneSubjects,
|
||||
@@ -300,6 +378,16 @@ impl LocalSubjectHost {
|
||||
"local Subject Host is closed".into(),
|
||||
));
|
||||
}
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal::journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"Subject writer binding was released".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn body(&self, context: &SubjektivHostContext) -> Result<(), SubjektivHostError> {
|
||||
@@ -339,6 +427,9 @@ impl LocalSubjectHost {
|
||||
"Session does not belong to this local Subject/Worker scope".into(),
|
||||
));
|
||||
}
|
||||
store
|
||||
.bind_journal_session(&self.writer, record.clone())
|
||||
.map_err(journal::journal_error)?;
|
||||
return Ok(record);
|
||||
}
|
||||
if !create {
|
||||
@@ -355,9 +446,13 @@ impl LocalSubjectHost {
|
||||
&context.session_id,
|
||||
)
|
||||
.map_err(domain_error)?;
|
||||
store
|
||||
let record = store
|
||||
.record_session_attribution(record)
|
||||
.map_err(domain_error)
|
||||
.map_err(domain_error)?;
|
||||
store
|
||||
.bind_journal_session(&self.writer, record.clone())
|
||||
.map_err(journal::journal_error)?;
|
||||
Ok(record)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -698,6 +793,53 @@ impl LocalSubjectHost {
|
||||
}
|
||||
}
|
||||
impl SubjektivHost for LocalSubjectHost {
|
||||
fn memory_journal(
|
||||
&self,
|
||||
context: &SubjektivHostContext,
|
||||
request: subjektiv::journal::memory_links::MemoryJournalRequest,
|
||||
) -> Result<subjektiv::journal::memory_links::MemoryJournalResponse, SubjektivHostError> {
|
||||
self.body(context)?;
|
||||
let reader = subjektiv::journal::JournalHostContext::validated_reader(
|
||||
&self.catalog.store,
|
||||
&self.binding.subject_id,
|
||||
)
|
||||
.map_err(journal::journal_error)?;
|
||||
self.catalog
|
||||
.store
|
||||
.execute_memory_journal_with_source_presence(&reader, request, |session| {
|
||||
crate::subjektiv_sessions::original_session_presence(
|
||||
&self.catalog.state_root,
|
||||
&self.binding.scope_id,
|
||||
&self.binding.subject_id,
|
||||
&self.catalog.store,
|
||||
session,
|
||||
)
|
||||
})
|
||||
.map_err(journal::journal_error)
|
||||
}
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
self.live.load(Ordering::Acquire)
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
context: &SubjektivHostContext,
|
||||
) -> Result<subjektiv::journal::startup::JournalStartupSnapshot, SubjektivHostError> {
|
||||
self.body(context)?;
|
||||
let reader = subjektiv::journal::JournalHostContext::validated_reader(
|
||||
&self.catalog.store,
|
||||
&self.binding.subject_id,
|
||||
)
|
||||
.map_err(journal::journal_error)?;
|
||||
let predecessor = self
|
||||
.catalog
|
||||
.store
|
||||
.journal_predecessor_session(&self.writer)
|
||||
.map_err(journal::journal_error)?;
|
||||
self.catalog
|
||||
.store
|
||||
.journal_startup_snapshot(&reader, predecessor.as_deref())
|
||||
.map_err(journal::journal_error)
|
||||
}
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
self.live.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ const INPUT_BUDGET: usize = 48 * 1024;
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LocalJournalGrant {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
writer_binding_id: Option<String>,
|
||||
binding: StandaloneSubjectBinding,
|
||||
session_id: String,
|
||||
journal_id: String,
|
||||
@@ -59,73 +61,72 @@ impl StandaloneSubjects {
|
||||
"Session does not belong to the selected Subject/storage scope".into(),
|
||||
)
|
||||
})?;
|
||||
let worker = attribution
|
||||
.worker_id
|
||||
.parse()
|
||||
.map_err(|_| SubjectError::InvalidScope("invalid attributed Worker identity".into()))?;
|
||||
let mut connection = self.connect(subject_id, worker)?;
|
||||
let result = (|| -> Result<(), SubjektivHostError> {
|
||||
let host = &connection.host;
|
||||
let context = authority(host, session_id)?;
|
||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&host.catalog.state_root,
|
||||
&host.binding.scope_id,
|
||||
subject_id,
|
||||
&host.catalog.store,
|
||||
session_id,
|
||||
)?;
|
||||
// Persist this even if source erasure subsequently fails. A retry resumes physical
|
||||
// deletion with the same permanent fence; it must never generate from leftovers.
|
||||
host.catalog
|
||||
.store
|
||||
.erase_journal_session(&context, request_id)
|
||||
.map_err(journal_error)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&connection.jobs_path,
|
||||
host.catalog.state_root.clone(),
|
||||
let leases = self.root.join("leases");
|
||||
private_dir(&leases, true)?;
|
||||
let lease = private_file(&leases.join(format!("{subject_id}.lock")), true)?;
|
||||
if !lease.try_lock_exclusive()? {
|
||||
return Err(SubjectError::Active(subject_id.into()));
|
||||
}
|
||||
let context = JournalHostContext::validated_session(&self.store, attribution)?;
|
||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.state_root,
|
||||
&self.scope_id,
|
||||
subject_id,
|
||||
&self.store,
|
||||
session_id,
|
||||
)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
// Erasure is not a logical reconnect. Never mint a new writer lifetime in order
|
||||
// to delete a released bodys text; the permanent fence precedes physical deletion.
|
||||
self.store.erase_journal_session(&context, request_id)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&self.root.join("jobs").join(subject_id).join("jobs.sqlite3"),
|
||||
self.state_root.clone(),
|
||||
)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
for snapshot in jobs
|
||||
.pending()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
.into_iter()
|
||||
.chain(
|
||||
jobs.unknown_granted()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
for snapshot in jobs
|
||||
.pending()
|
||||
.map_err(service_error)?
|
||||
.into_iter()
|
||||
.chain(jobs.unknown_granted().map_err(service_error)?)
|
||||
{
|
||||
if serde_json::from_value::<LocalJournalGrant>(
|
||||
snapshot.domain_grant.unwrap_or_default(),
|
||||
)
|
||||
.is_ok_and(|grant| grant.binding == host.binding && grant.session_id == session_id)
|
||||
{
|
||||
jobs.cancel(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
}
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
jobs.cancel(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
for snapshot in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if serde_json::from_value::<LocalJournalGrant>(
|
||||
snapshot.domain_grant.unwrap_or_default(),
|
||||
)
|
||||
.is_ok_and(|grant| grant.binding == host.binding && grant.session_id == session_id)
|
||||
{
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
}
|
||||
}
|
||||
for snapshot in jobs
|
||||
.unacknowledged_results()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
session_store::FsStore::new_with_retention(
|
||||
path,
|
||||
crate::store::session_retention(&host.catalog.state_root),
|
||||
)
|
||||
.map_err(source_error)?
|
||||
.delete_session_with_intent(
|
||||
session_id
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
Ok(())
|
||||
})();
|
||||
connection.close()?;
|
||||
result.map_err(|error| SubjectError::InvalidScope(error.to_string()))
|
||||
}
|
||||
session_store::FsStore::new_with_retention(
|
||||
path,
|
||||
crate::store::session_retention(&self.state_root),
|
||||
)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
.delete_session_with_intent(
|
||||
session_id
|
||||
.parse()
|
||||
.map_err(|_| SubjectError::InvalidScope("invalid Session".into()))?,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
self.store.release_erased_journal_body(&context)?;
|
||||
self.manager.shutdown()?;
|
||||
lease.unlock()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,17 +134,17 @@ pub(crate) fn retained_workers_root(state_root: &Path) -> PathBuf {
|
||||
state_root.join("subjektiv").join("retained-workers")
|
||||
}
|
||||
|
||||
/// Called while the generic Worker deletion mutation lease excludes restore/final commits.
|
||||
/// Returning a destination means logical removal must move, not copy/delete, original material.
|
||||
/// There is no model call, and failure leaves the original Worker and its sources in place.
|
||||
/// Ordinary removal cannot turn a stopped/unknown body into an unoccupied Subject. The
|
||||
/// live Host must first release it under the Controller Idle reservation. The saved
|
||||
/// release fence also prevents newly appended old-body history from being recorded here.
|
||||
pub(crate) fn prepare_worker_deletion(
|
||||
state_root: &Path,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
) -> Result<Option<PathBuf>, crate::store::StandaloneStoreError> {
|
||||
) -> Result<(), crate::store::StandaloneStoreError> {
|
||||
let Some(binding) = &record.subject else {
|
||||
return Ok(None);
|
||||
return Ok(());
|
||||
};
|
||||
let prepare = || -> Result<Option<PathBuf>, SubjektivHostError> {
|
||||
let prepare = || -> Result<(), SubjektivHostError> {
|
||||
let catalog = StandaloneSubjects::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
if catalog.scope_id != binding.scope_id {
|
||||
@@ -151,89 +152,68 @@ pub(crate) fn prepare_worker_deletion(
|
||||
"deletion Subject scope mismatch".into(),
|
||||
));
|
||||
}
|
||||
let mut connection = catalog
|
||||
.connect(&binding.subject_id, record.worker_id)
|
||||
let erased = catalog
|
||||
.store
|
||||
.session_attribution(&record.active_session_id.to_string())
|
||||
.map_err(domain_error)?
|
||||
.filter(|attr| {
|
||||
attr.subject_id == binding.subject_id
|
||||
&& attr.runtime_id == binding.scope_id
|
||||
&& attr.worker_id == record.worker_id.to_string()
|
||||
})
|
||||
.map(|attr| JournalHostContext::validated_session(&catalog.store, attr))
|
||||
.transpose()
|
||||
.map_err(journal_error)?;
|
||||
let destructively_released = match erased.as_ref() {
|
||||
Some(context) => catalog
|
||||
.store
|
||||
.release_erased_journal_body(context)
|
||||
.map_err(journal_error)?,
|
||||
None => false,
|
||||
};
|
||||
if !destructively_released
|
||||
&& !catalog
|
||||
.store
|
||||
.journal_worker_was_released(
|
||||
&binding.subject_id,
|
||||
&binding.scope_id,
|
||||
&record.worker_id.to_string(),
|
||||
)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::Conflict { code: "subject_occupied".into(), message: "Subject binding is retained; restore this body and explicitly disconnect while Idle after Journal activity, position and checkpoint complete".into() });
|
||||
}
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let result = (|| {
|
||||
let host = &connection.host;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&connection.jobs_path,
|
||||
record.cwd.canonical_path.clone(),
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
host.bind_jobs(jobs)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(state_root)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let sessions = workers.sessions_dir(record.worker_id);
|
||||
let mut pending = false;
|
||||
// Check every original Session, including a save that preceded its notification.
|
||||
// An unknown old Session attribution/lineage blocks removal instead of being erased.
|
||||
if !sessions.is_dir() {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"original Session storage is missing".into(),
|
||||
));
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(record.worker_id),
|
||||
crate::store::session_retention(state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in session_store::Store::list_sessions(&source_store).map_err(source_error)? {
|
||||
if destructively_released {
|
||||
source_store
|
||||
.delete_session_with_intent(
|
||||
session,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
} else {
|
||||
source_store
|
||||
.release_session_pin(
|
||||
session,
|
||||
session_store::retention::SessionRetentionPin(session),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
}
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
&sessions,
|
||||
crate::store::session_retention(state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in
|
||||
session_store::Store::list_sessions(&source_store).map_err(source_error)?
|
||||
{
|
||||
let context = SubjektivHostContext {
|
||||
worker_id: record.worker_id.to_string(),
|
||||
session_id: session.to_string(),
|
||||
};
|
||||
host.attribution(&context, session == record.active_session_id)?;
|
||||
host.reconcile_journal_session(&context.session_id)?;
|
||||
pending |= !host.session_fully_reflected(&context.session_id)?;
|
||||
if source_store
|
||||
.retention()
|
||||
.pins(session)
|
||||
.map_err(source_error)?
|
||||
.iter()
|
||||
.any(|pin| pin.0 != session)
|
||||
{
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"another consumer retains this Session; relocation is not authorized"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if !pending {
|
||||
for session in
|
||||
session_store::Store::list_sessions(&source_store).map_err(source_error)?
|
||||
{
|
||||
host.prune_source_session(&source_store, session)?;
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
// Reservation is durable before sources leave the public Worker namespace. The
|
||||
// same Subject's next Host consumes it with the retained attribution after reopen.
|
||||
let _ = host.next_journal_job()?;
|
||||
let archive = retained_workers_root(state_root);
|
||||
private_dir(&archive, true)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
let destination = archive.join(record.worker_id.to_string());
|
||||
if destination.exists() {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"retained Worker already exists; reconcile deletion outcome".into(),
|
||||
));
|
||||
}
|
||||
Ok(Some(destination))
|
||||
})();
|
||||
connection
|
||||
.close()
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
|
||||
result
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
prepare()
|
||||
.map_err(|error| crate::store::StandaloneStoreError::JournalRetention(error.to_string()))
|
||||
}
|
||||
|
||||
fn journal_error(error: JournalError) -> SubjektivHostError {
|
||||
pub(super) fn journal_error(error: JournalError) -> SubjektivHostError {
|
||||
match error {
|
||||
JournalError::Conflict(message) => SubjektivHostError::Conflict {
|
||||
code: "journal_conflict".into(),
|
||||
@@ -270,6 +250,89 @@ fn authority(
|
||||
}
|
||||
|
||||
impl LocalSubjectHost {
|
||||
/// The reservation is obtained by this Host from its actual Controller and kept through
|
||||
/// final reconciliation and durable revocation. A stopped body cannot supply this proof.
|
||||
pub(crate) fn release_for_handoff(
|
||||
&self,
|
||||
record: &crate::store::StandaloneWorkerRecord,
|
||||
_reservation: &worker::SubjectReleaseReservation,
|
||||
) -> Result<(), SubjektivHostError> {
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Ok(()); // reconcile an earlier committed revocation, never renew it
|
||||
}
|
||||
self.ensure_live()?;
|
||||
let dispatch = self
|
||||
.journal_dispatch
|
||||
.lock()
|
||||
.expect("Journal dispatcher poisoned");
|
||||
if *dispatch {
|
||||
return Err(SubjektivHostError::Conflict {
|
||||
code: "journal_running".into(),
|
||||
message: "Journal dispatcher is still running; binding retained".into(),
|
||||
});
|
||||
}
|
||||
let workers = crate::store::StandaloneWorkerStore::open_existing(&self.catalog.state_root)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?;
|
||||
let source_store = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(self.worker_id),
|
||||
crate::store::session_retention(&self.catalog.state_root),
|
||||
)
|
||||
.map_err(source_error)?;
|
||||
for session in session_store::Store::list_sessions(&source_store).map_err(source_error)? {
|
||||
let context = SubjektivHostContext {
|
||||
worker_id: self.worker_id.to_string(),
|
||||
session_id: session.to_string(),
|
||||
};
|
||||
self.attribution(&context, session == record.active_session_id)?;
|
||||
self.reconcile_journal_session(&context.session_id)?;
|
||||
if !self.session_fully_reflected(&context.session_id)? {
|
||||
return Err(SubjektivHostError::Conflict {
|
||||
code: "journal_unreflected".into(),
|
||||
message:
|
||||
"Final saved public history has not been fully recorded; binding retained"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
let jobs = self.jobs()?;
|
||||
for result in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if self.journal_grant(&result).is_ok() {
|
||||
self.complete_journal_job(&jobs, &result)?;
|
||||
}
|
||||
}
|
||||
let mut job_closure = jobs.reserve_quiescent_closure().map_err(service_error)?;
|
||||
for job in jobs
|
||||
.pending()
|
||||
.map_err(service_error)?
|
||||
.into_iter()
|
||||
.chain(jobs.unknown_granted().map_err(service_error)?)
|
||||
.chain(jobs.unacknowledged_results().map_err(service_error)?)
|
||||
{
|
||||
let _ = job;
|
||||
return Err(SubjektivHostError::Conflict { code: "journal_outcome_unconfirmed".into(), message: "Subject has queued/running or Unknown/unacknowledged delegated work; binding retained".into() });
|
||||
}
|
||||
let released = self
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&self.writer)
|
||||
.map_err(journal_error);
|
||||
if released.is_ok()
|
||||
|| !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.unwrap_or(false)
|
||||
{
|
||||
job_closure.commit();
|
||||
}
|
||||
released
|
||||
}
|
||||
|
||||
fn journal_session_erased(&self, session: &str) -> Result<bool, SubjektivHostError> {
|
||||
let context = authority(self, session)?;
|
||||
self.catalog
|
||||
@@ -425,6 +488,11 @@ impl LocalSubjectHost {
|
||||
// Old Workers' retained sources are reconciled as well; possession of the
|
||||
// Subject lease alone does not bypass the original Worker binding checks.
|
||||
if attribution.runtime_id == self.binding.scope_id
|
||||
&& self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &attribution.session_id)
|
||||
.map_err(journal_error)?
|
||||
&& let Err(error) = self.reconcile_journal_session(&attribution.session_id)
|
||||
{
|
||||
eprintln!(
|
||||
@@ -519,6 +587,7 @@ impl LocalSubjectHost {
|
||||
fn next_journal_job(
|
||||
&self,
|
||||
) -> Result<Option<(crate::jobs::StandaloneJobs, String)>, SubjektivHostError> {
|
||||
self.ensure_live()?;
|
||||
let jobs = self.jobs()?;
|
||||
for saved in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if self.journal_grant(&saved).is_ok() {
|
||||
@@ -578,6 +647,14 @@ impl LocalSubjectHost {
|
||||
.list_journals(&reader, after.as_deref(), MAX_JOURNAL_PAGE)
|
||||
.map_err(journal_error)?;
|
||||
for journal in page.items {
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &journal.source_session_id)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let context = authority(self, &journal.source_session_id)?;
|
||||
if let Some(active) = self
|
||||
.catalog
|
||||
@@ -618,6 +695,7 @@ impl LocalSubjectHost {
|
||||
.collect::<Vec<_>>();
|
||||
let origins = self.journal_origins(&ranges)?;
|
||||
let grant = LocalJournalGrant {
|
||||
writer_binding_id: Some(self.writer.binding_id.clone()),
|
||||
binding: self.binding.clone(),
|
||||
session_id: journal.source_session_id,
|
||||
journal_id: journal.id.clone(),
|
||||
@@ -1012,6 +1090,10 @@ impl LocalSubjectHost {
|
||||
})?)
|
||||
.map_err(|error| SubjektivHostError::InvalidArgument(error.to_string()))?;
|
||||
if grant.binding != self.binding
|
||||
|| grant
|
||||
.writer_binding_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &self.writer.binding_id)
|
||||
|| snapshot.request.purpose != PURPOSE
|
||||
|| snapshot.request.serialization_key.as_deref()
|
||||
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|
||||
@@ -1021,6 +1103,24 @@ impl LocalSubjectHost {
|
||||
"Job input/grant does not match this Journal capability".into(),
|
||||
));
|
||||
}
|
||||
// Pre-binding persisted Jobs lack the new field. They may resume only after the
|
||||
// Host has bound their ORIGINAL Session to this lifetime. Session binding is immutable;
|
||||
// missing grant metadata never maps an old Session onto a later body/reconnection.
|
||||
if !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_session_writer_matches(&self.writer, &grant.session_id)
|
||||
.map_err(journal_error)?
|
||||
|| !self
|
||||
.catalog
|
||||
.store
|
||||
.journal_writer_active(&self.writer)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"Journal Job's original Session is not bound to this active body lifetime".into(),
|
||||
));
|
||||
}
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
let journal = self
|
||||
.catalog
|
||||
@@ -1060,10 +1160,18 @@ impl LocalSubjectHost {
|
||||
"Journal completion binding mismatch".into(),
|
||||
));
|
||||
}
|
||||
self.catalog
|
||||
.store
|
||||
.close_journal_generation(&context, &generation.id, &generation.job)
|
||||
let recovery = JournalJobContext::validated_recovery(
|
||||
&self.catalog.store,
|
||||
&context,
|
||||
&generation.id,
|
||||
generation.job.clone(),
|
||||
)
|
||||
.map_err(journal_error)?;
|
||||
let saved = subjektiv::journal::service::finish(&self.catalog.store, &recovery)
|
||||
.map_err(journal_error)?;
|
||||
if saved != completion {
|
||||
return Err(SubjektivHostError::Conflict { code: "journal_completion_mismatch".into(), message: "Job result differs from the saved Journal activity, position and checkpoint completion".into() });
|
||||
}
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
if let Err(error) = self.prune_retained_workers() {
|
||||
|
||||
@@ -8,6 +8,8 @@ use session_store::{Store, WorkerMetadataStore};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
mod compaction;
|
||||
mod handoff;
|
||||
mod memory_links;
|
||||
|
||||
struct Fixture {
|
||||
_temp: tempfile::TempDir,
|
||||
@@ -407,10 +409,19 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
|
||||
JournalAction::ConfirmUnchanged,
|
||||
state.selected_ranges.clone(),
|
||||
);
|
||||
let position = save(
|
||||
&adapter,
|
||||
"position",
|
||||
&nochange.state_token,
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "First prefix checked; last input still pending".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
save(
|
||||
&adapter,
|
||||
"checkpoint",
|
||||
&nochange.state_token,
|
||||
&position.state_token,
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["checked".into()],
|
||||
},
|
||||
@@ -675,6 +686,7 @@ impl agen::llm_client::LlmClient for JournalScript {
|
||||
#[derive(Clone, Default)]
|
||||
struct DialogAndJournal {
|
||||
journal: JournalScript,
|
||||
dialog_requests: Arc<std::sync::Mutex<Vec<agen::llm_client::Request>>>,
|
||||
dialog_calls: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
@@ -730,6 +742,7 @@ impl agen::llm_client::LlmClient for DialogAndJournal {
|
||||
};
|
||||
return Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))));
|
||||
}
|
||||
self.dialog_requests.lock().unwrap().push(request.clone());
|
||||
self.dialog_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Box::pin(futures::stream::iter(
|
||||
[
|
||||
@@ -1000,6 +1013,60 @@ async fn real_job_tools_save_and_drain_last_input_without_another_turn() {
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_pre_binding_job_recovers_only_under_original_session_lifetime_and_never_after_release()
|
||||
{
|
||||
let mut fixture = Fixture::new("pre-upgrade pending Journal");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, modern_id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let saved = jobs.get(&modern_id).unwrap();
|
||||
jobs.cancel(&modern_id).unwrap();
|
||||
let mut request = saved.request;
|
||||
request.job_id = format!("legacy-journal:{}", uuid::Uuid::now_v7());
|
||||
let mut legacy = saved.domain_grant.unwrap();
|
||||
legacy.as_object_mut().unwrap().remove("writer_binding_id");
|
||||
let pending = jobs.request_granted(request, legacy.clone()).unwrap();
|
||||
assert_eq!(
|
||||
jobs.get(&pending.request.job_id).unwrap().domain_grant,
|
||||
Some(legacy)
|
||||
);
|
||||
jobs.start_with_model_client(&pending.request.job_id, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(20),
|
||||
jobs.wait(&pending.request.job_id),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let subject = fixture.connection.binding.subject_id.clone();
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&fixture.host().writer)
|
||||
.unwrap();
|
||||
assert!(fixture.host().journal_grant(&completed).is_err());
|
||||
jobs.shutdown().await.unwrap();
|
||||
fixture.connection.close().unwrap();
|
||||
let mut next = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&subject, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
assert!(
|
||||
next.host.journal_grant(&completed).is_err(),
|
||||
"missing legacy metadata cannot rebind its original Session to the next body"
|
||||
);
|
||||
next.close().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_create() {
|
||||
let mut fixture = Fixture::new("proposed, corrected, partially performed, then interrupted");
|
||||
@@ -1088,51 +1155,38 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopped_worker_removal_retains_originals_and_the_existing_durable_request() {
|
||||
async fn stopped_worker_removal_and_replacement_require_explicit_idle_handoff_even_after_recording()
|
||||
{
|
||||
let mut fixture = Fixture::new("Unrecorded stopped Worker");
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let binding = fixture.connection.binding.clone();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(matches!(
|
||||
workers.load(worker),
|
||||
Err(crate::store::StandaloneStoreError::WorkerNotFound(_))
|
||||
));
|
||||
let retained = retained_workers_root(&state).join(worker.to_string());
|
||||
assert!(workers.delete(worker).is_err());
|
||||
assert!(workers.load(worker).is_ok());
|
||||
assert!(
|
||||
retained
|
||||
.join("sessions")
|
||||
workers
|
||||
.sessions_dir(worker)
|
||||
.join(&fixture.context.session_id)
|
||||
.is_dir()
|
||||
);
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.is_err()
|
||||
);
|
||||
let mut reopened = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.connect(&binding.subject_id, worker)
|
||||
.unwrap();
|
||||
let jobs = crate::jobs::StandaloneJobs::open(&reopened.jobs_path, fixture._temp.path().into())
|
||||
.unwrap();
|
||||
reopened.host.bind_jobs(jobs.clone()).unwrap();
|
||||
assert_eq!(
|
||||
jobs.pending().unwrap().len(),
|
||||
1,
|
||||
"repeated deletion must not multiply requests"
|
||||
);
|
||||
reopened
|
||||
.host
|
||||
.reconcile_journal_session(&fixture.context.session_id)
|
||||
.unwrap();
|
||||
let (_, id) = reopened.host.next_journal_job().unwrap().unwrap();
|
||||
let snapshot = jobs.get(&id).unwrap();
|
||||
assert!(
|
||||
grant(
|
||||
reopened.host.clone(),
|
||||
&snapshot,
|
||||
Arc::new(TestFence(AtomicBool::new(true)))
|
||||
)
|
||||
.is_ok(),
|
||||
"retained originals remain readable through the real grant path"
|
||||
);
|
||||
assert!(
|
||||
retained.is_dir(),
|
||||
"unfinished generation cannot prune original material"
|
||||
);
|
||||
jobs.start_with_model_client(&id, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1146,31 +1200,23 @@ async fn stopped_worker_removal_retains_originals_and_the_existing_durable_reque
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!retained.exists(),
|
||||
"only reflected original material may be pruned"
|
||||
);
|
||||
let context = authority(&reopened.host, &fixture.context.session_id).unwrap();
|
||||
let journal = reopened
|
||||
.host
|
||||
.catalog
|
||||
.store
|
||||
.list_journals(&context, None, 1)
|
||||
.unwrap()
|
||||
.items
|
||||
.pop()
|
||||
.unwrap();
|
||||
let (_, open) = reopened
|
||||
.host
|
||||
.catalog
|
||||
.store
|
||||
.journal_resume_state(&context, &journal.id)
|
||||
.unwrap();
|
||||
assert!(
|
||||
open.unwrap().body.contains("Only part performed"),
|
||||
"Journal remains readable without the removed Worker/Session"
|
||||
reopened
|
||||
.host
|
||||
.session_fully_reflected(&fixture.context.session_id)
|
||||
.unwrap()
|
||||
);
|
||||
jobs.shutdown().await.unwrap();
|
||||
reopened.close().unwrap();
|
||||
assert!(
|
||||
workers.delete(worker).is_err(),
|
||||
"recording completion is not actual Controller Idle/release"
|
||||
);
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&binding.subject_id, protocol::WorkerId::now_v7())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1415,8 +1461,12 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
|
||||
.unwrap();
|
||||
let descriptor = old_reader.descriptors().next().unwrap().clone();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(workers.delete(worker).is_err());
|
||||
// Reproduce a T-734 archive already persisted before durable body-release bindings.
|
||||
// New ordinary deletion must not create this state, but its erasure contract remains.
|
||||
let retained = retained_workers_root(&state).join(worker.to_string());
|
||||
fs::create_dir_all(retained.parent().unwrap()).unwrap();
|
||||
fs::rename(workers.root().join(worker.to_string()), &retained).unwrap();
|
||||
let retained_sessions = retained.join("sessions");
|
||||
let retained_store = session_store::FsStore::new_with_retention(
|
||||
&retained_sessions,
|
||||
|
||||
@@ -183,7 +183,7 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
|
||||
retained_tokens: 0,
|
||||
..Default::default()
|
||||
});
|
||||
let host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
|
||||
let mut host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let session = host.record().active_session_id;
|
||||
@@ -422,6 +422,13 @@ async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_a
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!position.text.is_empty());
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(crate::StandaloneSubjectDisconnectError::Incomplete(message)) if message.contains("dispatcher") || message.contains("cleanup") => tokio::task::yield_now().await,
|
||||
Err(error) => panic!("recorded compaction/cancellation lineage must remain releasable while work is unfinished: {error}"),
|
||||
}
|
||||
}
|
||||
host.shutdown().await.unwrap();
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
//! Real standalone Subject/Controller/Journal Job/initial input connection. Only external
|
||||
//! model streams are scripted. The scenario checks persisted data and ordering, not whether
|
||||
//! a real model understands the correction or can autonomously continue the work.
|
||||
|
||||
use super::*;
|
||||
use crate::{StandaloneHost, StandaloneLaunchConfig, StandaloneSubjectDisconnectError};
|
||||
use agen::llm_client::types::Item;
|
||||
|
||||
fn launch(root: &Path, state: &Path, subject: &str, name: &str) -> crate::ResolvedStandaloneLaunch {
|
||||
StandaloneLaunchConfig::new(
|
||||
root,
|
||||
state,
|
||||
manifest::ProfileSelector::source_named(
|
||||
manifest::ProfileRegistrySource::Builtin,
|
||||
"standalone-subjektiv",
|
||||
),
|
||||
name,
|
||||
)
|
||||
.with_subject(subject.into())
|
||||
.resolve()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recorded_idle_body_handoff_preserves_open_after_source_deletion_and_seeds_next_body_once()
|
||||
{
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("continuity", Some("BEHAVIOR_ORIGINAL_UNCHANGED")).unwrap();
|
||||
let model = DialogAndJournal { journal: JournalScript { gate_first: true, ..Default::default() }, ..Default::default() };
|
||||
let mut first = StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "first"), model.clone()).await.unwrap();
|
||||
let old_worker = first.worker_id();
|
||||
let old_session = first.record().active_session_id;
|
||||
let old_writer = catalog.store.claim_journal_writer(&subject.id, &catalog.scope_id, &old_worker.to_string()).unwrap();
|
||||
submit_dialog(&first, "A was proposed, user corrected its constraint; B selected because A is unusable. One step performed; tests not performed; external outcome Unknown. Work is unfinished.").await;
|
||||
model.journal.entered.notified().await;
|
||||
assert!(matches!(first.disconnect_subject().await, Err(StandaloneSubjectDisconnectError::Incomplete(_))));
|
||||
assert!(catalog.store.journal_writer_active(&old_writer).unwrap());
|
||||
assert!(StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "premature"), DialogAndJournal::default()).await.is_err());
|
||||
let context = JournalHostContext::validated_session(&catalog.store, catalog.store.session_attribution(&old_session.to_string()).unwrap().unwrap()).unwrap();
|
||||
let id = catalog.store.journal_for_session(&context).unwrap().unwrap();
|
||||
let generation = catalog.store.active_journal_generation(&context, &id).unwrap().unwrap();
|
||||
model.journal.release.notify_one();
|
||||
first.jobs().wait(&generation.job.job_id).await.unwrap();
|
||||
// Observe the authoritative gate after cleanup; yielding is cooperative scheduling,
|
||||
// never elapsed time used as proof of recording completion.
|
||||
loop {
|
||||
match first.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(message)) if message.contains("dispatcher") || message.contains("cleanup") => tokio::task::yield_now().await,
|
||||
Err(error) => panic!("unexpected handoff blocker: {error}"),
|
||||
}
|
||||
}
|
||||
assert!(!catalog.store.journal_writer_active(&old_writer).unwrap());
|
||||
assert!(catalog.store.accept_journal_sources(&context, "late old activity", &[generation.selected_ranges[0].clone()]).is_err());
|
||||
first.shutdown().await.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
workers.delete(old_worker).unwrap();
|
||||
assert!(!workers.sessions_dir(old_worker).exists());
|
||||
let reader = JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap();
|
||||
let historical = catalog.store.journal_startup_snapshot(&reader, Some(&old_session.to_string())).unwrap();
|
||||
assert_eq!(historical.journals[0].entries[0].state, JournalEntryState::Open);
|
||||
assert!(historical.journals[0].entries[0].ongoing.contains("unknown"));
|
||||
assert!(catalog.store.list_memories(&subject.id).unwrap().is_empty());
|
||||
|
||||
let second_model = DialogAndJournal::default();
|
||||
let second = StandaloneHost::start_with_model_client(launch(temp.path(), &state, &subject.id, "second"), second_model.clone()).await.unwrap();
|
||||
submit_dialog(&second, "Continue B; describe what is still unperformed and unconfirmed.").await;
|
||||
let requests = second_model.dialog_requests.lock().unwrap();
|
||||
let initial = &requests[0];
|
||||
let context_index = initial.items.iter().position(|item| item.as_text().is_some_and(|text| text.contains("Proposed A; user corrected it to B"))).expect("saved Journal visible to first model request");
|
||||
let user_index = initial.items.iter().position(|item| matches!(item, Item::Message { role: agen::llm_client::types::Role::User, .. })).unwrap();
|
||||
assert!(context_index < user_index);
|
||||
assert_eq!(initial.items.iter().filter(|item| item.as_text().is_some_and(|text| text.contains("Proposed A; user corrected it to B"))).count(), 1);
|
||||
drop(requests);
|
||||
let session = second.record().active_session_id;
|
||||
let sessions = session_store::FsStore::new_with_retention(workers.sessions_dir(second.worker_id()), crate::store::session_retention(&state)).unwrap();
|
||||
let pointer = session_store::FsWorkerStore::new(workers.worker_metadata_dir(second.worker_id())).unwrap().read_by_name(&second.record().storage_key).unwrap().unwrap().active.unwrap();
|
||||
let entries = sessions.read_all(session, pointer.segment_id.unwrap()).unwrap();
|
||||
let journal_items = entries.iter().filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, session_store::SystemItem::JournalStartup { .. }))).count();
|
||||
assert_eq!(journal_items, 1);
|
||||
assert_eq!(catalog.store.subject(&subject.id).unwrap().unwrap().behavior_md, "BEHAVIOR_ORIGINAL_UNCHANGED");
|
||||
second.shutdown().await.unwrap();
|
||||
}).await.expect("bounded complete Subject handoff connection");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saved_public_history_without_admission_notification_blocks_release_until_actual_recording()
|
||||
{
|
||||
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("saved-before-notified", None).unwrap();
|
||||
let model = DialogAndJournal {
|
||||
journal: JournalScript {
|
||||
gate_first: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "gap"),
|
||||
model.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
let record = host.record().clone();
|
||||
let writer = catalog
|
||||
.store
|
||||
.claim_journal_writer(
|
||||
&subject.id,
|
||||
&catalog.scope_id,
|
||||
&host.worker_id().to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let sessions = session_store::FsStore::new_with_retention(
|
||||
workers.sessions_dir(host.worker_id()),
|
||||
crate::store::session_retention(&state),
|
||||
)
|
||||
.unwrap();
|
||||
// The production saved-history boundary, with its admission callback deliberately
|
||||
// omitted. The real Host release must discover these bytes, not trust Job status.
|
||||
sessions
|
||||
.append(
|
||||
record.active_session_id,
|
||||
record.active_segment_id.unwrap(),
|
||||
&session_store::LogEntry::AnnotatedUserInput {
|
||||
ts: 3,
|
||||
segments: vec![],
|
||||
extensions: vec![],
|
||||
history: vec![message(
|
||||
"missed-admission",
|
||||
"A corrected to B, partially performed, outcome Unknown",
|
||||
)],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(host.disconnect_subject().await.is_err());
|
||||
assert!(catalog.store.journal_writer_active(&writer).unwrap());
|
||||
model.journal.entered.notified().await;
|
||||
let context = JournalHostContext::validated_session(
|
||||
&catalog.store,
|
||||
catalog
|
||||
.store
|
||||
.session_attribution(&record.active_session_id.to_string())
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let id = catalog
|
||||
.store
|
||||
.journal_for_session(&context)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let generation = catalog
|
||||
.store
|
||||
.active_journal_generation(&context, &id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
model.journal.release.notify_one();
|
||||
host.jobs().wait(&generation.job.job_id).await.unwrap();
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => break,
|
||||
Err(StandaloneSubjectDisconnectError::Incomplete(message))
|
||||
if message.contains("dispatcher") || message.contains("cleanup") =>
|
||||
{
|
||||
tokio::task::yield_now().await
|
||||
}
|
||||
Err(error) => {
|
||||
panic!("unexpected release blocker after saved-history recording: {error}")
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(!catalog.store.journal_writer_active(&writer).unwrap());
|
||||
let snapshot = catalog
|
||||
.store
|
||||
.journal_startup_snapshot(
|
||||
&JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!snapshot.journals[0].current_position.is_empty());
|
||||
assert_eq!(
|
||||
snapshot.journals[0].entries[0].state,
|
||||
JournalEntryState::Open
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
})
|
||||
.await
|
||||
.expect("saved-history/notification-gap handoff is bounded");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_and_normal_deletion_cannot_release_subject_occupancy() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||
let subject = catalog.create("retained-owner", None).unwrap();
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "retained"),
|
||||
DialogAndJournal::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker = host.worker_id();
|
||||
host.shutdown().await.unwrap();
|
||||
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||
assert!(workers.delete(worker).is_err());
|
||||
assert!(
|
||||
StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "not-an-owner"),
|
||||
DialogAndJournal::default()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut restored =
|
||||
StandaloneHost::restore_with_model_client(state, worker, DialogAndJournal::default())
|
||||
.await
|
||||
.unwrap();
|
||||
restored.disconnect_subject().await.unwrap();
|
||||
restored.shutdown().await.unwrap();
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
//! Connected standalone Host -> Subjektiv Memory/Journal -> real FsStore coverage.
|
||||
//! Candidate and Journal decisions are explicit test inputs; no model, Runtime or live DB
|
||||
//! is used. The parent Journal fixture owns isolated temporary storage and binding cleanup.
|
||||
//! Selected by the ordinary standalone lib tests, with no additional feature or service.
|
||||
|
||||
use super::*;
|
||||
use memory::{
|
||||
backend::MemoryStageCandidateOperation,
|
||||
extract::{CandidateKind, ExtractedCandidate, StagingEvidence},
|
||||
schema::{EvidenceKind, EvidenceOriginKind, SourceEvidenceRef, SourceRef},
|
||||
};
|
||||
use subjektiv::{
|
||||
CandidateDecision, CandidateDecisionRequest, MemoryChangeTarget, MemoryDraft, MemoryRecord,
|
||||
StagingResolution, SubjectStagingRecord,
|
||||
journal::memory_links::{
|
||||
JournalTargetAvailability, MemoryJournalLinks, MemoryJournalRequest, MemoryJournalResponse,
|
||||
MemoryJournalSelector, SourceConnection, UnconnectedReason,
|
||||
},
|
||||
};
|
||||
|
||||
fn adopt_original(fixture: &Fixture) -> (MemoryRecord, SubjectStagingRecord, StagingResolution) {
|
||||
let evidence_index = session_store::public_index::read_fs_session_evidence_index(
|
||||
fixture.sessions.root_dir(),
|
||||
fixture.context.session_id.parse().unwrap(),
|
||||
session_store::SessionPublicIndexLimits {
|
||||
max_bytes: 8 * 1024 * 1024,
|
||||
max_entries: 20_000,
|
||||
max_segments: 64,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let range = *evidence_index
|
||||
.ranges
|
||||
.get(&(fixture.segment.to_string(), "Efirst".into()))
|
||||
.unwrap()
|
||||
.first()
|
||||
.unwrap();
|
||||
let staged = fixture
|
||||
.host()
|
||||
.stage_candidate(
|
||||
&fixture.context,
|
||||
MemoryStageCandidateOperation {
|
||||
source: SourceRef {
|
||||
segment_id: fixture.segment.to_string(),
|
||||
range,
|
||||
},
|
||||
extract_run_id: "explicit-test-extraction-input".into(),
|
||||
candidate: ExtractedCandidate {
|
||||
kind: CandidateKind::Preference,
|
||||
claim: "Keep correction history".into(),
|
||||
why_useful: "Preserves the reason for later decisions".into(),
|
||||
staleness: None,
|
||||
evidence_ids: vec!["Efirst".into()],
|
||||
},
|
||||
evidence: vec![StagingEvidence {
|
||||
id: "Efirst".into(),
|
||||
kind: EvidenceKind::new(EvidenceKind::MESSAGE),
|
||||
entry_range: Some(range),
|
||||
origin: None,
|
||||
excerpt: None,
|
||||
summary: None,
|
||||
}],
|
||||
source_refs: vec![SourceEvidenceRef {
|
||||
session_id: Some(fixture.context.session_id.clone()),
|
||||
segment_id: Some(fixture.segment.to_string()),
|
||||
entry_range: Some(range),
|
||||
evidence_id: Some("Efirst".into()),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let store = &fixture.host().catalog.store;
|
||||
let subject = &fixture.connection.binding.subject_id;
|
||||
let candidate = store
|
||||
.staging_candidate(subject, &staged.staging_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let receipt = store
|
||||
.decide_candidate(
|
||||
subject,
|
||||
CandidateDecisionRequest {
|
||||
request_id: "explicit-test-adoption-input".into(),
|
||||
candidate_id: staged.staging_id,
|
||||
reason: "Adopt the committed original evidence, not the Journal summary".into(),
|
||||
decision: CandidateDecision::Apply {
|
||||
target: MemoryChangeTarget::Create,
|
||||
draft: MemoryDraft::active(
|
||||
CandidateKind::Preference,
|
||||
"Keep correction history",
|
||||
"Preserve earlier decisions and their corrections",
|
||||
"Preserves the reason for later decisions",
|
||||
"Adopt original HumanInput",
|
||||
),
|
||||
},
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
(receipt.memory.unwrap(), candidate, receipt.resolution)
|
||||
}
|
||||
|
||||
fn create_entry(adapter: &LocalJournalHost, state: JournalEntryState) -> JournalReceipt {
|
||||
save(
|
||||
adapter,
|
||||
"record-original-context",
|
||||
&adapter.generation.initial_journal.state_token,
|
||||
JournalAction::Create {
|
||||
title: "A proposal was corrected".into(),
|
||||
body: "Proposal A changed to B; only part was performed, outcome unconfirmed.".into(),
|
||||
ongoing: if state == JournalEntryState::Open {
|
||||
"Remaining work is not complete".into()
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
state,
|
||||
source_end: (state == JournalEntryState::Fixed).then(|| {
|
||||
adapter
|
||||
.generation
|
||||
.selected_ranges
|
||||
.last()
|
||||
.unwrap()
|
||||
.source
|
||||
.pointer
|
||||
.clone()
|
||||
}),
|
||||
correction_refs: vec![],
|
||||
},
|
||||
adapter.generation.selected_ranges.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
fn assert_originals_unchanged(
|
||||
fixture: &Fixture,
|
||||
memory: &MemoryRecord,
|
||||
candidate: &SubjectStagingRecord,
|
||||
resolution: &StagingResolution,
|
||||
) {
|
||||
let catalog = StandaloneSubjects::open_existing(&fixture.host().catalog.state_root).unwrap();
|
||||
let store = &catalog.store;
|
||||
let subject = &fixture.connection.binding.subject_id;
|
||||
assert_eq!(
|
||||
serde_json::to_value(store.memory(subject, &memory.id).unwrap().unwrap()).unwrap(),
|
||||
serde_json::to_value(memory).unwrap(),
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(
|
||||
store
|
||||
.staging_candidate(subject, &candidate.id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
)
|
||||
.unwrap(),
|
||||
serde_json::to_value(candidate).unwrap(),
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(
|
||||
store
|
||||
.staging_resolution(subject, &candidate.id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
)
|
||||
.unwrap(),
|
||||
serde_json::to_value(resolution).unwrap(),
|
||||
);
|
||||
assert_eq!(store.list_memories(subject).unwrap().len(), 1);
|
||||
}
|
||||
|
||||
fn selector(memory: &MemoryRecord, candidate: &SubjectStagingRecord) -> MemoryJournalSelector {
|
||||
MemoryJournalSelector {
|
||||
memory_id: memory.id.clone(),
|
||||
change_id: memory.change_id.clone(),
|
||||
candidate_id: candidate.id.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn links(
|
||||
fixture: &Fixture,
|
||||
selector: &MemoryJournalSelector,
|
||||
reconcile: bool,
|
||||
) -> MemoryJournalLinks {
|
||||
let request = if reconcile {
|
||||
MemoryJournalRequest::Reconcile {
|
||||
selector: selector.clone(),
|
||||
}
|
||||
} else {
|
||||
MemoryJournalRequest::Links {
|
||||
selector: selector.clone(),
|
||||
}
|
||||
};
|
||||
let MemoryJournalResponse::Links(links) = fixture
|
||||
.host()
|
||||
.memory_journal(&fixture.context, request)
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected Memory Journal links")
|
||||
};
|
||||
links
|
||||
}
|
||||
|
||||
fn read_request(
|
||||
selector: &MemoryJournalSelector,
|
||||
journal: &str,
|
||||
entry: &str,
|
||||
) -> MemoryJournalRequest {
|
||||
MemoryJournalRequest::Read {
|
||||
selector: selector.clone(),
|
||||
source_index: 0,
|
||||
journal_id: journal.into(),
|
||||
entry_id: entry.into(),
|
||||
expected_token: None,
|
||||
byte_offset: 0,
|
||||
max_bytes: MAX_JOURNAL_READ_BYTES,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_link(links: &MemoryJournalLinks, journal: &str, entry: &str, reconsideration: bool) {
|
||||
assert_eq!(links.sources.len(), 1);
|
||||
assert_eq!(links.sources[0].source_index, 0);
|
||||
assert_eq!(
|
||||
links.sources[0].connection,
|
||||
SourceConnection::Linked {
|
||||
journal_id: journal.into(),
|
||||
entry_id: entry.into(),
|
||||
target: JournalTargetAvailability::Available,
|
||||
reconsideration,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn saved_link_reads_after_original_worker_and_session_deletion_from_replacement_host() {
|
||||
let mut fixture = Fixture::new("Please preserve the reasons for my corrected decisions");
|
||||
let (memory, candidate, resolution) = adopt_original(&fixture);
|
||||
let selector = selector(&memory, &candidate);
|
||||
assert_eq!(
|
||||
candidate.evidence[0].origin.as_ref().unwrap().kind,
|
||||
EvidenceOriginKind::HumanInput
|
||||
);
|
||||
assert_eq!(
|
||||
candidate.source_refs[0].origin,
|
||||
candidate.evidence[0].origin
|
||||
);
|
||||
assert!(
|
||||
candidate.evidence[0]
|
||||
.excerpt
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains("corrected decisions")
|
||||
);
|
||||
let (adapter, _, _) = fixture.adapter();
|
||||
let recorded = create_entry(&adapter, JournalEntryState::Fixed);
|
||||
let entry = recorded.entry_id.clone().unwrap();
|
||||
let journal = adapter.grant.journal_id.clone();
|
||||
let connected = links(&fixture, &selector, true);
|
||||
assert_link(&connected, &journal, &entry, false);
|
||||
let request = read_request(&selector, &journal, &entry);
|
||||
let before = fixture
|
||||
.host()
|
||||
.memory_journal(&fixture.context, request.clone())
|
||||
.unwrap();
|
||||
let positioned = save(
|
||||
&adapter,
|
||||
"position-original",
|
||||
&recorded.state_token,
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "Recorded correction; remaining outcome unconfirmed".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
save(
|
||||
&adapter,
|
||||
"checkpoint-original",
|
||||
&positioned.state_token,
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec![recorded.request_id],
|
||||
},
|
||||
adapter.generation.selected_ranges.clone(),
|
||||
);
|
||||
adapter.finish().unwrap();
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.close_journal_generation(
|
||||
&adapter.context,
|
||||
&adapter.generation.id,
|
||||
&adapter.generation.job,
|
||||
)
|
||||
.unwrap();
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let subject = fixture.connection.binding.subject_id.clone();
|
||||
let original_session = fixture.context.session_id.clone();
|
||||
// This Host/storage suite has no Controller. Complete the explicit persisted
|
||||
// writer-release boundary before deletion; the sibling handoff suite exercises
|
||||
// the production Idle reservation + Host lifecycle that authorizes that release.
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.release_journal_writer(&fixture.host().writer)
|
||||
.unwrap();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
workers.delete(worker).unwrap();
|
||||
assert!(matches!(
|
||||
workers.load(worker),
|
||||
Err(crate::store::StandaloneStoreError::WorkerNotFound(_))
|
||||
));
|
||||
assert!(!fixture.sessions.root_dir().join(&original_session).exists());
|
||||
assert!(
|
||||
!retained_workers_root(&state)
|
||||
.join(worker.to_string())
|
||||
.exists()
|
||||
);
|
||||
|
||||
let replacement_worker = protocol::WorkerId::now_v7();
|
||||
let mut replacement = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.connect(&subject, replacement_worker)
|
||||
.unwrap();
|
||||
let replacement_context = SubjektivHostContext {
|
||||
worker_id: replacement_worker.to_string(),
|
||||
session_id: session_store::SessionId::now_v7().to_string(),
|
||||
};
|
||||
let MemoryJournalResponse::Links(saved) = replacement
|
||||
.host
|
||||
.memory_journal(
|
||||
&replacement_context,
|
||||
MemoryJournalRequest::Links {
|
||||
selector: selector.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected saved links")
|
||||
};
|
||||
assert_link(&saved, &journal, &entry, false);
|
||||
let after = replacement
|
||||
.host
|
||||
.memory_journal(&replacement_context, request)
|
||||
.unwrap();
|
||||
let MemoryJournalResponse::Read(read) = &after else {
|
||||
panic!("expected saved Journal text")
|
||||
};
|
||||
assert!(
|
||||
read.page
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.text
|
||||
.contains("outcome unconfirmed")
|
||||
);
|
||||
assert_eq!(
|
||||
read.source.original_source,
|
||||
subjektiv::journal::memory_links::OriginalSourceAvailability::Missing
|
||||
);
|
||||
// Reading the saved edge does not depend on a live Session fetch or a replacement Job.
|
||||
let MemoryJournalResponse::Read(before) = before else {
|
||||
panic!("expected Journal text")
|
||||
};
|
||||
assert_eq!(before.page, read.page);
|
||||
assert!(
|
||||
replacement.host.jobs().is_err(),
|
||||
"the reader has no model/Job executor"
|
||||
);
|
||||
assert_originals_unchanged(&fixture, &memory, &candidate, &resolution);
|
||||
replacement.close().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn late_journal_fix_connects_existing_memory_without_using_checkpoint_as_event_end() {
|
||||
let fixture = Fixture::new("Keep the reasons behind corrected decisions");
|
||||
let (memory, candidate, resolution) = adopt_original(&fixture);
|
||||
let selector = selector(&memory, &candidate);
|
||||
assert_eq!(
|
||||
links(&fixture, &selector, false).sources[0].connection,
|
||||
SourceConnection::Unconnected {
|
||||
reason: UnconnectedReason::NotReconciled
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
links(&fixture, &selector, true).sources[0].connection,
|
||||
SourceConnection::Unconnected {
|
||||
reason: UnconnectedReason::SourceNotAdmitted
|
||||
}
|
||||
);
|
||||
let (adapter, _, _) = fixture.adapter();
|
||||
let recorded = create_entry(&adapter, JournalEntryState::Open);
|
||||
let entry = recorded.entry_id.clone().unwrap();
|
||||
let positioned = save(
|
||||
&adapter,
|
||||
"position-open",
|
||||
&recorded.state_token,
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "Work remains open; no fixed endpoint yet".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
// Checkpoint a proper prefix, leaving source bytes for the later Fix.
|
||||
// A fully checkpointed generation cannot admit another activity write.
|
||||
let mut reflected = adapter.generation.selected_ranges.clone();
|
||||
reflected.last_mut().unwrap().bytes.end -= 1;
|
||||
let checkpoint = save(
|
||||
&adapter,
|
||||
"checkpoint-open",
|
||||
&positioned.state_token,
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec![recorded.request_id],
|
||||
},
|
||||
reflected.clone(),
|
||||
);
|
||||
let unconnected = links(&fixture, &selector, true);
|
||||
assert_eq!(
|
||||
unconnected.sources[0].connection,
|
||||
SourceConnection::Unconnected {
|
||||
reason: UnconnectedReason::JournalNotFixed
|
||||
}
|
||||
);
|
||||
assert_eq!(links(&fixture, &selector, true), unconnected);
|
||||
let state = adapter.state().unwrap();
|
||||
let open = state.current_open.unwrap();
|
||||
assert_eq!(open.id, entry);
|
||||
assert_eq!(open.state, JournalEntryState::Open);
|
||||
assert!(open.source_end.is_none());
|
||||
let last = reflected.last().unwrap();
|
||||
assert_eq!(state.current_journal.process_order, last.source.order);
|
||||
assert_eq!(state.current_journal.process_byte, last.bytes.end);
|
||||
save(
|
||||
&adapter,
|
||||
"fix-later",
|
||||
&checkpoint.state_token,
|
||||
JournalAction::Fix {
|
||||
entry_id: entry.clone(),
|
||||
source_end: adapter
|
||||
.generation
|
||||
.selected_ranges
|
||||
.last()
|
||||
.unwrap()
|
||||
.source
|
||||
.pointer
|
||||
.clone(),
|
||||
},
|
||||
adapter.generation.selected_ranges.clone(),
|
||||
);
|
||||
// A plain saved-state read must not silently perform reconciliation.
|
||||
assert_eq!(links(&fixture, &selector, false), unconnected);
|
||||
let connected = links(&fixture, &selector, true);
|
||||
assert_link(&connected, &adapter.grant.journal_id, &entry, false);
|
||||
assert_eq!(links(&fixture, &selector, true), connected);
|
||||
assert_eq!(links(&fixture, &selector, false), connected);
|
||||
assert_originals_unchanged(&fixture, &memory, &candidate, &resolution);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_rejects_foreign_subject_worker_context_and_closed_connection_for_all_link_operations() {
|
||||
let mut fixture = Fixture::new("Keep correction history");
|
||||
let (memory, candidate, resolution) = adopt_original(&fixture);
|
||||
let selector = selector(&memory, &candidate);
|
||||
let (adapter, _, _) = fixture.adapter();
|
||||
let recorded = create_entry(&adapter, JournalEntryState::Fixed);
|
||||
let journal = adapter.grant.journal_id.clone();
|
||||
let entry = recorded.entry_id.unwrap();
|
||||
assert_link(&links(&fixture, &selector, true), &journal, &entry, false);
|
||||
let requests = [
|
||||
MemoryJournalRequest::Reconcile {
|
||||
selector: selector.clone(),
|
||||
},
|
||||
MemoryJournalRequest::Links {
|
||||
selector: selector.clone(),
|
||||
},
|
||||
read_request(&selector, &journal, &entry),
|
||||
MemoryJournalRequest::Related {
|
||||
journal_id: journal,
|
||||
entry_id: entry,
|
||||
after: None,
|
||||
limit: 32,
|
||||
},
|
||||
];
|
||||
let catalog = StandaloneSubjects::open_existing(&fixture.host().catalog.state_root).unwrap();
|
||||
let foreign_subject = catalog.create("different-reader", None).unwrap();
|
||||
let foreign_worker = protocol::WorkerId::now_v7();
|
||||
let mut foreign = catalog
|
||||
.connect(&foreign_subject.id, foreign_worker)
|
||||
.unwrap();
|
||||
let foreign_context = SubjektivHostContext {
|
||||
worker_id: foreign_worker.to_string(),
|
||||
session_id: session_store::SessionId::now_v7().to_string(),
|
||||
};
|
||||
let wrong_worker = SubjektivHostContext {
|
||||
worker_id: foreign_worker.to_string(),
|
||||
..fixture.context.clone()
|
||||
};
|
||||
for request in &requests {
|
||||
assert!(
|
||||
foreign
|
||||
.host
|
||||
.memory_journal(&foreign_context, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.host()
|
||||
.memory_journal(&wrong_worker, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
foreign.close().unwrap();
|
||||
fixture.connection.close().unwrap();
|
||||
for request in requests {
|
||||
assert!(
|
||||
fixture
|
||||
.host()
|
||||
.memory_journal(&fixture.context, request)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
assert_originals_unchanged(&fixture, &memory, &candidate, &resolution);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_correction_finds_related_memory_without_revising_or_retargeting_its_evidence() {
|
||||
let fixture = Fixture::new("Keep correction history");
|
||||
let (memory, candidate, resolution) = adopt_original(&fixture);
|
||||
let selector = selector(&memory, &candidate);
|
||||
fixture.append("correction", "The earlier result was not confirmed");
|
||||
let (adapter, _, _) = fixture.adapter();
|
||||
let ranges = &adapter.generation.selected_ranges;
|
||||
assert_eq!(ranges.len(), 2);
|
||||
let original = save(
|
||||
&adapter,
|
||||
"original-event",
|
||||
&adapter.generation.initial_journal.state_token,
|
||||
JournalAction::Create {
|
||||
title: "Original context".into(),
|
||||
body: "A change was proposed; execution remained uncertain".into(),
|
||||
ongoing: String::new(),
|
||||
state: JournalEntryState::Fixed,
|
||||
source_end: Some(ranges[0].source.pointer.clone()),
|
||||
correction_refs: vec![],
|
||||
},
|
||||
vec![ranges[0].clone()],
|
||||
);
|
||||
let entry = original.entry_id.unwrap();
|
||||
let journal = &adapter.grant.journal_id;
|
||||
assert_link(&links(&fixture, &selector, true), journal, &entry, false);
|
||||
let corrected = save(
|
||||
&adapter,
|
||||
"correction-event",
|
||||
&original.state_token,
|
||||
JournalAction::Create {
|
||||
title: "Correction".into(),
|
||||
body: "The earlier result was unconfirmed; reconsider its associated Memory".into(),
|
||||
ongoing: String::new(),
|
||||
state: JournalEntryState::Fixed,
|
||||
source_end: Some(ranges[1].source.pointer.clone()),
|
||||
correction_refs: vec![entry.clone()],
|
||||
},
|
||||
vec![ranges[1].clone()],
|
||||
);
|
||||
assert_link(&links(&fixture, &selector, false), journal, &entry, true);
|
||||
let correction_entry = corrected.entry_id.unwrap();
|
||||
for target in [&entry, &correction_entry] {
|
||||
let MemoryJournalResponse::Related(related) = fixture
|
||||
.host()
|
||||
.memory_journal(
|
||||
&fixture.context,
|
||||
MemoryJournalRequest::Related {
|
||||
journal_id: journal.clone(),
|
||||
entry_id: target.clone(),
|
||||
after: None,
|
||||
limit: 32,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected related Memory")
|
||||
};
|
||||
assert_eq!(related.items.len(), 1);
|
||||
assert_eq!(related.items[0].selector, selector);
|
||||
assert_eq!(related.items[0].source_index, 0);
|
||||
assert!(related.items[0].reconsideration);
|
||||
assert!(related.next.is_none());
|
||||
}
|
||||
let MemoryJournalResponse::Read(read) = fixture
|
||||
.host()
|
||||
.memory_journal(&fixture.context, read_request(&selector, journal, &entry))
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected original context")
|
||||
};
|
||||
assert!(
|
||||
read.page
|
||||
.unwrap()
|
||||
.text
|
||||
.contains("execution remained uncertain")
|
||||
);
|
||||
assert_originals_unchanged(&fixture, &memory, &candidate, &resolution);
|
||||
}
|
||||
@@ -37,9 +37,8 @@ fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
|
||||
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
|
||||
let a = catalog.create("a", None).unwrap();
|
||||
let b = catalog.create("b", None).unwrap();
|
||||
let mut first = catalog
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
let first_id = protocol::WorkerId::now_v7();
|
||||
let mut first = catalog.connect(&a.id, first_id).unwrap();
|
||||
assert!(matches!(
|
||||
StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
@@ -51,11 +50,18 @@ fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
|
||||
.connect(&b.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
first.close().unwrap();
|
||||
let mut replacement = StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
assert!(
|
||||
StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.is_err(),
|
||||
"kernel lease release is not durable body release"
|
||||
);
|
||||
let mut resumed = StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.connect(&a.id, first_id)
|
||||
.unwrap();
|
||||
replacement.close().unwrap();
|
||||
resumed.close().unwrap();
|
||||
second.close().unwrap();
|
||||
}
|
||||
|
||||
|
||||
@@ -178,6 +178,58 @@ pub(crate) fn authorized_sessions_path(
|
||||
.map_err(|error: Failure| SubjektivHostError::Unavailable(error.message))
|
||||
}
|
||||
|
||||
/// Checks only saved ownership and filesystem metadata, never Session bytes or a Worker restore.
|
||||
/// Inaccessible/corrupt storage stays unknown rather than being reported as absent.
|
||||
pub(crate) fn original_session_presence(
|
||||
state_root: &Path,
|
||||
scope_id: &str,
|
||||
subject_id: &str,
|
||||
store: &SubjektivStore,
|
||||
session_id: &str,
|
||||
) -> Option<bool> {
|
||||
let attribution = store.session_attribution(session_id).ok()??;
|
||||
if store.scope_id() != scope_id
|
||||
|| attribution.subject_id != subject_id
|
||||
|| attribution.runtime_id != scope_id
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let worker: WorkerId = attribution.worker_id.parse().ok()?;
|
||||
let session: SessionId = session_id.parse().ok()?;
|
||||
let active = StandaloneWorkerStore::open_existing(state_root).ok()?;
|
||||
let (workers, record) = match active.load(worker) {
|
||||
Ok(record) => (active, record),
|
||||
Err(StandaloneStoreError::WorkerNotFound(_)) => {
|
||||
let root = state_root.join("subjektiv").join("retained-workers");
|
||||
match std::fs::metadata(&root) {
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Some(false),
|
||||
Err(_) => return None,
|
||||
Ok(_) => {}
|
||||
}
|
||||
let retained = StandaloneWorkerStore::open_existing(root).ok()?;
|
||||
match retained.load(worker) {
|
||||
Ok(record) => (retained, record),
|
||||
Err(StandaloneStoreError::WorkerNotFound(_)) => return Some(false),
|
||||
Err(_) => return None,
|
||||
}
|
||||
}
|
||||
Err(_) => return None,
|
||||
};
|
||||
if record.worker_id != worker
|
||||
|| !record
|
||||
.subject
|
||||
.as_ref()
|
||||
.is_some_and(|binding| binding.scope_id == scope_id && binding.subject_id == subject_id)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
match std::fs::metadata(workers.sessions_dir(worker).join(session.to_string())) {
|
||||
Ok(metadata) if metadata.is_dir() => Some(true),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Some(false),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn authorized_index_with_reader(
|
||||
state_root: &Path,
|
||||
scope_id: &str,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! T-709: real standalone Hosts and test-owned processes, with scripted models.
|
||||
//! These tests do not create a Workspace or contact a model provider.
|
||||
//! Integration of real standalone Subject lifecycle, dialog, Journal and Memory services.
|
||||
//! Only model streams are scripted; temporary storage and test-owned subprocesses isolate
|
||||
//! persistence and kernel leases. Run with `cargo test -p standalone --test subjektiv`.
|
||||
//! No Workspace or live model provider is used; this is not a continuity meaning evaluation.
|
||||
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::io::{Read, Write};
|
||||
@@ -354,17 +356,16 @@ impl LlmClient for ScriptedClient {
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
// This suite observes Memory and dialog requests. Journal's independent model
|
||||
// boundary fails explicitly rather than consuming their script or using live transport;
|
||||
// the Journal adapter suite exercises its successful sequential tool execution.
|
||||
// Journal is a separate scripted model boundary. Successful production recording
|
||||
// is required before these Memory scenarios may hand the Subject to a new body.
|
||||
if request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalState")
|
||||
{
|
||||
return Err(ClientError::Config(
|
||||
"Journal model is outside this Memory suite".into(),
|
||||
));
|
||||
return Ok(Box::pin(stream::iter(
|
||||
journal_response(&request).into_iter().map(Ok),
|
||||
)));
|
||||
}
|
||||
// Lifecycle-triggered consolidation uses a separate Internal Worker. It
|
||||
// must not consume the dialog script or silently become a live request.
|
||||
@@ -423,6 +424,134 @@ impl LlmClient for ScriptedClient {
|
||||
}
|
||||
}
|
||||
|
||||
// The outer model script is not a continuity meaning evaluation. Persistence, complete
|
||||
// source delivery and the handoff gate remain the production implementations.
|
||||
fn journal_response(request: &Request) -> Vec<LlmEvent> {
|
||||
let result = |id: &str| -> Option<Value> {
|
||||
request.items.iter().find_map(|item| match item {
|
||||
Item::ToolResult {
|
||||
call_id,
|
||||
content,
|
||||
is_error,
|
||||
summary,
|
||||
..
|
||||
} if call_id == id => {
|
||||
assert!(!is_error, "{id}: {summary}; {content:?}");
|
||||
Some(
|
||||
content
|
||||
.as_deref()
|
||||
.map(|v| serde_json::from_str(v).unwrap())
|
||||
.unwrap_or(Value::Null),
|
||||
)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
if result("journal-submit").is_some() {
|
||||
return prose();
|
||||
}
|
||||
if result("journal-checkpoint").is_some() {
|
||||
return tool("journal-submit", "SubmitJobResult", json!({"result":{}}));
|
||||
}
|
||||
let Some(state) = result("journal-state") else {
|
||||
return tool("journal-state", "ReadJournalState", json!({}));
|
||||
};
|
||||
let state = &state["value"];
|
||||
let generation = state["generation_id"].as_str().unwrap();
|
||||
if let Some(position) = result("journal-position") {
|
||||
return tool(
|
||||
"journal-checkpoint",
|
||||
"CheckpointJournal",
|
||||
json!({
|
||||
"request_id":format!("checkpoint:{generation}"), "expected_state":position["value"]["state_token"],
|
||||
"action":{"action":"checkpoint","record_operation_ids":[format!("record:{generation}")]},
|
||||
"covered_ranges":state["selected_ranges"]
|
||||
}),
|
||||
);
|
||||
}
|
||||
if let Some(record) = result("journal-record") {
|
||||
return tool(
|
||||
"journal-position",
|
||||
"UpdateJournalPosition",
|
||||
json!({
|
||||
"request_id":format!("position:{generation}"), "expected_state":record["value"]["state_token"],
|
||||
"action":{"action":"update_position","current_position":"Memory scenario recorded; represented work may remain unfinished"}, "covered_ranges":[]
|
||||
}),
|
||||
);
|
||||
}
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("journal-source-{index}-{offset}");
|
||||
match result(&id) {
|
||||
Some(page) => match page["value"]["next_byte"].as_u64() {
|
||||
Some(next) => {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["value"]["content_token"].clone();
|
||||
}
|
||||
None => break,
|
||||
},
|
||||
None => {
|
||||
return tool(
|
||||
&id,
|
||||
"ReadJournalText",
|
||||
json!({
|
||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
||||
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let (name, action) = if state["current_open"].is_null() {
|
||||
(
|
||||
"CreateJournalEntry",
|
||||
json!({"action":"create","title":"Memory conversation","body":"Conversation recorded by the scripted model fixture","ongoing":"Unfinished work is not declared complete","state":"open","source_end":null,"correction_refs":[]}),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"RewriteJournalEntry",
|
||||
json!({"action":"rewrite","entry_id":state["current_open"]["id"],"title":"Memory conversation","body":"Further conversation recorded by the scripted model fixture","ongoing":"Unfinished work is not declared complete"}),
|
||||
)
|
||||
};
|
||||
tool(
|
||||
"journal-record",
|
||||
name,
|
||||
json!({
|
||||
"request_id":format!("record:{generation}"), "expected_state":state["current_journal"]["state_token"],
|
||||
"action":action,"covered_ranges":state["selected_ranges"]
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
async fn disconnect_recorded(host: &mut StandaloneHost) {
|
||||
let mut last_error = None;
|
||||
let completed = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
match host.disconnect_subject().await {
|
||||
Ok(()) => return,
|
||||
Err(error) => {
|
||||
last_error = Some(error.to_string());
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
completed.is_ok(),
|
||||
"actual Idle/Journal completion is required before Memory scenario handoff: {last_error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
fn prose() -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
@@ -762,7 +891,7 @@ async fn explicit_remember_is_committed_staging_and_sessions_follow_subject_acro
|
||||
tool("query-restored", "SubjektivMemoryQuery", json!({})),
|
||||
prose(),
|
||||
]);
|
||||
let restored =
|
||||
let mut restored =
|
||||
StandaloneHost::restore_with_model_client(state.clone(), worker_id, restored_model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -773,6 +902,7 @@ async fn explicit_remember_is_committed_staging_and_sessions_follow_subject_acro
|
||||
assert!(results["recall"].to_string().contains(claim));
|
||||
assert_eq!(results["query-restored"]["items"], json!([]));
|
||||
assert!(format!("{:?}", restored_model.requests()[0].items).contains("Remember this lesson"));
|
||||
disconnect_recorded(&mut restored).await;
|
||||
restored.shutdown().await.unwrap();
|
||||
|
||||
let second_model = ScriptedClient::new(vec![
|
||||
@@ -992,7 +1122,7 @@ async fn kernel_subject_lease_excludes_other_processes_and_recovers_after_abnorm
|
||||
let independent = catalog.create("independent", None).unwrap();
|
||||
drop(catalog);
|
||||
let ready = temp.path().join("ready");
|
||||
let parent = StandaloneHost::start_with_model_client(
|
||||
let mut parent = StandaloneHost::start_with_model_client(
|
||||
launch(&parent_cwd, &state, &subject.id, "parent-owner"),
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
@@ -1014,6 +1144,7 @@ async fn kernel_subject_lease_excludes_other_processes_and_recovers_after_abnorm
|
||||
.await
|
||||
.unwrap();
|
||||
independent_host.shutdown().await.unwrap();
|
||||
disconnect_recorded(&mut parent).await;
|
||||
parent.shutdown().await.unwrap();
|
||||
|
||||
let mut owner = OwnedChild(
|
||||
@@ -1082,7 +1213,7 @@ async fn automatic_generic_consolidation_applies_candidate_publishes_surface_and
|
||||
drop(catalog);
|
||||
let claim = "Validate the immutable result before reporting success";
|
||||
let model = ScriptedClient::consolidating(remember_script(claim), false);
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "consolidation-body"),
|
||||
model.clone(),
|
||||
)
|
||||
@@ -1157,6 +1288,7 @@ async fn automatic_generic_consolidation_applies_candidate_publishes_surface_and
|
||||
.exists(),
|
||||
"selected Subject owns the shared generic Job ledger"
|
||||
);
|
||||
disconnect_recorded(&mut host).await;
|
||||
host.shutdown().await.unwrap();
|
||||
let memory = inspect_store(&state, |store| {
|
||||
assert!(
|
||||
@@ -1236,7 +1368,7 @@ async fn transport_loss_after_candidate_apply_preserves_memory_and_later_job_onl
|
||||
drop(catalog);
|
||||
let claim = "Candidate receipts survive transport failure";
|
||||
let losing = ScriptedClient::consolidating(remember_script(claim), true);
|
||||
let host = StandaloneHost::start_with_model_client(
|
||||
let mut host = StandaloneHost::start_with_model_client(
|
||||
launch(temp.path(), &state, &subject.id, "losing-body"),
|
||||
losing.clone(),
|
||||
)
|
||||
@@ -1261,6 +1393,7 @@ async fn transport_loss_after_candidate_apply_preserves_memory_and_later_job_onl
|
||||
failed.result.is_none(),
|
||||
"partial apply is not an accepted Job result"
|
||||
);
|
||||
disconnect_recorded(&mut host).await;
|
||||
host.shutdown().await.unwrap();
|
||||
let original = inspect_store(&state, |store| {
|
||||
assert!(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
Shared Host domain implementation for per-Subject Memory, change history, candidate dispositions, derived surfaces, and Session activity Journals.
|
||||
`feature-storage` manages physical SQLite storage; the calling Host handles authentication and execution authority.
|
||||
Journal storage does not generate prose, launch Jobs, or fetch Session history.
|
||||
Hosts retain body ownership across inactivity, reserve actual execution quiescence, and reconcile final committed history before releasing a Journal writer lifetime. [`journal::binding`] fences new mutations inside storage transactions; historical attribution and exact receipt recovery cannot renew a released lifetime. [`journal::startup`] selects bounded reference context independently of source retention and long-term Memory.
|
||||
|
||||
## Main entry points
|
||||
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
//! Durable Journal writer lifetime, issued only from the Host's body binding. Kernel leases
|
||||
//! and Runtime liveness do not end ownership. The immutable binding ID fences retained contexts
|
||||
//! even when the same Worker later reconnects. Hosts serialize release with body lifecycle and
|
||||
//! reconcile final committed history before calling release; this store checks saved effects.
|
||||
|
||||
use super::*;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JournalWriterBinding {
|
||||
pub subject_id: String,
|
||||
pub runtime_id: String,
|
||||
pub worker_id: String,
|
||||
pub binding_id: String,
|
||||
}
|
||||
|
||||
pub(crate) fn migrate(tx: &Transaction<'_>) -> Result<(), FeatureStorageError> {
|
||||
tx.execute_batch("CREATE TABLE journal_writer_bindings (
|
||||
binding_id TEXT PRIMARY KEY,
|
||||
subject_id TEXT NOT NULL REFERENCES subjects(subject_id) ON DELETE RESTRICT,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
active INTEGER NOT NULL CHECK(active IN (0,1))
|
||||
);
|
||||
CREATE UNIQUE INDEX journal_one_body_binding ON journal_writer_bindings(subject_id) WHERE active=1;
|
||||
CREATE TABLE journal_session_bindings (
|
||||
session_id TEXT PRIMARY KEY REFERENCES subject_session_attributions(session_id) ON DELETE RESTRICT,
|
||||
binding_id TEXT NOT NULL REFERENCES journal_writer_bindings(binding_id) ON DELETE RESTRICT
|
||||
);
|
||||
CREATE INDEX journal_body_history ON journal_writer_bindings(subject_id,active);
|
||||
CREATE INDEX journal_body_sessions ON journal_session_bindings(binding_id);
|
||||
CREATE INDEX journal_recent_subject ON journals(subject_id,updated_at DESC,id DESC);")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn denied() -> JournalError {
|
||||
JournalError::PermissionDenied("Journal writer binding is absent, released or replaced".into())
|
||||
}
|
||||
|
||||
pub(super) fn check_writer(tx: &Connection, host: &JournalHostContext) -> JournalResult<()> {
|
||||
let attr = host.attribution.as_ref().ok_or_else(denied)?;
|
||||
let binding = host.writer_binding_id.as_ref().ok_or_else(denied)?;
|
||||
if !tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings b JOIN journal_session_bindings s ON s.binding_id=b.binding_id WHERE b.binding_id=?1 AND b.subject_id=?2 AND b.runtime_id=?3 AND b.worker_id=?4 AND b.active=1 AND s.session_id=?5)", params![binding,host.subject_id,attr.runtime_id,attr.worker_id,attr.session_id], |r| r.get::<_,bool>(0))? {
|
||||
return Err(denied());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn require_saved_position(tx: &Connection, generation: &str) -> JournalResult<()> {
|
||||
let saved: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_operations p WHERE p.generation_id=?1 AND json_extract(p.request_json,'$.action.action')='update_position' AND p.rowid > COALESCE((SELECT MAX(r.rowid) FROM journal_operations r WHERE r.generation_id=?1 AND json_extract(r.request_json,'$.action.action') IN ('create','rewrite','fix','confirm_unchanged')),0))",[generation],|r|r.get(0))?;
|
||||
if !saved {
|
||||
return Err(JournalError::Conflict("Journal current position must be saved after the generation's activity records before completion".into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl SubjektivStore {
|
||||
/// The last released body's most recently updated Journal Session, independent of source
|
||||
/// retention. Session attachment order may differ from the order of recorded activity.
|
||||
pub fn journal_predecessor_session(
|
||||
&self,
|
||||
binding: &JournalWriterBinding,
|
||||
) -> JournalResult<Option<String>> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.query_row("SELECT j.source_session_id FROM journals j JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE j.subject_id=?1 AND s.binding_id=(SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=0 ORDER BY rowid DESC LIMIT 1) ORDER BY j.updated_at DESC,j.id DESC LIMIT 1", [&binding.subject_id], |r| r.get(0)).optional()?)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn journal_worker_was_released(
|
||||
&self,
|
||||
subject: &str,
|
||||
runtime: &str,
|
||||
worker: &str,
|
||||
) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 AND active=0)", params![subject,runtime,worker], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn journal_writer_active(&self, binding: &JournalWriterBinding) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4 AND active=1)", params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn journal_session_writer_matches(
|
||||
&self,
|
||||
binding: &JournalWriterBinding,
|
||||
session: &str,
|
||||
) -> JournalResult<bool> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_session_bindings WHERE binding_id=?1 AND session_id=?2)", params![binding.binding_id,session], |r| r.get(0))?)
|
||||
})
|
||||
}
|
||||
|
||||
/// Called under the existing Host Subject lifecycle authority. A stopped or unavailable
|
||||
/// owner remains the owner; only the exact same body may resume an unreleased binding.
|
||||
pub fn claim_journal_writer(
|
||||
&self,
|
||||
subject: &str,
|
||||
runtime: &str,
|
||||
worker: &str,
|
||||
) -> JournalResult<JournalWriterBinding> {
|
||||
for identifier in [subject, runtime, worker] {
|
||||
if identifier.is_empty()
|
||||
|| identifier.len() > 128
|
||||
|| identifier.trim() != identifier
|
||||
|| identifier.chars().any(char::is_control)
|
||||
{
|
||||
return Err(JournalError::InvalidInput(
|
||||
"invalid Journal binding identity".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
self.database.try_transaction(|tx| {
|
||||
let state: Option<String> = tx.query_row("SELECT state FROM subjects WHERE subject_id=?1", [subject], |r| r.get(0)).optional()?;
|
||||
if state.as_deref() != Some("active") { return Err(JournalError::PermissionDenied("Subject is absent or retired".into())); }
|
||||
let current: Option<JournalWriterBinding> = tx.query_row("SELECT subject_id,runtime_id,worker_id,binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=1", [subject], |r| Ok(JournalWriterBinding { subject_id:r.get(0)?,runtime_id:r.get(1)?,worker_id:r.get(2)?,binding_id:r.get(3)? })).optional()?;
|
||||
if let Some(current) = current {
|
||||
if current.runtime_id != runtime || current.worker_id != worker {
|
||||
return Err(JournalError::Conflict("Subject is still occupied; explicit Idle and Journal-complete release required".into()));
|
||||
}
|
||||
return Ok(current);
|
||||
}
|
||||
// Upgraded T-734 data can contain a removed body's durable recording handoff.
|
||||
// Retention is not permission for a different body to occupy that Subject.
|
||||
let legacy_busy: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE j.subject_id=?1 AND (a.runtime_id<>?2 OR a.worker_id<>?3) AND NOT EXISTS(SELECT 1 FROM journal_session_bindings b WHERE b.session_id=j.source_session_id) AND (EXISTS(SELECT 1 FROM journal_generations g WHERE g.journal_id=j.id AND g.active=1) OR EXISTS(SELECT 1 FROM journal_sources s,json_each(s.accepted_json) r WHERE s.journal_id=j.id AND json_extract(r.value,'$.end')>s.reflected_bytes)))", params![subject,runtime,worker], |r| r.get(0))?;
|
||||
if legacy_busy { return Err(JournalError::Conflict("previous body's retained Journal handoff is not complete".into())); }
|
||||
let binding = JournalWriterBinding { subject_id:subject.into(),runtime_id:runtime.into(),worker_id:worker.into(),binding_id:crate::issued_id("journal-binding") };
|
||||
tx.execute("INSERT INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,1)", params![binding.binding_id,subject,runtime,worker])?;
|
||||
Ok(binding)
|
||||
})
|
||||
}
|
||||
|
||||
/// Binds a new Session once. Reconnecting a Worker under a later binding cannot acquire
|
||||
/// mutation authority for its old Session. Attribution and binding are Host-only input.
|
||||
pub fn bind_journal_session(
|
||||
&self,
|
||||
binding: &JournalWriterBinding,
|
||||
attribution: SubjectSessionAttribution,
|
||||
) -> JournalResult<JournalHostContext> {
|
||||
let mut host = JournalHostContext::validated_session(self, attribution)?;
|
||||
self.database.try_transaction(|tx| {
|
||||
let attr = host.attribution.as_ref().expect("validated Session");
|
||||
if binding.subject_id != attr.subject_id || binding.runtime_id != attr.runtime_id || binding.worker_id != attr.worker_id {
|
||||
return Err(denied());
|
||||
}
|
||||
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT ?1,binding_id FROM journal_writer_bindings WHERE binding_id=?2 AND subject_id=?3 AND runtime_id=?4 AND worker_id=?5 AND active=1",params![attr.session_id,binding.binding_id,attr.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
host.writer_binding_id = Some(binding.binding_id.clone());
|
||||
check_writer(tx,&host)?;
|
||||
Ok(host.clone())
|
||||
})
|
||||
}
|
||||
|
||||
/// Destructive release is distinct from a continuity handoff: every attributed Session
|
||||
/// of this body must already have a permanent erasure tombstone. This also records a
|
||||
/// released lifetime for pre-binding bodies, without granting them new mutation authority.
|
||||
pub fn release_erased_journal_body(&self, host: &JournalHostContext) -> JournalResult<bool> {
|
||||
self.database.try_transaction(|tx| {
|
||||
super::store::check_session_host(self, tx, host)?;
|
||||
let attr = host.attribution.as_ref().expect("checked Session authority");
|
||||
let remaining: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_session_attributions a WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3 AND NOT EXISTS(SELECT 1 FROM journal_erased_sessions e WHERE e.session_id=a.session_id AND e.subject_id=a.subject_id))", params![host.subject_id,attr.runtime_id,attr.worker_id], |r| r.get(0))?;
|
||||
if remaining {
|
||||
return Ok(false);
|
||||
}
|
||||
let existing: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3)", params![host.subject_id,attr.runtime_id,attr.worker_id], |r| r.get(0))?;
|
||||
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
if !existing {
|
||||
tx.execute("INSERT INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)", params![crate::issued_id("journal-erased-body"),host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
}
|
||||
Ok(true)
|
||||
})
|
||||
}
|
||||
|
||||
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
||||
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
||||
/// This transaction rejects retained writers, pending source bytes and active generations.
|
||||
/// It never fixes an open entry or declares the represented work complete.
|
||||
pub fn release_journal_writer(&self, binding: &JournalWriterBinding) -> JournalResult<()> {
|
||||
self.database.try_transaction(|tx| {
|
||||
let active: Option<bool> = tx.query_row("SELECT active FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0)).optional()?;
|
||||
let Some(active) = active else { return Err(denied()); };
|
||||
if !active { return Ok(()); }
|
||||
let unbound: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_session_attributions a WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3 AND NOT EXISTS(SELECT 1 FROM journal_session_bindings s WHERE s.session_id=a.session_id) AND NOT EXISTS(SELECT 1 FROM journal_erased_sessions e WHERE e.session_id=a.session_id AND e.subject_id=a.subject_id))",params![binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0))?;
|
||||
if unbound { return Err(JournalError::Conflict("body has legacy Sessions whose committed history has not been reconciled under its writer binding".into())); }
|
||||
let busy: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_generations g JOIN journals j ON j.id=g.journal_id JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE s.binding_id=?1 AND g.active=1)",[&binding.binding_id],|r|r.get(0))?;
|
||||
if busy { return Err(JournalError::Conflict("Journal generation remains active or its result is unconfirmed".into())); }
|
||||
let mut stmt = tx.prepare("SELECT s.accepted_json,s.reflected_bytes,s.unavailable_json FROM journal_sources s JOIN journals j ON j.id=s.journal_id JOIN journal_session_bindings b ON b.session_id=j.source_session_id WHERE b.binding_id=?1")?;
|
||||
let mut rows = stmt.query([&binding.binding_id])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let accepted: Vec<JournalByteRange> = serde_json::from_str(&row.get::<_,String>(0)?)?;
|
||||
let reflected: u64 = row.get(1)?;
|
||||
if accepted.iter().any(|r| r.end > reflected) {
|
||||
return Err(JournalError::Conflict("Journal activity/position/checkpoint has unreflected source bytes".into()));
|
||||
}
|
||||
}
|
||||
let mut checkpoints = tx.prepare("SELECT p.generation_id FROM journal_operations p JOIN journals j ON j.id=p.journal_id JOIN journal_session_bindings b ON b.session_id=j.source_session_id WHERE b.binding_id=?1 AND p.rowid=(SELECT MAX(c.rowid) FROM journal_operations c WHERE c.journal_id=j.id AND json_extract(c.request_json,'$.action.action')='checkpoint')")?;
|
||||
let generations = checkpoints.query_map([&binding.binding_id], |r| r.get::<_,String>(0))?;
|
||||
for generation in generations { require_saved_position(tx, &generation?)?; }
|
||||
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE binding_id=?1 AND active=1",[&binding.binding_id])?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,668 @@
|
||||
//! Additive, immutable provenance-to-context links, not additional evidence for a Memory claim.
|
||||
//!
|
||||
//! Reconciliation reads only saved candidate anchors and the Host-attested Journal source ledger.
|
||||
//! It never fetches history, starts Workers, changes extraction/consolidation, fixes an open event,
|
||||
//! or mutates Memory. A stable evidence reference identifies a public entry, not its legacy numeric
|
||||
//! display range. Missing stable identity stays unconnected; numeric ranges are not guessed to be
|
||||
//! canonical order. Saved edges remain usable without the original runtime and cannot be retargeted.
|
||||
//! Journal content is never copied here. Explicit erasure retains edges and exposes the tombstone.
|
||||
|
||||
use super::{
|
||||
JournalError, JournalHostContext, JournalResult, JournalSourcePointer, JournalTextField,
|
||||
JournalTextPage, MAX_JOURNAL_PAGE, MAX_JOURNAL_RESPONSE_BYTES,
|
||||
};
|
||||
use crate::{MAX_STAGING_ANCHORS, SubjectStagingRecord, SubjektivStore};
|
||||
use memory::schema::SourceEvidenceRef;
|
||||
use rusqlite::{Connection, OptionalExtension, Transaction, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub(crate) fn migrate(tx: &Transaction<'_>) -> feature_storage::Result<()> {
|
||||
tx.execute_batch(include_str!("schema.sql"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MemoryJournalSelector {
|
||||
pub memory_id: String,
|
||||
pub change_id: String,
|
||||
pub candidate_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum MemoryJournalRequest {
|
||||
Reconcile {
|
||||
selector: MemoryJournalSelector,
|
||||
},
|
||||
/// Reads saved link state only; does not implicitly retry reconciliation.
|
||||
Links {
|
||||
selector: MemoryJournalSelector,
|
||||
},
|
||||
/// Continues reconciliation over immutable source indices, at most MAX_STAGING_ANCHORS per call.
|
||||
ReconcilePage {
|
||||
selector: MemoryJournalSelector,
|
||||
source_offset: usize,
|
||||
limit: usize,
|
||||
},
|
||||
/// Reads another saved source page without reconciliation.
|
||||
LinksPage {
|
||||
selector: MemoryJournalSelector,
|
||||
source_offset: usize,
|
||||
limit: usize,
|
||||
},
|
||||
Read {
|
||||
selector: MemoryJournalSelector,
|
||||
source_index: usize,
|
||||
journal_id: String,
|
||||
entry_id: String,
|
||||
expected_token: Option<String>,
|
||||
byte_offset: usize,
|
||||
max_bytes: usize,
|
||||
},
|
||||
Related {
|
||||
journal_id: String,
|
||||
entry_id: String,
|
||||
after: Option<RelatedMemoryCursor>,
|
||||
limit: usize,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(
|
||||
tag = "kind",
|
||||
content = "value",
|
||||
rename_all = "snake_case",
|
||||
deny_unknown_fields
|
||||
)]
|
||||
pub enum MemoryJournalResponse {
|
||||
Links(MemoryJournalLinks),
|
||||
Read(MemoryJournalRead),
|
||||
Related(RelatedMemories),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum OriginalSourceAvailability {
|
||||
/// No saved Host attestation. None of these values imply a live history fetch.
|
||||
Unknown,
|
||||
Missing,
|
||||
Available,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JournalTargetAvailability {
|
||||
Available,
|
||||
Unavailable,
|
||||
Erased,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UnconnectedReason {
|
||||
NotReconciled,
|
||||
InsufficientSource,
|
||||
SourceNotAdmitted,
|
||||
AmbiguousSource,
|
||||
JournalNotFixed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum SourceConnection {
|
||||
Unconnected {
|
||||
reason: UnconnectedReason,
|
||||
},
|
||||
Linked {
|
||||
journal_id: String,
|
||||
entry_id: String,
|
||||
target: JournalTargetAvailability,
|
||||
/// A saved correction invites reconsideration, never an automatic Memory edit.
|
||||
reconsideration: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MemoryJournalSourceLink {
|
||||
pub source_index: usize,
|
||||
pub original_source: OriginalSourceAvailability,
|
||||
pub connection: SourceConnection,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MemoryJournalLinks {
|
||||
pub selector: MemoryJournalSelector,
|
||||
pub sources: Vec<MemoryJournalSourceLink>,
|
||||
pub total_sources: usize,
|
||||
pub next_source_index: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MemoryJournalRead {
|
||||
pub source: MemoryJournalSourceLink,
|
||||
/// Absent for an unavailable or explicitly erased target; never falls back to another entry.
|
||||
pub page: Option<JournalTextPage>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RelatedMemoryCursor {
|
||||
pub memory_id: String,
|
||||
pub change_id: String,
|
||||
pub candidate_id: String,
|
||||
pub source_index: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RelatedMemory {
|
||||
pub selector: MemoryJournalSelector,
|
||||
pub source_index: usize,
|
||||
pub reconsideration: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RelatedMemories {
|
||||
pub items: Vec<RelatedMemory>,
|
||||
pub next: Option<RelatedMemoryCursor>,
|
||||
}
|
||||
|
||||
fn invalid(message: &str) -> JournalError {
|
||||
JournalError::InvalidInput(message.into())
|
||||
}
|
||||
|
||||
fn identifier(value: &str) -> JournalResult<()> {
|
||||
if value.is_empty() || value.len() > 256 || value.chars().any(char::is_control) {
|
||||
return Err(invalid("invalid Memory/Journal selector identifier"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn selector_valid(selector: &MemoryJournalSelector) -> JournalResult<()> {
|
||||
identifier(&selector.memory_id)?;
|
||||
identifier(&selector.change_id)?;
|
||||
identifier(&selector.candidate_id)
|
||||
}
|
||||
|
||||
fn candidate(
|
||||
store: &SubjektivStore,
|
||||
tx: &Connection,
|
||||
host: &JournalHostContext,
|
||||
selector: &MemoryJournalSelector,
|
||||
) -> JournalResult<SubjectStagingRecord> {
|
||||
super::store::check_host(store, tx, host, false)?;
|
||||
selector_valid(selector)?;
|
||||
// The immutable change-candidate edge, not the current Memory projection or a supplied
|
||||
// candidate ID alone, authorizes this provenance selector.
|
||||
let raw: Option<String> = tx
|
||||
.query_row(
|
||||
"SELECT s.record_json FROM staging_records s JOIN memory_change_candidates c
|
||||
ON c.subject_id=s.subject_id AND c.candidate_id=s.candidate_id
|
||||
WHERE c.subject_id=?1 AND c.memory_id=?2 AND c.change_id=?3 AND c.candidate_id=?4",
|
||||
params![
|
||||
host.subject_id,
|
||||
selector.memory_id,
|
||||
selector.change_id,
|
||||
selector.candidate_id
|
||||
],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
let record: SubjectStagingRecord = serde_json::from_str(
|
||||
&raw.ok_or_else(|| JournalError::NotFound("Memory change candidate membership".into()))?,
|
||||
)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SavedLink {
|
||||
reason: String,
|
||||
journal_id: Option<String>,
|
||||
entry_id: Option<String>,
|
||||
session_id: Option<String>,
|
||||
order: Option<u64>,
|
||||
}
|
||||
|
||||
fn saved(
|
||||
tx: &Connection,
|
||||
host: &JournalHostContext,
|
||||
candidate_id: &str,
|
||||
index: usize,
|
||||
) -> JournalResult<Option<SavedLink>> {
|
||||
Ok(tx.query_row(
|
||||
"SELECT reason,journal_id,entry_id,source_session_id,source_order FROM memory_journal_links
|
||||
WHERE subject_id=?1 AND candidate_id=?2 AND source_index=?3",
|
||||
params![host.subject_id, candidate_id, index],
|
||||
|r| Ok(SavedLink { reason:r.get(0)?,journal_id:r.get(1)?,entry_id:r.get(2)?,session_id:r.get(3)?,order:r.get(4)? }),
|
||||
).optional()?)
|
||||
}
|
||||
|
||||
fn original_availability(
|
||||
tx: &Connection,
|
||||
journal: &str,
|
||||
order: u64,
|
||||
) -> JournalResult<OriginalSourceAvailability> {
|
||||
let unavailable: Option<String> = tx
|
||||
.query_row(
|
||||
"SELECT unavailable_json FROM journal_sources WHERE journal_id=?1 AND source_order=?2",
|
||||
params![journal, order],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
Ok(match unavailable {
|
||||
None => OriginalSourceAvailability::Unknown,
|
||||
Some(raw) => {
|
||||
let ranges: Vec<super::JournalByteRange> = serde_json::from_str(&raw)?;
|
||||
if ranges.is_empty() {
|
||||
OriginalSourceAvailability::Available
|
||||
} else {
|
||||
OriginalSourceAvailability::Missing
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn project(
|
||||
tx: &Connection,
|
||||
host: &JournalHostContext,
|
||||
candidate_id: &str,
|
||||
index: usize,
|
||||
) -> JournalResult<MemoryJournalSourceLink> {
|
||||
let Some(link) = saved(tx, host, candidate_id, index)? else {
|
||||
return Ok(MemoryJournalSourceLink {
|
||||
source_index: index,
|
||||
original_source: OriginalSourceAvailability::Unknown,
|
||||
connection: SourceConnection::Unconnected {
|
||||
reason: UnconnectedReason::NotReconciled,
|
||||
},
|
||||
});
|
||||
};
|
||||
if link.reason != "linked" {
|
||||
let reason = match link.reason.as_str() {
|
||||
"insufficient_source" => UnconnectedReason::InsufficientSource,
|
||||
"source_not_admitted" => UnconnectedReason::SourceNotAdmitted,
|
||||
"ambiguous_source" => UnconnectedReason::AmbiguousSource,
|
||||
"journal_not_fixed" => UnconnectedReason::JournalNotFixed,
|
||||
_ => return Err(invalid("invalid saved connection reason")),
|
||||
};
|
||||
return Ok(MemoryJournalSourceLink {
|
||||
source_index: index,
|
||||
original_source: OriginalSourceAvailability::Unknown,
|
||||
connection: SourceConnection::Unconnected { reason },
|
||||
});
|
||||
}
|
||||
let journal_id = link
|
||||
.journal_id
|
||||
.ok_or_else(|| invalid("saved link lacks Journal"))?;
|
||||
let entry_id = link
|
||||
.entry_id
|
||||
.ok_or_else(|| invalid("saved link lacks entry"))?;
|
||||
let session = link
|
||||
.session_id
|
||||
.ok_or_else(|| invalid("saved link lacks Session"))?;
|
||||
let order = link
|
||||
.order
|
||||
.ok_or_else(|| invalid("saved link lacks source order"))?;
|
||||
let erased: bool = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM journal_erased_sessions WHERE subject_id=?1 AND session_id=?2)",
|
||||
params![host.subject_id, session], |r| r.get(0),
|
||||
)?;
|
||||
let exists: bool = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM journal_entries e JOIN journals j ON j.id=e.journal_id
|
||||
WHERE e.subject_id=?1 AND e.journal_id=?2 AND e.id=?3 AND j.source_session_id=?4)",
|
||||
params![host.subject_id, journal_id, entry_id, session],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let reconsideration = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM journal_corrections WHERE subject_id=?1 AND target_id=?2)",
|
||||
params![host.subject_id, entry_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(MemoryJournalSourceLink {
|
||||
source_index: index,
|
||||
original_source: original_availability(tx, &journal_id, order)?,
|
||||
connection: SourceConnection::Linked {
|
||||
journal_id,
|
||||
entry_id,
|
||||
target: if erased {
|
||||
JournalTargetAvailability::Erased
|
||||
} else if exists {
|
||||
JournalTargetAvailability::Available
|
||||
} else {
|
||||
JournalTargetAvailability::Unavailable
|
||||
},
|
||||
reconsideration,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
struct Resolved {
|
||||
journal: String,
|
||||
entry: String,
|
||||
session: String,
|
||||
order: u64,
|
||||
}
|
||||
|
||||
fn resolve(
|
||||
tx: &Connection,
|
||||
host: &JournalHostContext,
|
||||
source: &SourceEvidenceRef,
|
||||
) -> JournalResult<Result<Resolved, &'static str>> {
|
||||
let Some(evidence) = source.evidence_id.as_deref() else {
|
||||
return Ok(Err("insufficient_source"));
|
||||
};
|
||||
if !evidence.starts_with('E')
|
||||
|| evidence.len() < 2
|
||||
|| evidence.len() > 129
|
||||
|| !evidence[1..]
|
||||
.bytes()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == b'-')
|
||||
|| (source.session_id.is_none() && source.segment_id.is_none())
|
||||
{
|
||||
return Ok(Err("insufficient_source"));
|
||||
}
|
||||
// Segment identity may be omitted only when an immutable Session attribution disambiguates it.
|
||||
// A candidate's primary legacy source range is not substituted for an individual anchor.
|
||||
let segment = match source.segment_id.as_deref() {
|
||||
Some(raw) => match uuid::Uuid::parse_str(raw) {
|
||||
Ok(id) => Some(id),
|
||||
Err(_) => return Ok(Err("insufficient_source")),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let mut stmt = tx.prepare(
|
||||
"SELECT j.id,j.source_session_id,s.source_order,s.pointer_json FROM journal_sources s
|
||||
JOIN journals j ON j.id=s.journal_id
|
||||
JOIN subject_session_attributions a ON a.subject_id=j.subject_id AND a.session_id=j.source_session_id
|
||||
WHERE j.subject_id=?1 AND s.entry_ref=?2
|
||||
AND (?3 IS NULL OR j.source_session_id=?3)
|
||||
AND (?4 IS NULL OR json_extract(s.pointer_json,'$.segment_id')=?4)
|
||||
LIMIT 2",
|
||||
)?;
|
||||
let found = stmt
|
||||
.query_map(
|
||||
params![
|
||||
host.subject_id,
|
||||
evidence,
|
||||
source.session_id,
|
||||
segment.map(|s| s.to_string())
|
||||
],
|
||||
|r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, u64>(2)?,
|
||||
r.get::<_, String>(3)?,
|
||||
))
|
||||
},
|
||||
)?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
if found.is_empty() {
|
||||
return Ok(Err("source_not_admitted"));
|
||||
}
|
||||
if found.len() != 1 {
|
||||
return Ok(Err("ambiguous_source"));
|
||||
}
|
||||
let (journal, session, order, pointer) = &found[0];
|
||||
let pointer: JournalSourcePointer = serde_json::from_str(pointer)?;
|
||||
if pointer.entry_ref != evidence || segment.is_some_and(|s| s != pointer.segment_id) {
|
||||
return Ok(Err("ambiguous_source"));
|
||||
}
|
||||
// A fixed endpoint covers (previous_fixed_end, source_end]. The Session start is the first
|
||||
// interval's lower boundary. Processing checkpoints and open entries cannot supply an end.
|
||||
let mut stmt = tx.prepare(
|
||||
"SELECT e.id FROM journal_entries e WHERE e.subject_id=?1 AND e.journal_id=?2
|
||||
AND e.state='fixed' AND e.source_order>=?3
|
||||
AND NOT EXISTS(SELECT 1 FROM journal_entries p WHERE p.journal_id=e.journal_id
|
||||
AND p.position<e.position AND p.state='fixed' AND p.source_order>=?3)
|
||||
ORDER BY e.position LIMIT 2",
|
||||
)?;
|
||||
let entries = stmt
|
||||
.query_map(params![host.subject_id, journal, order], |r| {
|
||||
r.get::<_, String>(0)
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
match entries.as_slice() {
|
||||
[entry] => Ok(Ok(Resolved {
|
||||
journal: journal.clone(),
|
||||
entry: entry.clone(),
|
||||
session: session.clone(),
|
||||
order: *order,
|
||||
})),
|
||||
[] => Ok(Err("journal_not_fixed")),
|
||||
_ => Ok(Err("ambiguous_source")),
|
||||
}
|
||||
}
|
||||
|
||||
fn reconcile(
|
||||
tx: &Connection,
|
||||
host: &JournalHostContext,
|
||||
record: &SubjectStagingRecord,
|
||||
indices: std::ops::Range<usize>,
|
||||
) -> JournalResult<()> {
|
||||
for index in indices {
|
||||
let source = &record.source_refs[index];
|
||||
if saved(tx, host, &record.id, index)?.is_some_and(|s| s.reason == "linked") {
|
||||
continue;
|
||||
}
|
||||
let (reason, resolved) = match resolve(tx, host, source)? {
|
||||
Ok(link) => ("linked", Some(link)),
|
||||
Err(reason) => (reason, None),
|
||||
};
|
||||
tx.execute(
|
||||
"INSERT INTO memory_journal_links(subject_id,candidate_id,source_index,reason,
|
||||
journal_id,entry_id,source_session_id,source_order) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)
|
||||
ON CONFLICT(subject_id,candidate_id,source_index) DO UPDATE SET
|
||||
reason=excluded.reason,journal_id=excluded.journal_id,entry_id=excluded.entry_id,
|
||||
source_session_id=excluded.source_session_id,source_order=excluded.source_order",
|
||||
params![
|
||||
host.subject_id,
|
||||
record.id,
|
||||
index,
|
||||
reason,
|
||||
resolved.as_ref().map(|r| r.journal.as_str()),
|
||||
resolved.as_ref().map(|r| r.entry.as_str()),
|
||||
resolved.as_ref().map(|r| r.session.as_str()),
|
||||
resolved.as_ref().map(|r| r.order)
|
||||
],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl SubjektivStore {
|
||||
/// Scoped Memory provenance bridge. Reconcile is additive and idempotent, so a lost response
|
||||
/// needs no operation ledger: retry the same selector and reread saved edges. Reads never reconcile.
|
||||
pub fn execute_memory_journal(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
request: MemoryJournalRequest,
|
||||
) -> JournalResult<MemoryJournalResponse> {
|
||||
let response = match request {
|
||||
MemoryJournalRequest::Reconcile { selector } => {
|
||||
self.memory_journal_links(host, selector, true, 0, MAX_STAGING_ANCHORS)?
|
||||
}
|
||||
MemoryJournalRequest::Links { selector } => {
|
||||
self.memory_journal_links(host, selector, false, 0, MAX_STAGING_ANCHORS)?
|
||||
}
|
||||
MemoryJournalRequest::ReconcilePage {
|
||||
selector,
|
||||
source_offset,
|
||||
limit,
|
||||
} => self.memory_journal_links(host, selector, true, source_offset, limit)?,
|
||||
MemoryJournalRequest::LinksPage {
|
||||
selector,
|
||||
source_offset,
|
||||
limit,
|
||||
} => self.memory_journal_links(host, selector, false, source_offset, limit)?,
|
||||
MemoryJournalRequest::Read {
|
||||
selector,
|
||||
source_index,
|
||||
journal_id,
|
||||
entry_id,
|
||||
expected_token,
|
||||
byte_offset,
|
||||
max_bytes,
|
||||
} => {
|
||||
identifier(&journal_id)?;
|
||||
identifier(&entry_id)?;
|
||||
if !(4..=super::MAX_JOURNAL_READ_BYTES).contains(&max_bytes) {
|
||||
return Err(invalid("text page must be 4..16384 bytes"));
|
||||
}
|
||||
if let Some(token) = expected_token.as_deref() {
|
||||
identifier(token)?;
|
||||
}
|
||||
let mut source = self.database.try_with_connection(|tx| {
|
||||
let record = candidate(self, tx, host, &selector)?;
|
||||
if source_index >= record.source_refs.len() {
|
||||
return Err(invalid("source index outside candidate anchors"));
|
||||
}
|
||||
let link = project(tx, host, &selector.candidate_id, source_index)?;
|
||||
match &link.connection {
|
||||
SourceConnection::Linked {
|
||||
journal_id: saved_journal,
|
||||
entry_id: saved_entry,
|
||||
..
|
||||
} if saved_journal == &journal_id && saved_entry == &entry_id => Ok(link),
|
||||
_ => Err(JournalError::PermissionDenied(
|
||||
"read requires the exact saved Journal link".into(),
|
||||
)),
|
||||
}
|
||||
})?;
|
||||
let page = match &source.connection {
|
||||
SourceConnection::Linked {
|
||||
target: JournalTargetAvailability::Available,
|
||||
..
|
||||
} => {
|
||||
match self.read_journal_text(
|
||||
host,
|
||||
&journal_id,
|
||||
Some(&entry_id),
|
||||
JournalTextField::Body,
|
||||
expected_token.as_deref(),
|
||||
byte_offset,
|
||||
max_bytes,
|
||||
) {
|
||||
Ok(page) => Some(page),
|
||||
// Erasure between the saved-edge check and shared text read must not
|
||||
// fall back, expose cached text, or pretend the link never existed.
|
||||
Err(JournalError::NotFound(_)) => {
|
||||
source = self.database.try_with_connection(|tx| {
|
||||
project(tx, host, &selector.candidate_id, source_index)
|
||||
})?;
|
||||
None
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
MemoryJournalResponse::Read(MemoryJournalRead { source, page })
|
||||
}
|
||||
MemoryJournalRequest::Related {
|
||||
journal_id,
|
||||
entry_id,
|
||||
after,
|
||||
limit,
|
||||
} => self.related_memories(host, &journal_id, &entry_id, after.as_ref(), limit)?,
|
||||
};
|
||||
if serde_json::to_string_pretty(&response)?.len() > MAX_JOURNAL_RESPONSE_BYTES {
|
||||
return Err(invalid("Memory Journal response exceeds byte budget"));
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn memory_journal_links(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
selector: MemoryJournalSelector,
|
||||
do_reconcile: bool,
|
||||
source_offset: usize,
|
||||
limit: usize,
|
||||
) -> JournalResult<MemoryJournalResponse> {
|
||||
if !(1..=MAX_STAGING_ANCHORS).contains(&limit) {
|
||||
return Err(invalid("source page exceeds staging anchor bound"));
|
||||
}
|
||||
self.database.try_transaction(|tx| {
|
||||
let record = candidate(self, tx, host, &selector)?;
|
||||
let total_sources = record.source_refs.len();
|
||||
if source_offset > total_sources {
|
||||
return Err(invalid("source offset outside immutable candidate anchors"));
|
||||
}
|
||||
let end = source_offset.saturating_add(limit).min(total_sources);
|
||||
if do_reconcile {
|
||||
reconcile(tx, host, &record, source_offset..end)?;
|
||||
}
|
||||
let sources = (source_offset..end)
|
||||
.map(|index| project(tx, host, &selector.candidate_id, index))
|
||||
.collect::<JournalResult<Vec<_>>>()?;
|
||||
Ok(MemoryJournalResponse::Links(MemoryJournalLinks {
|
||||
selector,
|
||||
sources,
|
||||
total_sources,
|
||||
next_source_index: (end < total_sources).then_some(end),
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
fn related_memories(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
journal_id: &str,
|
||||
entry_id: &str,
|
||||
after: Option<&RelatedMemoryCursor>,
|
||||
limit: usize,
|
||||
) -> JournalResult<MemoryJournalResponse> {
|
||||
identifier(journal_id)?;
|
||||
identifier(entry_id)?;
|
||||
if !(1..=MAX_JOURNAL_PAGE).contains(&limit) {
|
||||
return Err(invalid("related page must be 1..32"));
|
||||
}
|
||||
if let Some(cursor) = after {
|
||||
identifier(&cursor.memory_id)?;
|
||||
identifier(&cursor.change_id)?;
|
||||
identifier(&cursor.candidate_id)?;
|
||||
if i64::try_from(cursor.source_index).is_err() {
|
||||
return Err(invalid("invalid related source index"));
|
||||
}
|
||||
}
|
||||
self.database.try_with_connection(|tx| {
|
||||
super::store::check_host(self,tx,host,false)?;
|
||||
// Saved target identity authorizes reverse lookup after erasure as well as before it.
|
||||
let known:bool=tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM journal_entries WHERE subject_id=?1 AND journal_id=?2 AND id=?3)
|
||||
OR EXISTS(SELECT 1 FROM memory_journal_links WHERE subject_id=?1 AND journal_id=?2 AND entry_id=?3)",
|
||||
params![host.subject_id,journal_id,entry_id],|r|r.get(0))?;
|
||||
if !known { return Err(JournalError::NotFound("Journal entry".into())); }
|
||||
let mut stmt=tx.prepare(
|
||||
"SELECT m.memory_id,m.change_id,l.candidate_id,l.source_index,
|
||||
EXISTS(SELECT 1 FROM journal_corrections c WHERE c.subject_id=l.subject_id AND c.target_id=l.entry_id)
|
||||
FROM memory_journal_links l JOIN memory_change_candidates m
|
||||
ON m.subject_id=l.subject_id AND m.candidate_id=l.candidate_id
|
||||
WHERE l.subject_id=?1 AND l.reason='linked'
|
||||
AND ((l.journal_id=?2 AND l.entry_id=?3)
|
||||
OR EXISTS(SELECT 1 FROM journal_corrections c WHERE c.subject_id=?1 AND c.entry_id=?3 AND c.target_id=l.entry_id))
|
||||
AND (?4 IS NULL OR (m.memory_id,m.change_id,l.candidate_id,l.source_index)>(?4,?5,?6,?7))
|
||||
ORDER BY m.memory_id,m.change_id,l.candidate_id,l.source_index LIMIT ?8")?;
|
||||
let mut items=stmt.query_map(params![host.subject_id,journal_id,entry_id,
|
||||
after.map(|c|c.memory_id.as_str()),after.map(|c|c.change_id.as_str()),
|
||||
after.map(|c|c.candidate_id.as_str()),after.map(|c|c.source_index),limit+1],
|
||||
|r|Ok(RelatedMemory {selector:MemoryJournalSelector {memory_id:r.get(0)?,change_id:r.get(1)?,candidate_id:r.get(2)?},
|
||||
source_index:r.get(3)?,reconsideration:r.get(4)?}))?.collect::<Result<Vec<_>,_>>()?;
|
||||
let more=items.len()>limit;
|
||||
items.truncate(limit);
|
||||
let next=if more {items.last().map(|last|RelatedMemoryCursor {
|
||||
memory_id:last.selector.memory_id.clone(),change_id:last.selector.change_id.clone(),
|
||||
candidate_id:last.selector.candidate_id.clone(),source_index:last.source_index })} else {None};
|
||||
Ok(MemoryJournalResponse::Related(RelatedMemories {items,next}))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
-- References outlive explicit Journal erasure. Do not add target-row foreign keys:
|
||||
-- lifecycle erasure deletes Journal content but retains its Session tombstone.
|
||||
CREATE TABLE memory_journal_links (
|
||||
subject_id TEXT NOT NULL,
|
||||
candidate_id TEXT NOT NULL,
|
||||
source_index INTEGER NOT NULL CHECK(source_index >= 0),
|
||||
reason TEXT NOT NULL CHECK(reason IN (
|
||||
'insufficient_source', 'source_not_admitted', 'ambiguous_source',
|
||||
'journal_not_fixed', 'linked'
|
||||
)),
|
||||
journal_id TEXT,
|
||||
entry_id TEXT,
|
||||
source_session_id TEXT,
|
||||
source_order INTEGER,
|
||||
PRIMARY KEY(subject_id, candidate_id, source_index),
|
||||
FOREIGN KEY(subject_id, candidate_id)
|
||||
REFERENCES staging_records(subject_id, candidate_id) ON DELETE RESTRICT,
|
||||
CHECK((reason = 'linked' AND journal_id IS NOT NULL AND entry_id IS NOT NULL
|
||||
AND source_session_id IS NOT NULL AND source_order IS NOT NULL AND source_order >= 0)
|
||||
OR (reason <> 'linked' AND journal_id IS NULL AND entry_id IS NULL
|
||||
AND source_session_id IS NULL AND source_order IS NULL))
|
||||
);
|
||||
CREATE INDEX memory_journal_links_reverse
|
||||
ON memory_journal_links(subject_id, journal_id, entry_id, candidate_id, source_index);
|
||||
CREATE TRIGGER memory_journal_links_identity BEFORE UPDATE ON memory_journal_links
|
||||
WHEN OLD.subject_id <> NEW.subject_id OR OLD.candidate_id <> NEW.candidate_id
|
||||
OR OLD.source_index <> NEW.source_index
|
||||
BEGIN SELECT RAISE(ABORT, 'Journal source reference identity is immutable'); END;
|
||||
CREATE TRIGGER memory_journal_links_connected_immutable BEFORE UPDATE ON memory_journal_links
|
||||
WHEN OLD.reason = 'linked'
|
||||
BEGIN SELECT RAISE(ABORT, 'connected Journal references are immutable'); END;
|
||||
CREATE TRIGGER memory_journal_links_retained BEFORE DELETE ON memory_journal_links
|
||||
BEGIN SELECT RAISE(ABORT, 'Journal references are retained'); END;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -37,10 +37,16 @@
|
||||
//! Hosts resuming at the processing cursor can use a bounded source point lookup instead of
|
||||
//! traversing already reflected pages. Receipt/entry keysets remain append-stable.
|
||||
|
||||
pub mod binding;
|
||||
pub use binding::JournalWriterBinding;
|
||||
pub(crate) use binding::migrate as migrate_binding;
|
||||
mod lifecycle;
|
||||
pub mod memory_links;
|
||||
pub(crate) use lifecycle::migrate as migrate_lifecycle;
|
||||
pub mod recording;
|
||||
pub mod service;
|
||||
mod source_status;
|
||||
pub mod startup;
|
||||
mod store;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -110,8 +116,14 @@ pub struct JournalHostContext {
|
||||
scope_id: String,
|
||||
subject_id: String,
|
||||
attribution: Option<SubjectSessionAttribution>,
|
||||
writer_binding_id: Option<String>,
|
||||
}
|
||||
impl JournalHostContext {
|
||||
/// Evidence for a retained Job grant, not permission to mint or renew a binding.
|
||||
pub fn writer_binding_id(&self) -> Option<&str> {
|
||||
self.writer_binding_id.as_deref()
|
||||
}
|
||||
|
||||
pub fn validated_reader(store: &SubjektivStore, subject_id: &str) -> JournalResult<Self> {
|
||||
store
|
||||
.subject(subject_id)?
|
||||
@@ -120,6 +132,7 @@ impl JournalHostContext {
|
||||
scope_id: store.scope_id().into(),
|
||||
subject_id: subject_id.into(),
|
||||
attribution: None,
|
||||
writer_binding_id: None,
|
||||
})
|
||||
}
|
||||
/// External authentication, public-history validation and execution authority are Host duties.
|
||||
@@ -133,6 +146,17 @@ impl JournalHostContext {
|
||||
));
|
||||
}
|
||||
let mut context = Self::validated_reader(store, &attribution.subject_id)?;
|
||||
use rusqlite::OptionalExtension;
|
||||
context.writer_binding_id = store.database.try_with_connection(|tx| {
|
||||
Ok::<_, JournalError>(
|
||||
tx.query_row(
|
||||
"SELECT binding_id FROM journal_session_bindings WHERE session_id=?1",
|
||||
[&attribution.session_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?,
|
||||
)
|
||||
})?;
|
||||
context.attribution = Some(attribution);
|
||||
Ok(context)
|
||||
}
|
||||
|
||||
@@ -386,6 +386,7 @@ pub fn finish(
|
||||
return Ok(serde_json::from_str(&saved)?);
|
||||
}
|
||||
super::store::job(store, tx, context, true)?;
|
||||
super::binding::require_saved_position(tx, &generation.id)?;
|
||||
let last = generation
|
||||
.selected_ranges
|
||||
.last()
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
//! Optional Host metadata observations for original material, independent of saved Journal reading.
|
||||
//!
|
||||
//! Presence checks must not fetch original history or restore a Worker. A failed/unauthorized
|
||||
//! metadata lookup is unknown, not missing, and cannot prevent a saved Journal body from being read.
|
||||
|
||||
use super::memory_links::{
|
||||
MemoryJournalRequest, MemoryJournalResponse, OriginalSourceAvailability,
|
||||
};
|
||||
use super::{JournalHostContext, JournalResult, SubjektivStore};
|
||||
use rusqlite::{OptionalExtension, params};
|
||||
|
||||
impl SubjektivStore {
|
||||
/// Executes the saved-reference operation first, then annotates original-source absence using
|
||||
/// Host-owned metadata only. `Some(false)` is confirmed absence; `None` is an unavailable lookup.
|
||||
/// Presence alone cannot attest that a particular historical entry is still readable.
|
||||
pub fn execute_memory_journal_with_source_presence(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
request: MemoryJournalRequest,
|
||||
mut presence: impl FnMut(&str) -> Option<bool>,
|
||||
) -> JournalResult<MemoryJournalResponse> {
|
||||
let selector = match &request {
|
||||
MemoryJournalRequest::Reconcile { selector }
|
||||
| MemoryJournalRequest::Links { selector }
|
||||
| MemoryJournalRequest::ReconcilePage { selector, .. }
|
||||
| MemoryJournalRequest::LinksPage { selector, .. }
|
||||
| MemoryJournalRequest::Read { selector, .. } => Some(selector.clone()),
|
||||
MemoryJournalRequest::Related { .. } => None,
|
||||
};
|
||||
let mut response = self.execute_memory_journal(host, request)?;
|
||||
let Some(selector) = selector else {
|
||||
return Ok(response);
|
||||
};
|
||||
// Membership and scoped reader authority were checked by the operation. Candidate JSON is
|
||||
// immutable; never infer source ownership from its presence without checking attribution.
|
||||
let candidate = self.staging_candidate(&host.subject_id, &selector.candidate_id)?;
|
||||
let mut observations = std::collections::HashMap::new();
|
||||
let mut annotate =
|
||||
|link: &mut super::memory_links::MemoryJournalSourceLink| -> JournalResult<()> {
|
||||
let source = candidate
|
||||
.as_ref()
|
||||
.and_then(|c| c.source_refs.get(link.source_index));
|
||||
// Legacy anchors may lack a Session ID. A connected edge already resolved it
|
||||
// from immutable attribution and stable Segment/entry identity; reuse that saved
|
||||
// identity rather than fetching history or modifying the original candidate.
|
||||
let saved_session: Option<String> = self.database.try_with_connection(|tx| {
|
||||
Ok::<_, super::JournalError>(tx.query_row(
|
||||
"SELECT source_session_id FROM memory_journal_links
|
||||
WHERE subject_id=?1 AND candidate_id=?2 AND source_index=?3 AND reason='linked'",
|
||||
params![host.subject_id, selector.candidate_id, link.source_index],
|
||||
|row| row.get(0),
|
||||
).optional()?)
|
||||
})?;
|
||||
let session = saved_session
|
||||
.as_deref()
|
||||
.or_else(|| source.and_then(|s| s.session_id.as_deref()));
|
||||
let observed = if let Some(session) = session {
|
||||
if let Some(value) = observations.get(session) {
|
||||
*value
|
||||
} else {
|
||||
let owned = self
|
||||
.session_attribution(session)?
|
||||
.is_some_and(|a| a.subject_id == host.subject_id);
|
||||
let value = if owned { presence(session) } else { None };
|
||||
observations.insert(session.to_owned(), value);
|
||||
value
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
link.original_source = match observed {
|
||||
Some(false) => OriginalSourceAvailability::Missing,
|
||||
// Preserve explicit saved missing-byte attestations. Directory/registry presence
|
||||
// does not prove the referenced entry survived rewind or storage corruption.
|
||||
_ if link.original_source == OriginalSourceAvailability::Missing => {
|
||||
OriginalSourceAvailability::Missing
|
||||
}
|
||||
_ => OriginalSourceAvailability::Unknown,
|
||||
};
|
||||
Ok(())
|
||||
};
|
||||
match &mut response {
|
||||
MemoryJournalResponse::Links(page) => {
|
||||
for source in &mut page.sources {
|
||||
annotate(source)?;
|
||||
}
|
||||
}
|
||||
MemoryJournalResponse::Read(read) => annotate(&mut read.source)?,
|
||||
MemoryJournalResponse::Related(_) => {}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
//! Read-only, bounded Journal context for a new body Session. Selection and text projection
|
||||
//! share one storage transaction; no source history, Job, admission or checkpoint is consulted.
|
||||
//! This is derived reference material, never a new observation or an execution capability.
|
||||
|
||||
use super::*;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
pub const STARTUP_MAX_BYTES: usize = 32 * 1024;
|
||||
const JOURNALS: usize = 3;
|
||||
const ENTRIES: usize = 3;
|
||||
const TEXT_BYTES: usize = 2048;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JournalStartupSnapshot {
|
||||
pub subject_id: String,
|
||||
pub journals: Vec<StartupJournal>,
|
||||
pub omitted: bool,
|
||||
}
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StartupJournal {
|
||||
pub journal_id: String,
|
||||
pub source_session_id: String,
|
||||
pub state_token: String,
|
||||
pub process_order: u64,
|
||||
pub process_byte: u64,
|
||||
pub current_position: String,
|
||||
pub entries: Vec<StartupEntry>,
|
||||
pub omitted: bool,
|
||||
}
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StartupEntry {
|
||||
pub entry_id: String,
|
||||
pub position: u64,
|
||||
pub state: JournalEntryState,
|
||||
pub title: String,
|
||||
pub body: String,
|
||||
pub ongoing: String,
|
||||
pub source_end: Option<JournalSourcePointer>,
|
||||
pub correction_refs: Vec<String>,
|
||||
pub omitted: bool,
|
||||
}
|
||||
|
||||
// Read a bounded UTF-8 prefix at SQLite, not a full megabyte field followed by truncation.
|
||||
fn text(tx: &Connection, table: &str, id: &str, field: &str) -> JournalResult<(String, bool)> {
|
||||
let (mut bytes, length): (Vec<u8>, usize) = tx.query_row(
|
||||
&format!("SELECT COALESCE(substr(CAST({field} AS BLOB),1,?2),X''),length(CAST({field} AS BLOB)) FROM {table} WHERE id=?1"),
|
||||
params![id, TEXT_BYTES],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
let length_read = bytes.len();
|
||||
match std::str::from_utf8(&bytes) {
|
||||
Ok(_) => {}
|
||||
Err(error) if error.error_len().is_none() => bytes.truncate(error.valid_up_to()),
|
||||
Err(_) => {
|
||||
return Err(JournalError::Conflict(
|
||||
"stored Journal text is not UTF-8".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let omitted = bytes.len() < length || length_read < length;
|
||||
Ok((String::from_utf8(bytes).expect("validated UTF-8"), omitted))
|
||||
}
|
||||
|
||||
impl SubjektivStore {
|
||||
/// `previous_session` comes from the Host's retained predecessor binding, not model input.
|
||||
/// Recent Subject-local Journals supplement that predecessor under independent fixed budgets.
|
||||
pub fn journal_startup_snapshot(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
previous_session: Option<&str>,
|
||||
) -> JournalResult<JournalStartupSnapshot> {
|
||||
self.database.try_transaction(|tx| {
|
||||
super::store::check_host(self, tx, host, false)?;
|
||||
let mut ids = Vec::new();
|
||||
if let Some(session) = previous_session {
|
||||
let previous: Option<(String, String)> = tx.query_row(
|
||||
"SELECT id,subject_id FROM journals WHERE source_session_id=?1",
|
||||
[session], |r| Ok((r.get(0)?, r.get(1)?)),
|
||||
).optional()?;
|
||||
if let Some((id, subject)) = previous {
|
||||
if subject != host.subject_id { return Err(JournalError::PermissionDenied("predecessor Journal belongs to another Subject".into())); }
|
||||
ids.push(id);
|
||||
}
|
||||
}
|
||||
let mut query = tx.prepare("SELECT id FROM journals WHERE subject_id=?1 ORDER BY updated_at DESC,id DESC LIMIT ?2")?;
|
||||
let recent = query.query_map(params![host.subject_id, JOURNALS + 1], |r| r.get::<_, String>(0))?;
|
||||
for id in recent { let id = id?; if !ids.contains(&id) { ids.push(id); } }
|
||||
let omitted = ids.len() > JOURNALS;
|
||||
ids.truncate(JOURNALS);
|
||||
let mut snapshot = JournalStartupSnapshot { subject_id: host.subject_id.clone(), journals: Vec::new(), omitted };
|
||||
for id in ids {
|
||||
let (session, token, order, byte) = tx.query_row(
|
||||
"SELECT source_session_id,state_token,process_order,process_byte FROM journals WHERE id=?1 AND subject_id=?2",
|
||||
params![id, host.subject_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?)),
|
||||
)?;
|
||||
let (current_position, mut omitted) = text(tx, "journals", &id, "current_position")?;
|
||||
let mut stmt = tx.prepare("SELECT id,position,state,source_end_json FROM journal_entries WHERE journal_id=?1 ORDER BY position DESC LIMIT ?2")?;
|
||||
let rows = stmt.query_map(params![id, ENTRIES + 1], |r| Ok((r.get::<_,String>(0)?,r.get::<_,u64>(1)?,r.get::<_,String>(2)?,r.get::<_,Option<String>>(3)?)))?.collect::<Result<Vec<_>,_>>()?;
|
||||
omitted |= rows.len() > ENTRIES;
|
||||
let mut entries = Vec::new();
|
||||
for (entry_id, position, state, source_end) in rows.into_iter().take(ENTRIES) {
|
||||
let (title, title_cut) = text(tx, "journal_entries", &entry_id, "title")?;
|
||||
let (body, body_cut) = text(tx, "journal_entries", &entry_id, "body")?;
|
||||
let (ongoing, ongoing_cut) = text(tx, "journal_entries", &entry_id, "ongoing")?;
|
||||
let mut refs = tx.prepare("SELECT target_id FROM journal_corrections WHERE entry_id=?1 ORDER BY target_id LIMIT 17")?;
|
||||
let mut correction_refs = refs.query_map([&entry_id], |r| r.get::<_,String>(0))?.collect::<Result<Vec<_>,_>>()?;
|
||||
let omitted = title_cut || body_cut || ongoing_cut || correction_refs.len() > 16;
|
||||
correction_refs.truncate(16);
|
||||
entries.push(StartupEntry { entry_id, position, state: match state.as_str() { "open" => JournalEntryState::Open, "fixed" => JournalEntryState::Fixed, _ => return Err(JournalError::Conflict("invalid stored Journal entry state".into())) }, title, body, ongoing, source_end: source_end.map(|s| serde_json::from_str(&s)).transpose()?, correction_refs, omitted });
|
||||
}
|
||||
snapshot.journals.push(StartupJournal { journal_id: id, source_session_id: session, state_token: token, process_order: order, process_byte: byte, current_position, entries, omitted });
|
||||
}
|
||||
// JSON escaping also consumes the independent output budget. Drop least-priority
|
||||
// entries/Journals rather than returning an over-budget or silently partial document.
|
||||
while serde_json::to_vec(&snapshot)?.len() > STARTUP_MAX_BYTES {
|
||||
snapshot.omitted = true;
|
||||
let journal = snapshot.journals.last_mut().ok_or_else(|| JournalError::InvalidInput("startup metadata exceeds budget".into()))?;
|
||||
if journal.entries.pop().is_some() { journal.omitted = true; } else { snapshot.journals.pop(); }
|
||||
}
|
||||
Ok(snapshot)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -141,6 +141,7 @@ pub(super) fn check_host(
|
||||
return Err(JournalError::PermissionDenied("Subject is retired".into()));
|
||||
}
|
||||
check_session_host(store, tx, host)?;
|
||||
super::binding::check_writer(tx, host)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -373,7 +374,10 @@ pub(super) fn job(
|
||||
ctx: &JournalJobContext,
|
||||
active: bool,
|
||||
) -> JournalResult<JournalGeneration> {
|
||||
check_host(store, tx, &ctx.host, true)?;
|
||||
// Recovery can inspect the original generation and exact saved receipts after release;
|
||||
// only a new mutation requires the still-current writer lifetime.
|
||||
check_host(store, tx, &ctx.host, active)?;
|
||||
check_session_host(store, tx, &ctx.host)?;
|
||||
let (g, is_active) = generation(tx, &ctx.host, &ctx.generation_id)?;
|
||||
if g.job != ctx.binding {
|
||||
return Err(JournalError::PermissionDenied(
|
||||
@@ -629,6 +633,13 @@ impl SubjektivStore {
|
||||
if operation.expected_state!=j.state_token {return Err(conflict("stored Journal content changed; reread before applying"));}
|
||||
in_snapshot(&g,&operation.covered_ranges)?;
|
||||
if !operation.covered_ranges.is_empty() {check_ranges(tx,&j,&operation.covered_ranges,true)?;}
|
||||
// A fully checkpointed selection has no pending source to recover another
|
||||
// activity write if the Job subsequently fails. Receipts and position repair
|
||||
// remain available, but new activity belongs to the next selected prefix.
|
||||
if matches!(operation.action, JournalAction::Create {..}|JournalAction::Rewrite {..}|JournalAction::Fix {..}|JournalAction::ConfirmUnchanged)
|
||||
&& (j.process_order,j.process_byte)==prefix(&g.initial_journal,&g.selected_ranges)? {
|
||||
return Err(conflict("generation activity is already fully checkpointed; finish this Job before recording new input"));
|
||||
}
|
||||
let mut entry_id=None;
|
||||
match &operation.action {
|
||||
JournalAction::Create {title,body,ongoing,state,source_end,correction_refs}=> {
|
||||
@@ -680,6 +691,9 @@ impl SubjektivStore {
|
||||
if !contains(&merged,r.bytes) {return Err(invalid("checkpoint has input with no durable record correspondence"));}
|
||||
tx.execute("UPDATE journal_sources SET reflected_bytes=?3 WHERE journal_id=?1 AND source_order=?2",params![j.id,r.source.order,r.bytes.end])?;
|
||||
}
|
||||
// Advancing reflection without position would strand a terminal failed
|
||||
// Job: source selection would see no pending bytes to repair that memo.
|
||||
super::binding::require_saved_position(tx,&g.id)?;
|
||||
j.process_order=order;j.process_byte=byte;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,450 @@ use sha2::{Digest, Sha256};
|
||||
use std::sync::{Arc, Barrier};
|
||||
use std::thread;
|
||||
|
||||
#[test]
|
||||
fn explicit_erasure_releases_only_after_all_body_sessions_including_unadmitted_history_are_erased()
|
||||
{
|
||||
let f = Fixture::new();
|
||||
let subject = f.host.subject_id.clone();
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&subject, "runtime", "worker")
|
||||
.unwrap();
|
||||
let attr = f
|
||||
.store()
|
||||
.record_session_attribution(
|
||||
SubjectSessionAttribution::new(&subject, "runtime", "worker", "saved-before-admission")
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let unadmitted = f.store().bind_journal_session(&binding, attr).unwrap();
|
||||
assert!(!f.store().release_erased_journal_body(&f.host).unwrap());
|
||||
f.store()
|
||||
.erase_journal_session(&f.host, "erase-first")
|
||||
.unwrap();
|
||||
assert!(!f.store().release_erased_journal_body(&f.host).unwrap());
|
||||
assert!(f.store().journal_writer_active(&binding).unwrap());
|
||||
f.store()
|
||||
.erase_journal_session(&unadmitted, "erase-last")
|
||||
.unwrap();
|
||||
assert!(f.store().release_erased_journal_body(&unadmitted).unwrap());
|
||||
assert!(f.store().release_erased_journal_body(&unadmitted).unwrap());
|
||||
assert!(!f.store().journal_writer_active(&binding).unwrap());
|
||||
assert!(
|
||||
f.store()
|
||||
.journal_worker_was_released(&subject, "runtime", "worker")
|
||||
.unwrap()
|
||||
);
|
||||
let next = f
|
||||
.store()
|
||||
.claim_journal_writer(&subject, "runtime", "replacement")
|
||||
.unwrap();
|
||||
assert!(f.store().release_erased_journal_body(&f.host).unwrap());
|
||||
assert!(
|
||||
f.store().journal_writer_active(&next).unwrap(),
|
||||
"late erase retry cannot revoke another body"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn released_binding_preserves_open_and_receipts_but_never_revives_old_session_writes() {
|
||||
let mut f = Fixture::new();
|
||||
let subject = f.host.subject_id.clone();
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&subject, "runtime", "worker")
|
||||
.unwrap();
|
||||
assert!(
|
||||
f.store()
|
||||
.claim_journal_writer(&subject, "runtime", "other-worker")
|
||||
.is_err()
|
||||
);
|
||||
assert!(f.store().release_journal_writer(&binding).is_err());
|
||||
let (generation, context) = f.generation("first-body", f.ranges.clone());
|
||||
let create = f.operation(
|
||||
"create",
|
||||
open_action("A was proposed; user corrected a constraint; B selected and partly executed"),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let receipt = f.store().execute_journal(&context, create.clone()).unwrap();
|
||||
f.execute(
|
||||
&context,
|
||||
"position",
|
||||
JournalAction::UpdatePosition {
|
||||
current_position:
|
||||
"B: configuration changed; tests not executed; external result Unknown".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
assert!(f.store().release_journal_writer(&binding).is_err());
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["create".into()],
|
||||
},
|
||||
f.ranges.clone(),
|
||||
);
|
||||
// A saved checkpoint is not proof that the generation has stopped executing.
|
||||
assert!(f.store().release_journal_writer(&binding).is_err());
|
||||
f.close(&generation);
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
f.reopen();
|
||||
let old = f
|
||||
.store()
|
||||
.query_journal_operation(&f.host, &f.journal_id, "create")
|
||||
.unwrap();
|
||||
assert_eq!(old, Some(receipt.clone()));
|
||||
assert_eq!(
|
||||
f.store().execute_journal(&context, create).unwrap(),
|
||||
receipt
|
||||
);
|
||||
let reader = JournalHostContext::validated_reader(f.store(), &subject).unwrap();
|
||||
let snapshot = f
|
||||
.store()
|
||||
.journal_startup_snapshot(&reader, Some("source-session"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
snapshot.journals[0].entries[0].state,
|
||||
JournalEntryState::Open
|
||||
);
|
||||
assert!(snapshot.journals[0].current_position.contains("Unknown"));
|
||||
let new = f
|
||||
.store()
|
||||
.claim_journal_writer(&subject, "runtime", "worker")
|
||||
.unwrap();
|
||||
assert_ne!(new.binding_id, binding.binding_id);
|
||||
assert!(
|
||||
f.store()
|
||||
.bind_journal_session(&new, f.host.attribution.clone().unwrap())
|
||||
.is_err()
|
||||
);
|
||||
let late = source_range("source-session", 2, 64);
|
||||
assert!(matches!(
|
||||
f.store().accept_journal_sources(&f.host, "late", &[late]),
|
||||
Err(JournalError::PermissionDenied(_))
|
||||
));
|
||||
assert!(
|
||||
f.store()
|
||||
.prepare_journal_generation(
|
||||
&f.host,
|
||||
&f.journal_id,
|
||||
JobAttemptBinding {
|
||||
job_id: "late".into(),
|
||||
attempt_id: "late".into()
|
||||
},
|
||||
f.ranges.clone()
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let operation = JournalOperation {
|
||||
request_id: "late-position".into(),
|
||||
expected_state: snapshot.journals[0].state_token.clone(),
|
||||
action: JournalAction::UpdatePosition {
|
||||
current_position: "forged".into(),
|
||||
},
|
||||
covered_ranges: vec![],
|
||||
};
|
||||
assert!(f.store().execute_journal(&context, operation).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_rejects_unbound_legacy_body_sessions_even_without_admitted_sources() {
|
||||
let f = Fixture::new();
|
||||
let (generation, context) = f.generation("recorded-body", f.ranges.clone());
|
||||
f.execute(
|
||||
&context,
|
||||
"record",
|
||||
JournalAction::ConfirmUnchanged,
|
||||
f.ranges.clone(),
|
||||
);
|
||||
save_position(f.store(), &context, "position");
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
checkpoint(&["record"]),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.close(&generation);
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
let legacy = f
|
||||
.store()
|
||||
.record_session_attribution(
|
||||
SubjectSessionAttribution::new(
|
||||
&f.host.subject_id,
|
||||
"runtime",
|
||||
"worker",
|
||||
"legacy-not-admitted",
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
f.store().release_journal_writer(&binding),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
assert!(f.store().journal_writer_active(&binding).unwrap());
|
||||
// The Host must bind and reconcile this original Session, even if its canonical
|
||||
// history proves empty and therefore creates no Journal.
|
||||
f.store().bind_journal_session(&binding, legacy).unwrap();
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn predecessor_prefers_latest_journal_activity_not_session_attachment_order() {
|
||||
let f = Fixture::new();
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
let second = session_host(f.store(), &f.host.subject_id, "later-attached-session");
|
||||
let second_ranges = vec![source_range("later-attached-session", 0, 8)];
|
||||
let second_id = f
|
||||
.store()
|
||||
.accept_journal_sources(&second, "Earlier activity", &second_ranges)
|
||||
.unwrap();
|
||||
for (host, journal_id, ranges) in [
|
||||
(&f.host, &f.journal_id, &f.ranges),
|
||||
(&second, &second_id, &second_ranges),
|
||||
] {
|
||||
let job = JobAttemptBinding {
|
||||
job_id: journal_id.clone(),
|
||||
attempt_id: "recording".into(),
|
||||
};
|
||||
let generation = f
|
||||
.store()
|
||||
.prepare_journal_generation(host, journal_id, job.clone(), ranges.clone())
|
||||
.unwrap();
|
||||
let context = JournalJobContext::validated(f.store(), host, &generation.id, job).unwrap();
|
||||
let state = f
|
||||
.store()
|
||||
.read_journal(host, journal_id)
|
||||
.unwrap()
|
||||
.state_token;
|
||||
f.store()
|
||||
.execute_journal(
|
||||
&context,
|
||||
JournalOperation {
|
||||
request_id: "record".into(),
|
||||
expected_state: state,
|
||||
action: open_action("Unfinished activity"),
|
||||
covered_ranges: ranges.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let position = save_position(f.store(), &context, "position");
|
||||
f.store()
|
||||
.checkpoint_journal(
|
||||
&context,
|
||||
JournalOperation {
|
||||
request_id: "checkpoint".into(),
|
||||
expected_state: position.state_token,
|
||||
action: checkpoint(&["record"]),
|
||||
covered_ranges: ranges.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
f.store()
|
||||
.close_journal_generation(host, &generation.id, &generation.job)
|
||||
.unwrap();
|
||||
}
|
||||
// Stored timestamps model a later update to the earlier-attached Session without
|
||||
// using elapsed wall time as a synchronization or ordering assertion.
|
||||
f.store()
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
tx.execute(
|
||||
"UPDATE journals SET updated_at='2001-01-01T00:00:00Z' WHERE id=?1",
|
||||
[&f.journal_id],
|
||||
)?;
|
||||
tx.execute(
|
||||
"UPDATE journals SET updated_at='2000-01-01T00:00:00Z' WHERE id=?1",
|
||||
[&second_id],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let _empty = session_host(
|
||||
f.store(),
|
||||
&f.host.subject_id,
|
||||
"latest-attachment-no-journal",
|
||||
);
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
let replacement = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "replacement")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
f.store()
|
||||
.journal_predecessor_session(&replacement)
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("source-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_body_claims_from_independent_connections_bind_only_one_worker() {
|
||||
let f = Fixture::new();
|
||||
let subject = f
|
||||
.store()
|
||||
.create_subject(SubjectRole::new("claim-race").unwrap())
|
||||
.unwrap();
|
||||
let manager = FeatureStorage::new(f.temp.path().join("storage"));
|
||||
let workspace = manager.workspace("journal-tests").unwrap();
|
||||
let registration = SubjektivStore::register(&workspace).unwrap();
|
||||
let second = SubjektivStore::open(&workspace, ®istration).unwrap();
|
||||
let barrier = Arc::new(Barrier::new(3));
|
||||
let spawn = |store: SubjektivStore, worker: &'static str| {
|
||||
let barrier = barrier.clone();
|
||||
let subject = subject.id.clone();
|
||||
thread::spawn(move || {
|
||||
barrier.wait();
|
||||
store.claim_journal_writer(&subject, "runtime", worker)
|
||||
})
|
||||
};
|
||||
let first = spawn(f.store().clone(), "first");
|
||||
let second = spawn(second, "second");
|
||||
barrier.wait();
|
||||
let outcomes = [first.join().unwrap(), second.join().unwrap()];
|
||||
assert_eq!(outcomes.iter().filter(|r| r.is_ok()).count(), 1);
|
||||
let winner = outcomes.into_iter().find_map(Result::ok).unwrap();
|
||||
assert!(f.store().journal_writer_active(&winner).unwrap());
|
||||
let loser = if winner.worker_id == "first" {
|
||||
"second"
|
||||
} else {
|
||||
"first"
|
||||
};
|
||||
assert!(
|
||||
f.store()
|
||||
.claim_journal_writer(&subject.id, "runtime", loser)
|
||||
.is_err()
|
||||
);
|
||||
drop(workspace);
|
||||
manager.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_and_late_admission_are_serialized_by_the_persisted_binding_check() {
|
||||
let f = Fixture::new();
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
let (generation, context) = f.generation("release-race", f.ranges.clone());
|
||||
f.execute(
|
||||
&context,
|
||||
"record",
|
||||
open_action("unfinished"),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.execute(
|
||||
&context,
|
||||
"position",
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "remaining work".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["record".into()],
|
||||
},
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.close(&generation);
|
||||
let manager = FeatureStorage::new(f.temp.path().join("storage"));
|
||||
let workspace = manager.workspace("journal-tests").unwrap();
|
||||
let registration = SubjektivStore::register(&workspace).unwrap();
|
||||
let second = SubjektivStore::open(&workspace, ®istration).unwrap();
|
||||
let barrier = Arc::new(Barrier::new(3));
|
||||
let b = barrier.clone();
|
||||
let store = f.store().clone();
|
||||
let released = binding.clone();
|
||||
let release = thread::spawn(move || {
|
||||
b.wait();
|
||||
store.release_journal_writer(&released)
|
||||
});
|
||||
let b = barrier.clone();
|
||||
let host = f.host.clone();
|
||||
let late = source_range("source-session", 2, 16);
|
||||
let admission = thread::spawn(move || {
|
||||
b.wait();
|
||||
second.accept_journal_sources(&host, "late", &[late])
|
||||
});
|
||||
barrier.wait();
|
||||
let release = release.join().unwrap();
|
||||
let admission = admission.join().unwrap();
|
||||
assert_ne!(
|
||||
release.is_ok(),
|
||||
admission.is_ok(),
|
||||
"exactly one ordered effect may commit"
|
||||
);
|
||||
assert_eq!(
|
||||
f.store().journal_writer_active(&binding).unwrap(),
|
||||
admission.is_ok()
|
||||
);
|
||||
assert_eq!(
|
||||
f.store()
|
||||
.read_journal_source(&f.host, &f.journal_id, 2)
|
||||
.is_ok(),
|
||||
admission.is_ok()
|
||||
);
|
||||
drop(workspace);
|
||||
manager.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_snapshot_distinguishes_none_is_subject_local_and_bounds_utf8_at_storage() {
|
||||
let f = Fixture::new();
|
||||
let (_, context) = f.generation("snapshot", f.ranges.clone());
|
||||
f.execute(
|
||||
&context,
|
||||
"large-open",
|
||||
open_action(&"日本語\\\"\\n".repeat(6000)),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let reader = JournalHostContext::validated_reader(f.store(), &f.host.subject_id).unwrap();
|
||||
let snapshot = f
|
||||
.store()
|
||||
.journal_startup_snapshot(&reader, Some("source-session"))
|
||||
.unwrap();
|
||||
let encoded = serde_json::to_vec(&snapshot).unwrap();
|
||||
assert!(encoded.len() <= startup::STARTUP_MAX_BYTES);
|
||||
assert!(snapshot.journals[0].entries[0].omitted);
|
||||
assert!(snapshot.journals[0].entries[0].body.len() <= 2048);
|
||||
assert_eq!(snapshot.journals[0].source_session_id, "source-session");
|
||||
let other = f
|
||||
.store()
|
||||
.create_subject_with_behavior(
|
||||
SubjectRole::new("other").unwrap(),
|
||||
"behavior unchanged".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let other_reader = JournalHostContext::validated_reader(f.store(), &other.id).unwrap();
|
||||
assert!(matches!(
|
||||
f.store()
|
||||
.journal_startup_snapshot(&other_reader, Some("source-session")),
|
||||
Err(JournalError::PermissionDenied(_))
|
||||
));
|
||||
let none = f
|
||||
.store()
|
||||
.journal_startup_snapshot(&other_reader, None)
|
||||
.unwrap();
|
||||
assert!(none.journals.is_empty());
|
||||
assert!(!none.omitted);
|
||||
assert_eq!(
|
||||
f.store().subject(&other.id).unwrap().unwrap().behavior_md,
|
||||
"behavior unchanged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stable_legacy_run_references_are_accepted_without_relaxing_character_or_size_limits() {
|
||||
let f = Fixture::new();
|
||||
@@ -430,6 +874,111 @@ fn erasure_failure_rolls_back_tombstone_and_content_then_exact_retry_succeeds()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpointed_generation_rejects_late_activity_but_retains_saved_receipts() {
|
||||
let f = Fixture::new();
|
||||
let (generation, context) = f.generation("checkpointed-job", f.ranges.clone());
|
||||
let operation = f.operation("record", open_action("B partly executed"), f.ranges.clone());
|
||||
let receipt = f
|
||||
.store()
|
||||
.execute_journal(&context, operation.clone())
|
||||
.unwrap();
|
||||
let entry_id = receipt.entry_id.clone().unwrap();
|
||||
save_position(f.store(), &context, "position");
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
checkpoint(&["record"]),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let before = f.state();
|
||||
for (index, action) in [
|
||||
open_action("late create"),
|
||||
JournalAction::Rewrite {
|
||||
entry_id: entry_id.clone(),
|
||||
title: "late rewrite".into(),
|
||||
body: "late".into(),
|
||||
ongoing: "late".into(),
|
||||
},
|
||||
JournalAction::Fix {
|
||||
entry_id,
|
||||
source_end: f.ranges.last().unwrap().source.pointer.clone(),
|
||||
},
|
||||
JournalAction::ConfirmUnchanged,
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let late = f.operation(&format!("late-{index}"), action, f.ranges.clone());
|
||||
assert!(matches!(
|
||||
f.store().execute_journal(&context, late),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
assert_eq!(f.state(), before);
|
||||
}
|
||||
assert_eq!(
|
||||
f.store().execute_journal(&context, operation).unwrap(),
|
||||
receipt
|
||||
);
|
||||
f.close(&generation);
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_without_current_position_leaves_failed_job_input_recoverable_after_reopen() {
|
||||
let mut f = Fixture::new();
|
||||
let (failed, context) = f.generation("incomplete-position", f.ranges.clone());
|
||||
save_position(f.store(), &context, "outdated-position");
|
||||
f.execute(
|
||||
&context,
|
||||
"record",
|
||||
open_action("B selected; tests not run"),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let checkpoint_operation = f.operation("checkpoint", checkpoint(&["record"]), f.ranges.clone());
|
||||
let before = f.state();
|
||||
assert!(matches!(
|
||||
f.store().execute_journal(&context, checkpoint_operation),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
assert_eq!(f.state(), before);
|
||||
assert!(f.sources().iter().all(|source| source.reflected_bytes == 0));
|
||||
assert!(f.receipt("checkpoint").is_none());
|
||||
f.close(&failed);
|
||||
f.reopen();
|
||||
let (replacement, context) = f.generation("recover-position", f.ranges.clone());
|
||||
f.execute(
|
||||
&context,
|
||||
"recovered-position",
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "B partly performed; tests unperformed; external result Unknown"
|
||||
.into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
// Use the saved record receipt rather than duplicating the old activity entry.
|
||||
f.execute(
|
||||
&context,
|
||||
"checkpoint",
|
||||
checkpoint(&["record"]),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
f.close(&replacement);
|
||||
let binding = f
|
||||
.store()
|
||||
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
|
||||
.unwrap();
|
||||
assert_eq!(f.entries().len(), 1);
|
||||
assert_eq!(f.entries()[0].state, JournalEntryState::Open);
|
||||
assert!(f.state().0.current_position.contains("Unknown"));
|
||||
f.store().release_journal_writer(&binding).unwrap();
|
||||
assert!(!f.store().journal_writer_active(&binding).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_service_completion_requires_checkpoint_and_replays_saved_completion_after_new_generation() {
|
||||
let mut f = Fixture::new();
|
||||
@@ -476,22 +1025,43 @@ fn job_service_completion_requires_checkpoint_and_replays_saved_completion_after
|
||||
service::finish(f.store(), &context),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
service::execute(
|
||||
f.store(),
|
||||
&context,
|
||||
transport::JournalJobRequest::Execute {
|
||||
operation: JournalOperation {
|
||||
request_id: "service-checkpoint".into(),
|
||||
expected_state: receipt.state_token,
|
||||
action: JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["service-create".into()],
|
||||
},
|
||||
covered_ranges: f.ranges.clone(),
|
||||
},
|
||||
let checkpoint_operation = JournalOperation {
|
||||
request_id: "service-checkpoint".into(),
|
||||
expected_state: receipt.state_token,
|
||||
action: JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["service-create".into()],
|
||||
},
|
||||
|_| unreachable!(),
|
||||
)
|
||||
.unwrap();
|
||||
covered_ranges: f.ranges.clone(),
|
||||
};
|
||||
// Reject the checkpoint atomically: failed/terminal Jobs must leave pending input
|
||||
// recoverable rather than stranding a fully reflected prefix without a position memo.
|
||||
assert!(matches!(
|
||||
service::execute(
|
||||
f.store(),
|
||||
&context,
|
||||
transport::JournalJobRequest::Execute {
|
||||
operation: checkpoint_operation
|
||||
},
|
||||
|_| unreachable!(),
|
||||
),
|
||||
Err(JournalError::Conflict(_))
|
||||
));
|
||||
assert!(f.sources().iter().all(|source| source.reflected_bytes == 0));
|
||||
assert!(f.receipt("service-checkpoint").is_none());
|
||||
f.execute(
|
||||
&context,
|
||||
"service-position",
|
||||
JournalAction::UpdatePosition {
|
||||
current_position: "ongoing; work remains".into(),
|
||||
},
|
||||
vec![],
|
||||
);
|
||||
f.execute(
|
||||
&context,
|
||||
"service-checkpoint",
|
||||
checkpoint(&["service-create"]),
|
||||
f.ranges.clone(),
|
||||
);
|
||||
let completion = service::finish(f.store(), &context).unwrap();
|
||||
assert_eq!(completion.generation_id, generation.id);
|
||||
assert_eq!((completion.process_order, completion.process_byte), (2, 0));
|
||||
@@ -685,7 +1255,7 @@ fn retention_release_requires_checkpoint_not_saved_entry_or_unavailability() {
|
||||
let f = Fixture::new();
|
||||
let (_, job) = f.generation("retained-job", f.ranges.clone());
|
||||
let operation = f.operation("record", open_action("ongoing activity"), f.ranges.clone());
|
||||
let receipt = f.store().execute_journal(&job, operation).unwrap();
|
||||
f.store().execute_journal(&job, operation).unwrap();
|
||||
assert!(f.store().journal_retention_required(&f.host).unwrap());
|
||||
f.store()
|
||||
.mark_journal_sources_unavailable(&f.host, &f.journal_id, &f.ranges)
|
||||
@@ -694,6 +1264,7 @@ fn retention_release_requires_checkpoint_not_saved_entry_or_unavailability() {
|
||||
f.store()
|
||||
.mark_journal_sources_available(&f.host, &f.journal_id, &f.ranges)
|
||||
.unwrap();
|
||||
let receipt = save_position(f.store(), &job, "retained-position");
|
||||
f.store()
|
||||
.checkpoint_journal(
|
||||
&job,
|
||||
@@ -805,8 +1376,7 @@ fn recording_materials_resume_exact_saved_byte_prefix_after_reopen() {
|
||||
.unwrap();
|
||||
let job = JournalJobContext::validated(f.store(), &host, &generation.id, binding).unwrap();
|
||||
let state = f.store().read_journal(&host, &id).unwrap().state_token;
|
||||
let receipt = f
|
||||
.store()
|
||||
f.store()
|
||||
.execute_journal(
|
||||
&job,
|
||||
JournalOperation {
|
||||
@@ -817,6 +1387,7 @@ fn recording_materials_resume_exact_saved_byte_prefix_after_reopen() {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let receipt = save_position(f.store(), &job, "recording-position");
|
||||
f.store()
|
||||
.checkpoint_journal(
|
||||
&job,
|
||||
@@ -891,7 +1462,12 @@ impl Fixture {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let host = session_host(&store, &subject.id, "source-session");
|
||||
let attribution =
|
||||
SubjectSessionAttribution::new(&subject.id, "runtime", "worker", "source-session")
|
||||
.unwrap();
|
||||
store
|
||||
.record_session_attribution(attribution.clone())
|
||||
.unwrap();
|
||||
let (manager, workspace, store) = if upgrade {
|
||||
let has_journals: bool = store.database.try_with_connection::<_, JournalError>(|connection| {
|
||||
Ok(connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='journals')", [], |row| row.get(0))?)
|
||||
@@ -905,15 +1481,16 @@ impl Fixture {
|
||||
let workspace = manager.workspace("journal-tests").unwrap();
|
||||
let registration = SubjektivStore::register(&workspace).unwrap();
|
||||
let store = SubjektivStore::open(&workspace, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject));
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
assert_eq!(
|
||||
store.session_attribution("source-session").unwrap(),
|
||||
host.attribution
|
||||
Some(attribution)
|
||||
);
|
||||
(manager, workspace, store)
|
||||
} else {
|
||||
(manager, workspace, store)
|
||||
};
|
||||
let host = session_host(&store, &subject.id, "source-session");
|
||||
let ranges = sizes
|
||||
.iter()
|
||||
.enumerate()
|
||||
@@ -1070,10 +1647,11 @@ impl Drop for Fixture {
|
||||
fn session_host(store: &SubjektivStore, subject: &str, session: &str) -> JournalHostContext {
|
||||
let attribution =
|
||||
SubjectSessionAttribution::new(subject, "runtime", "worker", session).unwrap();
|
||||
store
|
||||
.record_session_attribution(attribution.clone())
|
||||
let attribution = store.record_session_attribution(attribution).unwrap();
|
||||
let binding = store
|
||||
.claim_journal_writer(subject, "runtime", "worker")
|
||||
.unwrap();
|
||||
JournalHostContext::validated_session(store, attribution).unwrap()
|
||||
store.bind_journal_session(&binding, attribution).unwrap()
|
||||
}
|
||||
|
||||
fn source_range(session: &str, order: u64, total_bytes: u64) -> JournalSourceRange {
|
||||
@@ -1127,6 +1705,32 @@ fn fixed_action(body: &str, range: &JournalSourceRange, corrections: Vec<String>
|
||||
}
|
||||
}
|
||||
|
||||
fn save_position(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
request_id: &str,
|
||||
) -> JournalReceipt {
|
||||
let generation = store
|
||||
.read_journal_generation(&context.host, &context.generation_id)
|
||||
.unwrap();
|
||||
let (journal, _) = store
|
||||
.journal_resume_state(&context.host, &generation.journal_id)
|
||||
.unwrap();
|
||||
store
|
||||
.execute_journal(
|
||||
context,
|
||||
JournalOperation {
|
||||
request_id: request_id.into(),
|
||||
expected_state: journal.state_token,
|
||||
action: JournalAction::UpdatePosition {
|
||||
current_position: journal.current_position,
|
||||
},
|
||||
covered_ranges: vec![],
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn checkpoint(ids: &[&str]) -> JournalAction {
|
||||
JournalAction::Checkpoint {
|
||||
record_operation_ids: ids.iter().map(|id| (*id).into()).collect(),
|
||||
@@ -1249,6 +1853,7 @@ fn sequential_open_rewrite_fix_and_correction_preserve_fixed_history() {
|
||||
assert_eq!(entries[1].correction_refs, vec![first_id]);
|
||||
assert_eq!(entries[1].previous_fixed_end, fixed.source_end);
|
||||
assert_eq!(fixture.state().0.process_order, 0);
|
||||
save_position(fixture.store(), &ctx, "final-position");
|
||||
let cp = fixture.execute(
|
||||
&ctx,
|
||||
"checkpoint",
|
||||
@@ -1966,6 +2571,7 @@ fn checkpoint_requires_saved_record_correspondence_and_never_position_only_evide
|
||||
JournalAction::ConfirmUnchanged,
|
||||
vec![fixture.ranges[1].clone()],
|
||||
);
|
||||
save_position(fixture.store(), &ctx, "checked-position");
|
||||
let receipt = fixture.execute(
|
||||
&ctx,
|
||||
"valid-checkpoint",
|
||||
@@ -2017,6 +2623,7 @@ fn huge_source_partial_generations_checkpoint_only_read_prefix_and_fix_only_at_e
|
||||
fixture.store().execute_journal(&first_ctx, unread_end),
|
||||
Err(JournalError::InvalidInput(_))
|
||||
));
|
||||
save_position(fixture.store(), &first_ctx, "partial-position");
|
||||
let cp = fixture.execute(
|
||||
&first_ctx,
|
||||
"partial-checkpoint",
|
||||
@@ -2082,6 +2689,7 @@ fn huge_source_partial_generations_checkpoint_only_read_prefix_and_fix_only_at_e
|
||||
vec![],
|
||||
);
|
||||
assert_eq!(fixture.state().0.process_byte, head.bytes.end);
|
||||
save_position(fixture.store(), &next_ctx, "tail-position");
|
||||
let cp = fixture.execute(
|
||||
&next_ctx,
|
||||
"tail-checkpoint",
|
||||
@@ -2913,6 +3521,7 @@ fn failure_before_receipt_rolls_back_record_and_checkpoint_then_exact_retry_succ
|
||||
fixture.store().execute_journal(&ctx, record).unwrap(),
|
||||
saved
|
||||
);
|
||||
save_position(fixture.store(), &ctx, "fault-position");
|
||||
let cp = fixture.operation(
|
||||
"fault-checkpoint",
|
||||
checkpoint(&["fault-record"]),
|
||||
@@ -2994,6 +3603,7 @@ fn per_operation_reopen_recovers_a_realistic_investigation_and_later_correction_
|
||||
assert!(fixture.state().1.is_none());
|
||||
assert_eq!(fixture.state().0.process_order, 0);
|
||||
let fixed_before_correction = fixture.entries()[0].clone();
|
||||
save_position(fixture.store(), &first_ctx, "investigation-fixed-position");
|
||||
fixture.execute_and_reopen(
|
||||
&first_ctx,
|
||||
"investigation-checkpoint",
|
||||
@@ -3117,6 +3727,7 @@ fn split_input_and_replacement_job_reconcile_old_record_coverage_without_duplica
|
||||
.unwrap();
|
||||
assert_eq!(coverage.items, vec![whole.clone()]);
|
||||
let (_, replacement) = fixture.generation("replacement-job", vec![whole.clone()]);
|
||||
save_position(fixture.store(), &replacement, "replacement-position");
|
||||
let cp = fixture.operation("recover-reflection", checkpoint(&["record"]), vec![whole]);
|
||||
let result = fixture
|
||||
.store()
|
||||
@@ -3200,86 +3811,95 @@ fn serialized_metadata_and_text_budgets_preserve_continuation_and_availability_r
|
||||
|
||||
#[test]
|
||||
fn journal_migration_failure_rolls_back_ddl_and_retries_without_changing_existing_subject() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("storage");
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = scope
|
||||
.register(feature_storage::FeatureRegistration::new(
|
||||
crate::SUBJEKTIV_FEATURE_ID,
|
||||
&crate::MIGRATIONS[..7],
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
SubjectRole::new("preserved").unwrap(),
|
||||
"Original behavior".into(),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
tx.execute_batch("CREATE TABLE journal_entries(injected INTEGER)")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = SubjektivStore::register(&scope).unwrap();
|
||||
assert!(matches!(
|
||||
SubjektivStore::open(&scope, ®istration),
|
||||
Err(crate::SubjektivError::Storage(
|
||||
feature_storage::FeatureStorageError::MigrationFailed { .. }
|
||||
))
|
||||
));
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = scope
|
||||
.register(feature_storage::FeatureRegistration::new(
|
||||
crate::SUBJEKTIV_FEATURE_ID,
|
||||
&crate::MIGRATIONS[..7],
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
assert!(!tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='journals')",
|
||||
[],
|
||||
|r| r.get::<_, bool>(0)
|
||||
)?);
|
||||
tx.execute_batch("DROP TABLE journal_entries")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = SubjektivStore::register(&scope).unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
let host = session_host(&store, &subject.id, "post-migration-session");
|
||||
let range = source_range("post-migration-session", 0, 8);
|
||||
let id = store
|
||||
.accept_journal_sources(&host, "Recovered migration", &[range])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.read_journal(&host, &id).unwrap().source_session_id,
|
||||
"post-migration-session"
|
||||
);
|
||||
for (baseline, collision, rolled_back) in [
|
||||
(&crate::MIGRATIONS[..7], "journal_entries", "journals"),
|
||||
(
|
||||
&crate::MIGRATIONS[..9],
|
||||
"journal_session_bindings",
|
||||
"journal_writer_bindings",
|
||||
),
|
||||
] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("storage");
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = scope
|
||||
.register(feature_storage::FeatureRegistration::new(
|
||||
crate::SUBJEKTIV_FEATURE_ID,
|
||||
baseline,
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
SubjectRole::new("preserved").unwrap(),
|
||||
"Original behavior".into(),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
tx.execute_batch(&format!("CREATE TABLE {collision}(injected INTEGER)"))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = SubjektivStore::register(&scope).unwrap();
|
||||
assert!(matches!(
|
||||
SubjektivStore::open(&scope, ®istration),
|
||||
Err(crate::SubjektivError::Storage(
|
||||
feature_storage::FeatureStorageError::MigrationFailed { .. }
|
||||
))
|
||||
));
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = scope
|
||||
.register(feature_storage::FeatureRegistration::new(
|
||||
crate::SUBJEKTIV_FEATURE_ID,
|
||||
baseline,
|
||||
))
|
||||
.unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
store
|
||||
.database
|
||||
.try_transaction::<_, JournalError>(|tx| {
|
||||
assert!(!tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name=?1)",
|
||||
[rolled_back],
|
||||
|r| r.get::<_, bool>(0)
|
||||
)?);
|
||||
tx.execute_batch(&format!("DROP TABLE {collision}"))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
drop(store);
|
||||
drop(scope);
|
||||
manager.shutdown().unwrap();
|
||||
drop(manager);
|
||||
let manager = FeatureStorage::new(&root);
|
||||
let scope = manager.scope("migration-rollback").unwrap();
|
||||
let registration = SubjektivStore::register(&scope).unwrap();
|
||||
let store = SubjektivStore::open(&scope, ®istration).unwrap();
|
||||
assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone()));
|
||||
let host = session_host(&store, &subject.id, "post-migration-session");
|
||||
let range = source_range("post-migration-session", 0, 8);
|
||||
let id = store
|
||||
.accept_journal_sources(&host, "Recovered migration", &[range])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.read_journal(&host, &id).unwrap().source_session_id,
|
||||
"post-migration-session"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3365,6 +3985,7 @@ fn late_gap_admission_and_partial_updates_reject_stale_source_page_cursors() {
|
||||
.list_journal_sources(&fixture.host, &fixture.journal_id, fresh.next.as_deref(), 2)
|
||||
.is_ok()
|
||||
);
|
||||
save_position(fixture.store(), &job, "listing-position");
|
||||
fixture.execute(
|
||||
&job,
|
||||
"checkpoint",
|
||||
|
||||
@@ -953,11 +953,12 @@ mod tests {
|
||||
store.session_attribution("session-1").unwrap().unwrap(),
|
||||
stored_attribution
|
||||
);
|
||||
let host = crate::journal::JournalHostContext::validated_session(
|
||||
&store,
|
||||
stored_attribution.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let binding = store
|
||||
.claim_journal_writer(&subject_id, "runtime-1", "worker-1")
|
||||
.unwrap();
|
||||
let host = store
|
||||
.bind_journal_session(&binding, stored_attribution.clone())
|
||||
.unwrap();
|
||||
let range = crate::journal::JournalSourceRange {
|
||||
source: crate::journal::JournalSource {
|
||||
session_id: stored_attribution.session_id.clone(),
|
||||
|
||||
@@ -613,6 +613,16 @@ static MIGRATIONS: &[FeatureMigration] = &[
|
||||
"fence erased journal Sessions and permit explicit content erasure",
|
||||
journal::migrate_lifecycle,
|
||||
),
|
||||
FeatureMigration::new(
|
||||
10,
|
||||
"fence journal writers by explicit body binding lifetime",
|
||||
journal::migrate_binding,
|
||||
),
|
||||
FeatureMigration::new(
|
||||
11,
|
||||
"add immutable candidate Journal context links",
|
||||
journal::memory_links::migrate,
|
||||
),
|
||||
];
|
||||
|
||||
pub const REGISTRATION: FeatureRegistration =
|
||||
|
||||
@@ -3034,6 +3034,7 @@ impl App {
|
||||
session_store::SystemItem::FeatureInvocationResult { body, .. }
|
||||
| session_store::SystemItem::FileAttachment { body, .. }
|
||||
| session_store::SystemItem::SkillActivation { body, .. }
|
||||
| session_store::SystemItem::JournalStartup { body, .. }
|
||||
| session_store::SystemItem::ResidentSummaryRefresh { body, .. }
|
||||
| session_store::SystemItem::SubjectBehaviorRefresh { body, .. }
|
||||
| session_store::SystemItem::Interrupt { body, .. } => {
|
||||
|
||||
@@ -23,6 +23,7 @@ http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
||||
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
|
||||
|
||||
[dependencies]
|
||||
agen.workspace = true
|
||||
async-trait.workspace = true
|
||||
base64.workspace = true
|
||||
axum = { workspace = true, optional = true }
|
||||
|
||||
@@ -567,6 +567,14 @@ impl fmt::Debug for WorkerProtocolTransport {
|
||||
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||
fn backend_id(&self) -> &str;
|
||||
|
||||
/// Must reserve the actual Controller, not infer Idle from a catalog snapshot.
|
||||
fn subject_release(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
Err("execution backend cannot reserve the actual Subject Controller".into())
|
||||
}
|
||||
|
||||
fn fetch_workspace_config(
|
||||
&self,
|
||||
_request: WorkspaceConfigFetchRequest,
|
||||
@@ -831,6 +839,13 @@ impl WorkerExecutionBackendRef {
|
||||
self.backend.fetch_workspace_config(request)
|
||||
}
|
||||
|
||||
pub(crate) fn subject_release(
|
||||
&self,
|
||||
request: &runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
self.backend.subject_release(request)
|
||||
}
|
||||
|
||||
pub(crate) fn worker_session(
|
||||
&self,
|
||||
request: WorkerSessionObservationRequest,
|
||||
|
||||
@@ -479,6 +479,10 @@ pub(crate) struct OpenedFsRuntimeStore {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct PersistedRuntimeState {
|
||||
pub(crate) subject_releases: Vec<(
|
||||
runtime_api::SubjectReleaseRequest,
|
||||
runtime_api::SubjectReleaseState,
|
||||
)>,
|
||||
pub(crate) display_name: Option<String>,
|
||||
pub(crate) status: RuntimeStatus,
|
||||
pub(crate) next_diagnostic_id: u64,
|
||||
@@ -1915,6 +1919,11 @@ fn migrate_runtime_store_in_place(
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
struct RuntimeSnapshot {
|
||||
#[serde(default)]
|
||||
subject_releases: Vec<(
|
||||
runtime_api::SubjectReleaseRequest,
|
||||
runtime_api::SubjectReleaseState,
|
||||
)>,
|
||||
schema_version: u32,
|
||||
display_name: Option<String>,
|
||||
backend: RuntimeBackendKind,
|
||||
@@ -1946,6 +1955,7 @@ fn record_worker_load_diagnostic(
|
||||
impl RuntimeSnapshot {
|
||||
fn from_persisted(state: &PersistedRuntimeState) -> Self {
|
||||
Self {
|
||||
subject_releases: state.subject_releases.clone(),
|
||||
schema_version: SCHEMA_VERSION,
|
||||
display_name: state.display_name.clone(),
|
||||
backend: RuntimeBackendKind::FsStore,
|
||||
@@ -1983,6 +1993,7 @@ impl RuntimeSnapshot {
|
||||
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
|
||||
) -> PersistedRuntimeState {
|
||||
PersistedRuntimeState {
|
||||
subject_releases: self.subject_releases,
|
||||
display_name: self.display_name,
|
||||
status: self.status,
|
||||
next_diagnostic_id: self.next_diagnostic_id,
|
||||
|
||||
@@ -2102,7 +2102,10 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
||||
{
|
||||
return Some("workers:input");
|
||||
}
|
||||
if path.ends_with("/stop") || path.ends_with("/cancel") {
|
||||
if path.ends_with("/stop")
|
||||
|| path.ends_with("/cancel")
|
||||
|| (path == "/v1/subject-release" && *method == Method::POST)
|
||||
{
|
||||
return Some("workers:stop");
|
||||
}
|
||||
if path == runtime_api::RUNTIME_ROUTE_PROTOCOL_WS {
|
||||
|
||||
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
}
|
||||
}
|
||||
|
||||
async fn subject_release(
|
||||
&self,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, runtime_api::RuntimeApiError> {
|
||||
let scope = material_scope(&self.state, &request.workspace_id)?;
|
||||
self.state
|
||||
.runtime
|
||||
.subject_release_scoped(&scope, request)
|
||||
.map_err(RuntimeHttpRestError::runtime)
|
||||
.map_err(api_error)
|
||||
}
|
||||
|
||||
async fn session_material_pin(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialPinRequest,
|
||||
|
||||
@@ -356,6 +356,109 @@ impl Drop for WorkerOperationLease {
|
||||
}
|
||||
|
||||
impl Runtime {
|
||||
/// Reserves the actual Controller while the Host reconciles final saved Journal material.
|
||||
/// A Runtime restart loses proof of a reservation, never reconstructs it from Worker status.
|
||||
pub fn subject_release_scoped(
|
||||
&self,
|
||||
scope: &RuntimeWorkspaceScope,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, RuntimeError> {
|
||||
use runtime_api::SubjectReleaseAction as Action;
|
||||
if request.workspace_id != scope.workspace_id
|
||||
|| request.binding_id.trim().is_empty()
|
||||
|| request.operation_id.trim().is_empty()
|
||||
|| request.binding_id.len() > 512
|
||||
|| request.operation_id.len() > 512
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"invalid Subject release identity".into(),
|
||||
));
|
||||
}
|
||||
let worker_ref = WorkerRef::new(request.worker_id);
|
||||
let lease = self.worker_operation_lock(request.worker_id)?;
|
||||
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
self.ensure_worker_in_workspace(scope, &worker_ref)?;
|
||||
let backend = {
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
let worker = state.worker(&worker_ref)?;
|
||||
if worker.has_pending_lifecycle_operation() {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject lifecycle is unresolved".into(),
|
||||
));
|
||||
}
|
||||
if let Some((saved, _)) = state
|
||||
.subject_release_operations
|
||||
.get(&(request.worker_id, request.operation_id.clone()))
|
||||
{
|
||||
if saved.operation_id != request.operation_id
|
||||
|| saved.binding_id != request.binding_id
|
||||
|| saved.workspace_id != request.workspace_id
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"different Subject release is retained".into(),
|
||||
));
|
||||
}
|
||||
} else if request.action != Action::Reserve {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject reservation is absent or lost; retain occupancy".into(),
|
||||
));
|
||||
} else {
|
||||
state.ensure_subject_not_reserved(&worker_ref, false)?;
|
||||
}
|
||||
state.execution_backend.clone().ok_or_else(|| {
|
||||
RuntimeError::InvalidRequest("actual Controller reservation is unavailable".into())
|
||||
})?
|
||||
};
|
||||
let key = (request.worker_id, request.operation_id.clone());
|
||||
{
|
||||
let mut state = self.lock()?;
|
||||
if let Some((_, saved)) = state.subject_release_operations.get(&key) {
|
||||
if *saved == runtime_api::SubjectReleaseState::Cancelled {
|
||||
if request.action != Action::Cancel {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"cancelled release cannot be rearmed; use a new operation".into(),
|
||||
));
|
||||
}
|
||||
return Ok(runtime_api::SubjectReleaseResponse {
|
||||
operation_id: request.operation_id,
|
||||
binding_id: request.binding_id,
|
||||
state: *saved,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Persist admission exclusion before any Controller ACK, under the Worker op
|
||||
// lock. Restart retains this fence without manufacturing disconnect proof.
|
||||
state.subject_release_operations.insert(
|
||||
key.clone(),
|
||||
(request.clone(), runtime_api::SubjectReleaseState::Reserved),
|
||||
);
|
||||
}
|
||||
state.persist_runtime_snapshot()?;
|
||||
}
|
||||
let response = backend
|
||||
.subject_release(&request)
|
||||
.map_err(RuntimeError::InvalidRequest)?;
|
||||
if response.binding_id != request.binding_id
|
||||
|| response.operation_id != request.operation_id
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject release acknowledgment mismatch".into(),
|
||||
));
|
||||
}
|
||||
let mut state = self.lock()?;
|
||||
let before = state
|
||||
.subject_release_operations
|
||||
.insert(key.clone(), (request.clone(), response.state));
|
||||
if let Err(error) = state.persist_runtime_snapshot() {
|
||||
if let Some(before) = before {
|
||||
state.subject_release_operations.insert(key, before);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn reserve_workdir_use<'a>(
|
||||
&self,
|
||||
ids: impl IntoIterator<Item = &'a str>,
|
||||
@@ -1961,6 +2064,8 @@ impl Runtime {
|
||||
let _operation_guard = operation_lock
|
||||
.lock()
|
||||
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
self.lock()?
|
||||
.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
if let Some(scope) = scope {
|
||||
let mut state = self.lock()?;
|
||||
if !state
|
||||
@@ -2332,6 +2437,8 @@ impl Runtime {
|
||||
mode: WorkerRestoreMode,
|
||||
preparation_authorized: bool,
|
||||
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
|
||||
self.lock()?
|
||||
.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
let owner = {
|
||||
let state = self.lock()?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
@@ -2586,6 +2693,8 @@ impl Runtime {
|
||||
worker_ref: &WorkerRef,
|
||||
mode: WorkerRestoreMode,
|
||||
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
|
||||
self.lock()?
|
||||
.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
let fresh_operation_id = {
|
||||
let state = self.lock()?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
@@ -3007,6 +3116,7 @@ impl Runtime {
|
||||
|
||||
fn ensure_worker_operation_settled(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
|
||||
let state = self.lock()?;
|
||||
state.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
if let Some(operation) = worker.pending_lifecycle_operation_name() {
|
||||
return Err(RuntimeError::WorkerExecutionUnavailable {
|
||||
@@ -3803,6 +3913,7 @@ impl Runtime {
|
||||
self.ensure_worker_in_workspace(scope, worker_ref)?;
|
||||
}
|
||||
|
||||
self.lock()?.ensure_subject_not_reserved(worker_ref, true)?;
|
||||
let (backend, pending_stop, pending_restore, status) = {
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
@@ -3980,6 +4091,7 @@ impl Runtime {
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
state.ensure_worker_ref(worker_ref)?;
|
||||
state.ensure_subject_not_reserved(worker_ref, true)?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
if let Some(operation) = worker.pending_lifecycle_operation_name() {
|
||||
return Err(RuntimeError::InvalidRequest(format!(
|
||||
@@ -4270,7 +4382,10 @@ impl Runtime {
|
||||
.workers
|
||||
.values()
|
||||
.filter(|worker| {
|
||||
worker.restore_guard.active_request_id.is_none()
|
||||
state
|
||||
.ensure_subject_not_reserved(&worker.worker_ref, false)
|
||||
.is_ok()
|
||||
&& worker.restore_guard.active_request_id.is_none()
|
||||
&& worker.pending_restore.is_none()
|
||||
&& worker.pending_stop.is_none()
|
||||
&& worker.execution_bound
|
||||
@@ -4313,7 +4428,12 @@ impl Runtime {
|
||||
state
|
||||
.workers
|
||||
.values()
|
||||
.filter(|worker| worker.pending_stop.is_none())
|
||||
.filter(|worker| {
|
||||
worker.pending_stop.is_none()
|
||||
&& state
|
||||
.ensure_subject_not_reserved(&worker.worker_ref, false)
|
||||
.is_ok()
|
||||
})
|
||||
.filter_map(|worker| {
|
||||
worker
|
||||
.pending_restore
|
||||
@@ -5362,6 +5482,13 @@ struct RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
next_diagnostic_id: u64,
|
||||
workers: BTreeMap<WorkerId, WorkerRecord>,
|
||||
subject_release_operations: BTreeMap<
|
||||
(WorkerId, String),
|
||||
(
|
||||
runtime_api::SubjectReleaseRequest,
|
||||
runtime_api::SubjectReleaseState,
|
||||
),
|
||||
>,
|
||||
workdir_operations: BTreeMap<String, WorkdirOperation>,
|
||||
workspace_owners: BTreeMap<String, String>,
|
||||
config_bundles: BTreeMap<String, ConfigBundle>,
|
||||
@@ -5392,6 +5519,7 @@ impl RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
next_diagnostic_id: 1,
|
||||
workers: BTreeMap::new(),
|
||||
subject_release_operations: BTreeMap::new(),
|
||||
workdir_operations: BTreeMap::new(),
|
||||
workspace_owners: BTreeMap::new(),
|
||||
config_bundles: BTreeMap::new(),
|
||||
@@ -5423,6 +5551,7 @@ impl RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
next_diagnostic_id: 1,
|
||||
workers: BTreeMap::new(),
|
||||
subject_release_operations: BTreeMap::new(),
|
||||
workdir_operations: BTreeMap::new(),
|
||||
workspace_owners: BTreeMap::new(),
|
||||
config_bundles: BTreeMap::new(),
|
||||
@@ -5568,6 +5697,16 @@ impl RuntimeState {
|
||||
workspace_backend_resource_clients: BTreeMap::new(),
|
||||
next_diagnostic_id,
|
||||
workers,
|
||||
subject_release_operations: persisted
|
||||
.subject_releases
|
||||
.into_iter()
|
||||
.map(|(request, state)| {
|
||||
(
|
||||
(request.worker_id, request.operation_id.clone()),
|
||||
(request, state),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
workdir_operations: BTreeMap::new(),
|
||||
config_bundles: BTreeMap::new(),
|
||||
workspace_config_latest: BTreeMap::new(),
|
||||
@@ -5588,6 +5727,7 @@ impl RuntimeState {
|
||||
#[cfg(feature = "fs-store")]
|
||||
fn persisted_state(&self) -> PersistedRuntimeState {
|
||||
PersistedRuntimeState {
|
||||
subject_releases: self.subject_release_operations.values().cloned().collect(),
|
||||
display_name: self.display_name.clone(),
|
||||
status: self.status,
|
||||
next_diagnostic_id: self.next_diagnostic_id,
|
||||
@@ -5731,6 +5871,28 @@ impl RuntimeState {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_subject_not_reserved(
|
||||
&self,
|
||||
worker: &WorkerRef,
|
||||
allow_disconnected: bool,
|
||||
) -> Result<(), RuntimeError> {
|
||||
use runtime_api::SubjectReleaseState as State;
|
||||
if self
|
||||
.subject_release_operations
|
||||
.iter()
|
||||
.any(|((id, _), (_, state))| {
|
||||
*id == worker.worker_id
|
||||
&& (*state == State::Reserved
|
||||
|| (!allow_disconnected && *state == State::Disconnected))
|
||||
})
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject Controller is reserved for release or disconnected".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_worker_ref(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
|
||||
if !self.workers.contains_key(&worker_ref.worker_id) {
|
||||
return Err(RuntimeError::WorkerNotFound {
|
||||
@@ -6777,8 +6939,7 @@ fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), R
|
||||
.validate()
|
||||
.map_err(|message| RuntimeError::InvalidRequest(message.into()))?;
|
||||
if request.workspace_api.is_none()
|
||||
|| ((binding.subjektiv_consolidation || binding.subjektiv_journal)
|
||||
&& !request.subjektiv_attached)
|
||||
|| (binding.subjektiv_consolidation && !request.subjektiv_attached)
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest("Backend Job capability requires Workspace authority and consolidation requires trusted subject attachment".into()));
|
||||
}
|
||||
@@ -8206,6 +8367,21 @@ mod tests {
|
||||
}
|
||||
|
||||
impl WorkerExecutionBackend for TestExecutionBackend {
|
||||
fn subject_release(
|
||||
&self,
|
||||
request: &runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
use runtime_api::{SubjectReleaseAction as A, SubjectReleaseState as S};
|
||||
Ok(runtime_api::SubjectReleaseResponse {
|
||||
operation_id: request.operation_id.clone(),
|
||||
binding_id: request.binding_id.clone(),
|
||||
state: match request.action {
|
||||
A::Reserve => S::Reserved,
|
||||
A::Cancel => S::Cancelled,
|
||||
A::Disconnect => S::Disconnected,
|
||||
},
|
||||
})
|
||||
}
|
||||
fn cleanup_working_directory(
|
||||
&self,
|
||||
_id: &str,
|
||||
@@ -13106,6 +13282,83 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
// Persistence component test; real Controller reservation has separate coverage.
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn subject_release_fence_survives_reopen_without_automatic_or_explicit_restore() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path())
|
||||
.with_runtime_id("test-runtime");
|
||||
let scoped = scope("workspace-a", "server-a");
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||
options.clone(),
|
||||
Arc::new(TestExecutionBackend::default()),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let worker = runtime
|
||||
.create_worker_scoped(
|
||||
&scoped,
|
||||
scoped_task_request("release durable", "workspace-a"),
|
||||
)
|
||||
.unwrap();
|
||||
let mut request = runtime_api::SubjectReleaseRequest {
|
||||
workspace_id: "workspace-a".into(),
|
||||
worker_id: worker.worker_id,
|
||||
binding_id: "writer-1".into(),
|
||||
operation_id: "release-1".into(),
|
||||
action: runtime_api::SubjectReleaseAction::Reserve,
|
||||
};
|
||||
runtime
|
||||
.subject_release_scoped(&scoped, request.clone())
|
||||
.unwrap();
|
||||
drop(runtime);
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
let reopened =
|
||||
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
|
||||
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
|
||||
assert!(
|
||||
reopened
|
||||
.send_input(&worker.worker_ref, WorkerInput::user("must remain fenced"))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.restore_worker(
|
||||
&worker.worker_ref,
|
||||
reopened.test_restore_request(&worker.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
request.action = runtime_api::SubjectReleaseAction::Cancel;
|
||||
assert_eq!(
|
||||
reopened
|
||||
.subject_release_scoped(&scoped, request.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
runtime_api::SubjectReleaseState::Cancelled
|
||||
);
|
||||
request.action = runtime_api::SubjectReleaseAction::Reserve;
|
||||
assert!(
|
||||
reopened
|
||||
.subject_release_scoped(&scoped, request.clone())
|
||||
.is_err(),
|
||||
"cancelled identity must never rearm"
|
||||
);
|
||||
drop(reopened);
|
||||
let reopened = Runtime::with_fs_store_and_execution_backend(
|
||||
options,
|
||||
Arc::new(TestExecutionBackend::default()),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
reopened
|
||||
.subject_release_scoped(&scoped, request.clone())
|
||||
.is_err(),
|
||||
"terminal cancellation survives restart"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[test]
|
||||
fn fs_store_restores_active_worker_executions() {
|
||||
|
||||
@@ -492,8 +492,12 @@ pub(crate) fn open_session_public_material_reader(
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
type SpawnModelClientFactory =
|
||||
Arc<dyn Fn() -> Box<dyn agen::llm_client::client::LlmClient> + Send + Sync>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ProfileRuntimeWorkerFactory {
|
||||
spawn_model_client_factory: Option<SpawnModelClientFactory>,
|
||||
observation_hub: Arc<RuntimeWorkerObservationHub>,
|
||||
profile_base_dir: PathBuf,
|
||||
worker_aggregate_root: Option<PathBuf>,
|
||||
@@ -521,6 +525,7 @@ impl ProfileRuntimeWorkerFactory {
|
||||
pub fn new(profile_base_dir: impl Into<PathBuf>) -> Self {
|
||||
let profile_base_dir = profile_base_dir.into();
|
||||
Self {
|
||||
spawn_model_client_factory: None,
|
||||
observation_hub: Arc::new(RuntimeWorkerObservationHub::default()),
|
||||
profile_base_dir,
|
||||
worker_aggregate_root: None,
|
||||
@@ -535,6 +540,17 @@ impl ProfileRuntimeWorkerFactory {
|
||||
}
|
||||
}
|
||||
|
||||
/// Supply the external model boundary for newly spawned Controllers. The factory is
|
||||
/// process-owned, never persisted into a Profile or restored as execution authority.
|
||||
/// Restores continue to use their saved model configuration.
|
||||
pub fn with_spawn_model_client_factory(
|
||||
mut self,
|
||||
factory: impl Fn() -> Box<dyn agen::llm_client::client::LlmClient> + Send + Sync + 'static,
|
||||
) -> Self {
|
||||
self.spawn_model_client_factory = Some(Arc::new(factory));
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_runtime_id(mut self, runtime_id: impl Into<String>) -> Self {
|
||||
self.runtime_id = Some(runtime_id.into());
|
||||
self
|
||||
@@ -1502,7 +1518,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
|
||||
let run_dir = self.worker_run_dir(&request.worker_ref)?;
|
||||
let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id);
|
||||
let mut prepared = WorkerBootstrap::new(
|
||||
let bootstrap = WorkerBootstrap::new(
|
||||
manifest,
|
||||
store,
|
||||
loader,
|
||||
@@ -1513,10 +1529,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
bash_output_dir,
|
||||
},
|
||||
self.controller_transport,
|
||||
)
|
||||
.prepare()
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
);
|
||||
let bootstrap = match &self.spawn_model_client_factory {
|
||||
Some(factory) => bootstrap.with_model_client(factory()),
|
||||
None => bootstrap,
|
||||
};
|
||||
let mut prepared = bootstrap.prepare().await.map_err(|error| match error {
|
||||
WorkerBootstrapError::Worker(source) => {
|
||||
format!("failed to create Worker from profile: {source}")
|
||||
}
|
||||
@@ -2331,6 +2349,12 @@ impl PendingRuntimeRestore {
|
||||
}
|
||||
}
|
||||
|
||||
struct SubjectControllerReservation {
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
guard: Option<worker::SubjectReleaseReservation>,
|
||||
state: runtime_api::SubjectReleaseState,
|
||||
}
|
||||
|
||||
/// Worker execution backend backed by real worker crate Workers.
|
||||
pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
||||
backend_id: String,
|
||||
@@ -2341,6 +2365,7 @@ pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
|
||||
pending_restores: Mutex<HashMap<WorkerRef, Arc<PendingRuntimeRestore>>>,
|
||||
worker_locks: Mutex<HashMap<WorkerRef, Arc<Mutex<()>>>>,
|
||||
spawn_restore_timeout: Duration,
|
||||
subject_reservations: Mutex<HashMap<(WorkerRef, String), SubjectControllerReservation>>,
|
||||
shutdown_admission_timeout: Duration,
|
||||
}
|
||||
|
||||
@@ -2363,6 +2388,7 @@ where
|
||||
pending_restores: Mutex::new(HashMap::new()),
|
||||
worker_locks: Mutex::new(HashMap::new()),
|
||||
spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT,
|
||||
subject_reservations: Mutex::new(HashMap::new()),
|
||||
shutdown_admission_timeout: Duration::from_secs(5),
|
||||
})
|
||||
}
|
||||
@@ -3343,6 +3369,119 @@ where
|
||||
&self.backend_id
|
||||
}
|
||||
|
||||
fn subject_release(
|
||||
&self,
|
||||
request: &runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
use runtime_api::{SubjectReleaseAction as Action, SubjectReleaseState as State};
|
||||
let worker_ref = WorkerRef::new(request.worker_id);
|
||||
let lock = self.worker_lock(&worker_ref)?;
|
||||
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
|
||||
let mut reservations = self
|
||||
.subject_reservations
|
||||
.lock()
|
||||
.map_err(|_| "Subject reservation registry poisoned")?;
|
||||
let key = (worker_ref.clone(), request.operation_id.clone());
|
||||
if let Some(saved) = reservations.get(&key) {
|
||||
if saved.request.operation_id != request.operation_id
|
||||
|| saved.request.binding_id != request.binding_id
|
||||
|| saved.request.workspace_id != request.workspace_id
|
||||
{
|
||||
return Err("Worker has another retained Subject release operation".into());
|
||||
}
|
||||
} else {
|
||||
if request.action == Action::Cancel {
|
||||
// Runtime has a durable admission intent, but no Controller reservation may
|
||||
// have been acquired (or this is a new process). Under the same Worker op
|
||||
// lock there is no late local Reserve left to publish. Retain a terminal
|
||||
// cancellation receipt so an old Reserve can never rearm this identity.
|
||||
reservations.insert(
|
||||
key,
|
||||
SubjectControllerReservation {
|
||||
request: request.clone(),
|
||||
guard: None,
|
||||
state: State::Cancelled,
|
||||
},
|
||||
);
|
||||
return Ok(runtime_api::SubjectReleaseResponse {
|
||||
operation_id: request.operation_id.clone(),
|
||||
binding_id: request.binding_id.clone(),
|
||||
state: State::Cancelled,
|
||||
});
|
||||
}
|
||||
if request.action != Action::Reserve {
|
||||
return Err(
|
||||
"Subject reservation is missing; no status-based recovery is allowed".into(),
|
||||
);
|
||||
}
|
||||
if reservations.iter().any(|((worker, _), saved)| {
|
||||
*worker == worker_ref && saved.state != State::Cancelled
|
||||
}) {
|
||||
return Err("Worker has another active Subject reservation".into());
|
||||
}
|
||||
if self
|
||||
.pending_restores
|
||||
.lock()
|
||||
.map_err(|_| "restore registry poisoned")?
|
||||
.contains_key(&worker_ref)
|
||||
{
|
||||
return Err("Subject Controller restore is unresolved".into());
|
||||
}
|
||||
let workers = self
|
||||
.workers
|
||||
.lock()
|
||||
.map_err(|_| "worker registry poisoned")?;
|
||||
let execution = workers
|
||||
.get(&worker_ref)
|
||||
.ok_or("Subject Controller is unavailable")?;
|
||||
if execution.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|
||||
|| execution.shutdown_requested.load(Ordering::Acquire)
|
||||
|| execution.handle.protocol_is_closed()
|
||||
{
|
||||
return Err("Subject Controller is unavailable or Workspace mismatched".into());
|
||||
}
|
||||
let handle = execution.handle.clone();
|
||||
drop(workers);
|
||||
let guard =
|
||||
self.run_on_adapter_runtime(async move { handle.reserve_subject_release().await })?;
|
||||
reservations.insert(
|
||||
key.clone(),
|
||||
SubjectControllerReservation {
|
||||
request: request.clone(),
|
||||
guard: Some(guard),
|
||||
state: State::Reserved,
|
||||
},
|
||||
);
|
||||
}
|
||||
let saved = reservations.get_mut(&key).expect("reservation inserted");
|
||||
match (request.action, saved.state) {
|
||||
(Action::Reserve, State::Reserved | State::Disconnected) => {}
|
||||
(Action::Disconnect, State::Reserved) => {
|
||||
saved
|
||||
.guard
|
||||
.as_mut()
|
||||
.ok_or("Controller reservation missing")?
|
||||
.commit_disconnect()?;
|
||||
saved.state = State::Disconnected;
|
||||
}
|
||||
(Action::Disconnect, State::Disconnected) | (Action::Cancel, State::Cancelled) => {}
|
||||
(Action::Cancel, State::Reserved) => {
|
||||
saved.guard.take();
|
||||
saved.state = State::Cancelled;
|
||||
}
|
||||
_ => {
|
||||
return Err(
|
||||
"Subject release operation already reached a different terminal state".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(runtime_api::SubjectReleaseResponse {
|
||||
operation_id: request.operation_id.clone(),
|
||||
binding_id: request.binding_id.clone(),
|
||||
state: saved.state,
|
||||
})
|
||||
}
|
||||
|
||||
fn fetch_workspace_config(
|
||||
&self,
|
||||
request: WorkspaceConfigFetchRequest,
|
||||
@@ -4932,7 +5071,7 @@ mod tests {
|
||||
let aggregate_dir = root.join("workers").join(worker_ref.worker_id.to_string());
|
||||
let session_store = WorkerSessionStore::new(aggregate_dir.join("session")).unwrap();
|
||||
let store = WorkerAggregateStore::new(&aggregate_dir, &worker_name).unwrap();
|
||||
let (saved, request) = saved_subjektiv_policy(root, &worker_ref);
|
||||
let (saved, mut request) = saved_subjektiv_policy(root, &worker_ref);
|
||||
let client = MockClient::new(Vec::new());
|
||||
let mut engine =
|
||||
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||
@@ -4979,11 +5118,7 @@ mod tests {
|
||||
Some(SAVED_SUBJEKTIV_SYSTEM_PROMPT)
|
||||
);
|
||||
} else {
|
||||
assert!(matches!(
|
||||
session_store.exists(session_id, segment_id),
|
||||
Err(session_store::StoreError::Corrupt { message, .. })
|
||||
if message.contains("no materialized Session")
|
||||
));
|
||||
assert!(!session_store.exists(session_id, segment_id).unwrap());
|
||||
}
|
||||
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
|
||||
drop(worker);
|
||||
@@ -5007,8 +5142,46 @@ mod tests {
|
||||
metadata.workspace_id.as_deref(),
|
||||
Some("workspace-saved-subjektiv")
|
||||
);
|
||||
// Attribution already committed before restart; the real restore needs
|
||||
// no network fixture or replacement current Profile.
|
||||
// Attribution history is already committed. Each new Controller must still
|
||||
// obtain current publication authority, even for a Confirmed Session.
|
||||
// This is a transport fixture for restore/stop component tests, not an
|
||||
// assertion that a real Backend authorized a released Subject body.
|
||||
#[cfg(feature = "http-server")]
|
||||
{
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
request.workspace_api.as_mut().unwrap().base_url =
|
||||
format!("http://{}", listener.local_addr().unwrap());
|
||||
let app = axum::Router::new().route(
|
||||
"/api/w/workspace-saved-subjektiv/subjektiv/sessions",
|
||||
axum::routing::post(
|
||||
move |axum::Json(input): axum::Json<
|
||||
server_api::SubjektivRecordSessionRequest,
|
||||
>| async move {
|
||||
assert!(
|
||||
!input.create_if_missing,
|
||||
"publication must not rewrite attribution"
|
||||
);
|
||||
assert_eq!(input.session_id, session_id.to_string());
|
||||
axum::Json(server_api::SubjektivRecordSessionResponse {
|
||||
session_id: input.session_id,
|
||||
subject_id: "saved-subject".into(),
|
||||
})
|
||||
},
|
||||
),
|
||||
);
|
||||
let app = app.route(
|
||||
"/api/w/workspace-saved-subjektiv/subjektiv/journal/startup",
|
||||
axum::routing::get(move || async move {
|
||||
assert!(!materialize_head, "normal restore must not refetch startup");
|
||||
axum::Json(serde_json::json!({
|
||||
"subject_id": "saved-subject", "journals": [], "omitted": false,
|
||||
}))
|
||||
}),
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
}
|
||||
metadata.subjektiv_session_attribution =
|
||||
Some(session_store::SubjektivSessionAttributionState::Confirmed {
|
||||
session_id,
|
||||
@@ -6077,7 +6250,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
async fn profile_factory_saved_subjektiv_manifest_survives_restart_and_explicit_restore() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
@@ -6147,7 +6320,7 @@ mod tests {
|
||||
assert!(!root.path().join("nonexistent-current-profiles").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
async fn profile_factory_pending_subjektiv_restore_requires_operation_owned_launch_material() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
@@ -9681,6 +9854,158 @@ mod tests {
|
||||
assert!(error.to_string().contains("WorkerNotify"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_release_reserves_real_controller_and_excludes_runtime_lifecycle() {
|
||||
use runtime_api::{SubjectReleaseAction as Action, SubjectReleaseState as State};
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
let cwd = tempfile::tempdir().unwrap();
|
||||
let store = tempfile::tempdir().unwrap();
|
||||
let backend = Arc::new(
|
||||
WorkerRuntimeExecutionBackend::new(MockFactory {
|
||||
client: MockClient::new(simple_text_events()),
|
||||
runtime_base: runtime_base.path().to_path_buf(),
|
||||
cwd: cwd.path().to_path_buf(),
|
||||
store_dir: store.path().join("sessions"),
|
||||
worker_metadata_dir: store.path().join("workers"),
|
||||
observed_cwds: Arc::new(Mutex::new(Vec::new())),
|
||||
observed_workspace_clients: Arc::new(Mutex::new(Vec::new())),
|
||||
})
|
||||
.unwrap(),
|
||||
);
|
||||
let runtime =
|
||||
EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend.clone())
|
||||
.unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let mut create = create_request("subject-release");
|
||||
create.workspace_api = Some(crate::catalog::WorkspaceApiRef {
|
||||
workspace_id: "ws-test".into(),
|
||||
base_url: "http://127.0.0.1:3999".into(),
|
||||
});
|
||||
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||
workspace_id: "ws-test".into(),
|
||||
language: "English".into(),
|
||||
});
|
||||
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "embedded-backend");
|
||||
let detail = runtime.create_worker_scoped(&scope, create).unwrap();
|
||||
let mut release = runtime_api::SubjectReleaseRequest {
|
||||
workspace_id: "ws-test".into(),
|
||||
worker_id: detail.worker_ref.worker_id,
|
||||
binding_id: "original-binding".into(),
|
||||
operation_id: "subject-release-1".into(),
|
||||
action: Action::Reserve,
|
||||
};
|
||||
let reserved = runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap();
|
||||
assert_eq!(reserved.state, State::Reserved);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap(),
|
||||
reserved
|
||||
);
|
||||
let before = backend
|
||||
.workers
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&detail.worker_ref)
|
||||
.unwrap()
|
||||
.handle
|
||||
.committed_entries();
|
||||
let blocked = runtime
|
||||
.send_input(
|
||||
&detail.worker_ref,
|
||||
WorkerInput::user("must not enter history"),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(blocked.to_string().contains("Subject"));
|
||||
assert!(
|
||||
runtime
|
||||
.stop_worker(&detail.worker_ref, None)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("Subject")
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.delete_worker(&detail.worker_ref)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("Subject")
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.restore_worker(
|
||||
&detail.worker_ref,
|
||||
runtime.test_restore_request(&detail.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let after = backend
|
||||
.workers
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&detail.worker_ref)
|
||||
.unwrap()
|
||||
.handle
|
||||
.committed_entries();
|
||||
assert_eq!(
|
||||
serde_json::to_value(after).unwrap(),
|
||||
serde_json::to_value(before).unwrap()
|
||||
);
|
||||
let mut conflicting = release.clone();
|
||||
conflicting.binding_id = "later-binding".into();
|
||||
assert!(runtime.subject_release_scoped(&scope, conflicting).is_err());
|
||||
release.action = Action::Cancel;
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
State::Cancelled
|
||||
);
|
||||
release.operation_id = "subject-release-2".into();
|
||||
release.action = Action::Reserve;
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
State::Reserved
|
||||
);
|
||||
release.action = Action::Disconnect;
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
State::Disconnected
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.subject_release_scoped(&scope, release.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
State::Disconnected
|
||||
);
|
||||
release.action = Action::Cancel;
|
||||
assert!(runtime.subject_release_scoped(&scope, release).is_err());
|
||||
assert!(
|
||||
runtime
|
||||
.send_input(&detail.worker_ref, WorkerInput::user("after disconnect"))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.restore_worker(
|
||||
&detail.worker_ref,
|
||||
runtime.test_restore_request(&detail.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
runtime.stop_worker(&detail.worker_ref, None).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stopped_runtime_worker_can_restore_and_accept_input() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
|
||||
@@ -78,7 +78,45 @@ impl WorkerHandle {
|
||||
self.method_tx.same_channel(&other.method_tx)
|
||||
}
|
||||
|
||||
/// Ask the actual Controller to acknowledge quiescence. No catalog status,
|
||||
/// profile, or model protocol message grants this Host lifecycle capability.
|
||||
pub async fn reserve_subject_release(
|
||||
&self,
|
||||
) -> Result<crate::SubjectReleaseReservation, String> {
|
||||
if self.protocol_is_closed() {
|
||||
return Err("Controller is unavailable".into());
|
||||
}
|
||||
if self.shared_state.snapshot().state != WorkerState::Idle {
|
||||
return Err("Controller is not Idle".into());
|
||||
}
|
||||
let control = self
|
||||
.shared_state
|
||||
.subject_release
|
||||
.get()
|
||||
.ok_or("Controller release authority unavailable")?;
|
||||
tokio::select! {
|
||||
result = control.request() => result,
|
||||
_ = self.method_tx.closed() => Err("Controller closed before release acknowledgement".into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn send(&self, method: Method) -> Result<(), mpsc::error::SendError<Method>> {
|
||||
// The lease covers enqueue itself. The Controller's exclusive barrier
|
||||
// cannot overlook an accepted send between its queue check and ack.
|
||||
let _admission = if release_fenced_method(&method) {
|
||||
match self
|
||||
.shared_state
|
||||
.subject_release
|
||||
.get()
|
||||
.map(|control| control.authority.enter())
|
||||
.transpose()
|
||||
{
|
||||
Ok(lease) => lease,
|
||||
Err(_) => return Err(mpsc::error::SendError(method)),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Reject known-busy sends before channel dispatch, including maintenance
|
||||
// that cannot receive methods until it finishes. The controller checks
|
||||
// again at admission to cover an Idle -> busy race after this snapshot.
|
||||
@@ -290,6 +328,49 @@ impl WorkerHandle {
|
||||
}
|
||||
}
|
||||
|
||||
fn answer_idle_release<St: Store + Clone>(
|
||||
control: &Arc<crate::subject_release::ReleaseControl>,
|
||||
shared_state: &WorkerSharedState,
|
||||
methods: &mpsc::Receiver<Method>,
|
||||
staged: bool,
|
||||
pending: &PendingSubmissionHandle<St>,
|
||||
notifications: &NotifyBuffer,
|
||||
attention: bool,
|
||||
) {
|
||||
control.respond(|| {
|
||||
// Read ingress and durable pending state only after acquiring exclusivity.
|
||||
// Actor-local staged/deferred/attention flags cannot change during this call.
|
||||
if shared_state.snapshot().state != WorkerState::Idle
|
||||
|| staged
|
||||
|| attention
|
||||
|| !methods.is_empty()
|
||||
|| methods.is_closed()
|
||||
|| !notifications.is_empty()
|
||||
|| pending.has_unfinished_admission()
|
||||
{
|
||||
return Err("Controller has pending, staged, busy, or unavailable work".into());
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
|
||||
fn release_fenced_method(method: &Method) -> bool {
|
||||
!matches!(
|
||||
method,
|
||||
Method::Shutdown { .. }
|
||||
| Method::ListWorkers
|
||||
| Method::ListPendingSubmissions
|
||||
| Method::ListCompletions { .. }
|
||||
)
|
||||
}
|
||||
|
||||
async fn release_requested(shared_state: &WorkerSharedState) {
|
||||
match shared_state.subject_release.get() {
|
||||
Some(control) => control.notified().await,
|
||||
None => std::future::pending::<()>().await,
|
||||
}
|
||||
}
|
||||
|
||||
fn command_admission_disposition(
|
||||
admission: WorkerCommandAdmission,
|
||||
) -> Result<(), WorkerCommandDisposition> {
|
||||
@@ -802,6 +883,10 @@ impl WorkerController {
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
worker
|
||||
.validate_subjektiv_body_publication()
|
||||
.map_err(std::io::Error::other)?;
|
||||
|
||||
// === 1. Initialization (channels / RuntimeDir / worker-immutable
|
||||
// snapshots / SpawnedWorkerRegistry / alerter attach /
|
||||
// bash-output scope) ===
|
||||
@@ -911,13 +996,13 @@ impl WorkerController {
|
||||
// the actual registered set instead of a hand-maintained mirror.
|
||||
worker.engine().tool_server_handle().flush_pending();
|
||||
|
||||
// Runtime-owned Workers persist their current model-visible head before
|
||||
// Fresh Workers persist their current model-visible head before
|
||||
// controller exposure. Feature installation above must happen first so a
|
||||
// fresh subject Worker captures the current resident surface in its durable
|
||||
// prompt. Restored Workers defer generic Feature restore notification until
|
||||
// the next model-visible context materialization; only an installed
|
||||
// Feature may then request a typed durable history append.
|
||||
if runtime_managed && worker.needs_initial_session_head_materialization() {
|
||||
if worker.needs_initial_session_head_materialization() {
|
||||
worker
|
||||
.materialize_durable_session_head()
|
||||
.await
|
||||
@@ -958,6 +1043,13 @@ impl WorkerController {
|
||||
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
|
||||
let session_id = worker.session_id();
|
||||
let pending_activations = worker.pending_activation_state();
|
||||
let release_control =
|
||||
crate::subject_release::ReleaseControl::new(worker.mutation_authority());
|
||||
shared_state
|
||||
.subject_release
|
||||
.set(release_control)
|
||||
.ok()
|
||||
.expect("one Controller release authority");
|
||||
let handle = WorkerHandle {
|
||||
method_tx,
|
||||
working_event_tx: working_event_tx.clone(),
|
||||
@@ -1561,6 +1653,18 @@ where
|
||||
{
|
||||
feature_registry.add_module(subjektiv_sessions);
|
||||
}
|
||||
if ordinary_subjektiv_features_enabled
|
||||
&& lifecycle_enabled
|
||||
&& let Some(host) = subjektiv_host.as_ref()
|
||||
&& host.host.journal_startup_granted()
|
||||
{
|
||||
worker.install_initial_context_source(Arc::new(
|
||||
crate::feature::builtin::journal::JournalStartupContextSource::new(
|
||||
host.clone(),
|
||||
worker.prompts(),
|
||||
),
|
||||
));
|
||||
}
|
||||
if ordinary_subjektiv_features_enabled
|
||||
&& lifecycle_enabled
|
||||
&& let Some(host) = subjektiv_host.as_ref()
|
||||
@@ -2232,6 +2336,12 @@ async fn controller_loop<C, St>(
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + 'static,
|
||||
{
|
||||
let release_control = shared_state
|
||||
.subject_release
|
||||
.get()
|
||||
.expect("Controller release authority installed")
|
||||
.clone();
|
||||
let _release_lifetime = crate::subject_release::ControllerLifetime(release_control.clone());
|
||||
// Hold an optional external attach server alive for the controller lifetime.
|
||||
// In-process runtimes retain and drive the WorkerHandle directly.
|
||||
let _socket_server = socket_server;
|
||||
@@ -2452,6 +2562,13 @@ async fn controller_loop<C, St>(
|
||||
break 'controller;
|
||||
}
|
||||
},
|
||||
_ = release_control.notified() => {
|
||||
answer_idle_release(&release_control, &shared_state, &method_rx,
|
||||
pending.is_some() || !deferred_methods.is_empty(),
|
||||
&pending_submissions, ¬ify_buffer,
|
||||
worker.has_pending_compaction_cleanup() || worker.has_failed_segment_activation());
|
||||
continue;
|
||||
}
|
||||
_ = notify_buffer.notified() => continue,
|
||||
_ = tokio::time::sleep(remaining) => {
|
||||
match prepare_pending_run(
|
||||
@@ -2486,10 +2603,32 @@ async fn controller_loop<C, St>(
|
||||
break 'controller;
|
||||
}
|
||||
},
|
||||
_ = release_control.notified() => {
|
||||
answer_idle_release(&release_control, &shared_state, &method_rx,
|
||||
pending.is_some() || !deferred_methods.is_empty(),
|
||||
&pending_submissions, ¬ify_buffer,
|
||||
worker.has_pending_compaction_cleanup() || worker.has_failed_segment_activation());
|
||||
continue;
|
||||
}
|
||||
_ = notify_buffer.notified() => continue,
|
||||
}
|
||||
};
|
||||
|
||||
let _method_admission = if release_fenced_method(&method) {
|
||||
match release_control.authority.enter() {
|
||||
Ok(lease) => Some(lease),
|
||||
Err(error) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: error.to_string(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if shared_state.snapshot().state != WorkerState::Idle
|
||||
&& reject_idle_only_submission(&method, &working_event_tx)
|
||||
{
|
||||
@@ -2885,6 +3024,7 @@ async fn controller_loop<C, St>(
|
||||
let mut compact = Box::pin(worker.manual_compact_with_cancel(cancel_rx));
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = release_control.notified() => { release_control.reject_busy(); }
|
||||
result = &mut compact => break result,
|
||||
method = method_rx.recv() => {
|
||||
match method {
|
||||
@@ -3142,6 +3282,7 @@ async fn controller_loop<C, St>(
|
||||
}
|
||||
}
|
||||
|
||||
release_control.close();
|
||||
// Close method admission before any fallible child/session cleanup. Existing
|
||||
// senders then fail instead of accepting work that this execution can no
|
||||
// longer process.
|
||||
@@ -3286,6 +3427,9 @@ where
|
||||
// Running only after processing the commit fence. This makes the
|
||||
// Running snapshot contract deterministic even for immediate clients.
|
||||
biased;
|
||||
_ = release_requested(shared_state) => {
|
||||
if let Some(control) = shared_state.subject_release.get() { control.reject_busy(); }
|
||||
}
|
||||
committed = async {
|
||||
input_commit
|
||||
.as_mut()
|
||||
@@ -3906,6 +4050,10 @@ fn worker_error_code(e: &WorkerError) -> ErrorCode {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "controller_subject_release_tests.rs"]
|
||||
mod subject_release_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -4251,7 +4399,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_session_head_is_materialized_after_feature_installation_before_exposure() {
|
||||
fn fresh_session_head_is_materialized_after_feature_installation_before_exposure() {
|
||||
let production = include_str!("controller.rs")
|
||||
.split_once("#[cfg(test)]\nmod tests")
|
||||
.map(|(production, _)| production)
|
||||
@@ -4270,8 +4418,8 @@ mod tests {
|
||||
.find("let handle = WorkerHandle")
|
||||
.expect("controller handle exposure");
|
||||
let materialize_gate = startup[..materialize]
|
||||
.rfind("if runtime_managed && worker.needs_initial_session_head_materialization() {")
|
||||
.expect("fresh runtime-managed materialization gate");
|
||||
.rfind("if worker.needs_initial_session_head_materialization() {")
|
||||
.expect("fresh materialization gate");
|
||||
|
||||
assert!(feature_install < materialize_gate);
|
||||
assert!(materialize_gate < materialize);
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
//! Production Direct bootstrap, Controller admission, pending persistence and FsStore.
|
||||
//! Only the provider is replaced by an explicitly released async boundary; no live Host,
|
||||
//! Journal policy, socket or model continuation semantics are claimed.
|
||||
use super::*;
|
||||
use crate::{
|
||||
PreparedWorker, WorkerBootstrap, WorkerBootstrapLayout, WorkerFilesystemAuthority,
|
||||
WorkerManifest, WorkerWorkspaceContext,
|
||||
};
|
||||
use session_store::{CombinedStore, FsStore, FsWorkerStore};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct HeldProvider {
|
||||
started: mpsc::UnboundedSender<()>,
|
||||
finish: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl LlmClient for HeldProvider {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
async fn stream(
|
||||
&self,
|
||||
_: agen::llm_client::Request,
|
||||
) -> Result<
|
||||
std::pin::Pin<
|
||||
Box<
|
||||
dyn futures_util::Stream<
|
||||
Item = Result<
|
||||
agen::llm_client::event::Event,
|
||||
agen::llm_client::ClientError,
|
||||
>,
|
||||
> + Send,
|
||||
>,
|
||||
>,
|
||||
agen::llm_client::ClientError,
|
||||
> {
|
||||
self.started.send(()).unwrap();
|
||||
self.finish.notified().await;
|
||||
Ok(Box::pin(futures_util::stream::iter([Ok(
|
||||
agen::llm_client::event::Event::Status(agen::llm_client::event::StatusEvent {
|
||||
status: agen::llm_client::event::ResponseStatus::Completed,
|
||||
}),
|
||||
)])))
|
||||
}
|
||||
}
|
||||
async fn prepared(
|
||||
temp: &TempDir,
|
||||
client: HeldProvider,
|
||||
) -> PreparedWorker<Box<dyn LlmClient>, CombinedStore<FsStore, FsWorkerStore>> {
|
||||
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='release-{}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]", uuid::Uuid::now_v7())).unwrap();
|
||||
manifest.compaction = Some(manifest::CompactionConfig {
|
||||
retained_tokens: 0,
|
||||
..Default::default()
|
||||
});
|
||||
WorkerBootstrap::new(
|
||||
manifest,
|
||||
CombinedStore::new(
|
||||
FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||
),
|
||||
crate::PromptCatalogSource::builtins_only(),
|
||||
WorkerWorkspaceContext::no_workspace(),
|
||||
WorkerFilesystemAuthority::None,
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
bash_output_dir: temp.path().join("bash"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(client)
|
||||
.prepare()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
async fn reserve(handle: &WorkerHandle) -> Result<crate::SubjectReleaseReservation, String> {
|
||||
tokio::time::timeout(Duration::from_secs(5), handle.reserve_subject_release())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
async fn shutdown(controller: crate::BootstrappedWorker) {
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Shutdown {
|
||||
command: WorkerCommandEnvelope::new(1),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
controller.controller_task.await.unwrap();
|
||||
}
|
||||
fn extension() -> LogEntry {
|
||||
LogEntry::Extension {
|
||||
ts: 1,
|
||||
domain: "release-test".into(),
|
||||
payload: serde_json::json!({}),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reservation_fences_ingress_fallback_and_history_drop_recovers_commit_stays_fenced() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, mut starts) = mpsc::unbounded_channel();
|
||||
let prepared = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: Arc::default(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let writer = prepared.worker().log_writer_handle();
|
||||
let pending = prepared.worker().pending_submission_handle();
|
||||
let buffer = prepared.worker().notify_buffer_handle();
|
||||
let capture = prepared.worker().committed_session_capture_handle();
|
||||
let extensions = prepared.worker().session_extension_handle();
|
||||
let fallback = durable_parent_notification_target(pending.clone(), buffer.clone());
|
||||
let controller = prepared.start().await.unwrap();
|
||||
let before = controller.handle.committed_entries().len();
|
||||
let expected = capture.capture().unwrap().location();
|
||||
let guard = reserve(&controller.handle).await.unwrap();
|
||||
assert!(
|
||||
controller
|
||||
.handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"blocked"
|
||||
))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "blocked".into()
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
pending
|
||||
.accept_notification("blocked".into(), "blocked".into())
|
||||
.is_err()
|
||||
);
|
||||
fallback.notify("child-session".into(), "must not be admitted".into());
|
||||
assert!(buffer.is_empty());
|
||||
assert!(!pending.has_unfinished_admission());
|
||||
assert!(writer.append_entry(extension()).is_err());
|
||||
assert!(
|
||||
extensions
|
||||
.append_if_current(&expected, "release-test", serde_json::json!({}))
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(controller.handle.committed_entries().len(), before);
|
||||
assert!(starts.try_recv().is_err());
|
||||
drop(guard);
|
||||
writer.append_entry(extension()).unwrap();
|
||||
let mut guard = reserve(&controller.handle).await.unwrap();
|
||||
guard.commit_disconnect().unwrap();
|
||||
guard.commit_disconnect().unwrap();
|
||||
drop(guard);
|
||||
assert!(writer.append_entry(extension()).is_err());
|
||||
assert!(
|
||||
pending
|
||||
.accept_notification("after-commit".into(), "blocked".into())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "blocked".into()
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
let _ = controller.handle.snapshot_event();
|
||||
controller
|
||||
.handle
|
||||
.send(Method::ListPendingSubmissions)
|
||||
.await
|
||||
.unwrap();
|
||||
let handle = controller.handle.clone();
|
||||
shutdown(controller).await;
|
||||
assert!(handle.protocol_is_closed());
|
||||
assert!(reserve(&handle).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_input_cannot_be_overtaken_by_release_and_running_request_is_rejected() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, mut starts) = mpsc::unbounded_channel();
|
||||
let finish = Arc::new(tokio::sync::Notify::new());
|
||||
let controller = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: finish.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.start()
|
||||
.await
|
||||
.unwrap();
|
||||
let mut events = controller.handle.subscribe();
|
||||
// Enqueue succeeds before submitting the separate internal release request.
|
||||
// Either the queue or the held provider is observed; neither is quiescence.
|
||||
controller
|
||||
.handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"accepted first",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
tokio::time::timeout(Duration::from_secs(5), starts.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
|
||||
}).await.unwrap();
|
||||
let guard = reserve(&controller.handle).await.unwrap();
|
||||
drop(guard);
|
||||
shutdown(controller).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_notification_fallback_blocks_release_until_the_actual_turn_finishes() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, mut starts) = mpsc::unbounded_channel();
|
||||
let finish = Arc::new(tokio::sync::Notify::new());
|
||||
let prepared = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: finish.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let pending = prepared.worker().pending_submission_handle();
|
||||
let buffer = prepared.worker().notify_buffer_handle();
|
||||
let fallback = durable_parent_notification_target(pending.clone(), buffer);
|
||||
let controller = prepared.start().await.unwrap();
|
||||
let mut events = controller.handle.subscribe();
|
||||
fallback.notify("child".into(), "durable accepted notification".into());
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
tokio::time::timeout(Duration::from_secs(5), starts.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
|
||||
}).await.unwrap();
|
||||
let guard = reserve(&controller.handle).await.unwrap();
|
||||
drop(guard);
|
||||
shutdown(controller).await;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct HeldRewrite {
|
||||
started: mpsc::UnboundedSender<()>,
|
||||
finish: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
impl crate::feature::FeatureModule for HeldRewrite {
|
||||
fn descriptor(&self) -> crate::feature::FeatureDescriptor {
|
||||
crate::feature::FeatureDescriptor::builtin("held-rewrite", "Held rewrite").with_hook(
|
||||
crate::feature::HookDeclaration::new(
|
||||
"hold",
|
||||
crate::feature::FeatureHookPoint::BeforeSessionRewrite,
|
||||
),
|
||||
)
|
||||
}
|
||||
fn install(
|
||||
&self,
|
||||
context: &mut crate::feature::FeatureInstallContext<'_>,
|
||||
) -> Result<(), crate::feature::FeatureInstallError> {
|
||||
context.hooks().add_before_session_rewrite(
|
||||
"hold",
|
||||
crate::hook::HookExecutionPolicy::fail_closed(),
|
||||
self.clone(),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl crate::hook::Hook<crate::hook::BeforeSessionRewrite> for HeldRewrite {
|
||||
async fn call(
|
||||
&self,
|
||||
context: &crate::hook::BeforeSessionRewriteContext,
|
||||
) -> Result<crate::hook::BeforeSessionRewriteAction, crate::hook::HookError> {
|
||||
if context.kind == crate::hook::SessionRewriteKind::Compact {
|
||||
self.started.send(()).unwrap();
|
||||
self.finish.notified().await;
|
||||
return Ok(crate::hook::BeforeSessionRewriteAction::Deny(
|
||||
"stop before compactor boundary".into(),
|
||||
));
|
||||
}
|
||||
Ok(crate::hook::BeforeSessionRewriteAction::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_maintenance_rejects_release_until_the_real_rewrite_boundary_exits() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, mut starts) = mpsc::unbounded_channel();
|
||||
let finish = Arc::new(tokio::sync::Notify::new());
|
||||
let mut prepared = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: finish.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let (rewrite_started, mut rewrites) = mpsc::unbounded_channel();
|
||||
let rewrite_finish = Arc::new(tokio::sync::Notify::new());
|
||||
let mut registry = FeatureRegistryBuilder::new();
|
||||
registry.add_module(HeldRewrite {
|
||||
started: rewrite_started,
|
||||
finish: rewrite_finish.clone(),
|
||||
});
|
||||
assert!(
|
||||
!prepared
|
||||
.worker_mut()
|
||||
.install_features(registry)
|
||||
.has_errors()
|
||||
);
|
||||
let controller = prepared.start().await.unwrap();
|
||||
let mut events = controller.handle.subscribe();
|
||||
controller
|
||||
.handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"history to compact ".repeat(128),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), starts.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
|
||||
}).await.unwrap();
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Compact {
|
||||
command: WorkerCommandEnvelope::new(1),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), rewrites.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(reserve(&controller.handle).await.is_err());
|
||||
assert!(matches!(
|
||||
controller.handle.shared_state.snapshot().state,
|
||||
WorkerState::Busy(WorkerBusyState::Maintenance(_))
|
||||
));
|
||||
rewrite_finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop { if matches!(events.recv().await.unwrap(), Event::WorkerState { snapshot } if snapshot.state == WorkerState::Idle) { break; } }
|
||||
}).await.unwrap();
|
||||
let guard = reserve(&controller.handle).await.unwrap();
|
||||
drop(guard);
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Shutdown {
|
||||
command: WorkerCommandEnvelope::new(2),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
controller.controller_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_or_actor_staged_work_cannot_pass_the_exclusive_ack_boundary() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let (started, _) = mpsc::unbounded_channel();
|
||||
let mut prepared = prepared(
|
||||
&temp,
|
||||
HeldProvider {
|
||||
started,
|
||||
finish: Arc::default(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
prepared
|
||||
.worker_mut()
|
||||
.materialize_durable_session_head()
|
||||
.await
|
||||
.unwrap();
|
||||
let pending = prepared.worker().pending_submission_handle();
|
||||
let buffer = prepared.worker().notify_buffer_handle();
|
||||
let shared = WorkerSharedState::new(
|
||||
"queue-boundary".into(),
|
||||
prepared.worker().segment_id(),
|
||||
String::new(),
|
||||
build_greeting(prepared.worker()),
|
||||
);
|
||||
let control =
|
||||
crate::subject_release::ReleaseControl::new(prepared.worker().mutation_authority());
|
||||
for condition in [
|
||||
"queued",
|
||||
"staged",
|
||||
"attention",
|
||||
"closed",
|
||||
"durable-notification",
|
||||
] {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
if condition == "queued" {
|
||||
sender
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"already enqueued",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
if condition == "durable-notification" {
|
||||
pending
|
||||
.accept_notification("already-admitted".into(), "pending".into())
|
||||
.unwrap();
|
||||
}
|
||||
if condition == "closed" {
|
||||
drop(sender);
|
||||
}
|
||||
let request = tokio::spawn({
|
||||
let control = control.clone();
|
||||
async move { control.request().await }
|
||||
});
|
||||
control.notified().await;
|
||||
answer_idle_release(
|
||||
&control,
|
||||
&shared,
|
||||
&receiver,
|
||||
condition == "staged",
|
||||
&pending,
|
||||
&buffer,
|
||||
condition == "attention",
|
||||
);
|
||||
assert!(request.await.unwrap().is_err(), "{condition}");
|
||||
// A failed request must not leave admission fenced for recovery.
|
||||
assert!(prepared.worker().mutation_authority().enter().is_ok());
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,59 @@ use subjektiv::journal::{
|
||||
MAX_JOURNAL_RESPONSE_BYTES,
|
||||
};
|
||||
|
||||
/// Subjektiv owns startup projection; Core only commits its typed initial items.
|
||||
pub(crate) struct JournalStartupContextSource {
|
||||
host: SubjektivHostConnection,
|
||||
prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>,
|
||||
}
|
||||
impl JournalStartupContextSource {
|
||||
pub(crate) fn new(
|
||||
host: SubjektivHostConnection,
|
||||
prompts: Arc<arc_swap::ArcSwap<crate::prompt::catalog::PromptCatalog>>,
|
||||
) -> Self {
|
||||
Self { host, prompts }
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl crate::worker::InitialContextSource for JournalStartupContextSource {
|
||||
async fn load(&self) -> Result<Vec<session_store::SystemItem>, String> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let snapshot = self
|
||||
.host
|
||||
.journal_startup_context()
|
||||
.map_err(|e| e.to_string())?;
|
||||
// Hash the same canonical JSON value retained by SystemItem, so replay
|
||||
// can verify provenance without depending on domain struct field order.
|
||||
let snapshot = serde_json::to_value(snapshot).map_err(|e| e.to_string())?;
|
||||
let snapshot_json = serde_json::to_string(&snapshot).map_err(|e| e.to_string())?;
|
||||
if snapshot_json.len() > subjektiv::journal::startup::STARTUP_MAX_BYTES {
|
||||
return Err("Host Journal startup snapshot exceeds independent byte budget".into());
|
||||
}
|
||||
let prompts = self.prompts.load();
|
||||
let body = prompts
|
||||
.subjektiv_journal_startup(&snapshot_json)
|
||||
.map_err(|e| e.to_string())?;
|
||||
if body.len() > subjektiv::journal::startup::STARTUP_MAX_BYTES + 4096 {
|
||||
return Err("Journal startup prompt exceeds rendered byte budget".into());
|
||||
}
|
||||
let projection = prompts.projection();
|
||||
Ok(vec![session_store::SystemItem::JournalStartup {
|
||||
body,
|
||||
snapshot,
|
||||
snapshot_sha256: Sha256::digest(snapshot_json.as_bytes())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect(),
|
||||
prompt_provenance: session_store::PromptRenderProvenance {
|
||||
workspace_id: self.host.host.workspace_id(),
|
||||
source_digest: projection.source_digest.clone(),
|
||||
projection_digest: projection.catalog_digest.clone(),
|
||||
logical_name: "internal.subjektiv_journal_startup".into(),
|
||||
},
|
||||
}])
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct JournalRecordingFeature {
|
||||
host: SubjektivHostConnection,
|
||||
|
||||
@@ -76,9 +76,16 @@ impl PendingNotify {
|
||||
pub struct NotifyBuffer {
|
||||
inner: Arc<Mutex<VecDeque<PendingNotify>>>,
|
||||
wake: Arc<tokio::sync::Notify>,
|
||||
admission: Arc<std::sync::OnceLock<Arc<crate::subject_release::MutationAuthority>>>,
|
||||
}
|
||||
|
||||
impl NotifyBuffer {
|
||||
pub(crate) fn bind_mutation_authority(
|
||||
&self,
|
||||
authority: Arc<crate::subject_release::MutationAuthority>,
|
||||
) {
|
||||
let _ = self.admission.set(authority);
|
||||
}
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
@@ -113,6 +120,18 @@ impl NotifyBuffer {
|
||||
}
|
||||
|
||||
fn push_entry(&self, entry: PendingNotify) {
|
||||
let _admission = match self
|
||||
.admission
|
||||
.get()
|
||||
.map(|authority| authority.enter())
|
||||
.transpose()
|
||||
{
|
||||
Ok(lease) => lease,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "notification rejected by Subject release fence");
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.inner
|
||||
.lock()
|
||||
.expect("notify buffer poisoned")
|
||||
|
||||
@@ -25,6 +25,8 @@ mod session_capture;
|
||||
mod session_history;
|
||||
pub mod shared_state;
|
||||
mod shutdown_after_idle;
|
||||
mod subject_release;
|
||||
pub use subject_release::SubjectReleaseReservation;
|
||||
pub mod skill;
|
||||
pub mod spawn;
|
||||
pub mod subjektiv;
|
||||
|
||||
@@ -102,6 +102,7 @@ pub enum WorkerPrompt {
|
||||
ResidentMemorySummarySection,
|
||||
ResidentMemoryRestoreSection,
|
||||
SubjektivResidentContext,
|
||||
SubjektivJournalStartup,
|
||||
WorkerOrchestrationGuidanceSection,
|
||||
SubWorkerSpawnToolDescription,
|
||||
}
|
||||
@@ -126,6 +127,7 @@ impl WorkerPrompt {
|
||||
Self::AgentsMdSection => "internal.agents_md_section",
|
||||
Self::ResidentMemorySummarySection => "internal.resident_memory_summary_section",
|
||||
Self::ResidentMemoryRestoreSection => "internal.resident_memory_restore_section",
|
||||
Self::SubjektivJournalStartup => "internal.subjektiv_journal_startup",
|
||||
Self::SubjektivResidentContext => "internal.subjektiv_resident_context",
|
||||
Self::WorkerOrchestrationGuidanceSection => {
|
||||
"internal.worker_orchestration_guidance_section"
|
||||
@@ -151,6 +153,7 @@ impl WorkerPrompt {
|
||||
WorkerPrompt::ResidentMemorySummarySection,
|
||||
WorkerPrompt::ResidentMemoryRestoreSection,
|
||||
WorkerPrompt::SubjektivResidentContext,
|
||||
WorkerPrompt::SubjektivJournalStartup,
|
||||
WorkerPrompt::WorkerOrchestrationGuidanceSection,
|
||||
WorkerPrompt::SubWorkerSpawnToolDescription,
|
||||
];
|
||||
@@ -540,6 +543,14 @@ impl PromptCatalog {
|
||||
},
|
||||
)
|
||||
}
|
||||
pub fn subjektiv_journal_startup(&self, snapshot_json: &str) -> Result<String, CatalogError> {
|
||||
self.render(
|
||||
WorkerPrompt::SubjektivJournalStartup,
|
||||
minijinja::context! {
|
||||
snapshot_json => snapshot_json,
|
||||
},
|
||||
)
|
||||
}
|
||||
pub fn subjektiv_resident_context(
|
||||
&self,
|
||||
behavior_md: &str,
|
||||
|
||||
@@ -50,6 +50,7 @@ pub struct WorkerSharedState {
|
||||
fs_view: OnceLock<WorkerFsView>,
|
||||
feature_invocations: RwLock<crate::feature::FeatureInvocationRegistry>,
|
||||
flow_transition_enabled: AtomicBool,
|
||||
pub(crate) subject_release: OnceLock<std::sync::Arc<crate::subject_release::ReleaseControl>>,
|
||||
}
|
||||
|
||||
impl WorkerSharedState {
|
||||
@@ -69,6 +70,7 @@ impl WorkerSharedState {
|
||||
fs_view: OnceLock::new(),
|
||||
feature_invocations: RwLock::new(crate::feature::FeatureInvocationRegistry::default()),
|
||||
flow_transition_enabled: AtomicBool::new(false),
|
||||
subject_release: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Controller-owned quiescence with a structural persistence/admission fence.
|
||||
//!
|
||||
//! A reservation is acknowledged only by the actual idle Controller. Shared read
|
||||
//! leases cover input enqueue, durable pending admission, and history writes; the
|
||||
//! exclusive lease spans the Host's reconciliation and ownership release. No
|
||||
//! Journal policy or catalog-status inference belongs here.
|
||||
|
||||
use session_store::StoreError;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use tokio::sync::{Notify, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock, oneshot};
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct MutationAuthority {
|
||||
barrier: Arc<RwLock<()>>,
|
||||
disconnected: AtomicBool,
|
||||
}
|
||||
impl MutationAuthority {
|
||||
pub(crate) fn enter(&self) -> Result<OwnedRwLockReadGuard<()>, StoreError> {
|
||||
let lease = self
|
||||
.barrier
|
||||
.clone()
|
||||
.try_read_owned()
|
||||
.map_err(|_| denied())?;
|
||||
if self.disconnected.load(Ordering::Acquire) {
|
||||
return Err(denied());
|
||||
}
|
||||
Ok(lease)
|
||||
}
|
||||
}
|
||||
fn denied() -> StoreError {
|
||||
StoreError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
"Subject release has fenced Worker admission and history mutation",
|
||||
))
|
||||
}
|
||||
|
||||
type Reply = oneshot::Sender<Result<SubjectReleaseReservation, String>>;
|
||||
struct Requests {
|
||||
alive: bool,
|
||||
queue: VecDeque<Reply>,
|
||||
}
|
||||
pub(crate) struct ReleaseControl {
|
||||
pub(crate) authority: Arc<MutationAuthority>,
|
||||
requests: Mutex<Requests>,
|
||||
changed: Notify,
|
||||
}
|
||||
impl ReleaseControl {
|
||||
pub(crate) fn new(authority: Arc<MutationAuthority>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
authority,
|
||||
requests: Mutex::new(Requests {
|
||||
alive: true,
|
||||
queue: VecDeque::new(),
|
||||
}),
|
||||
changed: Notify::new(),
|
||||
})
|
||||
}
|
||||
pub(crate) async fn request(self: &Arc<Self>) -> Result<SubjectReleaseReservation, String> {
|
||||
let (reply, receive) = oneshot::channel();
|
||||
{
|
||||
let mut requests = self
|
||||
.requests
|
||||
.lock()
|
||||
.map_err(|_| "Controller release requests poisoned")?;
|
||||
if !requests.alive {
|
||||
return Err("Controller is unavailable".into());
|
||||
}
|
||||
requests.queue.push_back(reply);
|
||||
}
|
||||
self.changed.notify_one();
|
||||
receive
|
||||
.await
|
||||
.map_err(|_| "Controller closed before release acknowledgement".to_string())?
|
||||
}
|
||||
pub(crate) async fn notified(&self) {
|
||||
self.changed.notified().await;
|
||||
}
|
||||
pub(crate) fn respond(self: &Arc<Self>, check_quiescent: impl Fn() -> Result<(), String>) {
|
||||
let requests = {
|
||||
let mut state = self
|
||||
.requests
|
||||
.lock()
|
||||
.expect("Controller release requests poisoned");
|
||||
state.queue.drain(..).collect::<Vec<_>>()
|
||||
};
|
||||
for reply in requests {
|
||||
if reply.is_closed() {
|
||||
continue;
|
||||
}
|
||||
let result = self
|
||||
.authority
|
||||
.barrier
|
||||
.clone()
|
||||
.try_write_owned()
|
||||
.map_err(|_| "Controller admission or history mutation is active".to_string())
|
||||
.and_then(|exclusive| {
|
||||
if self.authority.disconnected.load(Ordering::Acquire) {
|
||||
return Err("Controller body is disconnected".into());
|
||||
}
|
||||
check_quiescent()?;
|
||||
Ok(SubjectReleaseReservation {
|
||||
control: self.clone(),
|
||||
_exclusive: exclusive,
|
||||
committed: false,
|
||||
})
|
||||
});
|
||||
// If the waiter disappeared, dropping the unsent guard cancels the fence.
|
||||
let _ = reply.send(result);
|
||||
}
|
||||
}
|
||||
pub(crate) fn reject_busy(self: &Arc<Self>) {
|
||||
self.respond(|| Err("Controller is not quiescent Idle".into()));
|
||||
}
|
||||
pub(crate) fn close(&self) {
|
||||
let mut state = self
|
||||
.requests
|
||||
.lock()
|
||||
.expect("Controller release requests poisoned");
|
||||
state.alive = false;
|
||||
for reply in state.queue.drain(..) {
|
||||
let _ = reply.send(Err("Controller is unavailable".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ControllerLifetime(pub(crate) Arc<ReleaseControl>);
|
||||
impl Drop for ControllerLifetime {
|
||||
fn drop(&mut self) {
|
||||
self.0.close();
|
||||
}
|
||||
}
|
||||
|
||||
/// Exclusive permission to perform Host-side Subject handoff while the old body
|
||||
/// cannot accept input or change source history. Dropping an uncommitted guard
|
||||
/// cancels the reservation. Committing permanently revokes body admission; it
|
||||
/// does not stop the Controller or disable read-only observation and shutdown.
|
||||
pub struct SubjectReleaseReservation {
|
||||
control: Arc<ReleaseControl>,
|
||||
_exclusive: OwnedRwLockWriteGuard<()>,
|
||||
committed: bool,
|
||||
}
|
||||
impl SubjectReleaseReservation {
|
||||
pub fn commit_disconnect(&mut self) -> Result<(), String> {
|
||||
if self.committed {
|
||||
return Ok(());
|
||||
}
|
||||
let state = self
|
||||
.control
|
||||
.requests
|
||||
.lock()
|
||||
.map_err(|_| "Controller release requests poisoned")?;
|
||||
if !state.alive {
|
||||
return Err("Controller stopped before disconnect commit".into());
|
||||
}
|
||||
self.control
|
||||
.authority
|
||||
.disconnected
|
||||
.store(true, Ordering::Release);
|
||||
self.committed = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
// Structural lease test: the Controller's grant path is production code;
|
||||
// the held reader represents an already-entered synchronous persistence call.
|
||||
#[tokio::test]
|
||||
async fn in_flight_mutation_rejects_release_without_waiting_and_retry_recovers() {
|
||||
let authority = Arc::new(MutationAuthority::default());
|
||||
let control = ReleaseControl::new(authority.clone());
|
||||
let (entered, entry) = oneshot::channel();
|
||||
let (finish, finished) = oneshot::channel();
|
||||
let writer = tokio::spawn({
|
||||
let authority = authority.clone();
|
||||
async move {
|
||||
let _lease = authority.enter().unwrap();
|
||||
entered.send(()).unwrap();
|
||||
finished.await.unwrap();
|
||||
}
|
||||
});
|
||||
entry.await.unwrap();
|
||||
let request = tokio::spawn({
|
||||
let control = control.clone();
|
||||
async move { control.request().await }
|
||||
});
|
||||
control.notified().await;
|
||||
control.respond(|| Ok(()));
|
||||
assert!(request.await.unwrap().is_err());
|
||||
finish.send(()).unwrap();
|
||||
writer.await.unwrap();
|
||||
let request = tokio::spawn({
|
||||
let control = control.clone();
|
||||
async move { control.request().await }
|
||||
});
|
||||
control.notified().await;
|
||||
control.respond(|| Ok(()));
|
||||
let guard = request.await.unwrap().unwrap();
|
||||
assert!(authority.enter().is_err());
|
||||
drop(guard);
|
||||
assert!(authority.enter().is_ok());
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn closed_controller_rejects_unanswered_request_and_guard_disconnect() {
|
||||
let control = ReleaseControl::new(Arc::default());
|
||||
let request = tokio::spawn({
|
||||
let control = control.clone();
|
||||
async move { control.request().await }
|
||||
});
|
||||
control.notified().await;
|
||||
control.respond(|| Ok(()));
|
||||
let mut guard = request.await.unwrap().unwrap();
|
||||
control.close();
|
||||
assert!(guard.commit_disconnect().is_err());
|
||||
assert!(control.request().await.is_err());
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,8 @@ pub use crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifec
|
||||
pub use crate::feature::builtin::journal::{
|
||||
JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature, JournalRecordingFeature,
|
||||
};
|
||||
pub use subjektiv::journal::memory_links::{MemoryJournalRequest, MemoryJournalResponse};
|
||||
use subjektiv::journal::startup::JournalStartupSnapshot;
|
||||
pub use subjektiv::journal::transport;
|
||||
use transport::{JournalRecordingReason, JournalRecordingRequest, JournalRecordingResponse};
|
||||
|
||||
@@ -66,6 +68,18 @@ impl From<SubjektivHostError> for ToolError {
|
||||
/// identifier is not a grant. No method grants ambient filesystem access.
|
||||
pub trait SubjektivHost: Send + Sync + 'static {
|
||||
fn settings(&self) -> SubjektivHostSettings;
|
||||
/// Startup reads are a body-only capability, independent of recording Jobs.
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
_context: &SubjektivHostContext,
|
||||
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
|
||||
Err(SubjektivHostError::Unavailable(
|
||||
"Journal startup not granted".into(),
|
||||
))
|
||||
}
|
||||
/// Recording admission is separate from both Memory and Journal Job operation grants.
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
false
|
||||
@@ -79,6 +93,17 @@ pub trait SubjektivHost: Send + Sync + 'static {
|
||||
"Journal recording not granted".into(),
|
||||
))
|
||||
}
|
||||
/// Additive provenance links and direct Journal reads, separate from recording/Memory mutation.
|
||||
/// Restricted Job Hosts inherit denial rather than the main Subject reader capability.
|
||||
fn memory_journal(
|
||||
&self,
|
||||
_context: &SubjektivHostContext,
|
||||
_request: MemoryJournalRequest,
|
||||
) -> Result<MemoryJournalResponse, SubjektivHostError> {
|
||||
Err(SubjektivHostError::Unavailable(
|
||||
"Memory Journal references not granted".into(),
|
||||
))
|
||||
}
|
||||
/// Separate explicit grant for decision/surface execution; Profile policy
|
||||
/// alone must never authorize consolidation tools.
|
||||
fn consolidation_granted(&self) -> bool {
|
||||
@@ -207,6 +232,41 @@ fn decode<T: serde::de::DeserializeOwned>(
|
||||
}
|
||||
|
||||
impl SubjektivHost for BackendSubjektivHost {
|
||||
fn memory_journal(
|
||||
&self,
|
||||
_context: &SubjektivHostContext,
|
||||
request: MemoryJournalRequest,
|
||||
) -> Result<MemoryJournalResponse, SubjektivHostError> {
|
||||
decode(
|
||||
self.client
|
||||
.execute(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
format!("/api/w/{}/subjektiv/journal/memory", self.workspace_id),
|
||||
serde_json::to_string(&request)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?,
|
||||
))
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?,
|
||||
)
|
||||
}
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
_context: &SubjektivHostContext,
|
||||
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
|
||||
decode(
|
||||
self.client
|
||||
.execute_with_timeout(
|
||||
WorkspaceRequest::get(format!(
|
||||
"/api/w/{}/subjektiv/journal/startup",
|
||||
self.workspace_id
|
||||
)),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?,
|
||||
)
|
||||
}
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
// This adapter exists only for explicitly enabled, authenticated Host connections.
|
||||
// The Backend still verifies the main Worker/Subject binding on every request.
|
||||
@@ -420,6 +480,14 @@ pub struct SubjektivHostConnection {
|
||||
}
|
||||
|
||||
impl SubjektivHostConnection {
|
||||
pub fn journal_startup_context(&self) -> Result<JournalStartupSnapshot, SubjektivHostError> {
|
||||
if !self.host.journal_startup_granted() {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"Journal startup not granted".into(),
|
||||
));
|
||||
}
|
||||
self.host.journal_startup_context(&self.context)
|
||||
}
|
||||
pub fn new(
|
||||
host: Arc<dyn SubjektivHost>,
|
||||
context: SubjektivHostContext,
|
||||
@@ -438,6 +506,12 @@ impl SubjektivHostConnection {
|
||||
}
|
||||
Ok(Self { host, context })
|
||||
}
|
||||
pub fn memory_journal(
|
||||
&self,
|
||||
request: MemoryJournalRequest,
|
||||
) -> Result<MemoryJournalResponse, SubjektivHostError> {
|
||||
self.host.memory_journal(&self.context, request)
|
||||
}
|
||||
pub fn memory(
|
||||
&self,
|
||||
operation: SubjektivMemoryBackendOperation,
|
||||
@@ -546,12 +620,33 @@ mod tests {
|
||||
|
||||
#[derive(Default)]
|
||||
struct LocalHost {
|
||||
startup: Option<JournalStartupSnapshot>,
|
||||
startup_error: bool,
|
||||
startup_calls: std::sync::atomic::AtomicUsize,
|
||||
contexts: Mutex<Vec<SubjektivHostContext>>,
|
||||
attributed: Mutex<Vec<(SubjektivHostContext, bool)>>,
|
||||
journal_observer:
|
||||
Option<Arc<dyn Fn(&SubjektivHostContext, JournalRecordingReason) + Send + Sync>>,
|
||||
}
|
||||
impl SubjektivHost for LocalHost {
|
||||
fn journal_startup_granted(&self) -> bool {
|
||||
self.startup.is_some() || self.startup_error
|
||||
}
|
||||
fn journal_startup_context(
|
||||
&self,
|
||||
_context: &SubjektivHostContext,
|
||||
) -> Result<JournalStartupSnapshot, SubjektivHostError> {
|
||||
self.startup_calls
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
if self.startup_error {
|
||||
return Err(SubjektivHostError::Unavailable(
|
||||
"startup read failed".into(),
|
||||
));
|
||||
}
|
||||
self.startup
|
||||
.clone()
|
||||
.ok_or_else(|| SubjektivHostError::Unavailable("startup not granted".into()))
|
||||
}
|
||||
fn journal_recording_granted(&self) -> bool {
|
||||
self.journal_observer.is_some()
|
||||
}
|
||||
@@ -919,6 +1014,223 @@ enabled = true
|
||||
);
|
||||
}
|
||||
|
||||
// Production Direct bootstrap, Feature selection/rendering, Core, FsStore,
|
||||
// and model Request. The authenticated Host boundary and provider are fake;
|
||||
// real Backend/standalone authority and live continuation are not claimed.
|
||||
#[tokio::test]
|
||||
async fn direct_startup_journal_is_durable_before_first_user_and_provider_exactly_once() {
|
||||
use session_store::{LogEntry, LoggedSessionHistoryOrigin, Store, SystemItem};
|
||||
use std::sync::atomic::Ordering;
|
||||
for empty in [false, true] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let name = format!("startup-{}", uuid::Uuid::now_v7());
|
||||
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='{name}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]")).unwrap();
|
||||
manifest.feature.subjektiv.profile.enabled = true;
|
||||
manifest.feature.subjektiv.profile.extraction.enabled = false;
|
||||
let snapshot = JournalStartupSnapshot {
|
||||
subject_id: "subject".into(),
|
||||
omitted: false,
|
||||
journals: if empty {
|
||||
vec![]
|
||||
} else {
|
||||
vec![subjektiv::journal::startup::StartupJournal {
|
||||
journal_id: "journal".into(),
|
||||
source_session_id: "previous-session".into(),
|
||||
state_token: "consistent-token".into(),
|
||||
process_order: 4,
|
||||
process_byte: 0,
|
||||
current_position: "B adopted after correcting A; implementation unfinished"
|
||||
.into(),
|
||||
entries: vec![],
|
||||
omitted: true,
|
||||
}]
|
||||
},
|
||||
};
|
||||
let host = Arc::new(LocalHost {
|
||||
startup: Some(snapshot.clone()),
|
||||
..LocalHost::default()
|
||||
});
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||
);
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let prepared = WorkerBootstrap::new(
|
||||
manifest,
|
||||
store.clone(),
|
||||
crate::PromptCatalogSource::builtins_only(),
|
||||
WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::None,
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
bash_output_dir: temp.path().join("bash"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx))
|
||||
.with_subjektiv_host(host.clone())
|
||||
.prepare()
|
||||
.await
|
||||
.unwrap();
|
||||
let (session, segment) = (
|
||||
prepared.worker().session_id(),
|
||||
prepared.worker().segment_id(),
|
||||
);
|
||||
let controller = prepared.start().await.unwrap();
|
||||
let initial = store.read_all(session, segment).unwrap();
|
||||
let journal = initial
|
||||
.iter()
|
||||
.find_map(|entry| match entry {
|
||||
LogEntry::AnnotatedSystemItem { entry, .. }
|
||||
if matches!(entry.item, SystemItem::JournalStartup { .. }) =>
|
||||
{
|
||||
Some(entry)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
journal.metadata.origin,
|
||||
LoggedSessionHistoryOrigin::DerivedSummary
|
||||
);
|
||||
let SystemItem::JournalStartup {
|
||||
body,
|
||||
snapshot: saved,
|
||||
snapshot_sha256,
|
||||
prompt_provenance,
|
||||
} = &journal.item
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(*saved, serde_json::to_value(&snapshot).unwrap());
|
||||
use sha2::{Digest, Sha256};
|
||||
assert_eq!(
|
||||
*snapshot_sha256,
|
||||
Sha256::digest(serde_json::to_vec(saved).unwrap())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>()
|
||||
);
|
||||
assert_eq!(
|
||||
prompt_provenance.logical_name,
|
||||
"internal.subjektiv_journal_startup"
|
||||
);
|
||||
assert!(
|
||||
!initial
|
||||
.iter()
|
||||
.any(|e| matches!(e, LogEntry::AnnotatedUserInput { .. }))
|
||||
);
|
||||
let mut events = controller.handle.subscribe();
|
||||
for user in ["FIRST_USER_REQUEST", "SECOND_USER_REQUEST"] {
|
||||
controller
|
||||
.handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
user,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let request = tokio::time::timeout(Duration::from_secs(5), rx.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let index = request
|
||||
.items
|
||||
.iter()
|
||||
.position(|item| item.as_text() == Some(body.as_str()))
|
||||
.unwrap();
|
||||
let user_index = request
|
||||
.items
|
||||
.iter()
|
||||
.position(|item| item.as_text() == Some("FIRST_USER_REQUEST"))
|
||||
.unwrap();
|
||||
assert!(index < user_index);
|
||||
assert_eq!(
|
||||
request
|
||||
.items
|
||||
.iter()
|
||||
.filter(|item| item.as_text() == Some(body.as_str()))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if matches!(events.recv().await.unwrap(), protocol::Event::RunEnd { .. }) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(host.startup_calls.load(Ordering::SeqCst), 1);
|
||||
let logs = store.read_all(session, segment).unwrap();
|
||||
assert_eq!(logs.iter().filter(|entry| matches!(entry, LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, SystemItem::JournalStartup { .. }))).count(), 1);
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(1),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), controller.shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
controller.controller_task.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_startup_read_error_rejects_exposure_without_initialized_head() {
|
||||
use session_store::Store;
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let _sandbox =
|
||||
crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(temp.path());
|
||||
let mut manifest = WorkerManifest::from_toml(&format!("[worker]\nname='startup-error-{}'\n[model]\nscheme='anthropic'\nmodel_id='scripted'\n[engine]\n[scope]\nallow=[]", uuid::Uuid::now_v7())).unwrap();
|
||||
manifest.feature.subjektiv.profile.enabled = true;
|
||||
manifest.feature.subjektiv.profile.extraction.enabled = false;
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||
FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||
);
|
||||
let host = Arc::new(LocalHost {
|
||||
startup_error: true,
|
||||
..LocalHost::default()
|
||||
});
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let prepared = WorkerBootstrap::new(
|
||||
manifest,
|
||||
store.clone(),
|
||||
crate::PromptCatalogSource::builtins_only(),
|
||||
WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::None,
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
bash_output_dir: temp.path().join("bash"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx))
|
||||
.with_subjektiv_host(host.clone())
|
||||
.prepare()
|
||||
.await
|
||||
.unwrap();
|
||||
let (session, segment) = (
|
||||
prepared.worker().session_id(),
|
||||
prepared.worker().segment_id(),
|
||||
);
|
||||
assert!(prepared.start().await.is_err());
|
||||
assert!(!store.exists(session, segment).unwrap());
|
||||
assert!(rx.try_recv().is_err());
|
||||
assert_eq!(
|
||||
host.startup_calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
async fn launch_case(enabled: bool, attach: bool) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let _runtime_sandbox =
|
||||
@@ -1038,7 +1350,7 @@ enabled = true
|
||||
if enabled && attach {
|
||||
assert_eq!(
|
||||
*host.attributed.lock().unwrap(),
|
||||
vec![(expected.clone(), true)]
|
||||
vec![(expected.clone(), true), (expected.clone(), false)]
|
||||
);
|
||||
assert!(
|
||||
host.contexts
|
||||
|
||||
+878
-46
File diff suppressed because it is too large
Load Diff
@@ -53,7 +53,8 @@ ts-rs = "12.0.1"
|
||||
url.workspace = true
|
||||
uuid = { workspace = true, features = ["v7"] }
|
||||
zeroize.workspace = true
|
||||
webauthn-rs = { workspace = true }
|
||||
webauthn-rs = { workspace = true, features = ["conditional-ui"] }
|
||||
webauthn-rs-proto = "0.5.5"
|
||||
|
||||
[dev-dependencies]
|
||||
wip-protocol.workspace = true
|
||||
|
||||
@@ -529,7 +529,7 @@ fn activation_oldest_and_previous_schema_migrate_with_same_constraints_and_prese
|
||||
assert!(!store.is_user_pending_activation("old-user").unwrap());
|
||||
store.with_conn(|conn| {
|
||||
assert_eq!(schema(conn), expected);
|
||||
assert_eq!(current_schema_version(conn)?, 100);
|
||||
assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION);
|
||||
assert!(conn.execute("INSERT INTO user_activations VALUES('missing','issue','hash','future',0)", []).is_err());
|
||||
for attempts in [-1, 6] { assert!(conn.execute("INSERT INTO user_activations VALUES('old-user','issue','hash','future',?1)", [attempts]).is_err()); }
|
||||
Ok(())
|
||||
|
||||
@@ -474,7 +474,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
(97, "MR notifications and authenticated implementation results".into()),
|
||||
(98, "Assignment-independent implementation reports".into()),
|
||||
(99, "Explicit hosting administrators and self-registration policy".into()),
|
||||
(100, "Provisional users and scoped passkey activation".into())
|
||||
(100, "Provisional users and scoped passkey activation".into()),
|
||||
(101, "Durable explicit Subject body release".into())
|
||||
]
|
||||
);
|
||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||
|
||||
@@ -282,6 +282,10 @@ pub struct SubjektivConsolidationGrant {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubjektivJournalGrant {
|
||||
/// Frozen writer generation. Legacy absence preserves its original canonical wire/digest;
|
||||
/// recovery must use only the immutable original Session and still-current body.
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub binding_id: String,
|
||||
pub subject_id: String,
|
||||
pub session_id: String,
|
||||
pub journal_id: String,
|
||||
|
||||
@@ -1288,6 +1288,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_release(
|
||||
&self,
|
||||
_request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
Err("actual Subject Controller reservation is unavailable on this Runtime".into())
|
||||
}
|
||||
|
||||
fn session_material_pin(
|
||||
&self,
|
||||
_request: runtime_api::SessionMaterialPinRequest,
|
||||
@@ -2244,6 +2251,21 @@ impl RuntimeRegistry {
|
||||
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
|
||||
}
|
||||
|
||||
pub fn subject_release(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, RuntimeRegistryError> {
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
self.runtime(runtime_id)?
|
||||
.subject_release(request)
|
||||
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: runtime_id.into(),
|
||||
code: "subject_release_unproven".into(),
|
||||
message,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn session_material_pin(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -3297,6 +3319,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_release(
|
||||
&self,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
if request.workspace_id != self.workspace_id {
|
||||
return Err("Subject release Workspace mismatch".into());
|
||||
}
|
||||
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
|
||||
self.runtime
|
||||
.subject_release_scoped(&scope, request)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn session_material_pin(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialPinRequest,
|
||||
@@ -4114,7 +4149,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
|
||||
{
|
||||
return "workers:input";
|
||||
}
|
||||
if path.ends_with("/stop") || path.ends_with("/cancel") {
|
||||
if path.ends_with("/stop")
|
||||
|| path.ends_with("/cancel")
|
||||
|| (path == "/v1/subject-release" && method == "POST")
|
||||
{
|
||||
return "workers:stop";
|
||||
}
|
||||
if path == "/v1/protocol/ws" {
|
||||
@@ -5523,6 +5561,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_release(
|
||||
&self,
|
||||
request: runtime_api::SubjectReleaseRequest,
|
||||
) -> Result<runtime_api::SubjectReleaseResponse, String> {
|
||||
self.run_runtime_api(
|
||||
self.request_timeout,
|
||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||
move |client| async move { client.subject_release(request).await },
|
||||
)
|
||||
.map_err(|diagnostic| diagnostic.message)
|
||||
}
|
||||
|
||||
fn session_material_pin(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialPinRequest,
|
||||
|
||||
@@ -381,6 +381,8 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
|
||||
DROP TABLE ticket_implementation_results;
|
||||
DROP TABLE ticket_implementation_delegations;
|
||||
DROP TABLE merge_request_source_observations;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
@@ -513,6 +515,8 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
|
||||
// Reconstruct the prior composition before its new business capture existed.
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
@@ -671,7 +675,9 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
|
||||
if historical {
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TABLE user_activations;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||
|
||||
@@ -699,6 +699,22 @@ CREATE TABLE worker_create_reservations (
|
||||
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
|
||||
CHECK ((singleton_key IS NULL) = (singleton_generation IS NULL))
|
||||
);
|
||||
CREATE TABLE subject_worker_releases (
|
||||
workspace_id TEXT NOT NULL,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
singleton_key TEXT NOT NULL,
|
||||
singleton_generation INTEGER NOT NULL CHECK (singleton_generation > 0),
|
||||
binding_id TEXT NOT NULL,
|
||||
operation_id TEXT NOT NULL,
|
||||
proof TEXT CHECK (proof IN ('runtime_disconnected','erased')),
|
||||
completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0,1)),
|
||||
cancelled INTEGER NOT NULL DEFAULT 0 CHECK (cancelled IN (0,1)),
|
||||
PRIMARY KEY (workspace_id, operation_id),
|
||||
CHECK (cancelled = 0 OR (completed = 0 AND proof IS NULL)),
|
||||
FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_create_reservations(workspace_id, worker_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE UNIQUE INDEX subject_worker_release_active ON subject_worker_releases(workspace_id,worker_id) WHERE cancelled=0;
|
||||
CREATE TRIGGER worker_create_reservation_singleton_lease_insert
|
||||
BEFORE INSERT ON worker_create_reservations
|
||||
WHEN (NEW.singleton_key IS NULL AND NEW.singleton_generation IS NOT NULL)
|
||||
|
||||
@@ -57,6 +57,7 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
||||
"workspace_config_tree_history",
|
||||
"workspace_memory_settings",
|
||||
"worker_create_reservations",
|
||||
"subject_worker_releases",
|
||||
"backend_jobs",
|
||||
"backend_job_attempts",
|
||||
"flow_sources",
|
||||
|
||||
@@ -28,6 +28,7 @@ use subjektiv::api::*;
|
||||
mod backend_job_profile;
|
||||
mod subjektiv_job;
|
||||
mod subjektiv_journal;
|
||||
mod subjektiv_memory_journal;
|
||||
mod worker_operations;
|
||||
mod worker_protocol;
|
||||
pub(crate) use worker_operations::{
|
||||
@@ -1654,6 +1655,7 @@ pub struct WorkspaceApi {
|
||||
Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<Mutex<OrchestratorAttentionState>>>>>,
|
||||
notification_delivery_lifecycle: Arc<notification_delivery::NotificationDeliveryLifecycle>,
|
||||
backend_job_drain_requests: Arc<AtomicU64>,
|
||||
journal_lifecycle: Arc<Mutex<HashMap<String, Arc<Mutex<()>>>>>,
|
||||
journal_dispatch_owner: Option<Arc<WorkspaceApi>>,
|
||||
observation_proxy: BackendObservationProxy,
|
||||
runtime_subscription_broker: RuntimeSubscriptionBroker,
|
||||
@@ -3379,6 +3381,7 @@ fn workspace_request_is_callback(method: &Method, path: &str, workspace_id: &str
|
||||
|| path == format!("/api/runtime/v1/workspaces/{workspace_id}/resources/fetch")))
|
||||
|| (*method == Method::GET
|
||||
&& (path == format!("/api/w/{workspace_id}/runtime-config")
|
||||
|| path == format!("/api/w/{workspace_id}/subjektiv/journal/startup")
|
||||
|| path == format!("/api/w/{workspace_id}/worker-discovery/workers")))
|
||||
}
|
||||
|
||||
@@ -4160,6 +4163,7 @@ impl WorkspaceApi {
|
||||
notification_delivery_lifecycle: Arc::new(Default::default()),
|
||||
orchestrator_attention: Arc::new(Mutex::new(HashMap::new())),
|
||||
backend_job_drain_requests: Arc::new(AtomicU64::new(0)),
|
||||
journal_lifecycle: Arc::new(Mutex::new(HashMap::new())),
|
||||
journal_dispatch_owner: None,
|
||||
observation_proxy,
|
||||
runtime_subscription_broker,
|
||||
@@ -4362,8 +4366,9 @@ impl WorkspaceApi {
|
||||
resolved_worker_observation_grants: Vec::new(),
|
||||
resolved_workspace_api: None,
|
||||
resolved_memory_settings: None,
|
||||
resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some()
|
||||
|| request.grants.subjektiv_journal.is_some(),
|
||||
// Journal Jobs use their independent frozen Job Host grant, not the
|
||||
// main-Subject/Memory attachment or its resident/extraction policy.
|
||||
resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some(),
|
||||
resolved_control_operation: Some(control_operation),
|
||||
},
|
||||
) {
|
||||
@@ -5468,6 +5473,22 @@ impl WorkspaceApi {
|
||||
.map(|assignment| assignment.operation_id.clone())
|
||||
})
|
||||
.unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7()));
|
||||
let subject = singleton_key
|
||||
.as_deref()
|
||||
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
|
||||
let subject_gate = subject
|
||||
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
|
||||
.transpose()?;
|
||||
let subject_guard = subject_gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subject {
|
||||
subjektiv_journal::check_start_allowed(self, subject)?;
|
||||
}
|
||||
let reservation = match self.config_store.reserve_worker_create(
|
||||
&self.config.workspace_id,
|
||||
runtime_id,
|
||||
@@ -5490,6 +5511,7 @@ impl WorkspaceApi {
|
||||
return Err(ApiError::with_diagnostics(error, diagnostics));
|
||||
}
|
||||
};
|
||||
drop(subject_guard);
|
||||
let worker_id = reservation.worker_id;
|
||||
if let Some(assignment) = request.ticket_assignment.as_ref() {
|
||||
self.store.bind_ticket_assignment_operation_worker(
|
||||
@@ -5803,6 +5825,9 @@ impl WorkspaceApi {
|
||||
diagnostics,
|
||||
));
|
||||
}
|
||||
if let Err(error) = subjektiv_journal::reconcile_created_worker(self, &worker_ref) {
|
||||
tracing::warn!(%error, worker_id=%worker_ref.worker_id, "Journal first-run reconciliation remains recoverable; saved history retained");
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -13122,7 +13147,10 @@ fn build_inner_router(api: WorkspaceApi) -> Router {
|
||||
"/api/w/{workspace_id}/workers/self/backend-job-result",
|
||||
post(scoped_submit_backend_job_result),
|
||||
)
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/release", post(subjektiv_journal::release_route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/startup", get(subjektiv_journal::startup_route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/recording", post(subjektiv_journal::recording_route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/memory", post(subjektiv_memory_journal::route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/job", post(subjektiv_journal::job_route))
|
||||
.route("/api/w/{workspace_id}/subjektiv/journal/job/finish", post(subjektiv_journal::finish_route))
|
||||
.fallback(get(static_or_spa_fallback))
|
||||
@@ -22351,6 +22379,7 @@ async fn scoped_record_subjektiv_session(
|
||||
&request.session_id,
|
||||
)?
|
||||
};
|
||||
subjektiv_journal::bind_host_session(&api, attribution.clone())?;
|
||||
Ok(Json(server_api::SubjektivRecordSessionResponse {
|
||||
subject_id: attribution.subject_id,
|
||||
session_id: attribution.session_id,
|
||||
@@ -29670,11 +29699,13 @@ struct PasskeyRegistrationEnvelope {
|
||||
}
|
||||
|
||||
/// An account gets one stable WebAuthn user handle, without storing a new schema
|
||||
/// field. Migration leaves all existing credentials, their random registration
|
||||
/// handles and key material unchanged. Credential-ID-scoped authentication must
|
||||
/// accept signed assertions both without userHandle and with the original random
|
||||
/// or newly registered stable userHandle; it is not discoverable-handle lookup.
|
||||
/// The real-verifier compatibility tests exercise each representation explicitly.
|
||||
/// field. Legacy identified-account login accepts original random handles or no
|
||||
/// returned handle, always requiring the saved credential's signature. Normal
|
||||
/// discoverable login additionally requires the saved owner's stable handle.
|
||||
/// Random registration handles were never stored, so they cannot be recovered
|
||||
/// or safely mapped by trusting the unsigned assertion. Preserve these keys and
|
||||
/// use explicit legacy login to authenticate before adding a stable-handle RK.
|
||||
/// Merely having a stable handle does not prove the old key is discoverable.
|
||||
fn passkey_user_handle(account_id: &str) -> Uuid {
|
||||
let digest = Sha256::digest(account_id.as_bytes());
|
||||
Uuid::from_bytes(
|
||||
@@ -29778,7 +29809,7 @@ fn start_passkey_registration(
|
||||
let header_origin = request_origin(headers);
|
||||
let auth = auth_config_for_origin(&api.config, browser_origin.or(header_origin.as_deref()))?;
|
||||
let webauthn = webauthn_for_auth(&auth)?;
|
||||
let (public_key, verifier) = webauthn
|
||||
let (mut public_key, verifier) = webauthn
|
||||
.start_passkey_registration(
|
||||
passkey_user_handle(&user.account_id),
|
||||
&user.handle,
|
||||
@@ -29786,6 +29817,21 @@ fn start_passkey_registration(
|
||||
Some(exclude_credentials),
|
||||
)
|
||||
.map_err(|error| auth_error("webauthn_registration_options_failed", &error.to_string()))?;
|
||||
// webauthn-rs passkeys require UV but do not request residency by default.
|
||||
// These public protocol fields request an RK on conforming clients; residency
|
||||
// itself is not remotely attestable. Keep the library's UV/verifier state.
|
||||
let selection = public_key
|
||||
.public_key
|
||||
.authenticator_selection
|
||||
.as_mut()
|
||||
.ok_or_else(|| {
|
||||
auth_error(
|
||||
"webauthn_registration_options_failed",
|
||||
"authenticator selection is missing",
|
||||
)
|
||||
})?;
|
||||
selection.resident_key = Some(webauthn_rs_proto::ResidentKeyRequirement::Required);
|
||||
selection.require_resident_key = true;
|
||||
let challenge_id = new_id("webauthn-registration");
|
||||
// New users do not exist yet, so the nullable FK remains empty. The complete
|
||||
// target (including its allocated user ID) is inside the purpose-tagged state.
|
||||
@@ -29950,6 +29996,18 @@ fn complete_passkey_enrollment(
|
||||
issue_browser_session(api, user)
|
||||
}
|
||||
|
||||
/// Normal login has no preselected identity. Explicit handle requests are the
|
||||
/// legacy migration path, not an automatic fallback. Non-resident keys and keys
|
||||
/// with old random user handles can authenticate there, then add an RK from the
|
||||
/// live account. No saved key or owner is rewritten. Existing stable-handle RKs
|
||||
/// can already use normal login; discoverability cannot be inferred from storage.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "mode", content = "verifier", rename_all = "snake_case")]
|
||||
enum PasskeyLoginState {
|
||||
Discoverable(webauthn_rs::prelude::DiscoverableAuthentication),
|
||||
Legacy(PasskeyAuthentication),
|
||||
}
|
||||
|
||||
async fn post_passkey_login_options(
|
||||
State(api): State<ServerAuthApi>,
|
||||
headers: HeaderMap,
|
||||
@@ -29958,48 +30016,66 @@ async fn post_passkey_login_options(
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
let user = match request.handle.as_deref() {
|
||||
Some(handle) => api.store.get_user_by_handle(&normalize_handle(handle)?)?,
|
||||
None => {
|
||||
return Err(auth_error(
|
||||
"passkey_handle_required",
|
||||
"login requires an explicit handle",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
.ok_or_else(|| auth_error("unknown_auth_user", "no matching user account exists"))?;
|
||||
if api.store.is_user_pending_activation(&user.user_id)? {
|
||||
return Err(auth_error(
|
||||
"passkey_activation_required",
|
||||
"initial passkey activation is required",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let passkeys = passkeys_for_user(&api, &user.user_id)?;
|
||||
if passkeys.is_empty() {
|
||||
return Err(auth_error(
|
||||
"passkey_not_registered",
|
||||
"user has no registered passkey credentials",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let header_origin = request_origin(&headers);
|
||||
let requested_origin = request
|
||||
.browser_origin
|
||||
.as_deref()
|
||||
.or(header_origin.as_deref());
|
||||
let auth = auth_config_for_origin(&api.config, requested_origin)?;
|
||||
let auth = auth_config_for_origin(
|
||||
&api.config,
|
||||
request
|
||||
.browser_origin
|
||||
.as_deref()
|
||||
.or(header_origin.as_deref()),
|
||||
)?;
|
||||
let webauthn = webauthn_for_auth(&auth)?;
|
||||
let (public_key, state) = webauthn
|
||||
.start_passkey_authentication(&passkeys)
|
||||
.map_err(|error| auth_error("webauthn_login_options_failed", &error.to_string()))?;
|
||||
let (public_key, state, user_id) =
|
||||
if let Some(handle) = request.handle.as_deref() {
|
||||
let user = api
|
||||
.store
|
||||
.get_user_by_handle(&normalize_handle(handle)?)?
|
||||
.ok_or_else(|| {
|
||||
auth_error("unknown_auth_user", "no matching user account exists")
|
||||
})?;
|
||||
if api.store.is_user_pending_activation(&user.user_id)? {
|
||||
return Err(auth_error(
|
||||
"passkey_activation_required",
|
||||
"initial passkey activation is required",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let passkeys = passkeys_for_user(&api, &user.user_id)?;
|
||||
if passkeys.is_empty() {
|
||||
return Err(auth_error(
|
||||
"passkey_not_registered",
|
||||
"user has no registered passkey credentials",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let (options, state) = webauthn
|
||||
.start_passkey_authentication(&passkeys)
|
||||
.map_err(|error| {
|
||||
auth_error("webauthn_login_options_failed", &error.to_string())
|
||||
})?;
|
||||
(
|
||||
options,
|
||||
PasskeyLoginState::Legacy(state),
|
||||
Some(user.user_id),
|
||||
)
|
||||
} else {
|
||||
let (mut options, state) = webauthn
|
||||
.start_discoverable_authentication()
|
||||
.map_err(|error| {
|
||||
auth_error("webauthn_login_options_failed", &error.to_string())
|
||||
})?;
|
||||
// The feature exposes a conditional-UI builder. Use its empty allow
|
||||
// list and required UV verifier with an explicit user gesture, not
|
||||
// conditional/autofill mediation. No account inventory is returned.
|
||||
options.mediation = None;
|
||||
(options, PasskeyLoginState::Discoverable(state), None)
|
||||
};
|
||||
let challenge_id = new_id("webauthn-login");
|
||||
api.store.put_auth_challenge(&AuthChallengeRecord {
|
||||
challenge_id: challenge_id.clone(),
|
||||
ceremony: "passkey_login".to_string(),
|
||||
ceremony: "passkey_login".into(),
|
||||
challenge: challenge_id.clone(),
|
||||
user_id: Some(user.user_id),
|
||||
user_id,
|
||||
rp_id: auth.rp_id,
|
||||
origin: auth.origin,
|
||||
state_json: Some(serde_json::to_string(&state).map_err(|error| {
|
||||
@@ -30041,64 +30117,114 @@ fn complete_passkey_login(
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let user_id = challenge.user_id.clone().ok_or_else(|| {
|
||||
auth_error(
|
||||
"invalid_passkey_challenge",
|
||||
"passkey login challenge is not bound to a user",
|
||||
)
|
||||
})?;
|
||||
let user = api
|
||||
.store
|
||||
.get_user(&user_id)?
|
||||
.ok_or_else(|| auth_error("unknown_auth_user", "passkey user does not exist"))?;
|
||||
let state_json = challenge.state_json.clone().ok_or_else(|| {
|
||||
auth_error(
|
||||
"missing_webauthn_state",
|
||||
"passkey login state was not persisted",
|
||||
)
|
||||
})?;
|
||||
let state: PasskeyAuthentication = serde_json::from_str(&state_json)
|
||||
.map_err(|error| auth_error("webauthn_state_deserialize_failed", &error.to_string()))?;
|
||||
let webauthn = webauthn_for_challenge(&api.config, &challenge)?;
|
||||
let auth_result = webauthn
|
||||
.finish_passkey_authentication(&request.credential, &state)
|
||||
.map_err(|error| auth_error("webauthn_login_verification_failed", &error.to_string()))?;
|
||||
let credential_id = serde_json::to_value(auth_result.cred_id())
|
||||
.ok()
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.ok_or_else(|| {
|
||||
let state: PasskeyLoginState =
|
||||
serde_json::from_str(challenge.state_json.as_deref().ok_or_else(|| {
|
||||
auth_error(
|
||||
"invalid_webauthn_credential_id",
|
||||
"verified credential id was not serializable",
|
||||
"missing_webauthn_state",
|
||||
"passkey login state was not persisted",
|
||||
)
|
||||
})?;
|
||||
})?)
|
||||
.map_err(|error| auth_error("webauthn_state_deserialize_failed", &error.to_string()))?;
|
||||
let credential_id = serde_json::to_value(webauthn_rs::prelude::Base64UrlSafeData::from(
|
||||
request.credential.get_credential_id().to_vec(),
|
||||
))
|
||||
.ok()
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.ok_or_else(|| auth_error("invalid_webauthn_credential_id", "credential id is invalid"))?;
|
||||
let stored = api
|
||||
.store
|
||||
.get_passkey_credential(&credential_id)?
|
||||
.ok_or_else(|| {
|
||||
auth_error(
|
||||
"unknown_passkey_credential",
|
||||
"verified passkey credential is not registered",
|
||||
"passkey credential is not registered",
|
||||
)
|
||||
})?;
|
||||
if stored.user_id != user.user_id {
|
||||
let user = api
|
||||
.store
|
||||
.get_user(&stored.user_id)?
|
||||
.ok_or_else(|| auth_error("unknown_auth_user", "passkey user does not exist"))?;
|
||||
if api.store.is_user_pending_activation(&user.user_id)? {
|
||||
return Err(auth_error(
|
||||
"passkey_user_mismatch",
|
||||
"verified passkey credential does not belong to the challenged user",
|
||||
"passkey_activation_required",
|
||||
"initial passkey activation is required",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let mut passkey: Passkey = serde_json::from_str(&stored.public_key_cose)
|
||||
.map_err(|error| auth_error("webauthn_passkey_deserialize_failed", &error.to_string()))?;
|
||||
if passkey_credential_id(&passkey)? != credential_id {
|
||||
return Err(auth_error(
|
||||
"passkey_user_mismatch",
|
||||
"stored credential id does not match key material",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let webauthn = webauthn_for_challenge(&api.config, &challenge)?;
|
||||
let auth_result = match state {
|
||||
PasskeyLoginState::Discoverable(state) => {
|
||||
// userHandle is unsigned routing data, never identity authority. The
|
||||
// candidate owner comes only from the exact saved credential ID;
|
||||
// its stable handle must agree, and the saved key must verify below.
|
||||
if challenge.user_id.is_some()
|
||||
|| request.credential.get_user_unique_id()
|
||||
!= Some(passkey_user_handle(&user.account_id).as_bytes().as_slice())
|
||||
{
|
||||
return Err(auth_error(
|
||||
"passkey_user_mismatch",
|
||||
"credential owner/user handle mismatch; use explicit legacy login for old keys",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
webauthn.finish_discoverable_authentication(
|
||||
&request.credential,
|
||||
state,
|
||||
&[passkey.clone().into()],
|
||||
)
|
||||
}
|
||||
PasskeyLoginState::Legacy(state) => {
|
||||
if challenge.user_id.as_deref() != Some(user.user_id.as_str()) {
|
||||
return Err(auth_error(
|
||||
"passkey_user_mismatch",
|
||||
"credential does not belong to the challenged user",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
webauthn.finish_passkey_authentication(&request.credential, &state)
|
||||
}
|
||||
}
|
||||
.map_err(|error| auth_error("webauthn_login_verification_failed", &error.to_string()))?;
|
||||
if auth_result.cred_id().as_slice() != request.credential.get_credential_id() {
|
||||
return Err(auth_error(
|
||||
"passkey_user_mismatch",
|
||||
"verified credential differs from saved credential",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if stored.sign_count > 0
|
||||
&& auth_result.counter() > 0
|
||||
&& u64::from(auth_result.counter()) <= stored.sign_count
|
||||
{
|
||||
return Err(auth_error(
|
||||
"webauthn_login_verification_failed",
|
||||
"credential counter did not advance",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
passkey.update_credential(&auth_result);
|
||||
let public_key_cose = serde_json::to_string(&passkey)
|
||||
.map_err(|error| auth_error("webauthn_state_serialize_failed", &error.to_string()))?;
|
||||
api.store
|
||||
.upsert_passkey_credential(&PasskeyCredentialRecord {
|
||||
credential_id,
|
||||
user_id: user.user_id.clone(),
|
||||
public_key_cose: stored.public_key_cose,
|
||||
public_key_cose,
|
||||
transports_json: stored.transports_json,
|
||||
sign_count: u64::from(auth_result.counter()),
|
||||
created_at: stored.created_at,
|
||||
last_used_at: Some(crate::auth::now_rfc3339()),
|
||||
})?;
|
||||
issue_browser_session(&api, user)
|
||||
issue_browser_session(api, user)
|
||||
}
|
||||
|
||||
fn issue_browser_session(
|
||||
@@ -36815,6 +36941,7 @@ mod tests {
|
||||
mod subject_spawn_tests;
|
||||
mod subjektiv_jobs_tests;
|
||||
mod subjektiv_journal_tests;
|
||||
mod subjektiv_memory_journal_tests;
|
||||
mod ticket_evidence_tests;
|
||||
mod ticket_worker_claim_tests;
|
||||
mod ticket_worker_launch_tests;
|
||||
@@ -46462,8 +46589,8 @@ mod tests {
|
||||
let runtime = WorkdirlessFixtureRuntime::default();
|
||||
api.runtime.register_or_replace(runtime.clone());
|
||||
let request = |operation_id: &str| WorkerSpawnRequest {
|
||||
requested_worker_name: Some("Subject Worker".to_string()),
|
||||
singleton_key: Some("subjektiv:subject-retry".to_string()),
|
||||
requested_worker_name: Some("Generic singleton Worker".to_string()),
|
||||
singleton_key: Some("fixture:subject-retry".to_string()),
|
||||
intent: WorkerSpawnIntent::WorkspaceCompanion,
|
||||
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
|
||||
expected_segments: 0,
|
||||
@@ -46500,7 +46627,7 @@ mod tests {
|
||||
// Runtime dispatch. The restarted request receives a fresh manual allocation key, yet its
|
||||
// identical singleton intent replays the reserved Worker and dispatches that generation.
|
||||
let mut restart_request = request("ignored-manual-operation");
|
||||
restart_request.singleton_key = Some("subjektiv:restart-retry".to_string());
|
||||
restart_request.singleton_key = Some("fixture:restart-retry".to_string());
|
||||
restart_request.resolved_control_operation = None;
|
||||
let restart_fingerprint = worker_spawn_create_fingerprint(&restart_request).unwrap();
|
||||
let memory = api
|
||||
@@ -46547,7 +46674,7 @@ mod tests {
|
||||
assert_eq!(rejected.state, InternalWorkerOperationState::Rejected);
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "subjektiv:subject-retry")
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "fixture:subject-retry")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
@@ -46561,14 +46688,14 @@ mod tests {
|
||||
let worker = created.worker.unwrap();
|
||||
let current_owner = api
|
||||
.store
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "subjektiv:subject-retry")
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "fixture:subject-retry")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(current_owner.worker, worker.worker);
|
||||
assert_eq!(current_owner.generation, 3);
|
||||
assert_eq!(
|
||||
worker.singleton_key,
|
||||
Some("subjektiv:subject-retry".to_string())
|
||||
Some("fixture:subject-retry".to_string())
|
||||
);
|
||||
|
||||
// Idle and paused Runtime metadata never releases Backend ownership, and a conflict is
|
||||
@@ -46598,7 +46725,7 @@ mod tests {
|
||||
assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted);
|
||||
assert_eq!(
|
||||
restored.worker.unwrap().singleton_key.as_deref(),
|
||||
Some("subjektiv:subject-retry")
|
||||
Some("fixture:subject-retry")
|
||||
);
|
||||
|
||||
// Replacing the connected Runtime with an empty instance models an unavailable/restarted
|
||||
@@ -46618,10 +46745,7 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(
|
||||
&api.config.workspace_id,
|
||||
"subjektiv:subject-retry",
|
||||
)
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "fixture:subject-retry",)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.worker,
|
||||
@@ -46698,10 +46822,7 @@ mod tests {
|
||||
));
|
||||
assert_eq!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(
|
||||
&api.config.workspace_id,
|
||||
"subjektiv:subject-retry",
|
||||
)
|
||||
.current_worker_singleton_owner(&api.config.workspace_id, "fixture:subject-retry",)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.worker,
|
||||
@@ -51676,16 +51797,20 @@ mod tests {
|
||||
}
|
||||
|
||||
/// Generated HTTP admission, real SQLite and WebAuthn verification are connected
|
||||
/// here. Only the physical authenticator is replaced with SoftPasskey; all
|
||||
/// here. The authenticator is a resident software model with a SoftPasskey signing engine; all
|
||||
/// identities used for addition come from cookies/tokens issued by real login.
|
||||
mod passkey_security {
|
||||
use super::*;
|
||||
use webauthn_authenticator_rs::{WebauthnAuthenticator, softpasskey::SoftPasskey};
|
||||
mod resident_model {
|
||||
include!("../../../tests/e2e/src/resident_authenticator.rs");
|
||||
}
|
||||
use resident_model::ResidentAuthenticator;
|
||||
|
||||
include!("server_activation_tests.rs");
|
||||
include!("server_discoverable_tests.rs");
|
||||
|
||||
const ORIGIN_URL: &str = "https://workspace.test";
|
||||
type Authenticator = WebauthnAuthenticator<SoftPasskey>;
|
||||
type Authenticator = ResidentAuthenticator;
|
||||
|
||||
struct Fixture {
|
||||
_root: tempfile::TempDir,
|
||||
@@ -52013,7 +52138,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn authenticator() -> Authenticator {
|
||||
WebauthnAuthenticator::new(SoftPasskey::new(true))
|
||||
ResidentAuthenticator::new(true)
|
||||
}
|
||||
|
||||
fn rejected(result: (StatusCode, HeaderMap, Value), status: StatusCode, code: &str) {
|
||||
@@ -52204,17 +52329,17 @@ mod tests {
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
rejected(
|
||||
f.post(
|
||||
let (status, _, options) = f
|
||||
.post(
|
||||
"/api/auth/passkeys/login/options",
|
||||
None,
|
||||
Some(ORIGIN_URL),
|
||||
serde_json::json!({}),
|
||||
)
|
||||
.await,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"passkey_handle_required",
|
||||
);
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{options}");
|
||||
let options: PasskeyLoginOptionsResponse = serde_json::from_value(options).unwrap();
|
||||
assert!(options.public_key.public_key.allow_credentials.is_empty());
|
||||
assert_eq!(f.credentials(&user), originals);
|
||||
}
|
||||
|
||||
@@ -53059,113 +53184,152 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn legacy_random_user_handle_credential_authenticates_before_and_after_stable_handle_addition()
|
||||
{
|
||||
let f = Fixture::new().await;
|
||||
let api = ServerAuthApi::from(&f.api);
|
||||
let user = allocate_new_user(&api, "legacy", None).unwrap();
|
||||
let random_handle = Uuid::now_v7();
|
||||
assert_ne!(random_handle, passkey_user_handle(&user.account_id));
|
||||
let webauthn = webauthn_for_auth(&auth_public_config(&api.config)).unwrap();
|
||||
let (options, state) = webauthn
|
||||
.start_passkey_registration(random_handle, &user.handle, &user.display_name, None)
|
||||
.unwrap();
|
||||
let legacy_user_handle = options.public_key.user.id.clone();
|
||||
assert_eq!(legacy_user_handle.as_slice(), random_handle.as_bytes());
|
||||
let mut legacy_authenticator = authenticator();
|
||||
let credential = legacy_authenticator
|
||||
.do_registration(Url::parse(ORIGIN_URL).unwrap(), options)
|
||||
.unwrap();
|
||||
let passkey = webauthn
|
||||
.finish_passkey_registration(&credential, &state)
|
||||
.unwrap();
|
||||
let record = PasskeyCredentialRecord {
|
||||
credential_id: passkey_credential_id(&passkey).unwrap(),
|
||||
user_id: user.user_id.clone(),
|
||||
public_key_cose: serde_json::to_string(&passkey).unwrap(),
|
||||
transports_json: None,
|
||||
sign_count: 0,
|
||||
created_at: crate::auth::now_rfc3339(),
|
||||
last_used_at: None,
|
||||
};
|
||||
f.api
|
||||
.store
|
||||
.register_verified_passkey(
|
||||
&PasskeyRegistrationTarget::NewAccount { user: user.clone() },
|
||||
&record,
|
||||
)
|
||||
.unwrap();
|
||||
let expected = user_response(user.clone());
|
||||
for returned_handle in [None, Some(legacy_user_handle.as_slice())] {
|
||||
let (_, logged_in) = f
|
||||
.login_session_with_user_handle(
|
||||
"legacy",
|
||||
&mut legacy_authenticator,
|
||||
returned_handle,
|
||||
for resident in [false, true] {
|
||||
let f = Fixture::new().await;
|
||||
let api = ServerAuthApi::from(&f.api);
|
||||
let user = allocate_new_user(&api, "legacy", None).unwrap();
|
||||
let random_handle = Uuid::now_v7();
|
||||
assert_ne!(random_handle, passkey_user_handle(&user.account_id));
|
||||
let webauthn = webauthn_for_auth(&auth_public_config(&api.config)).unwrap();
|
||||
let (mut options, state) = webauthn
|
||||
.start_passkey_registration(
|
||||
random_handle,
|
||||
&user.handle,
|
||||
&user.display_name,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(logged_in, expected);
|
||||
}
|
||||
let (cookie, _) = f
|
||||
.login_session_with_user_handle(
|
||||
"legacy",
|
||||
&mut legacy_authenticator,
|
||||
Some(legacy_user_handle.as_slice()),
|
||||
)
|
||||
.await;
|
||||
let before_add = f.credentials(&expected);
|
||||
let mut added = authenticator();
|
||||
let options = f.options("addition", Some(&cookie), "").await;
|
||||
let stable_user_handle = options.public_key.public_key.user.id.clone();
|
||||
assert_eq!(
|
||||
stable_user_handle.as_slice(),
|
||||
passkey_user_handle(&user.account_id).as_bytes()
|
||||
);
|
||||
assert_ne!(stable_user_handle, legacy_user_handle);
|
||||
let request = Fixture::credential(options, &mut added);
|
||||
let (status, _, body) = f
|
||||
.post(
|
||||
"/api/auth/passkeys/addition/complete",
|
||||
Some(&cookie),
|
||||
Some(ORIGIN_URL),
|
||||
request,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(
|
||||
serde_json::from_value::<AuthUserResponse>(body)
|
||||
.unwrap()
|
||||
.user,
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
.unwrap();
|
||||
if resident {
|
||||
let selection = options.public_key.authenticator_selection.as_mut().unwrap();
|
||||
selection.require_resident_key = true;
|
||||
selection.resident_key =
|
||||
Some(webauthn_rs_proto::ResidentKeyRequirement::Required);
|
||||
}
|
||||
let legacy_user_handle = options.public_key.user.id.clone();
|
||||
assert_eq!(legacy_user_handle.as_slice(), random_handle.as_bytes());
|
||||
let mut legacy_authenticator = authenticator();
|
||||
let credential = legacy_authenticator
|
||||
.do_registration(Url::parse(ORIGIN_URL).unwrap(), options)
|
||||
.unwrap();
|
||||
let passkey = webauthn
|
||||
.finish_passkey_registration(&credential, &state)
|
||||
.unwrap();
|
||||
let record = PasskeyCredentialRecord {
|
||||
credential_id: passkey_credential_id(&passkey).unwrap(),
|
||||
user_id: user.user_id.clone(),
|
||||
public_key_cose: serde_json::to_string(&passkey).unwrap(),
|
||||
transports_json: None,
|
||||
sign_count: 0,
|
||||
created_at: crate::auth::now_rfc3339(),
|
||||
last_used_at: None,
|
||||
};
|
||||
f.api
|
||||
.store
|
||||
.get_passkey_credential(&record.credential_id)
|
||||
.unwrap()
|
||||
.as_ref(),
|
||||
Some(&before_add[0])
|
||||
);
|
||||
for returned_handle in [None, Some(legacy_user_handle.as_slice())] {
|
||||
let (_, logged_in) = f
|
||||
.register_verified_passkey(
|
||||
&PasskeyRegistrationTarget::NewAccount { user: user.clone() },
|
||||
&record,
|
||||
)
|
||||
.unwrap();
|
||||
let expected = user_response(user.clone());
|
||||
if resident {
|
||||
let normal = f.discoverable_request(&mut legacy_authenticator).await;
|
||||
rejected(
|
||||
f.post(
|
||||
"/api/auth/passkeys/login/complete",
|
||||
None,
|
||||
Some(ORIGIN_URL),
|
||||
serde_json::to_value(normal).unwrap(),
|
||||
)
|
||||
.await,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"passkey_user_mismatch",
|
||||
);
|
||||
} else {
|
||||
let normal = f.discoverable_options().await;
|
||||
assert!(
|
||||
legacy_authenticator
|
||||
.do_authentication(Url::parse(ORIGIN_URL).unwrap(), normal.public_key)
|
||||
.is_err(),
|
||||
"a non-resident key cannot be discovered without an allow list"
|
||||
);
|
||||
}
|
||||
for returned_handle in [None, Some(legacy_user_handle.as_slice())] {
|
||||
let (_, logged_in) = f
|
||||
.login_session_with_user_handle(
|
||||
"legacy",
|
||||
&mut legacy_authenticator,
|
||||
returned_handle,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(logged_in, expected);
|
||||
}
|
||||
let (cookie, _) = f
|
||||
.login_session_with_user_handle(
|
||||
"legacy",
|
||||
&mut legacy_authenticator,
|
||||
returned_handle,
|
||||
Some(legacy_user_handle.as_slice()),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(logged_in, expected);
|
||||
}
|
||||
for returned_handle in [None, Some(stable_user_handle.as_slice())] {
|
||||
let (_, logged_in) = f
|
||||
.login_session_with_user_handle("legacy", &mut added, returned_handle)
|
||||
let before_add = f.credentials(&expected);
|
||||
let mut added = authenticator();
|
||||
let options = f.options("addition", Some(&cookie), "").await;
|
||||
let stable_user_handle = options.public_key.public_key.user.id.clone();
|
||||
assert_eq!(
|
||||
stable_user_handle.as_slice(),
|
||||
passkey_user_handle(&user.account_id).as_bytes()
|
||||
);
|
||||
assert_ne!(stable_user_handle, legacy_user_handle);
|
||||
let request = Fixture::credential(options, &mut added);
|
||||
let (status, _, body) = f
|
||||
.post(
|
||||
"/api/auth/passkeys/addition/complete",
|
||||
Some(&cookie),
|
||||
Some(ORIGIN_URL),
|
||||
request,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(logged_in, expected);
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(
|
||||
serde_json::from_value::<AuthUserResponse>(body)
|
||||
.unwrap()
|
||||
.user,
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
f.api
|
||||
.store
|
||||
.get_passkey_credential(&record.credential_id)
|
||||
.unwrap()
|
||||
.as_ref(),
|
||||
Some(&before_add[0])
|
||||
);
|
||||
for returned_handle in [None, Some(legacy_user_handle.as_slice())] {
|
||||
let (_, logged_in) = f
|
||||
.login_session_with_user_handle(
|
||||
"legacy",
|
||||
&mut legacy_authenticator,
|
||||
returned_handle,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(logged_in, expected);
|
||||
}
|
||||
for returned_handle in [None, Some(stable_user_handle.as_slice())] {
|
||||
let (_, logged_in) = f
|
||||
.login_session_with_user_handle("legacy", &mut added, returned_handle)
|
||||
.await;
|
||||
assert_eq!(logged_in, expected);
|
||||
}
|
||||
assert_eq!(f.discoverable_login(&mut added).await, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generated_passkey_routes_complete_registration_login_and_logout_cookie_lifecycle() {
|
||||
use webauthn_authenticator_rs::{WebauthnAuthenticator, softpasskey::SoftPasskey};
|
||||
mod resident_model {
|
||||
include!("../../../tests/e2e/src/resident_authenticator.rs");
|
||||
}
|
||||
use resident_model::ResidentAuthenticator;
|
||||
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let mut api = test_api(workspace.path()).await;
|
||||
@@ -53182,7 +53346,7 @@ mod tests {
|
||||
let origin = origin.clone();
|
||||
let cookie_name = cookie_name.clone();
|
||||
let app = generated_auth_contract_router(ServerApiContractService::Workspace(api));
|
||||
let mut authenticator = WebauthnAuthenticator::new(SoftPasskey::new(true));
|
||||
let mut authenticator = ResidentAuthenticator::new(true);
|
||||
let authenticator_origin = url::Url::parse(&origin).unwrap();
|
||||
|
||||
let registration_options_response = app
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
||||
//! Authenticated Subject-body access to durable Memory/Journal links.
|
||||
//!
|
||||
//! The current singleton grants access; identifiers in the request do not. Saved
|
||||
//! Journal reads deliberately use a reader context, not a source Session, Runtime
|
||||
//! lookup or restore. Domain storage owns link matching, persistence and paging.
|
||||
use super::*;
|
||||
use subjektiv::journal::memory_links::{MemoryJournalRequest, MemoryJournalResponse};
|
||||
use subjektiv::journal::{JournalError, JournalHostContext};
|
||||
|
||||
fn domain(error: JournalError) -> Error {
|
||||
match error {
|
||||
JournalError::PermissionDenied(message) => Error::WorkspacePermissionDenied(message),
|
||||
JournalError::Conflict(message) => Error::RepositoryConflict(message),
|
||||
JournalError::InvalidInput(message) => Error::InvalidInput(message),
|
||||
JournalError::NotFound(id) => {
|
||||
Error::InvalidInput(format!("Journal record not found: {id}"))
|
||||
}
|
||||
JournalError::Subject(error) => subjektiv_store_error(error),
|
||||
other => Error::Store(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn route(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedWorkspacePath>,
|
||||
source: Option<axum::Extension<crate::worker_source::VerifiedRuntimeRequestSource>>,
|
||||
Json(request): Json<MemoryJournalRequest>,
|
||||
) -> ApiResult<Json<MemoryJournalResponse>> {
|
||||
let source = source
|
||||
.ok_or_else(|| {
|
||||
Error::WorkspacePermissionDenied(
|
||||
"Memory Journal requires signed Runtime request identity".into(),
|
||||
)
|
||||
})?
|
||||
.0;
|
||||
let worker_id = source.worker_id.ok_or_else(|| {
|
||||
Error::WorkspacePermissionDenied("Memory Journal requires bound Worker identity".into())
|
||||
})?;
|
||||
let worker = RuntimeWorkerRef::new(source.runtime_id, worker_id);
|
||||
// This is entirely store-owned, including callbacks while a Host parent waits
|
||||
// on Runtime. Do not put it behind Host or consult live Runtime/config state.
|
||||
api.store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
||||
let projection = api
|
||||
.store
|
||||
.worker_registry_projection(&path.workspace_id, &worker)?
|
||||
.ok_or_else(|| {
|
||||
Error::WorkspacePermissionDenied(
|
||||
"Memory Journal Worker registration missing".into(),
|
||||
)
|
||||
})?;
|
||||
// No Job grant implicitly expands to all of a Subject's saved Journal.
|
||||
if projection.job.is_some() {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"Memory Journal is not authorized for Job Workers".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let lease = api
|
||||
.store
|
||||
.require_current_worker_singleton_owner(&path.workspace_id, &worker)?
|
||||
.ok_or_else(|| {
|
||||
Error::WorkspacePermissionDenied(
|
||||
"Memory Journal requires the current Subject Worker".into(),
|
||||
)
|
||||
})?;
|
||||
let (subject_id, authority) = subjektiv_scope_from_singleton_key(&lease.key)?;
|
||||
require_subjektiv_worker_authority(
|
||||
authority,
|
||||
SubjektivWorkerAuthority::Subject,
|
||||
"Memory Journal access",
|
||||
)?;
|
||||
let store = open_subjektiv_store(&api)?;
|
||||
let host = JournalHostContext::validated_reader(&store, subject_id).map_err(domain)?;
|
||||
store
|
||||
.execute_memory_journal_with_source_presence(&host, request, |session| {
|
||||
// Absence is known only when neither retained Worker registration nor an
|
||||
// archive location remains. Runtime disconnection is not material deletion.
|
||||
// These SQL lookups never fetch Session bytes or restore the old Worker.
|
||||
let attribution = store.session_attribution(session).ok()??;
|
||||
if attribution.subject_id != subject_id {
|
||||
return None;
|
||||
}
|
||||
let target =
|
||||
RuntimeWorkerRef::new(&attribution.runtime_id, &attribution.worker_id);
|
||||
let archive = api
|
||||
.config_store
|
||||
.worker_session_archive(&path.workspace_id, &target, session)
|
||||
.ok()?;
|
||||
let registered = api
|
||||
.store
|
||||
.get_worker_registry(&path.workspace_id, &target)
|
||||
.ok()?;
|
||||
if archive.is_none() && registered.is_none() {
|
||||
Some(false)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.map(Json)
|
||||
.map_err(domain)
|
||||
.map_err(ApiError::from)
|
||||
})
|
||||
.await
|
||||
}
|
||||
@@ -110,7 +110,7 @@ async fn subject_callbacks_finish_before_waiting_host_parents_release_capacity()
|
||||
));
|
||||
}
|
||||
// Resident memory is not an allowed deletion-drain callback. Preserve that
|
||||
// admission fence; only the exact Session attribution route reenters below.
|
||||
// admission fence; only exact Session attribution and startup read callbacks reenter below.
|
||||
let mut deletion = Box::pin(gate.lock_deletion());
|
||||
assert!(futures::poll!(&mut deletion).is_pending());
|
||||
for (subject_id, worker_id, session_id) in &callbacks {
|
||||
@@ -139,6 +139,27 @@ async fn subject_callbacks_finish_before_waiting_host_parents_release_capacity()
|
||||
let stored = subjektiv.session_attribution(session_id).unwrap().unwrap();
|
||||
assert_eq!(stored.worker_id, *worker_id);
|
||||
assert_eq!(stored.subject_id, *subject_id);
|
||||
let startup_path = format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/journal/startup");
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
app.clone().oneshot(runtime_source_request(
|
||||
&identity,
|
||||
Some(worker_id),
|
||||
"GET",
|
||||
&startup_path,
|
||||
Vec::new(),
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("startup snapshot must not wait for Runtime's occupied Host parents")
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let snapshot: subjektiv::journal::startup::JournalStartupSnapshot =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(snapshot.subject_id, *subject_id);
|
||||
assert!(snapshot.journals.is_empty());
|
||||
assert!(!snapshot.omitted);
|
||||
}
|
||||
// Neither actual parent nor its admission lease was released to make these
|
||||
// callbacks succeed. Only durable attribution completion releases them.
|
||||
@@ -297,7 +318,7 @@ async fn only_exact_authenticated_callbacks_can_reenter_deletion_drain() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_create_releases_reservation_and_key_without_cleanup_errors() {
|
||||
async fn rejected_create_releases_key_but_retains_subject_occupancy_without_release_proof() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let api = test_api(workspace.path()).await;
|
||||
let memory = api
|
||||
@@ -349,11 +370,13 @@ async fn rejected_create_releases_reservation_and_key_without_cleanup_errors() {
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
assert_eq!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, "subjektiv:rejected-create")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
.expect("failed creation is not explicit Subject release authority")
|
||||
.worker,
|
||||
worker,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,623 @@
|
||||
//! Connected Backend → production Runtime factory → real Controller → Journal → signed
|
||||
//! Host startup reader → persisted first input. Only the external model stream is scripted.
|
||||
//! Browser authentication is outside this regression: creation enters the real scoped handler.
|
||||
use super::*;
|
||||
use agen::llm_client::{
|
||||
LlmClient, Request,
|
||||
error::ClientError,
|
||||
event::{Event, StopReason},
|
||||
types::Item,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ConnectedJournalModel {
|
||||
dialogs: Arc<Mutex<Vec<Request>>>,
|
||||
}
|
||||
|
||||
impl ConnectedJournalModel {
|
||||
fn journal_response(request: &Request) -> Vec<Event> {
|
||||
// Runtime Controllers deliberately retain alias-routed core schemas for their
|
||||
// lifetime even with zero attachments. Verify invocation authority, not schema
|
||||
// absence (standalone Internal Jobs use a different composition).
|
||||
for (id, tool, input) in [
|
||||
(
|
||||
"denied-bash",
|
||||
"Bash",
|
||||
json!({"command":"printf JOURNAL_MUST_NOT_EXECUTE","cwd":"."}),
|
||||
),
|
||||
(
|
||||
"denied-read",
|
||||
"Read",
|
||||
json!({"file_path":"journal-must-not-be-readable"}),
|
||||
),
|
||||
] {
|
||||
if let Some(error) = request.items.iter().find_map(|item| match item {
|
||||
Item::ToolResult {
|
||||
call_id, is_error, ..
|
||||
} if call_id == id => Some(*is_error),
|
||||
_ => None,
|
||||
}) {
|
||||
assert!(
|
||||
error,
|
||||
"Journal Job unexpectedly gained {tool} Workdir authority"
|
||||
);
|
||||
} else {
|
||||
return vec![
|
||||
Event::tool_use_start(0, id, tool),
|
||||
Event::tool_input_delta(0, input.to_string()),
|
||||
Event::tool_use_stop(0),
|
||||
];
|
||||
}
|
||||
}
|
||||
let result = |id: &str| -> Option<Value> {
|
||||
request.items.iter().find_map(|item| match item {
|
||||
Item::ToolResult {
|
||||
call_id,
|
||||
content,
|
||||
is_error,
|
||||
summary,
|
||||
..
|
||||
} if call_id == id => {
|
||||
assert!(!is_error, "{id}: {summary}; {content:?}");
|
||||
Some(
|
||||
content
|
||||
.as_deref()
|
||||
.map(|text| serde_json::from_str(text).unwrap())
|
||||
.unwrap_or(Value::Null),
|
||||
)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
};
|
||||
if result("submit").is_some() {
|
||||
return vec![
|
||||
Event::text_block_start(0),
|
||||
Event::text_delta(0, "saved"),
|
||||
Event::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||
];
|
||||
}
|
||||
let (tool, id, input) = if result("checkpoint").is_some() {
|
||||
("SubmitBackendJobResult", "submit", json!({"result":{}}))
|
||||
} else if let Some(position) = result("position") {
|
||||
let state = result("state").unwrap();
|
||||
(
|
||||
"CheckpointJournal",
|
||||
"checkpoint",
|
||||
json!({
|
||||
"request_id":format!("checkpoint:{}",state["value"]["generation_id"].as_str().unwrap()),
|
||||
"expected_state":position["value"]["state_token"],
|
||||
"action":{"action":"checkpoint","record_operation_ids":[format!("record:{}",state["value"]["generation_id"].as_str().unwrap())]},
|
||||
"covered_ranges":state["value"]["selected_ranges"],
|
||||
}),
|
||||
)
|
||||
} else if let Some(record) = result("record") {
|
||||
let state = result("state").unwrap();
|
||||
(
|
||||
"UpdateJournalPosition",
|
||||
"position",
|
||||
json!({
|
||||
"request_id":format!("position:{}",state["value"]["generation_id"].as_str().unwrap()),
|
||||
"expected_state":record["value"]["state_token"],
|
||||
"action":{"action":"update_position","current_position":"CONNECTED_HANDOFF: partially implemented; outcome unconfirmed"},
|
||||
"covered_ranges":[],
|
||||
}),
|
||||
)
|
||||
} else if let Some(state) = result("state") {
|
||||
let state = &state["value"];
|
||||
// Deliver the actual frozen Runtime source through authenticated Job tools, rather
|
||||
// than injecting a fixture Journal or claiming unseen source was processed.
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("source-{index}-{offset}");
|
||||
if let Some(page) = result(&id) {
|
||||
let page = &page["value"];
|
||||
if let Some(next) = page["next_byte"].as_u64() {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["content_token"].clone();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
assert!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalText")
|
||||
);
|
||||
return vec![Event::tool_use_start(0, &id, "ReadJournalText"), Event::tool_input_delta(0, json!({
|
||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
||||
"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token,
|
||||
}).to_string()), Event::tool_use_stop(0)];
|
||||
}
|
||||
}
|
||||
}
|
||||
let (tool, action) = if state["current_open"].is_null() {
|
||||
(
|
||||
"CreateJournalEntry",
|
||||
json!({"action":"create","title":"Connected handoff","body":"CONNECTED_HANDOFF: proposed A, user chose B; only partial work performed.","ongoing":"Remaining work; do not claim success","state":"open","source_end":null,"correction_refs":[]}),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"RewriteJournalEntry",
|
||||
json!({"action":"rewrite","entry_id":state["current_open"]["id"],"title":"Connected handoff","body":"CONNECTED_HANDOFF: B selected; partial work remains.","ongoing":"Remaining work; result unknown"}),
|
||||
)
|
||||
};
|
||||
(
|
||||
tool,
|
||||
"record",
|
||||
json!({
|
||||
"request_id":format!("record:{}",state["generation_id"].as_str().unwrap()),
|
||||
"expected_state":state["current_journal"]["state_token"],
|
||||
"action":action,"covered_ranges":state["selected_ranges"],
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
("ReadJournalState", "state", json!({}))
|
||||
};
|
||||
assert!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|definition| definition.name == tool),
|
||||
"missing {tool}"
|
||||
);
|
||||
for forbidden in ["MemoryApplyCandidate", "SubjektivSessionRead"] {
|
||||
assert!(
|
||||
!request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|definition| definition.name == forbidden),
|
||||
"Journal exposed forbidden {forbidden}: {:?}",
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| &tool.name)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
vec![
|
||||
Event::tool_use_start(0, id, tool),
|
||||
Event::tool_input_delta(0, input.to_string()),
|
||||
Event::tool_use_stop(0),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LlmClient for ConnectedJournalModel {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> std::result::Result<
|
||||
std::pin::Pin<
|
||||
Box<dyn futures::Stream<Item = std::result::Result<Event, ClientError>> + Send>,
|
||||
>,
|
||||
ClientError,
|
||||
> {
|
||||
let events = if request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalState")
|
||||
{
|
||||
Self::journal_response(&request)
|
||||
} else if request.tools.iter().any(|tool| {
|
||||
matches!(
|
||||
tool.name.as_str(),
|
||||
"SubmitBackendJobResult" | "SubmitMemorySurface"
|
||||
)
|
||||
}) {
|
||||
// The real companion also requests an empty Memory consolidation Job. Keep that
|
||||
// lifecycle real without counting its model requests as main-body dialog requests.
|
||||
if request.items.iter().any(|item| matches!(item, Item::ToolResult { call_id, is_error, .. } if call_id == "empty-memory" && !is_error)) {
|
||||
vec![Event::text_block_start(0), Event::text_delta(0, "empty Memory acknowledged"), Event::text_block_stop(0, Some(StopReason::EndTurn))]
|
||||
} else {
|
||||
let (tool, input) = if request.tools.iter().any(|tool| tool.name == "SubmitMemorySurface") {
|
||||
("SubmitMemorySurface", json!({"points":[]}))
|
||||
} else {
|
||||
let input: Value = request.items.iter().filter_map(Item::as_text).find_map(|text| text.lines().find_map(|line| line.strip_prefix("input_json: "))).map(|input| serde_json::from_str(input).unwrap()).expect("Backend Memory Job envelope");
|
||||
assert_eq!(input["candidate_ids"], json!([]));
|
||||
("SubmitBackendJobResult", json!({"result":{"subject_id":input["subject_id"],"candidate_ids":[]}}))
|
||||
};
|
||||
vec![Event::tool_use_start(0, "empty-memory", tool), Event::tool_input_delta(0, input.to_string()), Event::tool_use_stop(0)]
|
||||
}
|
||||
} else {
|
||||
self.dialogs.lock().unwrap().push(request);
|
||||
vec![
|
||||
Event::text_block_start(0),
|
||||
Event::text_delta(
|
||||
0,
|
||||
"Partial B implementation; remaining work is not confirmed.",
|
||||
),
|
||||
Event::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||
]
|
||||
};
|
||||
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
async fn connected_create(api: &WorkspaceApi, subject: &str, name: &str) -> RuntimeWorkerRef {
|
||||
let Json(created) = scoped_create_workspace_worker(
|
||||
State(api.clone()),
|
||||
AxumPath(ScopedWorkspacePath { workspace_id: TEST_WORKSPACE_ID.into() }),
|
||||
HeaderMap::new(),
|
||||
Json(CreateWorkspaceWorkerRequest {
|
||||
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), display_name: name.into(),
|
||||
singleton_key: None, profile: Some("builtin:companion".into()), ticket_assignment: None,
|
||||
initial_submit: vec![protocol::Segment::Text { content: "User chose B rather than A. Continue the partial work without assuming success.".into() }],
|
||||
workdir_attachments: vec![],
|
||||
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
|
||||
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest { subject_id: subject.into() }),
|
||||
}, control_operation_id: None,
|
||||
}),
|
||||
).await.unwrap();
|
||||
RuntimeWorkerRef::new(created.runtime_id, created.worker_id)
|
||||
}
|
||||
|
||||
async fn connected_wait_for_recording(
|
||||
api: &WorkspaceApi,
|
||||
store: &crate::subjektiv::SubjektivStore,
|
||||
attr: &crate::subjektiv::SubjectSessionAttribution,
|
||||
) {
|
||||
let host = JournalHostContext::validated_session(store, attr.clone()).unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
let mut tick = tokio::time::interval(std::time::Duration::from_millis(25));
|
||||
loop {
|
||||
tick.tick().await;
|
||||
let jobs = api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &attr.session_id, None, 100)
|
||||
.unwrap();
|
||||
let generation_closed =
|
||||
store
|
||||
.journal_for_session(&host)
|
||||
.unwrap()
|
||||
.is_some_and(|journal| {
|
||||
store
|
||||
.active_journal_generation(&host, &journal)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
});
|
||||
if !jobs.is_empty()
|
||||
&& generation_closed
|
||||
&& jobs
|
||||
.iter()
|
||||
.all(|job| job.state == BackendJobState::Completed)
|
||||
&& !store.journal_retention_required(&host).unwrap()
|
||||
&& api
|
||||
.runtime
|
||||
.worker(&RuntimeWorkerRef::new(&attr.runtime_id, &attr.worker_id))
|
||||
.unwrap()
|
||||
.worker_state
|
||||
.is_some_and(|state| state.catalog_status() == protocol::WorkerStatus::Idle)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!(
|
||||
"real Controller recording did not complete: {:?}",
|
||||
api.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &attr.session_id, None, 100)
|
||||
.unwrap()
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
|
||||
async fn backend_real_runtime_journal_release_and_next_body_persisted_startup() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(dir.path());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let mut config = test_server_config(dir.path());
|
||||
config.backend_base_url = Some(format!("http://{}", listener.local_addr().unwrap()));
|
||||
let control = Arc::new(test_control_store(&config));
|
||||
seed_test_registered_workspace(control.as_ref(), &config)
|
||||
.await
|
||||
.unwrap();
|
||||
let broker = BackendResourceBroker::default();
|
||||
let dispatcher = Arc::new(
|
||||
crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
|
||||
config.clone(),
|
||||
control.clone(),
|
||||
),
|
||||
);
|
||||
let model = ConnectedJournalModel::default();
|
||||
let model_factory = model.clone();
|
||||
let audience = embedded_runtime_request_audience(&config).unwrap();
|
||||
let execution = worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new(
|
||||
worker_runtime::worker_backend::ProfileRuntimeWorkerFactory::new(
|
||||
config.workspace_execution_root.clone(),
|
||||
)
|
||||
.with_embedded_worker_mutation_dispatcher(EMBEDDED_WORKER_RUNTIME_ID, dispatcher.clone())
|
||||
.with_workspace_request_client(
|
||||
worker_runtime::workspace_request::RuntimeWorkspaceRequestClient::new(
|
||||
TEST_WORKSPACE_ID,
|
||||
audience.clone(),
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
)
|
||||
.with_runtime_request_source(&EMBEDDED_RUNTIME_REQUEST_IDENTITY, audience),
|
||||
)
|
||||
.with_runtime_store_dir(config.embedded_runtime_store_root.clone())
|
||||
.with_resource_client(Arc::new(broker.clone()))
|
||||
.with_spawn_model_client_factory(move || Box::new(model_factory.clone())),
|
||||
)
|
||||
.unwrap();
|
||||
let api = WorkspaceApi::new_with_execution_backend_and_broker(
|
||||
config,
|
||||
control,
|
||||
Arc::new(execution),
|
||||
broker,
|
||||
HostExecutors::default(),
|
||||
Some(dispatcher),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
seed_test_delivery_binding(&api);
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
let observed = requests.clone();
|
||||
let app = build_router(api.clone()).layer(axum::middleware::from_fn(
|
||||
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
||||
let observed = observed.clone();
|
||||
async move {
|
||||
let path = request.uri().path().to_string();
|
||||
let response = next.run(request).await;
|
||||
observed.lock().unwrap().push((path, response.status()));
|
||||
response
|
||||
}
|
||||
},
|
||||
));
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("connected-handoff").unwrap())
|
||||
.unwrap();
|
||||
let old = connected_create(&api, &subject.id, "connected-old-body").await;
|
||||
let old_attr = store
|
||||
.subject_session_attributions(&subject.id)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|attr| attr.worker_id == old.worker_id)
|
||||
.unwrap();
|
||||
connected_wait_for_recording(&api, &store, &old_attr).await;
|
||||
let old_host = JournalHostContext::validated_session(&store, old_attr.clone()).unwrap();
|
||||
let old_binding = JournalWriterBinding {
|
||||
subject_id: subject.id.clone(),
|
||||
runtime_id: old.runtime_id.clone(),
|
||||
worker_id: old.worker_id.clone(),
|
||||
binding_id: old_host.writer_binding_id().unwrap().into(),
|
||||
};
|
||||
// Inject a durable result-acceptance gap after real Journal completion. This must
|
||||
// reject before writer revocation and confirm cancellation of the actual Controller
|
||||
// guard; merely returning Err would strand the body and its execution-admission fence.
|
||||
let recorded = api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &old_attr.session_id, None, 100)
|
||||
.unwrap();
|
||||
let uncertain_job = &recorded[0].request.job_id;
|
||||
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
|
||||
let saved_result: (String, String) = conn.query_row(
|
||||
"SELECT result_json,result_digest FROM backend_jobs WHERE workspace_id=?1 AND job_id=?2",
|
||||
rusqlite::params![TEST_WORKSPACE_ID, uncertain_job], |r| Ok((r.get(0)?, r.get(1)?)),
|
||||
).unwrap();
|
||||
conn.execute(
|
||||
"UPDATE backend_jobs SET state='unknown',result_json=NULL,result_digest=NULL WHERE workspace_id=?1 AND job_id=?2",
|
||||
rusqlite::params![TEST_WORKSPACE_ID, uncertain_job],
|
||||
)
|
||||
.unwrap();
|
||||
let incomplete = json!({
|
||||
"worker":old,"binding_id":old_binding.binding_id,"operation_id":"connected-incomplete-release",
|
||||
});
|
||||
let error = subjektiv_journal::release_subject(
|
||||
&api,
|
||||
serde_json::from_value(incomplete.clone()).unwrap(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("Unknown"), "{error}");
|
||||
assert!(store.journal_writer_active(&old_binding).unwrap());
|
||||
assert!(
|
||||
!api.config_store
|
||||
.subject_release_pending(TEST_WORKSPACE_ID, &old)
|
||||
.unwrap()
|
||||
);
|
||||
let cancelled: bool = conn.query_row("SELECT cancelled FROM subject_worker_releases WHERE workspace_id=?1 AND operation_id='connected-incomplete-release'", [TEST_WORKSPACE_ID], |r| r.get(0)).unwrap();
|
||||
assert!(
|
||||
cancelled,
|
||||
"checked pre-revoke refusal needs a terminal cancellation receipt"
|
||||
);
|
||||
assert!(
|
||||
subjektiv_journal::release_subject(&api, serde_json::from_value(incomplete).unwrap())
|
||||
.is_err(),
|
||||
"cancelled release identity must not rearm"
|
||||
);
|
||||
conn.execute(
|
||||
"UPDATE backend_jobs SET state='completed',result_json=?3,result_digest=?4 WHERE workspace_id=?1 AND job_id=?2",
|
||||
rusqlite::params![TEST_WORKSPACE_ID, uncertain_job, saved_result.0, saved_result.1],
|
||||
)
|
||||
.unwrap();
|
||||
// The new release below must reacquire the genuine Controller guard. It cannot
|
||||
// succeed if the failed pre-revoke operation left the body frozen.
|
||||
let release = subjektiv_journal::release_subject(&api, serde_json::from_value(json!({
|
||||
"worker":old,"binding_id":old_binding.binding_id,"operation_id":"connected-normal-release",
|
||||
})).unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
release.state,
|
||||
runtime_api::SubjectReleaseState::Disconnected
|
||||
);
|
||||
assert!(!store.journal_writer_active(&old_binding).unwrap());
|
||||
let key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap();
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(&old, WorkerLifecycleRequest::default())
|
||||
.unwrap();
|
||||
assert_eq!(stopped.state, InternalWorkerOperationState::Accepted);
|
||||
let removed = WorkerRemovalService::new(&api)
|
||||
.execute_target_removal(
|
||||
api.runtime.as_ref(),
|
||||
&old,
|
||||
"connected regression removes prior body",
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(removed.status, StatusCode::OK.as_u16(), "{}", removed.body);
|
||||
assert!(api.runtime.worker(&old).is_err());
|
||||
let original_root = api
|
||||
.config
|
||||
.embedded_runtime_store_root
|
||||
.join("workers")
|
||||
.join(&old.worker_id)
|
||||
.join("session");
|
||||
assert!(
|
||||
!original_root.exists(),
|
||||
"prior Worker original Session must actually be removed before startup"
|
||||
);
|
||||
// Purge retained original archive through Runtime, not fixture filesystem deletion.
|
||||
// Journal text is independent and remains readable for new-body startup.
|
||||
if let Some(archive) = api
|
||||
.config_store
|
||||
.worker_tombstone(TEST_WORKSPACE_ID, &old)
|
||||
.unwrap()
|
||||
.and_then(|t| t.archive_id)
|
||||
{
|
||||
api.runtime
|
||||
.session_material_erase(
|
||||
&old.runtime_id,
|
||||
runtime_api::SessionMaterialEraseRequest {
|
||||
workspace_id: TEST_WORKSPACE_ID.into(),
|
||||
source: runtime_api::SessionPublicSource::Archived {
|
||||
archive_id: archive.clone(),
|
||||
},
|
||||
expected_session_id: old_attr.session_id.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!api.config
|
||||
.embedded_runtime_store_root
|
||||
.join("archives/workers")
|
||||
.join(archive)
|
||||
.join("session")
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
let next = connected_create(&api, &subject.id, "connected-next-body").await;
|
||||
assert_ne!(next, old);
|
||||
let next_attr = store
|
||||
.subject_session_attributions(&subject.id)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|attr| attr.worker_id == next.worker_id)
|
||||
.unwrap();
|
||||
assert_ne!(next_attr.session_id, old_attr.session_id);
|
||||
connected_wait_for_recording(&api, &store, &next_attr).await;
|
||||
|
||||
// Read actual Session files produced by the real Controller; no fixture Session append,
|
||||
// startup-item injection, fake disconnect or store-only simulation is used in this test.
|
||||
use session_store::Store as _;
|
||||
let session = session_store::WorkerSessionStore::open_read_only(
|
||||
api.config
|
||||
.embedded_runtime_store_root
|
||||
.join("workers")
|
||||
.join(&next.worker_id)
|
||||
.join("session"),
|
||||
)
|
||||
.unwrap();
|
||||
let session_id = next_attr.session_id.parse().unwrap();
|
||||
let segments = session.list_segments(session_id).unwrap();
|
||||
assert_eq!(segments.len(), 1);
|
||||
let entries = session.read_all_read_only(session_id, segments[0]).unwrap();
|
||||
let startup: Vec<_> = entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, log)| match log {
|
||||
session_store::LogEntry::AnnotatedSystemItem { entry, .. } => match &entry.item {
|
||||
session_store::SystemItem::JournalStartup {
|
||||
snapshot,
|
||||
snapshot_sha256,
|
||||
body,
|
||||
prompt_provenance,
|
||||
} => Some((index, snapshot, snapshot_sha256, body, prompt_provenance)),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(startup.len(), 1, "startup must be durably committed once");
|
||||
let (startup_index, snapshot, digest, body, provenance) = startup[0];
|
||||
let first_input = entries
|
||||
.iter()
|
||||
.position(|entry| matches!(entry, session_store::LogEntry::AnnotatedUserInput { .. }))
|
||||
.expect("new body persisted its initial human input");
|
||||
assert!(
|
||||
startup_index < first_input,
|
||||
"startup reference must be persisted before the first human input/model request"
|
||||
);
|
||||
assert_eq!(snapshot["subject_id"], subject.id);
|
||||
assert_eq!(
|
||||
snapshot["journals"][0]["source_session_id"], old_attr.session_id,
|
||||
"Host-selected writer predecessor must lead the snapshot"
|
||||
);
|
||||
assert!(body.contains("CONNECTED_HANDOFF"));
|
||||
assert_eq!(
|
||||
provenance.logical_name,
|
||||
"internal.subjektiv_journal_startup"
|
||||
);
|
||||
assert_eq!(
|
||||
*digest,
|
||||
Sha256::digest(serde_json::to_vec(snapshot).unwrap())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>()
|
||||
);
|
||||
let dialogs = model.dialogs.lock().unwrap();
|
||||
assert!(dialogs.len() >= 2);
|
||||
assert!(
|
||||
dialogs[1]
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(Item::as_text)
|
||||
.any(|text| text.contains(body)),
|
||||
"first new-body model request must contain the same persisted startup body"
|
||||
);
|
||||
drop(dialogs);
|
||||
let requests = requests.lock().unwrap();
|
||||
assert_eq!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(
|
||||
|(path, status)| path.ends_with("/subjektiv/journal/startup")
|
||||
&& *status == StatusCode::OK
|
||||
)
|
||||
.count(),
|
||||
2,
|
||||
"both bodies must use the authenticated HTTP Host startup reader"
|
||||
);
|
||||
assert!(
|
||||
requests.iter().any(
|
||||
|(path, status)| path.ends_with("/subjektiv/sessions") && *status == StatusCode::OK
|
||||
)
|
||||
);
|
||||
drop(requests);
|
||||
server.abort();
|
||||
}
|
||||
@@ -8,6 +8,19 @@ use sha2::{Digest, Sha256};
|
||||
use subjektiv::journal::transport::*;
|
||||
use subjektiv::journal::*;
|
||||
|
||||
#[path = "subjektiv_journal_connected_tests.rs"]
|
||||
mod connected;
|
||||
|
||||
fn bind_fixture_session(
|
||||
store: &crate::subjektiv::SubjektivStore,
|
||||
attr: crate::subjektiv::SubjectSessionAttribution,
|
||||
) -> JournalHostContext {
|
||||
let binding = store
|
||||
.claim_journal_writer(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||
.unwrap();
|
||||
store.bind_journal_session(&binding, attr).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_job_reserves_before_binding_and_retains_partial_receipts_on_cancel() {
|
||||
let Fixture {
|
||||
@@ -195,7 +208,7 @@ impl Fixture {
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap();
|
||||
let host = bind_fixture_session(&store, attr.clone());
|
||||
let text = "original public input";
|
||||
let range = JournalSourceRange {
|
||||
source: JournalSource {
|
||||
@@ -228,6 +241,7 @@ impl Fixture {
|
||||
profile: "builtin:subjektiv-journal-recording".into(),
|
||||
grants: BackendJobGrants {
|
||||
subjektiv_journal: Some(SubjektivJournalGrant {
|
||||
binding_id: host.writer_binding_id().unwrap().into(),
|
||||
subject_id: subject.id,
|
||||
session_id: attr.session_id,
|
||||
journal_id: id.clone(),
|
||||
@@ -288,13 +302,14 @@ async fn journal_completion_replays_only_saved_checkpoint_and_fences_late_operat
|
||||
panic!("receipt required")
|
||||
};
|
||||
assert!(subjektiv_journal::finish(&f.api, worker).is_err());
|
||||
let position = save_fixture_position(&f.api, worker, receipt.state_token);
|
||||
subjektiv_journal::execute(
|
||||
&f.api,
|
||||
worker,
|
||||
JournalJobRequest::Execute {
|
||||
operation: JournalOperation {
|
||||
request_id: "checkpoint".into(),
|
||||
expected_state: receipt.state_token,
|
||||
expected_state: position,
|
||||
action: JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["no-change".into()],
|
||||
},
|
||||
@@ -332,6 +347,69 @@ async fn journal_completion_replays_only_saved_checkpoint_and_fences_late_operat
|
||||
);
|
||||
assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err());
|
||||
assert!(!f.store.journal_retention_required(&f.host).unwrap());
|
||||
// Store authority boundary: Host Job-state gating is separate from revocation itself.
|
||||
let grant = f.request.grants.subjektiv_journal.as_ref().unwrap();
|
||||
let attr = f
|
||||
.store
|
||||
.session_attribution(&grant.session_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let binding = JournalWriterBinding {
|
||||
subject_id: attr.subject_id.clone(),
|
||||
runtime_id: attr.runtime_id.clone(),
|
||||
worker_id: attr.worker_id.clone(),
|
||||
binding_id: grant.binding_id.clone(),
|
||||
};
|
||||
f.store.release_journal_writer(&binding).unwrap();
|
||||
let next = f
|
||||
.store
|
||||
.claim_journal_writer(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||
.unwrap();
|
||||
assert_ne!(next.binding_id, binding.binding_id);
|
||||
assert!(f.store.bind_journal_session(&next, attr).is_err());
|
||||
let mut stale = dispatched.clone();
|
||||
stale
|
||||
.job
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.binding_id = next.binding_id;
|
||||
assert!(subjektiv_journal::prepare(&f.api, &stale).is_err());
|
||||
assert!(
|
||||
f.store
|
||||
.query_journal_operation(&f.host, &f.id, "no-change")
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
// Exact saved completion is still recoverable; it is not a new write grant.
|
||||
subjektiv_journal::validate_result(&f.api, &f.request, &dispatched.attempt, &value).unwrap();
|
||||
}
|
||||
|
||||
fn save_fixture_position(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
expected_state: String,
|
||||
) -> String {
|
||||
let JournalJobResponse::Saved(receipt) = subjektiv_journal::execute(
|
||||
api,
|
||||
worker,
|
||||
JournalJobRequest::Execute {
|
||||
operation: JournalOperation {
|
||||
request_id: "position".into(),
|
||||
expected_state,
|
||||
action: JournalAction::UpdatePosition {
|
||||
current_position: "No activity change; work remains pending.".into(),
|
||||
},
|
||||
covered_ranges: vec![],
|
||||
},
|
||||
},
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("saved position receipt required")
|
||||
};
|
||||
receipt.state_token
|
||||
}
|
||||
|
||||
fn deliver_fixture_source(
|
||||
@@ -483,7 +561,7 @@ impl RuntimeFixture {
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap();
|
||||
let host = bind_fixture_session(&store, attr.clone());
|
||||
let src = runtime_api::SessionPublicSource::Retained {
|
||||
worker_id: worker_runtime::identity::WorkerId::parse(&source.worker_id).unwrap(),
|
||||
};
|
||||
@@ -543,13 +621,15 @@ impl RuntimeFixture {
|
||||
.accept_journal_sources(&host, "source", &[range.clone()])
|
||||
.unwrap();
|
||||
let request = subjektiv_journal::recording_request(
|
||||
&host,
|
||||
attr,
|
||||
&id,
|
||||
vec![subjektiv::journal::recording::JournalMaterial {
|
||||
range: range.clone(),
|
||||
text: SOURCE_SENTINEL.into(),
|
||||
}],
|
||||
);
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!serde_json::to_string(&request)
|
||||
.unwrap()
|
||||
@@ -650,13 +730,14 @@ impl RuntimeFixture {
|
||||
.unwrap() else {
|
||||
panic!("receipt")
|
||||
};
|
||||
let position = save_fixture_position(&self.api, worker, receipt.state_token);
|
||||
subjektiv_journal::execute(
|
||||
&self.api,
|
||||
worker,
|
||||
JournalJobRequest::Execute {
|
||||
operation: JournalOperation {
|
||||
request_id: "checkpoint".into(),
|
||||
expected_state: receipt.state_token,
|
||||
expected_state: position,
|
||||
action: JournalAction::Checkpoint {
|
||||
record_operation_ids: vec!["record".into()],
|
||||
},
|
||||
@@ -1661,7 +1742,7 @@ impl SubjectRemovalFixture {
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap();
|
||||
let host = bind_fixture_session(&store, attr.clone());
|
||||
seed_worker_control_grant(&api, &controller, &target, "subject-removal-fixture");
|
||||
api.runtime
|
||||
.stop_worker(
|
||||
@@ -1696,6 +1777,43 @@ impl SubjectRemovalFixture {
|
||||
registry.profile = Some("builtin:default".into());
|
||||
self.api.store.upsert_worker_registry(®istry).unwrap();
|
||||
}
|
||||
/// Seed the historical release boundary for retention-recovery component tests.
|
||||
/// This deterministic stopped-Worker fixture has no Controller and does NOT
|
||||
/// prove disconnect. The connected test below exercises actual release proof.
|
||||
fn seed_completed_release_for_archive_recovery(&self) {
|
||||
let lease = self
|
||||
.api
|
||||
.store
|
||||
.require_current_worker_singleton_owner(TEST_WORKSPACE_ID, &self.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let binding = JournalWriterBinding {
|
||||
subject_id: self.attr.subject_id.clone(),
|
||||
runtime_id: self.target.runtime_id.clone(),
|
||||
worker_id: self.target.worker_id.clone(),
|
||||
binding_id: self.host.writer_binding_id().unwrap().into(),
|
||||
};
|
||||
let operation = "historical-release-retention-fixture";
|
||||
self.api
|
||||
.config_store
|
||||
.reserve_subject_release(TEST_WORKSPACE_ID, &lease, &binding.binding_id, operation)
|
||||
.unwrap();
|
||||
self.store.release_journal_writer(&binding).unwrap();
|
||||
self.api
|
||||
.config_store
|
||||
.confirm_subject_release_proof(
|
||||
TEST_WORKSPACE_ID,
|
||||
&lease,
|
||||
&binding.binding_id,
|
||||
operation,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
self.api
|
||||
.config_store
|
||||
.release_subject_singleton(TEST_WORKSPACE_ID, &lease, &binding.binding_id, operation)
|
||||
.unwrap();
|
||||
}
|
||||
fn remove(&self) -> worker::WorkspaceResponse {
|
||||
let key = self
|
||||
.api
|
||||
@@ -1714,6 +1832,37 @@ impl SubjectRemovalFixture {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subject_release_never_substitutes_catalog_idle_or_stopped_for_controller_reservation() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
let input = serde_json::from_value(serde_json::json!({
|
||||
"worker":f.target,"binding_id":f.host.writer_binding_id().unwrap(),"operation_id":"actual-controller-required",
|
||||
})).unwrap();
|
||||
let error = subjektiv_journal::release_subject(&f.api, input).unwrap_err();
|
||||
assert!(
|
||||
error.to_string().contains("actual") || error.to_string().contains("Controller"),
|
||||
"{error}"
|
||||
);
|
||||
let key = crate::subjektiv::subject_worker_singleton_key(&f.attr.subject_id).unwrap();
|
||||
assert_eq!(
|
||||
f.api
|
||||
.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.worker,
|
||||
f.target
|
||||
);
|
||||
let binding = JournalWriterBinding {
|
||||
subject_id: f.attr.subject_id.clone(),
|
||||
runtime_id: f.target.runtime_id.clone(),
|
||||
worker_id: f.target.worker_id.clone(),
|
||||
binding_id: f.host.writer_binding_id().unwrap().into(),
|
||||
};
|
||||
assert!(f.store.journal_writer_active(&binding).unwrap());
|
||||
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_subject_removal_does_not_admit_new_public_history_or_reserve_a_job() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
@@ -1721,7 +1870,7 @@ async fn disabled_subject_removal_does_not_admit_new_public_history_or_reserve_a
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
StatusCode::CONFLICT.as_u16(),
|
||||
"{}",
|
||||
response.body
|
||||
);
|
||||
@@ -1730,7 +1879,7 @@ async fn disabled_subject_removal_does_not_admit_new_public_history_or_reserve_a
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
.is_some()
|
||||
);
|
||||
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
|
||||
assert!(
|
||||
@@ -1795,7 +1944,7 @@ async fn disabled_subject_removal_retains_only_the_already_admitted_prefix() {
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
StatusCode::CONFLICT.as_u16(),
|
||||
"{}",
|
||||
response.body
|
||||
);
|
||||
@@ -1804,62 +1953,25 @@ async fn disabled_subject_removal_retains_only_the_already_admitted_prefix() {
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
.is_some()
|
||||
);
|
||||
let jobs = f
|
||||
.api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.len(), 1);
|
||||
assert_eq!(
|
||||
jobs[0]
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
|
||||
.unwrap()
|
||||
.selected_ranges,
|
||||
ranges[..1]
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
f.store.journal_for_session(&f.host).unwrap().as_deref(),
|
||||
Some(id.as_str())
|
||||
);
|
||||
assert!(f.store.journal_retention_required(&f.host).unwrap());
|
||||
let archive = f
|
||||
.api
|
||||
.config_store
|
||||
.worker_session_archive(TEST_WORKSPACE_ID, &f.target, &f.attr.session_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(!session_root(f.dir.path(), &f.target).exists());
|
||||
let source = runtime_api::SessionPublicSource::Archived {
|
||||
archive_id: archive.archive_id,
|
||||
};
|
||||
let response = f
|
||||
.api
|
||||
.runtime
|
||||
.session_material_catalog(
|
||||
&f.target.runtime_id,
|
||||
runtime_api::SessionMaterialCatalogRequest {
|
||||
workspace_id: TEST_WORKSPACE_ID.into(),
|
||||
source,
|
||||
expected_session_id: f.attr.session_id.clone(),
|
||||
expected_catalog_sha256: None,
|
||||
offset: 0,
|
||||
limit: 100,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
response,
|
||||
runtime_api::SessionMaterialCatalogAvailability::Page { .. }
|
||||
));
|
||||
assert!(session_root(f.dir.path(), &f.target).exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attributed_unmaterialized_subject_removal_needs_no_journal_job() {
|
||||
async fn attributed_unmaterialized_subject_still_requires_actual_idle_release() {
|
||||
let f = SubjectRemovalFixture::with_materialized_source(false).await;
|
||||
let retention = source_retention(f.dir.path());
|
||||
let session = f.attr.session_id.parse().unwrap();
|
||||
@@ -1882,28 +1994,21 @@ async fn attributed_unmaterialized_subject_removal_needs_no_journal_job() {
|
||||
.unwrap();
|
||||
assert_eq!(retention.pins(session).unwrap().len(), 1);
|
||||
let response = f.remove();
|
||||
assert!(retention.pins(session).unwrap().is_empty());
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
StatusCode::CONFLICT.as_u16(),
|
||||
"{}",
|
||||
response.body
|
||||
);
|
||||
assert_eq!(retention.pins(session).unwrap().len(), 1);
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
.is_some()
|
||||
);
|
||||
assert!(f.store.journal_for_session(&f.host).unwrap().is_none());
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2237,6 +2342,7 @@ async fn assert_erased_archive_plan_replans_without_republishing_sources(
|
||||
) {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
f.seed_completed_release_for_archive_recovery();
|
||||
let policy = f
|
||||
.api
|
||||
.config_store
|
||||
@@ -2331,6 +2437,7 @@ journal_recovery_cases! {
|
||||
async fn assert_erased_completed_archive_receipt_is_audited_without_republishing(restart: bool) {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
f.seed_completed_release_for_archive_recovery();
|
||||
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
// Runtime has committed source_removed, but Backend commit/response is lost.
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected metadata commit interruption'); END;").unwrap();
|
||||
@@ -2406,6 +2513,7 @@ journal_recovery_cases! {
|
||||
async fn erased_archive_without_receipt_or_inventory_remains_unknown_without_replanning() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
f.seed_completed_release_for_archive_recovery();
|
||||
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE UPDATE OF state ON worker_removal_operations WHEN NEW.state='executing' BEGIN SELECT RAISE(ABORT,'injected interruption'); END;").unwrap();
|
||||
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||
@@ -2465,6 +2573,7 @@ async fn assert_erased_archive_retained_diagnostics_receipt_finalizes_only_after
|
||||
) {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
f.seed_completed_release_for_archive_recovery();
|
||||
let policy = f
|
||||
.api
|
||||
.config_store
|
||||
@@ -2662,3 +2771,36 @@ journal_recovery_cases! {
|
||||
erased_archive_retained_diagnostics_reopen_finalizes_only_after_physical_ack => (true, false),
|
||||
erased_archive_retained_diagnostics_interrupted_reopen_finalizes_only_after_physical_ack => (true, true),
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_journal_grant_preserves_old_json_and_only_recovers_original_current_body() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
let old = json!({"subject_id":f.attr.subject_id,"session_id":f.attr.session_id,"journal_id":"legacy-journal","selected_ranges":[]});
|
||||
let wire = serde_json::to_string(&old).unwrap();
|
||||
let grant: SubjektivJournalGrant = serde_json::from_str(&wire).unwrap();
|
||||
assert!(grant.binding_id.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::to_value(&grant).unwrap(),
|
||||
old,
|
||||
"old canonical request/digest must not acquire a new empty binding_id field"
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&serde_json::to_value(&grant).unwrap()).unwrap(),
|
||||
wire
|
||||
);
|
||||
assert!(subjektiv_journal::host(&f.api, &f.store, &grant).is_ok());
|
||||
let writer = JournalWriterBinding {
|
||||
subject_id: f.attr.subject_id.clone(),
|
||||
runtime_id: f.attr.runtime_id.clone(),
|
||||
worker_id: f.attr.worker_id.clone(),
|
||||
binding_id: f.host.writer_binding_id().unwrap().into(),
|
||||
};
|
||||
f.store.release_journal_writer(&writer).unwrap();
|
||||
f.store
|
||||
.claim_journal_writer(&f.attr.subject_id, &f.attr.runtime_id, "different-body")
|
||||
.unwrap();
|
||||
assert!(
|
||||
subjektiv_journal::host(&f.api, &f.store, &grant).is_err(),
|
||||
"legacy grant must never map to a later writer"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,731 @@
|
||||
//! Connected ordinary library tests: production router, signed Runtime authentication,
|
||||
//! singleton/Job authority and Subjektiv FeatureStorage/SQLite. Extraction output and
|
||||
//! Host admission/generation are fixture inputs; staging/adoption and canonical Session
|
||||
//! serialization are real. No Runtime execution, model, external config or network is used.
|
||||
use super::*;
|
||||
use subjektiv::journal::memory_links::*;
|
||||
use subjektiv::journal::transport::{JournalJobRequest, JournalTextRead, JournalTextTarget};
|
||||
use subjektiv::journal::*;
|
||||
|
||||
const RUNTIME: &str = "runtime-memory-journal";
|
||||
const JOURNAL_BODY: &str =
|
||||
"A was proposed; B was chosen after checking the constraint. Work remains.";
|
||||
|
||||
fn route_path() -> String {
|
||||
format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/journal/memory")
|
||||
}
|
||||
|
||||
async fn response(app: Router, request: Request<Body>) -> (StatusCode, Vec<u8>) {
|
||||
let response = app.oneshot(request).await.unwrap();
|
||||
let status = response.status();
|
||||
let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap();
|
||||
(status, bytes.to_vec())
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
_dir: tempfile::TempDir,
|
||||
api: WorkspaceApi,
|
||||
identity: worker_runtime::auth::RuntimeIdentityMaterial,
|
||||
store: crate::subjektiv::SubjektivStore,
|
||||
host: JournalHostContext,
|
||||
job: JournalJobContext,
|
||||
worker: RuntimeWorkerRef,
|
||||
source_worker: RuntimeWorkerRef,
|
||||
source_root: PathBuf,
|
||||
selector: MemoryJournalSelector,
|
||||
journal_id: String,
|
||||
entry_id: String,
|
||||
ranges: Vec<JournalSourceRange>,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
async fn new() -> Self {
|
||||
use session_store::{LogEntry, Store as _, WorkerSessionStore};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut api = test_api(dir.path()).await;
|
||||
let identity = worker_runtime::auth::RuntimeIdentityMaterial::generate(RUNTIME).unwrap();
|
||||
configure_runtime_request_auth(&mut api, &identity, RUNTIME);
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("memory-journal").unwrap())
|
||||
.unwrap();
|
||||
let worker = Self::body_worker(&api, &subject.id);
|
||||
let source_worker = RuntimeWorkerRef::new(RUNTIME, Uuid::new_v4().to_string());
|
||||
seed_worker_source_member(&api, RUNTIME, &source_worker.worker_id);
|
||||
let session = session_store::new_session_id();
|
||||
let segment = Uuid::new_v4();
|
||||
let source_root = dir.path().join("old-worker/session");
|
||||
let history = [
|
||||
"Choose B after checking the constraint",
|
||||
"Correction: B still needs validation",
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, text)| session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::Message {
|
||||
role: session_store::LoggedRole::User,
|
||||
content: vec![session_store::LoggedContentPart::Text { text: text.into() }],
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(format!("original-{index}")),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: None,
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
WorkerSessionStore::new(&source_root)
|
||||
.unwrap()
|
||||
.create_segment(
|
||||
session,
|
||||
segment,
|
||||
&[LogEntry::AnnotatedSegmentStart {
|
||||
ts: 1,
|
||||
session_id: session,
|
||||
system_prompt: None,
|
||||
config: agen::llm_client::RequestConfig::default(),
|
||||
history,
|
||||
forked_from: None,
|
||||
compacted_from: None,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
let reader = session_store::public_material::SessionPublicMaterialReader::open_worker(
|
||||
&source_root,
|
||||
segment,
|
||||
Default::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let ranges: Vec<_> = reader
|
||||
.descriptors()
|
||||
.map(|d| JournalSourceRange {
|
||||
source: JournalSource {
|
||||
session_id: session.to_string(),
|
||||
pointer: JournalSourcePointer {
|
||||
segment_id: d.identity.segment_id,
|
||||
entry_ref: d.identity.entry_ref.clone(),
|
||||
},
|
||||
order: d.canonical_order,
|
||||
content_sha256: d.sha256.clone(),
|
||||
total_bytes: d.total_bytes,
|
||||
},
|
||||
bytes: JournalByteRange {
|
||||
start: 0,
|
||||
end: d.total_bytes,
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(ranges.len(), 2);
|
||||
let attr = store
|
||||
.record_session_attribution(
|
||||
crate::subjektiv::SubjectSessionAttribution::new(
|
||||
&subject.id,
|
||||
RUNTIME,
|
||||
&source_worker.worker_id,
|
||||
session.to_string(),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let writer = store
|
||||
.claim_journal_writer(&subject.id, RUNTIME, &source_worker.worker_id)
|
||||
.unwrap();
|
||||
let host = store.bind_journal_session(&writer, attr.clone()).unwrap();
|
||||
let journal_id = store
|
||||
.accept_journal_sources(&host, "original activity", &ranges)
|
||||
.unwrap();
|
||||
let binding = subjektiv::api::JobAttemptBinding {
|
||||
job_id: "fixture-journal-generation".into(),
|
||||
attempt_id: "fixture-attempt".into(),
|
||||
};
|
||||
let generation = store
|
||||
.prepare_journal_generation(&host, &journal_id, binding.clone(), ranges.clone())
|
||||
.unwrap();
|
||||
let job = JournalJobContext::validated(&store, &host, &generation.id, binding).unwrap();
|
||||
for (descriptor, range) in reader.descriptors().zip(&ranges) {
|
||||
subjektiv::journal::service::execute_bounded(
|
||||
&store,
|
||||
&job,
|
||||
JournalJobRequest::ReadText {
|
||||
read: JournalTextRead {
|
||||
target: JournalTextTarget::Source {
|
||||
pointer: range.source.pointer.clone(),
|
||||
},
|
||||
offset: 0,
|
||||
limit: MAX_JOURNAL_READ_BYTES,
|
||||
content_token: None,
|
||||
},
|
||||
},
|
||||
|source, start, limit| {
|
||||
assert_eq!(source, &range.source);
|
||||
let page = reader.read_content(descriptor, start, limit).unwrap();
|
||||
assert_eq!(page.next_offset, range.bytes.end);
|
||||
Ok(subjektiv::journal::recording::JournalMaterial {
|
||||
range: range.clone(),
|
||||
text: page.content,
|
||||
})
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let receipt = store
|
||||
.execute_journal(
|
||||
&job,
|
||||
JournalOperation {
|
||||
request_id: "initial-open".into(),
|
||||
expected_state: generation.initial_journal.state_token,
|
||||
action: JournalAction::Create {
|
||||
title: "Choosing the approach".into(),
|
||||
body: JOURNAL_BODY.into(),
|
||||
ongoing: "Validation pending".into(),
|
||||
state: JournalEntryState::Open,
|
||||
source_end: None,
|
||||
correction_refs: vec![],
|
||||
},
|
||||
covered_ranges: vec![ranges[0].clone()],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let entry_id = receipt.entry_id.unwrap();
|
||||
let kind = memory::extract::CandidateKind::Lesson;
|
||||
let evidence_id = ranges[0].source.pointer.entry_ref.clone();
|
||||
let candidate = memory::extract::StagingRecord::from_candidate(
|
||||
"extracted-candidate",
|
||||
"extraction-run",
|
||||
memory::schema::SourceRef {
|
||||
segment_id: segment.to_string(),
|
||||
range: [0, 0],
|
||||
},
|
||||
memory::extract::ExtractedCandidate {
|
||||
kind: kind.clone(),
|
||||
claim: "Check the constraint before choosing".into(),
|
||||
why_useful: "Avoid repeating the rejected approach".into(),
|
||||
staleness: None,
|
||||
evidence_ids: vec![evidence_id.clone()],
|
||||
},
|
||||
vec![memory::extract::StagingEvidence {
|
||||
id: evidence_id.clone(),
|
||||
kind: memory::schema::EvidenceKind::new("message"),
|
||||
entry_range: Some([0, 0]),
|
||||
origin: Some(serde_json::from_value(json!({"kind":"human_input"})).unwrap()),
|
||||
excerpt: Some("Choose B after checking the constraint".into()),
|
||||
summary: None,
|
||||
}],
|
||||
vec![memory::schema::SourceEvidenceRef {
|
||||
session_id: Some(attr.session_id),
|
||||
segment_id: Some(segment.to_string()),
|
||||
entry_range: Some([0, 0]),
|
||||
evidence_id: Some(evidence_id),
|
||||
origin: Some(serde_json::from_value(json!({"kind":"human_input"})).unwrap()),
|
||||
evidence_kind: Some(memory::schema::EvidenceKind::new("message")),
|
||||
..Default::default()
|
||||
}],
|
||||
);
|
||||
let (staged, _) = store
|
||||
.stage_candidate_with_attribution(
|
||||
crate::subjektiv::SubjectStagingRecord::attach(&subject.id, candidate),
|
||||
store
|
||||
.session_attribution(&session.to_string())
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let (memory, _) = store
|
||||
.apply_candidate(
|
||||
&subject.id,
|
||||
&staged.id,
|
||||
crate::subjektiv::MemoryChangeTarget::Create,
|
||||
crate::subjektiv::MemoryDraft::active(
|
||||
kind,
|
||||
&staged.claim,
|
||||
"Check before choosing.",
|
||||
&staged.why_useful,
|
||||
"Adopt the extracted lesson",
|
||||
),
|
||||
"Accepted after comparison",
|
||||
)
|
||||
.unwrap();
|
||||
let selector = MemoryJournalSelector {
|
||||
memory_id: memory.id,
|
||||
change_id: memory.change_id,
|
||||
candidate_id: staged.id,
|
||||
};
|
||||
Self {
|
||||
_dir: dir,
|
||||
api,
|
||||
identity,
|
||||
store,
|
||||
host,
|
||||
job,
|
||||
worker,
|
||||
source_worker,
|
||||
source_root,
|
||||
selector,
|
||||
journal_id,
|
||||
entry_id,
|
||||
ranges,
|
||||
}
|
||||
}
|
||||
|
||||
fn body_worker(api: &WorkspaceApi, subject: &str) -> RuntimeWorkerRef {
|
||||
let memory = api
|
||||
.config_store
|
||||
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
|
||||
.unwrap();
|
||||
let key = crate::subjektiv::subject_worker_singleton_key(subject).unwrap();
|
||||
let reservation = api
|
||||
.config_store
|
||||
.reserve_worker_create(
|
||||
TEST_WORKSPACE_ID,
|
||||
RUNTIME,
|
||||
&format!("body-{subject}"),
|
||||
&format!("body-input-{subject}"),
|
||||
Some(&key),
|
||||
&memory,
|
||||
)
|
||||
.unwrap();
|
||||
let worker = RuntimeWorkerRef::new(RUNTIME, reservation.worker_id.to_string());
|
||||
seed_worker_source_member(api, RUNTIME, &worker.worker_id);
|
||||
api.config_store
|
||||
.complete_worker_create_reservation(TEST_WORKSPACE_ID, reservation.worker_id)
|
||||
.unwrap();
|
||||
worker
|
||||
}
|
||||
|
||||
fn request(&self, worker: Option<&str>, request: &MemoryJournalRequest) -> Request<Body> {
|
||||
runtime_source_request(
|
||||
&self.identity,
|
||||
worker,
|
||||
"POST",
|
||||
&route_path(),
|
||||
serde_json::to_vec(request).unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn call_as(
|
||||
&self,
|
||||
worker: &RuntimeWorkerRef,
|
||||
request: MemoryJournalRequest,
|
||||
) -> (StatusCode, Vec<u8>) {
|
||||
response(
|
||||
build_router(self.api.clone()),
|
||||
self.request(Some(&worker.worker_id), &request),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn call(&self, request: MemoryJournalRequest) -> MemoryJournalResponse {
|
||||
let (status, bytes) = self.call_as(&self.worker, request).await;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::OK,
|
||||
"{}",
|
||||
String::from_utf8_lossy(&bytes)
|
||||
);
|
||||
serde_json::from_slice(&bytes).unwrap()
|
||||
}
|
||||
|
||||
fn fix_initial(&self) {
|
||||
self.store
|
||||
.execute_journal(
|
||||
&self.job,
|
||||
JournalOperation {
|
||||
request_id: "fix-initial".into(),
|
||||
expected_state: self
|
||||
.store
|
||||
.read_journal(&self.host, &self.journal_id)
|
||||
.unwrap()
|
||||
.state_token,
|
||||
action: JournalAction::Fix {
|
||||
entry_id: self.entry_id.clone(),
|
||||
source_end: self.ranges[0].source.pointer.clone(),
|
||||
},
|
||||
covered_ranges: vec![self.ranges[0].clone()],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn add_correction(&self) -> String {
|
||||
self.store
|
||||
.execute_journal(
|
||||
&self.job,
|
||||
JournalOperation {
|
||||
request_id: "later-correction".into(),
|
||||
expected_state: self
|
||||
.store
|
||||
.read_journal(&self.host, &self.journal_id)
|
||||
.unwrap()
|
||||
.state_token,
|
||||
action: JournalAction::Create {
|
||||
title: "Correction".into(),
|
||||
body: "The chosen approach still needs validation.".into(),
|
||||
ongoing: String::new(),
|
||||
state: JournalEntryState::Fixed,
|
||||
source_end: Some(self.ranges[1].source.pointer.clone()),
|
||||
correction_refs: vec![self.entry_id.clone()],
|
||||
},
|
||||
covered_ranges: vec![self.ranges[1].clone()],
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.entry_id
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn read_request(&self) -> MemoryJournalRequest {
|
||||
MemoryJournalRequest::Read {
|
||||
selector: self.selector.clone(),
|
||||
source_index: 0,
|
||||
journal_id: self.journal_id.clone(),
|
||||
entry_id: self.entry_id.clone(),
|
||||
expected_token: None,
|
||||
byte_offset: 0,
|
||||
max_bytes: 4096,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn memory_journal_router_requires_signed_bound_worker_not_user_or_forged_identity() {
|
||||
let f = Fixture::new().await;
|
||||
let request = MemoryJournalRequest::Links {
|
||||
selector: f.selector.clone(),
|
||||
};
|
||||
let body = serde_json::to_vec(&request).unwrap();
|
||||
let app = build_router(f.api.clone());
|
||||
let anonymous = Request::builder()
|
||||
.method("POST")
|
||||
.uri(route_path())
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(body.clone()))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response(app.clone(), anonymous).await.0,
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
let forged = Request::builder()
|
||||
.method("POST")
|
||||
.uri(route_path())
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header(
|
||||
worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER,
|
||||
"forged",
|
||||
)
|
||||
.body(Body::from(body.clone()))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response(app.clone(), forged).await.0,
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
let token = seed_test_workspace_owner_token(f.api.store.as_ref(), "memory-journal-user").await;
|
||||
let user = Request::builder()
|
||||
.method("POST")
|
||||
.uri(route_path())
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header("authorization", format!("Bearer {token}"))
|
||||
.body(Body::from(body))
|
||||
.unwrap();
|
||||
assert_eq!(response(app.clone(), user).await.0, StatusCode::FORBIDDEN);
|
||||
assert_eq!(
|
||||
response(app, f.request(None, &request)).await.0,
|
||||
StatusCode::FORBIDDEN
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn memory_journal_router_default_worker_and_job_have_no_subject_capability() {
|
||||
let f = Fixture::new().await;
|
||||
let request = MemoryJournalRequest::Reconcile {
|
||||
selector: f.selector.clone(),
|
||||
};
|
||||
let ordinary = RuntimeWorkerRef::new(RUNTIME, Uuid::new_v4().to_string());
|
||||
seed_worker_source_member(&f.api, RUNTIME, &ordinary.worker_id);
|
||||
let (status, bytes) = f.call_as(&ordinary, request.clone()).await;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::FORBIDDEN,
|
||||
"{}",
|
||||
String::from_utf8_lossy(&bytes)
|
||||
);
|
||||
// Even a singleton owner cannot reuse its body authority through a restricted Job binding.
|
||||
let job = BackendJobRequest {
|
||||
job_id: "restricted-job".into(),
|
||||
purpose: "test".into(),
|
||||
input_ref: "fixture://job".into(),
|
||||
input: json!({}),
|
||||
instruction: "No Memory Journal grant".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: Default::default(),
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: Default::default(),
|
||||
};
|
||||
let now = Utc::now().to_rfc3339();
|
||||
let reserved = f
|
||||
.api
|
||||
.store
|
||||
.reserve_backend_job(TEST_WORKSPACE_ID, &job, &now)
|
||||
.unwrap();
|
||||
f.api
|
||||
.store
|
||||
.claim_backend_job_attempt_dispatch(
|
||||
TEST_WORKSPACE_ID,
|
||||
&job.job_id,
|
||||
&reserved.attempt.attempt_id,
|
||||
&now,
|
||||
)
|
||||
.unwrap();
|
||||
f.api
|
||||
.store
|
||||
.bind_backend_job_attempt_worker(
|
||||
TEST_WORKSPACE_ID,
|
||||
&job.job_id,
|
||||
&reserved.attempt.attempt_id,
|
||||
&f.worker,
|
||||
None,
|
||||
&now,
|
||||
)
|
||||
.unwrap();
|
||||
let (status, bytes) = f.call_as(&f.worker, request).await;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::FORBIDDEN,
|
||||
"{}",
|
||||
String::from_utf8_lossy(&bytes)
|
||||
);
|
||||
let reader = JournalHostContext::validated_reader(
|
||||
&f.store,
|
||||
&f.store
|
||||
.read_journal(&f.host, &f.journal_id)
|
||||
.unwrap()
|
||||
.subject_id,
|
||||
)
|
||||
.unwrap();
|
||||
let MemoryJournalResponse::Links(links) = f
|
||||
.store
|
||||
.execute_memory_journal(
|
||||
&reader,
|
||||
MemoryJournalRequest::Links {
|
||||
selector: f.selector.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("links required")
|
||||
};
|
||||
assert_eq!(
|
||||
links.sources[0].connection,
|
||||
SourceConnection::Unconnected {
|
||||
reason: UnconnectedReason::NotReconciled
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn memory_journal_router_reconciles_only_fixed_events_and_retries_without_duplication() {
|
||||
let f = Fixture::new().await;
|
||||
let reconcile = MemoryJournalRequest::Reconcile {
|
||||
selector: f.selector.clone(),
|
||||
};
|
||||
let MemoryJournalResponse::Links(open) = f.call(reconcile.clone()).await else {
|
||||
panic!("links required")
|
||||
};
|
||||
assert_eq!(
|
||||
open.sources[0].connection,
|
||||
SourceConnection::Unconnected {
|
||||
reason: UnconnectedReason::JournalNotFixed
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
f.store
|
||||
.list_journal_entries(&f.host, &f.journal_id, None, 32)
|
||||
.unwrap()
|
||||
.items[0]
|
||||
.state,
|
||||
JournalEntryState::Open
|
||||
);
|
||||
f.fix_initial();
|
||||
// Reading stored state alone must not silently trigger reconciliation.
|
||||
assert_eq!(
|
||||
f.call(MemoryJournalRequest::Links {
|
||||
selector: f.selector.clone()
|
||||
})
|
||||
.await,
|
||||
MemoryJournalResponse::Links(open)
|
||||
);
|
||||
let linked = f.call(reconcile.clone()).await;
|
||||
assert_eq!(f.call(reconcile).await, linked);
|
||||
assert_eq!(
|
||||
f.call(MemoryJournalRequest::Links {
|
||||
selector: f.selector.clone()
|
||||
})
|
||||
.await,
|
||||
linked
|
||||
);
|
||||
let MemoryJournalResponse::Links(links) = linked else {
|
||||
panic!("links required")
|
||||
};
|
||||
assert_eq!(links.sources.len(), 1);
|
||||
assert_eq!(
|
||||
links.sources[0].connection,
|
||||
SourceConnection::Linked {
|
||||
journal_id: f.journal_id.clone(),
|
||||
entry_id: f.entry_id.clone(),
|
||||
target: JournalTargetAvailability::Available,
|
||||
reconsideration: false,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
f.store
|
||||
.list_journal_entries(&f.host, &f.journal_id, None, 32)
|
||||
.unwrap()
|
||||
.items
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn memory_journal_router_reads_saved_text_after_original_session_and_source_worker_removal() {
|
||||
let f = Fixture::new().await;
|
||||
f.fix_initial();
|
||||
f.call(MemoryJournalRequest::Reconcile {
|
||||
selector: f.selector.clone(),
|
||||
})
|
||||
.await;
|
||||
assert!(f.source_root.exists());
|
||||
fs::remove_dir_all(&f.source_root).unwrap();
|
||||
f.api
|
||||
.store
|
||||
.delete_worker_registry(TEST_WORKSPACE_ID, &f.source_worker)
|
||||
.unwrap();
|
||||
assert!(!f.source_root.exists());
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &f.source_worker)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
// No Runtime adapter exists for RUNTIME, so a restore/history dependency would fail.
|
||||
let MemoryJournalResponse::Read(read) = f.call(f.read_request()).await else {
|
||||
panic!("read required")
|
||||
};
|
||||
assert_eq!(
|
||||
read.source.original_source,
|
||||
OriginalSourceAvailability::Missing
|
||||
);
|
||||
assert_eq!(read.page.unwrap().text, JOURNAL_BODY);
|
||||
let MemoryJournalResponse::Related(related) = f
|
||||
.call(MemoryJournalRequest::Related {
|
||||
journal_id: f.journal_id.clone(),
|
||||
entry_id: f.entry_id.clone(),
|
||||
after: None,
|
||||
limit: 32,
|
||||
})
|
||||
.await
|
||||
else {
|
||||
panic!("related required")
|
||||
};
|
||||
assert_eq!(related.items.len(), 1);
|
||||
assert_eq!(related.items[0].selector, f.selector);
|
||||
assert!(!related.items[0].reconsideration);
|
||||
assert!(related.next.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn memory_journal_router_ids_do_not_grant_cross_subject_reads_or_reconciliation() {
|
||||
let f = Fixture::new().await;
|
||||
f.fix_initial();
|
||||
f.call(MemoryJournalRequest::Reconcile {
|
||||
selector: f.selector.clone(),
|
||||
})
|
||||
.await;
|
||||
let other = f
|
||||
.store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("other").unwrap())
|
||||
.unwrap();
|
||||
let other_worker = Fixture::body_worker(&f.api, &other.id);
|
||||
for request in [
|
||||
MemoryJournalRequest::Reconcile {
|
||||
selector: f.selector.clone(),
|
||||
},
|
||||
MemoryJournalRequest::Links {
|
||||
selector: f.selector.clone(),
|
||||
},
|
||||
f.read_request(),
|
||||
MemoryJournalRequest::Related {
|
||||
journal_id: f.journal_id.clone(),
|
||||
entry_id: f.entry_id.clone(),
|
||||
after: None,
|
||||
limit: 32,
|
||||
},
|
||||
] {
|
||||
let (status, bytes) = f.call_as(&other_worker, request).await;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"{}",
|
||||
String::from_utf8_lossy(&bytes)
|
||||
);
|
||||
assert!(!String::from_utf8_lossy(&bytes).contains(JOURNAL_BODY));
|
||||
}
|
||||
// A model cannot override the authenticated Subject with an additional JSON field.
|
||||
let mut body = serde_json::to_value(f.read_request()).unwrap();
|
||||
body["subject_id"] = json!(other.id);
|
||||
let request = runtime_source_request(
|
||||
&f.identity,
|
||||
Some(&f.worker.worker_id),
|
||||
"POST",
|
||||
&route_path(),
|
||||
serde_json::to_vec(&body).unwrap(),
|
||||
);
|
||||
assert_eq!(
|
||||
response(build_router(f.api.clone()), request).await.0,
|
||||
StatusCode::UNPROCESSABLE_ENTITY
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn memory_journal_router_correction_reverse_lookup_preserves_memory_and_original_link() {
|
||||
let f = Fixture::new().await;
|
||||
f.fix_initial();
|
||||
f.call(MemoryJournalRequest::Reconcile {
|
||||
selector: f.selector.clone(),
|
||||
})
|
||||
.await;
|
||||
let subject = f
|
||||
.store
|
||||
.read_journal(&f.host, &f.journal_id)
|
||||
.unwrap()
|
||||
.subject_id;
|
||||
let before =
|
||||
serde_json::to_value(f.store.memory(&subject, &f.selector.memory_id).unwrap()).unwrap();
|
||||
let correction = f.add_correction();
|
||||
let MemoryJournalResponse::Related(related) = f
|
||||
.call(MemoryJournalRequest::Related {
|
||||
journal_id: f.journal_id.clone(),
|
||||
entry_id: correction,
|
||||
after: None,
|
||||
limit: 32,
|
||||
})
|
||||
.await
|
||||
else {
|
||||
panic!("related required")
|
||||
};
|
||||
assert_eq!(related.items.len(), 1);
|
||||
assert_eq!(related.items[0].selector, f.selector);
|
||||
assert!(related.items[0].reconsideration);
|
||||
let MemoryJournalResponse::Read(read) = f.call(f.read_request()).await else {
|
||||
panic!("read required")
|
||||
};
|
||||
assert_eq!(read.page.unwrap().text, JOURNAL_BODY);
|
||||
assert!(matches!(read.source.connection, SourceConnection::Linked {
|
||||
entry_id, reconsideration:true, ..
|
||||
} if entry_id == f.entry_id));
|
||||
assert_eq!(
|
||||
serde_json::to_value(f.store.memory(&subject, &f.selector.memory_id).unwrap()).unwrap(),
|
||||
before
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// Production generated routes, SQLite and WebAuthn verifier share the passkey
|
||||
// security fixture. Only the authenticator is SoftPasskey, not a browser/OS key.
|
||||
// security fixture. The authenticator is a resident model, not a browser/OS key.
|
||||
// The ordinary Server test target selects these isolated integration cases.
|
||||
|
||||
impl Fixture {
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
// The generated HTTP router, real SQLite and production verifier/session issuer
|
||||
// are connected. Only the authenticator is the shared resident software model;
|
||||
// SoftPasskey alone is not resident-capable and browser-native UX is not claimed.
|
||||
|
||||
impl Fixture {
|
||||
async fn discoverable_options(&self) -> PasskeyLoginOptionsResponse {
|
||||
let (status, _, body) = self
|
||||
.post(
|
||||
"/api/auth/passkeys/login/options",
|
||||
None,
|
||||
Some(ORIGIN_URL),
|
||||
serde_json::json!({}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
let options: PasskeyLoginOptionsResponse = serde_json::from_value(body).unwrap();
|
||||
assert!(options.public_key.public_key.allow_credentials.is_empty());
|
||||
assert!(options.public_key.mediation.is_none());
|
||||
assert_eq!(
|
||||
serde_json::to_value(&options.public_key).unwrap()["publicKey"]["userVerification"],
|
||||
"required"
|
||||
);
|
||||
let owner: Option<String> = self
|
||||
.api
|
||||
.config_store
|
||||
.with_conn(|conn| {
|
||||
Ok(conn.query_row(
|
||||
"SELECT user_id FROM auth_challenges WHERE challenge_id=?1",
|
||||
[&options.challenge_id],
|
||||
|row| row.get(0),
|
||||
)?)
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
owner.is_none(),
|
||||
"normal challenge must not preselect any account"
|
||||
);
|
||||
options
|
||||
}
|
||||
|
||||
async fn discoverable_request(&self, key: &mut Authenticator) -> PasskeyLoginCompleteRequest {
|
||||
let options = self.discoverable_options().await;
|
||||
let credential = key
|
||||
.do_authentication(Url::parse(ORIGIN_URL).unwrap(), options.public_key)
|
||||
.unwrap();
|
||||
PasskeyLoginCompleteRequest {
|
||||
challenge_id: options.challenge_id,
|
||||
credential,
|
||||
}
|
||||
}
|
||||
|
||||
async fn discoverable_login(&self, key: &mut Authenticator) -> AuthenticatedUser {
|
||||
let request = self.discoverable_request(key).await;
|
||||
let (status, headers, body) = self
|
||||
.post(
|
||||
"/api/auth/passkeys/login/complete",
|
||||
None,
|
||||
Some(ORIGIN_URL),
|
||||
serde_json::to_value(request).unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
let user: AuthUserResponse = serde_json::from_value(body).unwrap();
|
||||
let cookie = headers[axum::http::header::SET_COOKIE]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.split(';')
|
||||
.next()
|
||||
.unwrap();
|
||||
let token = cookie.split_once('=').unwrap().1;
|
||||
let live = crate::auth::resolve_request_actor_sync(
|
||||
self.api.store.as_ref(),
|
||||
None,
|
||||
Some(token.to_string()),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(live.user_id, user.user.user_id);
|
||||
assert_eq!(live.account_id, user.user.account_id);
|
||||
user.user
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resident_selection_with_multiple_accounts_and_all_enrollment_purposes_issues_only_owner_session()
|
||||
{
|
||||
let f = Fixture::new().await;
|
||||
let mut device = authenticator();
|
||||
let (alice_cookie, alice) = f.enroll("registration", None, "alice", &mut device).await;
|
||||
let alice_id = f.credentials(&alice)[0].credential_id.clone();
|
||||
let (_, bob) = f.enroll("registration", None, "bob", &mut device).await;
|
||||
let bob_id = f.credentials(&bob)[0].credential_id.clone();
|
||||
for (id, expected) in [(alice_id, alice.clone()), (bob_id, bob)] {
|
||||
let id: webauthn_rs::prelude::Base64UrlSafeData =
|
||||
serde_json::from_value(Value::String(id)).unwrap();
|
||||
device.select(&id);
|
||||
assert_eq!(f.discoverable_login(&mut device).await, expected);
|
||||
}
|
||||
let mut added = authenticator();
|
||||
f.enroll("addition", Some(&alice_cookie), "", &mut added)
|
||||
.await;
|
||||
assert_eq!(f.discoverable_login(&mut added).await, alice);
|
||||
// Registration policy controls account creation, not login or own addition.
|
||||
let admin = f.activation_admin().await;
|
||||
assert_eq!(f.discoverable_login(&mut added).await, alice);
|
||||
let issued = f.issue_activation(&admin, "pending-discoverable").await;
|
||||
let mut activated = authenticator();
|
||||
let options = f.activation_options(&issued).await;
|
||||
let body = Fixture::credential(options, &mut activated);
|
||||
let (status, _, body) = f
|
||||
.post(
|
||||
"/api/auth/activation/complete",
|
||||
None,
|
||||
Some(ORIGIN_URL),
|
||||
body,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(f.discoverable_login(&mut activated).await, issued.user);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discoverable_verifier_rejects_unknown_owner_signature_origin_rp_uv_expiry_reuse_and_revocation()
|
||||
{
|
||||
let f = Fixture::new().await;
|
||||
let mut key = authenticator();
|
||||
let (_, alice) = f.enroll("registration", None, "alice", &mut key).await;
|
||||
for defect in [
|
||||
"unknown",
|
||||
"handle",
|
||||
"signature",
|
||||
"origin",
|
||||
"rp",
|
||||
"uv",
|
||||
"expired",
|
||||
"consumed",
|
||||
"stored_id",
|
||||
"stored_owner",
|
||||
] {
|
||||
let mut request = f.discoverable_request(&mut key).await;
|
||||
let original = f.credentials(&alice)[0].clone();
|
||||
let change_byte =
|
||||
|value: &mut webauthn_rs::prelude::Base64UrlSafeData, index: usize, bit: u8| {
|
||||
let mut bytes = value.to_vec();
|
||||
bytes[index] ^= bit;
|
||||
*value = bytes.into();
|
||||
};
|
||||
match defect {
|
||||
"unknown" => change_byte(&mut request.credential.raw_id, 0, 1),
|
||||
"handle" => request.credential.response.user_handle = Some(vec![0; 16].into()),
|
||||
"signature" => change_byte(&mut request.credential.response.signature, 0, 1),
|
||||
"origin" => {
|
||||
let mut data: Value =
|
||||
serde_json::from_slice(&request.credential.response.client_data_json).unwrap();
|
||||
data["origin"] = Value::String("https://other.test".into());
|
||||
request.credential.response.client_data_json =
|
||||
serde_json::to_vec(&data).unwrap().into();
|
||||
}
|
||||
"rp" => change_byte(&mut request.credential.response.authenticator_data, 0, 1),
|
||||
"uv" => change_byte(&mut request.credential.response.authenticator_data, 32, 4),
|
||||
"expired" => f.expire_challenge(&request.challenge_id),
|
||||
"consumed" => {
|
||||
f.api
|
||||
.store
|
||||
.consume_auth_challenge_by_id(
|
||||
&request.challenge_id,
|
||||
"passkey_login",
|
||||
&crate::auth::now_rfc3339(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
"stored_id" => {
|
||||
let mut corrupt = original.clone();
|
||||
let mut value: Value = serde_json::from_str(&corrupt.public_key_cose).unwrap();
|
||||
// Keep the indexed ID, replace the embedded verified credential ID.
|
||||
value["cred"]["cred_id"] = Value::String("YW5vdGhlci1jcmVkZW50aWFs".into());
|
||||
corrupt.public_key_cose = serde_json::to_string(&value).unwrap();
|
||||
f.api.store.upsert_passkey_credential(&corrupt).unwrap();
|
||||
}
|
||||
"stored_owner" => {
|
||||
let (_, bob) = f
|
||||
.enroll("registration", None, "bob", &mut authenticator())
|
||||
.await;
|
||||
f.api
|
||||
.config_store
|
||||
.with_conn(|conn| {
|
||||
conn.execute(
|
||||
"UPDATE passkey_credentials SET user_id=?1 WHERE credential_id=?2",
|
||||
[&bob.user_id, &original.credential_id],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
let (status, headers, body) = f
|
||||
.post(
|
||||
"/api/auth/passkeys/login/complete",
|
||||
None,
|
||||
Some(ORIGIN_URL),
|
||||
serde_json::to_value(&request).unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert!(status.is_client_error(), "{defect}: {body}");
|
||||
assert!(
|
||||
!headers.contains_key(axum::http::header::SET_COOKIE),
|
||||
"{defect}"
|
||||
);
|
||||
let (status, headers, _) = f
|
||||
.post(
|
||||
"/api/auth/passkeys/login/complete",
|
||||
None,
|
||||
Some(ORIGIN_URL),
|
||||
serde_json::to_value(request).unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
status.is_client_error(),
|
||||
"failed attempts must consume their ceremony: {defect}"
|
||||
);
|
||||
assert!(!headers.contains_key(axum::http::header::SET_COOKIE));
|
||||
f.api
|
||||
.config_store
|
||||
.with_conn(|conn| {
|
||||
conn.execute(
|
||||
"UPDATE passkey_credentials SET user_id=?1 WHERE credential_id=?2",
|
||||
[&original.user_id, &original.credential_id],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
f.api.store.upsert_passkey_credential(&original).unwrap();
|
||||
}
|
||||
assert_eq!(f.discoverable_login(&mut key).await, alice);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn signed_assertions_with_foreign_origin_rp_or_without_uv_are_rejected_by_required_verifier()
|
||||
{
|
||||
let f = Fixture::new().await;
|
||||
let mut key = authenticator();
|
||||
let (_, alice) = f.enroll("registration", None, "alice", &mut key).await;
|
||||
let id: webauthn_rs::prelude::Base64UrlSafeData = serde_json::from_value(Value::String(
|
||||
f.credentials(&alice)[0].credential_id.clone(),
|
||||
))
|
||||
.unwrap();
|
||||
for defect in ["origin", "rp", "uv"] {
|
||||
let mut options = f.discoverable_options().await;
|
||||
let mut origin = ORIGIN_URL;
|
||||
match defect {
|
||||
"origin" => origin = "https://sub.workspace.test",
|
||||
"rp" => {
|
||||
origin = "https://other.test";
|
||||
options.public_key.public_key.rp_id = "other.test".into();
|
||||
// Explicit signer selection produces a genuinely signed assertion
|
||||
// for the wrong RP; normal Server options remain empty/unchanged.
|
||||
options.public_key.public_key.allow_credentials =
|
||||
vec![webauthn_rs_proto::AllowCredentials {
|
||||
type_: "public-key".into(),
|
||||
id: id.clone(),
|
||||
transports: None,
|
||||
}];
|
||||
}
|
||||
"uv" => {
|
||||
options.public_key.public_key.user_verification =
|
||||
webauthn_rs_proto::UserVerificationPolicy::Preferred
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
let mut credential = key
|
||||
.do_authentication(Url::parse(origin).unwrap(), options.public_key)
|
||||
.unwrap();
|
||||
if defect == "uv" {
|
||||
assert_eq!(credential.response.authenticator_data[32] & 4, 0);
|
||||
}
|
||||
credential.response.user_handle = Some(
|
||||
passkey_user_handle(&alice.account_id)
|
||||
.as_bytes()
|
||||
.to_vec()
|
||||
.into(),
|
||||
);
|
||||
let request = PasskeyLoginCompleteRequest {
|
||||
challenge_id: options.challenge_id,
|
||||
credential,
|
||||
};
|
||||
rejected(
|
||||
f.post(
|
||||
"/api/auth/passkeys/login/complete",
|
||||
None,
|
||||
Some(ORIGIN_URL),
|
||||
serde_json::to_value(request).unwrap(),
|
||||
)
|
||||
.await,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"webauthn_login_verification_failed",
|
||||
);
|
||||
}
|
||||
assert_eq!(f.discoverable_login(&mut key).await, alice);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_resident_library_authenticator_refuses_registration_without_creating_identity() {
|
||||
use webauthn_authenticator_rs::{WebauthnAuthenticator, softpasskey::SoftPasskey};
|
||||
let f = Fixture::new().await;
|
||||
let options = f.options("registration", None, "unsupported-key").await;
|
||||
let selection = options
|
||||
.public_key
|
||||
.public_key
|
||||
.authenticator_selection
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
assert!(selection.require_resident_key);
|
||||
assert_eq!(
|
||||
selection.resident_key,
|
||||
Some(webauthn_rs_proto::ResidentKeyRequirement::Required)
|
||||
);
|
||||
assert_eq!(
|
||||
selection.user_verification,
|
||||
webauthn_rs_proto::UserVerificationPolicy::Required
|
||||
);
|
||||
let result = WebauthnAuthenticator::new(SoftPasskey::new(true))
|
||||
.do_registration(Url::parse(ORIGIN_URL).unwrap(), options.public_key);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"SoftPasskey itself must not be reported as resident-capable"
|
||||
);
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.get_user_by_handle("unsupported-key")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
@@ -119,7 +119,7 @@ pub use ticket::{
|
||||
};
|
||||
|
||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 100;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 101;
|
||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||
@@ -427,6 +427,11 @@ const MIGRATIONS: &[Migration] = &[
|
||||
name: "Provisional users and scoped passkey activation",
|
||||
apply: activation::migrate,
|
||||
},
|
||||
Migration {
|
||||
version: 101,
|
||||
name: "Durable explicit Subject body release",
|
||||
apply: migrate_subject_release_v100_to_v101,
|
||||
},
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -1954,6 +1959,15 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
||||
workspace_id: &str,
|
||||
singleton_key: &str,
|
||||
) -> Result<Option<WorkerSingletonLease>>;
|
||||
fn retained_worker_singleton_lease(
|
||||
&self,
|
||||
_workspace_id: &str,
|
||||
_worker: &RuntimeWorkerRef,
|
||||
) -> Result<Option<WorkerSingletonLease>> {
|
||||
Err(Error::Store(
|
||||
"retained singleton lookup is unavailable".into(),
|
||||
))
|
||||
}
|
||||
fn require_current_worker_singleton_owner(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -3164,6 +3178,156 @@ impl SqliteWorkspaceStore {
|
||||
})
|
||||
}
|
||||
|
||||
/// Retained identity for lifecycle/retention reconciliation only, never execution authority.
|
||||
pub(crate) fn retained_worker_singleton_lease(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> Result<Option<WorkerSingletonLease>> {
|
||||
self.with_conn(|conn| {
|
||||
let row: Option<(Option<String>, Option<i64>)> = conn.query_row(
|
||||
"SELECT singleton_key,singleton_generation FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
|
||||
params![workspace_id,worker.runtime_id,worker.worker_id], |r| Ok((r.get(0)?,r.get(1)?)),
|
||||
).optional()?;
|
||||
match row {
|
||||
Some((Some(key),Some(generation))) => Ok(Some(WorkerSingletonLease {
|
||||
key, generation: generation.try_into().map_err(|_| Error::Store("invalid singleton generation".into()))?, worker:worker.clone(),
|
||||
})),
|
||||
Some((None,None)) | None => Ok(None),
|
||||
_ => Err(Error::Store("incomplete singleton lease".into())),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn retained_subject_release_operation(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
) -> Result<Option<String>> {
|
||||
self.with_conn(|c| Ok(c.query_row("SELECT operation_id FROM subject_worker_releases WHERE workspace_id=?1 AND worker_id=?2 AND runtime_id=?3 AND singleton_generation=?4 AND binding_id=?5 AND cancelled=0",params![workspace,lease.worker.worker_id,lease.worker.runtime_id,lease.generation as i64,binding],|r|r.get(0)).optional()?))
|
||||
}
|
||||
|
||||
pub(crate) fn subject_release_pending(
|
||||
&self,
|
||||
workspace: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> Result<bool> {
|
||||
self.with_conn(|c| Ok(c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND cancelled=0 AND completed=0)",params![workspace,worker.runtime_id,worker.worker_id],|r|r.get(0))?))
|
||||
}
|
||||
|
||||
pub(crate) fn require_subject_release_intent(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<()> {
|
||||
self.with_conn(|c| {
|
||||
let active: bool = c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND binding_id=?6 AND operation_id=?7 AND cancelled=0)",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,binding,operation],|r|r.get(0))?;
|
||||
if !active {return Err(Error::RepositoryConflict("Subject release intent absent, cancelled or mismatched".into()));} Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn reserve_subject_release(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<()> {
|
||||
if binding.trim().is_empty()
|
||||
|| operation.trim().is_empty()
|
||||
|| binding.len() > 512
|
||||
|| operation.len() > 512
|
||||
{
|
||||
return Err(Error::InvalidInput(
|
||||
"invalid Subject release identity".into(),
|
||||
));
|
||||
}
|
||||
self.with_conn_mut(|c| {
|
||||
let tx=c.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let saved: Option<(String,String,String,u64,String,bool)> = tx.query_row("SELECT runtime_id,worker_id,singleton_key,singleton_generation,binding_id,cancelled FROM subject_worker_releases WHERE workspace_id=?1 AND operation_id=?2",params![workspace,operation],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?))).optional()?;
|
||||
if let Some(saved)=saved {
|
||||
if saved != (lease.worker.runtime_id.clone(),lease.worker.worker_id.clone(),lease.key.clone(),lease.generation,binding.into(),false) {return Err(Error::RepositoryConflict("release operation mismatched or terminally cancelled; use a new operation".into()));} return Ok(());
|
||||
}
|
||||
require_current_worker_singleton_lease(&tx,workspace,lease)?;
|
||||
let busy: bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND worker_id=?2 AND cancelled=0)",params![workspace,lease.worker.worker_id],|r|r.get(0))?;
|
||||
if busy {return Err(Error::RepositoryConflict("another Subject release intent is active".into()));}
|
||||
tx.execute("INSERT INTO subject_worker_releases(workspace_id,runtime_id,worker_id,singleton_key,singleton_generation,binding_id,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7)",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,binding,operation])?;
|
||||
tx.commit()?; Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn cancel_subject_release(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<()> {
|
||||
self.with_conn(|c| {
|
||||
let changed=c.execute("UPDATE subject_worker_releases SET cancelled=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_generation=?4 AND binding_id=?5 AND operation_id=?6 AND proof IS NULL AND completed=0",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.generation as i64,binding,operation])?;
|
||||
if changed!=1 {return Err(Error::RepositoryConflict("cannot confirm cancellation of different or committed Subject release".into()));} Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn subject_release_proven(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<bool> {
|
||||
self.with_conn(|c|Ok(c.query_row("SELECT EXISTS(SELECT 1 FROM subject_worker_releases WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND binding_id=?6 AND operation_id=?7 AND proof IS NOT NULL AND cancelled=0)",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,binding,operation],|r|r.get(0))?))
|
||||
}
|
||||
|
||||
pub(crate) fn confirm_subject_release_proof(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
erased: bool,
|
||||
) -> Result<()> {
|
||||
let proof = if erased {
|
||||
"erased"
|
||||
} else {
|
||||
"runtime_disconnected"
|
||||
};
|
||||
self.with_conn(|c| {
|
||||
let changed=c.execute("UPDATE subject_worker_releases SET proof=?8 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND binding_id=?6 AND operation_id=?7 AND cancelled=0 AND (proof IS NULL OR proof=?8)",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,binding,operation,proof])?;
|
||||
if changed!=1 {return Err(Error::RepositoryConflict("Subject release proof does not match active intent".into()));} Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn release_subject_singleton(
|
||||
&self,
|
||||
workspace: &str,
|
||||
lease: &WorkerSingletonLease,
|
||||
binding: &str,
|
||||
operation: &str,
|
||||
) -> Result<()> {
|
||||
self.require_subject_release_intent(workspace, lease, binding, operation)?;
|
||||
if !lease
|
||||
.key
|
||||
.starts_with(crate::subjektiv::SUBJECT_WORKER_SINGLETON_PREFIX)
|
||||
{
|
||||
return Err(Error::InvalidInput("not a Subject singleton".into()));
|
||||
}
|
||||
self.with_conn_mut(|c| {
|
||||
let tx=c.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let (proof,completed,cancelled):(Option<String>,bool,bool)=tx.query_row("SELECT proof,completed,cancelled FROM subject_worker_releases WHERE workspace_id=?1 AND operation_id=?2",params![workspace,operation],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?;
|
||||
if cancelled || proof.is_none() {return Err(Error::RepositoryConflict("Subject release lacks confirmed proof; retain occupancy".into()));}
|
||||
if completed {return Ok(());}
|
||||
require_current_worker_singleton_lease(&tx,workspace,lease)?;
|
||||
tx.execute("UPDATE subject_worker_releases SET completed=1 WHERE workspace_id=?1 AND operation_id=?2 AND cancelled=0",params![workspace,operation])?;
|
||||
let changed=tx.execute("UPDATE worker_create_reservations SET state='removed',updated_at=?6 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND singleton_key=?4 AND singleton_generation=?5 AND state IN ('reserved','created','removed')",params![workspace,lease.worker.runtime_id,lease.worker.worker_id,lease.key,lease.generation as i64,chrono::Utc::now().to_rfc3339()])?;
|
||||
if changed!=1 {return Err(Error::RepositoryConflict("Subject owner changed before release".into()));}
|
||||
tx.commit()?; Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn current_worker_singleton_owner(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -7931,6 +8095,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
SqliteWorkspaceStore::current_worker_singleton_owner(self, workspace_id, singleton_key)
|
||||
}
|
||||
|
||||
fn retained_worker_singleton_lease(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> Result<Option<WorkerSingletonLease>> {
|
||||
SqliteWorkspaceStore::retained_worker_singleton_lease(self, workspace_id, worker)
|
||||
}
|
||||
|
||||
fn require_current_worker_singleton_owner(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -14840,6 +15012,22 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_subject_release_v100_to_v101(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? != 100 {
|
||||
return Err(Error::Store(
|
||||
"Subject release migration requires schema 100".into(),
|
||||
));
|
||||
}
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
tx.execute_batch(include_str!("subject_release_schema.sql"))?;
|
||||
tx.execute(
|
||||
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(101,?1)",
|
||||
["Durable explicit Subject body release"],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_hosting_policy_v98_to_v99(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? != 98 {
|
||||
return Err(Error::Store(
|
||||
@@ -15521,7 +15709,10 @@ fn current_worker_singleton_owner(
|
||||
) -> Result<Option<(WorkerSingletonLease, Option<String>)>> {
|
||||
let owner = conn
|
||||
.query_row(
|
||||
"SELECT owner.runtime_id, owner.worker_id, owner.generation, reservation.state \
|
||||
"SELECT owner.runtime_id, owner.worker_id, owner.generation, \
|
||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||
THEN 'created' ELSE reservation.state END \
|
||||
FROM worker_singleton_owners owner \
|
||||
LEFT JOIN worker_create_reservations reservation \
|
||||
ON reservation.workspace_id = owner.workspace_id \
|
||||
@@ -18547,6 +18738,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
WorkspaceSchemaMigrationStep {version:98, name:"Assignment-independent implementation reports".into()},
|
||||
WorkspaceSchemaMigrationStep {version:99, name:"Explicit hosting administrators and self-registration policy".into()},
|
||||
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
|
||||
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -18657,6 +18849,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(98, "Assignment-independent implementation reports".to_string()),
|
||||
(99, "Explicit hosting administrators and self-registration policy".to_string()),
|
||||
(100, "Provisional users and scoped passkey activation".to_string()),
|
||||
(101, "Durable explicit Subject body release".to_string()),
|
||||
]
|
||||
);
|
||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||
@@ -20734,7 +20927,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let key = "subjektiv:主体 id/?!";
|
||||
// Generic singleton lifecycle; Subject-prefixed keys have a separate explicit release contract.
|
||||
let key = "fixture:主体 id/?!";
|
||||
let memory_a = store.get_workspace_memory_settings("workspace-a").unwrap();
|
||||
let first = store
|
||||
.reserve_worker_create(
|
||||
@@ -20842,6 +21036,175 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_release_migration_rolls_back_schema_and_marker_then_reopens() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let database = dir.path().join("server.db");
|
||||
prepare_retained_schema(&database, 100);
|
||||
let conn = Connection::open(&database).unwrap();
|
||||
configure_sqlite(&conn).unwrap();
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_subject_release_migration BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=101 BEGIN SELECT RAISE(ABORT,'injected Subject release migration interruption'); END;").unwrap();
|
||||
assert!(migrate_subject_release_v100_to_v101(&conn).is_err());
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 100);
|
||||
assert!(!table_exists(&conn, "subject_worker_releases").unwrap());
|
||||
conn.execute_batch("DROP TRIGGER interrupt_subject_release_migration")
|
||||
.unwrap();
|
||||
migrate_subject_release_v100_to_v101(&conn).unwrap();
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 101);
|
||||
assert!(table_exists(&conn, "subject_worker_releases").unwrap());
|
||||
assert!(migrate_subject_release_v100_to_v101(&conn).is_err());
|
||||
drop(conn);
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(database).unwrap();
|
||||
reopened
|
||||
.with_conn(|conn| verify_schema_history(conn, 101))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subject_generic_removal_retains_occupancy_until_exact_explicit_release_receipt() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let database = dir.path().join("server.db");
|
||||
let store = SqliteWorkspaceStore::open_api_fixture(&database).unwrap();
|
||||
store
|
||||
.upsert_workspace(&WorkspaceRecord {
|
||||
workspace_id: "workspace-a".into(),
|
||||
owner_account_id: "owner-account".into(),
|
||||
display_name: "Workspace A".into(),
|
||||
state: "active".into(),
|
||||
created_at: "2026-10-01T00:00:00Z".into(),
|
||||
updated_at: "2026-10-01T00:00:00Z".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let memory = store.get_workspace_memory_settings("workspace-a").unwrap();
|
||||
let key = "subjektiv:subject-a";
|
||||
let first = store
|
||||
.reserve_worker_create(
|
||||
"workspace-a",
|
||||
"runtime-a",
|
||||
"first",
|
||||
"sha256:first",
|
||||
Some(key),
|
||||
&memory,
|
||||
)
|
||||
.unwrap();
|
||||
let lease = first.singleton.unwrap();
|
||||
store
|
||||
.fail_worker_create_reservation(
|
||||
"workspace-a",
|
||||
"runtime-a",
|
||||
first.worker_id,
|
||||
&first.create_fingerprint,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.current_worker_singleton_owner("workspace-a", key)
|
||||
.unwrap(),
|
||||
Some(lease.clone())
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.reserve_worker_create(
|
||||
"workspace-a",
|
||||
"runtime-b",
|
||||
"early",
|
||||
"sha256:early",
|
||||
Some(key),
|
||||
&memory
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
store
|
||||
.reserve_subject_release("workspace-a", &lease, "binding-a", "cancelled-first")
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.subject_release_pending("workspace-a", &lease.worker)
|
||||
.unwrap()
|
||||
);
|
||||
store
|
||||
.cancel_subject_release("workspace-a", &lease, "binding-a", "cancelled-first")
|
||||
.unwrap();
|
||||
assert!(
|
||||
!store
|
||||
.subject_release_pending("workspace-a", &lease.worker)
|
||||
.unwrap()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.reserve_subject_release("workspace-a", &lease, "binding-a", "cancelled-first")
|
||||
.is_err()
|
||||
);
|
||||
store
|
||||
.reserve_subject_release("workspace-a", &lease, "binding-a", "release-a")
|
||||
.unwrap();
|
||||
drop(store);
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.current_worker_singleton_owner("workspace-a", key)
|
||||
.unwrap(),
|
||||
Some(lease.clone())
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.reserve_subject_release("workspace-a", &lease, "binding-later", "release-a")
|
||||
.is_err()
|
||||
);
|
||||
// SQL component boundary: the Host's actual Controller and durable writer-revoke proof
|
||||
// are tested at their integration boundaries, not certified by this private SQL helper.
|
||||
assert!(
|
||||
reopened
|
||||
.release_subject_singleton("workspace-a", &lease, "binding-a", "release-a")
|
||||
.is_err()
|
||||
);
|
||||
reopened
|
||||
.confirm_subject_release_proof("workspace-a", &lease, "binding-a", "release-a", false)
|
||||
.unwrap();
|
||||
reopened
|
||||
.release_subject_singleton("workspace-a", &lease, "binding-a", "release-a")
|
||||
.unwrap();
|
||||
assert!(
|
||||
reopened
|
||||
.current_worker_singleton_owner("workspace-a", key)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let replacement = reopened
|
||||
.reserve_worker_create(
|
||||
"workspace-a",
|
||||
"runtime-b",
|
||||
"next",
|
||||
"sha256:next",
|
||||
Some(key),
|
||||
&memory,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
replacement.singleton.unwrap().generation,
|
||||
lease.generation + 1
|
||||
);
|
||||
reopened
|
||||
.release_subject_singleton("workspace-a", &lease, "binding-a", "release-a")
|
||||
.unwrap();
|
||||
assert!(
|
||||
reopened
|
||||
.release_subject_singleton(
|
||||
"workspace-a",
|
||||
&lease,
|
||||
"binding-a",
|
||||
"different-operation"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.require_current_worker_singleton_owner("workspace-a", &lease.worker)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn concurrent_worker_singleton_reservations_have_one_winner() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -21272,7 +21635,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(97, "MR notifications and authenticated implementation results".into()),
|
||||
(98, "Assignment-independent implementation reports".into()),
|
||||
(99, "Explicit hosting administrators and self-registration policy".into()),
|
||||
(100, "Provisional users and scoped passkey activation".into())]);
|
||||
(100, "Provisional users and scoped passkey activation".into()),
|
||||
(101, "Durable explicit Subject body release".into())]);
|
||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE subject_worker_releases (
|
||||
workspace_id TEXT NOT NULL,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
singleton_key TEXT NOT NULL,
|
||||
singleton_generation INTEGER NOT NULL CHECK (singleton_generation > 0),
|
||||
binding_id TEXT NOT NULL,
|
||||
operation_id TEXT NOT NULL,
|
||||
proof TEXT CHECK (proof IN ('runtime_disconnected','erased')),
|
||||
completed INTEGER NOT NULL DEFAULT 0 CHECK (completed IN (0,1)),
|
||||
cancelled INTEGER NOT NULL DEFAULT 0 CHECK (cancelled IN (0,1)),
|
||||
PRIMARY KEY (workspace_id, operation_id),
|
||||
CHECK (cancelled = 0 OR (completed = 0 AND proof IS NULL)),
|
||||
FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_create_reservations(workspace_id, worker_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE UNIQUE INDEX subject_worker_release_active ON subject_worker_releases(workspace_id,worker_id) WHERE cancelled=0;
|
||||
@@ -94,6 +94,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
||||
"workdir_registry",
|
||||
"workdir_removal_operations",
|
||||
"worker_control_grants",
|
||||
"subject_worker_releases",
|
||||
"worker_create_reservations",
|
||||
"worker_diagnostics_archives",
|
||||
"worker_mutation_source_proof_jtis",
|
||||
|
||||
Reference in New Issue
Block a user