chore: merge develop into hare/develop

This commit is contained in:
2026-10-03 19:14:17 +09:00
147 changed files with 33175 additions and 2270 deletions
+1
View File
@@ -51,6 +51,7 @@ webauthn-rs = { workspace = true }
[dev-dependencies]
agen.workspace = true
session-store.workspace = true
tempfile.workspace = true
tools.workspace = true
yoi = { path = "../yoi" }
+3 -5
View File
@@ -820,11 +820,9 @@ impl SqliteWorkspaceAuthority {
.map(|worker| (assignment, worker))
})
.map(|(assignment, worker)| {
let worker_resource_key = self.store.resource_key(
&self.workspace_id,
WorkspaceResourceKind::Worker,
&worker.worker_id,
)?;
let worker_resource_key = self
.store
.worker_resource_key(&self.workspace_id, &worker)?;
Ok::<_, Error>(TicketAssignmentSummary {
assignment_id: assignment.assignment_id.clone(),
runtime_id: worker.runtime_id,
+117 -1
View File
@@ -534,6 +534,9 @@ pub struct WorkerSpawnRequest {
/// Backend-authored immutable Workspace Memory settings snapshot.
#[serde(skip, default)]
pub resolved_memory_settings: Option<manifest::WorkspaceMemorySettingsSnapshot>,
/// Backend-attested subjektiv singleton attachment. Browser/model input cannot set it.
#[serde(skip, default)]
pub resolved_subjektiv_attached: bool,
/// Backend-owned feature enablement; client input cannot set it.
#[serde(skip, default)]
pub resolved_worker_observation_enabled: bool,
@@ -1217,6 +1220,26 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
})
}
fn session_public_search(
&self,
_request: runtime_api::SessionPublicSearchRequest,
) -> Result<runtime_api::SessionPublicSearchAvailability, String> {
Ok(runtime_api::SessionPublicSearchAvailability::Unavailable {
reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable,
message: "Session public search is not supported by this Runtime".to_string(),
})
}
fn session_public_read(
&self,
_request: runtime_api::SessionPublicReadRequest,
) -> Result<runtime_api::SessionPublicReadAvailability, String> {
Ok(runtime_api::SessionPublicReadAvailability::Unavailable {
reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable,
message: "Session public read is not supported by this Runtime".to_string(),
})
}
fn observation_source(
&self,
_worker_id: &str,
@@ -2073,6 +2096,36 @@ impl RuntimeRegistry {
})
}
pub fn session_public_search(
&self,
runtime_id: &str,
request: runtime_api::SessionPublicSearchRequest,
) -> Result<runtime_api::SessionPublicSearchAvailability, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.session_public_search(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.to_string(),
code: "session_public_search_failed".to_string(),
message,
})
}
pub fn session_public_read(
&self,
runtime_id: &str,
request: runtime_api::SessionPublicReadRequest,
) -> Result<runtime_api::SessionPublicReadAvailability, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.session_public_read(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.to_string(),
code: "session_public_read_failed".to_string(),
message,
})
}
pub fn observation_source(
&self,
worker: &RuntimeWorkerRef,
@@ -2964,6 +3017,36 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
.map_err(|error| error.to_string())
}
fn session_public_search(
&self,
request: runtime_api::SessionPublicSearchRequest,
) -> Result<runtime_api::SessionPublicSearchAvailability, String> {
if request.workspace_id != self.workspace_id {
return Err(
"Session public search Workspace does not match embedded Runtime".to_string(),
);
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
self.runtime
.session_public_search_scoped(&scope, &request)
.map_err(|error| error.to_string())
}
fn session_public_read(
&self,
request: runtime_api::SessionPublicReadRequest,
) -> Result<runtime_api::SessionPublicReadAvailability, String> {
if request.workspace_id != self.workspace_id {
return Err(
"Session public read Workspace does not match embedded Runtime".to_string(),
);
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
self.runtime
.session_public_read_scoped(&scope, &request)
.map_err(|error| error.to_string())
}
fn observation_source(
&self,
worker_id: &str,
@@ -3669,6 +3752,9 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
if path.contains("/retention/") || (path.starts_with("/v1/workers/") && method == "DELETE") {
return "workers:delete";
}
if path.starts_with("/v1/session-public/") && method == "POST" {
return "workers:read";
}
if path.starts_with("/v1/workers/") && method == "GET" {
return "workers:read";
}
@@ -5007,6 +5093,30 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
.map_err(|diagnostic| diagnostic.message)
}
fn session_public_search(
&self,
request: runtime_api::SessionPublicSearchRequest,
) -> Result<runtime_api::SessionPublicSearchAvailability, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.session_public_search(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn session_public_read(
&self,
request: runtime_api::SessionPublicReadRequest,
) -> Result<runtime_api::SessionPublicReadAvailability, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.session_public_read(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn observation_source(
&self,
worker_id: &str,
@@ -5246,6 +5356,7 @@ fn runtime_create_worker_request(
worker_observation_grants: request.resolved_worker_observation_grants.clone(),
workspace_api: Some(workspace_api),
memory_settings: request.resolved_memory_settings.clone(),
subjektiv_attached: request.resolved_subjektiv_attached,
}
}
@@ -5370,6 +5481,7 @@ fn embedded_profile_label(profile: &ProfileSelector) -> Option<String> {
const MEMORY_CONSOLIDATION_PROFILE: &str = "memory-consolidation";
pub(crate) const MEMORY_CONSOLIDATION_SINGLETON_KEY: &str = "workspace-memory-consolidation";
pub(crate) const SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX: &str = "subjektiv-consolidation:";
const WORKSPACE_ORCHESTRATOR_PROFILE: &str = "orchestrator";
pub(crate) const WORKSPACE_ORCHESTRATOR_SINGLETON_KEY: &str = "workspace-orchestrator";
@@ -5377,7 +5489,7 @@ pub(crate) fn is_reserved_internal_worker_singleton_key(key: &str) -> bool {
matches!(
key,
MEMORY_CONSOLIDATION_SINGLETON_KEY | WORKSPACE_ORCHESTRATOR_SINGLETON_KEY
)
) || key.starts_with(SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX)
}
struct WorkerDisplayMetadata {
@@ -7030,6 +7142,7 @@ mod tests {
resolved_control_operation: None,
resolved_workspace_api: Some(test_workspace_api()),
resolved_memory_settings: Some(test_memory_settings()),
resolved_subjektiv_attached: false,
}
}
@@ -7318,6 +7431,7 @@ mod tests {
resolved_control_operation: None,
resolved_workspace_api: Some(test_workspace_api()),
resolved_memory_settings: Some(test_memory_settings()),
resolved_subjektiv_attached: false,
},
)
.unwrap();
@@ -7420,6 +7534,7 @@ mod tests {
resolved_control_operation: None,
resolved_workspace_api: Some(test_workspace_api()),
resolved_memory_settings: Some(test_memory_settings()),
resolved_subjektiv_attached: false,
},
)
.unwrap();
@@ -7460,6 +7575,7 @@ mod tests {
resolved_control_operation: None,
resolved_workspace_api: Some(test_workspace_api()),
resolved_memory_settings: Some(test_memory_settings()),
resolved_subjektiv_attached: false,
},
)
.unwrap();
+25 -30
View File
@@ -753,11 +753,11 @@ CREATE TABLE "worker_control_grants" (
created_at TEXT NOT NULL,
revoked_at TEXT,
PRIMARY KEY (workspace_id, grant_id),
UNIQUE (workspace_id, controller_worker_id, operation_id),
FOREIGN KEY (workspace_id, controller_worker_id)
REFERENCES "worker_registry"(workspace_id, worker_id) ON DELETE CASCADE,
FOREIGN KEY (workspace_id, subject_worker_id)
REFERENCES "worker_registry"(workspace_id, worker_id) ON DELETE CASCADE
UNIQUE (workspace_id, controller_runtime_id, controller_worker_id, operation_id),
FOREIGN KEY (workspace_id, controller_runtime_id, controller_worker_id)
REFERENCES "worker_registry"(workspace_id, runtime_id, worker_id) ON DELETE CASCADE,
FOREIGN KEY (workspace_id, subject_runtime_id, subject_worker_id)
REFERENCES "worker_registry"(workspace_id, runtime_id, worker_id) ON DELETE CASCADE
);
CREATE TABLE worker_create_reservations (
workspace_id TEXT NOT NULL,
@@ -832,7 +832,7 @@ CREATE TABLE "worker_registry" (
diagnostics_ref TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (workspace_id, worker_id),
PRIMARY KEY (workspace_id, runtime_id, worker_id),
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE TABLE backend_jobs (
@@ -914,31 +914,15 @@ CREATE TABLE worker_registry_observations (
worker_id TEXT NOT NULL,
availability TEXT NOT NULL CHECK (availability IN ('observed', 'unavailable')),
worker_json TEXT,
connection_generation INTEGER NOT NULL,
subject_revision INTEGER NOT NULL,
snapshot_revision INTEGER NOT NULL,
projection_revision INTEGER NOT NULL,
observed_at TEXT NOT NULL,
PRIMARY KEY (workspace_id, runtime_id, worker_id),
FOREIGN KEY (workspace_id, worker_id)
REFERENCES worker_registry(workspace_id, worker_id) ON DELETE CASCADE
);
CREATE TABLE worker_registry_projection_cursors (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
connection_generation INTEGER NOT NULL,
snapshot_revision INTEGER NOT NULL,
PRIMARY KEY (workspace_id, runtime_id)
);
CREATE TABLE worker_registry_projection_revisions (
workspace_id TEXT PRIMARY KEY,
revision INTEGER NOT NULL
FOREIGN KEY (workspace_id, runtime_id, worker_id)
REFERENCES worker_registry(workspace_id, runtime_id, worker_id) ON DELETE CASCADE
);
CREATE TABLE worker_registry_projection_removals (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
projection_revision INTEGER NOT NULL,
PRIMARY KEY (workspace_id, runtime_id, worker_id)
);
CREATE TABLE worker_registry_projection_diagnostics (
@@ -973,6 +957,17 @@ CREATE TABLE worker_session_archives (
committed_at TEXT NOT NULL, expires_at TEXT,
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
FOREIGN KEY(operation_id) REFERENCES worker_removal_operations(operation_id));
CREATE TABLE worker_session_archive_observe_grants (
workspace_id TEXT NOT NULL, archive_id TEXT NOT NULL,
controller_runtime_id TEXT NOT NULL, controller_worker_id TEXT NOT NULL,
subject_runtime_id TEXT NOT NULL, subject_worker_id TEXT NOT NULL,
source_grant_id TEXT NOT NULL, granted_at TEXT NOT NULL, revoked_at TEXT,
PRIMARY KEY(workspace_id,archive_id,controller_runtime_id,controller_worker_id,source_grant_id),
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
FOREIGN KEY(archive_id) REFERENCES worker_session_archives(archive_id) ON DELETE CASCADE);
CREATE INDEX worker_session_archive_observe_grants_controller
ON worker_session_archive_observe_grants(
workspace_id,controller_runtime_id,controller_worker_id,archive_id);
CREATE TABLE worker_tombstones (
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
display_name TEXT NOT NULL, profile TEXT, worker_created_at TEXT NOT NULL, removed_at TEXT NOT NULL,
@@ -998,9 +993,9 @@ CREATE TABLE "worker_workdir_links" (
alias TEXT NOT NULL,
linked_at TEXT NOT NULL,
unlinked_at TEXT,
PRIMARY KEY (workspace_id, worker_id, workdir_id, alias),
FOREIGN KEY (workspace_id, worker_id)
REFERENCES "worker_registry"(workspace_id, worker_id) ON DELETE CASCADE,
PRIMARY KEY (workspace_id, runtime_id, worker_id, workdir_id, alias),
FOREIGN KEY (workspace_id, runtime_id, worker_id)
REFERENCES "worker_registry"(workspace_id, runtime_id, worker_id) ON DELETE CASCADE,
FOREIGN KEY (workspace_id, workdir_id)
REFERENCES workdir_registry(workspace_id, workdir_id) ON DELETE CASCADE
);
@@ -1217,11 +1212,11 @@ CREATE UNIQUE INDEX ux_worker_workdir_attachment_reservation_id
ON worker_workdir_attachment_reservations(workspace_id, reservation_id);
CREATE INDEX worker_control_grants_controller
ON worker_control_grants(
workspace_id, controller_worker_id, revoked_at
workspace_id, controller_runtime_id, controller_worker_id, revoked_at
);
CREATE INDEX worker_control_grants_subject
ON worker_control_grants(
workspace_id, subject_worker_id, revoked_at
workspace_id, subject_runtime_id, subject_worker_id, revoked_at
);
CREATE INDEX worker_create_reservations_worker
ON worker_create_reservations(workspace_id, worker_id);
@@ -1232,7 +1227,7 @@ CREATE UNIQUE INDEX worker_workdir_links_active_workdir_unique
ON worker_workdir_links(workspace_id, workdir_id)
WHERE unlinked_at IS NULL;
CREATE UNIQUE INDEX worker_workdir_links_active_alias_unique
ON worker_workdir_links(workspace_id, worker_id, alias)
ON worker_workdir_links(workspace_id, runtime_id, worker_id, alias)
WHERE unlinked_at IS NULL;
CREATE INDEX worker_workdir_links_workdir
ON worker_workdir_links(workspace_id, workdir_id);
+2
View File
@@ -112,6 +112,8 @@ pub enum Error {
},
#[error("unknown local repository `{0}`")]
UnknownRepository(String),
#[error("subjektiv subject `{0}` was not found in this Workspace")]
SubjektivSubjectNotFound(String),
#[error(
"Merge Request reopen requires an authenticated Browser session and explicit confirmation"
)]
+11 -8
View File
@@ -1,8 +1,9 @@
use memory::backend::{
MemoryBackendAckOutput, MemoryBackendOperation, MemoryBackendOperationResult,
MemoryDocumentReadOperation, MemoryDocumentUpdateOperation, MemoryQueryOperation,
MemoryStagingCloseAction, MemoryStagingCloseOperation, MemoryStagingListOperation,
MemoryStagingReadOperation, MemoryStagingWriteOutput, MemoryToolOutput,
MemoryResidentSummaryAvailability, MemoryResidentSummaryOutput, MemoryStagingCloseAction,
MemoryStagingCloseOperation, MemoryStagingListOperation, MemoryStagingReadOperation,
MemoryStagingWriteOutput, MemoryToolOutput,
};
use memory::extract::{ExtractedCandidate, StagingEvidence, StagingRecord};
use memory::schema::{SourceEvidenceRef, SourceRef};
@@ -32,7 +33,7 @@ pub fn execute_memory_backend_operation_with_authority<A: MemoryAuthority>(
.map(MemoryBackendOperationResult::ToolOutput)
}
MemoryBackendOperation::ResidentSummary(_operation) => Ok(
MemoryBackendOperationResult::ToolOutput(execute_resident_summary(authority)?),
MemoryBackendOperationResult::ResidentSummary(execute_resident_summary(authority)?),
),
MemoryBackendOperation::AppendAudit(_operation) => Ok(
MemoryBackendOperationResult::Acknowledged(MemoryBackendAckOutput {
@@ -88,16 +89,18 @@ pub fn execute_memory_backend_operation_with_authority<A: MemoryAuthority>(
}
}
fn execute_resident_summary<A: MemoryAuthority>(authority: &A) -> Result<MemoryToolOutput> {
fn execute_resident_summary<A: MemoryAuthority>(
authority: &A,
) -> Result<MemoryResidentSummaryOutput> {
let document = authority.ensure_memory_document()?;
if document.body_md.trim().is_empty() {
Ok(MemoryToolOutput {
summary: "resident memory summary unavailable".to_string(),
Ok(MemoryResidentSummaryOutput {
availability: MemoryResidentSummaryAvailability::Unavailable,
content: None,
})
} else {
Ok(MemoryToolOutput {
summary: "resident memory document collected".to_string(),
Ok(MemoryResidentSummaryOutput {
availability: MemoryResidentSummaryAvailability::Ready,
content: Some(document.body_md),
})
}
+183
View File
@@ -132,6 +132,21 @@ pub struct WorkerTombstone {
pub operation_id: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerSessionArchiveRecord {
pub archive_id: String,
pub workspace_id: String,
pub worker: RuntimeWorkerRef,
pub session_id: String,
pub checksum_sha256: String,
pub content_bytes: u64,
pub policy_id: String,
pub policy_revision: u64,
pub operation_id: String,
pub committed_at: String,
pub expires_at: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerOrphanDiagnostic {
pub diagnostic_id: String,
@@ -454,6 +469,51 @@ impl SqliteWorkspaceStore {
if plan.archive_id.as_deref()!=Some(&a.archive_id)||a.workspace_id!=workspace_id||a.source_runtime_id!=plan.worker.runtime_id||a.source_worker_id.to_string()!=plan.worker.worker_id||a.policy_id!=plan.policy_id||a.policy_revision!=plan.policy_revision{return Err(StoreError::InvalidInput("archive manifest mismatch".into()));}
let expires_at=match plan.archive_retention { ArchiveRetention::Forever=>None, ArchiveRetention::ForSeconds{seconds}=>{let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("archive retention deadline overflow".into()))?;Some((Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339())} };
tx.execute("INSERT OR IGNORE INTO worker_session_archives(archive_id,workspace_id,runtime_id,worker_id,session_id,checksum_sha256,content_bytes,policy_id,policy_revision,operation_id,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",params![a.archive_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,a.source_session_id,a.content_checksum_sha256,a.content_bytes,plan.policy_id,plan.policy_revision,operation_id,now,expires_at])?;
let observe_grants = {
let mut statement = tx.prepare(
"SELECT grant_id, controller_runtime_id, controller_worker_id,
permissions_json, created_at
FROM worker_control_grants
WHERE workspace_id = ?1 AND subject_runtime_id = ?2
AND subject_worker_id = ?3 AND revoked_at IS NULL",
)?;
statement
.query_map(
params![workspace_id, plan.worker.runtime_id, plan.worker.worker_id],
|row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
row.get::<_, String>(3)?,
row.get::<_, String>(4)?,
))
},
)?
.collect::<std::result::Result<Vec<_>, _>>()?
};
for (grant_id, controller_runtime_id, controller_worker_id, permissions, created_at) in observe_grants {
let permissions: Vec<String> = serde_json::from_str(&permissions)
.map_err(|error| StoreError::Store(format!("invalid Worker control grant permissions: {error}")))?;
if permissions.iter().any(|permission| permission == "observe") {
tx.execute(
"INSERT OR IGNORE INTO worker_session_archive_observe_grants
(workspace_id,archive_id,controller_runtime_id,controller_worker_id,
subject_runtime_id,subject_worker_id,source_grant_id,granted_at)
VALUES(?1,?2,?3,?4,?5,?6,?7,?8)",
params![
workspace_id,
a.archive_id,
controller_runtime_id,
controller_worker_id,
plan.worker.runtime_id,
plan.worker.worker_id,
grant_id,
created_at
],
)?;
}
}
}else if plan.session_disposition==SessionDisposition::Archive{return Err(StoreError::InvalidInput("archive manifest missing".into()));}
match plan.diagnostics_disposition {
DiagnosticsDisposition::Purge if result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime retained diagnostics for purge disposition".into())),
@@ -592,6 +652,78 @@ impl SqliteWorkspaceStore {
) -> crate::Result<Option<WorkerTombstone>> {
self.with_conn(|conn|conn.query_row("SELECT display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_revision,operation_id FROM worker_tombstones WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",params![workspace_id,worker.runtime_id,worker.worker_id],|r|Ok(WorkerTombstone{workspace_id:workspace_id.into(),worker:worker.clone(),display_name:r.get(0)?,profile:r.get(1)?,created_at:r.get(2)?,removed_at:r.get(3)?,archive_id:r.get(4)?,policy_id:r.get(5)?,policy_revision:r.get::<_,i64>(6)? as u64,operation_id:r.get(7)?})).optional().map_err(StoreError::from))
}
pub fn worker_session_archive(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
session_id: &str,
) -> crate::Result<Option<WorkerSessionArchiveRecord>> {
self.with_conn(|connection| {
connection
.query_row(
"SELECT archive_id, checksum_sha256, content_bytes, policy_id,
policy_revision, operation_id, committed_at, expires_at
FROM worker_session_archives
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3
AND session_id = ?4",
params![
workspace_id,
worker.runtime_id,
worker.worker_id,
session_id
],
|row| {
Ok(WorkerSessionArchiveRecord {
archive_id: row.get(0)?,
workspace_id: workspace_id.to_string(),
worker: worker.clone(),
session_id: session_id.to_string(),
checksum_sha256: row.get(1)?,
content_bytes: row.get::<_, i64>(2)? as u64,
policy_id: row.get(3)?,
policy_revision: row.get::<_, i64>(4)? as u64,
operation_id: row.get(5)?,
committed_at: row.get(6)?,
expires_at: row.get(7)?,
})
},
)
.optional()
.map_err(StoreError::from)
})
}
pub fn worker_session_archive_observable_by(
&self,
workspace_id: &str,
archive_id: &str,
controller: &RuntimeWorkerRef,
subject: &RuntimeWorkerRef,
) -> crate::Result<bool> {
self.with_conn(|connection| {
connection
.query_row(
"SELECT EXISTS(
SELECT 1 FROM worker_session_archive_observe_grants
WHERE workspace_id = ?1 AND archive_id = ?2
AND controller_runtime_id = ?3 AND controller_worker_id = ?4
AND subject_runtime_id = ?5 AND subject_worker_id = ?6
AND revoked_at IS NULL
)",
params![
workspace_id,
archive_id,
controller.runtime_id,
controller.worker_id,
subject.runtime_id,
subject.worker_id
],
|row| row.get(0),
)
.map_err(StoreError::from)
})
}
}
fn orphan_diagnostic(
@@ -1170,6 +1302,32 @@ mod tests {
#[test]
fn purge_tombstone_commit_is_idempotent() {
let s = setup();
let archived_grant_id = "archive-observe-grant";
let archived_controller =
RuntimeWorkerRef::new("r", WorkerId::from_legacy_u64(2).to_string());
s.with_conn(|conn| {
conn.execute(
"INSERT INTO worker_registry(\
workspace_id,worker_id,runtime_id,display_name,profile,retention_state,created_at,updated_at\
) VALUES('w',?1,'r','controller','builtin:companion','normal','created','controller-rev1')",
[&archived_controller.worker_id],
)?;
Ok(())
})
.unwrap();
s.create_worker_control_grant(&crate::store::WorkerControlGrantRecord {
workspace_id: "w".into(),
grant_id: archived_grant_id.into(),
controller: archived_controller.clone(),
subject: req().worker,
relation: "subject_session".into(),
origin: "test".into(),
permissions: vec!["observe".into()],
operation_id: archived_grant_id.into(),
created_at: "2026-10-02T00:00:00Z".into(),
revoked_at: None,
})
.unwrap();
s.with_conn(|conn| {
conn.execute("INSERT INTO typed_tickets(workspace_id,ticket_id,slug,title,status,kind,priority,body,workflow_state,workflow_state_explicit) VALUES('w','ticket-old','ticket-old','Old Ticket','open','task','normal','','planning',1)", [])?;
conn.execute("INSERT INTO worker_registry(workspace_id,worker_id,runtime_id,display_name,profile,retention_state,created_at,updated_at) VALUES('w','1','r','old worker','builtin:coder','normal','created','rev1')", [])?;
@@ -1220,6 +1378,31 @@ mod tests {
.state,
WorkerRemovalPlanState::Succeeded
);
assert!(
s.worker_session_archive_observable_by(
"w",
p.archive_id.as_deref().unwrap(),
&archived_controller,
&p.worker,
)
.unwrap(),
"the explicit observe grant survives Worker removal for the committed archive"
);
assert!(
s.revoke_worker_control_grant("w", archived_grant_id, "2026-10-02T00:01:00Z")
.unwrap(),
"the existing grant revocation authority must also revoke its archive projection"
);
assert!(
!s.worker_session_archive_observable_by(
"w",
p.archive_id.as_deref().unwrap(),
&archived_controller,
&p.worker,
)
.unwrap(),
"every later cursor page and direct Read must observe archive grant revocation"
);
assert!(s.worker_tombstone("w", &p.worker).unwrap().is_some());
let reservation_state: String = s.with_conn(|conn| {
conn.query_row(
@@ -33,26 +33,19 @@ pub enum RuntimeSubscriptionBrokerError {
#[derive(Clone, Debug)]
pub enum BrokerSubscriptionEvent {
Snapshot {
connection_generation: u64,
snapshot_revision: u64,
snapshot: SubscriptionSnapshot,
},
Event {
connection_generation: u64,
subject_revision: u64,
payload: SubscriptionEventPayload,
},
Disconnected {
connection_generation: u64,
message: String,
},
Rejected {
connection_generation: u64,
code: SubscriptionRejectionCode,
message: String,
},
Closed {
connection_generation: u64,
code: SubscriptionTerminationCode,
message: String,
},
@@ -60,7 +53,6 @@ pub enum BrokerSubscriptionEvent {
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct RuntimeSubscriptionBrokerStatus {
pub connection_generation: u64,
pub connected: bool,
pub desired_selectors: usize,
pub upstream_subscriptions: usize,
@@ -82,7 +74,30 @@ impl BrokerSubscription {
&self.selector
}
pub async fn recv(&mut self) -> Option<BrokerSubscriptionEvent> {
self.receiver.recv().await
loop {
if let Some(event) = self.receiver.recv().await {
return Some(event);
}
if self.restart().is_err() {
return None;
}
}
}
pub(crate) fn restart(&mut self) -> Result<(), RuntimeSubscriptionBrokerError> {
self.commands
.send(Command::Unsubscribe(self.downstream_id))
.map_err(|_| RuntimeSubscriptionBrokerError::Closed)?;
let (events, receiver) = mpsc::channel(DOWNSTREAM_QUEUE_CAPACITY);
self.commands
.send(Command::Subscribe {
downstream_id: self.downstream_id,
selector: self.selector.clone(),
events,
})
.map_err(|_| RuntimeSubscriptionBrokerError::Closed)?;
self.receiver = receiver;
Ok(())
}
}
@@ -120,10 +135,7 @@ impl RuntimeSubscriptionBroker {
pub fn register_remote_runtime(&self, config: RemoteRuntimeConfig) -> u64 {
let generation = self.next_generation.fetch_add(1, Ordering::Relaxed);
let (commands, receiver) = mpsc::unbounded_channel();
let status = Arc::new(RwLock::new(RuntimeSubscriptionBrokerStatus {
connection_generation: generation,
..Default::default()
}));
let status = Arc::new(RwLock::new(RuntimeSubscriptionBrokerStatus::default()));
let registration = Registration {
generation,
commands: commands.clone(),
@@ -156,7 +168,6 @@ impl RuntimeSubscriptionBroker {
let generation = self.next_generation.fetch_add(1, Ordering::Relaxed);
let (commands, receiver) = mpsc::unbounded_channel();
let status = Arc::new(RwLock::new(RuntimeSubscriptionBrokerStatus {
connection_generation: generation,
connected: true,
..Default::default()
}));
@@ -257,7 +268,6 @@ impl RuntimeSubscriptionBroker {
.clone();
if !initial_status.connected {
let _ = initial_events.try_send(BrokerSubscriptionEvent::Disconnected {
connection_generation: initial_status.connection_generation,
message: "Runtime subscription connection is not currently available".to_string(),
});
}
@@ -286,8 +296,7 @@ struct SelectorState {
downstreams: HashMap<u64, mpsc::Sender<BrokerSubscriptionEvent>>,
upstream_id: Option<SubscriptionId>,
pending: bool,
snapshot: Option<(u64, SubscriptionSnapshot)>,
revisions: HashMap<String, u64>,
snapshot: Option<SubscriptionSnapshot>,
}
impl SelectorState {
fn new() -> Self {
@@ -296,7 +305,6 @@ impl SelectorState {
upstream_id: None,
pending: false,
snapshot: None,
revisions: HashMap::new(),
}
}
}
@@ -334,11 +342,9 @@ impl State {
selector.upstream_id = None;
selector.pending = false;
selector.snapshot = None;
selector.revisions.clear();
broadcast(
&mut selector.downstreams,
BrokerSubscriptionEvent::Disconnected {
connection_generation: self.generation,
message: message.clone(),
},
);
@@ -347,12 +353,77 @@ impl State {
}
struct EmbeddedEntry {
lifetime: u64,
downstreams: HashMap<u64, mpsc::Sender<BrokerSubscriptionEvent>>,
snapshot_revision: u64,
snapshot: SubscriptionSnapshot,
task: tokio::task::JoinHandle<()>,
}
enum EmbeddedUpdate {
Event {
selector: EventSubscriptionSelector,
lifetime: u64,
payload: SubscriptionEventPayload,
},
Closed {
selector: EventSubscriptionSelector,
lifetime: u64,
code: SubscriptionTerminationCode,
message: String,
},
}
fn start_embedded_subscription(
runtime: &worker_runtime::Runtime,
runtime_id: &str,
selector: EventSubscriptionSelector,
lifetime: u64,
updates: mpsc::UnboundedSender<EmbeddedUpdate>,
) -> Result<(SubscriptionSnapshot, tokio::task::JoinHandle<()>), worker_runtime::error::RuntimeError>
{
let mut subscription = runtime.subscribe_event_selector(selector.clone())?;
let snapshot = project_snapshot_runtime(subscription.snapshot().clone(), runtime_id);
let task_runtime_id = runtime_id.to_string();
let task = tokio::spawn(async move {
loop {
match subscription.recv().await {
Ok(update) => {
let payload = project_payload_runtime(update.payload, &task_runtime_id);
if updates
.send(EmbeddedUpdate::Event {
selector: selector.clone(),
lifetime,
payload,
})
.is_err()
{
return;
}
}
Err(worker_runtime::RuntimeSubscriptionRecvError::Lagged) => {
let _ = updates.send(EmbeddedUpdate::Closed {
selector,
lifetime,
code: SubscriptionTerminationCode::Lagged,
message: "embedded Runtime subscription lagged; resubscribing from a fresh snapshot".to_string(),
});
return;
}
Err(worker_runtime::RuntimeSubscriptionRecvError::Closed) => {
let _ = updates.send(EmbeddedUpdate::Closed {
selector,
lifetime,
code: SubscriptionTerminationCode::ServerShutdown,
message: "embedded Runtime subscription closed".to_string(),
});
return;
}
}
}
});
Ok((snapshot, task))
}
async fn run_embedded_connection(
runtime_id: String,
runtime: worker_runtime::Runtime,
@@ -364,6 +435,7 @@ async fn run_embedded_connection(
let (updates, mut update_receiver) = mpsc::unbounded_channel();
let mut entries = HashMap::<EventSubscriptionSelector, EmbeddedEntry>::new();
let mut downstream_index = HashMap::<u64, EventSubscriptionSelector>::new();
let mut next_lifetime = 1_u64;
loop {
tokio::select! {
command = commands.recv() => match command {
@@ -371,30 +443,38 @@ async fn run_embedded_connection(
downstream_index.insert(downstream_id, selector.clone());
if let Some(entry) = entries.get_mut(&selector) {
let _ = events.try_send(BrokerSubscriptionEvent::Snapshot {
connection_generation: generation,
snapshot_revision: entry.snapshot_revision,
snapshot: entry.snapshot.clone(),
});
entry.downstreams.insert(downstream_id, events);
} else {
match runtime.subscribe_event_selector(selector.clone()) {
Ok(mut subscription) => {
let snapshot_revision = subscription.snapshot_revision();
let snapshot = project_snapshot_runtime(subscription.snapshot().clone(), &runtime_id);
let _ = events.try_send(BrokerSubscriptionEvent::Snapshot { connection_generation: generation, snapshot_revision, snapshot: snapshot.clone() });
let sender = updates.clone();
let task_selector = selector.clone();
let task_runtime_id = runtime_id.clone();
let task = tokio::spawn(async move {
while let Ok(update) = subscription.recv().await {
let payload = project_payload_runtime(update.payload, &task_runtime_id);
if sender.send((task_selector.clone(), update.subject_revision, payload)).is_err() { break; }
}
let lifetime = next_lifetime;
next_lifetime = next_lifetime.saturating_add(1);
match start_embedded_subscription(
&runtime,
&runtime_id,
selector.clone(),
lifetime,
updates.clone(),
) {
Ok((snapshot, task)) => {
let _ = events.try_send(BrokerSubscriptionEvent::Snapshot {
snapshot: snapshot.clone(),
});
entries.insert(selector, EmbeddedEntry { downstreams: HashMap::from([(downstream_id, events)]), snapshot_revision, snapshot, task });
entries.insert(
selector,
EmbeddedEntry {
lifetime,
downstreams: HashMap::from([(downstream_id, events)]),
snapshot,
task,
},
);
}
Err(error) => {
let _ = events.try_send(BrokerSubscriptionEvent::Rejected { connection_generation: generation, code: SubscriptionRejectionCode::UnsupportedSelector, message: error.to_string() });
let _ = events.try_send(BrokerSubscriptionEvent::Rejected {
code: SubscriptionRejectionCode::UnsupportedSelector,
message: error.to_string(),
});
}
}
}
@@ -407,7 +487,7 @@ async fn run_embedded_connection(
}
Some(Command::Shutdown(replacement)) => {
for entry in entries.values_mut() {
broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Closed { connection_generation: generation, code: SubscriptionTerminationCode::ServerShutdown, message: format!("embedded Runtime generation {generation} was fenced by {replacement}") });
broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Closed { code: SubscriptionTerminationCode::ServerShutdown, message: format!("embedded Runtime registration {generation} was replaced by {replacement}") });
entry.task.abort();
}
return;
@@ -415,16 +495,68 @@ async fn run_embedded_connection(
None => return,
},
update = update_receiver.recv() => {
let Some((selector, subject_revision, payload)) = update else { return; };
if let Some(entry) = entries.get_mut(&selector) {
entry.snapshot_revision = entry.snapshot_revision.saturating_add(1);
apply_event_to_cached_snapshot(&mut entry.snapshot, &payload);
broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Event { connection_generation: generation, subject_revision, payload });
let Some(update) = update else { return; };
match update {
EmbeddedUpdate::Event { selector, lifetime, payload } => {
if let Some(entry) = entries
.get_mut(&selector)
.filter(|entry| entry.lifetime == lifetime)
{
apply_event_to_cached_snapshot(&mut entry.snapshot, &payload);
broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Event { payload });
}
}
EmbeddedUpdate::Closed { selector, lifetime, code, message } => {
if !entries
.get(&selector)
.is_some_and(|entry| entry.lifetime == lifetime)
{
continue;
}
let Some(mut entry) = entries.remove(&selector) else { continue; };
broadcast(
&mut entry.downstreams,
BrokerSubscriptionEvent::Closed { code, message },
);
if entry.downstreams.is_empty() {
continue;
}
let new_lifetime = next_lifetime;
next_lifetime = next_lifetime.saturating_add(1);
match start_embedded_subscription(
&runtime,
&runtime_id,
selector.clone(),
new_lifetime,
updates.clone(),
) {
Ok((snapshot, task)) => {
broadcast(
&mut entry.downstreams,
BrokerSubscriptionEvent::Snapshot {
snapshot: snapshot.clone(),
},
);
entry.snapshot = snapshot;
entry.task = task;
entry.lifetime = new_lifetime;
entries.insert(selector, entry);
}
Err(error) => {
broadcast(
&mut entry.downstreams,
BrokerSubscriptionEvent::Rejected {
code: SubscriptionRejectionCode::Internal,
message: error.to_string(),
},
);
}
}
}
}
}
}
*status.write().expect("broker status poisoned") = RuntimeSubscriptionBrokerStatus {
connection_generation: generation,
connected: true,
desired_selectors: entries.len(),
upstream_subscriptions: entries.len(),
@@ -597,10 +729,8 @@ async fn apply_online(
.selectors
.entry(selector.clone())
.or_insert_with(SelectorState::new);
if let Some((revision, snapshot)) = &entry.snapshot {
if let Some(snapshot) = &entry.snapshot {
let _ = events.try_send(BrokerSubscriptionEvent::Snapshot {
connection_generation: state.generation,
snapshot_revision: *revision,
snapshot: snapshot.clone(),
});
}
@@ -696,7 +826,6 @@ async fn handle_frame(
request_id,
subscription_id,
selector,
snapshot_revision,
snapshot,
}) => {
if state.pending.remove(&request_id) != Some(selector.clone()) {
@@ -706,18 +835,13 @@ async fn handle_frame(
let entry = state.selectors.get_mut(&selector).ok_or(())?;
entry.pending = false;
entry.upstream_id = Some(subscription_id.clone());
entry.snapshot = Some((snapshot_revision, snapshot.clone()));
entry.revisions = snapshot_revisions(&snapshot);
entry.snapshot = Some(snapshot.clone());
state
.upstream_index
.insert(subscription_id, selector.clone());
broadcast(
&mut entry.downstreams,
BrokerSubscriptionEvent::Snapshot {
connection_generation: state.generation,
snapshot_revision,
snapshot,
},
BrokerSubscriptionEvent::Snapshot { snapshot },
);
if entry.downstreams.is_empty() {
maybe_unsubscribe(socket, state, selector).await?;
@@ -734,18 +858,13 @@ async fn handle_frame(
if let Some(mut entry) = state.selectors.remove(&selector) {
broadcast(
&mut entry.downstreams,
BrokerSubscriptionEvent::Rejected {
connection_generation: state.generation,
code,
message,
},
BrokerSubscriptionEvent::Rejected { code, message },
);
}
}
}
SubscriptionFramePayload::Event(SubscriptionEvent::Event {
subscription_id,
subject_revision,
payload,
}) => {
let selector = state
@@ -756,24 +875,12 @@ async fn handle_frame(
payload.validate_for_selector(&selector).map_err(|_| ())?;
let payload = project_payload_runtime(payload, &state.runtime_id);
let entry = state.selectors.get_mut(&selector).ok_or(())?;
if let Some(subject) = event_subject(&payload) {
let revision = entry.revisions.entry(subject).or_insert(0);
if subject_revision <= *revision {
return Ok(());
}
*revision = subject_revision;
}
if let Some((snapshot_revision, snapshot)) = entry.snapshot.as_mut() {
*snapshot_revision = snapshot_revision.saturating_add(1);
if let Some(snapshot) = entry.snapshot.as_mut() {
apply_event_to_cached_snapshot(snapshot, &payload);
}
broadcast(
&mut entry.downstreams,
BrokerSubscriptionEvent::Event {
connection_generation: state.generation,
subject_revision,
payload,
},
BrokerSubscriptionEvent::Event { payload },
);
}
SubscriptionFramePayload::Event(SubscriptionEvent::SubscriptionClosed {
@@ -785,14 +892,9 @@ async fn handle_frame(
let should_resubscribe = if let Some(entry) = state.selectors.get_mut(&selector) {
entry.upstream_id = None;
entry.snapshot = None;
entry.revisions.clear();
broadcast(
&mut entry.downstreams,
BrokerSubscriptionEvent::Closed {
connection_generation: state.generation,
code,
message,
},
BrokerSubscriptionEvent::Closed { code, message },
);
!entry.downstreams.is_empty()
} else {
@@ -822,10 +924,9 @@ fn close_all(state: &mut State, replacement: u64) {
broadcast(
&mut entry.downstreams,
BrokerSubscriptionEvent::Closed {
connection_generation: state.generation,
code: SubscriptionTerminationCode::ServerShutdown,
message: format!(
"Runtime connection generation {} was fenced by generation {replacement}",
"Runtime registration {} was replaced by {replacement}",
state.generation
),
},
@@ -846,51 +947,6 @@ fn broadcast(
downstreams.retain(|id, _| !closed.contains(id));
}
fn snapshot_revisions(snapshot: &SubscriptionSnapshot) -> HashMap<String, u64> {
match snapshot {
SubscriptionSnapshot::Workers { workers } => workers
.iter()
.map(|worker| {
(
format!(
"{}:{}",
worker.runtime_id.as_deref().unwrap_or_default(),
worker.worker_id
),
worker.subject_revision,
)
})
.collect(),
SubscriptionSnapshot::WorkerProtocol { worker_id, .. } => {
HashMap::from([(worker_id.to_string(), 0)])
}
SubscriptionSnapshot::WorkspaceWorkdirs { .. } => HashMap::new(),
}
}
fn event_subject(payload: &SubscriptionEventPayload) -> Option<String> {
Some(match payload {
SubscriptionEventPayload::WorkerUpserted { worker } => format!(
"{}:{}",
worker.runtime_id.as_deref().unwrap_or_default(),
worker.worker_id
),
SubscriptionEventPayload::WorkerRemoved {
worker_id,
runtime_id,
} => format!(
"{}:{}",
runtime_id.as_deref().unwrap_or_default(),
worker_id
),
SubscriptionEventPayload::WorkerProtocol { worker_id, .. } => worker_id.to_string(),
SubscriptionEventPayload::WorkdirUpserted { workdir } => {
workdir.working_directory_id.to_string()
}
SubscriptionEventPayload::WorkdirRemoved {
working_directory_id,
} => working_directory_id.to_string(),
})
}
async fn send_frame(socket: &mut RuntimeSocket, frame: SubscriptionFrame) -> Result<(), ()> {
frame.validate().map_err(|_| ())?;
socket
@@ -979,7 +1035,6 @@ fn runtime_token(
}
fn update_status(status: &RwLock<RuntimeSubscriptionBrokerStatus>, state: &State, connected: bool) {
*status.write().expect("broker status poisoned") = RuntimeSubscriptionBrokerStatus {
connection_generation: state.generation,
connected,
desired_selectors: state
.selectors
@@ -74,6 +74,7 @@ fn create_request(name: &str) -> CreateWorkerRequest {
settings_revision: 1,
language: "English".to_string(),
}),
subjektiv_attached: false,
}
}
@@ -264,16 +265,9 @@ async fn replacing_runtime_registration_fences_the_old_generation() {
assert!(second_generation > first_generation);
assert!(matches!(
next_event(&mut old).await,
BrokerSubscriptionEvent::Closed {
connection_generation,
..
} if connection_generation == first_generation
BrokerSubscriptionEvent::Closed { .. }
));
let status = wait_for_status(&broker, "runtime-test", |status| {
status.connection_generation == second_generation && status.connected
})
.await;
assert_eq!(status.connection_generation, second_generation);
wait_for_status(&broker, "runtime-test", |status| status.connected).await;
server.abort();
}
@@ -430,6 +424,75 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
server.abort();
}
#[tokio::test]
async fn embedded_downstream_overflow_recovers_through_a_fresh_snapshot() {
let (runtime, _config, server) = fixture().await;
let worker = runtime.list_workers().unwrap().remove(0);
let broker = RuntimeSubscriptionBroker::new("local");
broker.register_embedded_runtime("embedded-worker-runtime", runtime.clone());
let mut subscription = broker
.subscribe(
"embedded-worker-runtime",
EventSubscriptionSelector::RuntimeWorkers,
)
.unwrap();
assert!(matches!(
next_snapshot(&mut subscription).await,
BrokerSubscriptionEvent::Snapshot { .. }
));
let final_command_id = (DOWNSTREAM_QUEUE_CAPACITY as u64) * 3;
for command_id in 1..=final_command_id {
let mut snapshot = worker.worker_state.clone().unwrap();
snapshot.last_command_id = command_id;
runtime
.observe_worker_event(
&worker.worker_ref,
protocol::Event::WorkerState { snapshot },
)
.unwrap();
}
let mut saw_recovery_snapshot = false;
let mut converged = false;
tokio::time::timeout(Duration::from_secs(5), async {
while !converged {
match subscription
.recv()
.await
.expect("broker subscription closed")
{
BrokerSubscriptionEvent::Snapshot {
snapshot: SubscriptionSnapshot::Workers { workers },
} => {
saw_recovery_snapshot = true;
converged = workers.iter().any(|candidate| {
candidate.worker_id.as_str() == worker.worker_id.to_string()
&& candidate
.worker_state
.as_ref()
.is_some_and(|state| state.last_command_id == final_command_id)
});
}
BrokerSubscriptionEvent::Event {
payload: SubscriptionEventPayload::WorkerUpserted { worker: candidate },
} if saw_recovery_snapshot => {
converged = candidate.worker_id.as_str() == worker.worker_id.to_string()
&& candidate
.worker_state
.as_ref()
.is_some_and(|state| state.last_command_id == final_command_id);
}
_ => {}
}
}
})
.await
.expect("overflow recovery did not converge");
assert!(saw_recovery_snapshot);
server.abort();
}
#[tokio::test]
async fn strict_runtime_subscription_rejects_private_endpoint_before_websocket_connect() {
let config = RemoteRuntimeConfig::new(
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+318 -110
View File
@@ -11,7 +11,10 @@ use tokio::task::JoinHandle;
use crate::runtime_subscription::{
BrokerSubscriptionEvent, RuntimeSubscriptionBroker, RuntimeSubscriptionBrokerError,
};
use crate::store::{ControlPlaneStore, WorkerCatalogChange, WorkerRegistryProjectionCommit};
use crate::store::{
ControlPlaneStore, WorkerCatalogChange, WorkerRegistryProjectionCommit,
WorkerRegistryProjectionRecord,
};
use worker_runtime::identity::RuntimeWorkerRef;
const PROJECTION_EVENT_CAPACITY: usize = 256;
@@ -20,21 +23,42 @@ pub type WorkerProjectionChange = WorkerCatalogChange;
#[derive(Debug, Clone)]
pub struct WorkerProjectionEvent {
pub revision: u64,
pub changes: Vec<WorkerProjectionChange>,
}
/// An authoritative durable snapshot followed by changes committed after that snapshot.
pub struct WorkerProjectionSubscription {
snapshot: Vec<WorkerRegistryProjectionRecord>,
receiver: broadcast::Receiver<WorkerProjectionEvent>,
}
impl WorkerProjectionSubscription {
pub fn take_snapshot(&mut self) -> Vec<WorkerRegistryProjectionRecord> {
std::mem::take(&mut self.snapshot)
}
pub async fn recv(&mut self) -> Result<WorkerProjectionEvent, broadcast::error::RecvError> {
self.receiver.recv().await
}
#[cfg(test)]
pub fn try_recv(&mut self) -> Result<WorkerProjectionEvent, broadcast::error::TryRecvError> {
self.receiver.try_recv()
}
}
/// Owns the durable Runtime-to-Workspace Worker projection and its independent
/// low-frequency publication stream.
///
/// One Runtime subscription is kept alive per registered Runtime even when no
/// browser is connected. Browser subscriptions therefore never become the
/// authority for populating `worker_registry` observations.
/// The ordering gate covers both durable commits and subscription snapshot creation.
/// Every subscription therefore starts with one authoritative snapshot and receives
/// only commits that completed after it, in the same order in which they were stored.
pub struct WorkerProjectionService {
workspace_id: String,
store: Arc<dyn ControlPlaneStore>,
events: broadcast::Sender<WorkerProjectionEvent>,
connection_generation_epoch: u64,
ordering: Mutex<()>,
attach_gate: tokio::sync::Mutex<()>,
tasks: Mutex<HashMap<String, JoinHandle<()>>>,
}
@@ -45,8 +69,8 @@ impl WorkerProjectionService {
workspace_id: workspace_id.into(),
store,
events,
connection_generation_epoch: Utc::now().timestamp_nanos_opt().unwrap_or_default().max(0)
as u64,
ordering: Mutex::new(()),
attach_gate: tokio::sync::Mutex::new(()),
tasks: Mutex::new(HashMap::new()),
})
}
@@ -62,8 +86,38 @@ impl WorkerProjectionService {
}
}
pub fn subscribe(&self) -> broadcast::Receiver<WorkerProjectionEvent> {
self.events.subscribe()
pub fn subscribe_ordered(&self, limit: usize) -> crate::Result<WorkerProjectionSubscription> {
let _ordering = self
.ordering
.lock()
.expect("Worker projection ordering lock poisoned");
let receiver = self.events.subscribe();
let snapshot_limit = limit.saturating_add(1);
let mut snapshot = self
.store
.worker_registry_projection_snapshot(self.workspace_id.as_str(), snapshot_limit)?;
if snapshot.len() > limit {
return Err(crate::Error::Store(format!(
"authoritative Worker projection exceeds the configured snapshot limit of {limit} records"
)));
}
snapshot.shrink_to_fit();
Ok(WorkerProjectionSubscription { snapshot, receiver })
}
/// Serialize one durable projection mutation with its publication. The closure
/// must return the exact projection payload captured by the same transaction.
pub(crate) fn publish_ordered<T, E>(
&self,
mutation: impl FnOnce() -> std::result::Result<(T, WorkerRegistryProjectionCommit), E>,
) -> std::result::Result<T, E> {
let _ordering = self
.ordering
.lock()
.expect("Worker projection ordering lock poisoned");
let (result, commit) = mutation()?;
self.publish_commit(commit);
Ok(result)
}
pub async fn attach_runtime(
@@ -71,6 +125,20 @@ impl WorkerProjectionService {
broker: &RuntimeSubscriptionBroker,
runtime_id: &str,
) -> Result<(), RuntimeSubscriptionBrokerError> {
// Serialize replacement and wait for the old consumer to stop before the new
// lifetime can apply its snapshot. Late notifications can therefore never
// cross a Runtime ownership boundary.
let _attach = self.attach_gate.lock().await;
let previous = self
.tasks
.lock()
.expect("Worker projection task lock poisoned")
.remove(runtime_id);
if let Some(previous) = previous {
previous.abort();
let _ = previous.await;
}
let mut subscription =
broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?;
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
@@ -80,12 +148,14 @@ impl WorkerProjectionService {
break;
};
let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. });
if let Err(error) = self.apply_broker_event(runtime_id, event).await {
if let Err(error) = self.apply_broker_event(runtime_id, event) {
tracing::warn!(
runtime_id,
error = %error,
"failed to initialize durable Worker projection"
"failed to initialize durable Worker projection; restarting from a fresh snapshot"
);
subscription.restart()?;
continue;
}
if initialized {
break;
@@ -99,26 +169,22 @@ impl WorkerProjectionService {
let Some(service) = service.upgrade() else {
return;
};
if let Err(error) = service
.apply_broker_event(task_runtime_id.as_str(), event)
.await
{
if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) {
tracing::warn!(
runtime_id = %task_runtime_id,
error = %error,
"failed to update durable Worker projection"
"failed to update durable Worker projection; restarting from a fresh snapshot"
);
if subscription.restart().is_err() {
return;
}
}
}
});
if let Some(previous) = self
.tasks
self.tasks
.lock()
.expect("Worker projection task lock poisoned")
.insert(runtime_id, task)
{
previous.abort();
}
.insert(runtime_id, task);
Ok(())
}
@@ -128,110 +194,96 @@ impl WorkerProjectionService {
worker: &protocol::subscription::SubscriptionWorker,
observed_at: &str,
) -> crate::Result<()> {
let commit = self.store.apply_worker_registry_observation(
self.workspace_id.as_str(),
runtime_id,
0,
worker,
observed_at,
)?;
self.publish_commit(commit)
self.publish_ordered(|| {
self.store
.apply_worker_registry_observation(
self.workspace_id.as_str(),
runtime_id,
worker,
observed_at,
)
.map(|commit| ((), commit))
})
}
pub fn refresh(&self, worker: &RuntimeWorkerRef) -> crate::Result<()> {
let commit = self
.store
.commit_worker_registry_projection_refresh(self.workspace_id.as_str(), worker)?;
self.publish_commit(commit)
self.publish_ordered(|| {
self.store
.commit_worker_registry_projection_refresh(self.workspace_id.as_str(), worker)
.map(|commit| ((), commit))
})
}
async fn apply_broker_event(
fn apply_broker_event(
&self,
runtime_id: &str,
event: BrokerSubscriptionEvent,
) -> crate::Result<()> {
let observed_at = Utc::now().to_rfc3339();
let projection_generation = |connection_generation: u64| {
self.connection_generation_epoch
.saturating_add(connection_generation)
};
let commit = match event {
BrokerSubscriptionEvent::Snapshot {
connection_generation,
snapshot_revision,
snapshot: SubscriptionSnapshot::Workers { workers },
} => self.store.reconcile_worker_registry_snapshot(
self.workspace_id.as_str(),
runtime_id,
projection_generation(connection_generation),
snapshot_revision,
workers.as_slice(),
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Snapshot { .. } => return Ok(()),
BrokerSubscriptionEvent::Event {
connection_generation,
payload: SubscriptionEventPayload::WorkerUpserted { worker },
..
} => self.store.apply_worker_registry_observation(
self.workspace_id.as_str(),
runtime_id,
projection_generation(connection_generation),
&worker,
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Event {
connection_generation,
subject_revision,
payload: SubscriptionEventPayload::WorkerRemoved { worker_id, .. },
} => self.store.mark_worker_registry_observation_unavailable(
self.workspace_id.as_str(),
runtime_id,
projection_generation(connection_generation),
Some(worker_id.as_str()),
Some(subject_revision),
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Event { .. } => return Ok(()),
BrokerSubscriptionEvent::Disconnected {
connection_generation,
..
}
| BrokerSubscriptionEvent::Rejected {
connection_generation,
..
}
| BrokerSubscriptionEvent::Closed {
connection_generation,
..
} => self.store.mark_worker_registry_observation_unavailable(
self.workspace_id.as_str(),
runtime_id,
projection_generation(connection_generation),
None,
None,
observed_at.as_str(),
)?,
};
self.publish_commit(commit)
self.publish_ordered(|| {
let commit = match event {
BrokerSubscriptionEvent::Snapshot {
snapshot: SubscriptionSnapshot::Workers { workers },
} => self.store.reconcile_worker_registry_snapshot(
self.workspace_id.as_str(),
runtime_id,
workers.as_slice(),
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Snapshot { .. } => {
return Ok((
(),
WorkerRegistryProjectionCommit {
changes: Vec::new(),
},
));
}
BrokerSubscriptionEvent::Event {
payload: SubscriptionEventPayload::WorkerUpserted { worker },
} => self.store.apply_worker_registry_observation(
self.workspace_id.as_str(),
runtime_id,
&worker,
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Event {
payload: SubscriptionEventPayload::WorkerRemoved { worker_id, .. },
} => self.store.mark_worker_registry_observation_unavailable(
self.workspace_id.as_str(),
runtime_id,
Some(worker_id.as_str()),
observed_at.as_str(),
)?,
BrokerSubscriptionEvent::Event { .. } => {
return Ok((
(),
WorkerRegistryProjectionCommit {
changes: Vec::new(),
},
));
}
BrokerSubscriptionEvent::Disconnected { .. }
| BrokerSubscriptionEvent::Rejected { .. }
| BrokerSubscriptionEvent::Closed { .. } => {
self.store.mark_worker_registry_observation_unavailable(
self.workspace_id.as_str(),
runtime_id,
None,
observed_at.as_str(),
)?
}
};
Ok(((), commit))
})
}
/// Broadcast an already committed catalog payload. Persistence always happens
/// before this call; a process crash therefore closes subscribers, whose next
/// subscription starts from the durable snapshot. Retrying a removal returns
/// the same durable change and can heal an in-process missed broadcast.
pub(crate) fn publish_commit(
&self,
commit: WorkerRegistryProjectionCommit,
) -> crate::Result<()> {
fn publish_commit(&self, commit: WorkerRegistryProjectionCommit) {
if commit.changes.is_empty() {
return Ok(());
return;
}
let _ = self.events.send(WorkerProjectionEvent {
revision: commit.revision,
changes: commit.changes,
});
Ok(())
}
}
@@ -240,3 +292,159 @@ impl Drop for WorkerProjectionService {
self.shutdown();
}
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Barrier, Mutex};
use crate::store::{
AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerCatalogChange,
WorkerRegistryRecord, WorkspaceRecord,
};
use worker_runtime::identity::RuntimeWorkerRef;
use super::WorkerProjectionService;
const WORKSPACE_ID: &str = "projection-ordering-test";
async fn fixture() -> (
Arc<SqliteWorkspaceStore>,
Arc<WorkerProjectionService>,
tempfile::TempDir,
) {
let temp = tempfile::tempdir().unwrap();
let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap());
store
.upsert_account(&AccountRecord {
account_id: "owner".to_string(),
kind: "user".to_string(),
handle: "owner".to_string(),
display_name: "Owner".to_string(),
created_at: "1".to_string(),
updated_at: "1".to_string(),
})
.unwrap();
store
.upsert_workspace(&WorkspaceRecord {
workspace_id: WORKSPACE_ID.to_string(),
owner_account_id: "owner".to_string(),
display_name: "Projection ordering".to_string(),
state: "active".to_string(),
created_at: "1".to_string(),
updated_at: "1".to_string(),
})
.await
.unwrap();
let service = WorkerProjectionService::new(WORKSPACE_ID, store.clone());
(store, service, temp)
}
fn worker(runtime_id: &str, worker_id: &str) -> WorkerRegistryRecord {
WorkerRegistryRecord {
workspace_id: WORKSPACE_ID.to_string(),
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
display_name: format!("{runtime_id}-{worker_id}"),
profile: Some("builtin:coder".to_string()),
retention_state: "normal".to_string(),
transcript_ref: None,
session_ref: None,
summary_ref: None,
diagnostics_ref: None,
created_at: "1".to_string(),
updated_at: "1".to_string(),
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn snapshot_registration_is_atomic_with_concurrent_publication() {
let (store, service, _temp) = fixture().await;
let barrier = Arc::new(Barrier::new(2));
let subscriber_service = service.clone();
let subscriber_barrier = barrier.clone();
let subscriber = std::thread::spawn(move || {
subscriber_barrier.wait();
subscriber_service.subscribe_ordered(10).unwrap()
});
let publisher_service = service.clone();
let publisher_barrier = barrier.clone();
let publisher_store = store.clone();
let record = worker("runtime-a", "worker-a");
let publisher = std::thread::spawn(move || {
publisher_barrier.wait();
publisher_service
.publish_ordered(|| {
publisher_store
.upsert_worker_registry(&record)
.map(|commit| ((), commit))
})
.unwrap();
});
let mut subscription = subscriber.join().unwrap();
publisher.join().unwrap();
let in_snapshot = subscription
.take_snapshot()
.iter()
.any(|record| record.registry.worker.worker_id == "worker-a");
let in_event = match subscription.try_recv() {
Ok(event) => matches!(
event.changes.as_slice(),
[WorkerCatalogChange::Upsert(record)]
if record.registry.worker.worker_id == "worker-a"
),
Err(tokio::sync::broadcast::error::TryRecvError::Empty) => false,
Err(error) => panic!("unexpected projection receive error: {error}"),
};
assert_ne!(
in_snapshot, in_event,
"the commit must cross the boundary exactly once"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn concurrent_commits_are_published_in_durable_execution_order() {
let (store, service, _temp) = fixture().await;
let mut subscription = service.subscribe_ordered(10).unwrap();
assert!(subscription.take_snapshot().is_empty());
let barrier = Arc::new(Barrier::new(3));
let durable_order = Arc::new(Mutex::new(Vec::new()));
let mut publishers = Vec::new();
for worker_id in ["worker-a", "worker-b"] {
let service = service.clone();
let store = store.clone();
let barrier = barrier.clone();
let durable_order = durable_order.clone();
let record = worker("runtime-a", worker_id);
publishers.push(std::thread::spawn(move || {
barrier.wait();
service
.publish_ordered(|| {
durable_order
.lock()
.unwrap()
.push(record.worker.worker_id.clone());
store
.upsert_worker_registry(&record)
.map(|commit| ((), commit))
})
.unwrap();
}));
}
barrier.wait();
for publisher in publishers {
publisher.join().unwrap();
}
let published_order = [
subscription.try_recv().unwrap(),
subscription.try_recv().unwrap(),
]
.into_iter()
.map(|event| match event.changes.as_slice() {
[WorkerCatalogChange::Upsert(record)] => record.registry.worker.worker_id.clone(),
changes => panic!("unexpected projection changes: {changes:?}"),
})
.collect::<Vec<_>>();
assert_eq!(published_order, *durable_order.lock().unwrap());
}
}
@@ -296,7 +296,7 @@ mod tests {
display_name: "Workspace A".to_string(),
repository: InitialRepositoryIntent {
uri: repository.path().display().to_string(),
repository_key: "main".to_string(),
repository_key: "platform".to_string(),
default_ref: None,
},
};
@@ -322,13 +322,11 @@ mod tests {
assert_eq!(signing_identity.algorithm, "ed25519");
assert!(signing_identity.public_key.is_some());
assert!(signing_identity.public_key_fingerprint.is_some());
assert_eq!(
store
.list_repositories(&created.workspace.workspace_id)
.unwrap()
.len(),
1
);
let repositories = store
.list_repositories(&created.workspace.workspace_id)
.unwrap();
assert_eq!(repositories.len(), 1);
assert_eq!(repositories[0].repository_key, "platform");
assert!(
store
.load_workspace_config(&created.workspace.workspace_id)
@@ -337,6 +335,42 @@ mod tests {
);
}
#[test]
fn create_rejects_noncanonical_repository_keys_without_persisting_a_workspace() {
let store = Arc::new(SqliteWorkspaceStore::in_memory().unwrap());
let service = WorkspaceCatalogService::new(
store.clone(),
Arc::new(
crate::workspace_signing_identity::InMemoryWorkspaceSigningMaterialStore::default(),
),
);
let repository = git_repository();
let owner_account_id = owner_account(store.as_ref());
for (operation_key, repository_key) in [
("blank-repository-key", " "),
("spaced-repository-key", " platform "),
] {
let request = WorkspaceCreateRequest {
operation_key: operation_key.to_string(),
display_name: "Workspace A".to_string(),
repository: InitialRepositoryIntent {
uri: repository.path().display().to_string(),
repository_key: repository_key.to_string(),
default_ref: None,
},
};
let error = service
.create(request, owner_account_id.clone())
.unwrap_err()
.to_string();
assert!(error.contains("invalid Repository key"), "{error}");
assert!(store.list_workspaces().unwrap().is_empty());
}
}
#[tokio::test]
async fn create_recovers_same_reserved_identity_after_material_write_failure() {
let temp = tempfile::tempdir().unwrap();
@@ -73,12 +73,11 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"worker_orphan_diagnostics",
"worker_registry",
"worker_registry_observations",
"worker_registry_projection_cursors",
"worker_registry_projection_diagnostics",
"worker_registry_projection_removals",
"worker_registry_projection_revisions",
"worker_removal_operations",
"worker_retention_audit_events",
"worker_session_archive_observe_grants",
"worker_session_archives",
"worker_singleton_owners",
"worker_tombstones",
@@ -271,7 +271,6 @@ async fn run_worker_protocol(
worker_id: worker_id.clone(),
runtime_id: Some(runtime_id),
},
snapshot_revision: 0,
snapshot: SubscriptionSnapshot::WorkerProtocol {
worker_id: worker_id.clone(),
events: vec![initial_event],
@@ -284,13 +283,10 @@ async fn run_worker_protocol(
{
return;
}
let mut subject_revision = 0_u64;
while let Some(event) = events.recv().await {
subject_revision = subject_revision.saturating_add(1);
let frame =
SubscriptionFrame::new(SubscriptionFramePayload::Event(SubscriptionEvent::Event {
subscription_id: subscription_id.clone(),
subject_revision,
payload: SubscriptionEventPayload::WorkerProtocol {
worker_id: worker_id.clone(),
event,
@@ -333,12 +329,10 @@ async fn run_workspace_workers(
subscription_id: SubscriptionId,
outbound: mpsc::Sender<WsMessage>,
) {
// Subscribe before reading so commits racing with the snapshot are replayed.
let mut events = api.worker_projection.subscribe();
let Ok((snapshot_revision, records)) = api.store.worker_registry_projection_snapshot(
&api.config.workspace_id,
api.config.max_records.max(1),
) else {
let Ok(mut projection) = api
.worker_projection
.subscribe_ordered(api.config.max_records.max(1))
else {
let _ = send_rejected(
&outbound,
request_id,
@@ -348,7 +342,8 @@ async fn run_workspace_workers(
.await;
return;
};
let mut workers = records
let mut workers = projection
.take_snapshot()
.into_iter()
.filter_map(|record| project_registry_worker(&api, record))
.collect::<Vec<_>>();
@@ -360,7 +355,6 @@ async fn run_workspace_workers(
request_id,
subscription_id: subscription_id.clone(),
selector: EventSubscriptionSelector::WorkspaceWorkers,
snapshot_revision,
snapshot: SubscriptionSnapshot::Workers { workers },
},
)),
@@ -372,16 +366,14 @@ async fn run_workspace_workers(
}
loop {
match events.recv().await {
Ok(event) if event.revision <= snapshot_revision => continue,
match projection.recv().await {
Ok(event) => {
for change in event.changes {
let payload = match change {
crate::worker_projection::WorkerProjectionChange::Upsert(record) => {
let Some(mut worker) = project_registry_worker(&api, record) else {
let Some(worker) = project_registry_worker(&api, record) else {
continue;
};
worker.subject_revision = event.revision;
SubscriptionEventPayload::WorkerUpserted { worker }
}
crate::worker_projection::WorkerProjectionChange::Removed(worker) => {
@@ -396,7 +388,7 @@ async fn run_workspace_workers(
}
}
};
if send_event(&outbound, &subscription_id, event.revision, payload)
if send_event(&outbound, &subscription_id, payload)
.await
.is_err()
{
@@ -431,7 +423,6 @@ fn project_registry_worker(
let mut worker = if let Some(observation) = record.observation {
let mut worker = observation.worker;
worker.availability = observation.availability;
worker.subject_revision = observation.projection_revision;
worker
} else {
SubscriptionWorker {
@@ -442,7 +433,6 @@ fn project_registry_worker(
runtime_id: Some(runtime_id.clone()),
resource_key: record.resource_key.clone(),
availability: protocol::subscription::SubscriptionWorkerAvailability::Unavailable,
subject_revision: 0,
worker_state: None,
state: protocol::subscription::SubscriptionWorkerState::Stopped,
has_running_internal_workers: false,
@@ -559,14 +549,12 @@ fn sort_workers(workers: &mut [SubscriptionWorker]) {
async fn send_event(
outbound: &mpsc::Sender<WsMessage>,
subscription_id: &SubscriptionId,
subject_revision: u64,
payload: SubscriptionEventPayload,
) -> Result<(), ()> {
send_frame(
outbound,
SubscriptionFrame::new(SubscriptionFramePayload::Event(SubscriptionEvent::Event {
subscription_id: subscription_id.clone(),
subject_revision,
payload,
})),
)
@@ -617,7 +605,6 @@ mod tests {
runtime_id: Some("runtime-1".to_string()),
resource_key: Some("W-1".to_string()),
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
subject_revision: 1,
worker_state: None,
state: protocol::subscription::SubscriptionWorkerState::Idle,
has_running_internal_workers: false,