runtime: prove Worker mutation source authority

This commit is contained in:
2026-08-12 04:03:09 +09:00
parent 86be3a6865
commit 8cc0aaf8d2
21 changed files with 2037 additions and 230 deletions
+1 -2
View File
@@ -55,7 +55,7 @@ use worker_runtime::interaction::{
use worker_runtime::management::{RuntimeOptions as EmbeddedRuntimeOptions, RuntimeStatus};
use worker_runtime::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveInput};
const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime";
pub(crate) const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime";
const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host";
const REMOTE_HOST_KIND: &str = "remote-worker-runtime-host";
const MAX_DIAGNOSTICS: usize = 16;
@@ -4039,7 +4039,6 @@ mod tests {
WorkspaceApiRef {
workspace_id: "workspace-test".to_string(),
base_url: "http://127.0.0.1:8787".to_string(),
runtime_id: Some("runtime-test".to_string()),
}
}
+1
View File
@@ -24,6 +24,7 @@ pub mod runtime_subscription;
pub mod server;
pub mod skills;
pub mod store;
pub mod worker_source;
mod workspace_subscription;
pub use authority::{
+1 -1
View File
@@ -1354,7 +1354,7 @@ mod tests {
let path = temp.path().join("server.db");
{
let s = SqliteWorkspaceStore::open(&path).unwrap();
s.with_conn(|c|{c.execute("INSERT INTO workspaces(workspace_id,display_name,state,created_at,updated_at)VALUES('legacy','Legacy','active','old','old')",[])?;c.execute_batch("DROP TRIGGER seed_worker_retention_policy_after_workspace_insert;DROP TABLE worker_retention_audit_events;DROP TABLE worker_tombstones;DROP TABLE worker_session_archives;DROP TABLE worker_diagnostics_archives;DROP TABLE worker_orphan_diagnostics;DROP TABLE worker_removal_operations;DROP TABLE workspace_worker_retention_policies;DROP TABLE workspace_worker_retention_policy_revisions;DELETE FROM __yoi_schema_migrations WHERE version=28;")?;Ok(())}).unwrap();
s.with_conn(|c|{c.execute("INSERT INTO workspaces(workspace_id,display_name,state,created_at,updated_at)VALUES('legacy','Legacy','active','old','old')",[])?;c.execute_batch("DROP TRIGGER seed_worker_retention_policy_after_workspace_insert;DROP TABLE worker_retention_audit_events;DROP TABLE worker_tombstones;DROP TABLE worker_session_archives;DROP TABLE worker_diagnostics_archives;DROP TABLE worker_orphan_diagnostics;DROP TABLE worker_removal_operations;DROP TABLE workspace_worker_retention_policies;DROP TABLE workspace_worker_retention_policy_revisions;DELETE FROM __yoi_schema_migrations WHERE version>=28;")?;Ok(())}).unwrap();
}
let reopened = SqliteWorkspaceStore::open(&path).unwrap();
let p = reopened.worker_retention_policy("legacy").unwrap().unwrap();
+453 -17
View File
@@ -62,15 +62,15 @@ use crate::companion::{
};
use crate::config::{BackendRuntimesConfigFile, RemoteRuntimeConfigFile, resolve_remote_runtime};
use crate::hosts::{
ConfigBundleCheckResult, ConfigBundleSyncResult, DiagnosticSeverity, EmbeddedWorkerRuntime,
HostSummary, RemoteRuntimeConfig, RemoteWorkerRuntime, RuntimeDiagnostic, RuntimeRegistry,
RuntimeRegistryError, RuntimeRegistryUnregisterResult, RuntimeSummary, TicketWorkerRole,
WorkerCapabilitySummary, WorkerCompletionsRequest, WorkerCompletionsResult,
WorkerImplementationSummary, WorkerInputKind, WorkerInputRequest, WorkerInputResult,
WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult,
WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult,
WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerTicketAssignmentRequest,
WorkerWorkspaceSummary,
ConfigBundleCheckResult, ConfigBundleSyncResult, DiagnosticSeverity, EMBEDDED_RUNTIME_ID,
EmbeddedWorkerRuntime, HostSummary, RemoteRuntimeConfig, RemoteWorkerRuntime,
RuntimeDiagnostic, RuntimeRegistry, RuntimeRegistryError, RuntimeRegistryUnregisterResult,
RuntimeSummary, TicketWorkerRole, WorkerCapabilitySummary, WorkerCompletionsRequest,
WorkerCompletionsResult, WorkerImplementationSummary, WorkerInputKind, WorkerInputRequest,
WorkerInputResult, WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState,
WorkerRestoreResult, WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest,
WorkerSpawnResult, WorkerSpawnWorkingDirectoryRequest, WorkerSummary,
WorkerTicketAssignmentRequest, WorkerWorkspaceSummary,
};
use crate::identity::WorkspaceIdentity;
use crate::memory_backend::execute_memory_backend_operation_with_authority;
@@ -250,8 +250,8 @@ const ORCHESTRATOR_ATTENTION_PROMPT: &str = include_str!(concat!(
#[derive(Clone)]
pub struct WorkspaceApi {
config: ServerConfig,
store: Arc<dyn ControlPlaneStore>,
pub(crate) config: ServerConfig,
pub(crate) store: Arc<dyn ControlPlaneStore>,
authority: SqliteWorkspaceAuthority,
runtime: Arc<RuntimeRegistry>,
companion: Arc<CompanionConsole>,
@@ -269,6 +269,15 @@ impl WorkspaceApi {
let resource_broker = BackendResourceBroker::default();
let execution_backend = WorkerRuntimeExecutionBackend::new(
ProfileRuntimeWorkerFactory::new(config.workspace_root.clone())
.with_embedded_worker_mutation_dispatcher(
EMBEDDED_RUNTIME_ID,
Arc::new(
crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
config.clone(),
store.clone(),
),
),
)
.with_runtime_store_dir(config.embedded_runtime_store_root.clone())
.with_resource_client(Arc::new(resource_broker.clone())),
)
@@ -394,7 +403,7 @@ impl WorkspaceApi {
&self.runtime_subscription_broker
}
fn workspace_api_ref(&self, runtime_id: &str) -> WorkspaceApiRef {
fn workspace_api_ref(&self, _runtime_id: &str) -> WorkspaceApiRef {
WorkspaceApiRef {
workspace_id: self.config.workspace_id.clone(),
base_url: self
@@ -404,7 +413,6 @@ impl WorkspaceApi {
.unwrap_or_else(|| "http://127.0.0.1:8787".to_string())
.trim_end_matches('/')
.to_string(),
runtime_id: Some(runtime_id.to_string()),
}
}
@@ -1118,6 +1126,10 @@ pub fn build_router(api: WorkspaceApi) -> Router {
"/api/w/{workspace_id}/companion/cancel",
post(scoped_post_companion_cancel),
)
.route(
"/api/w/{workspace_id}/workers/remove",
post(scoped_worker_remove_source_boundary),
)
.route(
"/api/runtimes/{runtime_id}/workers",
get(list_runtime_workers).post(create_runtime_worker),
@@ -4811,6 +4823,71 @@ async fn scoped_workspace_protocol_ws(
.into_response())
}
#[derive(Debug, Deserialize)]
struct WorkerRemoveBoundaryRequest {
target_runtime_id: String,
target_worker_id: String,
}
async fn scoped_worker_remove_source_boundary(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedWorkspacePath>,
headers: HeaderMap,
Json(request): Json<WorkerRemoveBoundaryRequest>,
) -> Response {
if let Err(error) = validate_workspace_scope(&api, &path.workspace_id) {
return error.into_response();
}
let proof = match crate::worker_source::presented_worker_remove_source(&headers, None) {
Ok(proof) => proof,
Err(error) => {
return (
StatusCode::UNAUTHORIZED,
Json(serde_json::json!({ "error": error.to_string() })),
)
.into_response();
}
};
match crate::worker_source::verify_worker_remove_source(
&api,
proof,
&request.target_runtime_id,
&request.target_worker_id,
)
.await
{
Ok(source) => (
StatusCode::NOT_IMPLEMENTED,
Json(serde_json::json!({
"error": "WorkerRemove lifecycle is deferred to its consumer Ticket",
"source": {
"runtime_id": source.runtime_id,
"worker_id": source.worker_id,
"actor_kind": source.actor_kind,
"permission": source.permission,
}
})),
)
.into_response(),
Err(error) => {
let status = match error {
crate::worker_source::WorkerMutationSourceProofError::Replay => {
StatusCode::CONFLICT
}
crate::worker_source::WorkerMutationSourceProofError::Authority(_) => {
StatusCode::INTERNAL_SERVER_ERROR
}
_ => StatusCode::FORBIDDEN,
};
(
status,
Json(serde_json::json!({ "error": error.to_string() })),
)
.into_response()
}
}
}
async fn scoped_list_workers(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedWorkspacePath>,
@@ -10561,12 +10638,19 @@ mod tests {
use tokio_tungstenite::connect_async;
use tokio_tungstenite::tungstenite::Message;
use tower::ServiceExt;
use worker_runtime::auth::{
RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner, WORKER_REMOVE_PERMISSION,
decode_worker_mutation_source_claims,
};
use worker_runtime::resource::BackendResourceClient;
use worker_runtime::worker_source::{
RuntimeOwnedWorkerMutationProof, RuntimeWorkerMutationSourceAuthority,
};
use worker_runtime::working_directory::WorkingDirectoryMaterializer;
use crate::hosts::{
TicketWorkerRole, WorkerInputKind, WorkerOperationState, WorkerSpawnAcceptanceRequirement,
WorkerSpawnIntent,
RemoteRuntimeAuthConfig, RuntimeCapabilitySummary, TicketWorkerRole, WorkerInputKind,
WorkerOperationState, WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent,
};
use crate::store::{
MemoryDocumentRecord, MemoryStagingRecord, ObjectiveRecord, ObjectiveResourceRecord,
@@ -10643,11 +10727,10 @@ mod tests {
const TEST_REPOSITORY_ID: &str = "main";
const TEST_CREATED_AT: &str = "2026-06-23T06:43:28Z";
fn test_worker_workspace_api(runtime_id: &str) -> WorkspaceApiRef {
fn test_worker_workspace_api(_runtime_id: &str) -> WorkspaceApiRef {
WorkspaceApiRef {
workspace_id: TEST_WORKSPACE_ID.to_string(),
base_url: "http://127.0.0.1:8787".to_string(),
runtime_id: Some(runtime_id.to_string()),
}
}
@@ -13141,6 +13224,359 @@ mod tests {
.unwrap()
}
#[tokio::test]
async fn destructive_worker_remove_rejects_browser_and_legacy_source_headers() {
let headers = HeaderMap::new();
assert!(matches!(
crate::worker_source::presented_worker_remove_source(&headers, None),
Err(crate::worker_source::WorkerMutationSourceProofError::Missing)
));
let mut spoofed = HeaderMap::new();
spoofed.insert("x-yoi-runtime-id", "runtime-spoofed".parse().unwrap());
spoofed.insert("x-yoi-worker-id", "worker-spoofed".parse().unwrap());
assert!(matches!(
crate::worker_source::presented_worker_remove_source(&spoofed, None),
Err(crate::worker_source::WorkerMutationSourceProofError::Missing)
));
let temp = tempfile::tempdir().unwrap();
let app = build_router(test_api(temp.path()).await);
let body = r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker"}"#;
let browser = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri(format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove"))
.header(CONTENT_TYPE, "application/json")
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(browser.status(), StatusCode::UNAUTHORIZED);
let legacy = app
.oneshot(
Request::builder()
.method("POST")
.uri(format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove"))
.header(CONTENT_TYPE, "application/json")
.header("x-yoi-runtime-id", "runtime-spoofed")
.header("x-yoi-worker-id", "worker-spoofed")
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(legacy.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn embedded_worker_remove_proof_derives_source_and_rejects_replay_and_wrong_target() {
let temp = tempfile::tempdir().unwrap();
let api = test_api(temp.path()).await;
seed_worker_source_member(&api, EMBEDDED_RUNTIME_ID, "7");
let scope = worker_runtime::RuntimeWorkspaceScope::new(
api.config.workspace_id.clone(),
"server-unused-for-embedded",
);
let authority = RuntimeWorkerMutationSourceAuthority::embedded(
EMBEDDED_RUNTIME_ID,
&api.config.workspace_id,
);
let RuntimeOwnedWorkerMutationProof::InProcess(proof) = authority
.issue_worker_remove(&scope, "7", "runtime-target", "target-worker")
.unwrap()
else {
panic!("embedded Runtime must produce in-process claims");
};
let wrong_target = crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::InProcess(proof.clone()),
"runtime-target",
"different-worker",
)
.await;
assert!(matches!(
wrong_target,
Err(crate::worker_source::WorkerMutationSourceProofError::Invalid)
));
let verified = crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::InProcess(proof.clone()),
"runtime-target",
"target-worker",
)
.await
.unwrap();
assert_eq!(verified.runtime_id, EMBEDDED_RUNTIME_ID);
assert_eq!(verified.worker_id, "7");
assert_eq!(verified.permission, WORKER_REMOVE_PERMISSION);
let replay = crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::InProcess(proof),
"runtime-target",
"target-worker",
)
.await;
assert!(matches!(
replay,
Err(crate::worker_source::WorkerMutationSourceProofError::Replay)
));
let RuntimeOwnedWorkerMutationProof::InProcess(fresh_proof) = authority
.issue_worker_remove(&scope, "7", "runtime-target", "target-worker")
.unwrap()
else {
panic!("embedded Runtime must produce in-process claims");
};
let dispatcher = crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
api.config.clone(),
api.store.clone(),
);
let response =
worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher::execute_worker_remove(
&dispatcher,
fresh_proof,
"runtime-target",
"target-worker",
)
.unwrap();
assert_eq!(response.status, 501);
}
#[tokio::test]
async fn remote_worker_remove_proof_requires_current_runtime_trust_scope_and_catalog_member() {
let temp = tempfile::tempdir().unwrap();
let identity = RuntimeIdentityMaterial::generate("runtime-remote").unwrap();
let mut config = test_server_config(temp.path());
config.remote_runtime_sources.push(RemoteRuntimeConfig {
runtime_id: "runtime-remote".to_string(),
display_name: "Remote Runtime".to_string(),
base_url: "https://runtime.invalid".to_string(),
bearer_token: None,
auth: Some(RemoteRuntimeAuthConfig {
server_id: "server-main".to_string(),
server_private_key: identity.private_key.clone(),
}),
cached_capabilities: RuntimeCapabilitySummary {
can_list_hosts: true,
can_list_workers: true,
can_get_worker: true,
can_spawn_worker: true,
can_stop_worker: true,
has_workspace_fs: false,
has_shell: false,
has_git: false,
supports_worktrees: false,
supports_backend_internal_tools: false,
workspace_scope: TEST_WORKSPACE_ID.to_string(),
max_workers: 1,
os: "test".to_string(),
arch: "test".to_string(),
},
cached_status: "connected".to_string(),
timeout: std::time::Duration::from_secs(1),
});
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
let trust = crate::store::TrustedRuntimeRecord {
runtime_id: "runtime-remote".to_string(),
display_name: "Remote Runtime".to_string(),
base_url: "https://runtime.invalid".to_string(),
public_key: identity.public_key.clone(),
created_at: "2026-08-11T00:00:00Z".to_string(),
updated_at: "2026-08-11T00:00:00Z".to_string(),
revoked_at: None,
};
store.upsert_trusted_runtime(&trust).unwrap();
let api = WorkspaceApi::new_with_execution_backend(
config,
Arc::new(store),
Arc::new(DeterministicExecutionBackend::default()),
)
.await
.unwrap();
seed_worker_source_member(&api, "runtime-remote", "7");
let signer = RuntimeWorkerMutationSourceSigner::from_identity(&identity);
let token = signer
.issue_worker_remove(
"server-main",
&api.config.workspace_id,
"7",
"runtime-target",
"target-worker",
60,
)
.unwrap();
let verified = crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&token),
"runtime-target",
"target-worker",
)
.await
.unwrap();
assert_eq!(verified.runtime_id, "runtime-remote");
assert_eq!(verified.worker_id, "7");
let wrong_scope = signer
.issue_worker_remove(
"server-wrong",
&api.config.workspace_id,
"7",
"runtime-target",
"target-worker",
60,
)
.unwrap();
assert!(matches!(
crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&wrong_scope),
"runtime-target",
"target-worker",
)
.await,
Err(crate::worker_source::WorkerMutationSourceProofError::WrongAudience)
));
let wrong_workspace = signer
.issue_worker_remove(
"server-main",
"workspace-other",
"7",
"runtime-target",
"target-worker",
60,
)
.unwrap();
assert!(matches!(
crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&wrong_workspace),
"runtime-target",
"target-worker",
)
.await,
Err(crate::worker_source::WorkerMutationSourceProofError::WrongWorkspace)
));
let missing_worker = signer
.issue_worker_remove(
"server-main",
&api.config.workspace_id,
"999",
"runtime-target",
"target-worker",
60,
)
.unwrap();
assert!(matches!(
crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&missing_worker),
"runtime-target",
"target-worker",
)
.await,
Err(crate::worker_source::WorkerMutationSourceProofError::WorkerCatalogMembership)
));
let mut expired_claims = decode_worker_mutation_source_claims(&token).unwrap();
expired_claims.iat = 1;
expired_claims.exp = 2;
expired_claims.jti = "expired-proof".to_string();
let expired = signer.sign(&expired_claims).unwrap();
assert!(matches!(
crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&expired),
"runtime-target",
"target-worker",
)
.await,
Err(crate::worker_source::WorkerMutationSourceProofError::Expired)
));
let route_token = signer
.issue_worker_remove(
"server-main",
&api.config.workspace_id,
"7",
"runtime-target",
"target-worker",
60,
)
.unwrap();
let route_response = build_router(api.clone())
.oneshot(
Request::builder()
.method("POST")
.uri(format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove"))
.header(CONTENT_TYPE, "application/json")
.header(
worker_runtime::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER,
route_token,
)
.body(Body::from(
r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker"}"#,
))
.unwrap(),
)
.await
.unwrap();
assert_eq!(route_response.status(), StatusCode::NOT_IMPLEMENTED);
let mut revoked = trust;
revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string());
let authority = SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap();
authority.upsert_trusted_runtime(&revoked).unwrap();
let revoked_token = signer
.issue_worker_remove(
"server-main",
&api.config.workspace_id,
"7",
"runtime-target",
"target-worker",
60,
)
.unwrap();
assert!(matches!(
crate::worker_source::verify_worker_remove_source(
&api,
crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&revoked_token),
"runtime-target",
"target-worker",
)
.await,
Err(crate::worker_source::WorkerMutationSourceProofError::RevokedRuntimeTrust)
));
}
fn seed_worker_source_member(api: &WorkspaceApi, runtime_id: &str, worker_id: &str) {
let now = now_registry_timestamp();
api.store
.upsert_worker_registry(&WorkerRegistryRecord {
workspace_id: api.config.workspace_id.clone(),
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
display_name: worker_id.to_string(),
profile: None,
retention_state: "normal".to_string(),
transcript_ref: None,
session_ref: None,
summary_ref: None,
diagnostics_ref: None,
created_at: now.clone(),
updated_at: now,
})
.unwrap();
}
fn seed_cleanup_worker(
api: &WorkspaceApi,
runtime_worker_id: u64,
+123 -9
View File
@@ -161,6 +161,11 @@ const MIGRATIONS: &[Migration] = &[
name: "create Worker retention authority",
apply: crate::retention::create_worker_retention_tables,
},
Migration {
version: 29,
name: "create Worker mutation source proof replay guard",
apply: create_worker_mutation_source_proof_replay_guard,
},
];
struct Migration {
@@ -460,6 +465,15 @@ pub trait ControlPlaneStore: Send + Sync {
async fn schema_version(&self) -> Result<i64>;
async fn upsert_workspace(&self, record: &WorkspaceRecord) -> Result<()>;
async fn get_workspace(&self, workspace_id: &str) -> Result<Option<WorkspaceRecord>>;
async fn get_trusted_runtime(&self, runtime_id: &str) -> Result<Option<TrustedRuntimeRecord>>;
async fn consume_worker_mutation_source_jti(
&self,
runtime_id: &str,
jti: &str,
expires_at: u64,
now_seconds: u64,
consumed_at: &str,
) -> Result<bool>;
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>>;
fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>;
fn get_repository(
@@ -896,6 +910,44 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
})
}
async fn get_trusted_runtime(&self, runtime_id: &str) -> Result<Option<TrustedRuntimeRecord>> {
self.with_conn(|conn| {
conn.query_row(
r#"SELECT runtime_id, display_name, base_url, public_key, created_at, updated_at, revoked_at
FROM trusted_runtime_records WHERE runtime_id = ?1"#,
params![runtime_id],
read_trusted_runtime_record,
)
.optional()
.map_err(Error::from)
})
}
async fn consume_worker_mutation_source_jti(
&self,
runtime_id: &str,
jti: &str,
expires_at: u64,
now_seconds: u64,
consumed_at: &str,
) -> Result<bool> {
self.with_conn(|conn| {
let transaction = conn.unchecked_transaction()?;
transaction.execute(
"DELETE FROM worker_mutation_source_proof_jtis WHERE expires_at < ?1",
params![now_seconds],
)?;
let inserted = transaction.execute(
r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis (
runtime_id, jti, expires_at, consumed_at
) VALUES (?1, ?2, ?3, ?4)"#,
params![runtime_id, jti, expires_at, consumed_at],
)?;
transaction.commit()?;
Ok(inserted == 1)
})
}
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>> {
self.with_conn(|conn| {
let mut stmt = conn.prepare(
@@ -4326,6 +4378,23 @@ fn current_schema_version(conn: &Connection) -> Result<i64> {
.map_err(Error::from)
}
fn create_worker_mutation_source_proof_replay_guard(conn: &Connection) -> Result<()> {
conn.execute_batch(
r#"
CREATE TABLE IF NOT EXISTS worker_mutation_source_proof_jtis (
runtime_id TEXT NOT NULL,
jti TEXT NOT NULL,
expires_at INTEGER NOT NULL,
consumed_at TEXT NOT NULL,
PRIMARY KEY (runtime_id, jti)
);
CREATE INDEX IF NOT EXISTS idx_worker_mutation_source_proof_jtis_expiry
ON worker_mutation_source_proof_jtis(expires_at);
"#,
)?;
Ok(())
}
fn apply_migrations(conn: &Connection) -> Result<()> {
let current = current_schema_version(conn)?;
for migration in MIGRATIONS
@@ -4915,7 +4984,7 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
apply_migrations(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 28);
assert_eq!(current_schema_version(&conn).unwrap(), 29);
assert!(table_exists(&conn, "worker_workdir_attachment_reservations").unwrap());
}
@@ -4948,7 +5017,7 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY);
apply_migrations(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 28);
assert_eq!(current_schema_version(&conn).unwrap(), 29);
assert!(table_exists(&conn, "flow_sources").unwrap());
assert!(table_exists(&conn, "flow_source_revisions").unwrap());
assert!(!table_exists(&conn, "flow_instances").unwrap());
@@ -5015,7 +5084,7 @@ INSERT INTO worker_workdir_attachment_reservations (
apply_migrations(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 28);
assert_eq!(current_schema_version(&conn).unwrap(), 29);
let repositories_sql: String = conn
.query_row(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'repositories'",
@@ -5195,7 +5264,7 @@ INSERT INTO workdir_registry (
let db = dir.path().join("control-plane.sqlite");
let store = SqliteWorkspaceStore::open(&db).unwrap();
assert_eq!(store.schema_version().await.unwrap(), 28);
assert_eq!(store.schema_version().await.unwrap(), 29);
assert!(
!store
.with_conn(|conn| table_exists(conn, "worker_workspace_credentials"))
@@ -5212,7 +5281,7 @@ INSERT INTO workdir_registry (
store.upsert_workspace(&record).await.unwrap();
let reopened = SqliteWorkspaceStore::open(&db).unwrap();
assert_eq!(reopened.schema_version().await.unwrap(), 28);
assert_eq!(reopened.schema_version().await.unwrap(), 29);
assert_eq!(
reopened.get_workspace("local-dev").await.unwrap(),
Some(record)
@@ -5759,7 +5828,7 @@ INSERT INTO workdir_registry (
.unwrap();
let store = SqliteWorkspaceStore::from_connection(conn).unwrap();
assert_eq!(store.schema_version().await.unwrap(), 28);
assert_eq!(store.schema_version().await.unwrap(), 29);
store
.with_conn(|conn| {
@@ -5948,7 +6017,7 @@ CREATE TABLE ticket_assignment_operations (
#[tokio::test]
async fn repository_records_round_trip() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
assert_eq!(store.schema_version().await.unwrap(), 28);
assert_eq!(store.schema_version().await.unwrap(), 29);
let workspace = WorkspaceRecord {
workspace_id: "local-dev".to_string(),
owner_account_id: None,
@@ -6014,7 +6083,7 @@ CREATE TABLE ticket_assignment_operations (
#[tokio::test]
async fn memory_authority_records_round_trip_and_close_staging() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
assert_eq!(store.schema_version().await.unwrap(), 28);
assert_eq!(store.schema_version().await.unwrap(), 29);
let workspace = WorkspaceRecord {
workspace_id: "local-dev".to_string(),
owner_account_id: None,
@@ -6277,7 +6346,7 @@ CREATE TABLE ticket_assignment_operations (
#[tokio::test]
async fn account_and_login_records_round_trip() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
assert_eq!(store.schema_version().await.unwrap(), 28);
assert_eq!(store.schema_version().await.unwrap(), 29);
let now = "2026-07-22T00:00:00Z".to_string();
let account = AccountRecord {
account_id: "acct-user-alice".to_string(),
@@ -6455,6 +6524,51 @@ CREATE TABLE ticket_assignment_operations (
);
}
#[tokio::test]
async fn worker_mutation_source_jti_replay_guard_survives_reopen() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("server.db");
let store = SqliteWorkspaceStore::open(&path).unwrap();
store
.upsert_trusted_runtime(&TrustedRuntimeRecord {
runtime_id: "runtime-a".to_string(),
display_name: "Runtime A".to_string(),
base_url: "https://runtime.invalid".to_string(),
public_key: "public-key".to_string(),
created_at: "2026-08-11T00:00:00Z".to_string(),
updated_at: "2026-08-11T00:00:00Z".to_string(),
revoked_at: None,
})
.unwrap();
assert!(
store
.consume_worker_mutation_source_jti(
"runtime-a",
"proof-1",
2_000,
1_000,
"2026-08-11T00:00:00Z",
)
.await
.unwrap()
);
drop(store);
let reopened = SqliteWorkspaceStore::open(&path).unwrap();
assert!(
!reopened
.consume_worker_mutation_source_jti(
"runtime-a",
"proof-1",
2_000,
1_001,
"2026-08-11T00:00:01Z",
)
.await
.unwrap()
);
}
fn table_names(conn: &Connection) -> BTreeSet<String> {
let mut stmt = conn
.prepare(
@@ -0,0 +1,288 @@
use std::time::{SystemTime, UNIX_EPOCH};
use axum::http::HeaderMap;
use worker_runtime::auth::{
WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims,
WorkerMutationSourceExpectation, decode_worker_mutation_source_claims,
verify_worker_mutation_source_proof,
};
use worker_runtime::worker_source::InProcessWorkerMutationProof;
use crate::hosts::RemoteRuntimeConfig;
use crate::server::WorkspaceApi;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum PresentedWorkerMutationSourceProof<'a> {
Remote(&'a str),
InProcess(InProcessWorkerMutationProof),
}
pub fn presented_worker_remove_source<'a>(
headers: &'a HeaderMap,
in_process: Option<InProcessWorkerMutationProof>,
) -> Result<PresentedWorkerMutationSourceProof<'a>, WorkerMutationSourceProofError> {
if let Some(claims) = in_process {
return Ok(PresentedWorkerMutationSourceProof::InProcess(claims));
}
headers
.get(worker_runtime::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER)
.and_then(|value| value.to_str().ok())
.filter(|value| !value.trim().is_empty())
.map(PresentedWorkerMutationSourceProof::Remote)
.ok_or(WorkerMutationSourceProofError::Missing)
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct VerifiedWorkerMutationSource {
pub runtime_id: String,
pub worker_id: String,
pub actor_kind: WorkerMutationActorKind,
pub permission: String,
pub jti: String,
}
#[derive(Debug, thiserror::Error)]
pub enum WorkerMutationSourceProofError {
#[error("Worker mutation source proof is required")]
Missing,
#[error("Worker mutation source proof is invalid")]
Invalid,
#[error("Worker mutation source proof is not authorized for this Server")]
WrongAudience,
#[error("Worker mutation source proof is not authorized for this Workspace")]
WrongWorkspace,
#[error("Worker mutation source proof actor is not allowed")]
WrongActor,
#[error("Worker mutation source proof lacks `{0}` permission")]
MissingPermission(String),
#[error("Worker mutation source proof is expired")]
Expired,
#[error("Runtime trust is missing or revoked")]
RevokedRuntimeTrust,
#[error("Worker mutation source proof was already consumed")]
Replay,
#[error("source Worker is not a current member of this Workspace Runtime catalog")]
WorkerCatalogMembership,
#[error("source proof authority failed: {0}")]
Authority(String),
}
pub async fn verify_worker_remove_source(
api: &WorkspaceApi,
proof: PresentedWorkerMutationSourceProof<'_>,
target_runtime_id: &str,
target_worker_id: &str,
) -> Result<VerifiedWorkerMutationSource, WorkerMutationSourceProofError> {
verify_worker_remove_source_with(
&api.config,
&api.store,
proof,
target_runtime_id,
target_worker_id,
)
.await
}
async fn verify_worker_remove_source_with(
config: &crate::server::ServerConfig,
store: &std::sync::Arc<dyn crate::store::ControlPlaneStore>,
proof: PresentedWorkerMutationSourceProof<'_>,
target_runtime_id: &str,
target_worker_id: &str,
) -> Result<VerifiedWorkerMutationSource, WorkerMutationSourceProofError> {
let required_permission = worker_runtime::auth::WORKER_REMOVE_PERMISSION;
let now = unix_now_seconds();
let claims = match proof {
PresentedWorkerMutationSourceProof::Remote(token) => {
let unverified = decode_worker_mutation_source_claims(token)
.map_err(|_| WorkerMutationSourceProofError::Invalid)?;
let audience = remote_audience(config, &unverified.iss)?;
let trusted = store
.get_trusted_runtime(&unverified.iss)
.await
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?
.filter(|record| record.revoked_at.is_none())
.ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?;
let expected = WorkerMutationSourceExpectation {
runtime_id: &unverified.iss,
audience,
workspace_id: &config.workspace_id,
worker_id: None,
actor_kind: WorkerMutationActorKind::Worker,
operation: WorkerMutationOperation::WorkerRemove,
target_runtime_id,
target_worker_id,
permission: required_permission,
};
verify_worker_mutation_source_proof(&trusted.public_key, token, &expected, now)
.map_err(map_auth_error)?
}
PresentedWorkerMutationSourceProof::InProcess(proof) => {
let claims = proof.into_claims();
if config
.remote_runtime_sources
.iter()
.any(|runtime| runtime.runtime_id == claims.iss)
{
return Err(WorkerMutationSourceProofError::Invalid);
}
validate_in_process_claims(
&claims,
&format!("embedded:{}", config.workspace_id),
&config.workspace_id,
target_runtime_id,
target_worker_id,
required_permission,
now,
)?;
claims
}
};
let worker = worker_runtime::identity::RuntimeWorkerRef {
runtime_id: claims.iss.clone(),
worker_id: claims.worker_id.clone(),
};
let member = store
.get_worker_registry(&config.workspace_id, &worker)
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
if member.is_none() {
return Err(WorkerMutationSourceProofError::WorkerCatalogMembership);
}
let consumed_at = chrono::Utc::now().to_rfc3339();
let consumed = store
.consume_worker_mutation_source_jti(&claims.iss, &claims.jti, claims.exp, now, &consumed_at)
.await
.map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?;
if !consumed {
return Err(WorkerMutationSourceProofError::Replay);
}
Ok(VerifiedWorkerMutationSource {
runtime_id: claims.iss,
worker_id: claims.worker_id,
actor_kind: claims.actor_kind,
permission: claims.permission,
jti: claims.jti,
})
}
#[derive(Clone)]
pub(crate) struct EmbeddedServerWorkerMutationDispatcher {
config: crate::server::ServerConfig,
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
}
impl EmbeddedServerWorkerMutationDispatcher {
pub(crate) fn new(
config: crate::server::ServerConfig,
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
) -> Self {
Self { config, store }
}
}
impl worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher
for EmbeddedServerWorkerMutationDispatcher
{
fn execute_worker_remove(
&self,
proof: InProcessWorkerMutationProof,
target_runtime_id: &str,
target_worker_id: &str,
) -> Result<
worker::WorkspaceResponse,
worker_runtime::worker_source::RuntimeWorkerMutationForwardError,
> {
futures::executor::block_on(verify_worker_remove_source_with(
&self.config,
&self.store,
PresentedWorkerMutationSourceProof::InProcess(proof),
target_runtime_id,
target_worker_id,
))
.map_err(|error| {
worker_runtime::worker_source::RuntimeWorkerMutationForwardError::Embedded(
error.to_string(),
)
})?;
Ok(worker::WorkspaceResponse {
status: 501,
body: "WorkerRemove lifecycle is not implemented by this operation boundary"
.to_string(),
})
}
}
fn remote_audience<'a>(
config: &'a crate::server::ServerConfig,
runtime_id: &str,
) -> Result<&'a str, WorkerMutationSourceProofError> {
config
.remote_runtime_sources
.iter()
.find(|runtime| runtime.runtime_id == runtime_id)
.and_then(|runtime: &RemoteRuntimeConfig| runtime.auth.as_ref())
.map(|auth| auth.server_id.as_str())
.ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)
}
fn validate_in_process_claims(
claims: &WorkerMutationSourceClaims,
audience: &str,
workspace_id: &str,
target_runtime_id: &str,
target_worker_id: &str,
permission: &str,
now: u64,
) -> Result<(), WorkerMutationSourceProofError> {
if claims.aud != audience {
return Err(WorkerMutationSourceProofError::WrongAudience);
}
if claims.workspace_id != workspace_id {
return Err(WorkerMutationSourceProofError::WrongWorkspace);
}
if claims.actor_kind != WorkerMutationActorKind::Worker {
return Err(WorkerMutationSourceProofError::WrongActor);
}
if claims.operation != WorkerMutationOperation::WorkerRemove
|| claims.target_runtime_id != target_runtime_id
|| claims.target_worker_id != target_worker_id
{
return Err(WorkerMutationSourceProofError::Invalid);
}
if claims.permission != permission {
return Err(WorkerMutationSourceProofError::MissingPermission(
permission.to_string(),
));
}
if claims.exp <= now || claims.iat > now.saturating_add(60) || claims.jti.trim().is_empty() {
return Err(WorkerMutationSourceProofError::Expired);
}
Ok(())
}
fn map_auth_error(error: worker_runtime::auth::RuntimeAuthError) -> WorkerMutationSourceProofError {
use worker_runtime::auth::RuntimeAuthError;
match error {
RuntimeAuthError::WrongAudience { .. } => WorkerMutationSourceProofError::WrongAudience,
RuntimeAuthError::WrongWorkspace { .. } => WorkerMutationSourceProofError::WrongWorkspace,
RuntimeAuthError::WrongActorKind => WorkerMutationSourceProofError::WrongActor,
RuntimeAuthError::WrongOperation | RuntimeAuthError::WrongMutationTarget => {
WorkerMutationSourceProofError::Invalid
}
RuntimeAuthError::MissingPermission(permission) => {
WorkerMutationSourceProofError::MissingPermission(permission)
}
RuntimeAuthError::Expired => WorkerMutationSourceProofError::Expired,
_ => WorkerMutationSourceProofError::Invalid,
}
}
fn unix_now_seconds() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}