From f29c3438798e765b84a23a67179e5714fe94764a Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 8 Sep 2026 08:13:58 +0900 Subject: [PATCH] feat: verify Workspace-signed Runtime bindings --- crates/workdir/Cargo.toml | 1 + crates/workdir/src/http.rs | 33 +- crates/worker-runtime/src/http_server.rs | 546 ++++++++++- crates/worker-runtime/src/main.rs | 66 +- crates/worker-runtime/src/workspace_issuer.rs | 882 +++++++++++++++++- crates/workspace-server/src/hosts.rs | 492 +++++++++- crates/workspace-server/src/latest_schema.sql | 22 + .../src/runtime_subscription.rs | 6 + crates/workspace-server/src/server.rs | 306 +++++- crates/workspace-server/src/store.rs | 516 +++++++++- .../src/workspace_deletion.rs | 1 + .../src/workspace_signing_identity.rs | 21 + 12 files changed, 2817 insertions(+), 75 deletions(-) diff --git a/crates/workdir/Cargo.toml b/crates/workdir/Cargo.toml index 8bd23773..33f6c72b 100644 --- a/crates/workdir/Cargo.toml +++ b/crates/workdir/Cargo.toml @@ -14,6 +14,7 @@ fs-operation.workspace = true manifest.workspace = true reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"], optional = true } serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true sha2.workspace = true tempfile.workspace = true thiserror.workspace = true diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index f89b608b..cd02e19b 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -293,7 +293,12 @@ mod client { /// implementations can mint short-lived capability tokens without making a /// Worker-bound session expire with the token used to open it. pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync { - fn bearer_token(&self) -> Result; + fn bearer_token( + &self, + method: &str, + path_and_query: &str, + body: &[u8], + ) -> Result; } struct FixedBearerToken(Arc); @@ -305,7 +310,12 @@ mod client { } impl WorkdirHttpAuthorization for FixedBearerToken { - fn bearer_token(&self) -> Result { + fn bearer_token( + &self, + _method: &str, + _path_and_query: &str, + _body: &[u8], + ) -> Result { Ok(self.0.to_string()) } } @@ -354,10 +364,14 @@ mod client { &base_url, &["v1", "working-directories", workdir_id.as_str(), "sessions"], )?; + let body = serde_json::to_vec(&request) + .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; + let token = authorization.bearer_token("POST", url.path(), &body)?; let response = client .post(url) - .bearer_auth(authorization.bearer_token()?) - .json(&request) + .bearer_auth(token) + .header("content-type", "application/json") + .body(body) .send() .await .map_err(http_unavailable)?; @@ -401,11 +415,15 @@ mod client { ], )?; let operation = WorkdirSessionOperationRequest { operation }; + let body = serde_json::to_vec(&operation) + .map_err(|error| WorkdirError::Unavailable(error.to_string()))?; + let token = self.authorization.bearer_token("POST", url.path(), &body)?; let response = self .client .post(url) - .bearer_auth(self.authorization.bearer_token()?) - .json(&operation) + .bearer_auth(token) + .header("content-type", "application/json") + .body(body) .send() .await .map_err(http_unavailable)?; @@ -543,10 +561,11 @@ mod client { &self.base_url, &["v1", "workdir-sessions", self.session_id.as_str()], )?; + let token = self.authorization.bearer_token("DELETE", url.path(), &[])?; let response = self .client .delete(url) - .bearer_auth(self.authorization.bearer_token()?) + .bearer_auth(token) .send() .await .map_err(http_unavailable)?; diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index c8c13013..e14b75d3 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -27,6 +27,15 @@ use crate::retention::{ }; #[cfg(feature = "ws-server")] use crate::runtime::RuntimeSubscriptionRecvError; +use crate::workspace_issuer::{ + RuntimeVerificationSigner, VerifiedWorkspaceCapability, WORKSPACE_VERIFICATION_ACK_PATH, + WORKSPACE_VERIFICATION_CHALLENGE_PATH, WORKSPACE_VERIFICATION_OPERATION, + WorkspaceCapabilityExpectation, WorkspaceCapabilityVerifier, + WorkspaceRuntimeVerificationAcknowledgement, WorkspaceRuntimeVerificationAuthority, + WorkspaceRuntimeVerificationChallenge, WorkspaceRuntimeVerificationReceipt, + WorkspaceRuntimeVerificationRecord, WorkspaceRuntimeVerificationResponse, + inspect_workspace_capability_claims, workspace_request_body_digest, +}; use crate::{Runtime, RuntimeWorkspaceScope}; use axum::body::{Body, Bytes}; use axum::extract::rejection::{JsonRejection, QueryRejection}; @@ -155,7 +164,22 @@ pub async fn serve_runtime_http_with_auth( } axum::serve( listener, - runtime_http_router_with_optional_auth(runtime, local_token, auth), + runtime_http_router_with_optional_auth(runtime, local_token, auth, None), + ) + .await?; + Ok(()) +} + +pub async fn serve_runtime_http_with_workspace_auth( + runtime: Runtime, + listener: TcpListener, + local_token: Option, + auth: Option, + workspace_auth: WorkspaceRuntimeHttpAuth, +) -> Result<(), RuntimeHttpServerError> { + axum::serve( + listener, + runtime_http_router_with_optional_auth(runtime, local_token, auth, Some(workspace_auth)), ) .await?; Ok(()) @@ -167,7 +191,7 @@ pub async fn serve_runtime_http_with_auth( /// The path contains only a Runtime-local `worker_id`; backend aliases are not /// accepted or forwarded as Runtime authority. pub fn runtime_http_router(runtime: Runtime, local_token: String) -> Router { - runtime_http_router_with_optional_auth(runtime, Some(local_token), None) + runtime_http_router_with_optional_auth(runtime, Some(local_token), None, None) } /// Build the REST router for an existing Runtime with signed capability-token auth. @@ -176,23 +200,42 @@ pub fn runtime_http_router_with_auth( local_token: Option, auth: RuntimeHttpAuthConfig, ) -> Router { - runtime_http_router_with_optional_auth(runtime, local_token, Some(auth)) + runtime_http_router_with_optional_auth(runtime, local_token, Some(auth), None) +} + +pub fn runtime_http_router_with_workspace_auth( + runtime: Runtime, + local_token: Option, + auth: RuntimeHttpAuthConfig, + workspace_auth: WorkspaceRuntimeHttpAuth, +) -> Router { + runtime_http_router_with_optional_auth(runtime, local_token, Some(auth), Some(workspace_auth)) } fn runtime_http_router_with_optional_auth( runtime: Runtime, local_token: Option, auth: Option, + workspace_auth: Option, ) -> Router { let state = RuntimeHttpState { runtime, local_token: local_token.map(Arc::::from), auth: auth.map(Arc::new), + workspace_auth: workspace_auth.map(Arc::new), workdir_sessions: Arc::new(Mutex::new(HashMap::new())), }; let router = Router::new() .route("/v1/ping", get(get_runtime_ping)) + .route( + WORKSPACE_VERIFICATION_CHALLENGE_PATH, + post(post_workspace_verification_challenge), + ) + .route( + WORKSPACE_VERIFICATION_ACK_PATH, + post(post_workspace_verification_acknowledgement), + ) .route("/v1/runtime", get(get_runtime)) .route( "/v1/config-bundles", @@ -286,9 +329,17 @@ struct RuntimeHttpState { runtime: Runtime, local_token: Option>, auth: Option>, + workspace_auth: Option>, workdir_sessions: Arc>>, } +#[derive(Clone, Debug)] +pub struct WorkspaceRuntimeHttpAuth { + pub verifier: WorkspaceCapabilityVerifier, + pub signer: RuntimeVerificationSigner, + pub verifications: Arc, +} + struct RuntimeHttpWorkdirSession { owner: RuntimeWorkspaceScope, session: WorkdirSessionHandle, @@ -475,6 +526,164 @@ struct RuntimeWorkerEventsWsQuery { type RestResult = Result, RuntimeHttpRestError>; +fn unix_now_i64() -> i64 { + i64::try_from(unix_now_seconds()).unwrap_or(i64::MAX) +} + +async fn post_workspace_verification_challenge( + State(state): State, + Extension(verified): Extension, + Json(challenge): Json, +) -> RestResult { + let auth = state.workspace_auth.as_deref().ok_or_else(|| { + RuntimeHttpRestError::new( + StatusCode::NOT_IMPLEMENTED, + "workspace_runtime_verification_unavailable", + "Workspace Runtime verification is not configured", + ) + })?; + if challenge.workspace_id != verified.workspace_id + || challenge.runtime_id != verified.runtime_id + || challenge.binding_revision != verified.binding_revision + || challenge.workspace_key_id != verified.issuer_key_id + || challenge.workspace_identity_revision != verified.issuer_identity_revision + || challenge.workspace_trust_generation != verified.trust_generation + || challenge.runtime_id != auth.signer.runtime_id() + || challenge.runtime_public_key_fingerprint != auth.signer.public_key_fingerprint() + { + return Err(RuntimeHttpRestError::new( + StatusCode::UNAUTHORIZED, + "workspace_runtime_verification_rejected", + "Workspace Runtime verification challenge does not match authenticated authority", + )); + } + let response = auth + .signer + .sign_response(&challenge, uuid::Uuid::now_v7().to_string(), unix_now_i64()) + .map_err(|error| { + RuntimeHttpRestError::new( + StatusCode::UNAUTHORIZED, + "workspace_runtime_verification_rejected", + error.to_string(), + ) + })?; + Ok(Json(response)) +} + +async fn post_workspace_verification_acknowledgement( + State(state): State, + Extension(verified): Extension, + Json(acknowledgement): Json, +) -> RestResult { + let auth = state.workspace_auth.as_deref().ok_or_else(|| { + RuntimeHttpRestError::new( + StatusCode::NOT_IMPLEMENTED, + "workspace_runtime_verification_unavailable", + "Workspace Runtime verification is not configured", + ) + })?; + if acknowledgement.workspace_id != verified.workspace_id + || acknowledgement.runtime_id != verified.runtime_id + || acknowledgement.binding_revision != verified.binding_revision + || acknowledgement.workspace_key_id != verified.issuer_key_id + || acknowledgement.workspace_identity_revision != verified.issuer_identity_revision + || acknowledgement.workspace_trust_generation != verified.trust_generation + || acknowledgement.runtime_id != auth.signer.runtime_id() + || acknowledgement.runtime_public_key_fingerprint != auth.signer.public_key_fingerprint() + || acknowledgement.expires_at <= unix_now_i64() + || acknowledgement.binding_revision == 0 + || acknowledgement.runtime_identity_revision == 0 + || acknowledgement.workspace_identity_revision == 0 + || acknowledgement.workspace_trust_generation == 0 + || acknowledgement.workspace_nonce.is_empty() + || acknowledgement.runtime_nonce.is_empty() + || acknowledgement.response_digest.len() != workspace_request_body_digest(&[]).len() + { + return Err(RuntimeHttpRestError::new( + StatusCode::UNAUTHORIZED, + "workspace_runtime_verification_rejected", + "Workspace Runtime verification acknowledgement is invalid", + )); + } + let challenge = WorkspaceRuntimeVerificationChallenge { + challenge_id: acknowledgement.challenge_id.clone(), + workspace_id: acknowledgement.workspace_id.clone(), + runtime_id: acknowledgement.runtime_id.clone(), + binding_revision: acknowledgement.binding_revision, + workspace_key_id: acknowledgement.workspace_key_id.clone(), + workspace_identity_revision: acknowledgement.workspace_identity_revision, + workspace_trust_generation: acknowledgement.workspace_trust_generation, + runtime_public_key_fingerprint: acknowledgement.runtime_public_key_fingerprint.clone(), + runtime_identity_revision: acknowledgement.runtime_identity_revision, + workspace_nonce: acknowledgement.workspace_nonce.clone(), + expires_at: acknowledgement.expires_at, + }; + let response = acknowledgement.response.clone(); + if response.runtime_nonce != acknowledgement.runtime_nonce + || response.workspace_nonce != acknowledgement.workspace_nonce + || response.response_proof.is_empty() + { + return Err(RuntimeHttpRestError::new( + StatusCode::UNAUTHORIZED, + "workspace_runtime_verification_rejected", + "Workspace Runtime verification acknowledgement does not match its response", + )); + } + let response_bytes = serde_json::to_vec(&response).map_err(|_| { + RuntimeHttpRestError::new( + StatusCode::BAD_REQUEST, + "workspace_runtime_verification_rejected", + "Workspace Runtime verification response could not be canonicalized", + ) + })?; + if workspace_request_body_digest(&response_bytes) != acknowledgement.response_digest { + return Err(RuntimeHttpRestError::new( + StatusCode::UNAUTHORIZED, + "workspace_runtime_verification_rejected", + "Workspace Runtime verification acknowledgement has the wrong response digest", + )); + } + crate::workspace_issuer::verify_runtime_verification_response( + &response, + &challenge, + auth.signer.public_key(), + unix_now_i64(), + ) + .map_err(|error| { + RuntimeHttpRestError::new( + StatusCode::UNAUTHORIZED, + "workspace_runtime_verification_rejected", + error.to_string(), + ) + })?; + auth.verifications + .record(WorkspaceRuntimeVerificationRecord { + workspace_id: acknowledgement.workspace_id.clone(), + runtime_id: acknowledgement.runtime_id.clone(), + binding_revision: acknowledgement.binding_revision, + workspace_key_id: acknowledgement.workspace_key_id.clone(), + workspace_identity_revision: acknowledgement.workspace_identity_revision, + workspace_trust_generation: acknowledgement.workspace_trust_generation, + runtime_public_key_fingerprint: acknowledgement.runtime_public_key_fingerprint.clone(), + runtime_identity_revision: acknowledgement.runtime_identity_revision, + verified_at: unix_now_i64(), + }) + .map_err(|error| { + RuntimeHttpRestError::new( + StatusCode::SERVICE_UNAVAILABLE, + "workspace_runtime_verification_unavailable", + error.to_string(), + ) + })?; + Ok(Json(WorkspaceRuntimeVerificationReceipt { + challenge_id: acknowledgement.challenge_id, + workspace_id: acknowledgement.workspace_id, + runtime_id: acknowledgement.runtime_id, + binding_revision: acknowledgement.binding_revision, + accepted_at: unix_now_i64(), + })) +} + async fn get_runtime_ping( State(state): State, Extension(auth): Extension, @@ -1797,10 +2006,112 @@ async fn require_runtime_auth( .headers() .get(header::AUTHORIZATION) .and_then(|value| value.to_str().ok()) - .and_then(|value| value.strip_prefix("Bearer ")); + .and_then(|value| value.strip_prefix("Bearer ")) + .map(str::to_owned); + + if let Some(workspace_auth) = state.workspace_auth.as_deref() + && let Some(token) = supplied.as_deref() + && let Ok(claims) = inspect_workspace_capability_claims(token) + { + let method = request.method().as_str().to_string(); + let path_and_query = request + .uri() + .path_and_query() + .map_or_else(|| request.uri().path().to_string(), ToString::to_string); + let required_permission = + workspace_runtime_operation(request.method(), request.uri().path()); + let expected_worker_id = worker_id_from_runtime_path(request.uri().path()); + let (parts, body) = request.into_parts(); + let body = match axum::body::to_bytes(body, 8 * 1024 * 1024).await { + Ok(body) => body, + Err(_) => { + return RuntimeHttpRestError::new( + StatusCode::PAYLOAD_TOO_LARGE, + "request_body_too_large", + "Runtime request body exceeds the verification limit", + ) + .into_response(); + } + }; + let body_digest = workspace_request_body_digest(&body); + let expected = WorkspaceCapabilityExpectation { + workspace_id: &claims.issuer_workspace_id, + binding_revision: claims.binding_revision, + runtime_id: workspace_auth.signer.runtime_id(), + worker_id: expected_worker_id.as_deref(), + operation: required_permission, + method: &method, + path_and_query: &path_and_query, + body_digest: &body_digest, + now_unix: unix_now_i64(), + }; + match workspace_auth.verifier.verify(token, &expected) { + Ok(verified) => { + let is_verification = path_and_query == WORKSPACE_VERIFICATION_CHALLENGE_PATH + || path_and_query == WORKSPACE_VERIFICATION_ACK_PATH; + if !is_verification { + let record = match workspace_auth + .verifications + .get(&verified.workspace_id, workspace_auth.signer.runtime_id()) + { + Ok(Some(record)) => record, + Ok(None) => { + return RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "workspace_runtime_verification_required", + "Workspace Runtime binding has not completed signed verification", + ) + .into_response(); + } + Err(error) => { + return RuntimeHttpRestError::new( + StatusCode::SERVICE_UNAVAILABLE, + "workspace_runtime_verification_unavailable", + error.to_string(), + ) + .into_response(); + } + }; + if record.binding_revision != verified.binding_revision + || record.workspace_key_id != verified.issuer_key_id + || record.workspace_identity_revision != verified.issuer_identity_revision + || record.workspace_trust_generation != verified.trust_generation + || record.runtime_public_key_fingerprint + != workspace_auth.signer.public_key_fingerprint() + || record.runtime_identity_revision == 0 + { + return RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "workspace_runtime_verification_stale", + "Workspace Runtime verification does not match current request authority", + ) + .into_response(); + } + } + request = Request::from_parts(parts, Body::from(body)); + request.extensions_mut().insert(verified.clone()); + request.extensions_mut().insert(RuntimeAuthContext { + server_id: verified.issuer, + workspace_id: verified.workspace_id, + permissions: vec![required_permission.to_string()], + token_id: verified.token_id, + expires_at: u64::try_from(verified.expires_at).unwrap_or(0), + }); + return next.run(request).await; + } + Err(error) => { + return RuntimeHttpRestError::new( + StatusCode::UNAUTHORIZED, + "unauthorized", + format!("invalid Workspace capability token: {error}"), + ) + .into_response(); + } + } + } if let Some(auth) = state.auth.as_deref() { - let Some(token) = supplied else { + let Some(token) = supplied.as_deref() else { return RuntimeHttpRestError::new( StatusCode::UNAUTHORIZED, "unauthorized", @@ -1815,6 +2126,18 @@ async fn require_runtime_auth( unix_now_seconds(), ) { Ok(context) => { + if state.workspace_auth.as_ref().is_some_and(|workspace_auth| { + workspace_auth + .verifier + .has_active_workspace_issuer(&context.workspace_id) + }) { + return RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "workspace_identity_required", + "Legacy Server-issued capability is disabled for this Workspace", + ) + .into_response(); + } request.extensions_mut().insert(context); return next.run(request).await; } @@ -1825,7 +2148,7 @@ async fn require_runtime_auth( } if let Some(expected) = state.local_token.as_deref() { - if supplied != Some(expected) { + if supplied.as_deref() != Some(expected) { return RuntimeHttpRestError::new( StatusCode::UNAUTHORIZED, "unauthorized", @@ -1897,6 +2220,21 @@ fn auth_workspace_scope( Ok(Some(RuntimeWorkspaceScope::new(workspace_id, server_id))) } +fn workspace_runtime_operation(method: &Method, path: &str) -> &'static str { + if (path == WORKSPACE_VERIFICATION_CHALLENGE_PATH || path == WORKSPACE_VERIFICATION_ACK_PATH) + && *method == Method::POST + { + return WORKSPACE_VERIFICATION_OPERATION; + } + required_runtime_permission(method, path).unwrap_or("runtime:read") +} + +fn worker_id_from_runtime_path(path: &str) -> Option { + let rest = path.strip_prefix("/v1/workers/")?; + let worker_id = rest.split('/').next()?; + (!worker_id.is_empty()).then(|| worker_id.to_string()) +} + fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static str> { if path == "/v1/ping" && *method == Method::GET { return Some(RUNTIME_PING_PERMISSION); @@ -2209,15 +2547,210 @@ mod tests { WorkerExecutionSpawnResult, }; use crate::management::RuntimeOptions; + use crate::workspace_issuer::{ + InMemoryWorkspaceClaimReplayProtection, InMemoryWorkspaceRuntimeVerificationAuthority, + WorkspaceCapabilityClaims, WorkspaceCapabilityVerifier, WorkspaceIssuerTrustRecord, + WorkspaceIssuerTrustState, issue_workspace_capability_token, + verify_runtime_verification_response, + }; use axum::body::to_bytes; use axum::http::Method; use manifest::{Scope, SharedScope}; + use sha2::Digest as _; use tower::ServiceExt; use workdir::{ GrepOutputMode, GrepRequest, LocalWorkdirSession, StatRequest, Workdir, WorkdirPath, WorkdirSessionCapabilities, }; + #[tokio::test] + async fn workspace_signed_verification_requires_exact_request_and_acknowledges_response() { + let runtime = Runtime::new_memory(); + let (legacy_auth, _) = auth_config_and_signer(); + let workspace_identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap(); + let runtime_identity = RuntimeIdentityMaterial::generate("runtime-test").unwrap(); + let workspace_public_key = + crate::auth::decode_public_key(&workspace_identity.public_key).unwrap(); + let workspace_fingerprint = format!( + "sha256:{}", + crate::workspace_issuer::hex_lower(&sha2::Sha256::digest(workspace_public_key)) + ); + let runtime_public_key = + crate::auth::decode_public_key(&runtime_identity.public_key).unwrap(); + let runtime_fingerprint = format!( + "sha256:{}", + crate::workspace_issuer::hex_lower(&sha2::Sha256::digest(runtime_public_key)) + ); + let verifier = WorkspaceCapabilityVerifier::new( + vec![WorkspaceIssuerTrustRecord { + workspace_id: "workspace-a".to_string(), + backend_url: "https://backend.test".to_string(), + key_id: "workspace-key".to_string(), + algorithm: "ed25519".to_string(), + public_key: workspace_identity.public_key.clone(), + public_key_fingerprint: workspace_fingerprint, + identity_revision: 1, + trust_generation: 1, + state: WorkspaceIssuerTrustState::Active, + registered_at_unix: 1, + updated_at_unix: 1, + }], + Arc::new(InMemoryWorkspaceClaimReplayProtection::default()), + ) + .unwrap(); + let app = runtime_http_router_with_workspace_auth( + runtime, + None, + legacy_auth, + WorkspaceRuntimeHttpAuth { + verifier, + signer: RuntimeVerificationSigner::from_identity(&runtime_identity).unwrap(), + verifications: Arc::new(InMemoryWorkspaceRuntimeVerificationAuthority::default()), + }, + ); + let challenge = WorkspaceRuntimeVerificationChallenge { + challenge_id: "challenge-1".to_string(), + workspace_id: "workspace-a".to_string(), + runtime_id: "runtime-test".to_string(), + binding_revision: 4, + workspace_key_id: "workspace-key".to_string(), + workspace_identity_revision: 1, + workspace_trust_generation: 1, + runtime_public_key_fingerprint: runtime_fingerprint, + runtime_identity_revision: 1, + workspace_nonce: "workspace-nonce".to_string(), + expires_at: unix_now_i64() + 60, + }; + let body = serde_json::to_vec(&challenge).unwrap(); + let claims = WorkspaceCapabilityClaims { + issuer: "https://backend.test".to_string(), + issuer_workspace_id: "workspace-a".to_string(), + issuer_key_id: "workspace-key".to_string(), + issuer_identity_revision: 1, + trust_generation: 1, + binding_revision: 4, + runtime_id: "runtime-test".to_string(), + worker_id: None, + operation: WORKSPACE_VERIFICATION_OPERATION.to_string(), + method: "POST".to_string(), + path_and_query: WORKSPACE_VERIFICATION_CHALLENGE_PATH.to_string(), + body_digest: workspace_request_body_digest(&body), + iat: unix_now_i64(), + exp: challenge.expires_at, + jti: "challenge-token".to_string(), + }; + let token = + issue_workspace_capability_token(&workspace_identity.signing_key().unwrap(), &claims) + .unwrap(); + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::POST) + .uri(WORKSPACE_VERIFICATION_CHALLENGE_PATH) + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + status, + StatusCode::OK, + "{}", + String::from_utf8_lossy(&response_body) + ); + let verification_response = + serde_json::from_slice::(&response_body).unwrap(); + verify_runtime_verification_response( + &verification_response, + &challenge, + &runtime_identity.public_key, + unix_now_i64(), + ) + .unwrap(); + + let acknowledgement = WorkspaceRuntimeVerificationAcknowledgement { + challenge_id: verification_response.challenge_id.clone(), + workspace_id: verification_response.workspace_id.clone(), + runtime_id: verification_response.runtime_id.clone(), + binding_revision: verification_response.binding_revision, + workspace_key_id: verification_response.workspace_key_id.clone(), + workspace_identity_revision: verification_response.workspace_identity_revision, + workspace_trust_generation: verification_response.workspace_trust_generation, + runtime_public_key_fingerprint: verification_response + .runtime_public_key_fingerprint + .clone(), + runtime_identity_revision: verification_response.runtime_identity_revision, + workspace_nonce: verification_response.workspace_nonce.clone(), + runtime_nonce: verification_response.runtime_nonce.clone(), + response_digest: workspace_request_body_digest(&response_body), + response: verification_response.clone(), + expires_at: verification_response.expires_at, + }; + let acknowledgement_body = serde_json::to_vec(&acknowledgement).unwrap(); + let acknowledgement_claims = WorkspaceCapabilityClaims { + path_and_query: WORKSPACE_VERIFICATION_ACK_PATH.to_string(), + body_digest: workspace_request_body_digest(&acknowledgement_body), + jti: "ack-token".to_string(), + ..claims + }; + let acknowledgement_token = issue_workspace_capability_token( + &workspace_identity.signing_key().unwrap(), + &acknowledgement_claims, + ) + .unwrap(); + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::POST) + .uri(WORKSPACE_VERIFICATION_ACK_PATH) + .header( + header::AUTHORIZATION, + format!("Bearer {acknowledgement_token}"), + ) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(acknowledgement_body)) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + + let ping_claims = WorkspaceCapabilityClaims { + operation: RUNTIME_PING_PERMISSION.to_string(), + method: "GET".to_string(), + path_and_query: "/v1/ping".to_string(), + body_digest: workspace_request_body_digest(&[]), + jti: "ping-token".to_string(), + ..acknowledgement_claims + }; + let ping_token = issue_workspace_capability_token( + &workspace_identity.signing_key().unwrap(), + &ping_claims, + ) + .unwrap(); + let response = app + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(header::AUTHORIZATION, format!("Bearer {ping_token}")) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + #[tokio::test] async fn ping_requires_scoped_permission_and_returns_versioned_identity() { let runtime = Runtime::new_memory(); @@ -2786,6 +3319,7 @@ mod tests { .expect("runtime"), local_token: Some(Arc::from("token")), auth: None, + workspace_auth: None, workdir_sessions: Arc::new(Mutex::new(HashMap::from([( "session-1".to_string(), RuntimeHttpWorkdirSession { diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index c6b7e2b1..081f8c1e 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -22,13 +22,16 @@ use worker_runtime::error::RuntimeError; use worker_runtime::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions}; use worker_runtime::http_server::{ RuntimeHttpServerConfig, RuntimeHttpServerError, RuntimeHttpStoreSelection, + WorkspaceRuntimeHttpAuth, }; use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend}; use worker_runtime::working_directory::RuntimeGitCacheMaterializer; use worker_runtime::workspace_issuer::{ - MAX_WORKSPACE_ISSUER_TRUST_RECORDS, WorkspaceIssuerTrustError, WorkspaceIssuerTrustMutation, - WorkspaceIssuerTrustRecord, add_workspace_issuer_trust, replace_workspace_issuer_trust, - revoke_workspace_issuer_trust, validate_workspace_issuer_trust_records, + FileWorkspaceClaimReplayProtection, FileWorkspaceRuntimeVerificationAuthority, + MAX_WORKSPACE_ISSUER_TRUST_RECORDS, RuntimeVerificationSigner, WorkspaceCapabilityVerifier, + WorkspaceIssuerTrustError, WorkspaceIssuerTrustMutation, WorkspaceIssuerTrustRecord, + add_workspace_issuer_trust, replace_workspace_issuer_trust, revoke_workspace_issuer_trust, + validate_workspace_issuer_trust_records, }; use worker_runtime::{Runtime, RuntimeOptions}; @@ -88,6 +91,7 @@ fn run() -> Result<(), ProcessError> { }; init_serve_tracing(); config.http.auth = load_runtime_http_auth(&config)?; + let workspace_http_auth = load_workspace_runtime_http_auth(&config)?; let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -99,14 +103,25 @@ fn run() -> Result<(), ProcessError> { eprintln!( "yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser" ); - worker_runtime::http_server::serve_runtime_http_with_auth( - worker_runtime, - listener, - config.http.local_token, - config.http.auth, - ) - .await - .map_err(ProcessError::from) + let server = if let Some(workspace_auth) = workspace_http_auth { + worker_runtime::http_server::serve_runtime_http_with_workspace_auth( + worker_runtime, + listener, + config.http.local_token, + config.http.auth, + workspace_auth, + ) + .await + } else { + worker_runtime::http_server::serve_runtime_http_with_auth( + worker_runtime, + listener, + config.http.local_token, + config.http.auth, + ) + .await + }; + server.map_err(ProcessError::from) })?; Ok(()) } @@ -880,6 +895,35 @@ fn write_secret_file(path: &Path, contents: &[u8]) -> Result<(), ProcessError> { write_result } +fn load_workspace_runtime_http_auth( + config: &ProcessConfig, +) -> Result, ProcessError> { + let auth = read_runtime_auth_file(&runtime_auth_path(config))?; + let Some(identity) = auth.identity else { + return Ok(None); + }; + if auth.workspace_issuers.is_empty() { + return Ok(None); + } + let replay_path = runtime_auth_path(config).with_extension("workspace-replay.json"); + let verifier = WorkspaceCapabilityVerifier::new( + auth.workspace_issuers, + Arc::new(FileWorkspaceClaimReplayProtection::new(replay_path)), + ) + .map_err(|error| ProcessError::auth(format!("invalid Workspace issuer trust: {error}")))?; + let signer = RuntimeVerificationSigner::from_identity(&identity) + .map_err(|error| ProcessError::auth(format!("invalid Runtime identity: {error}")))?; + let verifications_path = + runtime_auth_path(config).with_extension("workspace-verifications.json"); + Ok(Some(WorkspaceRuntimeHttpAuth { + verifier, + signer, + verifications: Arc::new(FileWorkspaceRuntimeVerificationAuthority::new( + verifications_path, + )), + })) +} + fn load_runtime_http_auth( config: &ProcessConfig, ) -> Result, ProcessError> { diff --git a/crates/worker-runtime/src/workspace_issuer.rs b/crates/worker-runtime/src/workspace_issuer.rs index ce5880f7..c73ce9e4 100644 --- a/crates/worker-runtime/src/workspace_issuer.rs +++ b/crates/worker-runtime/src/workspace_issuer.rs @@ -1,7 +1,11 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fmt; +use std::fs; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; use ring::signature::Ed25519KeyPair; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -16,6 +20,7 @@ const MAX_TOKEN_BYTES: usize = 16 * 1024; const MAX_ID_BYTES: usize = 256; const MAX_ISSUER_BYTES: usize = 2 * 1024; const MAX_OPERATION_BYTES: usize = 128; +const MAX_PATH_AND_QUERY_BYTES: usize = 4 * 1024; pub const MAX_WORKSPACE_ISSUER_TRUST_RECORDS: usize = 4_096; const MAX_REPLAY_ENTRIES: usize = 65_536; const MAX_TOKEN_LIFETIME_SECONDS: i64 = 300; @@ -266,6 +271,474 @@ fn validate_id(value: &str) -> Result<(), WorkspaceIssuerTrustError> { Ok(()) } +pub const WORKSPACE_VERIFICATION_CHALLENGE_PATH: &str = + "/v1/workspace-runtime-verification/challenge"; +pub const WORKSPACE_VERIFICATION_ACK_PATH: &str = + "/v1/workspace-runtime-verification/acknowledgement"; +pub const WORKSPACE_VERIFICATION_OPERATION: &str = "workspace.runtime.verify"; +const RUNTIME_VERIFICATION_TOKEN_PREFIX: &str = "yoi-runtime-verification-v1"; +const RUNTIME_VERIFICATION_SIGNING_INPUT_PREFIX: &str = "yoi.runtime.verification.v1."; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkspaceRuntimeVerificationChallenge { + pub challenge_id: String, + pub workspace_id: String, + pub runtime_id: String, + pub binding_revision: u64, + pub workspace_key_id: String, + pub workspace_identity_revision: u64, + pub workspace_trust_generation: u64, + pub runtime_public_key_fingerprint: String, + pub runtime_identity_revision: u64, + pub workspace_nonce: String, + pub expires_at: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkspaceRuntimeVerificationResponse { + pub challenge_id: String, + pub workspace_id: String, + pub runtime_id: String, + pub binding_revision: u64, + pub workspace_key_id: String, + pub workspace_identity_revision: u64, + pub workspace_trust_generation: u64, + pub runtime_public_key_fingerprint: String, + pub runtime_identity_revision: u64, + pub workspace_nonce: String, + pub runtime_nonce: String, + pub expires_at: i64, + pub response_proof: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkspaceRuntimeVerificationAcknowledgement { + pub challenge_id: String, + pub workspace_id: String, + pub runtime_id: String, + pub binding_revision: u64, + pub workspace_key_id: String, + pub workspace_identity_revision: u64, + pub workspace_trust_generation: u64, + pub runtime_public_key_fingerprint: String, + pub runtime_identity_revision: u64, + pub workspace_nonce: String, + pub runtime_nonce: String, + pub response_digest: String, + pub response: WorkspaceRuntimeVerificationResponse, + pub expires_at: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkspaceRuntimeVerificationReceipt { + pub challenge_id: String, + pub workspace_id: String, + pub runtime_id: String, + pub binding_revision: u64, + pub accepted_at: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkspaceRuntimeVerificationRecord { + pub workspace_id: String, + pub runtime_id: String, + pub binding_revision: u64, + pub workspace_key_id: String, + pub workspace_identity_revision: u64, + pub workspace_trust_generation: u64, + pub runtime_public_key_fingerprint: String, + pub runtime_identity_revision: u64, + pub verified_at: i64, +} + +pub trait WorkspaceRuntimeVerificationAuthority: fmt::Debug + Send + Sync { + fn get( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result, WorkspaceCapabilityVerificationError>; + fn record( + &self, + record: WorkspaceRuntimeVerificationRecord, + ) -> Result<(), WorkspaceCapabilityVerificationError>; +} + +#[derive(Debug, Default)] +pub struct InMemoryWorkspaceRuntimeVerificationAuthority { + records: Mutex>, +} + +impl WorkspaceRuntimeVerificationAuthority for InMemoryWorkspaceRuntimeVerificationAuthority { + fn get( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result, WorkspaceCapabilityVerificationError> + { + Ok(self + .records + .lock() + .map_err(|_| WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable)? + .get(&(workspace_id.to_string(), runtime_id.to_string())) + .cloned()) + } + + fn record( + &self, + record: WorkspaceRuntimeVerificationRecord, + ) -> Result<(), WorkspaceCapabilityVerificationError> { + validate_verification_record(&record)?; + self.records + .lock() + .map_err(|_| WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable)? + .insert( + (record.workspace_id.clone(), record.runtime_id.clone()), + record, + ); + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub struct FileWorkspaceRuntimeVerificationAuthority { + path: PathBuf, + lock: Arc>, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceRuntimeVerificationDocument { + version: u32, + records: Vec, +} + +impl FileWorkspaceRuntimeVerificationAuthority { + pub fn new(path: impl Into) -> Self { + Self { + path: path.into(), + lock: Arc::new(Mutex::new(())), + } + } + + fn read( + &self, + ) -> Result { + match fs::read(&self.path) { + Ok(bytes) => { + let document: WorkspaceRuntimeVerificationDocument = serde_json::from_slice(&bytes) + .map_err(|_| { + WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable + })?; + if document.version != 1 + || document.records.len() > MAX_WORKSPACE_ISSUER_TRUST_RECORDS + { + return Err( + WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable, + ); + } + let mut identities = HashSet::new(); + for record in &document.records { + validate_verification_record(record)?; + if !identities + .insert((record.workspace_id.as_str(), record.runtime_id.as_str())) + { + return Err( + WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable, + ); + } + } + Ok(document) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + Ok(WorkspaceRuntimeVerificationDocument { + version: 1, + records: Vec::new(), + }) + } + Err(_) => Err(WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable), + } + } + + fn write( + &self, + document: &WorkspaceRuntimeVerificationDocument, + ) -> Result<(), WorkspaceCapabilityVerificationError> { + let parent = self.path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent) + .map_err(|_| WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable)?; + let temporary = parent.join(format!( + ".workspace-runtime-verification-{}.tmp", + uuid::Uuid::now_v7() + )); + let bytes = serde_json::to_vec(document) + .map_err(|_| WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable)?; + fs::write(&temporary, bytes) + .map_err(|_| WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(&temporary, fs::Permissions::from_mode(0o600)).map_err(|_| { + WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable + })?; + } + fs::rename(&temporary, &self.path).map_err(|_| { + let _ = fs::remove_file(&temporary); + WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable + })?; + Ok(()) + } +} + +impl WorkspaceRuntimeVerificationAuthority for FileWorkspaceRuntimeVerificationAuthority { + fn get( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result, WorkspaceCapabilityVerificationError> + { + let _guard = self + .lock + .lock() + .map_err(|_| WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable)?; + Ok(self + .read()? + .records + .into_iter() + .find(|record| record.workspace_id == workspace_id && record.runtime_id == runtime_id)) + } + + fn record( + &self, + record: WorkspaceRuntimeVerificationRecord, + ) -> Result<(), WorkspaceCapabilityVerificationError> { + validate_verification_record(&record)?; + let _guard = self + .lock + .lock() + .map_err(|_| WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable)?; + let mut document = self.read()?; + document.records.retain(|current| { + current.workspace_id != record.workspace_id || current.runtime_id != record.runtime_id + }); + document.records.push(record); + self.write(&document) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RuntimeVerificationResponseClaims { + response: WorkspaceRuntimeVerificationResponseUnsigned, + method: String, + path_and_query: String, + body_digest: String, + iat: i64, + exp: i64, + jti: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceRuntimeVerificationResponseUnsigned { + challenge_id: String, + workspace_id: String, + runtime_id: String, + binding_revision: u64, + workspace_key_id: String, + workspace_identity_revision: u64, + workspace_trust_generation: u64, + runtime_public_key_fingerprint: String, + runtime_identity_revision: u64, + workspace_nonce: String, + runtime_nonce: String, + expires_at: i64, +} + +impl WorkspaceRuntimeVerificationResponse { + fn unsigned(&self) -> WorkspaceRuntimeVerificationResponseUnsigned { + WorkspaceRuntimeVerificationResponseUnsigned { + challenge_id: self.challenge_id.clone(), + workspace_id: self.workspace_id.clone(), + runtime_id: self.runtime_id.clone(), + binding_revision: self.binding_revision, + workspace_key_id: self.workspace_key_id.clone(), + workspace_identity_revision: self.workspace_identity_revision, + workspace_trust_generation: self.workspace_trust_generation, + runtime_public_key_fingerprint: self.runtime_public_key_fingerprint.clone(), + runtime_identity_revision: self.runtime_identity_revision, + workspace_nonce: self.workspace_nonce.clone(), + runtime_nonce: self.runtime_nonce.clone(), + expires_at: self.expires_at, + } + } +} + +#[derive(Clone)] +pub struct RuntimeVerificationSigner { + runtime_id: String, + public_key: String, + public_key_fingerprint: String, + signing_key: Arc, +} + +impl fmt::Debug for RuntimeVerificationSigner { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RuntimeVerificationSigner") + .field("runtime_id", &self.runtime_id) + .field("public_key_fingerprint", &self.public_key_fingerprint) + .finish_non_exhaustive() + } +} + +impl RuntimeVerificationSigner { + pub fn from_identity( + identity: &crate::auth::RuntimeIdentityMaterial, + ) -> Result { + let public_key = crate::auth::decode_public_key(&identity.public_key) + .map_err(|_| WorkspaceCapabilityVerificationError::TrustRecordCorrupt)?; + let fingerprint = format!("sha256:{}", hex_lower(&Sha256::digest(public_key))); + Ok(Self { + runtime_id: identity.identity_id.clone(), + public_key: identity.public_key.clone(), + public_key_fingerprint: fingerprint, + signing_key: Arc::new( + identity + .signing_key() + .map_err(|_| WorkspaceCapabilityVerificationError::TrustRecordCorrupt)?, + ), + }) + } + + pub fn runtime_id(&self) -> &str { + &self.runtime_id + } + + pub fn public_key_fingerprint(&self) -> &str { + &self.public_key_fingerprint + } + + pub fn public_key(&self) -> &str { + &self.public_key + } + + pub fn sign_response( + &self, + challenge: &WorkspaceRuntimeVerificationChallenge, + runtime_nonce: String, + now_unix: i64, + ) -> Result { + validate_verification_challenge(challenge)?; + if challenge.runtime_id != self.runtime_id + || challenge.runtime_public_key_fingerprint != self.public_key_fingerprint + || challenge.runtime_identity_revision == 0 + || challenge.expires_at <= now_unix + { + return Err(WorkspaceCapabilityVerificationError::VerificationChallengeMismatch); + } + let unsigned = WorkspaceRuntimeVerificationResponseUnsigned { + challenge_id: challenge.challenge_id.clone(), + workspace_id: challenge.workspace_id.clone(), + runtime_id: challenge.runtime_id.clone(), + binding_revision: challenge.binding_revision, + workspace_key_id: challenge.workspace_key_id.clone(), + workspace_identity_revision: challenge.workspace_identity_revision, + workspace_trust_generation: challenge.workspace_trust_generation, + runtime_public_key_fingerprint: challenge.runtime_public_key_fingerprint.clone(), + runtime_identity_revision: challenge.runtime_identity_revision, + workspace_nonce: challenge.workspace_nonce.clone(), + runtime_nonce, + expires_at: challenge.expires_at, + }; + let claims = RuntimeVerificationResponseClaims { + response: unsigned.clone(), + method: "POST".to_string(), + path_and_query: WORKSPACE_VERIFICATION_CHALLENGE_PATH.to_string(), + body_digest: workspace_request_body_digest( + &serde_json::to_vec(challenge) + .map_err(|_| WorkspaceCapabilityVerificationError::MalformedClaims)?, + ), + iat: now_unix, + exp: challenge.expires_at, + jti: uuid::Uuid::now_v7().to_string(), + }; + let proof = crate::auth::sign_json_token( + RUNTIME_VERIFICATION_TOKEN_PREFIX, + RUNTIME_VERIFICATION_SIGNING_INPUT_PREFIX, + self.signing_key.as_ref(), + &claims, + ) + .map_err(|_| WorkspaceCapabilityVerificationError::MalformedClaims)?; + Ok(WorkspaceRuntimeVerificationResponse { + challenge_id: unsigned.challenge_id, + workspace_id: unsigned.workspace_id, + runtime_id: unsigned.runtime_id, + binding_revision: unsigned.binding_revision, + workspace_key_id: unsigned.workspace_key_id, + workspace_identity_revision: unsigned.workspace_identity_revision, + workspace_trust_generation: unsigned.workspace_trust_generation, + runtime_public_key_fingerprint: unsigned.runtime_public_key_fingerprint, + runtime_identity_revision: unsigned.runtime_identity_revision, + workspace_nonce: unsigned.workspace_nonce, + runtime_nonce: unsigned.runtime_nonce, + expires_at: unsigned.expires_at, + response_proof: proof, + }) + } +} + +pub fn verify_runtime_verification_response( + response: &WorkspaceRuntimeVerificationResponse, + challenge: &WorkspaceRuntimeVerificationChallenge, + runtime_public_key: &str, + now_unix: i64, +) -> Result<(), WorkspaceCapabilityVerificationError> { + validate_verification_challenge(challenge)?; + let signed = crate::auth::decode_signed_json_token::( + &response.response_proof, + RUNTIME_VERIFICATION_TOKEN_PREFIX, + ) + .map_err(|_| WorkspaceCapabilityVerificationError::MalformedToken)?; + crate::auth::verify_signed_json_token( + RUNTIME_VERIFICATION_SIGNING_INPUT_PREFIX, + &signed.payload, + &signed.signature, + runtime_public_key, + ) + .map_err(|_| WorkspaceCapabilityVerificationError::InvalidSignature)?; + let expected_body_digest = workspace_request_body_digest( + &serde_json::to_vec(challenge) + .map_err(|_| WorkspaceCapabilityVerificationError::MalformedClaims)?, + ); + if signed.claims.response != response.unsigned() + || signed.claims.method != "POST" + || signed.claims.path_and_query != WORKSPACE_VERIFICATION_CHALLENGE_PATH + || signed.claims.body_digest != expected_body_digest + || signed.claims.exp != response.expires_at + || signed.claims.iat > now_unix.saturating_add(MAX_CLOCK_SKEW_SECONDS) + || signed.claims.exp <= now_unix + || response.challenge_id != challenge.challenge_id + || response.workspace_id != challenge.workspace_id + || response.runtime_id != challenge.runtime_id + || response.binding_revision != challenge.binding_revision + || response.workspace_key_id != challenge.workspace_key_id + || response.workspace_identity_revision != challenge.workspace_identity_revision + || response.workspace_trust_generation != challenge.workspace_trust_generation + || response.runtime_public_key_fingerprint != challenge.runtime_public_key_fingerprint + || response.runtime_identity_revision != challenge.runtime_identity_revision + || response.workspace_nonce != challenge.workspace_nonce + || response.runtime_nonce.is_empty() + { + return Err(WorkspaceCapabilityVerificationError::VerificationChallengeMismatch); + } + Ok(()) +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct WorkspaceCapabilityClaims { @@ -279,6 +752,8 @@ pub struct WorkspaceCapabilityClaims { #[serde(skip_serializing_if = "Option::is_none")] pub worker_id: Option, pub operation: String, + pub method: String, + pub path_and_query: String, pub body_digest: String, pub iat: i64, pub exp: i64, @@ -292,6 +767,8 @@ pub struct WorkspaceCapabilityExpectation<'a> { pub runtime_id: &'a str, pub worker_id: Option<&'a str>, pub operation: &'a str, + pub method: &'a str, + pub path_and_query: &'a str, pub body_digest: &'a str, pub now_unix: i64, } @@ -307,6 +784,8 @@ pub struct VerifiedWorkspaceCapability { pub runtime_id: String, pub worker_id: Option, pub operation: String, + pub method: String, + pub path_and_query: String, pub token_id: String, pub expires_at: i64, } @@ -357,6 +836,123 @@ impl WorkspaceClaimReplayProtection for InMemoryWorkspaceClaimReplayProtection { } } +#[derive(Clone, Debug)] +pub struct FileWorkspaceClaimReplayProtection { + path: PathBuf, + lock: Arc>, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceClaimReplayDocument { + version: u32, + entries: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceClaimReplayEntry { + workspace_id: String, + trust_generation: u64, + token_id: String, + expires_at: i64, +} + +impl FileWorkspaceClaimReplayProtection { + pub fn new(path: impl Into) -> Self { + Self { + path: path.into(), + lock: Arc::new(Mutex::new(())), + } + } + + fn read(&self) -> Result { + match fs::read(&self.path) { + Ok(bytes) => { + let document: WorkspaceClaimReplayDocument = serde_json::from_slice(&bytes) + .map_err(|_| { + WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable + })?; + if document.version != 1 || document.entries.len() > MAX_REPLAY_ENTRIES { + return Err(WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable); + } + Ok(document) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + Ok(WorkspaceClaimReplayDocument { + version: 1, + entries: Vec::new(), + }) + } + Err(_) => Err(WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable), + } + } + + fn write( + &self, + document: &WorkspaceClaimReplayDocument, + ) -> Result<(), WorkspaceCapabilityVerificationError> { + let parent = self.path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent) + .map_err(|_| WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable)?; + let bytes = serde_json::to_vec(document) + .map_err(|_| WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable)?; + let temporary = parent.join(format!( + ".workspace-claim-replay-{}.tmp", + uuid::Uuid::now_v7() + )); + fs::write(&temporary, bytes) + .map_err(|_| WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(&temporary, fs::Permissions::from_mode(0o600)) + .map_err(|_| WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable)?; + } + fs::rename(&temporary, &self.path).map_err(|_| { + let _ = fs::remove_file(&temporary); + WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable + })?; + Ok(()) + } +} + +impl WorkspaceClaimReplayProtection for FileWorkspaceClaimReplayProtection { + fn consume_once( + &self, + workspace_id: &str, + trust_generation: u64, + token_id: &str, + expires_at: i64, + now_unix: i64, + ) -> Result { + let _guard = self + .lock + .lock() + .map_err(|_| WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable)?; + let mut document = self.read()?; + document.entries.retain(|entry| entry.expires_at > now_unix); + if document.entries.iter().any(|entry| { + entry.workspace_id == workspace_id + && entry.trust_generation == trust_generation + && entry.token_id == token_id + }) { + return Ok(false); + } + if document.entries.len() >= MAX_REPLAY_ENTRIES { + return Err(WorkspaceCapabilityVerificationError::ReplayAuthorityUnavailable); + } + document.entries.push(WorkspaceClaimReplayEntry { + workspace_id: workspace_id.to_string(), + trust_generation, + token_id: token_id.to_string(), + expires_at, + }); + self.write(&document)?; + Ok(true) + } +} + #[derive(Clone)] pub struct WorkspaceCapabilityVerifier { records: Arc<[WorkspaceIssuerTrustRecord]>, @@ -387,6 +983,12 @@ impl WorkspaceCapabilityVerifier { }) } + pub fn has_active_workspace_issuer(&self, workspace_id: &str) -> bool { + self.records.iter().any(|record| { + record.workspace_id == workspace_id && record.state == WorkspaceIssuerTrustState::Active + }) + } + pub fn verify( &self, token: &str, @@ -452,6 +1054,12 @@ impl WorkspaceCapabilityVerifier { if claims.operation != expected.operation { return Err(WorkspaceCapabilityVerificationError::WrongOperation); } + if claims.method != expected.method { + return Err(WorkspaceCapabilityVerificationError::WrongMethod); + } + if claims.path_and_query != expected.path_and_query { + return Err(WorkspaceCapabilityVerificationError::WrongPathAndQuery); + } if claims.body_digest != expected.body_digest { return Err(WorkspaceCapabilityVerificationError::WrongBodyDigest); } @@ -486,6 +1094,8 @@ impl WorkspaceCapabilityVerifier { runtime_id: claims.runtime_id, worker_id: claims.worker_id, operation: claims.operation, + method: claims.method, + path_and_query: claims.path_and_query, token_id: claims.jti, expires_at: claims.exp, }) @@ -534,6 +1144,10 @@ pub enum WorkspaceCapabilityVerificationError { WrongWorker, #[error("Workspace capability does not authorize this operation")] WrongOperation, + #[error("Workspace capability does not bind this HTTP method")] + WrongMethod, + #[error("Workspace capability does not bind this path and query")] + WrongPathAndQuery, #[error("Workspace capability does not bind this request body")] WrongBodyDigest, #[error("Workspace capability has expired")] @@ -546,20 +1160,73 @@ pub enum WorkspaceCapabilityVerificationError { Replay, #[error("Workspace capability replay authority is unavailable")] ReplayAuthorityUnavailable, + #[error("Workspace Runtime verification authority is unavailable")] + VerificationAuthorityUnavailable, + #[error( + "Workspace Runtime verification challenge or response does not match current authority" + )] + VerificationChallengeMismatch, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkspaceCapabilitySigningInput { + payload: String, + bytes: Vec, +} + +impl WorkspaceCapabilitySigningInput { + pub fn bytes(&self) -> &[u8] { + &self.bytes + } +} + +pub fn workspace_capability_signing_input( + claims: &WorkspaceCapabilityClaims, +) -> Result { + validate_claim_shape(claims)?; + let encoded = serde_json::to_vec(claims) + .map_err(|_| WorkspaceCapabilityVerificationError::MalformedClaims)?; + let payload = URL_SAFE_NO_PAD.encode(encoded); + let bytes = format!("{WORKSPACE_SIGNING_INPUT_PREFIX}{payload}").into_bytes(); + Ok(WorkspaceCapabilitySigningInput { payload, bytes }) +} + +pub fn assemble_workspace_capability_token( + input: WorkspaceCapabilitySigningInput, + signature: &[u8], +) -> Result { + if signature.len() != 64 { + return Err(WorkspaceCapabilityVerificationError::InvalidSignature); + } + Ok(format!( + "{WORKSPACE_TOKEN_PREFIX}.{}.{}", + input.payload, + URL_SAFE_NO_PAD.encode(signature) + )) +} + +pub fn inspect_workspace_capability_claims( + token: &str, +) -> Result { + if token.len() > MAX_TOKEN_BYTES { + return Err(WorkspaceCapabilityVerificationError::MalformedToken); + } + let signed = crate::auth::decode_signed_json_token::( + token, + WORKSPACE_TOKEN_PREFIX, + ) + .map_err(|_| WorkspaceCapabilityVerificationError::MalformedToken)?; + validate_claim_shape(&signed.claims)?; + Ok(signed.claims) } pub fn issue_workspace_capability_token( signing_key: &Ed25519KeyPair, claims: &WorkspaceCapabilityClaims, ) -> Result { - validate_claim_shape(claims)?; - crate::auth::sign_json_token( - WORKSPACE_TOKEN_PREFIX, - WORKSPACE_SIGNING_INPUT_PREFIX, - signing_key, - claims, - ) - .map_err(|_| WorkspaceCapabilityVerificationError::MalformedClaims) + let input = workspace_capability_signing_input(claims)?; + let signature = signing_key.sign(input.bytes()); + assemble_workspace_capability_token(input, signature.as_ref()) } fn validate_trust_record( @@ -581,6 +1248,71 @@ fn validate_trust_record( Ok(()) } +fn validate_verification_record( + record: &WorkspaceRuntimeVerificationRecord, +) -> Result<(), WorkspaceCapabilityVerificationError> { + for value in [ + record.workspace_id.as_str(), + record.runtime_id.as_str(), + record.workspace_key_id.as_str(), + ] { + if value.is_empty() + || value.len() > MAX_ID_BYTES + || value.trim() != value + || !value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':') + }) + { + return Err(WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable); + } + } + if record.binding_revision == 0 + || record.workspace_identity_revision == 0 + || record.workspace_trust_generation == 0 + || record.runtime_identity_revision == 0 + || record.verified_at <= 0 + || record.runtime_public_key_fingerprint.len() > 128 + || !record.runtime_public_key_fingerprint.starts_with("sha256:") + { + return Err(WorkspaceCapabilityVerificationError::VerificationAuthorityUnavailable); + } + Ok(()) +} + +fn validate_verification_challenge( + challenge: &WorkspaceRuntimeVerificationChallenge, +) -> Result<(), WorkspaceCapabilityVerificationError> { + for value in [ + challenge.challenge_id.as_str(), + challenge.workspace_id.as_str(), + challenge.runtime_id.as_str(), + challenge.workspace_key_id.as_str(), + challenge.workspace_nonce.as_str(), + ] { + if value.is_empty() + || value.len() > MAX_ID_BYTES + || value.trim() != value + || !value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':') + }) + { + return Err(WorkspaceCapabilityVerificationError::VerificationChallengeMismatch); + } + } + if challenge.binding_revision == 0 + || challenge.workspace_identity_revision == 0 + || challenge.workspace_trust_generation == 0 + || challenge.runtime_identity_revision == 0 + || challenge.runtime_public_key_fingerprint.len() > 128 + || !challenge + .runtime_public_key_fingerprint + .starts_with("sha256:") + { + return Err(WorkspaceCapabilityVerificationError::VerificationChallengeMismatch); + } + Ok(()) +} + fn validate_claim_shape( claims: &WorkspaceCapabilityClaims, ) -> Result<(), WorkspaceCapabilityVerificationError> { @@ -641,6 +1373,17 @@ fn validate_claim_shape( { return Err(WorkspaceCapabilityVerificationError::MalformedClaims); } + if !matches!(claims.method.as_str(), "GET" | "POST" | "DELETE") { + return Err(WorkspaceCapabilityVerificationError::MalformedClaims); + } + if claims.path_and_query.is_empty() + || claims.path_and_query.len() > MAX_PATH_AND_QUERY_BYTES + || !claims.path_and_query.starts_with('/') + || claims.path_and_query.contains('#') + || claims.path_and_query.chars().any(char::is_control) + { + return Err(WorkspaceCapabilityVerificationError::MalformedClaims); + } if !is_sha256_digest(&claims.body_digest) { return Err(WorkspaceCapabilityVerificationError::InvalidBodyDigest); } @@ -655,7 +1398,7 @@ pub fn workspace_request_body_digest(body: &[u8]) -> String { crate::auth::request_body_digest(body) } -fn hex_lower(bytes: &[u8]) -> String { +pub(crate) fn hex_lower(bytes: &[u8]) -> String { const HEX: &[u8; 16] = b"0123456789abcdef"; let mut output = String::with_capacity(bytes.len() * 2); for byte in bytes { @@ -769,7 +1512,9 @@ mod tests { binding_revision: 7, runtime_id: "runtime-1".to_string(), worker_id: Some("worker-1".to_string()), - operation: "POST /v1/workers/worker-1/submit".to_string(), + operation: "worker.submit".to_string(), + method: "POST".to_string(), + path_and_query: "/v1/workers/worker-1/submit".to_string(), body_digest: workspace_request_body_digest(br#"{"content":"hello"}"#), iat: 1_000, exp: 1_060, @@ -783,7 +1528,9 @@ mod tests { binding_revision: 7, runtime_id: "runtime-1", worker_id: Some("worker-1"), - operation: "POST /v1/workers/worker-1/submit", + operation: "worker.submit", + method: "POST", + path_and_query: "/v1/workers/worker-1/submit", body_digest, now_unix: 1_001, } @@ -864,7 +1611,9 @@ mod tests { binding_revision: 7, runtime_id: "runtime-1", worker_id: Some("worker-1"), - operation: "POST /v1/workers/worker-1/submit", + operation: "worker.submit", + method: "POST", + path_and_query: "/v1/workers/worker-1/submit", body_digest: &expected_body, now_unix: 1_001, }; @@ -914,6 +1663,19 @@ mod tests { }) as ClaimsMutation, WorkspaceCapabilityVerificationError::WrongIssuer, ), + ( + "method", + (|claims: &mut WorkspaceCapabilityClaims| claims.method = "DELETE".to_string()) + as ClaimsMutation, + WorkspaceCapabilityVerificationError::WrongMethod, + ), + ( + "path_and_query", + (|claims: &mut WorkspaceCapabilityClaims| { + claims.path_and_query = "/v1/workers/worker-1/submit?retry=1".to_string() + }) as ClaimsMutation, + WorkspaceCapabilityVerificationError::WrongPathAndQuery, + ), ( "operation", (|claims: &mut WorkspaceCapabilityClaims| { @@ -1047,4 +1809,98 @@ mod tests { Ok(VerifiedRuntimeCapability::WorkspaceIssuer(_)) )); } + + #[test] + fn file_replay_protection_survives_reconstruction() { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("workspace-replay.json"); + let first = FileWorkspaceClaimReplayProtection::new(&path); + assert!( + first + .consume_once("workspace-a", 3, "token-a", 1_000, 100) + .unwrap() + ); + let restored = FileWorkspaceClaimReplayProtection::new(&path); + assert!( + !restored + .consume_once("workspace-a", 3, "token-a", 1_000, 101) + .unwrap() + ); + assert!( + restored + .consume_once("workspace-a", 4, "token-a", 1_000, 101) + .unwrap() + ); + } + + #[test] + fn file_runtime_verification_authority_survives_reconstruction() { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("workspace-verifications.json"); + let authority = FileWorkspaceRuntimeVerificationAuthority::new(&path); + let record = WorkspaceRuntimeVerificationRecord { + workspace_id: "workspace-a".to_string(), + runtime_id: "runtime-a".to_string(), + binding_revision: 7, + workspace_key_id: "WK-a".to_string(), + workspace_identity_revision: 2, + workspace_trust_generation: 3, + runtime_public_key_fingerprint: "sha256:runtime".to_string(), + runtime_identity_revision: 1, + verified_at: 100, + }; + authority.record(record.clone()).unwrap(); + let restored = FileWorkspaceRuntimeVerificationAuthority::new(&path); + assert_eq!( + restored.get("workspace-a", "runtime-a").unwrap(), + Some(record) + ); + } + + #[test] + fn verification_response_binds_the_exact_challenge_and_runtime_identity() { + let runtime_identity = RuntimeIdentityMaterial::generate("runtime-1").unwrap(); + let signer = RuntimeVerificationSigner::from_identity(&runtime_identity).unwrap(); + let public_key_fingerprint = format!( + "sha256:{}", + hex_lower(&Sha256::digest( + crate::auth::decode_public_key(&runtime_identity.public_key).unwrap() + )) + ); + let challenge = WorkspaceRuntimeVerificationChallenge { + challenge_id: "challenge-1".to_string(), + workspace_id: "workspace-a".to_string(), + runtime_id: "runtime-1".to_string(), + binding_revision: 7, + workspace_key_id: "WK-1".to_string(), + workspace_identity_revision: 2, + workspace_trust_generation: 3, + runtime_public_key_fingerprint: public_key_fingerprint, + runtime_identity_revision: 1, + workspace_nonce: "workspace-nonce".to_string(), + expires_at: 1_100, + }; + let response = signer + .sign_response(&challenge, "runtime-nonce".to_string(), 1_000) + .unwrap(); + verify_runtime_verification_response( + &response, + &challenge, + &runtime_identity.public_key, + 1_001, + ) + .unwrap(); + + let mut tampered = response.clone(); + tampered.binding_revision += 1; + assert_eq!( + verify_runtime_verification_response( + &tampered, + &challenge, + &runtime_identity.public_key, + 1_001, + ), + Err(WorkspaceCapabilityVerificationError::VerificationChallengeMismatch) + ); + } } diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index b31796c8..5280e43a 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -2,6 +2,10 @@ use crate::Error; use crate::resource_broker::BackendResourceBroker; #[cfg(test)] use crate::resource_broker::BackendResourceTarget; +use crate::store::{ + ControlPlaneStore, WorkspaceRuntimeAuthenticationMode, WorkspaceRuntimeBindingState, +}; +use crate::workspace_signing_identity::WorkspaceSigningIdentityService; use chrono::Utc; use protocol::Segment; use reqwest::blocking::{Client as BlockingHttpClient, RequestBuilder}; @@ -64,6 +68,11 @@ use worker_runtime::profile_archive::ProfileSourceArchive; use worker_runtime::retention::{ WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory, }; +use worker_runtime::workspace_issuer::{ + WorkspaceCapabilityClaims, WorkspaceRuntimeVerificationAcknowledgement, + WorkspaceRuntimeVerificationChallenge, WorkspaceRuntimeVerificationReceipt, + WorkspaceRuntimeVerificationResponse, workspace_request_body_digest, +}; pub const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime"; const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host"; @@ -818,6 +827,32 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { )) } + fn activate_workspace_authorization(&self, _binding: crate::store::WorkspaceRuntimeBinding) {} + + fn send_workspace_verification_challenge( + &self, + _challenge: &WorkspaceRuntimeVerificationChallenge, + _bearer_token: &str, + ) -> Result { + Err(RuntimePingFailure::new( + RuntimePingFailureKind::Unsupported, + "runtime_workspace_verification_unsupported", + "Workspace Runtime verification is unavailable for this Runtime provider", + )) + } + + fn send_workspace_verification_acknowledgement( + &self, + _acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement, + _bearer_token: &str, + ) -> Result { + Err(RuntimePingFailure::new( + RuntimePingFailureKind::Unsupported, + "runtime_workspace_verification_unsupported", + "Workspace Runtime verification is unavailable for this Runtime provider", + )) + } + fn list_hosts(&self, limit: usize) -> RuntimeList; fn list_workers(&self, limit: usize) -> RuntimeList; @@ -1877,6 +1912,48 @@ impl RuntimeRegistry { runtime.ping() } + pub fn activate_workspace_authorization( + &self, + runtime_id: &str, + binding: crate::store::WorkspaceRuntimeBinding, + ) -> Result<(), RuntimeRegistryError> { + self.runtime(runtime_id)? + .activate_workspace_authorization(binding); + Ok(()) + } + + pub fn send_workspace_verification_challenge( + &self, + runtime_id: &str, + challenge: &WorkspaceRuntimeVerificationChallenge, + bearer_token: &str, + ) -> Result { + let runtime = self.runtime(runtime_id).map_err(|_| { + RuntimePingFailure::new( + RuntimePingFailureKind::Configuration, + "runtime_verification_registration_unavailable", + "Registered Runtime binding is unavailable", + ) + })?; + runtime.send_workspace_verification_challenge(challenge, bearer_token) + } + + pub fn send_workspace_verification_acknowledgement( + &self, + runtime_id: &str, + acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement, + bearer_token: &str, + ) -> Result { + let runtime = self.runtime(runtime_id).map_err(|_| { + RuntimePingFailure::new( + RuntimePingFailureKind::Configuration, + "runtime_verification_registration_unavailable", + "Registered Runtime binding is unavailable", + ) + })?; + runtime.send_workspace_verification_acknowledgement(acknowledgement, bearer_token) + } + fn runtimes_snapshot(&self) -> Vec> { self.runtimes .read() @@ -2851,6 +2928,7 @@ pub struct RemoteRuntimeConfig { pub base_url: String, pub bearer_token: Option, pub auth: Option, + pub workspace_authorization: Option, pub strict_public_egress: bool, pub cached_worker_creation_available: bool, pub cached_os: String, @@ -2859,6 +2937,168 @@ pub struct RemoteRuntimeConfig { pub timeout: Duration, } +#[derive(Clone)] +pub struct WorkspaceRuntimeAuthorization { + store: Arc, + signing_identities: WorkspaceSigningIdentityService, + backend_url: String, + binding: Arc>>, +} + +impl std::fmt::Debug for WorkspaceRuntimeAuthorization { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("WorkspaceRuntimeAuthorization") + .field("backend_url", &"") + .finish_non_exhaustive() + } +} + +impl WorkspaceRuntimeAuthorization { + pub fn new( + store: Arc, + signing_identities: WorkspaceSigningIdentityService, + backend_url: impl Into, + binding: Option, + ) -> Self { + Self { + store, + signing_identities, + backend_url: backend_url.into(), + binding: Arc::new(RwLock::new(binding)), + } + } + + fn activate(&self, binding: crate::store::WorkspaceRuntimeBinding) { + if let Ok(mut current) = self.binding.write() { + *current = Some(binding); + } + } + + pub(crate) fn issue( + &self, + method: &str, + path_and_query: &str, + operation: &str, + worker_id: Option<&str>, + body: &[u8], + ) -> Result { + let binding = self + .binding + .read() + .map_err(|_| { + diagnostic( + "workspace_runtime_authorization_unavailable", + DiagnosticSeverity::Error, + "Workspace Runtime authorization is unavailable".to_string(), + ) + })? + .clone() + .ok_or_else(|| { + diagnostic( + "workspace_runtime_verification_required", + DiagnosticSeverity::Error, + "Workspace Runtime binding is not verified".to_string(), + ) + })?; + if binding.state != WorkspaceRuntimeBindingState::Verified + || binding.authentication_mode != WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity + || binding.revoked_at.is_some() + || !self + .store + .workspace_runtime_binding_matches(&binding) + .map_err(|error| { + diagnostic( + "workspace_runtime_authorization_unavailable", + DiagnosticSeverity::Error, + error.to_string(), + ) + })? + { + return Err(diagnostic( + "workspace_runtime_authorization_stale", + DiagnosticSeverity::Error, + "Workspace Runtime binding changed or was revoked".to_string(), + )); + } + let identity = self + .signing_identities + .get_validated(&binding.workspace_id) + .map_err(|error| { + diagnostic( + "workspace_runtime_authorization_unavailable", + DiagnosticSeverity::Error, + error.to_string(), + ) + })?; + let workspace_key_id = binding.workspace_key_id.as_deref().ok_or_else(|| { + diagnostic( + "workspace_runtime_authorization_invalid", + DiagnosticSeverity::Error, + "Workspace Runtime binding is missing its Workspace key".to_string(), + ) + })?; + let trust_generation = binding.workspace_key_generation.ok_or_else(|| { + diagnostic( + "workspace_runtime_authorization_invalid", + DiagnosticSeverity::Error, + "Workspace Runtime binding is missing its trust generation".to_string(), + ) + })?; + if identity.state != "active" + || identity.key_id != workspace_key_id + || identity.revision != trust_generation + || !self + .store + .workspace_runtime_verification_matches( + &binding, + identity.revision, + trust_generation, + ) + .map_err(|error| { + diagnostic( + "workspace_runtime_authorization_unavailable", + DiagnosticSeverity::Error, + error.to_string(), + ) + })? + { + return Err(diagnostic( + "workspace_runtime_authorization_stale", + DiagnosticSeverity::Error, + "Workspace signing identity no longer matches the verified binding".to_string(), + )); + } + let now = Utc::now().timestamp(); + let claims = WorkspaceCapabilityClaims { + issuer: self.backend_url.clone(), + issuer_workspace_id: binding.workspace_id.clone(), + issuer_key_id: identity.key_id, + issuer_identity_revision: identity.revision, + trust_generation, + binding_revision: binding.binding_revision, + runtime_id: binding.runtime_id.clone(), + worker_id: worker_id.map(str::to_string), + operation: operation.to_string(), + method: method.to_string(), + path_and_query: path_and_query.to_string(), + body_digest: workspace_request_body_digest(body), + iat: now, + exp: now.saturating_add(60), + jti: uuid::Uuid::now_v7().to_string(), + }; + self.signing_identities + .issue_workspace_capability(&binding.workspace_id, &claims) + .map_err(|error| { + diagnostic( + "workspace_runtime_authorization_sign_failed", + DiagnosticSeverity::Error, + error.to_string(), + ) + }) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct RemoteRuntimeAuthConfig { pub server_id: String, @@ -2903,6 +3143,7 @@ impl RemoteRuntimeConfig { base_url: base_url.into(), bearer_token, auth: None, + workspace_authorization: None, strict_public_egress: false, cached_worker_creation_available: false, cached_os: "unknown".to_string(), @@ -2943,6 +3184,7 @@ struct RemoteWorkdirAuthorization { runtime_id: String, workspace_id: String, auth: Option, + workspace_authorization: Option, fallback_bearer_token: Option, } @@ -2962,7 +3204,23 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization { } impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization { - fn bearer_token(&self) -> Result { + fn bearer_token( + &self, + method: &str, + path_and_query: &str, + body: &[u8], + ) -> Result { + if let Some(authorization) = &self.workspace_authorization { + return authorization + .issue( + method, + path_and_query, + workspace_runtime_operation(method, path_and_query), + None, + body, + ) + .map_err(|error| WorkdirError::Unavailable(error.message)); + } if let Some(auth) = self.auth.as_ref() { let claims = capability_claims( &auth.server_id, @@ -3093,6 +3351,7 @@ pub struct RemoteWorkerRuntime { workspace_id: String, bearer_token: Option, auth: Option, + workspace_authorization: Option, cached_worker_creation_available: bool, cached_os: String, cached_arch: String, @@ -3146,6 +3405,69 @@ fn remote_runtime_ping_transport_failure(error: reqwest::Error) -> RuntimePingFa ) } +fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static str { + let path = path_and_query.split('?').next().unwrap_or(path_and_query); + if path == "/v1/ping" && method == "GET" { + return RUNTIME_PING_PERMISSION; + } + if path == "/v1/workers" && method == "GET" { + return "workers:list"; + } + if path == "/v1/workers" && method == "POST" { + return "workers:create"; + } + if (path == "/v1/working-directories/repository-access" + || path == "/v1/repository-refs/observe") + && method == "POST" + { + return "workdirs:operate"; + } + if path.starts_with("/v1/workdir-sessions") + || (path.starts_with("/v1/working-directories/") && path.ends_with("/sessions")) + { + return "workdirs:operate"; + } + if path.starts_with("/v1/config-bundles") + || path.starts_with("/v1/workspace-prompt-projections") + || path.starts_with("/v1/working-directories") + { + return "workers:create"; + } + if path.ends_with("/input") + || path.ends_with("/restore") + || path.ends_with("/workspace-api") + || path.contains("/attachments") + { + return "workers:input"; + } + if path.ends_with("/stop") || path.ends_with("/cancel") { + return "workers:stop"; + } + if path == "/v1/protocol/ws" { + return "workers:list"; + } + if path.ends_with("/protocol") || path.ends_with("/protocol/ws") { + return "workers:protocol"; + } + if path.ends_with("/completions") { + return "workers:read"; + } + if path.contains("/retention/") || (path.starts_with("/v1/workers/") && method == "DELETE") { + return "workers:delete"; + } + if path.starts_with("/v1/workers/") && method == "GET" { + return "workers:read"; + } + "runtime:read" +} + +fn worker_id_from_remote_path(path_and_query: &str) -> Option { + let path = path_and_query.split('?').next().unwrap_or(path_and_query); + let rest = path.strip_prefix("/v1/workers/")?; + let worker_id = rest.split('/').next()?; + (!worker_id.is_empty()).then(|| worker_id.to_string()) +} + fn all_remote_runtime_permissions() -> Vec { [ "workers:list", @@ -3226,6 +3548,7 @@ impl RemoteWorkerRuntime { workspace_id, bearer_token: config.bearer_token, auth: config.auth, + workspace_authorization: config.workspace_authorization, cached_worker_creation_available: config.cached_worker_creation_available, cached_os: config.cached_os, cached_arch: config.cached_arch, @@ -3254,6 +3577,7 @@ impl RemoteWorkerRuntime { runtime_id: self.runtime_id.clone(), workspace_id: self.workspace_id.clone(), auth: self.auth.clone(), + workspace_authorization: self.workspace_authorization.clone(), fallback_bearer_token: self.bearer_token.clone(), }); RemoteWorkdirSession::open_with_authorization( @@ -3290,11 +3614,54 @@ impl RemoteWorkerRuntime { format!("{base}/v1/workers/{worker_id}/protocol/ws") } + fn post_bearer_json( + &self, + path: &str, + body: &T, + bearer_token: &str, + ) -> Result + where + T: Serialize + ?Sized, + U: DeserializeOwned, + { + let response = self + .http + .post(self.endpoint(path)) + .bearer_auth(bearer_token) + .json(body) + .send() + .map_err(|error| { + RuntimePingFailure::new( + RuntimePingFailureKind::NetworkUnreachable, + "runtime_workspace_verification_unreachable", + format!("Runtime verification request failed: {error}"), + ) + })?; + let status = response.status(); + if !status.is_success() { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::MalformedResponse, + "runtime_workspace_verification_rejected", + format!( + "Runtime verification request returned HTTP {}", + status.as_u16() + ), + )); + } + response.json::().map_err(|error| { + RuntimePingFailure::new( + RuntimePingFailureKind::MalformedResponse, + "runtime_workspace_verification_invalid_response", + format!("Runtime verification response was invalid: {error}"), + ) + }) + } + fn get_json(&self, path: &str) -> Result where T: DeserializeOwned + Send + 'static, { - self.send_json(path, self.http.get(self.endpoint(path))) + self.send_json(path, "GET", &[], self.http.get(self.endpoint(path))) } fn post_json(&self, path: &str, body: &B) -> Result @@ -3302,7 +3669,22 @@ impl RemoteWorkerRuntime { B: Serialize + ?Sized, T: DeserializeOwned + Send + 'static, { - self.send_json(path, self.http.post(self.endpoint(path)).json(body)) + let body = serde_json::to_vec(body).map_err(|error| { + diagnostic( + "remote_runtime_request_encode_failed", + DiagnosticSeverity::Error, + error.to_string(), + ) + })?; + self.send_json( + path, + "POST", + &body, + self.http + .post(self.endpoint(path)) + .header(CONTENT_TYPE, "application/json") + .body(body.clone()), + ) } fn post_bytes(&self, path: &str, body: &[u8]) -> Result @@ -3311,7 +3693,12 @@ impl RemoteWorkerRuntime { { self.send_json( path, - self.http.post(self.endpoint(path)).body(body.to_vec()), + "POST", + body, + self.http + .post(self.endpoint(path)) + .header(CONTENT_TYPE, "application/json") + .body(body.to_vec()), ) } @@ -3319,7 +3706,7 @@ impl RemoteWorkerRuntime { where T: DeserializeOwned + Send + 'static, { - self.send_json(path, self.http.delete(self.endpoint(path))) + self.send_json(path, "DELETE", &[], self.http.delete(self.endpoint(path))) } fn runtime_capability_token_with_permissions( @@ -3364,10 +3751,23 @@ impl RemoteWorkerRuntime { const PATH: &str = "/v1/ping"; let workspace_id = self.workspace_id.clone(); let bearer_token = self.bearer_token.clone(); - let capability_token = self.runtime_capability_token_with_permissions( - PATH, - vec![RUNTIME_PING_PERMISSION.to_string()], - ); + let capability_token = match &self.workspace_authorization { + Some(authorization) => Some( + authorization + .issue("GET", PATH, RUNTIME_PING_PERMISSION, None, &[]) + .map_err(|diagnostic| { + RuntimePingFailure::new( + RuntimePingFailureKind::Authentication, + diagnostic.code, + diagnostic.message, + ) + })?, + ), + None => self.runtime_capability_token_with_permissions( + PATH, + vec![RUNTIME_PING_PERMISSION.to_string()], + ), + }; let request = self .http .get(self.endpoint(PATH)) @@ -3432,15 +3832,33 @@ impl RemoteWorkerRuntime { }) } - fn send_json(&self, path: &str, request: RequestBuilder) -> Result + fn send_json( + &self, + path: &str, + method: &str, + body: &[u8], + request: RequestBuilder, + ) -> Result where T: DeserializeOwned + Send + 'static, { let runtime_id = self.runtime_id.clone(); + let workspace_id = self.workspace_id.clone(); let bearer_token = self.bearer_token.clone(); - let capability_token = self.runtime_capability_token(path); + let capability_token = match &self.workspace_authorization { + Some(authorization) => Some(authorization.issue( + method, + path, + workspace_runtime_operation(method, path), + worker_id_from_remote_path(path).as_deref(), + body, + )?), + None => self.runtime_capability_token(path), + }; run_blocking_http(move || { - let request = request.header(CONTENT_TYPE, "application/json"); + let request = request + .header(CONTENT_TYPE, "application/json") + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, &workspace_id); let request = if let Some(token) = capability_token.as_deref().or(bearer_token.as_deref()) { request.header(AUTHORIZATION, format!("Bearer {token}")) @@ -3649,6 +4067,36 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { self.ping_http() } + fn activate_workspace_authorization(&self, binding: crate::store::WorkspaceRuntimeBinding) { + if let Some(authorization) = &self.workspace_authorization { + authorization.activate(binding); + } + } + + fn send_workspace_verification_challenge( + &self, + challenge: &WorkspaceRuntimeVerificationChallenge, + bearer_token: &str, + ) -> Result { + self.post_bearer_json( + worker_runtime::workspace_issuer::WORKSPACE_VERIFICATION_CHALLENGE_PATH, + challenge, + bearer_token, + ) + } + + fn send_workspace_verification_acknowledgement( + &self, + acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement, + bearer_token: &str, + ) -> Result { + self.post_bearer_json( + worker_runtime::workspace_issuer::WORKSPACE_VERIFICATION_ACK_PATH, + acknowledgement, + bearer_token, + ) + } + fn list_hosts(&self, limit: usize) -> RuntimeList { if limit == 0 { return RuntimeList::new(Vec::new(), Vec::new()); @@ -3936,12 +4384,30 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { workspace_api: Some(workspace_api), memory_settings: request.resolved_memory_settings.clone(), }; + let create_body = match serde_json::to_vec(&create) { + Ok(body) => body, + Err(error) => { + return WorkerSpawnResult { + state: WorkerOperationState::Rejected, + worker: None, + acceptance_evidence: Vec::new(), + diagnostics: vec![diagnostic( + "remote_runtime_request_encode_failed", + DiagnosticSeverity::Error, + error.to_string(), + )], + }; + } + }; match self.send_json::( "/v1/workers", + "POST", + &create_body, self.http .post(self.endpoint("/v1/workers")) .timeout(REMOTE_WORKER_CREATE_TIMEOUT) - .json(&create), + .header(CONTENT_TYPE, "application/json") + .body(create_body.clone()), ) { Ok(response) => WorkerSpawnResult { state: WorkerOperationState::Accepted, diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index fb0aca3e..eca08f1f 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -462,6 +462,28 @@ CREATE TABLE workspace_runtime_bindings ( (state != 'revoked' AND revoked_at IS NULL) ) ); +CREATE TABLE workspace_runtime_verifications ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + binding_revision INTEGER NOT NULL CHECK(binding_revision > 0), + workspace_key_id TEXT NOT NULL, + workspace_identity_revision INTEGER NOT NULL CHECK(workspace_identity_revision > 0), + workspace_trust_generation INTEGER NOT NULL CHECK(workspace_trust_generation > 0), + runtime_public_key_fingerprint TEXT NOT NULL, + runtime_identity_revision INTEGER NOT NULL CHECK(runtime_identity_revision > 0), + challenge_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('pending', 'verified', 'failed')), + last_outcome TEXT NOT NULL, + verified_at TEXT, + checked_at TEXT NOT NULL, + PRIMARY KEY(workspace_id, runtime_id), + FOREIGN KEY(workspace_id, runtime_id) + REFERENCES workspace_runtime_bindings(workspace_id, runtime_id) ON DELETE CASCADE, + CHECK((state = 'verified' AND verified_at IS NOT NULL) + OR (state != 'verified' AND verified_at IS NULL)) +); +CREATE INDEX workspace_runtime_verifications_state_idx + ON workspace_runtime_verifications(workspace_id, state, checked_at DESC); CREATE TABLE workspace_runtime_binding_audit ( workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, diff --git a/crates/workspace-server/src/runtime_subscription.rs b/crates/workspace-server/src/runtime_subscription.rs index 0397d7f4..74356c1f 100644 --- a/crates/workspace-server/src/runtime_subscription.rs +++ b/crates/workspace-server/src/runtime_subscription.rs @@ -970,6 +970,12 @@ fn runtime_token( config: &RemoteRuntimeConfig, workspace_id: &str, ) -> Result, String> { + if let Some(authorization) = config.workspace_authorization.as_ref() { + return authorization + .issue("GET", "/v1/protocol/ws", "workers:list", None, &[]) + .map(Some) + .map_err(|error| error.message); + } let Some(auth) = config.auth.as_ref() else { return Ok(config.bearer_token.clone()); }; diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index a8f5b3b5..a99d465e 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -61,6 +61,12 @@ use worker_runtime::http_server::{ }; use worker_runtime::resource::{BackendResourceError, BackendResourceFetchRequest}; use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend}; +use worker_runtime::workspace_issuer::{ + WORKSPACE_VERIFICATION_ACK_PATH, WORKSPACE_VERIFICATION_CHALLENGE_PATH, + WORKSPACE_VERIFICATION_OPERATION, WorkspaceCapabilityClaims, + WorkspaceRuntimeVerificationAcknowledgement, WorkspaceRuntimeVerificationChallenge, + verify_runtime_verification_response, workspace_request_body_digest, +}; use workspace_api::{ ActorAuthMethod, AuthBootstrapUserRequest, AuthPublicConfig, AuthUserResponse, AuthenticatedUser, BrowserCreateWorkerResponse, BrowserWorkspaceOrchestratorResponse, @@ -128,8 +134,8 @@ use crate::hosts::{ WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult, WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult, WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerTicketAssignmentRequest, - WorkerWorkspaceSummary, is_disallowed_remote_runtime_address, worker_spawn_create_fingerprint, - workspace_worker_summary, + WorkerWorkspaceSummary, WorkspaceRuntimeAuthorization, is_disallowed_remote_runtime_address, + worker_spawn_create_fingerprint, workspace_worker_summary, }; use crate::identity::WorkspaceIdentity; use crate::memory_backend::execute_memory_backend_operation_with_authority; @@ -2229,23 +2235,6 @@ impl WorkspaceApi { RuntimeSubscriptionBroker::new(config.workspace_id.clone()); runtime_subscription_broker .register_embedded_runtime(embedded_runtime_id, embedded_subscription_runtime); - for remote_config in config.remote_runtime_sources.iter().cloned() { - let remote_runtime = RemoteWorkerRuntime::new( - remote_config.clone(), - config.workspace_id.clone(), - config - .backend_base_url - .clone() - .unwrap_or_else(|| "http://127.0.0.1:8787".to_string()), - ) - .map(|host| host.with_resource_broker(resource_broker.clone())) - .map_err(|err| err.into_error())?; - runtime.register(remote_runtime); - runtime_subscription_broker.register_remote_runtime(remote_config); - } - let runtime = Arc::new(runtime); - let companion = Arc::new(CompanionConsole::disabled()); - let observation_proxy = BackendObservationProxy::new(config.runtime_event_sources.clone()); let config_store = Arc::new(crate::SqliteWorkspaceStore::open( config.database_path.clone(), )?); @@ -2260,6 +2249,40 @@ impl WorkspaceApi { let signing_identities = WorkspaceSigningIdentityService::new(store.clone(), signing_materials); signing_identities.get_validated(&config.workspace_id)?; + let backend_url = config + .backend_base_url + .clone() + .unwrap_or_else(|| "http://127.0.0.1:8787".to_string()); + for mut remote_config in config.remote_runtime_sources.iter().cloned() { + let current_binding = store + .get_workspace_runtime_binding(&config.workspace_id, &remote_config.runtime_id) + .await?; + if current_binding.as_ref().is_some_and(|binding| { + binding.authentication_mode == StoredRuntimeAuthenticationMode::WorkspaceIdentity + && binding.revoked_at.is_none() + }) { + let verified_binding = current_binding + .filter(|binding| binding.state == StoredRuntimeBindingState::Verified); + remote_config.workspace_authorization = Some(WorkspaceRuntimeAuthorization::new( + store.clone(), + signing_identities.clone(), + backend_url.clone(), + verified_binding, + )); + } + let remote_runtime = RemoteWorkerRuntime::new( + remote_config.clone(), + config.workspace_id.clone(), + backend_url.clone(), + ) + .map(|host| host.with_resource_broker(resource_broker.clone())) + .map_err(|err| err.into_error())?; + runtime.register(remote_runtime); + runtime_subscription_broker.register_remote_runtime(remote_config); + } + let runtime = Arc::new(runtime); + let companion = Arc::new(CompanionConsole::disabled()); + let observation_proxy = BackendObservationProxy::new(config.runtime_event_sources.clone()); let config_schema_registry = crate::config_source::WorkspaceConfigSchemaRegistry::default() .with_provider(Arc::new( crate::profile_settings::ProfileConfigSchemaProvider, @@ -13674,6 +13697,197 @@ async fn delete_remote_runtime( Ok(StatusCode::NO_CONTENT) } +async fn perform_workspace_runtime_verification( + api: &WorkspaceApi, + runtime: Arc, + binding: &WorkspaceRuntimeBinding, +) -> std::result::Result { + if binding.authentication_mode != StoredRuntimeAuthenticationMode::WorkspaceIdentity { + return Ok(binding.clone()); + } + if binding.state == crate::store::WorkspaceRuntimeBindingState::Revoked + || binding.revoked_at.is_some() + { + return Err("Runtime binding is revoked".to_string()); + } + let backend_url = api.config.backend_base_url.as_deref().ok_or_else(|| { + "Workspace identity verification requires configured backend_base_url".to_string() + })?; + let identity = api + .signing_identities + .get_validated(&binding.workspace_id) + .map_err(|error| error.to_string())?; + if identity.state != "active" { + return Err("Workspace signing identity is not active".to_string()); + } + let workspace_key_id = binding + .workspace_key_id + .as_deref() + .ok_or_else(|| "Runtime binding is missing the Workspace key identity".to_string())?; + let workspace_trust_generation = binding + .workspace_key_generation + .ok_or_else(|| "Runtime binding is missing the Workspace trust generation".to_string())?; + if identity.key_id != workspace_key_id || identity.revision != workspace_trust_generation { + return Err( + "Runtime binding no longer matches the active Workspace or Runtime identity" + .to_string(), + ); + } + + let now = Utc::now(); + let expires_at = (now + Duration::seconds(60)).timestamp(); + let challenge = WorkspaceRuntimeVerificationChallenge { + challenge_id: Uuid::now_v7().to_string(), + workspace_id: binding.workspace_id.clone(), + runtime_id: binding.runtime_id.clone(), + binding_revision: binding.binding_revision, + workspace_key_id: workspace_key_id.to_string(), + workspace_identity_revision: identity.revision, + workspace_trust_generation, + runtime_public_key_fingerprint: binding.public_key_fingerprint.clone(), + runtime_identity_revision: 1, + workspace_nonce: Uuid::now_v7().to_string(), + expires_at, + }; + let checked_at = now.to_rfc3339_opts(SecondsFormat::Millis, true); + let pending = crate::store::WorkspaceRuntimeVerificationEvidence { + workspace_id: binding.workspace_id.clone(), + runtime_id: binding.runtime_id.clone(), + binding_revision: binding.binding_revision, + workspace_key_id: workspace_key_id.to_string(), + workspace_identity_revision: identity.revision, + workspace_trust_generation, + runtime_public_key_fingerprint: binding.public_key_fingerprint.clone(), + runtime_identity_revision: 1, + challenge_id: challenge.challenge_id.clone(), + state: "pending".to_string(), + last_outcome: "challenge_issued".to_string(), + verified_at: None, + checked_at: checked_at.clone(), + }; + api.store + .record_workspace_runtime_verification_attempt(&pending) + .await + .map_err(|error| error.to_string())?; + + let challenge_body = serde_json::to_vec(&challenge).map_err(|error| error.to_string())?; + let challenge_claims = WorkspaceCapabilityClaims { + issuer: backend_url.to_string(), + issuer_workspace_id: binding.workspace_id.clone(), + issuer_key_id: identity.key_id.clone(), + issuer_identity_revision: identity.revision, + trust_generation: workspace_trust_generation, + binding_revision: binding.binding_revision, + runtime_id: binding.runtime_id.clone(), + worker_id: None, + operation: WORKSPACE_VERIFICATION_OPERATION.to_string(), + method: "POST".to_string(), + path_and_query: WORKSPACE_VERIFICATION_CHALLENGE_PATH.to_string(), + body_digest: workspace_request_body_digest(&challenge_body), + iat: now.timestamp(), + exp: expires_at, + jti: Uuid::now_v7().to_string(), + }; + let challenge_token = api + .signing_identities + .issue_workspace_capability(&binding.workspace_id, &challenge_claims) + .map_err(|error| error.to_string())?; + let challenge_runtime = runtime.clone(); + let challenge_runtime_id = binding.runtime_id.clone(); + let challenge_request = challenge.clone(); + let response = tokio::task::spawn_blocking(move || { + challenge_runtime.send_workspace_verification_challenge( + &challenge_runtime_id, + &challenge_request, + &challenge_token, + ) + }) + .await + .map_err(|_| "Runtime verification challenge task failed".to_string())? + .map_err(|failure| failure.diagnostic.message)?; + verify_runtime_verification_response( + &response, + &challenge, + &binding.public_key, + Utc::now().timestamp(), + ) + .map_err(|error| error.to_string())?; + + let response_bytes = serde_json::to_vec(&response).map_err(|error| error.to_string())?; + let acknowledgement = WorkspaceRuntimeVerificationAcknowledgement { + challenge_id: response.challenge_id.clone(), + workspace_id: response.workspace_id.clone(), + runtime_id: response.runtime_id.clone(), + binding_revision: response.binding_revision, + workspace_key_id: response.workspace_key_id.clone(), + workspace_identity_revision: response.workspace_identity_revision, + workspace_trust_generation: response.workspace_trust_generation, + runtime_public_key_fingerprint: response.runtime_public_key_fingerprint.clone(), + runtime_identity_revision: response.runtime_identity_revision, + workspace_nonce: response.workspace_nonce.clone(), + runtime_nonce: response.runtime_nonce.clone(), + response_digest: workspace_request_body_digest(&response_bytes), + response: response.clone(), + expires_at: response.expires_at, + }; + let acknowledgement_body = + serde_json::to_vec(&acknowledgement).map_err(|error| error.to_string())?; + let acknowledgement_claims = WorkspaceCapabilityClaims { + issuer: backend_url.to_string(), + issuer_workspace_id: binding.workspace_id.clone(), + issuer_key_id: identity.key_id.clone(), + issuer_identity_revision: identity.revision, + trust_generation: workspace_trust_generation, + binding_revision: binding.binding_revision, + runtime_id: binding.runtime_id.clone(), + worker_id: None, + operation: WORKSPACE_VERIFICATION_OPERATION.to_string(), + method: "POST".to_string(), + path_and_query: WORKSPACE_VERIFICATION_ACK_PATH.to_string(), + body_digest: workspace_request_body_digest(&acknowledgement_body), + iat: Utc::now().timestamp(), + exp: expires_at, + jti: Uuid::now_v7().to_string(), + }; + let acknowledgement_token = api + .signing_identities + .issue_workspace_capability(&binding.workspace_id, &acknowledgement_claims) + .map_err(|error| error.to_string())?; + let acknowledgement_runtime = runtime; + let acknowledgement_runtime_id = binding.runtime_id.clone(); + let acknowledgement_request = acknowledgement.clone(); + let receipt = tokio::task::spawn_blocking(move || { + acknowledgement_runtime.send_workspace_verification_acknowledgement( + &acknowledgement_runtime_id, + &acknowledgement_request, + &acknowledgement_token, + ) + }) + .await + .map_err(|_| "Runtime verification acknowledgement task failed".to_string())? + .map_err(|failure| failure.diagnostic.message)?; + if receipt.challenge_id != challenge.challenge_id + || receipt.workspace_id != binding.workspace_id + || receipt.runtime_id != binding.runtime_id + || receipt.binding_revision != binding.binding_revision + { + return Err("Runtime verification acknowledgement receipt mismatched".to_string()); + } + + let verified_at = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true); + let verified = crate::store::WorkspaceRuntimeVerificationEvidence { + state: "verified".to_string(), + last_outcome: "verified".to_string(), + verified_at: Some(verified_at.clone()), + checked_at: verified_at, + ..pending + }; + api.store + .complete_workspace_runtime_verification(&verified) + .await + .map_err(|error| error.to_string()) +} + async fn test_runtime_connection( State(api): State, AxumPath(runtime_id): AxumPath, @@ -13686,12 +13900,61 @@ async fn test_runtime_connection( } .into()); } - api.store + let binding = api + .store .get_workspace_runtime_binding(api.workspace_id(), &runtime_id) .await? .filter(|binding| binding.revoked_at.is_none()) .ok_or_else(|| Error::UnknownRuntime(runtime_id.clone()))?; + if binding.authentication_mode == StoredRuntimeAuthenticationMode::WorkspaceIdentity { + match perform_workspace_runtime_verification(&api, api.runtime.clone(), &binding).await { + Ok(verified_binding) => { + api.runtime_binding_expectations + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + ( + verified_binding.workspace_id.clone(), + verified_binding.runtime_id.clone(), + ), + verified_binding.clone(), + ); + api.runtime + .activate_workspace_authorization(&runtime_id, verified_binding.clone()) + .map_err(|error| Error::Store(format!("{error:?}")))?; + } + Err(message) => { + if let Ok(Some(mut evidence)) = api + .store + .get_workspace_runtime_verification(api.workspace_id(), &runtime_id) + .await + { + evidence.state = "failed".to_string(); + evidence.last_outcome = "verification_failed".to_string(); + evidence.verified_at = None; + evidence.checked_at = Utc::now().to_rfc3339(); + let _ = api + .store + .record_workspace_runtime_verification_attempt(&evidence) + .await; + } + return Ok(Json(runtime_connection_test_failure( + api.workspace_id(), + &runtime_id, + Utc::now().to_rfc3339(), + RuntimeConnectionTestFailureKind::Authentication, + None, + RuntimeDiagnostic::new( + "runtime_workspace_verification_failed", + "error", + message, + ), + ))); + } + } + } + let checked_at = Utc::now().to_rfc3339(); let runtime = api.runtime.clone(); let ping_runtime_id = runtime_id.clone(); @@ -18283,6 +18546,7 @@ mod tests { server_id: "server-test".to_owned(), server_private_key: "unused".to_owned(), }), + workspace_authorization: None, strict_public_egress: false, cached_worker_creation_available: true, cached_os: "test".to_owned(), @@ -24847,6 +25111,7 @@ mod tests { server_id: "server-main".to_string(), server_private_key: identity.private_key.clone(), }), + workspace_authorization: None, strict_public_egress: false, cached_worker_creation_available: true, cached_os: "test".to_string(), @@ -26374,6 +26639,7 @@ mod tests { base_url: endpoint, bearer_token: Some("test-connection-token".to_string()), auth: None, + workspace_authorization: None, strict_public_egress: false, cached_worker_creation_available: true, cached_os: "linux".to_string(), diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index ca9ea392..3e4fe69b 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -18,7 +18,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 55; +const LATEST_SCHEMA_VERSION: i64 = 56; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings"; const RUNTIME_BINDING_AUDIT_MIGRATION_NAME: &str = "workspace Runtime binding revision and audit"; @@ -26,6 +26,8 @@ const WORKSPACE_DELETION_MIGRATION_NAME: &str = "durable Workspace deletion oper const WORKSPACE_SIGNING_IDENTITY_MIGRATION_NAME: &str = "Workspace signing identity authority"; const WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME: &str = "Workspace Runtime binding state and identity mode"; +const WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME: &str = + "Workspace-signed Runtime verification evidence"; const MIGRATIONS: &[Migration] = &[ Migration { @@ -53,6 +55,11 @@ const MIGRATIONS: &[Migration] = &[ name: WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME, apply: migrate_workspace_runtime_binding_state_v54_to_v55, }, + Migration { + version: 56, + name: WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME, + apply: migrate_workspace_runtime_verification_v55_to_v56, + }, ]; #[derive(Clone, Copy)] @@ -208,6 +215,23 @@ pub struct WorkspaceSigningIdentityProvisioningOperation { pub completed_at: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct WorkspaceRuntimeVerificationEvidence { + pub workspace_id: String, + pub runtime_id: String, + pub binding_revision: u64, + pub workspace_key_id: String, + pub workspace_identity_revision: u64, + pub workspace_trust_generation: u64, + pub runtime_public_key_fingerprint: String, + pub runtime_identity_revision: u64, + pub challenge_id: String, + pub state: String, + pub last_outcome: String, + pub verified_at: Option, + pub checked_at: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct WorkspaceRuntimeBinding { pub workspace_id: String, @@ -732,6 +756,25 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { ) -> Result; fn workspace_runtime_binding_matches(&self, expected: &WorkspaceRuntimeBinding) -> Result; + fn workspace_runtime_verification_matches( + &self, + binding: &WorkspaceRuntimeBinding, + workspace_identity_revision: u64, + workspace_trust_generation: u64, + ) -> Result; + async fn get_workspace_runtime_verification( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result>; + async fn record_workspace_runtime_verification_attempt( + &self, + evidence: &WorkspaceRuntimeVerificationEvidence, + ) -> Result<()>; + async fn complete_workspace_runtime_verification( + &self, + evidence: &WorkspaceRuntimeVerificationEvidence, + ) -> Result; async fn get_workspace_runtime_binding( &self, workspace_id: &str, @@ -2179,6 +2222,183 @@ impl SqliteWorkspaceStore { }) } + pub fn workspace_runtime_verification_matches( + &self, + binding: &WorkspaceRuntimeBinding, + workspace_identity_revision: u64, + workspace_trust_generation: u64, + ) -> Result { + let Some(evidence) = + self.get_workspace_runtime_verification(&binding.workspace_id, &binding.runtime_id)? + else { + return Ok(false); + }; + Ok(evidence.state == "verified" + && evidence.verified_at.is_some() + && evidence.binding_revision == binding.binding_revision + && evidence.workspace_key_id == binding.workspace_key_id.as_deref().unwrap_or_default() + && evidence.workspace_identity_revision == workspace_identity_revision + && evidence.workspace_trust_generation == workspace_trust_generation + && evidence.runtime_public_key_fingerprint == binding.public_key_fingerprint + && evidence.runtime_identity_revision > 0) + } + + pub fn get_workspace_runtime_verification( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result> { + validate_identifier("workspace_id", workspace_id)?; + validate_identifier("runtime_id", runtime_id)?; + self.with_conn(|conn| { + conn.query_row( + r#"SELECT workspace_id, runtime_id, binding_revision, workspace_key_id, + workspace_identity_revision, workspace_trust_generation, + runtime_public_key_fingerprint, runtime_identity_revision, + challenge_id, state, last_outcome, verified_at, checked_at + FROM workspace_runtime_verifications + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![workspace_id, runtime_id], + read_workspace_runtime_verification, + ) + .optional() + .map_err(Error::from) + }) + } + + pub fn record_workspace_runtime_verification_attempt( + &self, + evidence: &WorkspaceRuntimeVerificationEvidence, + ) -> Result<()> { + validate_workspace_runtime_verification(evidence)?; + self.with_conn(|conn| { + conn.execute( + r#"INSERT INTO workspace_runtime_verifications ( + workspace_id, runtime_id, binding_revision, workspace_key_id, + workspace_identity_revision, workspace_trust_generation, + runtime_public_key_fingerprint, runtime_identity_revision, + challenge_id, state, last_outcome, verified_at, checked_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13) + ON CONFLICT(workspace_id, runtime_id) DO UPDATE SET + binding_revision = excluded.binding_revision, + workspace_key_id = excluded.workspace_key_id, + workspace_identity_revision = excluded.workspace_identity_revision, + workspace_trust_generation = excluded.workspace_trust_generation, + runtime_public_key_fingerprint = excluded.runtime_public_key_fingerprint, + runtime_identity_revision = excluded.runtime_identity_revision, + challenge_id = excluded.challenge_id, + state = excluded.state, + last_outcome = excluded.last_outcome, + verified_at = excluded.verified_at, + checked_at = excluded.checked_at"#, + params![ + evidence.workspace_id, + evidence.runtime_id, + evidence.binding_revision, + evidence.workspace_key_id, + evidence.workspace_identity_revision, + evidence.workspace_trust_generation, + evidence.runtime_public_key_fingerprint, + evidence.runtime_identity_revision, + evidence.challenge_id, + evidence.state, + evidence.last_outcome, + evidence.verified_at, + evidence.checked_at, + ], + )?; + Ok(()) + }) + } + + pub fn complete_workspace_runtime_verification( + &self, + evidence: &WorkspaceRuntimeVerificationEvidence, + ) -> Result { + validate_workspace_runtime_verification(evidence)?; + if evidence.state != "verified" || evidence.verified_at.is_none() { + return Err(Error::Store( + "completed Runtime verification evidence must be verified".to_string(), + )); + } + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let changed = tx.execute( + r#"UPDATE workspace_runtime_bindings + SET state = 'verified', updated_at = ?7 + WHERE workspace_id = ?1 AND runtime_id = ?2 + AND binding_revision = ?3 + AND state IN ('configured', 'verified') + AND authentication_mode = 'workspace_identity' + AND workspace_key_id = ?4 + AND workspace_key_generation = ?5 + AND public_key_fingerprint = ?6 + AND revoked_at IS NULL"#, + params![ + evidence.workspace_id, + evidence.runtime_id, + evidence.binding_revision, + evidence.workspace_key_id, + evidence.workspace_trust_generation, + evidence.runtime_public_key_fingerprint, + evidence.checked_at, + ], + )?; + if changed != 1 { + return Err(Error::RuntimeBindingConflict( + "Runtime verification evidence no longer matches the configured binding" + .to_string(), + )); + } + tx.execute( + r#"INSERT INTO workspace_runtime_verifications ( + workspace_id, runtime_id, binding_revision, workspace_key_id, + workspace_identity_revision, workspace_trust_generation, + runtime_public_key_fingerprint, runtime_identity_revision, + challenge_id, state, last_outcome, verified_at, checked_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13) + ON CONFLICT(workspace_id, runtime_id) DO UPDATE SET + binding_revision = excluded.binding_revision, + workspace_key_id = excluded.workspace_key_id, + workspace_identity_revision = excluded.workspace_identity_revision, + workspace_trust_generation = excluded.workspace_trust_generation, + runtime_public_key_fingerprint = excluded.runtime_public_key_fingerprint, + runtime_identity_revision = excluded.runtime_identity_revision, + challenge_id = excluded.challenge_id, + state = excluded.state, + last_outcome = excluded.last_outcome, + verified_at = excluded.verified_at, + checked_at = excluded.checked_at"#, + params![ + evidence.workspace_id, + evidence.runtime_id, + evidence.binding_revision, + evidence.workspace_key_id, + evidence.workspace_identity_revision, + evidence.workspace_trust_generation, + evidence.runtime_public_key_fingerprint, + evidence.runtime_identity_revision, + evidence.challenge_id, + evidence.state, + evidence.last_outcome, + evidence.verified_at, + evidence.checked_at, + ], + )?; + let binding = tx.query_row( + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, state, authentication_mode, + workspace_key_id, workspace_key_generation, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![evidence.workspace_id, evidence.runtime_id], + read_workspace_runtime_binding, + )?; + tx.commit()?; + Ok(binding) + }) + } + pub fn list_workspace_runtime_binding_audit( &self, workspace_id: &str, @@ -2919,6 +3139,42 @@ impl ControlPlaneStore for SqliteWorkspaceStore { .is_some_and(|binding| binding == *expected && binding.revoked_at.is_none())) } + fn workspace_runtime_verification_matches( + &self, + binding: &WorkspaceRuntimeBinding, + workspace_identity_revision: u64, + workspace_trust_generation: u64, + ) -> Result { + SqliteWorkspaceStore::workspace_runtime_verification_matches( + self, + binding, + workspace_identity_revision, + workspace_trust_generation, + ) + } + + async fn get_workspace_runtime_verification( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result> { + SqliteWorkspaceStore::get_workspace_runtime_verification(self, workspace_id, runtime_id) + } + + async fn record_workspace_runtime_verification_attempt( + &self, + evidence: &WorkspaceRuntimeVerificationEvidence, + ) -> Result<()> { + SqliteWorkspaceStore::record_workspace_runtime_verification_attempt(self, evidence) + } + + async fn complete_workspace_runtime_verification( + &self, + evidence: &WorkspaceRuntimeVerificationEvidence, + ) -> Result { + SqliteWorkspaceStore::complete_workspace_runtime_verification(self, evidence) + } + async fn get_workspace_runtime_binding( &self, workspace_id: &str, @@ -6424,6 +6680,56 @@ fn account_select_sql(where_clause: &str) -> String { ) } +fn read_workspace_runtime_verification( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(WorkspaceRuntimeVerificationEvidence { + workspace_id: row.get(0)?, + runtime_id: row.get(1)?, + binding_revision: row.get(2)?, + workspace_key_id: row.get(3)?, + workspace_identity_revision: row.get(4)?, + workspace_trust_generation: row.get(5)?, + runtime_public_key_fingerprint: row.get(6)?, + runtime_identity_revision: row.get(7)?, + challenge_id: row.get(8)?, + state: row.get(9)?, + last_outcome: row.get(10)?, + verified_at: row.get(11)?, + checked_at: row.get(12)?, + }) +} + +fn validate_workspace_runtime_verification( + evidence: &WorkspaceRuntimeVerificationEvidence, +) -> Result<()> { + for (field, value) in [ + ("workspace_id", evidence.workspace_id.as_str()), + ("runtime_id", evidence.runtime_id.as_str()), + ("workspace_key_id", evidence.workspace_key_id.as_str()), + ("challenge_id", evidence.challenge_id.as_str()), + ] { + validate_identifier(field, value)?; + } + if evidence.binding_revision == 0 + || evidence.workspace_identity_revision == 0 + || evidence.workspace_trust_generation == 0 + || evidence.runtime_identity_revision == 0 + { + return Err(Error::InvalidInput( + "Runtime verification revisions and generations must be positive".to_string(), + )); + } + validate_non_empty( + "runtime_public_key_fingerprint", + &evidence.runtime_public_key_fingerprint, + )?; + validate_non_empty("verification state", &evidence.state)?; + validate_non_empty("verification outcome", &evidence.last_outcome)?; + validate_non_empty("checked_at", &evidence.checked_at)?; + Ok(()) +} + fn read_workspace_runtime_binding( row: &rusqlite::Row<'_>, ) -> rusqlite::Result { @@ -7817,6 +8123,80 @@ fn migrate_workspace_runtime_binding_state_v54_to_v55(conn: &Connection) -> Resu Ok(()) } +fn migrate_workspace_runtime_verification_v55_to_v56(conn: &Connection) -> Result<()> { + let current = current_schema_version(conn)?; + if current != 55 { + return Err(Error::Store(format!( + "expected schema version 55 before {WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME} migration, found {current}" + ))); + } + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS workspace_runtime_verifications ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + binding_revision INTEGER NOT NULL CHECK(binding_revision > 0), + workspace_key_id TEXT NOT NULL, + workspace_identity_revision INTEGER NOT NULL CHECK(workspace_identity_revision > 0), + workspace_trust_generation INTEGER NOT NULL CHECK(workspace_trust_generation > 0), + runtime_public_key_fingerprint TEXT NOT NULL, + runtime_identity_revision INTEGER NOT NULL CHECK(runtime_identity_revision > 0), + challenge_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('pending', 'verified', 'failed')), + last_outcome TEXT NOT NULL, + verified_at TEXT, + checked_at TEXT NOT NULL, + PRIMARY KEY(workspace_id, runtime_id), + FOREIGN KEY(workspace_id, runtime_id) + REFERENCES workspace_runtime_bindings(workspace_id, runtime_id) + ON DELETE CASCADE, + CHECK((state = 'verified' AND verified_at IS NOT NULL) + OR (state != 'verified' AND verified_at IS NULL)) + ); + CREATE INDEX IF NOT EXISTS workspace_runtime_verifications_state_idx + ON workspace_runtime_verifications(workspace_id, state, checked_at DESC); + "#, + )?; + verify_workspace_runtime_verification_schema(&tx)?; + tx.execute( + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", + params![56_i64, WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME], + )?; + tx.commit()?; + Ok(()) +} + +fn verify_workspace_runtime_verification_schema(conn: &Connection) -> Result<()> { + let actual = table_columns(conn, "workspace_runtime_verifications")? + .into_iter() + .collect::>(); + let expected = [ + "workspace_id", + "runtime_id", + "binding_revision", + "workspace_key_id", + "workspace_identity_revision", + "workspace_trust_generation", + "runtime_public_key_fingerprint", + "runtime_identity_revision", + "challenge_id", + "state", + "last_outcome", + "verified_at", + "checked_at", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + if actual != expected { + return Err(Error::Store(format!( + "workspace_runtime_verifications schema does not match schema-56: {actual:?}" + ))); + } + Ok(()) +} + fn verify_workspace_signing_identity_schema(conn: &Connection) -> Result<()> { for (table, expected) in [ ( @@ -8679,6 +9059,7 @@ fn apply_migrations(conn: &Connection) -> Result<()> { verify_schema_history(conn, LATEST_SCHEMA_VERSION)?; verify_workspace_runtime_binding_schema(conn)?; + verify_workspace_runtime_verification_schema(conn)?; verify_workspace_deletion_schema(conn)?; verify_workspace_signing_identity_schema(conn) } @@ -8902,6 +9283,10 @@ mod tests { version: 55, name: WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME.to_string(), }, + WorkspaceSchemaMigrationStep { + version: 56, + name: WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME.to_string(), + }, ] ); @@ -8928,6 +9313,10 @@ mod tests { 55, WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME.to_string(), ), + ( + 56, + WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME.to_string(), + ), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -8997,7 +9386,7 @@ mod tests { .iter() .map(|migration| migration.version) .collect::>(), - vec![52, 53, 54, 55] + vec![52, 53, 54, 55, 56] ); SqliteWorkspaceStore::migrate_database(&path).unwrap(); let conn = Connection::open(&path).unwrap(); @@ -9005,7 +9394,7 @@ mod tests { current_schema_version(&conn).unwrap(), LATEST_SCHEMA_VERSION ); - assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 6); + assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 7); } #[test] @@ -9371,6 +9760,123 @@ mod tests { ); } + #[test] + fn workspace_runtime_verification_is_revision_bound_and_restart_safe() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("server.db"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + store + .with_conn(|conn| { + conn.execute_batch( + r#" + INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('owner', 'user', 'owner', 'Owner', '1', '1'); + INSERT INTO workspaces(workspace_id, owner_account_id, display_name, state, created_at, updated_at) + VALUES ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1'); + INSERT INTO workspace_signing_identities( + workspace_id, key_id, algorithm, public_key, public_key_fingerprint, + private_material_ref, revision, state, created_at, provisioned_at, updated_at + ) VALUES ('workspace-a', 'WK-a', 'ed25519', 'key', 'sha256:key', + 'workspace-signing/workspace-a/ed25519-v1', 1, 'active', '1', '1', '1'); + "#, + )?; + Ok(()) + }) + .unwrap(); + let runtime_identity = + worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-a").unwrap(); + store + .upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: "workspace-a".to_string(), + runtime_id: "runtime-a".to_string(), + display_name: "runtime-a".to_string(), + base_url: "https://runtime.test".to_string(), + public_key: runtime_identity.public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + state: WorkspaceRuntimeBindingState::Configured, + authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, + workspace_key_id: Some("WK-a".to_string()), + workspace_key_generation: Some(1), + created_at: "1".to_string(), + updated_at: "1".to_string(), + revoked_at: None, + }, + false, + ) + .unwrap(); + let persisted = store + .get_workspace_runtime_binding("workspace-a", "runtime-a") + .unwrap() + .unwrap(); + let evidence = WorkspaceRuntimeVerificationEvidence { + workspace_id: "workspace-a".to_string(), + runtime_id: "runtime-a".to_string(), + binding_revision: persisted.binding_revision, + workspace_key_id: "WK-a".to_string(), + workspace_identity_revision: 1, + workspace_trust_generation: 1, + runtime_public_key_fingerprint: persisted.public_key_fingerprint.clone(), + runtime_identity_revision: 1, + challenge_id: "challenge-a".to_string(), + state: "verified".to_string(), + last_outcome: "verified".to_string(), + verified_at: Some("2".to_string()), + checked_at: "2".to_string(), + }; + let verified = store + .complete_workspace_runtime_verification(&evidence) + .unwrap(); + assert_eq!(verified.state, WorkspaceRuntimeBindingState::Verified); + drop(store); + + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + assert_eq!( + reopened + .get_workspace_runtime_verification("workspace-a", "runtime-a") + .unwrap(), + Some(evidence) + ); + assert_eq!( + reopened + .get_workspace_runtime_binding("workspace-a", "runtime-a") + .unwrap() + .unwrap() + .state, + WorkspaceRuntimeBindingState::Verified + ); + } + + #[test] + fn schema_v55_migrates_runtime_verification_table_atomically() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("server.db"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + store + .with_conn(|conn| { + conn.execute_batch( + "DROP TABLE workspace_runtime_verifications; + DELETE FROM __yoi_schema_migrations; + INSERT INTO __yoi_schema_migrations(version, name) + VALUES (55, 'Workspace Runtime binding state and identity mode');", + )?; + Ok(()) + }) + .unwrap(); + drop(store); + + let conn = Connection::open(&path).unwrap(); + configure_sqlite(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 55); + migrate_workspace_runtime_verification_v55_to_v56(&conn).unwrap(); + drop(conn); + let migrated = Connection::open(&path).unwrap(); + configure_sqlite(&migrated).unwrap(); + assert_eq!(current_schema_version(&migrated).unwrap(), 56); + assert!(table_exists(&migrated, "workspace_runtime_verifications").unwrap()); + } + #[test] fn runtime_binding_key_mutations_are_revisioned_idempotent_and_audited() { let store = SqliteWorkspaceStore::in_memory().unwrap(); @@ -10691,13 +11197,13 @@ INSERT INTO worker_registry ( let conn = Connection::open_in_memory().unwrap(); configure_sqlite(&conn).unwrap(); conn.execute( - "INSERT INTO __yoi_schema_migrations (version, name) VALUES (56, 'future')", + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (57, 'future')", [], ) .unwrap(); let error = apply_migrations(&conn).unwrap_err().to_string(); - assert!(error.contains("schema version 56 is newer"), "{error}"); + assert!(error.contains("schema version 57 is newer"), "{error}"); assert!(error.contains("refusing to serve"), "{error}"); } diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index 0c6f35ea..5dda2f84 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -80,6 +80,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ "workspace_resource_keys", "workspace_runtime_binding_audit", "workspace_runtime_bindings", + "workspace_runtime_verifications", "workspace_signing_identities", "workspace_signing_identity_audit", "workspace_signing_identity_provisioning_operations", diff --git a/crates/workspace-server/src/workspace_signing_identity.rs b/crates/workspace-server/src/workspace_signing_identity.rs index eec48496..c57169b6 100644 --- a/crates/workspace-server/src/workspace_signing_identity.rs +++ b/crates/workspace-server/src/workspace_signing_identity.rs @@ -9,6 +9,10 @@ use chrono::{SecondsFormat, Utc}; use ring::signature::KeyPair; use serde::{Deserialize, Serialize}; use worker_runtime::auth::{RuntimeIdentityMaterial, encode_public_key}; +use worker_runtime::workspace_issuer::{ + WorkspaceCapabilityClaims, WorkspaceCapabilityVerificationError, + assemble_workspace_capability_token, workspace_capability_signing_input, +}; use zeroize::Zeroize; use crate::store::{ @@ -254,6 +258,16 @@ impl WorkspaceSigningIdentityService { Ok(signing_key.sign(payload).as_ref().to_vec()) } + pub fn issue_workspace_capability( + &self, + workspace_id: &str, + claims: &WorkspaceCapabilityClaims, + ) -> Result { + let input = workspace_capability_signing_input(claims).map_err(capability_error)?; + let signature = self.sign(workspace_id, input.bytes())?; + assemble_workspace_capability_token(input, &signature).map_err(capability_error) + } + pub fn delete_material(&self, workspace_id: &str) -> Result<()> { if let Some(identity) = self.store.get_workspace_signing_identity(workspace_id)? { self.materials.delete(&identity.private_material_ref)?; @@ -572,6 +586,13 @@ fn material_io_error(action: &str, error: std::io::Error) -> Error { ) } +fn capability_error(error: WorkspaceCapabilityVerificationError) -> Error { + identity_error( + "workspace_capability_issuance_failed", + format!("failed to issue Workspace capability: {error}"), + ) +} + pub fn identity_error(code: impl Into, message: impl Into) -> Error { Error::WorkspaceSigningIdentity { code: code.into(),