Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: verify Workspace-signed Runtime bindings

Browse Source
This commit is contained in:
Hare
2026-09-08 08:13:58 +09:00
parent f5e9f49a13
commit f29c343879
12 changed files with 2817 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workdir/Cargo.toml
+1
View File
@@ -14,6 +14,7 @@ fs-operation.workspace = true
manifest.workspace = true
reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"], optional = true }
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
sha2.workspace = true
tempfile.workspace = true
thiserror.workspace = true
crates/workdir/src/http.rs
+26 -7
View File
@@ -293,7 +293,12 @@ mod client {
/// implementations can mint short-lived capability tokens without making a
/// Worker-bound session expire with the token used to open it.
pub trait WorkdirHttpAuthorization: std::fmt::Debug + Send + Sync {
fn bearer_token(&self) -> Result<String, WorkdirError>;
fn bearer_token(
&self,
method: &str,
path_and_query: &str,
body: &[u8],
) -> Result<String, WorkdirError>;
}
struct FixedBearerToken(Arc<str>);
@@ -305,7 +310,12 @@ mod client {
}
impl WorkdirHttpAuthorization for FixedBearerToken {
fn bearer_token(&self) -> Result<String, WorkdirError> {
fn bearer_token(
&self,
_method: &str,
_path_and_query: &str,
_body: &[u8],
) -> Result<String, WorkdirError> {
Ok(self.0.to_string())
}
}
@@ -354,10 +364,14 @@ mod client {
&base_url,
&["v1", "working-directories", workdir_id.as_str(), "sessions"],
)?;
let body = serde_json::to_vec(&request)
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
let token = authorization.bearer_token("POST", url.path(), &body)?;
let response = client
.post(url)
.bearer_auth(authorization.bearer_token()?)
.json(&request)
.bearer_auth(token)
.header("content-type", "application/json")
.body(body)
.send()
.await
.map_err(http_unavailable)?;
@@ -401,11 +415,15 @@ mod client {
],
)?;
let operation = WorkdirSessionOperationRequest { operation };
let body = serde_json::to_vec(&operation)
.map_err(|error| WorkdirError::Unavailable(error.to_string()))?;
let token = self.authorization.bearer_token("POST", url.path(), &body)?;
let response = self
.client
.post(url)
.bearer_auth(self.authorization.bearer_token()?)
.json(&operation)
.bearer_auth(token)
.header("content-type", "application/json")
.body(body)
.send()
.await
.map_err(http_unavailable)?;
@@ -543,10 +561,11 @@ mod client {
&self.base_url,
&["v1", "workdir-sessions", self.session_id.as_str()],
)?;
let token = self.authorization.bearer_token("DELETE", url.path(), &[])?;
let response = self
.client
.delete(url)
.bearer_auth(self.authorization.bearer_token()?)
.bearer_auth(token)
.send()
.await
.map_err(http_unavailable)?;
crates/worker-runtime/src/http_server.rs
+540 -6
View File
@@ -27,6 +27,15 @@ use crate::retention::{
};
#[cfg(feature = "ws-server")]
use crate::runtime::RuntimeSubscriptionRecvError;
use crate::workspace_issuer::{
RuntimeVerificationSigner, VerifiedWorkspaceCapability, WORKSPACE_VERIFICATION_ACK_PATH,
WORKSPACE_VERIFICATION_CHALLENGE_PATH, WORKSPACE_VERIFICATION_OPERATION,
WorkspaceCapabilityExpectation, WorkspaceCapabilityVerifier,
WorkspaceRuntimeVerificationAcknowledgement, WorkspaceRuntimeVerificationAuthority,
WorkspaceRuntimeVerificationChallenge, WorkspaceRuntimeVerificationReceipt,
WorkspaceRuntimeVerificationRecord, WorkspaceRuntimeVerificationResponse,
inspect_workspace_capability_claims, workspace_request_body_digest,
};
use crate::{Runtime, RuntimeWorkspaceScope};
use axum::body::{Body, Bytes};
use axum::extract::rejection::{JsonRejection, QueryRejection};
@@ -155,7 +164,22 @@ pub async fn serve_runtime_http_with_auth(
}
axum::serve(
listener,
runtime_http_router_with_optional_auth(runtime, local_token, auth),
runtime_http_router_with_optional_auth(runtime, local_token, auth, None),
)
.await?;
Ok(())
}
pub async fn serve_runtime_http_with_workspace_auth(
runtime: Runtime,
listener: TcpListener,
local_token: Option<String>,
auth: Option<RuntimeHttpAuthConfig>,
workspace_auth: WorkspaceRuntimeHttpAuth,
) -> Result<(), RuntimeHttpServerError> {
axum::serve(
listener,
runtime_http_router_with_optional_auth(runtime, local_token, auth, Some(workspace_auth)),
)
.await?;
Ok(())
@@ -167,7 +191,7 @@ pub async fn serve_runtime_http_with_auth(
/// The path contains only a Runtime-local `worker_id`; backend aliases are not
/// accepted or forwarded as Runtime authority.
pub fn runtime_http_router(runtime: Runtime, local_token: String) -> Router {
runtime_http_router_with_optional_auth(runtime, Some(local_token), None)
runtime_http_router_with_optional_auth(runtime, Some(local_token), None, None)
}
/// Build the REST router for an existing Runtime with signed capability-token auth.
@@ -176,23 +200,42 @@ pub fn runtime_http_router_with_auth(
local_token: Option<String>,
auth: RuntimeHttpAuthConfig,
) -> Router {
runtime_http_router_with_optional_auth(runtime, local_token, Some(auth))
runtime_http_router_with_optional_auth(runtime, local_token, Some(auth), None)
}
pub fn runtime_http_router_with_workspace_auth(
runtime: Runtime,
local_token: Option<String>,
auth: RuntimeHttpAuthConfig,
workspace_auth: WorkspaceRuntimeHttpAuth,
) -> Router {
runtime_http_router_with_optional_auth(runtime, local_token, Some(auth), Some(workspace_auth))
}
fn runtime_http_router_with_optional_auth(
runtime: Runtime,
local_token: Option<String>,
auth: Option<RuntimeHttpAuthConfig>,
workspace_auth: Option<WorkspaceRuntimeHttpAuth>,
) -> Router {
let state = RuntimeHttpState {
runtime,
local_token: local_token.map(Arc::<str>::from),
auth: auth.map(Arc::new),
workspace_auth: workspace_auth.map(Arc::new),
workdir_sessions: Arc::new(Mutex::new(HashMap::new())),
};
let router = Router::new()
.route("/v1/ping", get(get_runtime_ping))
.route(
WORKSPACE_VERIFICATION_CHALLENGE_PATH,
post(post_workspace_verification_challenge),
)
.route(
WORKSPACE_VERIFICATION_ACK_PATH,
post(post_workspace_verification_acknowledgement),
)
.route("/v1/runtime", get(get_runtime))
.route(
"/v1/config-bundles",
@@ -286,9 +329,17 @@ struct RuntimeHttpState {
runtime: Runtime,
local_token: Option<Arc<str>>,
auth: Option<Arc<RuntimeHttpAuthConfig>>,
workspace_auth: Option<Arc<WorkspaceRuntimeHttpAuth>>,
workdir_sessions: Arc<Mutex<HashMap<String, RuntimeHttpWorkdirSession>>>,
}
#[derive(Clone, Debug)]
pub struct WorkspaceRuntimeHttpAuth {
pub verifier: WorkspaceCapabilityVerifier,
pub signer: RuntimeVerificationSigner,
pub verifications: Arc<dyn WorkspaceRuntimeVerificationAuthority>,
}
struct RuntimeHttpWorkdirSession {
owner: RuntimeWorkspaceScope,
session: WorkdirSessionHandle,
@@ -475,6 +526,164 @@ struct RuntimeWorkerEventsWsQuery {
type RestResult<T> = Result<Json<T>, RuntimeHttpRestError>;
fn unix_now_i64() -> i64 {
i64::try_from(unix_now_seconds()).unwrap_or(i64::MAX)
}
async fn post_workspace_verification_challenge(
State(state): State<RuntimeHttpState>,
Extension(verified): Extension<VerifiedWorkspaceCapability>,
Json(challenge): Json<WorkspaceRuntimeVerificationChallenge>,
) -> RestResult<WorkspaceRuntimeVerificationResponse> {
let auth = state.workspace_auth.as_deref().ok_or_else(|| {
RuntimeHttpRestError::new(
StatusCode::NOT_IMPLEMENTED,
"workspace_runtime_verification_unavailable",
"Workspace Runtime verification is not configured",
)
})?;
if challenge.workspace_id != verified.workspace_id
|| challenge.runtime_id != verified.runtime_id
|| challenge.binding_revision != verified.binding_revision
|| challenge.workspace_key_id != verified.issuer_key_id
|| challenge.workspace_identity_revision != verified.issuer_identity_revision
|| challenge.workspace_trust_generation != verified.trust_generation
|| challenge.runtime_id != auth.signer.runtime_id()
|| challenge.runtime_public_key_fingerprint != auth.signer.public_key_fingerprint()
{
return Err(RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"workspace_runtime_verification_rejected",
"Workspace Runtime verification challenge does not match authenticated authority",
));
}
let response = auth
.signer
.sign_response(&challenge, uuid::Uuid::now_v7().to_string(), unix_now_i64())
.map_err(|error| {
RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"workspace_runtime_verification_rejected",
error.to_string(),
)
})?;
Ok(Json(response))
}
async fn post_workspace_verification_acknowledgement(
State(state): State<RuntimeHttpState>,
Extension(verified): Extension<VerifiedWorkspaceCapability>,
Json(acknowledgement): Json<WorkspaceRuntimeVerificationAcknowledgement>,
) -> RestResult<WorkspaceRuntimeVerificationReceipt> {
let auth = state.workspace_auth.as_deref().ok_or_else(|| {
RuntimeHttpRestError::new(
StatusCode::NOT_IMPLEMENTED,
"workspace_runtime_verification_unavailable",
"Workspace Runtime verification is not configured",
)
})?;
if acknowledgement.workspace_id != verified.workspace_id
|| acknowledgement.runtime_id != verified.runtime_id
|| acknowledgement.binding_revision != verified.binding_revision
|| acknowledgement.workspace_key_id != verified.issuer_key_id
|| acknowledgement.workspace_identity_revision != verified.issuer_identity_revision
|| acknowledgement.workspace_trust_generation != verified.trust_generation
|| acknowledgement.runtime_id != auth.signer.runtime_id()
|| acknowledgement.runtime_public_key_fingerprint != auth.signer.public_key_fingerprint()
|| acknowledgement.expires_at <= unix_now_i64()
|| acknowledgement.binding_revision == 0
|| acknowledgement.runtime_identity_revision == 0
|| acknowledgement.workspace_identity_revision == 0
|| acknowledgement.workspace_trust_generation == 0
|| acknowledgement.workspace_nonce.is_empty()
|| acknowledgement.runtime_nonce.is_empty()
|| acknowledgement.response_digest.len() != workspace_request_body_digest(&[]).len()
{
return Err(RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"workspace_runtime_verification_rejected",
"Workspace Runtime verification acknowledgement is invalid",
));
}
let challenge = WorkspaceRuntimeVerificationChallenge {
challenge_id: acknowledgement.challenge_id.clone(),
workspace_id: acknowledgement.workspace_id.clone(),
runtime_id: acknowledgement.runtime_id.clone(),
binding_revision: acknowledgement.binding_revision,
workspace_key_id: acknowledgement.workspace_key_id.clone(),
workspace_identity_revision: acknowledgement.workspace_identity_revision,
workspace_trust_generation: acknowledgement.workspace_trust_generation,
runtime_public_key_fingerprint: acknowledgement.runtime_public_key_fingerprint.clone(),
runtime_identity_revision: acknowledgement.runtime_identity_revision,
workspace_nonce: acknowledgement.workspace_nonce.clone(),
expires_at: acknowledgement.expires_at,
};
let response = acknowledgement.response.clone();
if response.runtime_nonce != acknowledgement.runtime_nonce
|| response.workspace_nonce != acknowledgement.workspace_nonce
|| response.response_proof.is_empty()
{
return Err(RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"workspace_runtime_verification_rejected",
"Workspace Runtime verification acknowledgement does not match its response",
));
}
let response_bytes = serde_json::to_vec(&response).map_err(|_| {
RuntimeHttpRestError::new(
StatusCode::BAD_REQUEST,
"workspace_runtime_verification_rejected",
"Workspace Runtime verification response could not be canonicalized",
)
})?;
if workspace_request_body_digest(&response_bytes) != acknowledgement.response_digest {
return Err(RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"workspace_runtime_verification_rejected",
"Workspace Runtime verification acknowledgement has the wrong response digest",
));
}
crate::workspace_issuer::verify_runtime_verification_response(
&response,
&challenge,
auth.signer.public_key(),
unix_now_i64(),
)
.map_err(|error| {
RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"workspace_runtime_verification_rejected",
error.to_string(),
)
})?;
auth.verifications
.record(WorkspaceRuntimeVerificationRecord {
workspace_id: acknowledgement.workspace_id.clone(),
runtime_id: acknowledgement.runtime_id.clone(),
binding_revision: acknowledgement.binding_revision,
workspace_key_id: acknowledgement.workspace_key_id.clone(),
workspace_identity_revision: acknowledgement.workspace_identity_revision,
workspace_trust_generation: acknowledgement.workspace_trust_generation,
runtime_public_key_fingerprint: acknowledgement.runtime_public_key_fingerprint.clone(),
runtime_identity_revision: acknowledgement.runtime_identity_revision,
verified_at: unix_now_i64(),
})
.map_err(|error| {
RuntimeHttpRestError::new(
StatusCode::SERVICE_UNAVAILABLE,
"workspace_runtime_verification_unavailable",
error.to_string(),
)
})?;
Ok(Json(WorkspaceRuntimeVerificationReceipt {
challenge_id: acknowledgement.challenge_id,
workspace_id: acknowledgement.workspace_id,
runtime_id: acknowledgement.runtime_id,
binding_revision: acknowledgement.binding_revision,
accepted_at: unix_now_i64(),
}))
}
async fn get_runtime_ping(
State(state): State<RuntimeHttpState>,
Extension(auth): Extension<RuntimeAuthContext>,
@@ -1797,10 +2006,112 @@ async fn require_runtime_auth(
.headers()
.get(header::AUTHORIZATION)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.strip_prefix("Bearer "));
.and_then(|value| value.strip_prefix("Bearer "))
.map(str::to_owned);
if let Some(workspace_auth) = state.workspace_auth.as_deref()
&& let Some(token) = supplied.as_deref()
&& let Ok(claims) = inspect_workspace_capability_claims(token)
{
let method = request.method().as_str().to_string();
let path_and_query = request
.uri()
.path_and_query()
.map_or_else(|| request.uri().path().to_string(), ToString::to_string);
let required_permission =
workspace_runtime_operation(request.method(), request.uri().path());
let expected_worker_id = worker_id_from_runtime_path(request.uri().path());
let (parts, body) = request.into_parts();
let body = match axum::body::to_bytes(body, 8 * 1024 * 1024).await {
Ok(body) => body,
Err(_) => {
return RuntimeHttpRestError::new(
StatusCode::PAYLOAD_TOO_LARGE,
"request_body_too_large",
"Runtime request body exceeds the verification limit",
)
.into_response();
}
};
let body_digest = workspace_request_body_digest(&body);
let expected = WorkspaceCapabilityExpectation {
workspace_id: &claims.issuer_workspace_id,
binding_revision: claims.binding_revision,
runtime_id: workspace_auth.signer.runtime_id(),
worker_id: expected_worker_id.as_deref(),
operation: required_permission,
method: &method,
path_and_query: &path_and_query,
body_digest: &body_digest,
now_unix: unix_now_i64(),
};
match workspace_auth.verifier.verify(token, &expected) {
Ok(verified) => {
let is_verification = path_and_query == WORKSPACE_VERIFICATION_CHALLENGE_PATH
|| path_and_query == WORKSPACE_VERIFICATION_ACK_PATH;
if !is_verification {
let record = match workspace_auth
.verifications
.get(&verified.workspace_id, workspace_auth.signer.runtime_id())
{
Ok(Some(record)) => record,
Ok(None) => {
return RuntimeHttpRestError::new(
StatusCode::FORBIDDEN,
"workspace_runtime_verification_required",
"Workspace Runtime binding has not completed signed verification",
)
.into_response();
}
Err(error) => {
return RuntimeHttpRestError::new(
StatusCode::SERVICE_UNAVAILABLE,
"workspace_runtime_verification_unavailable",
error.to_string(),
)
.into_response();
}
};
if record.binding_revision != verified.binding_revision
|| record.workspace_key_id != verified.issuer_key_id
|| record.workspace_identity_revision != verified.issuer_identity_revision
|| record.workspace_trust_generation != verified.trust_generation
|| record.runtime_public_key_fingerprint
!= workspace_auth.signer.public_key_fingerprint()
|| record.runtime_identity_revision == 0
{
return RuntimeHttpRestError::new(
StatusCode::FORBIDDEN,
"workspace_runtime_verification_stale",
"Workspace Runtime verification does not match current request authority",
)
.into_response();
}
}
request = Request::from_parts(parts, Body::from(body));
request.extensions_mut().insert(verified.clone());
request.extensions_mut().insert(RuntimeAuthContext {
server_id: verified.issuer,
workspace_id: verified.workspace_id,
permissions: vec![required_permission.to_string()],
token_id: verified.token_id,
expires_at: u64::try_from(verified.expires_at).unwrap_or(0),
});
return next.run(request).await;
}
Err(error) => {
return RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"unauthorized",
format!("invalid Workspace capability token: {error}"),
)
.into_response();
}
}
}
if let Some(auth) = state.auth.as_deref() {
let Some(token) = supplied else {
let Some(token) = supplied.as_deref() else {
return RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"unauthorized",
@@ -1815,6 +2126,18 @@ async fn require_runtime_auth(
unix_now_seconds(),
) {
Ok(context) => {
if state.workspace_auth.as_ref().is_some_and(|workspace_auth| {
workspace_auth
.verifier
.has_active_workspace_issuer(&context.workspace_id)
}) {
return RuntimeHttpRestError::new(
StatusCode::FORBIDDEN,
"workspace_identity_required",
"Legacy Server-issued capability is disabled for this Workspace",
)
.into_response();
}
request.extensions_mut().insert(context);
return next.run(request).await;
}
@@ -1825,7 +2148,7 @@ async fn require_runtime_auth(
}
if let Some(expected) = state.local_token.as_deref() {
if supplied != Some(expected) {
if supplied.as_deref() != Some(expected) {
return RuntimeHttpRestError::new(
StatusCode::UNAUTHORIZED,
"unauthorized",
@@ -1897,6 +2220,21 @@ fn auth_workspace_scope(
Ok(Some(RuntimeWorkspaceScope::new(workspace_id, server_id)))
}
fn workspace_runtime_operation(method: &Method, path: &str) -> &'static str {
if (path == WORKSPACE_VERIFICATION_CHALLENGE_PATH || path == WORKSPACE_VERIFICATION_ACK_PATH)
&& *method == Method::POST
{
return WORKSPACE_VERIFICATION_OPERATION;
}
required_runtime_permission(method, path).unwrap_or("runtime:read")
}
fn worker_id_from_runtime_path(path: &str) -> Option<String> {
let rest = path.strip_prefix("/v1/workers/")?;
let worker_id = rest.split('/').next()?;
(!worker_id.is_empty()).then(|| worker_id.to_string())
}
fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static str> {
if path == "/v1/ping" && *method == Method::GET {
return Some(RUNTIME_PING_PERMISSION);
@@ -2209,15 +2547,210 @@ mod tests {
WorkerExecutionSpawnResult,
};
use crate::management::RuntimeOptions;
use crate::workspace_issuer::{
InMemoryWorkspaceClaimReplayProtection, InMemoryWorkspaceRuntimeVerificationAuthority,
WorkspaceCapabilityClaims, WorkspaceCapabilityVerifier, WorkspaceIssuerTrustRecord,
WorkspaceIssuerTrustState, issue_workspace_capability_token,
verify_runtime_verification_response,
};
use axum::body::to_bytes;
use axum::http::Method;
use manifest::{Scope, SharedScope};
use sha2::Digest as _;
use tower::ServiceExt;
use workdir::{
GrepOutputMode, GrepRequest, LocalWorkdirSession, StatRequest, Workdir, WorkdirPath,
WorkdirSessionCapabilities,
};
#[tokio::test]
async fn workspace_signed_verification_requires_exact_request_and_acknowledges_response() {
let runtime = Runtime::new_memory();
let (legacy_auth, _) = auth_config_and_signer();
let workspace_identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
let runtime_identity = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
let workspace_public_key =
crate::auth::decode_public_key(&workspace_identity.public_key).unwrap();
let workspace_fingerprint = format!(
"sha256:{}",
crate::workspace_issuer::hex_lower(&sha2::Sha256::digest(workspace_public_key))
);
let runtime_public_key =
crate::auth::decode_public_key(&runtime_identity.public_key).unwrap();
let runtime_fingerprint = format!(
"sha256:{}",
crate::workspace_issuer::hex_lower(&sha2::Sha256::digest(runtime_public_key))
);
let verifier = WorkspaceCapabilityVerifier::new(
vec![WorkspaceIssuerTrustRecord {
workspace_id: "workspace-a".to_string(),
backend_url: "https://backend.test".to_string(),
key_id: "workspace-key".to_string(),
algorithm: "ed25519".to_string(),
public_key: workspace_identity.public_key.clone(),
public_key_fingerprint: workspace_fingerprint,
identity_revision: 1,
trust_generation: 1,
state: WorkspaceIssuerTrustState::Active,
registered_at_unix: 1,
updated_at_unix: 1,
}],
Arc::new(InMemoryWorkspaceClaimReplayProtection::default()),
)
.unwrap();
let app = runtime_http_router_with_workspace_auth(
runtime,
None,
legacy_auth,
WorkspaceRuntimeHttpAuth {
verifier,
signer: RuntimeVerificationSigner::from_identity(&runtime_identity).unwrap(),
verifications: Arc::new(InMemoryWorkspaceRuntimeVerificationAuthority::default()),
},
);
let challenge = WorkspaceRuntimeVerificationChallenge {
challenge_id: "challenge-1".to_string(),
workspace_id: "workspace-a".to_string(),
runtime_id: "runtime-test".to_string(),
binding_revision: 4,
workspace_key_id: "workspace-key".to_string(),
workspace_identity_revision: 1,
workspace_trust_generation: 1,
runtime_public_key_fingerprint: runtime_fingerprint,
runtime_identity_revision: 1,
workspace_nonce: "workspace-nonce".to_string(),
expires_at: unix_now_i64() + 60,
};
let body = serde_json::to_vec(&challenge).unwrap();
let claims = WorkspaceCapabilityClaims {
issuer: "https://backend.test".to_string(),
issuer_workspace_id: "workspace-a".to_string(),
issuer_key_id: "workspace-key".to_string(),
issuer_identity_revision: 1,
trust_generation: 1,
binding_revision: 4,
runtime_id: "runtime-test".to_string(),
worker_id: None,
operation: WORKSPACE_VERIFICATION_OPERATION.to_string(),
method: "POST".to_string(),
path_and_query: WORKSPACE_VERIFICATION_CHALLENGE_PATH.to_string(),
body_digest: workspace_request_body_digest(&body),
iat: unix_now_i64(),
exp: challenge.expires_at,
jti: "challenge-token".to_string(),
};
let token =
issue_workspace_capability_token(&workspace_identity.signing_key().unwrap(), &claims)
.unwrap();
let response = app
.clone()
.oneshot(
Request::builder()
.method(Method::POST)
.uri(WORKSPACE_VERIFICATION_CHALLENGE_PATH)
.header(header::AUTHORIZATION, format!("Bearer {token}"))
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let response_body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(
status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&response_body)
);
let verification_response =
serde_json::from_slice::<WorkspaceRuntimeVerificationResponse>(&response_body).unwrap();
verify_runtime_verification_response(
&verification_response,
&challenge,
&runtime_identity.public_key,
unix_now_i64(),
)
.unwrap();
let acknowledgement = WorkspaceRuntimeVerificationAcknowledgement {
challenge_id: verification_response.challenge_id.clone(),
workspace_id: verification_response.workspace_id.clone(),
runtime_id: verification_response.runtime_id.clone(),
binding_revision: verification_response.binding_revision,
workspace_key_id: verification_response.workspace_key_id.clone(),
workspace_identity_revision: verification_response.workspace_identity_revision,
workspace_trust_generation: verification_response.workspace_trust_generation,
runtime_public_key_fingerprint: verification_response
.runtime_public_key_fingerprint
.clone(),
runtime_identity_revision: verification_response.runtime_identity_revision,
workspace_nonce: verification_response.workspace_nonce.clone(),
runtime_nonce: verification_response.runtime_nonce.clone(),
response_digest: workspace_request_body_digest(&response_body),
response: verification_response.clone(),
expires_at: verification_response.expires_at,
};
let acknowledgement_body = serde_json::to_vec(&acknowledgement).unwrap();
let acknowledgement_claims = WorkspaceCapabilityClaims {
path_and_query: WORKSPACE_VERIFICATION_ACK_PATH.to_string(),
body_digest: workspace_request_body_digest(&acknowledgement_body),
jti: "ack-token".to_string(),
..claims
};
let acknowledgement_token = issue_workspace_capability_token(
&workspace_identity.signing_key().unwrap(),
&acknowledgement_claims,
)
.unwrap();
let response = app
.clone()
.oneshot(
Request::builder()
.method(Method::POST)
.uri(WORKSPACE_VERIFICATION_ACK_PATH)
.header(
header::AUTHORIZATION,
format!("Bearer {acknowledgement_token}"),
)
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(acknowledgement_body))
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
let ping_claims = WorkspaceCapabilityClaims {
operation: RUNTIME_PING_PERMISSION.to_string(),
method: "GET".to_string(),
path_and_query: "/v1/ping".to_string(),
body_digest: workspace_request_body_digest(&[]),
jti: "ping-token".to_string(),
..acknowledgement_claims
};
let ping_token = issue_workspace_capability_token(
&workspace_identity.signing_key().unwrap(),
&ping_claims,
)
.unwrap();
let response = app
.oneshot(
Request::builder()
.method(Method::GET)
.uri("/v1/ping")
.header(header::AUTHORIZATION, format!("Bearer {ping_token}"))
.header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn ping_requires_scoped_permission_and_returns_versioned_identity() {
let runtime = Runtime::new_memory();
@@ -2786,6 +3319,7 @@ mod tests {
.expect("runtime"),
local_token: Some(Arc::from("token")),
auth: None,
workspace_auth: None,
workdir_sessions: Arc::new(Mutex::new(HashMap::from([(
"session-1".to_string(),
RuntimeHttpWorkdirSession {
crates/worker-runtime/src/main.rs
+48 -4
View File
@@ -22,13 +22,16 @@ use worker_runtime::error::RuntimeError;
use worker_runtime::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
use worker_runtime::http_server::{
RuntimeHttpServerConfig, RuntimeHttpServerError, RuntimeHttpStoreSelection,
WorkspaceRuntimeHttpAuth,
};
use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend};
use worker_runtime::working_directory::RuntimeGitCacheMaterializer;
use worker_runtime::workspace_issuer::{
MAX_WORKSPACE_ISSUER_TRUST_RECORDS, WorkspaceIssuerTrustError, WorkspaceIssuerTrustMutation,
WorkspaceIssuerTrustRecord, add_workspace_issuer_trust, replace_workspace_issuer_trust,
revoke_workspace_issuer_trust, validate_workspace_issuer_trust_records,
FileWorkspaceClaimReplayProtection, FileWorkspaceRuntimeVerificationAuthority,
MAX_WORKSPACE_ISSUER_TRUST_RECORDS, RuntimeVerificationSigner, WorkspaceCapabilityVerifier,
WorkspaceIssuerTrustError, WorkspaceIssuerTrustMutation, WorkspaceIssuerTrustRecord,
add_workspace_issuer_trust, replace_workspace_issuer_trust, revoke_workspace_issuer_trust,
validate_workspace_issuer_trust_records,
};
use worker_runtime::{Runtime, RuntimeOptions};
@@ -88,6 +91,7 @@ fn run() -> Result<(), ProcessError> {
};
init_serve_tracing();
config.http.auth = load_runtime_http_auth(&config)?;
let workspace_http_auth = load_workspace_runtime_http_auth(&config)?;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
@@ -99,6 +103,16 @@ fn run() -> Result<(), ProcessError> {
eprintln!(
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
);
let server = if let Some(workspace_auth) = workspace_http_auth {
worker_runtime::http_server::serve_runtime_http_with_workspace_auth(
worker_runtime,
listener,
config.http.local_token,
config.http.auth,
workspace_auth,
)
.await
} else {
worker_runtime::http_server::serve_runtime_http_with_auth(
worker_runtime,
listener,
@@ -106,7 +120,8 @@ fn run() -> Result<(), ProcessError> {
config.http.auth,
)
.await
.map_err(ProcessError::from)
};
server.map_err(ProcessError::from)
})?;
Ok(())
}
@@ -880,6 +895,35 @@ fn write_secret_file(path: &Path, contents: &[u8]) -> Result<(), ProcessError> {
write_result
}
fn load_workspace_runtime_http_auth(
config: &ProcessConfig,
) -> Result<Option<WorkspaceRuntimeHttpAuth>, ProcessError> {
let auth = read_runtime_auth_file(&runtime_auth_path(config))?;
let Some(identity) = auth.identity else {
return Ok(None);
};
if auth.workspace_issuers.is_empty() {
return Ok(None);
}
let replay_path = runtime_auth_path(config).with_extension("workspace-replay.json");
let verifier = WorkspaceCapabilityVerifier::new(
auth.workspace_issuers,
Arc::new(FileWorkspaceClaimReplayProtection::new(replay_path)),
)
.map_err(|error| ProcessError::auth(format!("invalid Workspace issuer trust: {error}")))?;
let signer = RuntimeVerificationSigner::from_identity(&identity)
.map_err(|error| ProcessError::auth(format!("invalid Runtime identity: {error}")))?;
let verifications_path =
runtime_auth_path(config).with_extension("workspace-verifications.json");
Ok(Some(WorkspaceRuntimeHttpAuth {
verifier,
signer,
verifications: Arc::new(FileWorkspaceRuntimeVerificationAuthority::new(
verifications_path,
)),
}))
}
fn load_runtime_http_auth(
config: &ProcessConfig,
) -> Result<Option<RuntimeHttpAuthConfig>, ProcessError> {
crates/worker-runtime/src/workspace_issuer.rs
+869 -13
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/hosts.rs
+477 -11
View File
@@ -2,6 +2,10 @@ use crate::Error;
use crate::resource_broker::BackendResourceBroker;
#[cfg(test)]
use crate::resource_broker::BackendResourceTarget;
use crate::store::{
ControlPlaneStore, WorkspaceRuntimeAuthenticationMode, WorkspaceRuntimeBindingState,
};
use crate::workspace_signing_identity::WorkspaceSigningIdentityService;
use chrono::Utc;
use protocol::Segment;
use reqwest::blocking::{Client as BlockingHttpClient, RequestBuilder};
@@ -64,6 +68,11 @@ use worker_runtime::profile_archive::ProfileSourceArchive;
use worker_runtime::retention::{
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
};
use worker_runtime::workspace_issuer::{
WorkspaceCapabilityClaims, WorkspaceRuntimeVerificationAcknowledgement,
WorkspaceRuntimeVerificationChallenge, WorkspaceRuntimeVerificationReceipt,
WorkspaceRuntimeVerificationResponse, workspace_request_body_digest,
};
pub const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime";
const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host";
@@ -818,6 +827,32 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
))
}
fn activate_workspace_authorization(&self, _binding: crate::store::WorkspaceRuntimeBinding) {}
fn send_workspace_verification_challenge(
&self,
_challenge: &WorkspaceRuntimeVerificationChallenge,
_bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationResponse, RuntimePingFailure> {
Err(RuntimePingFailure::new(
RuntimePingFailureKind::Unsupported,
"runtime_workspace_verification_unsupported",
"Workspace Runtime verification is unavailable for this Runtime provider",
))
}
fn send_workspace_verification_acknowledgement(
&self,
_acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement,
_bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationReceipt, RuntimePingFailure> {
Err(RuntimePingFailure::new(
RuntimePingFailureKind::Unsupported,
"runtime_workspace_verification_unsupported",
"Workspace Runtime verification is unavailable for this Runtime provider",
))
}
fn list_hosts(&self, limit: usize) -> RuntimeList<HostSummary>;
fn list_workers(&self, limit: usize) -> RuntimeList<WorkerSummary>;
@@ -1877,6 +1912,48 @@ impl RuntimeRegistry {
runtime.ping()
}
pub fn activate_workspace_authorization(
&self,
runtime_id: &str,
binding: crate::store::WorkspaceRuntimeBinding,
) -> Result<(), RuntimeRegistryError> {
self.runtime(runtime_id)?
.activate_workspace_authorization(binding);
Ok(())
}
pub fn send_workspace_verification_challenge(
&self,
runtime_id: &str,
challenge: &WorkspaceRuntimeVerificationChallenge,
bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationResponse, RuntimePingFailure> {
let runtime = self.runtime(runtime_id).map_err(|_| {
RuntimePingFailure::new(
RuntimePingFailureKind::Configuration,
"runtime_verification_registration_unavailable",
"Registered Runtime binding is unavailable",
)
})?;
runtime.send_workspace_verification_challenge(challenge, bearer_token)
}
pub fn send_workspace_verification_acknowledgement(
&self,
runtime_id: &str,
acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement,
bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationReceipt, RuntimePingFailure> {
let runtime = self.runtime(runtime_id).map_err(|_| {
RuntimePingFailure::new(
RuntimePingFailureKind::Configuration,
"runtime_verification_registration_unavailable",
"Registered Runtime binding is unavailable",
)
})?;
runtime.send_workspace_verification_acknowledgement(acknowledgement, bearer_token)
}
fn runtimes_snapshot(&self) -> Vec<Arc<dyn WorkspaceWorkerRuntime>> {
self.runtimes
.read()
@@ -2851,6 +2928,7 @@ pub struct RemoteRuntimeConfig {
pub base_url: String,
pub bearer_token: Option<String>,
pub auth: Option<RemoteRuntimeAuthConfig>,
pub workspace_authorization: Option<WorkspaceRuntimeAuthorization>,
pub strict_public_egress: bool,
pub cached_worker_creation_available: bool,
pub cached_os: String,
@@ -2859,6 +2937,168 @@ pub struct RemoteRuntimeConfig {
pub timeout: Duration,
}
#[derive(Clone)]
pub struct WorkspaceRuntimeAuthorization {
store: Arc<dyn ControlPlaneStore>,
signing_identities: WorkspaceSigningIdentityService,
backend_url: String,
binding: Arc<RwLock<Option<crate::store::WorkspaceRuntimeBinding>>>,
}
impl std::fmt::Debug for WorkspaceRuntimeAuthorization {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("WorkspaceRuntimeAuthorization")
.field("backend_url", &"<backend-private>")
.finish_non_exhaustive()
}
}
impl WorkspaceRuntimeAuthorization {
pub fn new(
store: Arc<dyn ControlPlaneStore>,
signing_identities: WorkspaceSigningIdentityService,
backend_url: impl Into<String>,
binding: Option<crate::store::WorkspaceRuntimeBinding>,
) -> Self {
Self {
store,
signing_identities,
backend_url: backend_url.into(),
binding: Arc::new(RwLock::new(binding)),
}
}
fn activate(&self, binding: crate::store::WorkspaceRuntimeBinding) {
if let Ok(mut current) = self.binding.write() {
*current = Some(binding);
}
}
pub(crate) fn issue(
&self,
method: &str,
path_and_query: &str,
operation: &str,
worker_id: Option<&str>,
body: &[u8],
) -> Result<String, RuntimeDiagnostic> {
let binding = self
.binding
.read()
.map_err(|_| {
diagnostic(
"workspace_runtime_authorization_unavailable",
DiagnosticSeverity::Error,
"Workspace Runtime authorization is unavailable".to_string(),
)
})?
.clone()
.ok_or_else(|| {
diagnostic(
"workspace_runtime_verification_required",
DiagnosticSeverity::Error,
"Workspace Runtime binding is not verified".to_string(),
)
})?;
if binding.state != WorkspaceRuntimeBindingState::Verified
|| binding.authentication_mode != WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity
|| binding.revoked_at.is_some()
|| !self
.store
.workspace_runtime_binding_matches(&binding)
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_unavailable",
DiagnosticSeverity::Error,
error.to_string(),
)
})?
{
return Err(diagnostic(
"workspace_runtime_authorization_stale",
DiagnosticSeverity::Error,
"Workspace Runtime binding changed or was revoked".to_string(),
));
}
let identity = self
.signing_identities
.get_validated(&binding.workspace_id)
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_unavailable",
DiagnosticSeverity::Error,
error.to_string(),
)
})?;
let workspace_key_id = binding.workspace_key_id.as_deref().ok_or_else(|| {
diagnostic(
"workspace_runtime_authorization_invalid",
DiagnosticSeverity::Error,
"Workspace Runtime binding is missing its Workspace key".to_string(),
)
})?;
let trust_generation = binding.workspace_key_generation.ok_or_else(|| {
diagnostic(
"workspace_runtime_authorization_invalid",
DiagnosticSeverity::Error,
"Workspace Runtime binding is missing its trust generation".to_string(),
)
})?;
if identity.state != "active"
|| identity.key_id != workspace_key_id
|| identity.revision != trust_generation
|| !self
.store
.workspace_runtime_verification_matches(
&binding,
identity.revision,
trust_generation,
)
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_unavailable",
DiagnosticSeverity::Error,
error.to_string(),
)
})?
{
return Err(diagnostic(
"workspace_runtime_authorization_stale",
DiagnosticSeverity::Error,
"Workspace signing identity no longer matches the verified binding".to_string(),
));
}
let now = Utc::now().timestamp();
let claims = WorkspaceCapabilityClaims {
issuer: self.backend_url.clone(),
issuer_workspace_id: binding.workspace_id.clone(),
issuer_key_id: identity.key_id,
issuer_identity_revision: identity.revision,
trust_generation,
binding_revision: binding.binding_revision,
runtime_id: binding.runtime_id.clone(),
worker_id: worker_id.map(str::to_string),
operation: operation.to_string(),
method: method.to_string(),
path_and_query: path_and_query.to_string(),
body_digest: workspace_request_body_digest(body),
iat: now,
exp: now.saturating_add(60),
jti: uuid::Uuid::now_v7().to_string(),
};
self.signing_identities
.issue_workspace_capability(&binding.workspace_id, &claims)
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_sign_failed",
DiagnosticSeverity::Error,
error.to_string(),
)
})
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RemoteRuntimeAuthConfig {
pub server_id: String,
@@ -2903,6 +3143,7 @@ impl RemoteRuntimeConfig {
base_url: base_url.into(),
bearer_token,
auth: None,
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: false,
cached_os: "unknown".to_string(),
@@ -2943,6 +3184,7 @@ struct RemoteWorkdirAuthorization {
runtime_id: String,
workspace_id: String,
auth: Option<RemoteRuntimeAuthConfig>,
workspace_authorization: Option<WorkspaceRuntimeAuthorization>,
fallback_bearer_token: Option<String>,
}
@@ -2962,7 +3204,23 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization {
}
impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization {
fn bearer_token(&self) -> Result<String, WorkdirError> {
fn bearer_token(
&self,
method: &str,
path_and_query: &str,
body: &[u8],
) -> Result<String, WorkdirError> {
if let Some(authorization) = &self.workspace_authorization {
return authorization
.issue(
method,
path_and_query,
workspace_runtime_operation(method, path_and_query),
None,
body,
)
.map_err(|error| WorkdirError::Unavailable(error.message));
}
if let Some(auth) = self.auth.as_ref() {
let claims = capability_claims(
&auth.server_id,
@@ -3093,6 +3351,7 @@ pub struct RemoteWorkerRuntime {
workspace_id: String,
bearer_token: Option<String>,
auth: Option<RemoteRuntimeAuthConfig>,
workspace_authorization: Option<WorkspaceRuntimeAuthorization>,
cached_worker_creation_available: bool,
cached_os: String,
cached_arch: String,
@@ -3146,6 +3405,69 @@ fn remote_runtime_ping_transport_failure(error: reqwest::Error) -> RuntimePingFa
)
}
fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static str {
let path = path_and_query.split('?').next().unwrap_or(path_and_query);
if path == "/v1/ping" && method == "GET" {
return RUNTIME_PING_PERMISSION;
}
if path == "/v1/workers" && method == "GET" {
return "workers:list";
}
if path == "/v1/workers" && method == "POST" {
return "workers:create";
}
if (path == "/v1/working-directories/repository-access"
|| path == "/v1/repository-refs/observe")
&& method == "POST"
{
return "workdirs:operate";
}
if path.starts_with("/v1/workdir-sessions")
|| (path.starts_with("/v1/working-directories/") && path.ends_with("/sessions"))
{
return "workdirs:operate";
}
if path.starts_with("/v1/config-bundles")
|| path.starts_with("/v1/workspace-prompt-projections")
|| path.starts_with("/v1/working-directories")
{
return "workers:create";
}
if path.ends_with("/input")
|| path.ends_with("/restore")
|| path.ends_with("/workspace-api")
|| path.contains("/attachments")
{
return "workers:input";
}
if path.ends_with("/stop") || path.ends_with("/cancel") {
return "workers:stop";
}
if path == "/v1/protocol/ws" {
return "workers:list";
}
if path.ends_with("/protocol") || path.ends_with("/protocol/ws") {
return "workers:protocol";
}
if path.ends_with("/completions") {
return "workers:read";
}
if path.contains("/retention/") || (path.starts_with("/v1/workers/") && method == "DELETE") {
return "workers:delete";
}
if path.starts_with("/v1/workers/") && method == "GET" {
return "workers:read";
}
"runtime:read"
}
fn worker_id_from_remote_path(path_and_query: &str) -> Option<String> {
let path = path_and_query.split('?').next().unwrap_or(path_and_query);
let rest = path.strip_prefix("/v1/workers/")?;
let worker_id = rest.split('/').next()?;
(!worker_id.is_empty()).then(|| worker_id.to_string())
}
fn all_remote_runtime_permissions() -> Vec<String> {
[
"workers:list",
@@ -3226,6 +3548,7 @@ impl RemoteWorkerRuntime {
workspace_id,
bearer_token: config.bearer_token,
auth: config.auth,
workspace_authorization: config.workspace_authorization,
cached_worker_creation_available: config.cached_worker_creation_available,
cached_os: config.cached_os,
cached_arch: config.cached_arch,
@@ -3254,6 +3577,7 @@ impl RemoteWorkerRuntime {
runtime_id: self.runtime_id.clone(),
workspace_id: self.workspace_id.clone(),
auth: self.auth.clone(),
workspace_authorization: self.workspace_authorization.clone(),
fallback_bearer_token: self.bearer_token.clone(),
});
RemoteWorkdirSession::open_with_authorization(
@@ -3290,11 +3614,54 @@ impl RemoteWorkerRuntime {
format!("{base}/v1/workers/{worker_id}/protocol/ws")
}
fn post_bearer_json<T, U>(
&self,
path: &str,
body: &T,
bearer_token: &str,
) -> Result<U, RuntimePingFailure>
where
T: Serialize + ?Sized,
U: DeserializeOwned,
{
let response = self
.http
.post(self.endpoint(path))
.bearer_auth(bearer_token)
.json(body)
.send()
.map_err(|error| {
RuntimePingFailure::new(
RuntimePingFailureKind::NetworkUnreachable,
"runtime_workspace_verification_unreachable",
format!("Runtime verification request failed: {error}"),
)
})?;
let status = response.status();
if !status.is_success() {
return Err(RuntimePingFailure::new(
RuntimePingFailureKind::MalformedResponse,
"runtime_workspace_verification_rejected",
format!(
"Runtime verification request returned HTTP {}",
status.as_u16()
),
));
}
response.json::<U>().map_err(|error| {
RuntimePingFailure::new(
RuntimePingFailureKind::MalformedResponse,
"runtime_workspace_verification_invalid_response",
format!("Runtime verification response was invalid: {error}"),
)
})
}
fn get_json<T>(&self, path: &str) -> Result<T, RuntimeDiagnostic>
where
T: DeserializeOwned + Send + 'static,
{
self.send_json(path, self.http.get(self.endpoint(path)))
self.send_json(path, "GET", &[], self.http.get(self.endpoint(path)))
}
fn post_json<B, T>(&self, path: &str, body: &B) -> Result<T, RuntimeDiagnostic>
@@ -3302,7 +3669,22 @@ impl RemoteWorkerRuntime {
B: Serialize + ?Sized,
T: DeserializeOwned + Send + 'static,
{
self.send_json(path, self.http.post(self.endpoint(path)).json(body))
let body = serde_json::to_vec(body).map_err(|error| {
diagnostic(
"remote_runtime_request_encode_failed",
DiagnosticSeverity::Error,
error.to_string(),
)
})?;
self.send_json(
path,
"POST",
&body,
self.http
.post(self.endpoint(path))
.header(CONTENT_TYPE, "application/json")
.body(body.clone()),
)
}
fn post_bytes<T>(&self, path: &str, body: &[u8]) -> Result<T, RuntimeDiagnostic>
@@ -3311,7 +3693,12 @@ impl RemoteWorkerRuntime {
{
self.send_json(
path,
self.http.post(self.endpoint(path)).body(body.to_vec()),
"POST",
body,
self.http
.post(self.endpoint(path))
.header(CONTENT_TYPE, "application/json")
.body(body.to_vec()),
)
}
@@ -3319,7 +3706,7 @@ impl RemoteWorkerRuntime {
where
T: DeserializeOwned + Send + 'static,
{
self.send_json(path, self.http.delete(self.endpoint(path)))
self.send_json(path, "DELETE", &[], self.http.delete(self.endpoint(path)))
}
fn runtime_capability_token_with_permissions(
@@ -3364,10 +3751,23 @@ impl RemoteWorkerRuntime {
const PATH: &str = "/v1/ping";
let workspace_id = self.workspace_id.clone();
let bearer_token = self.bearer_token.clone();
let capability_token = self.runtime_capability_token_with_permissions(
let capability_token = match &self.workspace_authorization {
Some(authorization) => Some(
authorization
.issue("GET", PATH, RUNTIME_PING_PERMISSION, None, &[])
.map_err(|diagnostic| {
RuntimePingFailure::new(
RuntimePingFailureKind::Authentication,
diagnostic.code,
diagnostic.message,
)
})?,
),
None => self.runtime_capability_token_with_permissions(
PATH,
vec![RUNTIME_PING_PERMISSION.to_string()],
);
),
};
let request = self
.http
.get(self.endpoint(PATH))
@@ -3432,15 +3832,33 @@ impl RemoteWorkerRuntime {
})
}
fn send_json<T>(&self, path: &str, request: RequestBuilder) -> Result<T, RuntimeDiagnostic>
fn send_json<T>(
&self,
path: &str,
method: &str,
body: &[u8],
request: RequestBuilder,
) -> Result<T, RuntimeDiagnostic>
where
T: DeserializeOwned + Send + 'static,
{
let runtime_id = self.runtime_id.clone();
let workspace_id = self.workspace_id.clone();
let bearer_token = self.bearer_token.clone();
let capability_token = self.runtime_capability_token(path);
let capability_token = match &self.workspace_authorization {
Some(authorization) => Some(authorization.issue(
method,
path,
workspace_runtime_operation(method, path),
worker_id_from_remote_path(path).as_deref(),
body,
)?),
None => self.runtime_capability_token(path),
};
run_blocking_http(move || {
let request = request.header(CONTENT_TYPE, "application/json");
let request = request
.header(CONTENT_TYPE, "application/json")
.header(RUNTIME_WORKSPACE_SCOPE_HEADER, &workspace_id);
let request =
if let Some(token) = capability_token.as_deref().or(bearer_token.as_deref()) {
request.header(AUTHORIZATION, format!("Bearer {token}"))
@@ -3649,6 +4067,36 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
self.ping_http()
}
fn activate_workspace_authorization(&self, binding: crate::store::WorkspaceRuntimeBinding) {
if let Some(authorization) = &self.workspace_authorization {
authorization.activate(binding);
}
}
fn send_workspace_verification_challenge(
&self,
challenge: &WorkspaceRuntimeVerificationChallenge,
bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationResponse, RuntimePingFailure> {
self.post_bearer_json(
worker_runtime::workspace_issuer::WORKSPACE_VERIFICATION_CHALLENGE_PATH,
challenge,
bearer_token,
)
}
fn send_workspace_verification_acknowledgement(
&self,
acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement,
bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationReceipt, RuntimePingFailure> {
self.post_bearer_json(
worker_runtime::workspace_issuer::WORKSPACE_VERIFICATION_ACK_PATH,
acknowledgement,
bearer_token,
)
}
fn list_hosts(&self, limit: usize) -> RuntimeList<HostSummary> {
if limit == 0 {
return RuntimeList::new(Vec::new(), Vec::new());
@@ -3936,12 +4384,30 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
workspace_api: Some(workspace_api),
memory_settings: request.resolved_memory_settings.clone(),
};
let create_body = match serde_json::to_vec(&create) {
Ok(body) => body,
Err(error) => {
return WorkerSpawnResult {
state: WorkerOperationState::Rejected,
worker: None,
acceptance_evidence: Vec::new(),
diagnostics: vec![diagnostic(
"remote_runtime_request_encode_failed",
DiagnosticSeverity::Error,
error.to_string(),
)],
};
}
};
match self.send_json::<RuntimeHttpWorkerResponse>(
"/v1/workers",
"POST",
&create_body,
self.http
.post(self.endpoint("/v1/workers"))
.timeout(REMOTE_WORKER_CREATE_TIMEOUT)
.json(&create),
.header(CONTENT_TYPE, "application/json")
.body(create_body.clone()),
) {
Ok(response) => WorkerSpawnResult {
state: WorkerOperationState::Accepted,
crates/workspace-server/src/latest_schema.sql
+22
View File
@@ -462,6 +462,28 @@ CREATE TABLE workspace_runtime_bindings (
(state != 'revoked' AND revoked_at IS NULL)
)
);
CREATE TABLE workspace_runtime_verifications (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
binding_revision INTEGER NOT NULL CHECK(binding_revision > 0),
workspace_key_id TEXT NOT NULL,
workspace_identity_revision INTEGER NOT NULL CHECK(workspace_identity_revision > 0),
workspace_trust_generation INTEGER NOT NULL CHECK(workspace_trust_generation > 0),
runtime_public_key_fingerprint TEXT NOT NULL,
runtime_identity_revision INTEGER NOT NULL CHECK(runtime_identity_revision > 0),
challenge_id TEXT NOT NULL,
state TEXT NOT NULL CHECK(state IN ('pending', 'verified', 'failed')),
last_outcome TEXT NOT NULL,
verified_at TEXT,
checked_at TEXT NOT NULL,
PRIMARY KEY(workspace_id, runtime_id),
FOREIGN KEY(workspace_id, runtime_id)
REFERENCES workspace_runtime_bindings(workspace_id, runtime_id) ON DELETE CASCADE,
CHECK((state = 'verified' AND verified_at IS NOT NULL)
OR (state != 'verified' AND verified_at IS NULL))
);
CREATE INDEX workspace_runtime_verifications_state_idx
ON workspace_runtime_verifications(workspace_id, state, checked_at DESC);
CREATE TABLE workspace_runtime_binding_audit (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
crates/workspace-server/src/runtime_subscription.rs
+6
View File
@@ -970,6 +970,12 @@ fn runtime_token(
config: &RemoteRuntimeConfig,
workspace_id: &str,
) -> Result<Option<String>, String> {
if let Some(authorization) = config.workspace_authorization.as_ref() {
return authorization
.issue("GET", "/v1/protocol/ws", "workers:list", None, &[])
.map(Some)
.map_err(|error| error.message);
}
let Some(auth) = config.auth.as_ref() else {
return Ok(config.bearer_token.clone());
};
crates/workspace-server/src/server.rs
+286 -20
View File
@@ -61,6 +61,12 @@ use worker_runtime::http_server::{
};
use worker_runtime::resource::{BackendResourceError, BackendResourceFetchRequest};
use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend};
use worker_runtime::workspace_issuer::{
WORKSPACE_VERIFICATION_ACK_PATH, WORKSPACE_VERIFICATION_CHALLENGE_PATH,
WORKSPACE_VERIFICATION_OPERATION, WorkspaceCapabilityClaims,
WorkspaceRuntimeVerificationAcknowledgement, WorkspaceRuntimeVerificationChallenge,
verify_runtime_verification_response, workspace_request_body_digest,
};
use workspace_api::{
ActorAuthMethod, AuthBootstrapUserRequest, AuthPublicConfig, AuthUserResponse,
AuthenticatedUser, BrowserCreateWorkerResponse, BrowserWorkspaceOrchestratorResponse,
@@ -128,8 +134,8 @@ use crate::hosts::{
WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult,
WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult,
WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerTicketAssignmentRequest,
WorkerWorkspaceSummary, is_disallowed_remote_runtime_address, worker_spawn_create_fingerprint,
workspace_worker_summary,
WorkerWorkspaceSummary, WorkspaceRuntimeAuthorization, is_disallowed_remote_runtime_address,
worker_spawn_create_fingerprint, workspace_worker_summary,
};
use crate::identity::WorkspaceIdentity;
use crate::memory_backend::execute_memory_backend_operation_with_authority;
@@ -2229,23 +2235,6 @@ impl WorkspaceApi {
RuntimeSubscriptionBroker::new(config.workspace_id.clone());
runtime_subscription_broker
.register_embedded_runtime(embedded_runtime_id, embedded_subscription_runtime);
for remote_config in config.remote_runtime_sources.iter().cloned() {
let remote_runtime = RemoteWorkerRuntime::new(
remote_config.clone(),
config.workspace_id.clone(),
config
.backend_base_url
.clone()
.unwrap_or_else(|| "http://127.0.0.1:8787".to_string()),
)
.map(|host| host.with_resource_broker(resource_broker.clone()))
.map_err(|err| err.into_error())?;
runtime.register(remote_runtime);
runtime_subscription_broker.register_remote_runtime(remote_config);
}
let runtime = Arc::new(runtime);
let companion = Arc::new(CompanionConsole::disabled());
let observation_proxy = BackendObservationProxy::new(config.runtime_event_sources.clone());
let config_store = Arc::new(crate::SqliteWorkspaceStore::open(
config.database_path.clone(),
)?);
@@ -2260,6 +2249,40 @@ impl WorkspaceApi {
let signing_identities =
WorkspaceSigningIdentityService::new(store.clone(), signing_materials);
signing_identities.get_validated(&config.workspace_id)?;
let backend_url = config
.backend_base_url
.clone()
.unwrap_or_else(|| "http://127.0.0.1:8787".to_string());
for mut remote_config in config.remote_runtime_sources.iter().cloned() {
let current_binding = store
.get_workspace_runtime_binding(&config.workspace_id, &remote_config.runtime_id)
.await?;
if current_binding.as_ref().is_some_and(|binding| {
binding.authentication_mode == StoredRuntimeAuthenticationMode::WorkspaceIdentity
&& binding.revoked_at.is_none()
}) {
let verified_binding = current_binding
.filter(|binding| binding.state == StoredRuntimeBindingState::Verified);
remote_config.workspace_authorization = Some(WorkspaceRuntimeAuthorization::new(
store.clone(),
signing_identities.clone(),
backend_url.clone(),
verified_binding,
));
}
let remote_runtime = RemoteWorkerRuntime::new(
remote_config.clone(),
config.workspace_id.clone(),
backend_url.clone(),
)
.map(|host| host.with_resource_broker(resource_broker.clone()))
.map_err(|err| err.into_error())?;
runtime.register(remote_runtime);
runtime_subscription_broker.register_remote_runtime(remote_config);
}
let runtime = Arc::new(runtime);
let companion = Arc::new(CompanionConsole::disabled());
let observation_proxy = BackendObservationProxy::new(config.runtime_event_sources.clone());
let config_schema_registry = crate::config_source::WorkspaceConfigSchemaRegistry::default()
.with_provider(Arc::new(
crate::profile_settings::ProfileConfigSchemaProvider,
@@ -13674,6 +13697,197 @@ async fn delete_remote_runtime(
Ok(StatusCode::NO_CONTENT)
}
async fn perform_workspace_runtime_verification(
api: &WorkspaceApi,
runtime: Arc<RuntimeRegistry>,
binding: &WorkspaceRuntimeBinding,
) -> std::result::Result<WorkspaceRuntimeBinding, String> {
if binding.authentication_mode != StoredRuntimeAuthenticationMode::WorkspaceIdentity {
return Ok(binding.clone());
}
if binding.state == crate::store::WorkspaceRuntimeBindingState::Revoked
|| binding.revoked_at.is_some()
{
return Err("Runtime binding is revoked".to_string());
}
let backend_url = api.config.backend_base_url.as_deref().ok_or_else(|| {
"Workspace identity verification requires configured backend_base_url".to_string()
})?;
let identity = api
.signing_identities
.get_validated(&binding.workspace_id)
.map_err(|error| error.to_string())?;
if identity.state != "active" {
return Err("Workspace signing identity is not active".to_string());
}
let workspace_key_id = binding
.workspace_key_id
.as_deref()
.ok_or_else(|| "Runtime binding is missing the Workspace key identity".to_string())?;
let workspace_trust_generation = binding
.workspace_key_generation
.ok_or_else(|| "Runtime binding is missing the Workspace trust generation".to_string())?;
if identity.key_id != workspace_key_id || identity.revision != workspace_trust_generation {
return Err(
"Runtime binding no longer matches the active Workspace or Runtime identity"
.to_string(),
);
}
let now = Utc::now();
let expires_at = (now + Duration::seconds(60)).timestamp();
let challenge = WorkspaceRuntimeVerificationChallenge {
challenge_id: Uuid::now_v7().to_string(),
workspace_id: binding.workspace_id.clone(),
runtime_id: binding.runtime_id.clone(),
binding_revision: binding.binding_revision,
workspace_key_id: workspace_key_id.to_string(),
workspace_identity_revision: identity.revision,
workspace_trust_generation,
runtime_public_key_fingerprint: binding.public_key_fingerprint.clone(),
runtime_identity_revision: 1,
workspace_nonce: Uuid::now_v7().to_string(),
expires_at,
};
let checked_at = now.to_rfc3339_opts(SecondsFormat::Millis, true);
let pending = crate::store::WorkspaceRuntimeVerificationEvidence {
workspace_id: binding.workspace_id.clone(),
runtime_id: binding.runtime_id.clone(),
binding_revision: binding.binding_revision,
workspace_key_id: workspace_key_id.to_string(),
workspace_identity_revision: identity.revision,
workspace_trust_generation,
runtime_public_key_fingerprint: binding.public_key_fingerprint.clone(),
runtime_identity_revision: 1,
challenge_id: challenge.challenge_id.clone(),
state: "pending".to_string(),
last_outcome: "challenge_issued".to_string(),
verified_at: None,
checked_at: checked_at.clone(),
};
api.store
.record_workspace_runtime_verification_attempt(&pending)
.await
.map_err(|error| error.to_string())?;
let challenge_body = serde_json::to_vec(&challenge).map_err(|error| error.to_string())?;
let challenge_claims = WorkspaceCapabilityClaims {
issuer: backend_url.to_string(),
issuer_workspace_id: binding.workspace_id.clone(),
issuer_key_id: identity.key_id.clone(),
issuer_identity_revision: identity.revision,
trust_generation: workspace_trust_generation,
binding_revision: binding.binding_revision,
runtime_id: binding.runtime_id.clone(),
worker_id: None,
operation: WORKSPACE_VERIFICATION_OPERATION.to_string(),
method: "POST".to_string(),
path_and_query: WORKSPACE_VERIFICATION_CHALLENGE_PATH.to_string(),
body_digest: workspace_request_body_digest(&challenge_body),
iat: now.timestamp(),
exp: expires_at,
jti: Uuid::now_v7().to_string(),
};
let challenge_token = api
.signing_identities
.issue_workspace_capability(&binding.workspace_id, &challenge_claims)
.map_err(|error| error.to_string())?;
let challenge_runtime = runtime.clone();
let challenge_runtime_id = binding.runtime_id.clone();
let challenge_request = challenge.clone();
let response = tokio::task::spawn_blocking(move || {
challenge_runtime.send_workspace_verification_challenge(
&challenge_runtime_id,
&challenge_request,
&challenge_token,
)
})
.await
.map_err(|_| "Runtime verification challenge task failed".to_string())?
.map_err(|failure| failure.diagnostic.message)?;
verify_runtime_verification_response(
&response,
&challenge,
&binding.public_key,
Utc::now().timestamp(),
)
.map_err(|error| error.to_string())?;
let response_bytes = serde_json::to_vec(&response).map_err(|error| error.to_string())?;
let acknowledgement = WorkspaceRuntimeVerificationAcknowledgement {
challenge_id: response.challenge_id.clone(),
workspace_id: response.workspace_id.clone(),
runtime_id: response.runtime_id.clone(),
binding_revision: response.binding_revision,
workspace_key_id: response.workspace_key_id.clone(),
workspace_identity_revision: response.workspace_identity_revision,
workspace_trust_generation: response.workspace_trust_generation,
runtime_public_key_fingerprint: response.runtime_public_key_fingerprint.clone(),
runtime_identity_revision: response.runtime_identity_revision,
workspace_nonce: response.workspace_nonce.clone(),
runtime_nonce: response.runtime_nonce.clone(),
response_digest: workspace_request_body_digest(&response_bytes),
response: response.clone(),
expires_at: response.expires_at,
};
let acknowledgement_body =
serde_json::to_vec(&acknowledgement).map_err(|error| error.to_string())?;
let acknowledgement_claims = WorkspaceCapabilityClaims {
issuer: backend_url.to_string(),
issuer_workspace_id: binding.workspace_id.clone(),
issuer_key_id: identity.key_id.clone(),
issuer_identity_revision: identity.revision,
trust_generation: workspace_trust_generation,
binding_revision: binding.binding_revision,
runtime_id: binding.runtime_id.clone(),
worker_id: None,
operation: WORKSPACE_VERIFICATION_OPERATION.to_string(),
method: "POST".to_string(),
path_and_query: WORKSPACE_VERIFICATION_ACK_PATH.to_string(),
body_digest: workspace_request_body_digest(&acknowledgement_body),
iat: Utc::now().timestamp(),
exp: expires_at,
jti: Uuid::now_v7().to_string(),
};
let acknowledgement_token = api
.signing_identities
.issue_workspace_capability(&binding.workspace_id, &acknowledgement_claims)
.map_err(|error| error.to_string())?;
let acknowledgement_runtime = runtime;
let acknowledgement_runtime_id = binding.runtime_id.clone();
let acknowledgement_request = acknowledgement.clone();
let receipt = tokio::task::spawn_blocking(move || {
acknowledgement_runtime.send_workspace_verification_acknowledgement(
&acknowledgement_runtime_id,
&acknowledgement_request,
&acknowledgement_token,
)
})
.await
.map_err(|_| "Runtime verification acknowledgement task failed".to_string())?
.map_err(|failure| failure.diagnostic.message)?;
if receipt.challenge_id != challenge.challenge_id
|| receipt.workspace_id != binding.workspace_id
|| receipt.runtime_id != binding.runtime_id
|| receipt.binding_revision != binding.binding_revision
{
return Err("Runtime verification acknowledgement receipt mismatched".to_string());
}
let verified_at = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true);
let verified = crate::store::WorkspaceRuntimeVerificationEvidence {
state: "verified".to_string(),
last_outcome: "verified".to_string(),
verified_at: Some(verified_at.clone()),
checked_at: verified_at,
..pending
};
api.store
.complete_workspace_runtime_verification(&verified)
.await
.map_err(|error| error.to_string())
}
async fn test_runtime_connection(
State(api): State<WorkspaceApi>,
AxumPath(runtime_id): AxumPath<String>,
@@ -13686,12 +13900,61 @@ async fn test_runtime_connection(
}
.into());
}
api.store
let binding = api
.store
.get_workspace_runtime_binding(api.workspace_id(), &runtime_id)
.await?
.filter(|binding| binding.revoked_at.is_none())
.ok_or_else(|| Error::UnknownRuntime(runtime_id.clone()))?;
if binding.authentication_mode == StoredRuntimeAuthenticationMode::WorkspaceIdentity {
match perform_workspace_runtime_verification(&api, api.runtime.clone(), &binding).await {
Ok(verified_binding) => {
api.runtime_binding_expectations
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
(
verified_binding.workspace_id.clone(),
verified_binding.runtime_id.clone(),
),
verified_binding.clone(),
);
api.runtime
.activate_workspace_authorization(&runtime_id, verified_binding.clone())
.map_err(|error| Error::Store(format!("{error:?}")))?;
}
Err(message) => {
if let Ok(Some(mut evidence)) = api
.store
.get_workspace_runtime_verification(api.workspace_id(), &runtime_id)
.await
{
evidence.state = "failed".to_string();
evidence.last_outcome = "verification_failed".to_string();
evidence.verified_at = None;
evidence.checked_at = Utc::now().to_rfc3339();
let _ = api
.store
.record_workspace_runtime_verification_attempt(&evidence)
.await;
}
return Ok(Json(runtime_connection_test_failure(
api.workspace_id(),
&runtime_id,
Utc::now().to_rfc3339(),
RuntimeConnectionTestFailureKind::Authentication,
None,
RuntimeDiagnostic::new(
"runtime_workspace_verification_failed",
"error",
message,
),
)));
}
}
}
let checked_at = Utc::now().to_rfc3339();
let runtime = api.runtime.clone();
let ping_runtime_id = runtime_id.clone();
@@ -18283,6 +18546,7 @@ mod tests {
server_id: "server-test".to_owned(),
server_private_key: "unused".to_owned(),
}),
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: true,
cached_os: "test".to_owned(),
@@ -24847,6 +25111,7 @@ mod tests {
server_id: "server-main".to_string(),
server_private_key: identity.private_key.clone(),
}),
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: true,
cached_os: "test".to_string(),
@@ -26374,6 +26639,7 @@ mod tests {
base_url: endpoint,
bearer_token: Some("test-connection-token".to_string()),
auth: None,
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: true,
cached_os: "linux".to_string(),
crates/workspace-server/src/store.rs
+511 -5
View File
@@ -18,7 +18,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore;
use crate::{Error, Result};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 55;
const LATEST_SCHEMA_VERSION: i64 = 56;
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings";
const RUNTIME_BINDING_AUDIT_MIGRATION_NAME: &str = "workspace Runtime binding revision and audit";
@@ -26,6 +26,8 @@ const WORKSPACE_DELETION_MIGRATION_NAME: &str = "durable Workspace deletion oper
const WORKSPACE_SIGNING_IDENTITY_MIGRATION_NAME: &str = "Workspace signing identity authority";
const WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME: &str =
"Workspace Runtime binding state and identity mode";
const WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME: &str =
"Workspace-signed Runtime verification evidence";
const MIGRATIONS: &[Migration] = &[
Migration {
@@ -53,6 +55,11 @@ const MIGRATIONS: &[Migration] = &[
name: WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME,
apply: migrate_workspace_runtime_binding_state_v54_to_v55,
},
Migration {
version: 56,
name: WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME,
apply: migrate_workspace_runtime_verification_v55_to_v56,
},
];
#[derive(Clone, Copy)]
@@ -208,6 +215,23 @@ pub struct WorkspaceSigningIdentityProvisioningOperation {
pub completed_at: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkspaceRuntimeVerificationEvidence {
pub workspace_id: String,
pub runtime_id: String,
pub binding_revision: u64,
pub workspace_key_id: String,
pub workspace_identity_revision: u64,
pub workspace_trust_generation: u64,
pub runtime_public_key_fingerprint: String,
pub runtime_identity_revision: u64,
pub challenge_id: String,
pub state: String,
pub last_outcome: String,
pub verified_at: Option<String>,
pub checked_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkspaceRuntimeBinding {
pub workspace_id: String,
@@ -732,6 +756,25 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
) -> Result<WorkspaceSigningIdentityRecord>;
fn workspace_runtime_binding_matches(&self, expected: &WorkspaceRuntimeBinding)
-> Result<bool>;
fn workspace_runtime_verification_matches(
&self,
binding: &WorkspaceRuntimeBinding,
workspace_identity_revision: u64,
workspace_trust_generation: u64,
) -> Result<bool>;
async fn get_workspace_runtime_verification(
&self,
workspace_id: &str,
runtime_id: &str,
) -> Result<Option<WorkspaceRuntimeVerificationEvidence>>;
async fn record_workspace_runtime_verification_attempt(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<()>;
async fn complete_workspace_runtime_verification(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<WorkspaceRuntimeBinding>;
async fn get_workspace_runtime_binding(
&self,
workspace_id: &str,
@@ -2179,6 +2222,183 @@ impl SqliteWorkspaceStore {
})
}
pub fn workspace_runtime_verification_matches(
&self,
binding: &WorkspaceRuntimeBinding,
workspace_identity_revision: u64,
workspace_trust_generation: u64,
) -> Result<bool> {
let Some(evidence) =
self.get_workspace_runtime_verification(&binding.workspace_id, &binding.runtime_id)?
else {
return Ok(false);
};
Ok(evidence.state == "verified"
&& evidence.verified_at.is_some()
&& evidence.binding_revision == binding.binding_revision
&& evidence.workspace_key_id == binding.workspace_key_id.as_deref().unwrap_or_default()
&& evidence.workspace_identity_revision == workspace_identity_revision
&& evidence.workspace_trust_generation == workspace_trust_generation
&& evidence.runtime_public_key_fingerprint == binding.public_key_fingerprint
&& evidence.runtime_identity_revision > 0)
}
pub fn get_workspace_runtime_verification(
&self,
workspace_id: &str,
runtime_id: &str,
) -> Result<Option<WorkspaceRuntimeVerificationEvidence>> {
validate_identifier("workspace_id", workspace_id)?;
validate_identifier("runtime_id", runtime_id)?;
self.with_conn(|conn| {
conn.query_row(
r#"SELECT workspace_id, runtime_id, binding_revision, workspace_key_id,
workspace_identity_revision, workspace_trust_generation,
runtime_public_key_fingerprint, runtime_identity_revision,
challenge_id, state, last_outcome, verified_at, checked_at
FROM workspace_runtime_verifications
WHERE workspace_id = ?1 AND runtime_id = ?2"#,
params![workspace_id, runtime_id],
read_workspace_runtime_verification,
)
.optional()
.map_err(Error::from)
})
}
pub fn record_workspace_runtime_verification_attempt(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<()> {
validate_workspace_runtime_verification(evidence)?;
self.with_conn(|conn| {
conn.execute(
r#"INSERT INTO workspace_runtime_verifications (
workspace_id, runtime_id, binding_revision, workspace_key_id,
workspace_identity_revision, workspace_trust_generation,
runtime_public_key_fingerprint, runtime_identity_revision,
challenge_id, state, last_outcome, verified_at, checked_at
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)
ON CONFLICT(workspace_id, runtime_id) DO UPDATE SET
binding_revision = excluded.binding_revision,
workspace_key_id = excluded.workspace_key_id,
workspace_identity_revision = excluded.workspace_identity_revision,
workspace_trust_generation = excluded.workspace_trust_generation,
runtime_public_key_fingerprint = excluded.runtime_public_key_fingerprint,
runtime_identity_revision = excluded.runtime_identity_revision,
challenge_id = excluded.challenge_id,
state = excluded.state,
last_outcome = excluded.last_outcome,
verified_at = excluded.verified_at,
checked_at = excluded.checked_at"#,
params![
evidence.workspace_id,
evidence.runtime_id,
evidence.binding_revision,
evidence.workspace_key_id,
evidence.workspace_identity_revision,
evidence.workspace_trust_generation,
evidence.runtime_public_key_fingerprint,
evidence.runtime_identity_revision,
evidence.challenge_id,
evidence.state,
evidence.last_outcome,
evidence.verified_at,
evidence.checked_at,
],
)?;
Ok(())
})
}
pub fn complete_workspace_runtime_verification(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<WorkspaceRuntimeBinding> {
validate_workspace_runtime_verification(evidence)?;
if evidence.state != "verified" || evidence.verified_at.is_none() {
return Err(Error::Store(
"completed Runtime verification evidence must be verified".to_string(),
));
}
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let changed = tx.execute(
r#"UPDATE workspace_runtime_bindings
SET state = 'verified', updated_at = ?7
WHERE workspace_id = ?1 AND runtime_id = ?2
AND binding_revision = ?3
AND state IN ('configured', 'verified')
AND authentication_mode = 'workspace_identity'
AND workspace_key_id = ?4
AND workspace_key_generation = ?5
AND public_key_fingerprint = ?6
AND revoked_at IS NULL"#,
params![
evidence.workspace_id,
evidence.runtime_id,
evidence.binding_revision,
evidence.workspace_key_id,
evidence.workspace_trust_generation,
evidence.runtime_public_key_fingerprint,
evidence.checked_at,
],
)?;
if changed != 1 {
return Err(Error::RuntimeBindingConflict(
"Runtime verification evidence no longer matches the configured binding"
.to_string(),
));
}
tx.execute(
r#"INSERT INTO workspace_runtime_verifications (
workspace_id, runtime_id, binding_revision, workspace_key_id,
workspace_identity_revision, workspace_trust_generation,
runtime_public_key_fingerprint, runtime_identity_revision,
challenge_id, state, last_outcome, verified_at, checked_at
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)
ON CONFLICT(workspace_id, runtime_id) DO UPDATE SET
binding_revision = excluded.binding_revision,
workspace_key_id = excluded.workspace_key_id,
workspace_identity_revision = excluded.workspace_identity_revision,
workspace_trust_generation = excluded.workspace_trust_generation,
runtime_public_key_fingerprint = excluded.runtime_public_key_fingerprint,
runtime_identity_revision = excluded.runtime_identity_revision,
challenge_id = excluded.challenge_id,
state = excluded.state,
last_outcome = excluded.last_outcome,
verified_at = excluded.verified_at,
checked_at = excluded.checked_at"#,
params![
evidence.workspace_id,
evidence.runtime_id,
evidence.binding_revision,
evidence.workspace_key_id,
evidence.workspace_identity_revision,
evidence.workspace_trust_generation,
evidence.runtime_public_key_fingerprint,
evidence.runtime_identity_revision,
evidence.challenge_id,
evidence.state,
evidence.last_outcome,
evidence.verified_at,
evidence.checked_at,
],
)?;
let binding = tx.query_row(
r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key,
public_key_fingerprint, binding_revision, state, authentication_mode,
workspace_key_id, workspace_key_generation, created_at, updated_at, revoked_at
FROM workspace_runtime_bindings
WHERE workspace_id = ?1 AND runtime_id = ?2"#,
params![evidence.workspace_id, evidence.runtime_id],
read_workspace_runtime_binding,
)?;
tx.commit()?;
Ok(binding)
})
}
pub fn list_workspace_runtime_binding_audit(
&self,
workspace_id: &str,
@@ -2919,6 +3139,42 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
.is_some_and(|binding| binding == *expected && binding.revoked_at.is_none()))
}
fn workspace_runtime_verification_matches(
&self,
binding: &WorkspaceRuntimeBinding,
workspace_identity_revision: u64,
workspace_trust_generation: u64,
) -> Result<bool> {
SqliteWorkspaceStore::workspace_runtime_verification_matches(
self,
binding,
workspace_identity_revision,
workspace_trust_generation,
)
}
async fn get_workspace_runtime_verification(
&self,
workspace_id: &str,
runtime_id: &str,
) -> Result<Option<WorkspaceRuntimeVerificationEvidence>> {
SqliteWorkspaceStore::get_workspace_runtime_verification(self, workspace_id, runtime_id)
}
async fn record_workspace_runtime_verification_attempt(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<()> {
SqliteWorkspaceStore::record_workspace_runtime_verification_attempt(self, evidence)
}
async fn complete_workspace_runtime_verification(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<WorkspaceRuntimeBinding> {
SqliteWorkspaceStore::complete_workspace_runtime_verification(self, evidence)
}
async fn get_workspace_runtime_binding(
&self,
workspace_id: &str,
@@ -6424,6 +6680,56 @@ fn account_select_sql(where_clause: &str) -> String {
)
}
fn read_workspace_runtime_verification(
row: &rusqlite::Row<'_>,
) -> rusqlite::Result<WorkspaceRuntimeVerificationEvidence> {
Ok(WorkspaceRuntimeVerificationEvidence {
workspace_id: row.get(0)?,
runtime_id: row.get(1)?,
binding_revision: row.get(2)?,
workspace_key_id: row.get(3)?,
workspace_identity_revision: row.get(4)?,
workspace_trust_generation: row.get(5)?,
runtime_public_key_fingerprint: row.get(6)?,
runtime_identity_revision: row.get(7)?,
challenge_id: row.get(8)?,
state: row.get(9)?,
last_outcome: row.get(10)?,
verified_at: row.get(11)?,
checked_at: row.get(12)?,
})
}
fn validate_workspace_runtime_verification(
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<()> {
for (field, value) in [
("workspace_id", evidence.workspace_id.as_str()),
("runtime_id", evidence.runtime_id.as_str()),
("workspace_key_id", evidence.workspace_key_id.as_str()),
("challenge_id", evidence.challenge_id.as_str()),
] {
validate_identifier(field, value)?;
}
if evidence.binding_revision == 0
|| evidence.workspace_identity_revision == 0
|| evidence.workspace_trust_generation == 0
|| evidence.runtime_identity_revision == 0
{
return Err(Error::InvalidInput(
"Runtime verification revisions and generations must be positive".to_string(),
));
}
validate_non_empty(
"runtime_public_key_fingerprint",
&evidence.runtime_public_key_fingerprint,
)?;
validate_non_empty("verification state", &evidence.state)?;
validate_non_empty("verification outcome", &evidence.last_outcome)?;
validate_non_empty("checked_at", &evidence.checked_at)?;
Ok(())
}
fn read_workspace_runtime_binding(
row: &rusqlite::Row<'_>,
) -> rusqlite::Result<WorkspaceRuntimeBinding> {
@@ -7817,6 +8123,80 @@ fn migrate_workspace_runtime_binding_state_v54_to_v55(conn: &Connection) -> Resu
Ok(())
}
fn migrate_workspace_runtime_verification_v55_to_v56(conn: &Connection) -> Result<()> {
let current = current_schema_version(conn)?;
if current != 55 {
return Err(Error::Store(format!(
"expected schema version 55 before {WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME} migration, found {current}"
)));
}
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
tx.execute_batch(
r#"
CREATE TABLE IF NOT EXISTS workspace_runtime_verifications (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
binding_revision INTEGER NOT NULL CHECK(binding_revision > 0),
workspace_key_id TEXT NOT NULL,
workspace_identity_revision INTEGER NOT NULL CHECK(workspace_identity_revision > 0),
workspace_trust_generation INTEGER NOT NULL CHECK(workspace_trust_generation > 0),
runtime_public_key_fingerprint TEXT NOT NULL,
runtime_identity_revision INTEGER NOT NULL CHECK(runtime_identity_revision > 0),
challenge_id TEXT NOT NULL,
state TEXT NOT NULL CHECK(state IN ('pending', 'verified', 'failed')),
last_outcome TEXT NOT NULL,
verified_at TEXT,
checked_at TEXT NOT NULL,
PRIMARY KEY(workspace_id, runtime_id),
FOREIGN KEY(workspace_id, runtime_id)
REFERENCES workspace_runtime_bindings(workspace_id, runtime_id)
ON DELETE CASCADE,
CHECK((state = 'verified' AND verified_at IS NOT NULL)
OR (state != 'verified' AND verified_at IS NULL))
);
CREATE INDEX IF NOT EXISTS workspace_runtime_verifications_state_idx
ON workspace_runtime_verifications(workspace_id, state, checked_at DESC);
"#,
)?;
verify_workspace_runtime_verification_schema(&tx)?;
tx.execute(
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)",
params![56_i64, WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME],
)?;
tx.commit()?;
Ok(())
}
fn verify_workspace_runtime_verification_schema(conn: &Connection) -> Result<()> {
let actual = table_columns(conn, "workspace_runtime_verifications")?
.into_iter()
.collect::<BTreeSet<_>>();
let expected = [
"workspace_id",
"runtime_id",
"binding_revision",
"workspace_key_id",
"workspace_identity_revision",
"workspace_trust_generation",
"runtime_public_key_fingerprint",
"runtime_identity_revision",
"challenge_id",
"state",
"last_outcome",
"verified_at",
"checked_at",
]
.into_iter()
.map(str::to_string)
.collect::<BTreeSet<_>>();
if actual != expected {
return Err(Error::Store(format!(
"workspace_runtime_verifications schema does not match schema-56: {actual:?}"
)));
}
Ok(())
}
fn verify_workspace_signing_identity_schema(conn: &Connection) -> Result<()> {
for (table, expected) in [
(
@@ -8679,6 +9059,7 @@ fn apply_migrations(conn: &Connection) -> Result<()> {
verify_schema_history(conn, LATEST_SCHEMA_VERSION)?;
verify_workspace_runtime_binding_schema(conn)?;
verify_workspace_runtime_verification_schema(conn)?;
verify_workspace_deletion_schema(conn)?;
verify_workspace_signing_identity_schema(conn)
}
@@ -8902,6 +9283,10 @@ mod tests {
version: 55,
name: WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME.to_string(),
},
WorkspaceSchemaMigrationStep {
version: 56,
name: WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME.to_string(),
},
]
);
@@ -8928,6 +9313,10 @@ mod tests {
55,
WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME.to_string(),
),
(
56,
WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME.to_string(),
),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -8997,7 +9386,7 @@ mod tests {
.iter()
.map(|migration| migration.version)
.collect::<Vec<_>>(),
vec![52, 53, 54, 55]
vec![52, 53, 54, 55, 56]
);
SqliteWorkspaceStore::migrate_database(&path).unwrap();
let conn = Connection::open(&path).unwrap();
@@ -9005,7 +9394,7 @@ mod tests {
current_schema_version(&conn).unwrap(),
LATEST_SCHEMA_VERSION
);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 6);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 7);
}
#[test]
@@ -9371,6 +9760,123 @@ mod tests {
);
}
#[test]
fn workspace_runtime_verification_is_revision_bound_and_restart_safe() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("server.db");
let store = SqliteWorkspaceStore::open(&path).unwrap();
store
.with_conn(|conn| {
conn.execute_batch(
r#"
INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at)
VALUES ('owner', 'user', 'owner', 'Owner', '1', '1');
INSERT INTO workspaces(workspace_id, owner_account_id, display_name, state, created_at, updated_at)
VALUES ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1');
INSERT INTO workspace_signing_identities(
workspace_id, key_id, algorithm, public_key, public_key_fingerprint,
private_material_ref, revision, state, created_at, provisioned_at, updated_at
) VALUES ('workspace-a', 'WK-a', 'ed25519', 'key', 'sha256:key',
'workspace-signing/workspace-a/ed25519-v1', 1, 'active', '1', '1', '1');
"#,
)?;
Ok(())
})
.unwrap();
let runtime_identity =
worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-a").unwrap();
store
.upsert_workspace_runtime_binding(
WorkspaceRuntimeBinding {
workspace_id: "workspace-a".to_string(),
runtime_id: "runtime-a".to_string(),
display_name: "runtime-a".to_string(),
base_url: "https://runtime.test".to_string(),
public_key: runtime_identity.public_key,
public_key_fingerprint: String::new(),
binding_revision: 1,
state: WorkspaceRuntimeBindingState::Configured,
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
workspace_key_id: Some("WK-a".to_string()),
workspace_key_generation: Some(1),
created_at: "1".to_string(),
updated_at: "1".to_string(),
revoked_at: None,
},
false,
)
.unwrap();
let persisted = store
.get_workspace_runtime_binding("workspace-a", "runtime-a")
.unwrap()
.unwrap();
let evidence = WorkspaceRuntimeVerificationEvidence {
workspace_id: "workspace-a".to_string(),
runtime_id: "runtime-a".to_string(),
binding_revision: persisted.binding_revision,
workspace_key_id: "WK-a".to_string(),
workspace_identity_revision: 1,
workspace_trust_generation: 1,
runtime_public_key_fingerprint: persisted.public_key_fingerprint.clone(),
runtime_identity_revision: 1,
challenge_id: "challenge-a".to_string(),
state: "verified".to_string(),
last_outcome: "verified".to_string(),
verified_at: Some("2".to_string()),
checked_at: "2".to_string(),
};
let verified = store
.complete_workspace_runtime_verification(&evidence)
.unwrap();
assert_eq!(verified.state, WorkspaceRuntimeBindingState::Verified);
drop(store);
let reopened = SqliteWorkspaceStore::open(&path).unwrap();
assert_eq!(
reopened
.get_workspace_runtime_verification("workspace-a", "runtime-a")
.unwrap(),
Some(evidence)
);
assert_eq!(
reopened
.get_workspace_runtime_binding("workspace-a", "runtime-a")
.unwrap()
.unwrap()
.state,
WorkspaceRuntimeBindingState::Verified
);
}
#[test]
fn schema_v55_migrates_runtime_verification_table_atomically() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("server.db");
let store = SqliteWorkspaceStore::open(&path).unwrap();
store
.with_conn(|conn| {
conn.execute_batch(
"DROP TABLE workspace_runtime_verifications;
DELETE FROM __yoi_schema_migrations;
INSERT INTO __yoi_schema_migrations(version, name)
VALUES (55, 'Workspace Runtime binding state and identity mode');",
)?;
Ok(())
})
.unwrap();
drop(store);
let conn = Connection::open(&path).unwrap();
configure_sqlite(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 55);
migrate_workspace_runtime_verification_v55_to_v56(&conn).unwrap();
drop(conn);
let migrated = Connection::open(&path).unwrap();
configure_sqlite(&migrated).unwrap();
assert_eq!(current_schema_version(&migrated).unwrap(), 56);
assert!(table_exists(&migrated, "workspace_runtime_verifications").unwrap());
}
#[test]
fn runtime_binding_key_mutations_are_revisioned_idempotent_and_audited() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
@@ -10691,13 +11197,13 @@ INSERT INTO worker_registry (
let conn = Connection::open_in_memory().unwrap();
configure_sqlite(&conn).unwrap();
conn.execute(
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (56, 'future')",
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (57, 'future')",
[],
)
.unwrap();
let error = apply_migrations(&conn).unwrap_err().to_string();
assert!(error.contains("schema version 56 is newer"), "{error}");
assert!(error.contains("schema version 57 is newer"), "{error}");
assert!(error.contains("refusing to serve"), "{error}");
}
crates/workspace-server/src/workspace_deletion.rs
+1
View File
@@ -80,6 +80,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"workspace_resource_keys",
"workspace_runtime_binding_audit",
"workspace_runtime_bindings",
"workspace_runtime_verifications",
"workspace_signing_identities",
"workspace_signing_identity_audit",
"workspace_signing_identity_provisioning_operations",
crates/workspace-server/src/workspace_signing_identity.rs
+21
View File
@@ -9,6 +9,10 @@ use chrono::{SecondsFormat, Utc};
use ring::signature::KeyPair;
use serde::{Deserialize, Serialize};
use worker_runtime::auth::{RuntimeIdentityMaterial, encode_public_key};
use worker_runtime::workspace_issuer::{
WorkspaceCapabilityClaims, WorkspaceCapabilityVerificationError,
assemble_workspace_capability_token, workspace_capability_signing_input,
};
use zeroize::Zeroize;
use crate::store::{
@@ -254,6 +258,16 @@ impl WorkspaceSigningIdentityService {
Ok(signing_key.sign(payload).as_ref().to_vec())
}
pub fn issue_workspace_capability(
&self,
workspace_id: &str,
claims: &WorkspaceCapabilityClaims,
) -> Result<String> {
let input = workspace_capability_signing_input(claims).map_err(capability_error)?;
let signature = self.sign(workspace_id, input.bytes())?;
assemble_workspace_capability_token(input, &signature).map_err(capability_error)
}
pub fn delete_material(&self, workspace_id: &str) -> Result<()> {
if let Some(identity) = self.store.get_workspace_signing_identity(workspace_id)? {
self.materials.delete(&identity.private_material_ref)?;
@@ -572,6 +586,13 @@ fn material_io_error(action: &str, error: std::io::Error) -> Error {
)
}
fn capability_error(error: WorkspaceCapabilityVerificationError) -> Error {
identity_error(
"workspace_capability_issuance_failed",
format!("failed to issue Workspace capability: {error}"),
)
}
pub fn identity_error(code: impl Into<String>, message: impl Into<String>) -> Error {
Error::WorkspaceSigningIdentity {
code: code.into(),