Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: verify Workspace-signed Runtime bindings

Browse Source
This commit is contained in:
Hare
2026-09-08 08:13:58 +09:00
parent f5e9f49a13
commit f29c343879
12 changed files with 2817 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/hosts.rs
+479 -13
View File
@@ -2,6 +2,10 @@ use crate::Error;
use crate::resource_broker::BackendResourceBroker;
#[cfg(test)]
use crate::resource_broker::BackendResourceTarget;
use crate::store::{
ControlPlaneStore, WorkspaceRuntimeAuthenticationMode, WorkspaceRuntimeBindingState,
};
use crate::workspace_signing_identity::WorkspaceSigningIdentityService;
use chrono::Utc;
use protocol::Segment;
use reqwest::blocking::{Client as BlockingHttpClient, RequestBuilder};
@@ -64,6 +68,11 @@ use worker_runtime::profile_archive::ProfileSourceArchive;
use worker_runtime::retention::{
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
};
use worker_runtime::workspace_issuer::{
WorkspaceCapabilityClaims, WorkspaceRuntimeVerificationAcknowledgement,
WorkspaceRuntimeVerificationChallenge, WorkspaceRuntimeVerificationReceipt,
WorkspaceRuntimeVerificationResponse, workspace_request_body_digest,
};
pub const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime";
const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host";
@@ -818,6 +827,32 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
))
}
fn activate_workspace_authorization(&self, _binding: crate::store::WorkspaceRuntimeBinding) {}
fn send_workspace_verification_challenge(
&self,
_challenge: &WorkspaceRuntimeVerificationChallenge,
_bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationResponse, RuntimePingFailure> {
Err(RuntimePingFailure::new(
RuntimePingFailureKind::Unsupported,
"runtime_workspace_verification_unsupported",
"Workspace Runtime verification is unavailable for this Runtime provider",
))
}
fn send_workspace_verification_acknowledgement(
&self,
_acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement,
_bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationReceipt, RuntimePingFailure> {
Err(RuntimePingFailure::new(
RuntimePingFailureKind::Unsupported,
"runtime_workspace_verification_unsupported",
"Workspace Runtime verification is unavailable for this Runtime provider",
))
}
fn list_hosts(&self, limit: usize) -> RuntimeList<HostSummary>;
fn list_workers(&self, limit: usize) -> RuntimeList<WorkerSummary>;
@@ -1877,6 +1912,48 @@ impl RuntimeRegistry {
runtime.ping()
}
pub fn activate_workspace_authorization(
&self,
runtime_id: &str,
binding: crate::store::WorkspaceRuntimeBinding,
) -> Result<(), RuntimeRegistryError> {
self.runtime(runtime_id)?
.activate_workspace_authorization(binding);
Ok(())
}
pub fn send_workspace_verification_challenge(
&self,
runtime_id: &str,
challenge: &WorkspaceRuntimeVerificationChallenge,
bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationResponse, RuntimePingFailure> {
let runtime = self.runtime(runtime_id).map_err(|_| {
RuntimePingFailure::new(
RuntimePingFailureKind::Configuration,
"runtime_verification_registration_unavailable",
"Registered Runtime binding is unavailable",
)
})?;
runtime.send_workspace_verification_challenge(challenge, bearer_token)
}
pub fn send_workspace_verification_acknowledgement(
&self,
runtime_id: &str,
acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement,
bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationReceipt, RuntimePingFailure> {
let runtime = self.runtime(runtime_id).map_err(|_| {
RuntimePingFailure::new(
RuntimePingFailureKind::Configuration,
"runtime_verification_registration_unavailable",
"Registered Runtime binding is unavailable",
)
})?;
runtime.send_workspace_verification_acknowledgement(acknowledgement, bearer_token)
}
fn runtimes_snapshot(&self) -> Vec<Arc<dyn WorkspaceWorkerRuntime>> {
self.runtimes
.read()
@@ -2851,6 +2928,7 @@ pub struct RemoteRuntimeConfig {
pub base_url: String,
pub bearer_token: Option<String>,
pub auth: Option<RemoteRuntimeAuthConfig>,
pub workspace_authorization: Option<WorkspaceRuntimeAuthorization>,
pub strict_public_egress: bool,
pub cached_worker_creation_available: bool,
pub cached_os: String,
@@ -2859,6 +2937,168 @@ pub struct RemoteRuntimeConfig {
pub timeout: Duration,
}
#[derive(Clone)]
pub struct WorkspaceRuntimeAuthorization {
store: Arc<dyn ControlPlaneStore>,
signing_identities: WorkspaceSigningIdentityService,
backend_url: String,
binding: Arc<RwLock<Option<crate::store::WorkspaceRuntimeBinding>>>,
}
impl std::fmt::Debug for WorkspaceRuntimeAuthorization {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("WorkspaceRuntimeAuthorization")
.field("backend_url", &"<backend-private>")
.finish_non_exhaustive()
}
}
impl WorkspaceRuntimeAuthorization {
pub fn new(
store: Arc<dyn ControlPlaneStore>,
signing_identities: WorkspaceSigningIdentityService,
backend_url: impl Into<String>,
binding: Option<crate::store::WorkspaceRuntimeBinding>,
) -> Self {
Self {
store,
signing_identities,
backend_url: backend_url.into(),
binding: Arc::new(RwLock::new(binding)),
}
}
fn activate(&self, binding: crate::store::WorkspaceRuntimeBinding) {
if let Ok(mut current) = self.binding.write() {
*current = Some(binding);
}
}
pub(crate) fn issue(
&self,
method: &str,
path_and_query: &str,
operation: &str,
worker_id: Option<&str>,
body: &[u8],
) -> Result<String, RuntimeDiagnostic> {
let binding = self
.binding
.read()
.map_err(|_| {
diagnostic(
"workspace_runtime_authorization_unavailable",
DiagnosticSeverity::Error,
"Workspace Runtime authorization is unavailable".to_string(),
)
})?
.clone()
.ok_or_else(|| {
diagnostic(
"workspace_runtime_verification_required",
DiagnosticSeverity::Error,
"Workspace Runtime binding is not verified".to_string(),
)
})?;
if binding.state != WorkspaceRuntimeBindingState::Verified
|| binding.authentication_mode != WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity
|| binding.revoked_at.is_some()
|| !self
.store
.workspace_runtime_binding_matches(&binding)
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_unavailable",
DiagnosticSeverity::Error,
error.to_string(),
)
})?
{
return Err(diagnostic(
"workspace_runtime_authorization_stale",
DiagnosticSeverity::Error,
"Workspace Runtime binding changed or was revoked".to_string(),
));
}
let identity = self
.signing_identities
.get_validated(&binding.workspace_id)
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_unavailable",
DiagnosticSeverity::Error,
error.to_string(),
)
})?;
let workspace_key_id = binding.workspace_key_id.as_deref().ok_or_else(|| {
diagnostic(
"workspace_runtime_authorization_invalid",
DiagnosticSeverity::Error,
"Workspace Runtime binding is missing its Workspace key".to_string(),
)
})?;
let trust_generation = binding.workspace_key_generation.ok_or_else(|| {
diagnostic(
"workspace_runtime_authorization_invalid",
DiagnosticSeverity::Error,
"Workspace Runtime binding is missing its trust generation".to_string(),
)
})?;
if identity.state != "active"
|| identity.key_id != workspace_key_id
|| identity.revision != trust_generation
|| !self
.store
.workspace_runtime_verification_matches(
&binding,
identity.revision,
trust_generation,
)
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_unavailable",
DiagnosticSeverity::Error,
error.to_string(),
)
})?
{
return Err(diagnostic(
"workspace_runtime_authorization_stale",
DiagnosticSeverity::Error,
"Workspace signing identity no longer matches the verified binding".to_string(),
));
}
let now = Utc::now().timestamp();
let claims = WorkspaceCapabilityClaims {
issuer: self.backend_url.clone(),
issuer_workspace_id: binding.workspace_id.clone(),
issuer_key_id: identity.key_id,
issuer_identity_revision: identity.revision,
trust_generation,
binding_revision: binding.binding_revision,
runtime_id: binding.runtime_id.clone(),
worker_id: worker_id.map(str::to_string),
operation: operation.to_string(),
method: method.to_string(),
path_and_query: path_and_query.to_string(),
body_digest: workspace_request_body_digest(body),
iat: now,
exp: now.saturating_add(60),
jti: uuid::Uuid::now_v7().to_string(),
};
self.signing_identities
.issue_workspace_capability(&binding.workspace_id, &claims)
.map_err(|error| {
diagnostic(
"workspace_runtime_authorization_sign_failed",
DiagnosticSeverity::Error,
error.to_string(),
)
})
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RemoteRuntimeAuthConfig {
pub server_id: String,
@@ -2903,6 +3143,7 @@ impl RemoteRuntimeConfig {
base_url: base_url.into(),
bearer_token,
auth: None,
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: false,
cached_os: "unknown".to_string(),
@@ -2943,6 +3184,7 @@ struct RemoteWorkdirAuthorization {
runtime_id: String,
workspace_id: String,
auth: Option<RemoteRuntimeAuthConfig>,
workspace_authorization: Option<WorkspaceRuntimeAuthorization>,
fallback_bearer_token: Option<String>,
}
@@ -2962,7 +3204,23 @@ impl std::fmt::Debug for RemoteWorkdirAuthorization {
}
impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization {
fn bearer_token(&self) -> Result<String, WorkdirError> {
fn bearer_token(
&self,
method: &str,
path_and_query: &str,
body: &[u8],
) -> Result<String, WorkdirError> {
if let Some(authorization) = &self.workspace_authorization {
return authorization
.issue(
method,
path_and_query,
workspace_runtime_operation(method, path_and_query),
None,
body,
)
.map_err(|error| WorkdirError::Unavailable(error.message));
}
if let Some(auth) = self.auth.as_ref() {
let claims = capability_claims(
&auth.server_id,
@@ -3093,6 +3351,7 @@ pub struct RemoteWorkerRuntime {
workspace_id: String,
bearer_token: Option<String>,
auth: Option<RemoteRuntimeAuthConfig>,
workspace_authorization: Option<WorkspaceRuntimeAuthorization>,
cached_worker_creation_available: bool,
cached_os: String,
cached_arch: String,
@@ -3146,6 +3405,69 @@ fn remote_runtime_ping_transport_failure(error: reqwest::Error) -> RuntimePingFa
)
}
fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static str {
let path = path_and_query.split('?').next().unwrap_or(path_and_query);
if path == "/v1/ping" && method == "GET" {
return RUNTIME_PING_PERMISSION;
}
if path == "/v1/workers" && method == "GET" {
return "workers:list";
}
if path == "/v1/workers" && method == "POST" {
return "workers:create";
}
if (path == "/v1/working-directories/repository-access"
|| path == "/v1/repository-refs/observe")
&& method == "POST"
{
return "workdirs:operate";
}
if path.starts_with("/v1/workdir-sessions")
|| (path.starts_with("/v1/working-directories/") && path.ends_with("/sessions"))
{
return "workdirs:operate";
}
if path.starts_with("/v1/config-bundles")
|| path.starts_with("/v1/workspace-prompt-projections")
|| path.starts_with("/v1/working-directories")
{
return "workers:create";
}
if path.ends_with("/input")
|| path.ends_with("/restore")
|| path.ends_with("/workspace-api")
|| path.contains("/attachments")
{
return "workers:input";
}
if path.ends_with("/stop") || path.ends_with("/cancel") {
return "workers:stop";
}
if path == "/v1/protocol/ws" {
return "workers:list";
}
if path.ends_with("/protocol") || path.ends_with("/protocol/ws") {
return "workers:protocol";
}
if path.ends_with("/completions") {
return "workers:read";
}
if path.contains("/retention/") || (path.starts_with("/v1/workers/") && method == "DELETE") {
return "workers:delete";
}
if path.starts_with("/v1/workers/") && method == "GET" {
return "workers:read";
}
"runtime:read"
}
fn worker_id_from_remote_path(path_and_query: &str) -> Option<String> {
let path = path_and_query.split('?').next().unwrap_or(path_and_query);
let rest = path.strip_prefix("/v1/workers/")?;
let worker_id = rest.split('/').next()?;
(!worker_id.is_empty()).then(|| worker_id.to_string())
}
fn all_remote_runtime_permissions() -> Vec<String> {
[
"workers:list",
@@ -3226,6 +3548,7 @@ impl RemoteWorkerRuntime {
workspace_id,
bearer_token: config.bearer_token,
auth: config.auth,
workspace_authorization: config.workspace_authorization,
cached_worker_creation_available: config.cached_worker_creation_available,
cached_os: config.cached_os,
cached_arch: config.cached_arch,
@@ -3254,6 +3577,7 @@ impl RemoteWorkerRuntime {
runtime_id: self.runtime_id.clone(),
workspace_id: self.workspace_id.clone(),
auth: self.auth.clone(),
workspace_authorization: self.workspace_authorization.clone(),
fallback_bearer_token: self.bearer_token.clone(),
});
RemoteWorkdirSession::open_with_authorization(
@@ -3290,11 +3614,54 @@ impl RemoteWorkerRuntime {
format!("{base}/v1/workers/{worker_id}/protocol/ws")
}
fn post_bearer_json<T, U>(
&self,
path: &str,
body: &T,
bearer_token: &str,
) -> Result<U, RuntimePingFailure>
where
T: Serialize + ?Sized,
U: DeserializeOwned,
{
let response = self
.http
.post(self.endpoint(path))
.bearer_auth(bearer_token)
.json(body)
.send()
.map_err(|error| {
RuntimePingFailure::new(
RuntimePingFailureKind::NetworkUnreachable,
"runtime_workspace_verification_unreachable",
format!("Runtime verification request failed: {error}"),
)
})?;
let status = response.status();
if !status.is_success() {
return Err(RuntimePingFailure::new(
RuntimePingFailureKind::MalformedResponse,
"runtime_workspace_verification_rejected",
format!(
"Runtime verification request returned HTTP {}",
status.as_u16()
),
));
}
response.json::<U>().map_err(|error| {
RuntimePingFailure::new(
RuntimePingFailureKind::MalformedResponse,
"runtime_workspace_verification_invalid_response",
format!("Runtime verification response was invalid: {error}"),
)
})
}
fn get_json<T>(&self, path: &str) -> Result<T, RuntimeDiagnostic>
where
T: DeserializeOwned + Send + 'static,
{
self.send_json(path, self.http.get(self.endpoint(path)))
self.send_json(path, "GET", &[], self.http.get(self.endpoint(path)))
}
fn post_json<B, T>(&self, path: &str, body: &B) -> Result<T, RuntimeDiagnostic>
@@ -3302,7 +3669,22 @@ impl RemoteWorkerRuntime {
B: Serialize + ?Sized,
T: DeserializeOwned + Send + 'static,
{
self.send_json(path, self.http.post(self.endpoint(path)).json(body))
let body = serde_json::to_vec(body).map_err(|error| {
diagnostic(
"remote_runtime_request_encode_failed",
DiagnosticSeverity::Error,
error.to_string(),
)
})?;
self.send_json(
path,
"POST",
&body,
self.http
.post(self.endpoint(path))
.header(CONTENT_TYPE, "application/json")
.body(body.clone()),
)
}
fn post_bytes<T>(&self, path: &str, body: &[u8]) -> Result<T, RuntimeDiagnostic>
@@ -3311,7 +3693,12 @@ impl RemoteWorkerRuntime {
{
self.send_json(
path,
self.http.post(self.endpoint(path)).body(body.to_vec()),
"POST",
body,
self.http
.post(self.endpoint(path))
.header(CONTENT_TYPE, "application/json")
.body(body.to_vec()),
)
}
@@ -3319,7 +3706,7 @@ impl RemoteWorkerRuntime {
where
T: DeserializeOwned + Send + 'static,
{
self.send_json(path, self.http.delete(self.endpoint(path)))
self.send_json(path, "DELETE", &[], self.http.delete(self.endpoint(path)))
}
fn runtime_capability_token_with_permissions(
@@ -3364,10 +3751,23 @@ impl RemoteWorkerRuntime {
const PATH: &str = "/v1/ping";
let workspace_id = self.workspace_id.clone();
let bearer_token = self.bearer_token.clone();
let capability_token = self.runtime_capability_token_with_permissions(
PATH,
vec![RUNTIME_PING_PERMISSION.to_string()],
);
let capability_token = match &self.workspace_authorization {
Some(authorization) => Some(
authorization
.issue("GET", PATH, RUNTIME_PING_PERMISSION, None, &[])
.map_err(|diagnostic| {
RuntimePingFailure::new(
RuntimePingFailureKind::Authentication,
diagnostic.code,
diagnostic.message,
)
})?,
),
None => self.runtime_capability_token_with_permissions(
PATH,
vec![RUNTIME_PING_PERMISSION.to_string()],
),
};
let request = self
.http
.get(self.endpoint(PATH))
@@ -3432,15 +3832,33 @@ impl RemoteWorkerRuntime {
})
}
fn send_json<T>(&self, path: &str, request: RequestBuilder) -> Result<T, RuntimeDiagnostic>
fn send_json<T>(
&self,
path: &str,
method: &str,
body: &[u8],
request: RequestBuilder,
) -> Result<T, RuntimeDiagnostic>
where
T: DeserializeOwned + Send + 'static,
{
let runtime_id = self.runtime_id.clone();
let workspace_id = self.workspace_id.clone();
let bearer_token = self.bearer_token.clone();
let capability_token = self.runtime_capability_token(path);
let capability_token = match &self.workspace_authorization {
Some(authorization) => Some(authorization.issue(
method,
path,
workspace_runtime_operation(method, path),
worker_id_from_remote_path(path).as_deref(),
body,
)?),
None => self.runtime_capability_token(path),
};
run_blocking_http(move || {
let request = request.header(CONTENT_TYPE, "application/json");
let request = request
.header(CONTENT_TYPE, "application/json")
.header(RUNTIME_WORKSPACE_SCOPE_HEADER, &workspace_id);
let request =
if let Some(token) = capability_token.as_deref().or(bearer_token.as_deref()) {
request.header(AUTHORIZATION, format!("Bearer {token}"))
@@ -3649,6 +4067,36 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
self.ping_http()
}
fn activate_workspace_authorization(&self, binding: crate::store::WorkspaceRuntimeBinding) {
if let Some(authorization) = &self.workspace_authorization {
authorization.activate(binding);
}
}
fn send_workspace_verification_challenge(
&self,
challenge: &WorkspaceRuntimeVerificationChallenge,
bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationResponse, RuntimePingFailure> {
self.post_bearer_json(
worker_runtime::workspace_issuer::WORKSPACE_VERIFICATION_CHALLENGE_PATH,
challenge,
bearer_token,
)
}
fn send_workspace_verification_acknowledgement(
&self,
acknowledgement: &WorkspaceRuntimeVerificationAcknowledgement,
bearer_token: &str,
) -> Result<WorkspaceRuntimeVerificationReceipt, RuntimePingFailure> {
self.post_bearer_json(
worker_runtime::workspace_issuer::WORKSPACE_VERIFICATION_ACK_PATH,
acknowledgement,
bearer_token,
)
}
fn list_hosts(&self, limit: usize) -> RuntimeList<HostSummary> {
if limit == 0 {
return RuntimeList::new(Vec::new(), Vec::new());
@@ -3936,12 +4384,30 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
workspace_api: Some(workspace_api),
memory_settings: request.resolved_memory_settings.clone(),
};
let create_body = match serde_json::to_vec(&create) {
Ok(body) => body,
Err(error) => {
return WorkerSpawnResult {
state: WorkerOperationState::Rejected,
worker: None,
acceptance_evidence: Vec::new(),
diagnostics: vec![diagnostic(
"remote_runtime_request_encode_failed",
DiagnosticSeverity::Error,
error.to_string(),
)],
};
}
};
match self.send_json::<RuntimeHttpWorkerResponse>(
"/v1/workers",
"POST",
&create_body,
self.http
.post(self.endpoint("/v1/workers"))
.timeout(REMOTE_WORKER_CREATE_TIMEOUT)
.json(&create),
.header(CONTENT_TYPE, "application/json")
.body(create_body.clone()),
) {
Ok(response) => WorkerSpawnResult {
state: WorkerOperationState::Accepted,
crates/workspace-server/src/latest_schema.sql
+22
View File
@@ -462,6 +462,28 @@ CREATE TABLE workspace_runtime_bindings (
(state != 'revoked' AND revoked_at IS NULL)
)
);
CREATE TABLE workspace_runtime_verifications (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
binding_revision INTEGER NOT NULL CHECK(binding_revision > 0),
workspace_key_id TEXT NOT NULL,
workspace_identity_revision INTEGER NOT NULL CHECK(workspace_identity_revision > 0),
workspace_trust_generation INTEGER NOT NULL CHECK(workspace_trust_generation > 0),
runtime_public_key_fingerprint TEXT NOT NULL,
runtime_identity_revision INTEGER NOT NULL CHECK(runtime_identity_revision > 0),
challenge_id TEXT NOT NULL,
state TEXT NOT NULL CHECK(state IN ('pending', 'verified', 'failed')),
last_outcome TEXT NOT NULL,
verified_at TEXT,
checked_at TEXT NOT NULL,
PRIMARY KEY(workspace_id, runtime_id),
FOREIGN KEY(workspace_id, runtime_id)
REFERENCES workspace_runtime_bindings(workspace_id, runtime_id) ON DELETE CASCADE,
CHECK((state = 'verified' AND verified_at IS NOT NULL)
OR (state != 'verified' AND verified_at IS NULL))
);
CREATE INDEX workspace_runtime_verifications_state_idx
ON workspace_runtime_verifications(workspace_id, state, checked_at DESC);
CREATE TABLE workspace_runtime_binding_audit (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
crates/workspace-server/src/runtime_subscription.rs
+6
View File
@@ -970,6 +970,12 @@ fn runtime_token(
config: &RemoteRuntimeConfig,
workspace_id: &str,
) -> Result<Option<String>, String> {
if let Some(authorization) = config.workspace_authorization.as_ref() {
return authorization
.issue("GET", "/v1/protocol/ws", "workers:list", None, &[])
.map(Some)
.map_err(|error| error.message);
}
let Some(auth) = config.auth.as_ref() else {
return Ok(config.bearer_token.clone());
};
crates/workspace-server/src/server.rs
+286 -20
View File
@@ -61,6 +61,12 @@ use worker_runtime::http_server::{
};
use worker_runtime::resource::{BackendResourceError, BackendResourceFetchRequest};
use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend};
use worker_runtime::workspace_issuer::{
WORKSPACE_VERIFICATION_ACK_PATH, WORKSPACE_VERIFICATION_CHALLENGE_PATH,
WORKSPACE_VERIFICATION_OPERATION, WorkspaceCapabilityClaims,
WorkspaceRuntimeVerificationAcknowledgement, WorkspaceRuntimeVerificationChallenge,
verify_runtime_verification_response, workspace_request_body_digest,
};
use workspace_api::{
ActorAuthMethod, AuthBootstrapUserRequest, AuthPublicConfig, AuthUserResponse,
AuthenticatedUser, BrowserCreateWorkerResponse, BrowserWorkspaceOrchestratorResponse,
@@ -128,8 +134,8 @@ use crate::hosts::{
WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult,
WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult,
WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerTicketAssignmentRequest,
WorkerWorkspaceSummary, is_disallowed_remote_runtime_address, worker_spawn_create_fingerprint,
workspace_worker_summary,
WorkerWorkspaceSummary, WorkspaceRuntimeAuthorization, is_disallowed_remote_runtime_address,
worker_spawn_create_fingerprint, workspace_worker_summary,
};
use crate::identity::WorkspaceIdentity;
use crate::memory_backend::execute_memory_backend_operation_with_authority;
@@ -2229,23 +2235,6 @@ impl WorkspaceApi {
RuntimeSubscriptionBroker::new(config.workspace_id.clone());
runtime_subscription_broker
.register_embedded_runtime(embedded_runtime_id, embedded_subscription_runtime);
for remote_config in config.remote_runtime_sources.iter().cloned() {
let remote_runtime = RemoteWorkerRuntime::new(
remote_config.clone(),
config.workspace_id.clone(),
config
.backend_base_url
.clone()
.unwrap_or_else(|| "http://127.0.0.1:8787".to_string()),
)
.map(|host| host.with_resource_broker(resource_broker.clone()))
.map_err(|err| err.into_error())?;
runtime.register(remote_runtime);
runtime_subscription_broker.register_remote_runtime(remote_config);
}
let runtime = Arc::new(runtime);
let companion = Arc::new(CompanionConsole::disabled());
let observation_proxy = BackendObservationProxy::new(config.runtime_event_sources.clone());
let config_store = Arc::new(crate::SqliteWorkspaceStore::open(
config.database_path.clone(),
)?);
@@ -2260,6 +2249,40 @@ impl WorkspaceApi {
let signing_identities =
WorkspaceSigningIdentityService::new(store.clone(), signing_materials);
signing_identities.get_validated(&config.workspace_id)?;
let backend_url = config
.backend_base_url
.clone()
.unwrap_or_else(|| "http://127.0.0.1:8787".to_string());
for mut remote_config in config.remote_runtime_sources.iter().cloned() {
let current_binding = store
.get_workspace_runtime_binding(&config.workspace_id, &remote_config.runtime_id)
.await?;
if current_binding.as_ref().is_some_and(|binding| {
binding.authentication_mode == StoredRuntimeAuthenticationMode::WorkspaceIdentity
&& binding.revoked_at.is_none()
}) {
let verified_binding = current_binding
.filter(|binding| binding.state == StoredRuntimeBindingState::Verified);
remote_config.workspace_authorization = Some(WorkspaceRuntimeAuthorization::new(
store.clone(),
signing_identities.clone(),
backend_url.clone(),
verified_binding,
));
}
let remote_runtime = RemoteWorkerRuntime::new(
remote_config.clone(),
config.workspace_id.clone(),
backend_url.clone(),
)
.map(|host| host.with_resource_broker(resource_broker.clone()))
.map_err(|err| err.into_error())?;
runtime.register(remote_runtime);
runtime_subscription_broker.register_remote_runtime(remote_config);
}
let runtime = Arc::new(runtime);
let companion = Arc::new(CompanionConsole::disabled());
let observation_proxy = BackendObservationProxy::new(config.runtime_event_sources.clone());
let config_schema_registry = crate::config_source::WorkspaceConfigSchemaRegistry::default()
.with_provider(Arc::new(
crate::profile_settings::ProfileConfigSchemaProvider,
@@ -13674,6 +13697,197 @@ async fn delete_remote_runtime(
Ok(StatusCode::NO_CONTENT)
}
async fn perform_workspace_runtime_verification(
api: &WorkspaceApi,
runtime: Arc<RuntimeRegistry>,
binding: &WorkspaceRuntimeBinding,
) -> std::result::Result<WorkspaceRuntimeBinding, String> {
if binding.authentication_mode != StoredRuntimeAuthenticationMode::WorkspaceIdentity {
return Ok(binding.clone());
}
if binding.state == crate::store::WorkspaceRuntimeBindingState::Revoked
|| binding.revoked_at.is_some()
{
return Err("Runtime binding is revoked".to_string());
}
let backend_url = api.config.backend_base_url.as_deref().ok_or_else(|| {
"Workspace identity verification requires configured backend_base_url".to_string()
})?;
let identity = api
.signing_identities
.get_validated(&binding.workspace_id)
.map_err(|error| error.to_string())?;
if identity.state != "active" {
return Err("Workspace signing identity is not active".to_string());
}
let workspace_key_id = binding
.workspace_key_id
.as_deref()
.ok_or_else(|| "Runtime binding is missing the Workspace key identity".to_string())?;
let workspace_trust_generation = binding
.workspace_key_generation
.ok_or_else(|| "Runtime binding is missing the Workspace trust generation".to_string())?;
if identity.key_id != workspace_key_id || identity.revision != workspace_trust_generation {
return Err(
"Runtime binding no longer matches the active Workspace or Runtime identity"
.to_string(),
);
}
let now = Utc::now();
let expires_at = (now + Duration::seconds(60)).timestamp();
let challenge = WorkspaceRuntimeVerificationChallenge {
challenge_id: Uuid::now_v7().to_string(),
workspace_id: binding.workspace_id.clone(),
runtime_id: binding.runtime_id.clone(),
binding_revision: binding.binding_revision,
workspace_key_id: workspace_key_id.to_string(),
workspace_identity_revision: identity.revision,
workspace_trust_generation,
runtime_public_key_fingerprint: binding.public_key_fingerprint.clone(),
runtime_identity_revision: 1,
workspace_nonce: Uuid::now_v7().to_string(),
expires_at,
};
let checked_at = now.to_rfc3339_opts(SecondsFormat::Millis, true);
let pending = crate::store::WorkspaceRuntimeVerificationEvidence {
workspace_id: binding.workspace_id.clone(),
runtime_id: binding.runtime_id.clone(),
binding_revision: binding.binding_revision,
workspace_key_id: workspace_key_id.to_string(),
workspace_identity_revision: identity.revision,
workspace_trust_generation,
runtime_public_key_fingerprint: binding.public_key_fingerprint.clone(),
runtime_identity_revision: 1,
challenge_id: challenge.challenge_id.clone(),
state: "pending".to_string(),
last_outcome: "challenge_issued".to_string(),
verified_at: None,
checked_at: checked_at.clone(),
};
api.store
.record_workspace_runtime_verification_attempt(&pending)
.await
.map_err(|error| error.to_string())?;
let challenge_body = serde_json::to_vec(&challenge).map_err(|error| error.to_string())?;
let challenge_claims = WorkspaceCapabilityClaims {
issuer: backend_url.to_string(),
issuer_workspace_id: binding.workspace_id.clone(),
issuer_key_id: identity.key_id.clone(),
issuer_identity_revision: identity.revision,
trust_generation: workspace_trust_generation,
binding_revision: binding.binding_revision,
runtime_id: binding.runtime_id.clone(),
worker_id: None,
operation: WORKSPACE_VERIFICATION_OPERATION.to_string(),
method: "POST".to_string(),
path_and_query: WORKSPACE_VERIFICATION_CHALLENGE_PATH.to_string(),
body_digest: workspace_request_body_digest(&challenge_body),
iat: now.timestamp(),
exp: expires_at,
jti: Uuid::now_v7().to_string(),
};
let challenge_token = api
.signing_identities
.issue_workspace_capability(&binding.workspace_id, &challenge_claims)
.map_err(|error| error.to_string())?;
let challenge_runtime = runtime.clone();
let challenge_runtime_id = binding.runtime_id.clone();
let challenge_request = challenge.clone();
let response = tokio::task::spawn_blocking(move || {
challenge_runtime.send_workspace_verification_challenge(
&challenge_runtime_id,
&challenge_request,
&challenge_token,
)
})
.await
.map_err(|_| "Runtime verification challenge task failed".to_string())?
.map_err(|failure| failure.diagnostic.message)?;
verify_runtime_verification_response(
&response,
&challenge,
&binding.public_key,
Utc::now().timestamp(),
)
.map_err(|error| error.to_string())?;
let response_bytes = serde_json::to_vec(&response).map_err(|error| error.to_string())?;
let acknowledgement = WorkspaceRuntimeVerificationAcknowledgement {
challenge_id: response.challenge_id.clone(),
workspace_id: response.workspace_id.clone(),
runtime_id: response.runtime_id.clone(),
binding_revision: response.binding_revision,
workspace_key_id: response.workspace_key_id.clone(),
workspace_identity_revision: response.workspace_identity_revision,
workspace_trust_generation: response.workspace_trust_generation,
runtime_public_key_fingerprint: response.runtime_public_key_fingerprint.clone(),
runtime_identity_revision: response.runtime_identity_revision,
workspace_nonce: response.workspace_nonce.clone(),
runtime_nonce: response.runtime_nonce.clone(),
response_digest: workspace_request_body_digest(&response_bytes),
response: response.clone(),
expires_at: response.expires_at,
};
let acknowledgement_body =
serde_json::to_vec(&acknowledgement).map_err(|error| error.to_string())?;
let acknowledgement_claims = WorkspaceCapabilityClaims {
issuer: backend_url.to_string(),
issuer_workspace_id: binding.workspace_id.clone(),
issuer_key_id: identity.key_id.clone(),
issuer_identity_revision: identity.revision,
trust_generation: workspace_trust_generation,
binding_revision: binding.binding_revision,
runtime_id: binding.runtime_id.clone(),
worker_id: None,
operation: WORKSPACE_VERIFICATION_OPERATION.to_string(),
method: "POST".to_string(),
path_and_query: WORKSPACE_VERIFICATION_ACK_PATH.to_string(),
body_digest: workspace_request_body_digest(&acknowledgement_body),
iat: Utc::now().timestamp(),
exp: expires_at,
jti: Uuid::now_v7().to_string(),
};
let acknowledgement_token = api
.signing_identities
.issue_workspace_capability(&binding.workspace_id, &acknowledgement_claims)
.map_err(|error| error.to_string())?;
let acknowledgement_runtime = runtime;
let acknowledgement_runtime_id = binding.runtime_id.clone();
let acknowledgement_request = acknowledgement.clone();
let receipt = tokio::task::spawn_blocking(move || {
acknowledgement_runtime.send_workspace_verification_acknowledgement(
&acknowledgement_runtime_id,
&acknowledgement_request,
&acknowledgement_token,
)
})
.await
.map_err(|_| "Runtime verification acknowledgement task failed".to_string())?
.map_err(|failure| failure.diagnostic.message)?;
if receipt.challenge_id != challenge.challenge_id
|| receipt.workspace_id != binding.workspace_id
|| receipt.runtime_id != binding.runtime_id
|| receipt.binding_revision != binding.binding_revision
{
return Err("Runtime verification acknowledgement receipt mismatched".to_string());
}
let verified_at = Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true);
let verified = crate::store::WorkspaceRuntimeVerificationEvidence {
state: "verified".to_string(),
last_outcome: "verified".to_string(),
verified_at: Some(verified_at.clone()),
checked_at: verified_at,
..pending
};
api.store
.complete_workspace_runtime_verification(&verified)
.await
.map_err(|error| error.to_string())
}
async fn test_runtime_connection(
State(api): State<WorkspaceApi>,
AxumPath(runtime_id): AxumPath<String>,
@@ -13686,12 +13900,61 @@ async fn test_runtime_connection(
}
.into());
}
api.store
let binding = api
.store
.get_workspace_runtime_binding(api.workspace_id(), &runtime_id)
.await?
.filter(|binding| binding.revoked_at.is_none())
.ok_or_else(|| Error::UnknownRuntime(runtime_id.clone()))?;
if binding.authentication_mode == StoredRuntimeAuthenticationMode::WorkspaceIdentity {
match perform_workspace_runtime_verification(&api, api.runtime.clone(), &binding).await {
Ok(verified_binding) => {
api.runtime_binding_expectations
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
(
verified_binding.workspace_id.clone(),
verified_binding.runtime_id.clone(),
),
verified_binding.clone(),
);
api.runtime
.activate_workspace_authorization(&runtime_id, verified_binding.clone())
.map_err(|error| Error::Store(format!("{error:?}")))?;
}
Err(message) => {
if let Ok(Some(mut evidence)) = api
.store
.get_workspace_runtime_verification(api.workspace_id(), &runtime_id)
.await
{
evidence.state = "failed".to_string();
evidence.last_outcome = "verification_failed".to_string();
evidence.verified_at = None;
evidence.checked_at = Utc::now().to_rfc3339();
let _ = api
.store
.record_workspace_runtime_verification_attempt(&evidence)
.await;
}
return Ok(Json(runtime_connection_test_failure(
api.workspace_id(),
&runtime_id,
Utc::now().to_rfc3339(),
RuntimeConnectionTestFailureKind::Authentication,
None,
RuntimeDiagnostic::new(
"runtime_workspace_verification_failed",
"error",
message,
),
)));
}
}
}
let checked_at = Utc::now().to_rfc3339();
let runtime = api.runtime.clone();
let ping_runtime_id = runtime_id.clone();
@@ -18283,6 +18546,7 @@ mod tests {
server_id: "server-test".to_owned(),
server_private_key: "unused".to_owned(),
}),
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: true,
cached_os: "test".to_owned(),
@@ -24847,6 +25111,7 @@ mod tests {
server_id: "server-main".to_string(),
server_private_key: identity.private_key.clone(),
}),
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: true,
cached_os: "test".to_string(),
@@ -26374,6 +26639,7 @@ mod tests {
base_url: endpoint,
bearer_token: Some("test-connection-token".to_string()),
auth: None,
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: true,
cached_os: "linux".to_string(),
crates/workspace-server/src/store.rs
+511 -5
View File
@@ -18,7 +18,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore;
use crate::{Error, Result};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 55;
const LATEST_SCHEMA_VERSION: i64 = 56;
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings";
const RUNTIME_BINDING_AUDIT_MIGRATION_NAME: &str = "workspace Runtime binding revision and audit";
@@ -26,6 +26,8 @@ const WORKSPACE_DELETION_MIGRATION_NAME: &str = "durable Workspace deletion oper
const WORKSPACE_SIGNING_IDENTITY_MIGRATION_NAME: &str = "Workspace signing identity authority";
const WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME: &str =
"Workspace Runtime binding state and identity mode";
const WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME: &str =
"Workspace-signed Runtime verification evidence";
const MIGRATIONS: &[Migration] = &[
Migration {
@@ -53,6 +55,11 @@ const MIGRATIONS: &[Migration] = &[
name: WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME,
apply: migrate_workspace_runtime_binding_state_v54_to_v55,
},
Migration {
version: 56,
name: WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME,
apply: migrate_workspace_runtime_verification_v55_to_v56,
},
];
#[derive(Clone, Copy)]
@@ -208,6 +215,23 @@ pub struct WorkspaceSigningIdentityProvisioningOperation {
pub completed_at: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkspaceRuntimeVerificationEvidence {
pub workspace_id: String,
pub runtime_id: String,
pub binding_revision: u64,
pub workspace_key_id: String,
pub workspace_identity_revision: u64,
pub workspace_trust_generation: u64,
pub runtime_public_key_fingerprint: String,
pub runtime_identity_revision: u64,
pub challenge_id: String,
pub state: String,
pub last_outcome: String,
pub verified_at: Option<String>,
pub checked_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkspaceRuntimeBinding {
pub workspace_id: String,
@@ -732,6 +756,25 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
) -> Result<WorkspaceSigningIdentityRecord>;
fn workspace_runtime_binding_matches(&self, expected: &WorkspaceRuntimeBinding)
-> Result<bool>;
fn workspace_runtime_verification_matches(
&self,
binding: &WorkspaceRuntimeBinding,
workspace_identity_revision: u64,
workspace_trust_generation: u64,
) -> Result<bool>;
async fn get_workspace_runtime_verification(
&self,
workspace_id: &str,
runtime_id: &str,
) -> Result<Option<WorkspaceRuntimeVerificationEvidence>>;
async fn record_workspace_runtime_verification_attempt(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<()>;
async fn complete_workspace_runtime_verification(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<WorkspaceRuntimeBinding>;
async fn get_workspace_runtime_binding(
&self,
workspace_id: &str,
@@ -2179,6 +2222,183 @@ impl SqliteWorkspaceStore {
})
}
pub fn workspace_runtime_verification_matches(
&self,
binding: &WorkspaceRuntimeBinding,
workspace_identity_revision: u64,
workspace_trust_generation: u64,
) -> Result<bool> {
let Some(evidence) =
self.get_workspace_runtime_verification(&binding.workspace_id, &binding.runtime_id)?
else {
return Ok(false);
};
Ok(evidence.state == "verified"
&& evidence.verified_at.is_some()
&& evidence.binding_revision == binding.binding_revision
&& evidence.workspace_key_id == binding.workspace_key_id.as_deref().unwrap_or_default()
&& evidence.workspace_identity_revision == workspace_identity_revision
&& evidence.workspace_trust_generation == workspace_trust_generation
&& evidence.runtime_public_key_fingerprint == binding.public_key_fingerprint
&& evidence.runtime_identity_revision > 0)
}
pub fn get_workspace_runtime_verification(
&self,
workspace_id: &str,
runtime_id: &str,
) -> Result<Option<WorkspaceRuntimeVerificationEvidence>> {
validate_identifier("workspace_id", workspace_id)?;
validate_identifier("runtime_id", runtime_id)?;
self.with_conn(|conn| {
conn.query_row(
r#"SELECT workspace_id, runtime_id, binding_revision, workspace_key_id,
workspace_identity_revision, workspace_trust_generation,
runtime_public_key_fingerprint, runtime_identity_revision,
challenge_id, state, last_outcome, verified_at, checked_at
FROM workspace_runtime_verifications
WHERE workspace_id = ?1 AND runtime_id = ?2"#,
params![workspace_id, runtime_id],
read_workspace_runtime_verification,
)
.optional()
.map_err(Error::from)
})
}
pub fn record_workspace_runtime_verification_attempt(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<()> {
validate_workspace_runtime_verification(evidence)?;
self.with_conn(|conn| {
conn.execute(
r#"INSERT INTO workspace_runtime_verifications (
workspace_id, runtime_id, binding_revision, workspace_key_id,
workspace_identity_revision, workspace_trust_generation,
runtime_public_key_fingerprint, runtime_identity_revision,
challenge_id, state, last_outcome, verified_at, checked_at
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)
ON CONFLICT(workspace_id, runtime_id) DO UPDATE SET
binding_revision = excluded.binding_revision,
workspace_key_id = excluded.workspace_key_id,
workspace_identity_revision = excluded.workspace_identity_revision,
workspace_trust_generation = excluded.workspace_trust_generation,
runtime_public_key_fingerprint = excluded.runtime_public_key_fingerprint,
runtime_identity_revision = excluded.runtime_identity_revision,
challenge_id = excluded.challenge_id,
state = excluded.state,
last_outcome = excluded.last_outcome,
verified_at = excluded.verified_at,
checked_at = excluded.checked_at"#,
params![
evidence.workspace_id,
evidence.runtime_id,
evidence.binding_revision,
evidence.workspace_key_id,
evidence.workspace_identity_revision,
evidence.workspace_trust_generation,
evidence.runtime_public_key_fingerprint,
evidence.runtime_identity_revision,
evidence.challenge_id,
evidence.state,
evidence.last_outcome,
evidence.verified_at,
evidence.checked_at,
],
)?;
Ok(())
})
}
pub fn complete_workspace_runtime_verification(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<WorkspaceRuntimeBinding> {
validate_workspace_runtime_verification(evidence)?;
if evidence.state != "verified" || evidence.verified_at.is_none() {
return Err(Error::Store(
"completed Runtime verification evidence must be verified".to_string(),
));
}
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let changed = tx.execute(
r#"UPDATE workspace_runtime_bindings
SET state = 'verified', updated_at = ?7
WHERE workspace_id = ?1 AND runtime_id = ?2
AND binding_revision = ?3
AND state IN ('configured', 'verified')
AND authentication_mode = 'workspace_identity'
AND workspace_key_id = ?4
AND workspace_key_generation = ?5
AND public_key_fingerprint = ?6
AND revoked_at IS NULL"#,
params![
evidence.workspace_id,
evidence.runtime_id,
evidence.binding_revision,
evidence.workspace_key_id,
evidence.workspace_trust_generation,
evidence.runtime_public_key_fingerprint,
evidence.checked_at,
],
)?;
if changed != 1 {
return Err(Error::RuntimeBindingConflict(
"Runtime verification evidence no longer matches the configured binding"
.to_string(),
));
}
tx.execute(
r#"INSERT INTO workspace_runtime_verifications (
workspace_id, runtime_id, binding_revision, workspace_key_id,
workspace_identity_revision, workspace_trust_generation,
runtime_public_key_fingerprint, runtime_identity_revision,
challenge_id, state, last_outcome, verified_at, checked_at
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)
ON CONFLICT(workspace_id, runtime_id) DO UPDATE SET
binding_revision = excluded.binding_revision,
workspace_key_id = excluded.workspace_key_id,
workspace_identity_revision = excluded.workspace_identity_revision,
workspace_trust_generation = excluded.workspace_trust_generation,
runtime_public_key_fingerprint = excluded.runtime_public_key_fingerprint,
runtime_identity_revision = excluded.runtime_identity_revision,
challenge_id = excluded.challenge_id,
state = excluded.state,
last_outcome = excluded.last_outcome,
verified_at = excluded.verified_at,
checked_at = excluded.checked_at"#,
params![
evidence.workspace_id,
evidence.runtime_id,
evidence.binding_revision,
evidence.workspace_key_id,
evidence.workspace_identity_revision,
evidence.workspace_trust_generation,
evidence.runtime_public_key_fingerprint,
evidence.runtime_identity_revision,
evidence.challenge_id,
evidence.state,
evidence.last_outcome,
evidence.verified_at,
evidence.checked_at,
],
)?;
let binding = tx.query_row(
r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key,
public_key_fingerprint, binding_revision, state, authentication_mode,
workspace_key_id, workspace_key_generation, created_at, updated_at, revoked_at
FROM workspace_runtime_bindings
WHERE workspace_id = ?1 AND runtime_id = ?2"#,
params![evidence.workspace_id, evidence.runtime_id],
read_workspace_runtime_binding,
)?;
tx.commit()?;
Ok(binding)
})
}
pub fn list_workspace_runtime_binding_audit(
&self,
workspace_id: &str,
@@ -2919,6 +3139,42 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
.is_some_and(|binding| binding == *expected && binding.revoked_at.is_none()))
}
fn workspace_runtime_verification_matches(
&self,
binding: &WorkspaceRuntimeBinding,
workspace_identity_revision: u64,
workspace_trust_generation: u64,
) -> Result<bool> {
SqliteWorkspaceStore::workspace_runtime_verification_matches(
self,
binding,
workspace_identity_revision,
workspace_trust_generation,
)
}
async fn get_workspace_runtime_verification(
&self,
workspace_id: &str,
runtime_id: &str,
) -> Result<Option<WorkspaceRuntimeVerificationEvidence>> {
SqliteWorkspaceStore::get_workspace_runtime_verification(self, workspace_id, runtime_id)
}
async fn record_workspace_runtime_verification_attempt(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<()> {
SqliteWorkspaceStore::record_workspace_runtime_verification_attempt(self, evidence)
}
async fn complete_workspace_runtime_verification(
&self,
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<WorkspaceRuntimeBinding> {
SqliteWorkspaceStore::complete_workspace_runtime_verification(self, evidence)
}
async fn get_workspace_runtime_binding(
&self,
workspace_id: &str,
@@ -6424,6 +6680,56 @@ fn account_select_sql(where_clause: &str) -> String {
)
}
fn read_workspace_runtime_verification(
row: &rusqlite::Row<'_>,
) -> rusqlite::Result<WorkspaceRuntimeVerificationEvidence> {
Ok(WorkspaceRuntimeVerificationEvidence {
workspace_id: row.get(0)?,
runtime_id: row.get(1)?,
binding_revision: row.get(2)?,
workspace_key_id: row.get(3)?,
workspace_identity_revision: row.get(4)?,
workspace_trust_generation: row.get(5)?,
runtime_public_key_fingerprint: row.get(6)?,
runtime_identity_revision: row.get(7)?,
challenge_id: row.get(8)?,
state: row.get(9)?,
last_outcome: row.get(10)?,
verified_at: row.get(11)?,
checked_at: row.get(12)?,
})
}
fn validate_workspace_runtime_verification(
evidence: &WorkspaceRuntimeVerificationEvidence,
) -> Result<()> {
for (field, value) in [
("workspace_id", evidence.workspace_id.as_str()),
("runtime_id", evidence.runtime_id.as_str()),
("workspace_key_id", evidence.workspace_key_id.as_str()),
("challenge_id", evidence.challenge_id.as_str()),
] {
validate_identifier(field, value)?;
}
if evidence.binding_revision == 0
|| evidence.workspace_identity_revision == 0
|| evidence.workspace_trust_generation == 0
|| evidence.runtime_identity_revision == 0
{
return Err(Error::InvalidInput(
"Runtime verification revisions and generations must be positive".to_string(),
));
}
validate_non_empty(
"runtime_public_key_fingerprint",
&evidence.runtime_public_key_fingerprint,
)?;
validate_non_empty("verification state", &evidence.state)?;
validate_non_empty("verification outcome", &evidence.last_outcome)?;
validate_non_empty("checked_at", &evidence.checked_at)?;
Ok(())
}
fn read_workspace_runtime_binding(
row: &rusqlite::Row<'_>,
) -> rusqlite::Result<WorkspaceRuntimeBinding> {
@@ -7817,6 +8123,80 @@ fn migrate_workspace_runtime_binding_state_v54_to_v55(conn: &Connection) -> Resu
Ok(())
}
fn migrate_workspace_runtime_verification_v55_to_v56(conn: &Connection) -> Result<()> {
let current = current_schema_version(conn)?;
if current != 55 {
return Err(Error::Store(format!(
"expected schema version 55 before {WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME} migration, found {current}"
)));
}
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
tx.execute_batch(
r#"
CREATE TABLE IF NOT EXISTS workspace_runtime_verifications (
workspace_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
binding_revision INTEGER NOT NULL CHECK(binding_revision > 0),
workspace_key_id TEXT NOT NULL,
workspace_identity_revision INTEGER NOT NULL CHECK(workspace_identity_revision > 0),
workspace_trust_generation INTEGER NOT NULL CHECK(workspace_trust_generation > 0),
runtime_public_key_fingerprint TEXT NOT NULL,
runtime_identity_revision INTEGER NOT NULL CHECK(runtime_identity_revision > 0),
challenge_id TEXT NOT NULL,
state TEXT NOT NULL CHECK(state IN ('pending', 'verified', 'failed')),
last_outcome TEXT NOT NULL,
verified_at TEXT,
checked_at TEXT NOT NULL,
PRIMARY KEY(workspace_id, runtime_id),
FOREIGN KEY(workspace_id, runtime_id)
REFERENCES workspace_runtime_bindings(workspace_id, runtime_id)
ON DELETE CASCADE,
CHECK((state = 'verified' AND verified_at IS NOT NULL)
OR (state != 'verified' AND verified_at IS NULL))
);
CREATE INDEX IF NOT EXISTS workspace_runtime_verifications_state_idx
ON workspace_runtime_verifications(workspace_id, state, checked_at DESC);
"#,
)?;
verify_workspace_runtime_verification_schema(&tx)?;
tx.execute(
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)",
params![56_i64, WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME],
)?;
tx.commit()?;
Ok(())
}
fn verify_workspace_runtime_verification_schema(conn: &Connection) -> Result<()> {
let actual = table_columns(conn, "workspace_runtime_verifications")?
.into_iter()
.collect::<BTreeSet<_>>();
let expected = [
"workspace_id",
"runtime_id",
"binding_revision",
"workspace_key_id",
"workspace_identity_revision",
"workspace_trust_generation",
"runtime_public_key_fingerprint",
"runtime_identity_revision",
"challenge_id",
"state",
"last_outcome",
"verified_at",
"checked_at",
]
.into_iter()
.map(str::to_string)
.collect::<BTreeSet<_>>();
if actual != expected {
return Err(Error::Store(format!(
"workspace_runtime_verifications schema does not match schema-56: {actual:?}"
)));
}
Ok(())
}
fn verify_workspace_signing_identity_schema(conn: &Connection) -> Result<()> {
for (table, expected) in [
(
@@ -8679,6 +9059,7 @@ fn apply_migrations(conn: &Connection) -> Result<()> {
verify_schema_history(conn, LATEST_SCHEMA_VERSION)?;
verify_workspace_runtime_binding_schema(conn)?;
verify_workspace_runtime_verification_schema(conn)?;
verify_workspace_deletion_schema(conn)?;
verify_workspace_signing_identity_schema(conn)
}
@@ -8902,6 +9283,10 @@ mod tests {
version: 55,
name: WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME.to_string(),
},
WorkspaceSchemaMigrationStep {
version: 56,
name: WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME.to_string(),
},
]
);
@@ -8928,6 +9313,10 @@ mod tests {
55,
WORKSPACE_RUNTIME_BINDING_STATE_MIGRATION_NAME.to_string(),
),
(
56,
WORKSPACE_RUNTIME_VERIFICATION_MIGRATION_NAME.to_string(),
),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -8997,7 +9386,7 @@ mod tests {
.iter()
.map(|migration| migration.version)
.collect::<Vec<_>>(),
vec![52, 53, 54, 55]
vec![52, 53, 54, 55, 56]
);
SqliteWorkspaceStore::migrate_database(&path).unwrap();
let conn = Connection::open(&path).unwrap();
@@ -9005,7 +9394,7 @@ mod tests {
current_schema_version(&conn).unwrap(),
LATEST_SCHEMA_VERSION
);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 6);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 7);
}
#[test]
@@ -9371,6 +9760,123 @@ mod tests {
);
}
#[test]
fn workspace_runtime_verification_is_revision_bound_and_restart_safe() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("server.db");
let store = SqliteWorkspaceStore::open(&path).unwrap();
store
.with_conn(|conn| {
conn.execute_batch(
r#"
INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at)
VALUES ('owner', 'user', 'owner', 'Owner', '1', '1');
INSERT INTO workspaces(workspace_id, owner_account_id, display_name, state, created_at, updated_at)
VALUES ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1');
INSERT INTO workspace_signing_identities(
workspace_id, key_id, algorithm, public_key, public_key_fingerprint,
private_material_ref, revision, state, created_at, provisioned_at, updated_at
) VALUES ('workspace-a', 'WK-a', 'ed25519', 'key', 'sha256:key',
'workspace-signing/workspace-a/ed25519-v1', 1, 'active', '1', '1', '1');
"#,
)?;
Ok(())
})
.unwrap();
let runtime_identity =
worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-a").unwrap();
store
.upsert_workspace_runtime_binding(
WorkspaceRuntimeBinding {
workspace_id: "workspace-a".to_string(),
runtime_id: "runtime-a".to_string(),
display_name: "runtime-a".to_string(),
base_url: "https://runtime.test".to_string(),
public_key: runtime_identity.public_key,
public_key_fingerprint: String::new(),
binding_revision: 1,
state: WorkspaceRuntimeBindingState::Configured,
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
workspace_key_id: Some("WK-a".to_string()),
workspace_key_generation: Some(1),
created_at: "1".to_string(),
updated_at: "1".to_string(),
revoked_at: None,
},
false,
)
.unwrap();
let persisted = store
.get_workspace_runtime_binding("workspace-a", "runtime-a")
.unwrap()
.unwrap();
let evidence = WorkspaceRuntimeVerificationEvidence {
workspace_id: "workspace-a".to_string(),
runtime_id: "runtime-a".to_string(),
binding_revision: persisted.binding_revision,
workspace_key_id: "WK-a".to_string(),
workspace_identity_revision: 1,
workspace_trust_generation: 1,
runtime_public_key_fingerprint: persisted.public_key_fingerprint.clone(),
runtime_identity_revision: 1,
challenge_id: "challenge-a".to_string(),
state: "verified".to_string(),
last_outcome: "verified".to_string(),
verified_at: Some("2".to_string()),
checked_at: "2".to_string(),
};
let verified = store
.complete_workspace_runtime_verification(&evidence)
.unwrap();
assert_eq!(verified.state, WorkspaceRuntimeBindingState::Verified);
drop(store);
let reopened = SqliteWorkspaceStore::open(&path).unwrap();
assert_eq!(
reopened
.get_workspace_runtime_verification("workspace-a", "runtime-a")
.unwrap(),
Some(evidence)
);
assert_eq!(
reopened
.get_workspace_runtime_binding("workspace-a", "runtime-a")
.unwrap()
.unwrap()
.state,
WorkspaceRuntimeBindingState::Verified
);
}
#[test]
fn schema_v55_migrates_runtime_verification_table_atomically() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("server.db");
let store = SqliteWorkspaceStore::open(&path).unwrap();
store
.with_conn(|conn| {
conn.execute_batch(
"DROP TABLE workspace_runtime_verifications;
DELETE FROM __yoi_schema_migrations;
INSERT INTO __yoi_schema_migrations(version, name)
VALUES (55, 'Workspace Runtime binding state and identity mode');",
)?;
Ok(())
})
.unwrap();
drop(store);
let conn = Connection::open(&path).unwrap();
configure_sqlite(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 55);
migrate_workspace_runtime_verification_v55_to_v56(&conn).unwrap();
drop(conn);
let migrated = Connection::open(&path).unwrap();
configure_sqlite(&migrated).unwrap();
assert_eq!(current_schema_version(&migrated).unwrap(), 56);
assert!(table_exists(&migrated, "workspace_runtime_verifications").unwrap());
}
#[test]
fn runtime_binding_key_mutations_are_revisioned_idempotent_and_audited() {
let store = SqliteWorkspaceStore::in_memory().unwrap();
@@ -10691,13 +11197,13 @@ INSERT INTO worker_registry (
let conn = Connection::open_in_memory().unwrap();
configure_sqlite(&conn).unwrap();
conn.execute(
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (56, 'future')",
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (57, 'future')",
[],
)
.unwrap();
let error = apply_migrations(&conn).unwrap_err().to_string();
assert!(error.contains("schema version 56 is newer"), "{error}");
assert!(error.contains("schema version 57 is newer"), "{error}");
assert!(error.contains("refusing to serve"), "{error}");
}
crates/workspace-server/src/workspace_deletion.rs
+1
View File
@@ -80,6 +80,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"workspace_resource_keys",
"workspace_runtime_binding_audit",
"workspace_runtime_bindings",
"workspace_runtime_verifications",
"workspace_signing_identities",
"workspace_signing_identity_audit",
"workspace_signing_identity_provisioning_operations",
crates/workspace-server/src/workspace_signing_identity.rs
+21
View File
@@ -9,6 +9,10 @@ use chrono::{SecondsFormat, Utc};
use ring::signature::KeyPair;
use serde::{Deserialize, Serialize};
use worker_runtime::auth::{RuntimeIdentityMaterial, encode_public_key};
use worker_runtime::workspace_issuer::{
WorkspaceCapabilityClaims, WorkspaceCapabilityVerificationError,
assemble_workspace_capability_token, workspace_capability_signing_input,
};
use zeroize::Zeroize;
use crate::store::{
@@ -254,6 +258,16 @@ impl WorkspaceSigningIdentityService {
Ok(signing_key.sign(payload).as_ref().to_vec())
}
pub fn issue_workspace_capability(
&self,
workspace_id: &str,
claims: &WorkspaceCapabilityClaims,
) -> Result<String> {
let input = workspace_capability_signing_input(claims).map_err(capability_error)?;
let signature = self.sign(workspace_id, input.bytes())?;
assemble_workspace_capability_token(input, &signature).map_err(capability_error)
}
pub fn delete_material(&self, workspace_id: &str) -> Result<()> {
if let Some(identity) = self.store.get_workspace_signing_identity(workspace_id)? {
self.materials.delete(&identity.private_material_ref)?;
@@ -572,6 +586,13 @@ fn material_io_error(action: &str, error: std::io::Error) -> Error {
)
}
fn capability_error(error: WorkspaceCapabilityVerificationError) -> Error {
identity_error(
"workspace_capability_issuance_failed",
format!("failed to issue Workspace capability: {error}"),
)
}
pub fn identity_error(code: impl Into<String>, message: impl Into<String>) -> Error {
Error::WorkspaceSigningIdentity {
code: code.into(),