fix: authenticate remote worker removal through workspace router
This commit is contained in:
@@ -134,6 +134,8 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync {
|
|||||||
enum RuntimeWorkerMutationTransport {
|
enum RuntimeWorkerMutationTransport {
|
||||||
Remote {
|
Remote {
|
||||||
base_url: String,
|
base_url: String,
|
||||||
|
request_source_signer: RuntimeRequestSourceSigner,
|
||||||
|
request_source_audience: String,
|
||||||
},
|
},
|
||||||
Embedded {
|
Embedded {
|
||||||
dispatcher: Arc<dyn EmbeddedWorkerMutationDispatcher>,
|
dispatcher: Arc<dyn EmbeddedWorkerMutationDispatcher>,
|
||||||
@@ -157,10 +159,12 @@ impl RuntimeWorkerMutationForwarder {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
authority: RuntimeWorkerMutationSourceAuthority::remote(identity),
|
authority: RuntimeWorkerMutationSourceAuthority::remote(identity),
|
||||||
scope,
|
scope: scope.clone(),
|
||||||
source_worker_id: source_worker_id.into(),
|
source_worker_id: source_worker_id.into(),
|
||||||
transport: RuntimeWorkerMutationTransport::Remote {
|
transport: RuntimeWorkerMutationTransport::Remote {
|
||||||
base_url: base_url.into().trim_end_matches('/').to_string(),
|
base_url: base_url.into().trim_end_matches('/').to_string(),
|
||||||
|
request_source_signer: RuntimeRequestSourceSigner::from_identity(identity),
|
||||||
|
request_source_audience: scope.server_id,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -197,11 +201,18 @@ impl RuntimeWorkerMutationForwarder {
|
|||||||
)?;
|
)?;
|
||||||
match (&self.transport, proof) {
|
match (&self.transport, proof) {
|
||||||
(
|
(
|
||||||
RuntimeWorkerMutationTransport::Remote { base_url },
|
RuntimeWorkerMutationTransport::Remote {
|
||||||
|
base_url,
|
||||||
|
request_source_signer,
|
||||||
|
request_source_audience,
|
||||||
|
},
|
||||||
RuntimeOwnedWorkerMutationProof::Remote(token),
|
RuntimeOwnedWorkerMutationProof::Remote(token),
|
||||||
) => execute_remote_worker_remove_http(RemoteWorkerRemoveHttpRequest {
|
) => execute_remote_worker_remove_http(RemoteWorkerRemoveHttpRequest {
|
||||||
base_url: base_url.clone(),
|
base_url: base_url.clone(),
|
||||||
workspace_id: self.scope.workspace_id.clone(),
|
workspace_id: self.scope.workspace_id.clone(),
|
||||||
|
source_worker_id: self.source_worker_id.clone(),
|
||||||
|
request_source_signer: request_source_signer.clone(),
|
||||||
|
request_source_audience: request_source_audience.clone(),
|
||||||
token,
|
token,
|
||||||
target_runtime_id: target_runtime_id.to_string(),
|
target_runtime_id: target_runtime_id.to_string(),
|
||||||
target_worker_id: target_worker_id.to_string(),
|
target_worker_id: target_worker_id.to_string(),
|
||||||
@@ -224,6 +235,9 @@ impl RuntimeWorkerMutationForwarder {
|
|||||||
struct RemoteWorkerRemoveHttpRequest {
|
struct RemoteWorkerRemoveHttpRequest {
|
||||||
base_url: String,
|
base_url: String,
|
||||||
workspace_id: String,
|
workspace_id: String,
|
||||||
|
source_worker_id: String,
|
||||||
|
request_source_signer: RuntimeRequestSourceSigner,
|
||||||
|
request_source_audience: String,
|
||||||
token: String,
|
token: String,
|
||||||
target_runtime_id: String,
|
target_runtime_id: String,
|
||||||
target_worker_id: String,
|
target_worker_id: String,
|
||||||
@@ -256,23 +270,35 @@ fn execute_remote_worker_remove_http(
|
|||||||
fn execute_remote_worker_remove_http_blocking(
|
fn execute_remote_worker_remove_http_blocking(
|
||||||
request: RemoteWorkerRemoveHttpRequest,
|
request: RemoteWorkerRemoveHttpRequest,
|
||||||
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
||||||
let url = format!(
|
let path = format!("/api/w/{}/workers/remove", request.workspace_id);
|
||||||
"{}/api/w/{}/workers/remove",
|
let url = format!("{}{}", request.base_url, path);
|
||||||
request.base_url, request.workspace_id
|
let body = serde_json::to_string(&serde_json::json!({
|
||||||
);
|
|
||||||
let body = serde_json::json!({
|
|
||||||
"target_runtime_id": request.target_runtime_id,
|
"target_runtime_id": request.target_runtime_id,
|
||||||
"target_worker_id": request.target_worker_id,
|
"target_worker_id": request.target_worker_id,
|
||||||
"reason": request.reason,
|
"reason": request.reason,
|
||||||
});
|
}))
|
||||||
|
.map_err(|error| RuntimeWorkerMutationForwardError::Transport(error.to_string()))?;
|
||||||
|
let request_source_proof = request.request_source_signer.issue(
|
||||||
|
&request.request_source_audience,
|
||||||
|
&request.workspace_id,
|
||||||
|
Some(&request.source_worker_id),
|
||||||
|
WORKSPACE_REQUEST_PERMISSION,
|
||||||
|
"POST",
|
||||||
|
&path,
|
||||||
|
body.as_bytes(),
|
||||||
|
i64::try_from(unix_now_seconds()).unwrap_or(i64::MAX),
|
||||||
|
30,
|
||||||
|
)?;
|
||||||
let client = reqwest::blocking::Client::new();
|
let client = reqwest::blocking::Client::new();
|
||||||
let response = client
|
let response = client
|
||||||
.post(url)
|
.post(url)
|
||||||
|
.header(RUNTIME_REQUEST_SOURCE_PROOF_HEADER, request_source_proof)
|
||||||
.header(
|
.header(
|
||||||
crate::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER,
|
crate::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER,
|
||||||
request.token,
|
request.token,
|
||||||
)
|
)
|
||||||
.json(&body)
|
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||||
|
.body(body)
|
||||||
.send()
|
.send()
|
||||||
.map_err(|error| RuntimeWorkerMutationForwardError::Transport(error.to_string()))?;
|
.map_err(|error| RuntimeWorkerMutationForwardError::Transport(error.to_string()))?;
|
||||||
let status = response.status().as_u16();
|
let status = response.status().as_u16();
|
||||||
@@ -697,7 +723,8 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::auth::{
|
use crate::auth::{
|
||||||
WorkerMutationSourceExpectation, decode_runtime_request_source_claims,
|
WorkerMutationSourceExpectation, decode_runtime_request_source_claims,
|
||||||
decode_worker_mutation_source_claims, verify_worker_mutation_source_proof,
|
decode_worker_mutation_source_claims, request_body_digest,
|
||||||
|
verify_worker_mutation_source_proof,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1119,6 +1146,41 @@ mod tests {
|
|||||||
assert!(request.contains("\"target_worker_id\":\"worker-target\""));
|
assert!(request.contains("\"target_worker_id\":\"worker-target\""));
|
||||||
assert!(!request.contains("expected_worker_revision"));
|
assert!(!request.contains("expected_worker_revision"));
|
||||||
assert!(request.contains("\"reason\":\"retire obsolete Worker\""));
|
assert!(request.contains("\"reason\":\"retire obsolete Worker\""));
|
||||||
|
let request_source_token = request
|
||||||
|
.lines()
|
||||||
|
.find_map(|line| {
|
||||||
|
line.split_once(':').and_then(|(name, value)| {
|
||||||
|
name.eq_ignore_ascii_case(RUNTIME_REQUEST_SOURCE_PROOF_HEADER)
|
||||||
|
.then(|| value.trim())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.expect("runtime request source proof header");
|
||||||
|
let request_source_claims =
|
||||||
|
decode_runtime_request_source_claims(request_source_token).unwrap();
|
||||||
|
assert_eq!(request_source_claims.iss, "runtime-a");
|
||||||
|
assert_eq!(request_source_claims.aud, "server-a");
|
||||||
|
assert_eq!(request_source_claims.workspace_id, "workspace-a");
|
||||||
|
assert_eq!(
|
||||||
|
request_source_claims.worker_id.as_deref(),
|
||||||
|
Some("worker-source")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
request_source_claims.permission,
|
||||||
|
WORKSPACE_REQUEST_PERMISSION
|
||||||
|
);
|
||||||
|
assert_eq!(request_source_claims.method, "POST");
|
||||||
|
assert_eq!(
|
||||||
|
request_source_claims.path,
|
||||||
|
"/api/w/workspace-a/workers/remove"
|
||||||
|
);
|
||||||
|
let request_body = request
|
||||||
|
.split_once("\r\n\r\n")
|
||||||
|
.map(|(_, body)| body)
|
||||||
|
.expect("WorkerRemove request body");
|
||||||
|
assert_eq!(
|
||||||
|
request_source_claims.body_digest,
|
||||||
|
request_body_digest(request_body.as_bytes())
|
||||||
|
);
|
||||||
let token = request
|
let token = request
|
||||||
.lines()
|
.lines()
|
||||||
.find_map(|line| {
|
.find_map(|line| {
|
||||||
|
|||||||
@@ -21778,6 +21778,119 @@ mod tests {
|
|||||||
assert!(!route_body.contains("source"));
|
assert!(!route_body.contains("source"));
|
||||||
assert!(!route_body.contains("proof"));
|
assert!(!route_body.contains("proof"));
|
||||||
|
|
||||||
|
let outer_path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove");
|
||||||
|
let outer_request_body = r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker","reason":"retire target Worker"}"#;
|
||||||
|
let outer_mutation_token = signer
|
||||||
|
.issue_worker_remove(
|
||||||
|
"server-main",
|
||||||
|
&api.config.workspace_id,
|
||||||
|
"7",
|
||||||
|
"runtime-target",
|
||||||
|
"target-worker",
|
||||||
|
60,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let outer_request_token =
|
||||||
|
worker_runtime::auth::RuntimeRequestSourceSigner::from_identity(&identity)
|
||||||
|
.issue(
|
||||||
|
"server-main",
|
||||||
|
&api.config.workspace_id,
|
||||||
|
Some("7"),
|
||||||
|
worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION,
|
||||||
|
"POST",
|
||||||
|
&outer_path,
|
||||||
|
outer_request_body.as_bytes(),
|
||||||
|
i64::try_from(worker_runtime::auth::unix_now_seconds()).unwrap_or(i64::MAX),
|
||||||
|
30,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let outer_response = build_router(api.clone())
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri(&outer_path)
|
||||||
|
.header(CONTENT_TYPE, "application/json")
|
||||||
|
.header(
|
||||||
|
worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER,
|
||||||
|
outer_request_token,
|
||||||
|
)
|
||||||
|
.header(
|
||||||
|
worker_runtime::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER,
|
||||||
|
outer_mutation_token,
|
||||||
|
)
|
||||||
|
.body(Body::from(outer_request_body))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(outer_response.status(), StatusCode::NOT_FOUND);
|
||||||
|
let outer_response_body = axum::body::to_bytes(outer_response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
String::from_utf8(outer_response_body.to_vec())
|
||||||
|
.unwrap()
|
||||||
|
.contains("unknown_worker")
|
||||||
|
);
|
||||||
|
|
||||||
|
let missing_outer_mutation_token = signer
|
||||||
|
.issue_worker_remove(
|
||||||
|
"server-main",
|
||||||
|
&api.config.workspace_id,
|
||||||
|
"7",
|
||||||
|
"runtime-target",
|
||||||
|
"target-worker",
|
||||||
|
60,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let missing_outer_response = build_router(api.clone())
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri(&outer_path)
|
||||||
|
.header(CONTENT_TYPE, "application/json")
|
||||||
|
.header(
|
||||||
|
worker_runtime::auth::WORKER_MUTATION_SOURCE_PROOF_HEADER,
|
||||||
|
missing_outer_mutation_token,
|
||||||
|
)
|
||||||
|
.body(Body::from(outer_request_body))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(missing_outer_response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
|
||||||
|
let missing_mutation_request_token =
|
||||||
|
worker_runtime::auth::RuntimeRequestSourceSigner::from_identity(&identity)
|
||||||
|
.issue(
|
||||||
|
"server-main",
|
||||||
|
&api.config.workspace_id,
|
||||||
|
Some("7"),
|
||||||
|
worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION,
|
||||||
|
"POST",
|
||||||
|
&outer_path,
|
||||||
|
outer_request_body.as_bytes(),
|
||||||
|
i64::try_from(worker_runtime::auth::unix_now_seconds()).unwrap_or(i64::MAX),
|
||||||
|
30,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let missing_mutation_response = build_router(api.clone())
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri(&outer_path)
|
||||||
|
.header(CONTENT_TYPE, "application/json")
|
||||||
|
.header(
|
||||||
|
worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER,
|
||||||
|
missing_mutation_request_token,
|
||||||
|
)
|
||||||
|
.body(Body::from(outer_request_body))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(missing_mutation_response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
|
||||||
let mut revoked = trust;
|
let mut revoked = trust;
|
||||||
revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string());
|
revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string());
|
||||||
let authority = SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap();
|
let authority = SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap();
|
||||||
|
|||||||
Reference in New Issue
Block a user