Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

worker: add guarded WorkerRemove lifecycle

Browse Source
This commit is contained in:
Hare
2026-08-12 17:00:32 +09:00
parent ebe0f93744
commit 8ae930c5fc
12 changed files with 1565 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/worker-runtime/src/http_server.rs
+81
View File
@@ -21,6 +21,9 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck};
use crate::management::{RuntimeSummary, WorkerDeleteResult};
#[cfg(feature = "ws-server")]
use crate::observation::WorkerObservationCursor;
use crate::retention::{
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
};
#[cfg(feature = "ws-server")]
use crate::runtime::RuntimeSubscriptionRecvError;
use crate::{Runtime, RuntimeWorkspaceScope};
@@ -217,6 +220,14 @@ fn runtime_http_router_with_optional_auth(
"/v1/workers/{worker_id}",
get(get_worker).delete(delete_worker),
)
.route(
"/v1/workers/{worker_id}/retention/inventory",
get(worker_retention_inventory),
)
.route(
"/v1/workers/{worker_id}/retention/execute",
post(execute_worker_retention),
)
.route("/v1/workers/{worker_id}/input", post(send_worker_input))
.route("/v1/workers/{worker_id}/restore", post(restore_worker))
.route(
@@ -1220,6 +1231,61 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
}
}
async fn worker_retention_inventory(
State(state): State<RuntimeHttpState>,
auth: Option<Extension<RuntimeAuthContext>>,
Path(worker_id): Path<String>,
) -> RestResult<WorkerRetentionInventory> {
let scope = auth_workspace_scope(&state, auth.as_ref())?.ok_or_else(|| {
RuntimeHttpRestError::new(
StatusCode::FORBIDDEN,
"workspace_scope_required",
"Worker retention inventory requires workspace-scoped authorization",
)
})?;
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
state
.runtime
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
.map(Json)
.map_err(RuntimeHttpRestError::runtime)
}
async fn execute_worker_retention(
State(state): State<RuntimeHttpState>,
auth: Option<Extension<RuntimeAuthContext>>,
Path(worker_id): Path<String>,
body: Result<Json<WorkerRetentionExecutionRequest>, JsonRejection>,
) -> RestResult<WorkerRetentionExecutionResult> {
let Json(request) = body.map_err(RuntimeHttpRestError::json_rejection)?;
if request.worker_id.to_string() != worker_id {
return Err(RuntimeHttpRestError::new(
StatusCode::BAD_REQUEST,
"worker_id_mismatch",
"Retention request worker_id does not match the route",
));
}
let scope = auth_workspace_scope(&state, auth.as_ref())?.ok_or_else(|| {
RuntimeHttpRestError::new(
StatusCode::FORBIDDEN,
"workspace_scope_required",
"Worker retention execution requires workspace-scoped authorization",
)
})?;
if request.workspace_id != scope.workspace_id {
return Err(RuntimeHttpRestError::new(
StatusCode::NOT_FOUND,
"worker_not_found",
"Worker was not found in the authenticated Workspace",
));
}
state
.runtime
.execute_worker_retention(&request)
.map(Json)
.map_err(RuntimeHttpRestError::runtime)
}
async fn send_worker_input(
State(state): State<RuntimeHttpState>,
auth: Option<Extension<RuntimeAuthContext>>,
@@ -1473,6 +1539,9 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
if path.ends_with("/completions") {
return Some("workers:read");
}
if path.contains("/retention/") {
return Some("workers:delete");
}
if path.starts_with("/v1/workers/") && *method == Method::DELETE {
return Some("workers:delete");
}
@@ -2066,6 +2135,18 @@ mod tests {
}
}
#[test]
fn retention_routes_require_worker_delete_permission() {
assert_eq!(
required_runtime_permission(&Method::GET, "/v1/workers/worker-1/retention/inventory",),
Some("workers:delete")
);
assert_eq!(
required_runtime_permission(&Method::POST, "/v1/workers/worker-1/retention/execute",),
Some("workers:delete")
);
}
#[test]
fn workdir_routes_require_dedicated_operation_permission() {
assert_eq!(
crates/worker-runtime/src/retention.rs
+4
View File
@@ -117,6 +117,7 @@ pub struct WorkerRetentionExecutionRequest {
pub workspace_id: String,
pub source_runtime_id: String,
pub worker_id: WorkerId,
pub expected_worker_revision: String,
pub expected_run_generation: u64,
pub source_created_at: String,
pub removed_at: String,
@@ -155,6 +156,7 @@ pub struct WorkerSessionArchiveManifest {
pub struct WorkerRetentionExecutionResult {
pub operation_id: String,
pub input_fingerprint: String,
pub expected_worker_revision: String,
pub worker_id: WorkerId,
pub session_disposition: SessionDisposition,
pub diagnostics_disposition: DiagnosticsDisposition,
@@ -521,6 +523,7 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
let mut result = WorkerRetentionExecutionResult {
operation_id: request.operation_id.clone(),
input_fingerprint: request.input_fingerprint.clone(),
expected_worker_revision: request.expected_worker_revision.clone(),
worker_id: request.worker_id,
session_disposition: request.session_disposition,
diagnostics_disposition: request.diagnostics_disposition,
@@ -1291,6 +1294,7 @@ mod tests {
WorkerRetentionExecutionRequest {
operation_id: "operation-a".to_string(),
input_fingerprint: "fingerprint-a".to_string(),
expected_worker_revision: "revision-a".to_string(),
archive_id: (disposition == SessionDisposition::Archive)
.then(|| "archive-a".to_string()),
workspace_id: "workspace-a".to_string(),
crates/worker-runtime/src/worker_source.rs
+45 -9
View File
@@ -124,6 +124,8 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync {
proof: InProcessWorkerMutationProof,
target_runtime_id: &str,
target_worker_id: &str,
expected_worker_revision: &str,
reason: &str,
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError>;
}
@@ -186,6 +188,8 @@ impl RuntimeWorkerMutationForwarder {
&self,
target_runtime_id: &str,
target_worker_id: &str,
expected_worker_revision: &str,
reason: &str,
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
let proof = self.authority.issue_worker_remove(
&self.scope,
@@ -205,6 +209,8 @@ impl RuntimeWorkerMutationForwarder {
let body = serde_json::json!({
"target_runtime_id": target_runtime_id,
"target_worker_id": target_worker_id,
"expected_worker_revision": expected_worker_revision,
"reason": reason,
});
let response = client
.post(url)
@@ -223,7 +229,13 @@ impl RuntimeWorkerMutationForwarder {
(
RuntimeWorkerMutationTransport::Embedded { dispatcher },
RuntimeOwnedWorkerMutationProof::InProcess(claims),
) => dispatcher.execute_worker_remove(claims, target_runtime_id, target_worker_id),
) => dispatcher.execute_worker_remove(
claims,
target_runtime_id,
target_worker_id,
expected_worker_revision,
reason,
),
_ => Err(RuntimeWorkerMutationForwardError::AuthorityTransportMismatch),
}
}
@@ -316,6 +328,8 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
&self,
target_runtime_id: &str,
target_worker_id: &str,
expected_worker_revision: &str,
reason: &str,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
self.worker_remove
.as_ref()
@@ -324,7 +338,12 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
"Runtime-owned WorkerRemove forwarding is unavailable".to_string(),
)
})?
.execute_worker_remove(target_runtime_id, target_worker_id)
.execute_worker_remove(
target_runtime_id,
target_worker_id,
expected_worker_revision,
reason,
)
.map_err(|error| WorkspaceClientError::Request(error.to_string()))
}
}
@@ -500,16 +519,22 @@ mod tests {
format!("http://{address}"),
);
let response = forwarder
.execute_worker_remove("runtime-target", "worker-target")
.execute_worker_remove(
"runtime-target",
"worker-target",
"revision-7",
"retire obsolete Worker",
)
.unwrap();
assert_eq!(response.status, 204);
server.join().unwrap();
let request = received.lock().unwrap().clone();
assert!(request.starts_with("POST /api/w/workspace-a/workers/remove HTTP/1.1"));
assert!(request.contains(
r#"{"target_runtime_id":"runtime-target","target_worker_id":"worker-target"}"#
));
assert!(request.contains("\"target_runtime_id\":\"runtime-target\""));
assert!(request.contains("\"target_worker_id\":\"worker-target\""));
assert!(request.contains("\"expected_worker_revision\":\"revision-7\""));
assert!(request.contains("\"reason\":\"retire obsolete Worker\""));
let token = request
.lines()
.find_map(|line| {
@@ -541,7 +566,7 @@ mod tests {
#[derive(Default)]
struct RecordingDispatcher {
seen: Mutex<Option<(WorkerMutationSourceClaims, String, String)>>,
seen: Mutex<Option<(WorkerMutationSourceClaims, String, String, String, String)>>,
}
impl EmbeddedWorkerMutationDispatcher for RecordingDispatcher {
fn execute_worker_remove(
@@ -549,11 +574,15 @@ mod tests {
proof: InProcessWorkerMutationProof,
target_runtime_id: &str,
target_worker_id: &str,
expected_worker_revision: &str,
reason: &str,
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
*self.seen.lock().unwrap() = Some((
proof.into_claims(),
target_runtime_id.to_string(),
target_worker_id.to_string(),
expected_worker_revision.to_string(),
reason.to_string(),
));
Ok(WorkspaceResponse {
status: 202,
@@ -571,10 +600,15 @@ mod tests {
dispatcher.clone(),
);
let response = forwarder
.execute_worker_remove("runtime-target", "worker-target")
.execute_worker_remove(
"runtime-target",
"worker-target",
"revision-7",
"retire obsolete Worker",
)
.unwrap();
assert_eq!(response.status, 202);
let (claims, target_runtime_id, target_worker_id) =
let (claims, target_runtime_id, target_worker_id, expected_revision, reason) =
dispatcher.seen.lock().unwrap().take().unwrap();
assert_eq!(claims.iss, "runtime-embedded");
assert_eq!(claims.worker_id, "worker-source");
@@ -582,6 +616,8 @@ mod tests {
assert_eq!(claims.target_worker_id, "worker-target");
assert_eq!(target_runtime_id, "runtime-target");
assert_eq!(target_worker_id, "worker-target");
assert_eq!(expected_revision, "revision-7");
assert_eq!(reason, "retire obsolete Worker");
}
#[test]