Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

worker: add guarded WorkerRemove lifecycle

Browse Source
This commit is contained in:
Hare
2026-08-12 17:00:32 +09:00
parent ebe0f93744
commit 8ae930c5fc
12 changed files with 1565 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/hosts.rs
+109 -1
View File
@@ -54,6 +54,9 @@ use worker_runtime::interaction::{
};
use worker_runtime::management::{RuntimeOptions as EmbeddedRuntimeOptions, RuntimeStatus};
use worker_runtime::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveInput};
use worker_runtime::retention::{
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
};
pub(crate) const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime";
const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host";
@@ -856,6 +859,25 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
}
}
fn worker_retention_inventory(
&self,
worker_id: &str,
) -> Result<WorkerRetentionInventory, String> {
Err(format!(
"runtime does not implement retention inventory for '{worker_id}'"
))
}
fn execute_worker_retention(
&self,
request: WorkerRetentionExecutionRequest,
) -> Result<WorkerRetentionExecutionResult, String> {
Err(format!(
"runtime does not implement retention execution for '{}'",
request.worker_id
))
}
fn observation_source(
&self,
_worker_id: &str,
@@ -1399,6 +1421,44 @@ impl RuntimeRegistry {
Ok(runtime.delete_worker(worker_id))
}
pub fn worker_retention_inventory(
&self,
worker: &RuntimeWorkerRef,
) -> Result<WorkerRetentionInventory, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", &worker.runtime_id)?;
validate_backend_identifier("worker_id", &worker.worker_id)?;
self.runtime(&worker.runtime_id)?
.worker_retention_inventory(&worker.worker_id)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: worker.runtime_id.clone(),
code: "worker_retention_inventory_failed".to_string(),
message,
})
}
pub fn execute_worker_retention(
&self,
worker: &RuntimeWorkerRef,
request: WorkerRetentionExecutionRequest,
) -> Result<WorkerRetentionExecutionResult, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", &worker.runtime_id)?;
validate_backend_identifier("worker_id", &worker.worker_id)?;
if request.worker_id.to_string() != worker.worker_id {
return Err(RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: worker.runtime_id.clone(),
code: "worker_id_mismatch".to_string(),
message: "retention request worker_id does not match target".to_string(),
});
}
self.runtime(&worker.runtime_id)?
.execute_worker_retention(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: worker.runtime_id.clone(),
code: "worker_retention_execution_failed".to_string(),
message,
})
}
pub fn observation_source(
&self,
worker: &RuntimeWorkerRef,
@@ -1438,6 +1498,7 @@ impl RuntimeRegistry {
#[derive(Clone)]
pub struct EmbeddedWorkerRuntime {
workspace_id: String,
runtime_id: String,
host_id: String,
runtime: worker_runtime::Runtime,
@@ -1501,8 +1562,9 @@ impl EmbeddedWorkerRuntime {
.bind_runtime_identity(EMBEDDED_RUNTIME_ID)
.expect("fresh embedded Runtime must accept its Backend-owned identity");
Self {
runtime_id: EMBEDDED_RUNTIME_ID.to_string(),
host_id: host_id_for_embedded_workspace(&workspace_id),
workspace_id,
runtime_id: EMBEDDED_RUNTIME_ID.to_string(),
runtime,
execution_enabled: false,
resource_broker: BackendResourceBroker::default(),
@@ -2106,6 +2168,30 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
}
}
fn worker_retention_inventory(
&self,
worker_id: &str,
) -> Result<WorkerRetentionInventory, String> {
let worker_ref = self
.worker_ref(worker_id)
.ok_or_else(|| format!("invalid embedded Worker id '{worker_id}'"))?;
self.runtime
.worker_retention_inventory(&self.workspace_id, &worker_ref)
.map_err(|error| error.to_string())
}
fn execute_worker_retention(
&self,
request: WorkerRetentionExecutionRequest,
) -> Result<WorkerRetentionExecutionResult, String> {
if request.workspace_id != self.workspace_id {
return Err("retention request Workspace does not match embedded Runtime".to_string());
}
self.runtime
.execute_worker_retention(&request)
.map_err(|error| error.to_string())
}
fn observation_source(
&self,
worker_id: &str,
@@ -3128,6 +3214,28 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
}
}
fn worker_retention_inventory(
&self,
worker_id: &str,
) -> Result<WorkerRetentionInventory, String> {
self.get_json::<WorkerRetentionInventory>(&format!(
"/v1/workers/{worker_id}/retention/inventory"
))
.map_err(|diagnostic| diagnostic.message)
}
fn execute_worker_retention(
&self,
request: WorkerRetentionExecutionRequest,
) -> Result<WorkerRetentionExecutionResult, String> {
let worker_id = request.worker_id.to_string();
self.post_json::<_, WorkerRetentionExecutionResult>(
&format!("/v1/workers/{worker_id}/retention/execute"),
&request,
)
.map_err(|diagnostic| diagnostic.message)
}
fn observation_source(
&self,
worker_id: &str,
crates/workspace-server/src/retention.rs
+185 -1
View File
@@ -377,6 +377,7 @@ impl SqliteWorkspaceStore {
workspace_id: plan.workspace_id.clone(),
source_runtime_id: plan.worker.runtime_id.clone(),
worker_id: worker_runtime::identity::WorkerId::new(worker_number),
expected_worker_revision: plan.worker_revision.clone(),
expected_run_generation: plan.run_generation,
source_created_at: worker.created_at,
removed_at,
@@ -391,6 +392,85 @@ impl SqliteWorkspaceStore {
})
}
pub fn recover_worker_removal_execution(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
expected_worker_revision: &str,
reason: &str,
) -> Result<Option<PreparedWorkerRemoval>, WorkerRetentionError> {
bounded("workspace", workspace_id, 160)?;
bounded("revision", expected_worker_revision, 256)?;
bounded("reason", reason, 512)?;
let plan = self.with_conn(|conn| {
conn.query_row(
"SELECT plan_id FROM worker_removal_operations
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
AND worker_revision=?4 AND reason=?5
AND state IN ('executing','failed','succeeded')
ORDER BY CASE state WHEN 'succeeded' THEN 0 ELSE 1 END,
created_at DESC LIMIT 1",
params![
workspace_id,
worker.runtime_id,
worker.worker_id,
expected_worker_revision,
reason,
],
|row| row.get::<_, String>(0),
)
.optional()
.map_err(StoreError::from)
.and_then(|plan_id| match plan_id {
Some(plan_id) => load_plan(conn, &plan_id),
None => Ok(None),
})
})?;
let Some(plan) = plan else {
return Ok(None);
};
let worker_number = plan.worker.worker_id.parse::<u64>().map_err(|_| {
WorkerRetentionError::Invalid(
"Runtime Worker id is not a canonical unsigned integer".to_string(),
)
})?;
let worker = if plan.state == WorkerRemovalPlanState::Succeeded {
None
} else {
Some(
self.with_conn(|conn| load_worker(conn, workspace_id, &plan.worker))?
.ok_or(WorkerRetentionError::WorkerNotFound)?,
)
};
Ok(Some(PreparedWorkerRemoval {
runtime_request: WorkerRetentionExecutionRequest {
operation_id: plan.operation_id.clone(),
input_fingerprint: plan.input_fingerprint.clone(),
archive_id: plan.archive_id.clone(),
workspace_id: plan.workspace_id.clone(),
source_runtime_id: plan.worker.runtime_id.clone(),
worker_id: worker_runtime::identity::WorkerId::new(worker_number),
expected_worker_revision: plan.worker_revision.clone(),
expected_run_generation: plan.run_generation,
source_created_at: worker
.as_ref()
.map(|worker| worker.created_at.clone())
.unwrap_or_else(|| plan.created_at.clone()),
removed_at: plan.created_at.clone(),
effective_profile: worker
.as_ref()
.map(|worker| worker.profile.clone())
.unwrap_or_else(|| Some("removed".to_string())),
retention_class: None,
policy_id: plan.policy_id.clone(),
policy_revision: plan.policy_revision,
session_disposition: plan.session_disposition,
diagnostics_disposition: plan.diagnostics_disposition,
},
plan,
}))
}
pub fn fail_worker_removal(
&self,
workspace_id: &str,
@@ -424,7 +504,8 @@ impl SqliteWorkspaceStore {
if plan.state != WorkerRemovalPlanState::Executing {
return Err(StoreError::InvalidInput(format!("stale:{}:plan state {} is not committable", plan.plan_id, state_s(plan.state))));
}
if result.worker_id.to_string() != plan.worker.worker_id
if result.expected_worker_revision != plan.worker_revision
|| result.worker_id.to_string() != plan.worker.worker_id
|| result.session_disposition != plan.session_disposition
|| result.diagnostics_disposition != plan.diagnostics_disposition
{
@@ -1107,6 +1188,7 @@ mod tests {
let result = WorkerRetentionExecutionResult {
operation_id: p.operation_id.clone(),
input_fingerprint: p.input_fingerprint.clone(),
expected_worker_revision: p.worker_revision.clone(),
worker_id: WorkerId::new(1),
session_disposition: p.session_disposition,
diagnostics_disposition: p.diagnostics_disposition,
@@ -1247,6 +1329,7 @@ mod tests {
let r = WorkerRetentionExecutionResult {
operation_id: p.operation_id.clone(),
input_fingerprint: p.input_fingerprint.clone(),
expected_worker_revision: p.worker_revision.clone(),
worker_id: WorkerId::new(1),
session_disposition: SessionDisposition::Purge,
diagnostics_disposition: DiagnosticsDisposition::Purge,
@@ -1265,6 +1348,7 @@ mod tests {
let mut result = WorkerRetentionExecutionResult {
operation_id: plan.operation_id.clone(),
input_fingerprint: plan.input_fingerprint.clone(),
expected_worker_revision: plan.worker_revision.clone(),
worker_id: WorkerId::new(1),
session_disposition: plan.session_disposition,
diagnostics_disposition: plan.diagnostics_disposition,
@@ -1348,6 +1432,106 @@ mod tests {
);
}
#[test]
fn runtime_result_must_match_prepared_worker_revision() {
let s = setup();
let plan = s.plan_worker_removal(&req(), &inv()).unwrap();
let prepared = s
.prepare_worker_removal_execution("w", &plan.plan_id, &plan.input_fingerprint)
.unwrap();
let mut runtime_result = WorkerRetentionExecutionResult {
operation_id: prepared.plan.operation_id.clone(),
input_fingerprint: prepared.plan.input_fingerprint.clone(),
expected_worker_revision: prepared.plan.worker_revision.clone(),
worker_id: WorkerId::new(1),
session_disposition: prepared.plan.session_disposition,
diagnostics_disposition: prepared.plan.diagnostics_disposition,
archive: None,
source_removed: true,
diagnostics_retained: false,
};
runtime_result.expected_worker_revision = "stale-revision".to_string();
let error = s
.commit_worker_removal(
"w",
&plan.operation_id,
&plan.input_fingerprint,
&runtime_result,
)
.unwrap_err();
assert!(
!error.to_string().is_empty(),
"mismatched Runtime revision must be rejected"
);
}
#[test]
fn succeeded_worker_removal_recovers_after_registry_purge() {
let s = setup();
let request = req();
let plan = s.plan_worker_removal(&request, &inv()).unwrap();
let prepared = s
.prepare_worker_removal_execution("w", &plan.plan_id, &plan.input_fingerprint)
.unwrap();
let runtime_result = WorkerRetentionExecutionResult {
operation_id: prepared.plan.operation_id.clone(),
input_fingerprint: prepared.plan.input_fingerprint.clone(),
expected_worker_revision: prepared.plan.worker_revision.clone(),
worker_id: WorkerId::new(1),
session_disposition: prepared.plan.session_disposition,
diagnostics_disposition: prepared.plan.diagnostics_disposition,
archive: Some(worker_runtime::retention::WorkerSessionArchiveManifest {
schema_version: 1,
archive_id: prepared.plan.archive_id.clone().unwrap(),
workspace_id: "w".into(),
source_runtime_id: "r".into(),
source_worker_id: WorkerId::new(1),
source_session_id: "s".into(),
segment_ids: vec!["a".into()],
source_created_at: "created".into(),
removed_at: "removed".into(),
archived_at_unix_seconds: 1,
effective_profile: None,
retention_class: None,
content_checksum_sha256: "sum".into(),
content_bytes: 1,
content_file_count: 1,
policy_id: prepared.plan.policy_id.clone(),
policy_revision: prepared.plan.policy_revision,
operation_id: prepared.plan.operation_id.clone(),
input_fingerprint: prepared.plan.input_fingerprint.clone(),
}),
source_removed: true,
diagnostics_retained: false,
};
s.commit_worker_removal(
"w",
&plan.operation_id,
&plan.input_fingerprint,
&runtime_result,
)
.unwrap();
assert!(
s.get_worker_registry("w", &request.worker)
.unwrap()
.is_none()
);
let recovered = s
.recover_worker_removal_execution(
"w",
&request.worker,
&request.expected_worker_revision,
&request.reason,
)
.unwrap()
.unwrap();
assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Succeeded);
assert_eq!(
recovered.runtime_request.expected_worker_revision,
request.expected_worker_revision
);
}
#[test]
fn old_schema_upgrade_seeds_existing_workspace() {
let temp = tempfile::tempdir().unwrap();
crates/workspace-server/src/server.rs
+691 -34
View File
@@ -1,7 +1,7 @@
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::{Component, Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::sync::{Arc, Mutex, Weak};
use axum::extract::ws::{Message as WsMessage, WebSocket, WebSocketUpgrade};
use axum::extract::{Path as AxumPath, Query, Request, State};
@@ -262,21 +262,328 @@ pub struct WorkspaceApi {
resource_broker: BackendResourceBroker,
workdir_sessions: Arc<Mutex<HashMap<RuntimeWorkerRef, WorkdirSessionHandle>>>,
workdir_session_locks: Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<tokio::sync::Mutex<()>>>>>,
worker_remove_locks: Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<tokio::sync::Mutex<()>>>>>,
}
#[derive(Clone)]
struct WorkspaceWorkerRemoveExecutor {
workspace_id: String,
store: Arc<dyn ControlPlaneStore>,
runtime: Weak<RuntimeRegistry>,
workdir_sessions: Arc<Mutex<HashMap<RuntimeWorkerRef, WorkdirSessionHandle>>>,
workdir_session_locks: Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<tokio::sync::Mutex<()>>>>>,
worker_remove_locks: Arc<Mutex<HashMap<RuntimeWorkerRef, Arc<tokio::sync::Mutex<()>>>>>,
}
impl WorkspaceWorkerRemoveExecutor {
fn new(api: &WorkspaceApi) -> Self {
Self {
workspace_id: api.config.workspace_id.clone(),
store: api.store.clone(),
runtime: Arc::downgrade(&api.runtime),
workdir_sessions: api.workdir_sessions.clone(),
workdir_session_locks: api.workdir_session_locks.clone(),
worker_remove_locks: api.worker_remove_locks.clone(),
}
}
async fn resume_worker_retention(
&self,
runtime: &RuntimeRegistry,
target: &RuntimeWorkerRef,
prepared: crate::retention::PreparedWorkerRemoval,
) -> std::result::Result<worker::WorkspaceResponse, String> {
let result =
match runtime.execute_worker_retention(target, prepared.runtime_request.clone()) {
Ok(result) => result,
Err(_) => {
return Ok(worker_remove_error_response(
StatusCode::SERVICE_UNAVAILABLE,
"runtime_retention_failed",
"Runtime retention recovery failed; removal can be retried",
));
}
};
match self.store.commit_worker_removal(
&self.workspace_id,
&prepared.plan.operation_id,
&prepared.plan.input_fingerprint,
&result,
) {
Ok(_) => Ok(worker_remove_success_response(target)),
Err(error) => Ok(worker_retention_error_response(error)),
}
}
async fn execute_async(
&self,
source: crate::worker_source::VerifiedWorkerMutationSource,
target_runtime_id: &str,
target_worker_id: &str,
expected_worker_revision: &str,
reason: &str,
) -> std::result::Result<worker::WorkspaceResponse, String> {
let reason = reason.trim();
if reason.is_empty() || reason.len() > 512 {
return Ok(worker_remove_error_response(
StatusCode::BAD_REQUEST,
"invalid_reason",
"WorkerRemove reason must be between 1 and 512 bytes",
));
}
let runtime = self.runtime.upgrade().ok_or_else(|| {
"Workspace Runtime registry is unavailable during WorkerRemove".to_string()
})?;
let source_is_current_orchestrator =
runtime.list_workers(1_000).items.into_iter().any(|worker| {
worker.worker.runtime_id == source.runtime_id
&& worker.worker.worker_id == source.worker_id
&& worker.singleton_key.as_deref()
== Some(crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY)
});
if !source_is_current_orchestrator {
return Ok(worker_remove_error_response(
StatusCode::FORBIDDEN,
"orchestrator_required",
"WorkerRemove is restricted to the current Workspace Orchestrator",
));
}
if source.runtime_id == target_runtime_id && source.worker_id == target_worker_id {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"self_removal_forbidden",
"The current Orchestrator cannot remove itself",
));
}
let target = RuntimeWorkerRef {
runtime_id: target_runtime_id.to_string(),
worker_id: target_worker_id.to_string(),
};
let remove_lock = {
let mut locks = self
.worker_remove_locks
.lock()
.map_err(|_| "WorkerRemove lock registry was poisoned".to_string())?;
locks
.entry(target.clone())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
};
let _remove_guard = remove_lock.lock().await;
let workdir_session_lock = {
let mut locks = self
.workdir_session_locks
.lock()
.map_err(|_| "Workdir session lock registry was poisoned".to_string())?;
locks
.entry(target.clone())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
};
let _workdir_session_guard = workdir_session_lock.lock().await;
let prepared = self
.store
.recover_worker_removal_execution(
&self.workspace_id,
&target,
expected_worker_revision,
reason,
)
.map_err(|_| "Worker removal recovery authority is unavailable".to_string())?;
if let Some(prepared) = prepared {
if prepared.plan.state == crate::retention::WorkerRemovalPlanState::Succeeded {
return Ok(worker_remove_success_response(&target));
}
return self
.resume_worker_retention(&runtime, &target, prepared)
.await;
}
let worker = match runtime.worker(&target) {
Ok(worker) => worker,
Err(_) => {
return Ok(worker_remove_error_response(
StatusCode::NOT_FOUND,
"worker_not_found",
"Worker was not found in this Workspace",
));
}
};
if worker.singleton_key.is_some() {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"internal_worker_forbidden",
"Internal service Workers cannot be removed with WorkerRemove",
));
}
if !worker.state.eq_ignore_ascii_case("stopped") {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"worker_not_stopped",
"Worker must be stopped before removal",
));
}
let inventory = match runtime.worker_retention_inventory(&target) {
Ok(inventory) => inventory,
Err(_) => {
return Ok(worker_remove_error_response(
StatusCode::SERVICE_UNAVAILABLE,
"retention_inventory_unavailable",
"Retention inventory could not be loaded; removal can be retried",
));
}
};
let request = crate::retention::WorkerRemovalPlanRequest {
workspace_id: self.workspace_id.clone(),
worker: target.clone(),
expected_worker_revision: expected_worker_revision.to_string(),
reason: reason.to_string(),
};
let plan = match self.store.plan_worker_removal(&request, &inventory) {
Ok(plan) => plan,
Err(error) => return Ok(worker_retention_error_response(error)),
};
let prepared = match self.store.prepare_worker_removal_execution(
&self.workspace_id,
&plan.plan_id,
&plan.input_fingerprint,
) {
Ok(prepared) => prepared,
Err(error) => return Ok(worker_retention_error_response(error)),
};
let session = self
.workdir_sessions
.lock()
.map_err(|_| "Workdir session registry was poisoned".to_string())?
.get(&target)
.cloned();
if let Some(session) = session {
if session.close().await.is_err() {
let _ = self.store.fail_worker_removal(
&self.workspace_id,
&plan.operation_id,
&plan.input_fingerprint,
"workdir_session_close_failed",
);
return Ok(worker_remove_error_response(
StatusCode::SERVICE_UNAVAILABLE,
"attachment_close_failed",
"Worker Workdir session could not be closed; removal can be retried",
));
}
self.workdir_sessions
.lock()
.map_err(|_| "Workdir session registry was poisoned".to_string())?
.remove(&target);
}
if let Err(_) = self.store.detach_worker_workdir(
&self.workspace_id,
&target,
None,
&Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true),
) {
let _ = self.store.fail_worker_removal(
&self.workspace_id,
&plan.operation_id,
&plan.input_fingerprint,
"workdir_attachment_release_failed",
);
return Ok(worker_remove_error_response(
StatusCode::SERVICE_UNAVAILABLE,
"attachment_release_failed",
"Worker Workdir attachment could not be released; removal can be retried",
));
}
let retention_result =
match runtime.execute_worker_retention(&target, prepared.runtime_request.clone()) {
Ok(result) => result,
Err(_) => {
let _ = self.store.fail_worker_removal(
&self.workspace_id,
&plan.operation_id,
&plan.input_fingerprint,
"runtime_retention_failed",
);
return Ok(worker_remove_error_response(
StatusCode::SERVICE_UNAVAILABLE,
"runtime_retention_failed",
"Runtime retention execution failed; removal can be retried",
));
}
};
match self.store.commit_worker_removal(
&self.workspace_id,
&plan.operation_id,
&plan.input_fingerprint,
&retention_result,
) {
Ok(_) => {}
Err(error) => {
let _ = self.store.fail_worker_removal(
&self.workspace_id,
&plan.operation_id,
&plan.input_fingerprint,
"metadata_commit_failed",
);
return Ok(worker_retention_error_response(error));
}
};
Ok(worker_remove_success_response(&target))
}
}
impl crate::worker_source::VerifiedWorkerRemoveExecutor for WorkspaceWorkerRemoveExecutor {
fn execute(
&self,
source: crate::worker_source::VerifiedWorkerMutationSource,
target_runtime_id: &str,
target_worker_id: &str,
expected_worker_revision: &str,
reason: &str,
) -> std::result::Result<worker::WorkspaceResponse, String> {
let executor = self.clone();
let target_runtime_id = target_runtime_id.to_string();
let target_worker_id = target_worker_id.to_string();
let expected_worker_revision = expected_worker_revision.to_string();
let reason = reason.to_string();
std::thread::spawn(move || {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|error| error.to_string())?
.block_on(executor.execute_async(
source,
&target_runtime_id,
&target_worker_id,
&expected_worker_revision,
&reason,
))
})
.join()
.map_err(|_| "embedded WorkerRemove executor thread panicked".to_string())?
}
}
impl WorkspaceApi {
pub async fn new(config: ServerConfig, store: Arc<dyn ControlPlaneStore>) -> Result<Self> {
let resource_broker = BackendResourceBroker::default();
let worker_remove_dispatcher = Arc::new(
crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
config.clone(),
store.clone(),
),
);
let execution_backend = WorkerRuntimeExecutionBackend::new(
ProfileRuntimeWorkerFactory::new(config.workspace_root.clone())
.with_embedded_worker_mutation_dispatcher(
EMBEDDED_RUNTIME_ID,
Arc::new(
crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
config.clone(),
store.clone(),
),
),
worker_remove_dispatcher.clone(),
)
.with_runtime_store_dir(config.embedded_runtime_store_root.clone())
.with_resource_client(Arc::new(resource_broker.clone())),
@@ -291,6 +598,7 @@ impl WorkspaceApi {
store,
Arc::new(execution_backend),
resource_broker,
Some(worker_remove_dispatcher),
)
.await
}
@@ -306,6 +614,7 @@ impl WorkspaceApi {
store,
execution_backend,
BackendResourceBroker::default(),
None,
)
.await
}
@@ -315,6 +624,9 @@ impl WorkspaceApi {
store: Arc<dyn ControlPlaneStore>,
execution_backend: Arc<dyn worker_runtime::execution::WorkerExecutionBackend>,
resource_broker: BackendResourceBroker,
worker_remove_dispatcher: Option<
Arc<crate::worker_source::EmbeddedServerWorkerMutationDispatcher>,
>,
) -> Result<Self> {
store
.upsert_workspace(&WorkspaceRecord {
@@ -376,7 +688,7 @@ impl WorkspaceApi {
let runtime = Arc::new(runtime);
let companion = Arc::new(CompanionConsole::disabled());
let observation_proxy = BackendObservationProxy::new(config.runtime_event_sources.clone());
Ok(Self {
let api = Self {
authority: SqliteWorkspaceAuthority::new(
config.database_path.clone(),
config.workspace_id.clone(),
@@ -392,7 +704,14 @@ impl WorkspaceApi {
resource_broker,
workdir_sessions: Arc::new(Mutex::new(HashMap::new())),
workdir_session_locks: Arc::new(Mutex::new(HashMap::new())),
})
worker_remove_locks: Arc::new(Mutex::new(HashMap::new())),
};
if let Some(dispatcher) = worker_remove_dispatcher {
dispatcher
.install_executor(Arc::new(WorkspaceWorkerRemoveExecutor::new(&api)))
.map_err(|message| Error::Config(message.to_string()))?;
}
Ok(api)
}
pub fn workspace_id(&self) -> &str {
@@ -4823,10 +5142,93 @@ async fn scoped_workspace_protocol_ws(
.into_response())
}
#[derive(Debug, Serialize)]
struct WorkerRemoveSuccessResponse<'a> {
removed: bool,
runtime_id: &'a str,
worker_id: &'a str,
}
#[derive(Debug, Serialize)]
struct WorkerRemoveErrorResponse<'a> {
code: &'a str,
message: &'a str,
}
fn worker_remove_success_response(worker: &RuntimeWorkerRef) -> worker::WorkspaceResponse {
let body = serde_json::to_string(&WorkerRemoveSuccessResponse {
removed: true,
runtime_id: &worker.runtime_id,
worker_id: &worker.worker_id,
})
.unwrap_or_else(|_| r#"{"removed":true}"#.to_string());
worker::WorkspaceResponse {
status: StatusCode::OK.as_u16(),
body,
}
}
fn worker_remove_error_response(
status: StatusCode,
code: &str,
message: &str,
) -> worker::WorkspaceResponse {
let body =
serde_json::to_string(&WorkerRemoveErrorResponse { code, message }).unwrap_or_else(|_| {
r#"{"code":"worker_remove_failed","message":"Worker removal failed"}"#.to_string()
});
worker::WorkspaceResponse {
status: status.as_u16(),
body,
}
}
fn worker_retention_error_response(
error: crate::retention::WorkerRetentionError,
) -> worker::WorkspaceResponse {
match error {
crate::retention::WorkerRetentionError::WorkerNotFound
| crate::retention::WorkerRetentionError::CrossWorkspace => worker_remove_error_response(
StatusCode::NOT_FOUND,
"worker_not_found",
"Worker was not found in this Workspace",
),
crate::retention::WorkerRetentionError::WorkerRevisionConflict { .. }
| crate::retention::WorkerRetentionError::PolicyRevisionConflict { .. }
| crate::retention::WorkerRetentionError::StalePlan { .. }
| crate::retention::WorkerRetentionError::OperationFingerprintConflict { .. } => {
worker_remove_error_response(
StatusCode::CONFLICT,
"worker_revision_conflict",
"Worker removal state changed; reread the Worker and retry",
)
}
crate::retention::WorkerRetentionError::Blocked(_) => worker_remove_error_response(
StatusCode::CONFLICT,
"worker_removal_blocked",
"Worker removal is blocked by current assignment, hold, or retention policy",
),
crate::retention::WorkerRetentionError::Invalid(_) => worker_remove_error_response(
StatusCode::BAD_REQUEST,
"invalid_worker_remove",
"Worker removal request is invalid",
),
crate::retention::WorkerRetentionError::PolicyMissing { .. }
| crate::retention::WorkerRetentionError::Store(_) => worker_remove_error_response(
StatusCode::SERVICE_UNAVAILABLE,
"worker_removal_authority_unavailable",
"Worker removal authority is unavailable; removal can be retried",
),
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct WorkerRemoveBoundaryRequest {
target_runtime_id: String,
target_worker_id: String,
expected_worker_revision: String,
reason: String,
}
async fn scoped_worker_remove_source_boundary(
@@ -4856,19 +5258,41 @@ async fn scoped_worker_remove_source_boundary(
)
.await
{
Ok(source) => (
StatusCode::NOT_IMPLEMENTED,
Json(serde_json::json!({
"error": "WorkerRemove lifecycle is deferred to its consumer Ticket",
"source": {
"runtime_id": source.runtime_id,
"worker_id": source.worker_id,
"actor_kind": source.actor_kind,
"permission": source.permission,
Ok(source) => {
let executor = WorkspaceWorkerRemoveExecutor::new(&api);
match executor
.execute_async(
source,
&request.target_runtime_id,
&request.target_worker_id,
&request.expected_worker_revision,
&request.reason,
)
.await
{
Ok(response) => (
StatusCode::from_u16(response.status)
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
[(CONTENT_TYPE, "application/json")],
response.body,
)
.into_response(),
Err(_) => {
let response = worker_remove_error_response(
StatusCode::INTERNAL_SERVER_ERROR,
"worker_remove_failed",
"Worker removal failed before lifecycle execution",
);
(
StatusCode::from_u16(response.status)
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
[(CONTENT_TYPE, "application/json")],
response.body,
)
.into_response()
}
})),
)
.into_response(),
}
}
Err(error) => {
let status = match error {
crate::worker_source::WorkerMutationSourceProofError::Replay => {
@@ -6009,7 +6433,7 @@ async fn scoped_check_runtime_config_bundle(
async fn scoped_get_runtime_worker(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
) -> ApiResult<Json<WorkerSummary>> {
) -> ApiResult<Json<WorkerShowProjection>> {
validate_workspace_scope(&api, &path.workspace_id)?;
get_runtime_worker(
State(api),
@@ -7715,10 +8139,17 @@ async fn post_companion_cancel(
Ok(Json(api.companion.cancel(request)))
}
#[derive(Debug, Serialize)]
struct WorkerShowProjection {
#[serde(flatten)]
worker: WorkerSummary,
updated_at: String,
}
async fn get_runtime_worker(
State(api): State<WorkspaceApi>,
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
) -> ApiResult<Json<WorkerSummary>> {
) -> ApiResult<Json<WorkerShowProjection>> {
let worker_ref = RuntimeWorkerRef::new(runtime_id, worker_id);
let worker = api
.runtime
@@ -7731,12 +8162,11 @@ async fn get_runtime_worker(
let workdirs = api
.store
.list_workdir_registry(&api.config.workspace_id, 500)?;
Ok(Json(merge_worker_registry_projection(
Some(&worker),
&record,
links,
&workdirs,
)))
let updated_at = record.updated_at.clone();
Ok(Json(WorkerShowProjection {
worker: merge_worker_registry_projection(Some(&worker), &record, links, &workdirs),
updated_at,
}))
}
async fn restore_runtime_worker(
@@ -13242,7 +13672,7 @@ mod tests {
let temp = tempfile::tempdir().unwrap();
let app = build_router(test_api(temp.path()).await);
let body = r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker"}"#;
let body = r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker","expected_worker_revision":"revision-1","reason":"retire target Worker"}"#;
let browser = app
.clone()
.oneshot(
@@ -13257,6 +13687,7 @@ mod tests {
.unwrap();
assert_eq!(browser.status(), StatusCode::UNAUTHORIZED);
let legacy = app
.clone()
.oneshot(
Request::builder()
.method("POST")
@@ -13270,6 +13701,20 @@ mod tests {
.await
.unwrap();
assert_eq!(legacy.status(), StatusCode::UNAUTHORIZED);
let body_spoof = app
.oneshot(
Request::builder()
.method("POST")
.uri(format!("/api/w/{TEST_WORKSPACE_ID}/workers/remove"))
.header(CONTENT_TYPE, "application/json")
.body(Body::from(
r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker","expected_worker_revision":"revision-1","reason":"retire target Worker","source_proof":"browser-controlled","actor":"orchestrator","policy":"purge"}"#,
))
.unwrap(),
)
.await
.unwrap();
assert_eq!(body_spoof.status(), StatusCode::UNPROCESSABLE_ENTITY);
}
#[tokio::test]
@@ -13338,15 +13783,220 @@ mod tests {
api.config.clone(),
api.store.clone(),
);
let response =
let error =
worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher::execute_worker_remove(
&dispatcher,
fresh_proof,
"runtime-target",
"target-worker",
"revision-1",
"retire target Worker",
)
.unwrap_err();
assert!(error.to_string().contains("executor is unavailable"));
}
#[tokio::test]
async fn worker_remove_rejects_self_running_and_stale_revision_at_caller_boundary() {
let temp = tempfile::tempdir().unwrap();
let api = test_api(temp.path()).await;
let Json(orchestrator) = scoped_start_workspace_orchestrator(
State(api.clone()),
AxumPath(ScopedWorkspacePath {
workspace_id: TEST_WORKSPACE_ID.to_string(),
}),
)
.await
.unwrap();
let source = orchestrator.worker.unwrap().worker;
let verified_source = || crate::worker_source::VerifiedWorkerMutationSource {
runtime_id: source.runtime_id.clone(),
worker_id: source.worker_id.clone(),
actor_kind: worker_runtime::auth::WorkerMutationActorKind::Worker,
permission: worker_runtime::auth::WORKER_REMOVE_PERMISSION.to_string(),
jti: "caller-guard-proof".to_string(),
};
let executor = WorkspaceWorkerRemoveExecutor::new(&api);
let self_response = executor
.execute_async(
verified_source(),
&source.runtime_id,
&source.worker_id,
"irrelevant",
"must reject self",
)
.await
.unwrap();
assert_eq!(self_response.status, StatusCode::CONFLICT.as_u16());
assert!(self_response.body.contains("self_removal_forbidden"));
let spawned = api
.spawn_workspace_worker(
EMBEDDED_WORKER_RUNTIME_ID,
WorkerSpawnRequest {
intent: WorkerSpawnIntent::WorkspaceCompanion,
requested_worker_name: Some("guard-target".to_string()),
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
expected_segments: 0,
},
profile: worker_runtime::catalog::ProfileSelector::Builtin(
"builtin:companion".to_string(),
),
ticket_assignment: None,
initial_submit: Vec::new(),
working_directory_request: None,
resolved_working_directory_request: None,
resolved_working_directory: None,
resolved_config_bundle: Some(runtime_test_bundle()),
resolved_worker_observation_enabled: false,
resolved_worker_observation_grants: Vec::new(),
resolved_workspace_api: None,
},
)
.unwrap();
assert_eq!(response.status, 501);
let target = spawned.worker.unwrap().worker;
let running_response = executor
.execute_async(
verified_source(),
&target.runtime_id,
&target.worker_id,
"irrelevant",
"must reject a live Worker",
)
.await
.unwrap();
assert_eq!(running_response.status, StatusCode::CONFLICT.as_u16());
assert!(running_response.body.contains("worker_not_stopped"));
api.runtime
.stop_worker(
&target,
WorkerLifecycleRequest {
reason: Some("prepare stale revision guard".to_string()),
ticket_assignment: None,
},
)
.unwrap();
let summary = api.runtime.worker(&target).unwrap();
let record = sync_worker_observation(&api, &summary).unwrap();
let stale_response = executor
.execute_async(
verified_source(),
&target.runtime_id,
&target.worker_id,
&format!("{}-stale", record.updated_at),
"must reject stale revision",
)
.await
.unwrap();
assert_eq!(stale_response.status, StatusCode::CONFLICT.as_u16());
assert!(stale_response.body.contains("worker_revision_conflict"));
}
#[tokio::test]
async fn embedded_worker_remove_executes_retention_and_returns_bounded_result() {
let temp = tempfile::tempdir().unwrap();
let api = test_api(temp.path()).await;
let Json(orchestrator) = scoped_start_workspace_orchestrator(
State(api.clone()),
AxumPath(ScopedWorkspacePath {
workspace_id: TEST_WORKSPACE_ID.to_string(),
}),
)
.await
.unwrap();
let source = orchestrator.worker.unwrap().worker;
let spawned = api
.spawn_workspace_worker(
EMBEDDED_WORKER_RUNTIME_ID,
WorkerSpawnRequest {
intent: WorkerSpawnIntent::WorkspaceCompanion,
requested_worker_name: Some("remove-target".to_string()),
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
expected_segments: 0,
},
profile: worker_runtime::catalog::ProfileSelector::Builtin(
"builtin:companion".to_string(),
),
ticket_assignment: None,
initial_submit: Vec::new(),
working_directory_request: None,
resolved_working_directory_request: None,
resolved_working_directory: None,
resolved_config_bundle: Some(runtime_test_bundle()),
resolved_worker_observation_enabled: false,
resolved_worker_observation_grants: Vec::new(),
resolved_workspace_api: None,
},
)
.unwrap();
let target = spawned.worker.unwrap().worker;
let stopped = api
.runtime
.stop_worker(
&target,
WorkerLifecycleRequest {
reason: Some("prepare WorkerRemove regression".to_string()),
ticket_assignment: None,
},
)
.unwrap();
assert_eq!(stopped.state, WorkerOperationState::Accepted);
let worker_root = temp
.path()
.join(".test-embedded-runtime-store/workers")
.join(&target.worker_id);
fs::create_dir_all(worker_root.join("session/segments")).unwrap();
fs::write(
worker_root.join("session/session.json"),
serde_json::to_vec_pretty(&json!({
"schema_version": 1,
"session_id": "worker-remove-session"
}))
.unwrap(),
)
.unwrap();
fs::write(
worker_root.join("session/segments/segment-a.jsonl"),
b"retained evidence\n",
)
.unwrap();
let summary = api.runtime.worker(&target).unwrap();
let record = sync_worker_observation(&api, &summary).unwrap();
let response = WorkspaceWorkerRemoveExecutor::new(&api)
.execute_async(
crate::worker_source::VerifiedWorkerMutationSource {
runtime_id: source.runtime_id,
worker_id: source.worker_id,
actor_kind: worker_runtime::auth::WorkerMutationActorKind::Worker,
permission: worker_runtime::auth::WORKER_REMOVE_PERMISSION.to_string(),
jti: "embedded-valid-proof".to_string(),
},
&target.runtime_id,
&target.worker_id,
&record.updated_at,
"retire completed Worker",
)
.await
.unwrap();
assert_eq!(
response.status,
StatusCode::OK.as_u16(),
"{}",
response.body
);
assert!(response.body.contains("\"removed\":true"));
assert!(!response.body.contains("disposition"));
assert!(!response.body.contains("stage"));
assert!(!response.body.contains("path"));
assert!(
api.store
.get_worker_registry(TEST_WORKSPACE_ID, &target)
.unwrap()
.is_none()
);
}
#[tokio::test]
@@ -13524,13 +14174,20 @@ mod tests {
route_token,
)
.body(Body::from(
r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker"}"#,
r#"{"target_runtime_id":"runtime-target","target_worker_id":"target-worker","expected_worker_revision":"revision-1","reason":"retire target Worker"}"#,
))
.unwrap(),
)
.await
.unwrap();
assert_eq!(route_response.status(), StatusCode::NOT_IMPLEMENTED);
assert_eq!(route_response.status(), StatusCode::FORBIDDEN);
let route_body = axum::body::to_bytes(route_response.into_body(), usize::MAX)
.await
.unwrap();
let route_body = String::from_utf8(route_body.to_vec()).unwrap();
assert!(route_body.contains("orchestrator_required"));
assert!(!route_body.contains("source"));
assert!(!route_body.contains("proof"));
let mut revoked = trust;
revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string());
crates/workspace-server/src/store.rs
+149
View File
@@ -474,6 +474,73 @@ pub trait ControlPlaneStore: Send + Sync {
now_seconds: u64,
consumed_at: &str,
) -> Result<bool>;
fn plan_worker_removal(
&self,
request: &crate::retention::WorkerRemovalPlanRequest,
inventory: &worker_runtime::retention::WorkerRetentionInventory,
) -> std::result::Result<
crate::retention::WorkerRemovalPlan,
crate::retention::WorkerRetentionError,
> {
let _ = (request, inventory);
Err(crate::retention::WorkerRetentionError::Invalid(
"Worker retention authority is unavailable".to_string(),
))
}
fn prepare_worker_removal_execution(
&self,
workspace_id: &str,
plan_id: &str,
input_fingerprint: &str,
) -> std::result::Result<
crate::retention::PreparedWorkerRemoval,
crate::retention::WorkerRetentionError,
> {
let _ = (workspace_id, plan_id, input_fingerprint);
Err(crate::retention::WorkerRetentionError::Invalid(
"Worker retention authority is unavailable".to_string(),
))
}
fn recover_worker_removal_execution(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
expected_worker_revision: &str,
reason: &str,
) -> std::result::Result<
Option<crate::retention::PreparedWorkerRemoval>,
crate::retention::WorkerRetentionError,
> {
let _ = (workspace_id, worker, expected_worker_revision, reason);
Ok(None)
}
fn fail_worker_removal(
&self,
workspace_id: &str,
operation_id: &str,
input_fingerprint: &str,
category: &str,
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
let _ = (workspace_id, operation_id, input_fingerprint, category);
Err(crate::retention::WorkerRetentionError::Invalid(
"Worker retention authority is unavailable".to_string(),
))
}
fn commit_worker_removal(
&self,
workspace_id: &str,
operation_id: &str,
input_fingerprint: &str,
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
) -> std::result::Result<
crate::retention::WorkerRemovalPlan,
crate::retention::WorkerRetentionError,
> {
let _ = (workspace_id, operation_id, input_fingerprint, result);
Err(crate::retention::WorkerRetentionError::Invalid(
"Worker retention authority is unavailable".to_string(),
))
}
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>>;
fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>;
fn get_repository(
@@ -948,6 +1015,88 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
})
}
fn plan_worker_removal(
&self,
request: &crate::retention::WorkerRemovalPlanRequest,
inventory: &worker_runtime::retention::WorkerRetentionInventory,
) -> std::result::Result<
crate::retention::WorkerRemovalPlan,
crate::retention::WorkerRetentionError,
> {
SqliteWorkspaceStore::plan_worker_removal(self, request, inventory)
}
fn prepare_worker_removal_execution(
&self,
workspace_id: &str,
plan_id: &str,
input_fingerprint: &str,
) -> std::result::Result<
crate::retention::PreparedWorkerRemoval,
crate::retention::WorkerRetentionError,
> {
SqliteWorkspaceStore::prepare_worker_removal_execution(
self,
workspace_id,
plan_id,
input_fingerprint,
)
}
fn recover_worker_removal_execution(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
expected_worker_revision: &str,
reason: &str,
) -> std::result::Result<
Option<crate::retention::PreparedWorkerRemoval>,
crate::retention::WorkerRetentionError,
> {
SqliteWorkspaceStore::recover_worker_removal_execution(
self,
workspace_id,
worker,
expected_worker_revision,
reason,
)
}
fn fail_worker_removal(
&self,
workspace_id: &str,
operation_id: &str,
input_fingerprint: &str,
category: &str,
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
SqliteWorkspaceStore::fail_worker_removal(
self,
workspace_id,
operation_id,
input_fingerprint,
category,
)
}
fn commit_worker_removal(
&self,
workspace_id: &str,
operation_id: &str,
input_fingerprint: &str,
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
) -> std::result::Result<
crate::retention::WorkerRemovalPlan,
crate::retention::WorkerRetentionError,
> {
SqliteWorkspaceStore::commit_worker_removal(
self,
workspace_id,
operation_id,
input_fingerprint,
result,
)
}
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>> {
self.with_conn(|conn| {
let mut stmt = conn.prepare(
crates/workspace-server/src/worker_source.rs
+46 -9
View File
@@ -1,3 +1,4 @@
use std::sync::{Arc, OnceLock};
use std::time::{SystemTime, UNIX_EPOCH};
use axum::http::HeaderMap;
@@ -168,18 +169,43 @@ async fn verify_worker_remove_source_with(
})
}
pub(crate) trait VerifiedWorkerRemoveExecutor: Send + Sync {
fn execute(
&self,
source: VerifiedWorkerMutationSource,
target_runtime_id: &str,
target_worker_id: &str,
expected_worker_revision: &str,
reason: &str,
) -> Result<worker::WorkspaceResponse, String>;
}
#[derive(Clone)]
pub(crate) struct EmbeddedServerWorkerMutationDispatcher {
config: crate::server::ServerConfig,
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
store: Arc<dyn crate::store::ControlPlaneStore>,
executor: Arc<OnceLock<Arc<dyn VerifiedWorkerRemoveExecutor>>>,
}
impl EmbeddedServerWorkerMutationDispatcher {
pub(crate) fn new(
config: crate::server::ServerConfig,
store: std::sync::Arc<dyn crate::store::ControlPlaneStore>,
store: Arc<dyn crate::store::ControlPlaneStore>,
) -> Self {
Self { config, store }
Self {
config,
store,
executor: Arc::new(OnceLock::new()),
}
}
pub(crate) fn install_executor(
&self,
executor: Arc<dyn VerifiedWorkerRemoveExecutor>,
) -> Result<(), &'static str> {
self.executor
.set(executor)
.map_err(|_| "WorkerRemove executor is already installed")
}
}
@@ -191,11 +217,13 @@ impl worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher
proof: InProcessWorkerMutationProof,
target_runtime_id: &str,
target_worker_id: &str,
expected_worker_revision: &str,
reason: &str,
) -> Result<
worker::WorkspaceResponse,
worker_runtime::worker_source::RuntimeWorkerMutationForwardError,
> {
futures::executor::block_on(verify_worker_remove_source_with(
let source = futures::executor::block_on(verify_worker_remove_source_with(
&self.config,
&self.store,
PresentedWorkerMutationSourceProof::InProcess(proof),
@@ -207,11 +235,20 @@ impl worker_runtime::worker_source::EmbeddedWorkerMutationDispatcher
error.to_string(),
)
})?;
Ok(worker::WorkspaceResponse {
status: 501,
body: "WorkerRemove lifecycle is not implemented by this operation boundary"
.to_string(),
})
let executor = self.executor.get().ok_or_else(|| {
worker_runtime::worker_source::RuntimeWorkerMutationForwardError::Embedded(
"WorkerRemove executor is unavailable".to_string(),
)
})?;
executor
.execute(
source,
target_runtime_id,
target_worker_id,
expected_worker_revision,
reason,
)
.map_err(worker_runtime::worker_source::RuntimeWorkerMutationForwardError::Embedded)
}
}