runtime: filter stopped workers

This commit is contained in:
2026-07-29 04:27:04 +09:00
parent 98c1599d1a
commit acc7281414
6 changed files with 387 additions and 19 deletions
+134
View File
@@ -253,6 +253,14 @@ pub struct WorkerSummary {
pub diagnostics: Vec<RuntimeDiagnostic>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkerRestoreResult {
pub state: WorkerOperationState,
#[serde(skip_serializing_if = "Option::is_none")]
pub worker: Option<WorkerSummary>,
pub diagnostics: Vec<RuntimeDiagnostic>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RuntimeList<T> {
pub items: Vec<T>,
@@ -576,8 +584,24 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
fn list_workers(&self, limit: usize) -> RuntimeList<WorkerSummary>;
fn list_stopped_workers(&self, _limit: usize) -> RuntimeList<WorkerSummary> {
RuntimeList::new(Vec::new(), Vec::new())
}
fn worker(&self, worker_id: &str) -> WorkerLookupResult;
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
WorkerRestoreResult {
state: WorkerOperationState::Unsupported,
worker: None,
diagnostics: vec![diagnostic(
"worker_restore_unsupported",
DiagnosticSeverity::Info,
format!("runtime does not implement worker restore for `{worker_id}`"),
)],
}
}
fn create_working_directory(
&self,
_request: WorkingDirectoryRequest,
@@ -955,6 +979,26 @@ impl RuntimeRegistry {
Ok(RuntimeList::new(items, diagnostics))
}
pub fn list_stopped_workers_for_runtime(
&self,
runtime_id: &str,
limit: usize,
) -> Result<RuntimeList<WorkerSummary>, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
let runtime = self.runtime(runtime_id)?;
let worker_list = runtime.list_stopped_workers(limit);
let mut items: Vec<_> = worker_list
.items
.into_iter()
.filter(|worker| !is_retired_companion_worker(worker))
.take(limit)
.collect();
items.truncate(limit);
let mut diagnostics = worker_list.diagnostics;
diagnostics.truncate(MAX_DIAGNOSTICS);
Ok(RuntimeList::new(items, diagnostics))
}
pub fn list_workers_for_host(
&self,
host_id: &str,
@@ -1015,6 +1059,17 @@ impl RuntimeRegistry {
Ok(worker)
}
pub fn restore_worker(
&self,
runtime_id: &str,
worker_id: &str,
) -> Result<WorkerRestoreResult, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
validate_backend_identifier("worker_id", worker_id)?;
let runtime = self.runtime(runtime_id)?;
Ok(runtime.restore_worker(worker_id))
}
pub fn spawn_worker(
&self,
runtime_id: &str,
@@ -1501,6 +1556,23 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
}
}
fn list_stopped_workers(&self, limit: usize) -> RuntimeList<WorkerSummary> {
if limit == 0 {
return RuntimeList::new(Vec::new(), Vec::new());
}
match self.runtime.list_stopped_workers() {
Ok(workers) => RuntimeList::new(
workers
.into_iter()
.take(limit)
.map(|worker| self.map_worker_summary(worker))
.collect(),
Vec::new(),
),
Err(err) => RuntimeList::new(Vec::new(), vec![embedded_runtime_diagnostic(&err)]),
}
}
fn worker(&self, worker_id: &str) -> WorkerLookupResult {
let Some(worker_ref) = self.worker_ref(worker_id) else {
return WorkerLookupResult {
@@ -1528,6 +1600,32 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
}
}
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
let Some(worker_ref) = self.worker_ref(worker_id) else {
return WorkerRestoreResult {
state: WorkerOperationState::Rejected,
worker: None,
diagnostics: vec![diagnostic(
"embedded_worker_id_invalid",
DiagnosticSeverity::Warning,
"Worker id was empty and cannot be restored".to_string(),
)],
};
};
match self.runtime.restore_worker(&worker_ref) {
Ok(detail) => WorkerRestoreResult {
state: WorkerOperationState::Accepted,
worker: Some(self.map_worker_detail(detail)),
diagnostics: Vec::new(),
},
Err(err) => WorkerRestoreResult {
state: WorkerOperationState::Rejected,
worker: None,
diagnostics: vec![embedded_runtime_diagnostic(&err)],
},
}
}
fn create_working_directory(
&self,
_request: WorkingDirectoryRequest,
@@ -2446,6 +2544,24 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
}
}
fn list_stopped_workers(&self, limit: usize) -> RuntimeList<WorkerSummary> {
if limit == 0 {
return RuntimeList::new(Vec::new(), Vec::new());
}
match self.get_json::<RuntimeHttpWorkersResponse>("/v1/workers?status=stopped") {
Ok(response) => RuntimeList::new(
response
.workers
.into_iter()
.take(limit)
.map(|worker| self.map_worker_summary(worker))
.collect(),
Vec::new(),
),
Err(diagnostic) => RuntimeList::new(Vec::new(), vec![diagnostic]),
}
}
fn worker(&self, worker_id: &str) -> WorkerLookupResult {
match self.get_json::<RuntimeHttpWorkerResponse>(&format!("/v1/workers/{worker_id}")) {
Ok(response) => WorkerLookupResult {
@@ -2463,6 +2579,24 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
}
}
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
match self.post_json::<_, RuntimeHttpWorkerResponse>(
&format!("/v1/workers/{worker_id}/restore"),
&serde_json::json!({}),
) {
Ok(response) => WorkerRestoreResult {
state: WorkerOperationState::Accepted,
worker: Some(self.map_worker_detail(response.worker)),
diagnostics: Vec::new(),
},
Err(diagnostic) => WorkerRestoreResult {
state: WorkerOperationState::Rejected,
worker: None,
diagnostics: vec![diagnostic],
},
}
}
fn create_working_directory(
&self,
request: WorkingDirectoryRequest,
+109 -10
View File
@@ -56,9 +56,9 @@ use crate::hosts::{
RuntimeRegistryError, RuntimeRegistryUnregisterResult, RuntimeSummary, WorkerCapabilitySummary,
WorkerCompletionsRequest, WorkerCompletionsResult, WorkerImplementationSummary,
WorkerInputKind, WorkerInputRequest, WorkerInputResult, WorkerLifecycleRequest,
WorkerLifecycleResult, WorkerOperationState, WorkerSpawnAcceptanceRequirement,
WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult, WorkerSpawnWorkingDirectoryRequest,
WorkerSummary, WorkerWorkspaceSummary,
WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult,
WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult,
WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerWorkspaceSummary,
};
use crate::identity::WorkspaceIdentity;
use crate::memory_backend::execute_memory_backend_operation_with_authority;
@@ -668,10 +668,18 @@ pub fn build_router(api: WorkspaceApi) -> Router {
"/api/runtimes/{runtime_id}/workers/{worker_id}",
get(get_runtime_worker),
)
.route(
"/api/runtimes/{runtime_id}/workers/{worker_id}/restore",
post(restore_runtime_worker),
)
.route(
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}",
get(scoped_get_runtime_worker),
)
.route(
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore",
post(scoped_restore_runtime_worker),
)
.route(
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/pin",
put(scoped_pin_runtime_worker).delete(scoped_unpin_runtime_worker),
@@ -786,6 +794,25 @@ pub struct RuntimeListResponse<T> {
pub diagnostics: Vec<RuntimeDiagnostic>,
}
#[derive(Debug, Default, Deserialize)]
struct RuntimeWorkersQuery {
status: Option<RuntimeWorkersStatusFilter>,
}
#[derive(Debug, Clone, Copy, Deserialize)]
#[serde(rename_all = "snake_case")]
enum RuntimeWorkersStatusFilter {
Stopped,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct WorkerRestoreResponse {
pub workspace_id: String,
pub runtime_id: String,
pub worker_id: String,
pub result: WorkerRestoreResult,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CleanupTargetKind {
@@ -2733,9 +2760,10 @@ async fn scoped_post_companion_cancel(
async fn scoped_list_runtime_workers(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedRuntimePath>,
Query(query): Query<RuntimeWorkersQuery>,
) -> ApiResult<Json<RuntimeListResponse<WorkerSummary>>> {
validate_workspace_scope(&api, &path.workspace_id)?;
list_runtime_workers(State(api), AxumPath(path.runtime_id)).await
list_runtime_workers(State(api), AxumPath(path.runtime_id), Query(query)).await
}
async fn scoped_create_runtime_worker(
@@ -2778,6 +2806,14 @@ async fn scoped_get_runtime_worker(
get_runtime_worker(State(api), AxumPath((path.runtime_id, path.worker_id))).await
}
async fn scoped_restore_runtime_worker(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
) -> ApiResult<Json<WorkerRestoreResponse>> {
validate_workspace_scope(&api, &path.workspace_id)?;
restore_runtime_worker(State(api), AxumPath((path.runtime_id, path.worker_id))).await
}
async fn scoped_pin_runtime_worker(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
@@ -4322,6 +4358,39 @@ async fn get_runtime_worker(
)))
}
async fn restore_runtime_worker(
State(api): State<WorkspaceApi>,
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
) -> ApiResult<Json<WorkerRestoreResponse>> {
let mut result = api
.runtime
.restore_worker(&runtime_id, &worker_id)
.map_err(|err| err.into_error())?;
if let Some(worker) = result.worker.as_ref() {
let record = sync_worker_observation(&api, worker)?;
let links = api.store.list_worker_workdir_links(
&api.config.workspace_id,
record.runtime_id.as_str(),
record.runtime_worker_id,
)?;
let workdirs = api
.store
.list_workdir_registry(&api.config.workspace_id, 500)?;
result.worker = Some(merge_worker_registry_projection(
Some(worker),
&record,
links,
&workdirs,
));
}
Ok(Json(WorkerRestoreResponse {
workspace_id: api.workspace_id().to_string(),
runtime_id,
worker_id,
result,
}))
}
#[derive(Debug, Serialize, Deserialize)]
pub struct RuntimeConfigBundleSyncRequest {
pub bundle: ConfigBundle,
@@ -4378,17 +4447,28 @@ fn reject_no_workdir_for_non_embedded_runtime(
async fn list_runtime_workers(
State(api): State<WorkspaceApi>,
AxumPath(runtime_id): AxumPath<String>,
Query(query): Query<RuntimeWorkersQuery>,
) -> ApiResult<Json<RuntimeListResponse<WorkerSummary>>> {
let limit = api.config.max_records.min(200);
let worker_list = api
.runtime
.list_workers_for_runtime(&runtime_id, limit)
.map_err(|err| err.into_error())?;
let (worker_list, source) = match query.status {
Some(RuntimeWorkersStatusFilter::Stopped) => (
api.runtime
.list_stopped_workers_for_runtime(&runtime_id, limit)
.map_err(|err| err.into_error())?,
"runtime_registry_stopped",
),
None => (
api.runtime
.list_workers_for_runtime(&runtime_id, limit)
.map_err(|err| err.into_error())?,
"runtime_registry",
),
};
Ok(Json(RuntimeListResponse {
workspace_id: api.workspace_id().to_string(),
limit,
items: worker_list.items,
source: "runtime_registry".to_string(),
source: source.to_string(),
diagnostics: worker_list.diagnostics,
}))
}
@@ -9322,6 +9402,22 @@ mod tests {
assert_eq!(worker["worker_id"], worker_id);
assert_eq!(worker["runtime_id"], "embedded-worker-runtime");
let stopped_workers = get_json(
app.clone(),
"/api/runtimes/embedded-worker-runtime/workers?status=stopped",
)
.await;
assert!(stopped_workers["items"].as_array().unwrap().is_empty());
let restored = post_json(
app.clone(),
&format!("/api/runtimes/embedded-worker-runtime/workers/{worker_id}/restore"),
json!({}),
)
.await;
assert_eq!(restored["result"]["state"], "accepted");
assert_eq!(restored["result"]["worker"]["worker_id"], worker_id);
let accepted = post_json(
app.clone(),
&format!("/api/runtimes/embedded-worker-runtime/workers/{worker_id}/input"),
@@ -9373,7 +9469,10 @@ mod tests {
.unwrap();
assert_eq!(wrong_runtime.status(), StatusCode::NOT_FOUND);
let projected = format!("{}{}{}{}", embedded_summary, spawned, worker, accepted);
let projected = format!(
"{}{}{}{}{}{}",
embedded_summary, spawned, worker, stopped_workers, restored, accepted
);
for forbidden in [
dir.path().to_string_lossy().as_ref(),
"metadata.json",