runtime: filter stopped workers
This commit is contained in:
@@ -253,6 +253,14 @@ pub struct WorkerSummary {
|
||||
pub diagnostics: Vec<RuntimeDiagnostic>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct WorkerRestoreResult {
|
||||
pub state: WorkerOperationState,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker: Option<WorkerSummary>,
|
||||
pub diagnostics: Vec<RuntimeDiagnostic>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct RuntimeList<T> {
|
||||
pub items: Vec<T>,
|
||||
@@ -576,8 +584,24 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
|
||||
fn list_workers(&self, limit: usize) -> RuntimeList<WorkerSummary>;
|
||||
|
||||
fn list_stopped_workers(&self, _limit: usize) -> RuntimeList<WorkerSummary> {
|
||||
RuntimeList::new(Vec::new(), Vec::new())
|
||||
}
|
||||
|
||||
fn worker(&self, worker_id: &str) -> WorkerLookupResult;
|
||||
|
||||
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
|
||||
WorkerRestoreResult {
|
||||
state: WorkerOperationState::Unsupported,
|
||||
worker: None,
|
||||
diagnostics: vec![diagnostic(
|
||||
"worker_restore_unsupported",
|
||||
DiagnosticSeverity::Info,
|
||||
format!("runtime does not implement worker restore for `{worker_id}`"),
|
||||
)],
|
||||
}
|
||||
}
|
||||
|
||||
fn create_working_directory(
|
||||
&self,
|
||||
_request: WorkingDirectoryRequest,
|
||||
@@ -955,6 +979,26 @@ impl RuntimeRegistry {
|
||||
Ok(RuntimeList::new(items, diagnostics))
|
||||
}
|
||||
|
||||
pub fn list_stopped_workers_for_runtime(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
limit: usize,
|
||||
) -> Result<RuntimeList<WorkerSummary>, RuntimeRegistryError> {
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
let runtime = self.runtime(runtime_id)?;
|
||||
let worker_list = runtime.list_stopped_workers(limit);
|
||||
let mut items: Vec<_> = worker_list
|
||||
.items
|
||||
.into_iter()
|
||||
.filter(|worker| !is_retired_companion_worker(worker))
|
||||
.take(limit)
|
||||
.collect();
|
||||
items.truncate(limit);
|
||||
let mut diagnostics = worker_list.diagnostics;
|
||||
diagnostics.truncate(MAX_DIAGNOSTICS);
|
||||
Ok(RuntimeList::new(items, diagnostics))
|
||||
}
|
||||
|
||||
pub fn list_workers_for_host(
|
||||
&self,
|
||||
host_id: &str,
|
||||
@@ -1015,6 +1059,17 @@ impl RuntimeRegistry {
|
||||
Ok(worker)
|
||||
}
|
||||
|
||||
pub fn restore_worker(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
worker_id: &str,
|
||||
) -> Result<WorkerRestoreResult, RuntimeRegistryError> {
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
validate_backend_identifier("worker_id", worker_id)?;
|
||||
let runtime = self.runtime(runtime_id)?;
|
||||
Ok(runtime.restore_worker(worker_id))
|
||||
}
|
||||
|
||||
pub fn spawn_worker(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -1501,6 +1556,23 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
fn list_stopped_workers(&self, limit: usize) -> RuntimeList<WorkerSummary> {
|
||||
if limit == 0 {
|
||||
return RuntimeList::new(Vec::new(), Vec::new());
|
||||
}
|
||||
match self.runtime.list_stopped_workers() {
|
||||
Ok(workers) => RuntimeList::new(
|
||||
workers
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|worker| self.map_worker_summary(worker))
|
||||
.collect(),
|
||||
Vec::new(),
|
||||
),
|
||||
Err(err) => RuntimeList::new(Vec::new(), vec![embedded_runtime_diagnostic(&err)]),
|
||||
}
|
||||
}
|
||||
|
||||
fn worker(&self, worker_id: &str) -> WorkerLookupResult {
|
||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||
return WorkerLookupResult {
|
||||
@@ -1528,6 +1600,32 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
|
||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||
return WorkerRestoreResult {
|
||||
state: WorkerOperationState::Rejected,
|
||||
worker: None,
|
||||
diagnostics: vec![diagnostic(
|
||||
"embedded_worker_id_invalid",
|
||||
DiagnosticSeverity::Warning,
|
||||
"Worker id was empty and cannot be restored".to_string(),
|
||||
)],
|
||||
};
|
||||
};
|
||||
match self.runtime.restore_worker(&worker_ref) {
|
||||
Ok(detail) => WorkerRestoreResult {
|
||||
state: WorkerOperationState::Accepted,
|
||||
worker: Some(self.map_worker_detail(detail)),
|
||||
diagnostics: Vec::new(),
|
||||
},
|
||||
Err(err) => WorkerRestoreResult {
|
||||
state: WorkerOperationState::Rejected,
|
||||
worker: None,
|
||||
diagnostics: vec![embedded_runtime_diagnostic(&err)],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn create_working_directory(
|
||||
&self,
|
||||
_request: WorkingDirectoryRequest,
|
||||
@@ -2446,6 +2544,24 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
fn list_stopped_workers(&self, limit: usize) -> RuntimeList<WorkerSummary> {
|
||||
if limit == 0 {
|
||||
return RuntimeList::new(Vec::new(), Vec::new());
|
||||
}
|
||||
match self.get_json::<RuntimeHttpWorkersResponse>("/v1/workers?status=stopped") {
|
||||
Ok(response) => RuntimeList::new(
|
||||
response
|
||||
.workers
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|worker| self.map_worker_summary(worker))
|
||||
.collect(),
|
||||
Vec::new(),
|
||||
),
|
||||
Err(diagnostic) => RuntimeList::new(Vec::new(), vec![diagnostic]),
|
||||
}
|
||||
}
|
||||
|
||||
fn worker(&self, worker_id: &str) -> WorkerLookupResult {
|
||||
match self.get_json::<RuntimeHttpWorkerResponse>(&format!("/v1/workers/{worker_id}")) {
|
||||
Ok(response) => WorkerLookupResult {
|
||||
@@ -2463,6 +2579,24 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
|
||||
match self.post_json::<_, RuntimeHttpWorkerResponse>(
|
||||
&format!("/v1/workers/{worker_id}/restore"),
|
||||
&serde_json::json!({}),
|
||||
) {
|
||||
Ok(response) => WorkerRestoreResult {
|
||||
state: WorkerOperationState::Accepted,
|
||||
worker: Some(self.map_worker_detail(response.worker)),
|
||||
diagnostics: Vec::new(),
|
||||
},
|
||||
Err(diagnostic) => WorkerRestoreResult {
|
||||
state: WorkerOperationState::Rejected,
|
||||
worker: None,
|
||||
diagnostics: vec![diagnostic],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn create_working_directory(
|
||||
&self,
|
||||
request: WorkingDirectoryRequest,
|
||||
|
||||
@@ -56,9 +56,9 @@ use crate::hosts::{
|
||||
RuntimeRegistryError, RuntimeRegistryUnregisterResult, RuntimeSummary, WorkerCapabilitySummary,
|
||||
WorkerCompletionsRequest, WorkerCompletionsResult, WorkerImplementationSummary,
|
||||
WorkerInputKind, WorkerInputRequest, WorkerInputResult, WorkerLifecycleRequest,
|
||||
WorkerLifecycleResult, WorkerOperationState, WorkerSpawnAcceptanceRequirement,
|
||||
WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult, WorkerSpawnWorkingDirectoryRequest,
|
||||
WorkerSummary, WorkerWorkspaceSummary,
|
||||
WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult,
|
||||
WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult,
|
||||
WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerWorkspaceSummary,
|
||||
};
|
||||
use crate::identity::WorkspaceIdentity;
|
||||
use crate::memory_backend::execute_memory_backend_operation_with_authority;
|
||||
@@ -668,10 +668,18 @@ pub fn build_router(api: WorkspaceApi) -> Router {
|
||||
"/api/runtimes/{runtime_id}/workers/{worker_id}",
|
||||
get(get_runtime_worker),
|
||||
)
|
||||
.route(
|
||||
"/api/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
||||
post(restore_runtime_worker),
|
||||
)
|
||||
.route(
|
||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}",
|
||||
get(scoped_get_runtime_worker),
|
||||
)
|
||||
.route(
|
||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
||||
post(scoped_restore_runtime_worker),
|
||||
)
|
||||
.route(
|
||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/pin",
|
||||
put(scoped_pin_runtime_worker).delete(scoped_unpin_runtime_worker),
|
||||
@@ -786,6 +794,25 @@ pub struct RuntimeListResponse<T> {
|
||||
pub diagnostics: Vec<RuntimeDiagnostic>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct RuntimeWorkersQuery {
|
||||
status: Option<RuntimeWorkersStatusFilter>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum RuntimeWorkersStatusFilter {
|
||||
Stopped,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct WorkerRestoreResponse {
|
||||
pub workspace_id: String,
|
||||
pub runtime_id: String,
|
||||
pub worker_id: String,
|
||||
pub result: WorkerRestoreResult,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CleanupTargetKind {
|
||||
@@ -2733,9 +2760,10 @@ async fn scoped_post_companion_cancel(
|
||||
async fn scoped_list_runtime_workers(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedRuntimePath>,
|
||||
Query(query): Query<RuntimeWorkersQuery>,
|
||||
) -> ApiResult<Json<RuntimeListResponse<WorkerSummary>>> {
|
||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
||||
list_runtime_workers(State(api), AxumPath(path.runtime_id)).await
|
||||
list_runtime_workers(State(api), AxumPath(path.runtime_id), Query(query)).await
|
||||
}
|
||||
|
||||
async fn scoped_create_runtime_worker(
|
||||
@@ -2778,6 +2806,14 @@ async fn scoped_get_runtime_worker(
|
||||
get_runtime_worker(State(api), AxumPath((path.runtime_id, path.worker_id))).await
|
||||
}
|
||||
|
||||
async fn scoped_restore_runtime_worker(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
|
||||
) -> ApiResult<Json<WorkerRestoreResponse>> {
|
||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
||||
restore_runtime_worker(State(api), AxumPath((path.runtime_id, path.worker_id))).await
|
||||
}
|
||||
|
||||
async fn scoped_pin_runtime_worker(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
|
||||
@@ -4322,6 +4358,39 @@ async fn get_runtime_worker(
|
||||
)))
|
||||
}
|
||||
|
||||
async fn restore_runtime_worker(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
||||
) -> ApiResult<Json<WorkerRestoreResponse>> {
|
||||
let mut result = api
|
||||
.runtime
|
||||
.restore_worker(&runtime_id, &worker_id)
|
||||
.map_err(|err| err.into_error())?;
|
||||
if let Some(worker) = result.worker.as_ref() {
|
||||
let record = sync_worker_observation(&api, worker)?;
|
||||
let links = api.store.list_worker_workdir_links(
|
||||
&api.config.workspace_id,
|
||||
record.runtime_id.as_str(),
|
||||
record.runtime_worker_id,
|
||||
)?;
|
||||
let workdirs = api
|
||||
.store
|
||||
.list_workdir_registry(&api.config.workspace_id, 500)?;
|
||||
result.worker = Some(merge_worker_registry_projection(
|
||||
Some(worker),
|
||||
&record,
|
||||
links,
|
||||
&workdirs,
|
||||
));
|
||||
}
|
||||
Ok(Json(WorkerRestoreResponse {
|
||||
workspace_id: api.workspace_id().to_string(),
|
||||
runtime_id,
|
||||
worker_id,
|
||||
result,
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct RuntimeConfigBundleSyncRequest {
|
||||
pub bundle: ConfigBundle,
|
||||
@@ -4378,17 +4447,28 @@ fn reject_no_workdir_for_non_embedded_runtime(
|
||||
async fn list_runtime_workers(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(runtime_id): AxumPath<String>,
|
||||
Query(query): Query<RuntimeWorkersQuery>,
|
||||
) -> ApiResult<Json<RuntimeListResponse<WorkerSummary>>> {
|
||||
let limit = api.config.max_records.min(200);
|
||||
let worker_list = api
|
||||
.runtime
|
||||
.list_workers_for_runtime(&runtime_id, limit)
|
||||
.map_err(|err| err.into_error())?;
|
||||
let (worker_list, source) = match query.status {
|
||||
Some(RuntimeWorkersStatusFilter::Stopped) => (
|
||||
api.runtime
|
||||
.list_stopped_workers_for_runtime(&runtime_id, limit)
|
||||
.map_err(|err| err.into_error())?,
|
||||
"runtime_registry_stopped",
|
||||
),
|
||||
None => (
|
||||
api.runtime
|
||||
.list_workers_for_runtime(&runtime_id, limit)
|
||||
.map_err(|err| err.into_error())?,
|
||||
"runtime_registry",
|
||||
),
|
||||
};
|
||||
Ok(Json(RuntimeListResponse {
|
||||
workspace_id: api.workspace_id().to_string(),
|
||||
limit,
|
||||
items: worker_list.items,
|
||||
source: "runtime_registry".to_string(),
|
||||
source: source.to_string(),
|
||||
diagnostics: worker_list.diagnostics,
|
||||
}))
|
||||
}
|
||||
@@ -9322,6 +9402,22 @@ mod tests {
|
||||
assert_eq!(worker["worker_id"], worker_id);
|
||||
assert_eq!(worker["runtime_id"], "embedded-worker-runtime");
|
||||
|
||||
let stopped_workers = get_json(
|
||||
app.clone(),
|
||||
"/api/runtimes/embedded-worker-runtime/workers?status=stopped",
|
||||
)
|
||||
.await;
|
||||
assert!(stopped_workers["items"].as_array().unwrap().is_empty());
|
||||
|
||||
let restored = post_json(
|
||||
app.clone(),
|
||||
&format!("/api/runtimes/embedded-worker-runtime/workers/{worker_id}/restore"),
|
||||
json!({}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(restored["result"]["state"], "accepted");
|
||||
assert_eq!(restored["result"]["worker"]["worker_id"], worker_id);
|
||||
|
||||
let accepted = post_json(
|
||||
app.clone(),
|
||||
&format!("/api/runtimes/embedded-worker-runtime/workers/{worker_id}/input"),
|
||||
@@ -9373,7 +9469,10 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(wrong_runtime.status(), StatusCode::NOT_FOUND);
|
||||
|
||||
let projected = format!("{}{}{}{}", embedded_summary, spawned, worker, accepted);
|
||||
let projected = format!(
|
||||
"{}{}{}{}{}{}",
|
||||
embedded_summary, spawned, worker, stopped_workers, restored, accepted
|
||||
);
|
||||
for forbidden in [
|
||||
dir.path().to_string_lossy().as_ref(),
|
||||
"metadata.json",
|
||||
|
||||
Reference in New Issue
Block a user