feat: guarded Worker Restore from shared Web actions (T-701)
This commit is contained in:
@@ -10,6 +10,7 @@ pub use server_api::{
|
|||||||
WorkerLaunchProfileCandidate as BackendWorkerLaunchProfileCandidate,
|
WorkerLaunchProfileCandidate as BackendWorkerLaunchProfileCandidate,
|
||||||
WorkerLaunchRuntimeOption as BackendWorkerLaunchRuntimeOption,
|
WorkerLaunchRuntimeOption as BackendWorkerLaunchRuntimeOption,
|
||||||
WorkerOperationState as BackendWorkerOperationState,
|
WorkerOperationState as BackendWorkerOperationState,
|
||||||
|
WorkerRestoreRequest as BackendWorkerRestoreRequest,
|
||||||
WorkerRestoreResponse as BackendWorkerRestoreResponse,
|
WorkerRestoreResponse as BackendWorkerRestoreResponse,
|
||||||
WorkerRestoreResult as BackendWorkerRestoreResult,
|
WorkerRestoreResult as BackendWorkerRestoreResult,
|
||||||
WorkerRestoreState as BackendWorkerRestoreState, WorkerSummary as BackendWorkerSummary,
|
WorkerRestoreState as BackendWorkerRestoreState, WorkerSummary as BackendWorkerSummary,
|
||||||
@@ -259,6 +260,15 @@ impl fmt::Display for BackendRuntimeClientError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl BackendRuntimeClientError {
|
||||||
|
/// A stale intent is not a rejected Restore operation and must not be retried
|
||||||
|
/// with a silently refreshed token.
|
||||||
|
pub fn is_restore_observation_conflict(&self) -> bool {
|
||||||
|
matches!(self, Self::ContractApi(server_api::client_support::ClientError::Public { status, error })
|
||||||
|
if *status == reqwest::StatusCode::CONFLICT && error.error == "restore_observation_conflict")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl std::error::Error for BackendRuntimeClientError {}
|
impl std::error::Error for BackendRuntimeClientError {}
|
||||||
|
|
||||||
impl From<server_api::ServerApiClientError> for BackendRuntimeClientError {
|
impl From<server_api::ServerApiClientError> for BackendRuntimeClientError {
|
||||||
@@ -502,6 +512,7 @@ pub async fn list_backend_stopped_workers(
|
|||||||
|
|
||||||
pub async fn restore_backend_worker(
|
pub async fn restore_backend_worker(
|
||||||
target: &BackendRuntimeTarget,
|
target: &BackendRuntimeTarget,
|
||||||
|
request: BackendWorkerRestoreRequest,
|
||||||
) -> Result<BackendWorkerRestoreResponse, BackendRuntimeClientError> {
|
) -> Result<BackendWorkerRestoreResponse, BackendRuntimeClientError> {
|
||||||
validate_target(target)?;
|
validate_target(target)?;
|
||||||
let api = BackendApiClient::from_stored_token(&target.base_url)?;
|
let api = BackendApiClient::from_stored_token(&target.base_url)?;
|
||||||
@@ -514,6 +525,7 @@ pub async fn restore_backend_worker(
|
|||||||
ticket_id: None,
|
ticket_id: None,
|
||||||
assignment_operation_id: None,
|
assignment_operation_id: None,
|
||||||
},
|
},
|
||||||
|
request,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
|
|||||||
@@ -26,12 +26,12 @@ pub use backend_runtime::{
|
|||||||
BackendRuntimeListTarget, BackendRuntimeSummary, BackendRuntimeTarget,
|
BackendRuntimeListTarget, BackendRuntimeSummary, BackendRuntimeTarget,
|
||||||
BackendWorkerImplementationSummary, BackendWorkerLaunchOptions,
|
BackendWorkerImplementationSummary, BackendWorkerLaunchOptions,
|
||||||
BackendWorkerLaunchProfileCandidate, BackendWorkerLaunchRuntimeOption,
|
BackendWorkerLaunchProfileCandidate, BackendWorkerLaunchRuntimeOption,
|
||||||
BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreResponse,
|
BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreRequest,
|
||||||
BackendWorkerRestoreResult, BackendWorkerRestoreState, BackendWorkerSummary,
|
BackendWorkerRestoreResponse, BackendWorkerRestoreResult, BackendWorkerRestoreState,
|
||||||
BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource, BackendWorkingDirectorySummary,
|
BackendWorkerSummary, BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource,
|
||||||
connect_backend_runtime, create_backend_worker, get_backend_worker_launch_options,
|
BackendWorkingDirectorySummary, connect_backend_runtime, create_backend_worker,
|
||||||
list_backend_stopped_workers, list_backend_workers, observe_backend_worker_session,
|
get_backend_worker_launch_options, list_backend_stopped_workers, list_backend_workers,
|
||||||
restore_backend_worker,
|
observe_backend_worker_session, restore_backend_worker,
|
||||||
};
|
};
|
||||||
pub use backend_workspace::{
|
pub use backend_workspace::{
|
||||||
BackendWorkspace, BackendWorkspaceCatalogTarget, BackendWorkspaceClientError,
|
BackendWorkspace, BackendWorkspaceCatalogTarget, BackendWorkspaceClientError,
|
||||||
|
|||||||
@@ -651,6 +651,10 @@ impl SubscriptionWorkerJob {
|
|||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||||
pub struct SubscriptionWorker {
|
pub struct SubscriptionWorker {
|
||||||
|
/// Opaque Runtime-owned Restore observation identity, never a display-state hash.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||||
|
pub restore_observation_token: Option<String>,
|
||||||
pub worker_id: SubscriptionWorkerId,
|
pub worker_id: SubscriptionWorkerId,
|
||||||
/// Set by the Workspace Server when projecting a Runtime-owned Worker to clients.
|
/// Set by the Workspace Server when projecting a Runtime-owned Worker to clients.
|
||||||
/// Runtime producers leave this unset because the connection identifies the Runtime.
|
/// Runtime producers leave this unset because the connection identifies the Runtime.
|
||||||
@@ -689,6 +693,9 @@ pub struct SubscriptionWorker {
|
|||||||
impl SubscriptionWorker {
|
impl SubscriptionWorker {
|
||||||
pub fn validate(&self) -> Result<(), SubscriptionValidationError> {
|
pub fn validate(&self) -> Result<(), SubscriptionValidationError> {
|
||||||
self.worker_id.validate()?;
|
self.worker_id.validate()?;
|
||||||
|
if let Some(token) = &self.restore_observation_token {
|
||||||
|
validate_identifier("restore_observation_token", token, MAX_RESOURCE_ID_BYTES)?;
|
||||||
|
}
|
||||||
if let Some(runtime_id) = &self.runtime_id {
|
if let Some(runtime_id) = &self.runtime_id {
|
||||||
validate_identifier("runtime_id", runtime_id, MAX_RESOURCE_ID_BYTES)?;
|
validate_identifier("runtime_id", runtime_id, MAX_RESOURCE_ID_BYTES)?;
|
||||||
}
|
}
|
||||||
@@ -962,6 +969,7 @@ mod tests {
|
|||||||
|
|
||||||
fn worker(value: &str) -> SubscriptionWorker {
|
fn worker(value: &str) -> SubscriptionWorker {
|
||||||
SubscriptionWorker {
|
SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: worker_id(value),
|
worker_id: worker_id(value),
|
||||||
runtime_id: None,
|
runtime_id: None,
|
||||||
resource_key: None,
|
resource_key: None,
|
||||||
|
|||||||
@@ -246,6 +246,7 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec<SubscriptionFrame> {
|
|||||||
let external_workdir_id = SubscriptionWorkdirId::new("fixture-external-workdir")
|
let external_workdir_id = SubscriptionWorkdirId::new("fixture-external-workdir")
|
||||||
.expect("fixture external Workdir id must be valid");
|
.expect("fixture external Workdir id must be valid");
|
||||||
let worker = SubscriptionWorker {
|
let worker = SubscriptionWorker {
|
||||||
|
restore_observation_token: Some("fixture-restore-observation".into()),
|
||||||
worker_id: worker_id.clone(),
|
worker_id: worker_id.clone(),
|
||||||
runtime_id: Some("fixture-runtime".to_string()),
|
runtime_id: Some("fixture-runtime".to_string()),
|
||||||
resource_key: Some("fixture-worker-resource".to_string()),
|
resource_key: Some("fixture-worker-resource".to_string()),
|
||||||
|
|||||||
@@ -378,6 +378,7 @@ function validateWorkers(workers: SubscriptionWorker[]): boolean {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function validateWorker(worker: SubscriptionWorker): boolean {
|
function validateWorker(worker: SubscriptionWorker): boolean {
|
||||||
|
if (worker.restore_observation_token != null && !validateIdentifier(worker.restore_observation_token, MAX_RESOURCE_ID_BYTES)) return false;
|
||||||
if (!validateIdentifier(worker.worker_id, MAX_RESOURCE_ID_BYTES)) return false;
|
if (!validateIdentifier(worker.worker_id, MAX_RESOURCE_ID_BYTES)) return false;
|
||||||
if (worker.runtime_id != null && !validateIdentifier(worker.runtime_id, MAX_RESOURCE_ID_BYTES)) {
|
if (worker.runtime_id != null && !validateIdentifier(worker.runtime_id, MAX_RESOURCE_ID_BYTES)) {
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -452,6 +452,8 @@ pub struct CreateWorkerRequest {
|
|||||||
|
|
||||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
pub struct WorkerSummary {
|
pub struct WorkerSummary {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub restore_observation_token: Option<String>,
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
pub worker_id: WorkerId,
|
pub worker_id: WorkerId,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
@@ -484,6 +486,62 @@ pub struct WorkerResponse {
|
|||||||
pub worker: WorkerDetail,
|
pub worker: WorkerDetail,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Explicit observed Restore intent. Preparation is authored by the Workspace
|
||||||
|
/// Backend, not browser/Tool JSON, and applied only after Runtime admission.
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct WorkerRestoreRequest {
|
||||||
|
pub expected_observation_token: String,
|
||||||
|
pub request_id: String,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub preparation: Option<WorkerRestorePreparation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct WorkerRestorePreparation {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub workspace_api: Option<WorkspaceApiRef>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub workdir_attachments: Option<Vec<LogicalWorkdirAttachment>>,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub repository_access: Vec<WorkingDirectoryRepositoryAccessRequest>,
|
||||||
|
/// Read-only references resolved by Workspace only after this operation owns admission.
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub repository_access_workdirs: Vec<RepositoryAccessWorkdirReference>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RepositoryAccessWorkdirReference {
|
||||||
|
pub runtime_id: String,
|
||||||
|
pub working_directory_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct WorkerRestoreCoordinationRequest {
|
||||||
|
pub expected_observation_token: String,
|
||||||
|
pub request_id: String,
|
||||||
|
/// None means owner/result recovery only; it can never admit a new intent.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub preparation: Option<WorkerRestorePreparation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
pub struct WorkerRestoreCoordinationResponse {
|
||||||
|
pub result: Option<WorkerRestoreResponse>,
|
||||||
|
/// Original read-only preparation, only when the existing operation still needs it.
|
||||||
|
pub preparation: Option<WorkerRestorePreparation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct WorkingDirectoryRepositoryAccessRequest {
|
||||||
|
pub working_directory_id: String,
|
||||||
|
pub materialization: RepositoryMaterializationContext,
|
||||||
|
}
|
||||||
|
|
||||||
/// Explicit restore outcome returned for every reachable Runtime restore attempt,
|
/// Explicit restore outcome returned for every reachable Runtime restore attempt,
|
||||||
/// including preflight rejection and uncertain post-side-effect reconciliation.
|
/// including preflight rejection and uncertain post-side-effect reconciliation.
|
||||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
@@ -1000,11 +1058,18 @@ pub trait RuntimeApi {
|
|||||||
#[body] request: WorkerLifecycleRequest,
|
#[body] request: WorkerLifecycleRequest,
|
||||||
) -> Result<WorkerLifecycleResponse, RuntimeApiError>;
|
) -> Result<WorkerLifecycleResponse, RuntimeApiError>;
|
||||||
|
|
||||||
#[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400)]
|
#[post("/v1/workers/{worker_id}/restore/coordinate", status = 200, error_status = 400, additional_error_statuses = [409])]
|
||||||
|
async fn coordinate_worker_restore(
|
||||||
|
&self,
|
||||||
|
#[path] worker_id: String,
|
||||||
|
#[body] request: WorkerRestoreCoordinationRequest,
|
||||||
|
) -> Result<WorkerRestoreCoordinationResponse, RuntimeApiError>;
|
||||||
|
|
||||||
|
#[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400, additional_error_statuses = [409])]
|
||||||
async fn restore_worker(
|
async fn restore_worker(
|
||||||
&self,
|
&self,
|
||||||
#[path] worker_id: String,
|
#[path] worker_id: String,
|
||||||
#[body] request: EmptyObjectRequest,
|
#[body] request: WorkerRestoreRequest,
|
||||||
) -> Result<WorkerRestoreResponse, RuntimeApiError>;
|
) -> Result<WorkerRestoreResponse, RuntimeApiError>;
|
||||||
|
|
||||||
#[post(
|
#[post(
|
||||||
@@ -1210,6 +1275,46 @@ pub const REMAINING_RUNTIME_ROUTES: &[RemainingRuntimeRoute] = &[
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn restore_request_is_mandatory_and_conflict_is_not_an_operation_state() {
|
||||||
|
for invalid in [
|
||||||
|
serde_json::json!({}),
|
||||||
|
serde_json::json!({"request_id":"r"}),
|
||||||
|
serde_json::json!({"expected_observation_token":"t"}),
|
||||||
|
] {
|
||||||
|
assert!(serde_json::from_value::<WorkerRestoreRequest>(invalid).is_err());
|
||||||
|
}
|
||||||
|
let request: WorkerRestoreRequest = serde_json::from_value(
|
||||||
|
serde_json::json!({"expected_observation_token":"t", "request_id":"r"}),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(request.preparation.is_none());
|
||||||
|
for state in [
|
||||||
|
"accepted",
|
||||||
|
"rejected",
|
||||||
|
"rolled_back",
|
||||||
|
"reconciliation_required",
|
||||||
|
] {
|
||||||
|
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
||||||
|
"restore_observation_conflict"
|
||||||
|
))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let conflict = RuntimeApiError::new(
|
||||||
|
409,
|
||||||
|
"restore_observation_conflict",
|
||||||
|
"refresh before a new intent",
|
||||||
|
);
|
||||||
|
assert_eq!(conflict.status(), 409);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(conflict).unwrap()["error"]["code"],
|
||||||
|
"restore_observation_conflict"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn feature_argument_completion_wire_preserves_source_and_argument() {
|
fn feature_argument_completion_wire_preserves_source_and_argument() {
|
||||||
let json = serde_json::json!({"kind":"feature_argument", "prefix":"資料/", "context": {"invocation":"builtin:test/prepare", "argument":"path"}});
|
let json = serde_json::json!({"kind":"feature_argument", "prefix":"資料/", "context": {"invocation":"builtin:test/prepare", "argument":"path"}});
|
||||||
@@ -1388,10 +1493,17 @@ mod tests {
|
|||||||
) -> Result<WorkerLifecycleResponse, RuntimeApiError> {
|
) -> Result<WorkerLifecycleResponse, RuntimeApiError> {
|
||||||
Err(test_error(501))
|
Err(test_error(501))
|
||||||
}
|
}
|
||||||
|
async fn coordinate_worker_restore(
|
||||||
|
&self,
|
||||||
|
_worker_id: String,
|
||||||
|
_request: WorkerRestoreCoordinationRequest,
|
||||||
|
) -> Result<WorkerRestoreCoordinationResponse, RuntimeApiError> {
|
||||||
|
Err(test_error(501))
|
||||||
|
}
|
||||||
async fn restore_worker(
|
async fn restore_worker(
|
||||||
&self,
|
&self,
|
||||||
_worker_id: String,
|
_worker_id: String,
|
||||||
_request: EmptyObjectRequest,
|
_request: WorkerRestoreRequest,
|
||||||
) -> Result<WorkerRestoreResponse, RuntimeApiError> {
|
) -> Result<WorkerRestoreResponse, RuntimeApiError> {
|
||||||
Err(test_error(501))
|
Err(test_error(501))
|
||||||
}
|
}
|
||||||
@@ -1491,7 +1603,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn contract_inventory_is_complete_and_unique() {
|
fn contract_inventory_is_complete_and_unique() {
|
||||||
let operations = RuntimeApiMetadata::OPERATIONS;
|
let operations = RuntimeApiMetadata::OPERATIONS;
|
||||||
assert_eq!(operations.len(), 19);
|
assert_eq!(operations.len(), 20);
|
||||||
let mut routes = operations
|
let mut routes = operations
|
||||||
.iter()
|
.iter()
|
||||||
.map(|operation| (format!("{:?}", operation.method), operation.path))
|
.map(|operation| (format!("{:?}", operation.method), operation.path))
|
||||||
|
|||||||
@@ -389,6 +389,7 @@ impl_openapi_schema!(
|
|||||||
WorkspaceRuntimeListResponse,
|
WorkspaceRuntimeListResponse,
|
||||||
RuntimeWorkerListResponse,
|
RuntimeWorkerListResponse,
|
||||||
WorkerRestoreResponse,
|
WorkerRestoreResponse,
|
||||||
|
WorkerRestoreRequest,
|
||||||
WorkerControlSubject,
|
WorkerControlSubject,
|
||||||
WorkerControlRecord,
|
WorkerControlRecord,
|
||||||
WorkerControlListResponse,
|
WorkerControlListResponse,
|
||||||
@@ -1552,6 +1553,7 @@ pub trait ServerApi {
|
|||||||
#[path] workspace_id: String,
|
#[path] workspace_id: String,
|
||||||
#[path] runtime_id: String,
|
#[path] runtime_id: String,
|
||||||
#[path] worker_id: String,
|
#[path] worker_id: String,
|
||||||
|
#[body] request: WorkerRestoreRequest,
|
||||||
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
||||||
#[get(
|
#[get(
|
||||||
"/api/w/{workspace_id}/worker-observation/sessions",
|
"/api/w/{workspace_id}/worker-observation/sessions",
|
||||||
@@ -2080,6 +2082,7 @@ pub trait ServerApi {
|
|||||||
#[extension] context: ServerRequestContext,
|
#[extension] context: ServerRequestContext,
|
||||||
#[path] runtime_id: String,
|
#[path] runtime_id: String,
|
||||||
#[path] worker_id: String,
|
#[path] worker_id: String,
|
||||||
|
#[body] request: WorkerRestoreRequest,
|
||||||
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
||||||
|
|
||||||
#[post(
|
#[post(
|
||||||
@@ -2097,6 +2100,7 @@ pub trait ServerApi {
|
|||||||
#[path] runtime_id: String,
|
#[path] runtime_id: String,
|
||||||
#[path] worker_id: String,
|
#[path] worker_id: String,
|
||||||
#[query] query: RestoreTicketAssignmentQuery,
|
#[query] query: RestoreTicketAssignmentQuery,
|
||||||
|
#[body] request: WorkerRestoreRequest,
|
||||||
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
||||||
|
|
||||||
#[put(
|
#[put(
|
||||||
@@ -9594,6 +9598,10 @@ pub struct RuntimeWorkerWorkdirAttachmentSummary {
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
pub struct WorkerSummary {
|
pub struct WorkerSummary {
|
||||||
|
/// Opaque Runtime-owned observation fence used by explicit Restore intent.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||||
|
pub restore_observation_token: Option<String>,
|
||||||
pub runtime_id: String,
|
pub runtime_id: String,
|
||||||
pub worker_id: String,
|
pub worker_id: String,
|
||||||
pub resource_key: String,
|
pub resource_key: String,
|
||||||
@@ -9830,6 +9838,16 @@ pub enum WorkerRestoreState {
|
|||||||
ReconciliationRequired,
|
ReconciliationRequired,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One explicit Restore intent. Retries retain both fields; conflicts require a
|
||||||
|
/// fresh observation and a deliberate new intent, not automatic token replacement.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct WorkerRestoreRequest {
|
||||||
|
pub expected_observation_token: String,
|
||||||
|
pub request_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
@@ -11694,6 +11712,28 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_restore_request_requires_observation_and_request_identity() {
|
||||||
|
let valid = serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request"});
|
||||||
|
let request: WorkerRestoreRequest = serde_json::from_value(valid.clone()).unwrap();
|
||||||
|
assert_eq!(serde_json::to_value(request).unwrap(), valid);
|
||||||
|
for invalid in [
|
||||||
|
serde_json::json!({}),
|
||||||
|
serde_json::json!({"request_id":"restore-request"}),
|
||||||
|
serde_json::json!({"expected_observation_token":"opaque-generation"}),
|
||||||
|
serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request", "command_id":701}),
|
||||||
|
] {
|
||||||
|
assert!(serde_json::from_value::<WorkerRestoreRequest>(invalid).is_err());
|
||||||
|
}
|
||||||
|
let contract = canonical_openapi_document().unwrap().to_json().unwrap();
|
||||||
|
let contract: serde_json::Value = serde_json::from_str(&contract).unwrap();
|
||||||
|
let restore = &contract["paths"]["/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore"]
|
||||||
|
["post"];
|
||||||
|
assert_eq!(restore["requestBody"]["required"], true);
|
||||||
|
assert!(restore["responses"]["409"].is_object());
|
||||||
|
assert!(contract["components"]["schemas"]["WorkerRestoreRequest"].is_object());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn worker_restore_result_rejects_unknown_fields() {
|
fn worker_restore_result_rejects_unknown_fields() {
|
||||||
let result = serde_json::from_value::<WorkerRestoreResult>(serde_json::json!({
|
let result = serde_json::from_value::<WorkerRestoreResult>(serde_json::json!({
|
||||||
|
|||||||
@@ -872,14 +872,14 @@ fn validate_runtime_operations(
|
|||||||
"/api/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
"/api/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
||||||
"post",
|
"post",
|
||||||
"runtime_worker_restore_alias",
|
"runtime_worker_restore_alias",
|
||||||
None,
|
Some("WorkerRestoreRequest"),
|
||||||
&[("200", "WorkerRestoreResponse")],
|
&[("200", "WorkerRestoreResponse")],
|
||||||
),
|
),
|
||||||
runtime_operation(
|
runtime_operation(
|
||||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
||||||
"post",
|
"post",
|
||||||
"runtime_worker_restore",
|
"runtime_worker_restore",
|
||||||
None,
|
Some("WorkerRestoreRequest"),
|
||||||
&[("200", "WorkerRestoreResponse")],
|
&[("200", "WorkerRestoreResponse")],
|
||||||
),
|
),
|
||||||
runtime_operation(
|
runtime_operation(
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ trait BackendWorkerLifecycle {
|
|||||||
async fn restore(
|
async fn restore(
|
||||||
&self,
|
&self,
|
||||||
target: &client::BackendRuntimeTarget,
|
target: &client::BackendRuntimeTarget,
|
||||||
|
request: client::BackendWorkerRestoreRequest,
|
||||||
) -> Result<BackendWorkerRestoreResponse, io::Error>;
|
) -> Result<BackendWorkerRestoreResponse, io::Error>;
|
||||||
|
|
||||||
async fn observe(
|
async fn observe(
|
||||||
@@ -44,13 +45,19 @@ impl BackendWorkerLifecycle for LiveBackendWorkerLifecycle {
|
|||||||
async fn restore(
|
async fn restore(
|
||||||
&self,
|
&self,
|
||||||
target: &client::BackendRuntimeTarget,
|
target: &client::BackendRuntimeTarget,
|
||||||
|
request: client::BackendWorkerRestoreRequest,
|
||||||
) -> Result<BackendWorkerRestoreResponse, io::Error> {
|
) -> Result<BackendWorkerRestoreResponse, io::Error> {
|
||||||
restore_backend_worker(target).await.map_err(|error| {
|
restore_backend_worker(target, request)
|
||||||
io::Error::other(format!(
|
.await
|
||||||
"failed to restore Backend Worker {}: {error}",
|
.map_err(|error| {
|
||||||
target.display_label()
|
if error.is_restore_observation_conflict() {
|
||||||
))
|
return io::Error::other("Restore observation changed; refresh the Worker list and select again. No new Restore was started.");
|
||||||
})
|
}
|
||||||
|
io::Error::other(format!(
|
||||||
|
"failed to restore Backend Worker {}: {error}",
|
||||||
|
target.display_label()
|
||||||
|
))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn observe(
|
async fn observe(
|
||||||
@@ -133,8 +140,20 @@ pub(crate) async fn run(
|
|||||||
let attach_target = target
|
let attach_target = target
|
||||||
.runtime_target(selected.runtime_id.clone(), selected.worker_id.clone())
|
.runtime_target(selected.runtime_id.clone(), selected.worker_id.clone())
|
||||||
.map_err(|error| io::Error::other(error.to_string()))?;
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
let attach_target =
|
let restore_request =
|
||||||
prepare_selected_worker(&LiveBackendWorkerLifecycle, attach_target, intent).await?;
|
selected
|
||||||
|
.restore_observation_token
|
||||||
|
.map(|token| client::BackendWorkerRestoreRequest {
|
||||||
|
expected_observation_token: token,
|
||||||
|
request_id: uuid::Uuid::now_v7().to_string(),
|
||||||
|
});
|
||||||
|
let attach_target = prepare_selected_worker(
|
||||||
|
&LiveBackendWorkerLifecycle,
|
||||||
|
attach_target,
|
||||||
|
intent,
|
||||||
|
restore_request,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
return console::run_backend_runtime(attach_target).await;
|
return console::run_backend_runtime(attach_target).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -143,9 +162,13 @@ async fn prepare_selected_worker(
|
|||||||
lifecycle: &impl BackendWorkerLifecycle,
|
lifecycle: &impl BackendWorkerLifecycle,
|
||||||
mut target: client::BackendRuntimeTarget,
|
mut target: client::BackendRuntimeTarget,
|
||||||
intent: BackendWorkerPickerIntent,
|
intent: BackendWorkerPickerIntent,
|
||||||
|
restore_request: Option<client::BackendWorkerRestoreRequest>,
|
||||||
) -> Result<client::BackendRuntimeTarget, io::Error> {
|
) -> Result<client::BackendRuntimeTarget, io::Error> {
|
||||||
if intent == BackendWorkerPickerIntent::Resume {
|
if intent == BackendWorkerPickerIntent::Resume {
|
||||||
let response = lifecycle.restore(&target).await?;
|
let request = restore_request.ok_or_else(|| {
|
||||||
|
io::Error::other("Worker has no current Restore observation; refresh and select again")
|
||||||
|
})?;
|
||||||
|
let response = lifecycle.restore(&target, request).await?;
|
||||||
ensure_restore_accepted(&response)?;
|
ensure_restore_accepted(&response)?;
|
||||||
let observation = lifecycle.observe(&target).await?;
|
let observation = lifecycle.observe(&target).await?;
|
||||||
require_live_resume_observation(observation.observation)?;
|
require_live_resume_observation(observation.observation)?;
|
||||||
@@ -158,7 +181,10 @@ async fn prepare_selected_worker(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn is_resume_candidate(worker: &BackendWorkerSummary) -> bool {
|
fn is_resume_candidate(worker: &BackendWorkerSummary) -> bool {
|
||||||
worker.worker_state.is_none()
|
worker.availability == protocol::subscription::SubscriptionWorkerAvailability::Observed
|
||||||
|
&& worker.state == "stopped"
|
||||||
|
&& worker.worker_state.is_none()
|
||||||
|
&& worker.restore_observation_token.is_some()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ensure_restore_accepted(response: &BackendWorkerRestoreResponse) -> Result<(), io::Error> {
|
fn ensure_restore_accepted(response: &BackendWorkerRestoreResponse) -> Result<(), io::Error> {
|
||||||
@@ -570,6 +596,7 @@ mod tests {
|
|||||||
|
|
||||||
fn worker(runtime_id: &str, worker_id: &str, profile: Option<&str>) -> BackendWorkerSummary {
|
fn worker(runtime_id: &str, worker_id: &str, profile: Option<&str>) -> BackendWorkerSummary {
|
||||||
BackendWorkerSummary {
|
BackendWorkerSummary {
|
||||||
|
restore_observation_token: Some("observed-worker-generation".into()),
|
||||||
runtime_id: runtime_id.to_string(),
|
runtime_id: runtime_id.to_string(),
|
||||||
worker_id: worker_id.to_string(),
|
worker_id: worker_id.to_string(),
|
||||||
resource_key: "W-1".to_string(),
|
resource_key: "W-1".to_string(),
|
||||||
@@ -642,6 +669,7 @@ mod tests {
|
|||||||
async fn restore(
|
async fn restore(
|
||||||
&self,
|
&self,
|
||||||
_target: &client::BackendRuntimeTarget,
|
_target: &client::BackendRuntimeTarget,
|
||||||
|
_request: client::BackendWorkerRestoreRequest,
|
||||||
) -> Result<BackendWorkerRestoreResponse, io::Error> {
|
) -> Result<BackendWorkerRestoreResponse, io::Error> {
|
||||||
self.calls.lock().unwrap().push("restore");
|
self.calls.lock().unwrap().push("restore");
|
||||||
Ok(self.restore_response.clone())
|
Ok(self.restore_response.clone())
|
||||||
@@ -689,10 +717,17 @@ mod tests {
|
|||||||
"worker-a",
|
"worker-a",
|
||||||
);
|
);
|
||||||
|
|
||||||
let prepared =
|
let prepared = prepare_selected_worker(
|
||||||
prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume)
|
&lifecycle,
|
||||||
.await
|
target,
|
||||||
.unwrap();
|
BackendWorkerPickerIntent::Resume,
|
||||||
|
Some(client::BackendWorkerRestoreRequest {
|
||||||
|
expected_observation_token: "observed-worker-generation".into(),
|
||||||
|
request_id: "resume-operation".into(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore", "observe"]);
|
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore", "observe"]);
|
||||||
assert!(prepared.initial_snapshot.is_none());
|
assert!(prepared.initial_snapshot.is_none());
|
||||||
@@ -716,10 +751,18 @@ mod tests {
|
|||||||
"worker-a",
|
"worker-a",
|
||||||
);
|
);
|
||||||
|
|
||||||
let error = prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume)
|
let error = prepare_selected_worker(
|
||||||
.await
|
&lifecycle,
|
||||||
.unwrap_err()
|
target,
|
||||||
.to_string();
|
BackendWorkerPickerIntent::Resume,
|
||||||
|
Some(client::BackendWorkerRestoreRequest {
|
||||||
|
expected_observation_token: "observed-worker-generation".into(),
|
||||||
|
request_id: "resume-operation".into(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string();
|
||||||
|
|
||||||
assert!(error.contains("Rejected"));
|
assert!(error.contains("Rejected"));
|
||||||
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
||||||
@@ -736,7 +779,7 @@ mod tests {
|
|||||||
|
|
||||||
assert!(!is_resume_candidate(&live));
|
assert!(!is_resume_candidate(&live));
|
||||||
assert!(is_resume_candidate(&stopped));
|
assert!(is_resume_candidate(&stopped));
|
||||||
assert!(is_resume_candidate(&unknown));
|
assert!(!is_resume_candidate(&unknown));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -334,6 +334,8 @@ pub(crate) enum WorkerRestoreIntent {
|
|||||||
/// Lightweight persisted Worker identity projection.
|
/// Lightweight persisted Worker identity projection.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct WorkerSummary {
|
pub struct WorkerSummary {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub restore_observation_token: Option<String>,
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
pub worker_id: WorkerId,
|
pub worker_id: WorkerId,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
@@ -359,6 +361,8 @@ pub struct WorkerSummary {
|
|||||||
/// Full persisted Worker identity and lifecycle detail.
|
/// Full persisted Worker identity and lifecycle detail.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct WorkerDetail {
|
pub struct WorkerDetail {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub restore_observation_token: Option<String>,
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
pub worker_id: WorkerId,
|
pub worker_id: WorkerId,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
|
|||||||
@@ -12,6 +12,9 @@ pub enum RuntimeError {
|
|||||||
#[error("initial worker input must be user input, got {kind}")]
|
#[error("initial worker input must be user input, got {kind}")]
|
||||||
InvalidInitialInputKind { kind: String },
|
InvalidInitialInputKind { kind: String },
|
||||||
|
|
||||||
|
#[error("worker {worker_id} Restore observation conflicts with the current operation")]
|
||||||
|
RestoreObservationConflict { worker_id: WorkerId },
|
||||||
|
|
||||||
#[error("worker {worker_id} was not found")]
|
#[error("worker {worker_id} was not found")]
|
||||||
WorkerNotFound { worker_id: WorkerId },
|
WorkerNotFound { worker_id: WorkerId },
|
||||||
|
|
||||||
|
|||||||
@@ -520,6 +520,7 @@ pub(crate) enum PersistedWorkerExecutionState {
|
|||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub(crate) struct PersistedWorkerRecord {
|
pub(crate) struct PersistedWorkerRecord {
|
||||||
|
pub(crate) restore_guard: crate::runtime::RestoreGuard,
|
||||||
pub(crate) worker_ref: WorkerRef,
|
pub(crate) worker_ref: WorkerRef,
|
||||||
pub(crate) worker_id: WorkerId,
|
pub(crate) worker_id: WorkerId,
|
||||||
pub(crate) profile: ProfileSelector,
|
pub(crate) profile: ProfileSelector,
|
||||||
@@ -1819,6 +1820,8 @@ impl RuntimeSnapshot {
|
|||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
struct WorkerAggregateRecord {
|
struct WorkerAggregateRecord {
|
||||||
|
#[serde(default)]
|
||||||
|
restore_guard: crate::runtime::RestoreGuard,
|
||||||
schema_version: u32,
|
schema_version: u32,
|
||||||
worker_ref: WorkerRef,
|
worker_ref: WorkerRef,
|
||||||
worker_id: WorkerId,
|
worker_id: WorkerId,
|
||||||
@@ -1874,6 +1877,7 @@ impl WorkerAggregateRecord {
|
|||||||
};
|
};
|
||||||
Self {
|
Self {
|
||||||
schema_version: SCHEMA_VERSION,
|
schema_version: SCHEMA_VERSION,
|
||||||
|
restore_guard: worker.restore_guard.clone(),
|
||||||
worker_ref: worker.worker_ref.clone(),
|
worker_ref: worker.worker_ref.clone(),
|
||||||
worker_id: worker.worker_id,
|
worker_id: worker.worker_id,
|
||||||
profile: worker.profile.clone(),
|
profile: worker.profile.clone(),
|
||||||
@@ -1902,6 +1906,7 @@ impl WorkerAggregateRecord {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn validate(self, path: &Path) -> Result<PersistedWorkerRecord, RuntimeError> {
|
fn validate(self, path: &Path) -> Result<PersistedWorkerRecord, RuntimeError> {
|
||||||
|
let restore_guard = self.restore_guard;
|
||||||
let last_finished_submission_request_id = self.last_finished_submission_request_id;
|
let last_finished_submission_request_id = self.last_finished_submission_request_id;
|
||||||
let identity = WorkerIdentityRecord {
|
let identity = WorkerIdentityRecord {
|
||||||
schema_version: self.schema_version,
|
schema_version: self.schema_version,
|
||||||
@@ -2007,6 +2012,7 @@ impl WorkerAggregateRecord {
|
|||||||
WorkerExecutionStateRecord::Unavailable => PersistedWorkerExecutionState::Unavailable,
|
WorkerExecutionStateRecord::Unavailable => PersistedWorkerExecutionState::Unavailable,
|
||||||
};
|
};
|
||||||
let mut persisted = identity.into_persisted(execution_state);
|
let mut persisted = identity.into_persisted(execution_state);
|
||||||
|
persisted.restore_guard = restore_guard;
|
||||||
persisted.last_finished_submission_request_id = last_finished_submission_request_id;
|
persisted.last_finished_submission_request_id = last_finished_submission_request_id;
|
||||||
Ok(persisted)
|
Ok(persisted)
|
||||||
}
|
}
|
||||||
@@ -2180,6 +2186,7 @@ impl WorkerIdentityRecord {
|
|||||||
self.workdir_attachments
|
self.workdir_attachments
|
||||||
};
|
};
|
||||||
PersistedWorkerRecord {
|
PersistedWorkerRecord {
|
||||||
|
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||||
worker_ref: self.worker_ref,
|
worker_ref: self.worker_ref,
|
||||||
worker_id: self.worker_id,
|
worker_id: self.worker_id,
|
||||||
profile: self.profile,
|
profile: self.profile,
|
||||||
|
|||||||
@@ -2302,6 +2302,7 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
|||||||
}
|
}
|
||||||
if path.ends_with("/input")
|
if path.ends_with("/input")
|
||||||
|| path.ends_with("/restore")
|
|| path.ends_with("/restore")
|
||||||
|
|| path.ends_with("/restore/coordinate")
|
||||||
|| (path.contains("/attachments") && *method != Method::GET)
|
|| (path.contains("/attachments") && *method != Method::GET)
|
||||||
|| path.contains("/workdir-attachments")
|
|| path.contains("/workdir-attachments")
|
||||||
{
|
{
|
||||||
@@ -2514,6 +2515,7 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
|
|||||||
StatusCode::CONFLICT
|
StatusCode::CONFLICT
|
||||||
}
|
}
|
||||||
RuntimeError::RuntimeStopped
|
RuntimeError::RuntimeStopped
|
||||||
|
| RuntimeError::RestoreObservationConflict { .. }
|
||||||
| RuntimeError::RuntimeStoreAlreadyOpen { .. }
|
| RuntimeError::RuntimeStoreAlreadyOpen { .. }
|
||||||
| RuntimeError::WorkerExecutionUnavailable { .. }
|
| RuntimeError::WorkerExecutionUnavailable { .. }
|
||||||
| RuntimeError::ExecutionBackendUnavailable { .. }
|
| RuntimeError::ExecutionBackendUnavailable { .. }
|
||||||
@@ -2538,6 +2540,9 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
|
|||||||
fn code_for_runtime_error(error: &RuntimeError) -> String {
|
fn code_for_runtime_error(error: &RuntimeError) -> String {
|
||||||
match error {
|
match error {
|
||||||
RuntimeError::RuntimeStopped => "runtime_stopped".to_string(),
|
RuntimeError::RuntimeStopped => "runtime_stopped".to_string(),
|
||||||
|
RuntimeError::RestoreObservationConflict { .. } => {
|
||||||
|
"restore_observation_conflict".to_string()
|
||||||
|
}
|
||||||
RuntimeError::RuntimeStoreAlreadyOpen { .. } => "runtime_store_already_open".to_string(),
|
RuntimeError::RuntimeStoreAlreadyOpen { .. } => "runtime_store_already_open".to_string(),
|
||||||
RuntimeError::WorkerNotFound { .. } => "worker_not_found".to_string(),
|
RuntimeError::WorkerNotFound { .. } => "worker_not_found".to_string(),
|
||||||
RuntimeError::WorkerExecutionUnavailable { .. } => {
|
RuntimeError::WorkerExecutionUnavailable { .. } => {
|
||||||
@@ -3642,7 +3647,17 @@ mod tests {
|
|||||||
client
|
client
|
||||||
.restore_worker(
|
.restore_worker(
|
||||||
generated_worker_id.clone(),
|
generated_worker_id.clone(),
|
||||||
runtime_api::EmptyObjectRequest::default(),
|
runtime_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: client
|
||||||
|
.get_worker(generated_worker_id.clone())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.worker
|
||||||
|
.restore_observation_token
|
||||||
|
.unwrap(),
|
||||||
|
request_id: "generated-restore".into(),
|
||||||
|
preparation: Some(runtime_api::WorkerRestorePreparation::default()),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -3786,18 +3801,86 @@ mod tests {
|
|||||||
let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await;
|
let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await;
|
||||||
assert_eq!(stop.ack.worker_ref, created.worker.worker_ref);
|
assert_eq!(stop.ack.worker_ref, created.worker.worker_ref);
|
||||||
|
|
||||||
let response = authed_empty_request(
|
let restore_uri = format!("/v1/workers/{}/restore", created.worker.worker_id);
|
||||||
|
let missing = authed_empty_request(app.clone(), Method::POST, &restore_uri, token).await;
|
||||||
|
assert_eq!(missing.status(), StatusCode::BAD_REQUEST);
|
||||||
|
let stale = runtime_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: created.worker.restore_observation_token.clone().unwrap(),
|
||||||
|
request_id: "stale-http-restore".into(),
|
||||||
|
preparation: None,
|
||||||
|
};
|
||||||
|
let response =
|
||||||
|
authed_json_request(app.clone(), Method::POST, &restore_uri, token, &stale).await;
|
||||||
|
assert_eq!(response.status(), StatusCode::CONFLICT);
|
||||||
|
let conflict: RuntimeHttpErrorResponse = read_json(response).await;
|
||||||
|
assert_eq!(conflict.error.code, "restore_observation_conflict");
|
||||||
|
|
||||||
|
let mut restore_request = runtime.test_restore_request(&created.worker.worker_ref);
|
||||||
|
let coordinate_uri = format!("{restore_uri}/coordinate");
|
||||||
|
let mut coordination = runtime_api::WorkerRestoreCoordinationRequest {
|
||||||
|
expected_observation_token: restore_request.expected_observation_token.clone(),
|
||||||
|
request_id: restore_request.request_id.clone(),
|
||||||
|
preparation: None,
|
||||||
|
};
|
||||||
|
let before_lookup = runtime.worker_detail(&created.worker.worker_ref).unwrap();
|
||||||
|
let response = authed_json_request(
|
||||||
app.clone(),
|
app.clone(),
|
||||||
Method::POST,
|
Method::POST,
|
||||||
&format!("/v1/workers/{}/restore", created.worker.worker_id),
|
&coordinate_uri,
|
||||||
token,
|
token,
|
||||||
|
&coordination,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let missing_owner: runtime_api::WorkerRestoreCoordinationResponse =
|
||||||
|
read_json(response).await;
|
||||||
|
assert!(missing_owner.result.is_none() && missing_owner.preparation.is_none());
|
||||||
|
assert_eq!(
|
||||||
|
runtime.worker_detail(&created.worker.worker_ref).unwrap(),
|
||||||
|
before_lookup
|
||||||
|
);
|
||||||
|
let response = authed_json_request(
|
||||||
|
app.clone(),
|
||||||
|
Method::POST,
|
||||||
|
&coordinate_uri,
|
||||||
|
"wrong-token",
|
||||||
|
&coordination,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
coordination.preparation = Some(runtime_api::WorkerRestorePreparation::default());
|
||||||
|
let response = authed_json_request(
|
||||||
|
app.clone(),
|
||||||
|
Method::POST,
|
||||||
|
&coordinate_uri,
|
||||||
|
token,
|
||||||
|
&coordination,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let admitted: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await;
|
||||||
|
assert!(admitted.result.is_none() && admitted.preparation.is_some());
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.worker_detail(&created.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.status,
|
||||||
|
crate::catalog::WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
restore_request.preparation = admitted.preparation;
|
||||||
|
let response = authed_json_request(
|
||||||
|
app.clone(),
|
||||||
|
Method::POST,
|
||||||
|
&restore_uri,
|
||||||
|
token,
|
||||||
|
&restore_request,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
let restored: runtime_api::WorkerRestoreResponse = read_json(response).await;
|
let restored: runtime_api::WorkerRestoreResponse = read_json(response).await;
|
||||||
assert_eq!(restored.state, WorkerRestoreState::Accepted);
|
assert_eq!(restored.state, WorkerRestoreState::Accepted);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
restored.worker.unwrap().status,
|
restored.worker.as_ref().unwrap().status,
|
||||||
runtime_api::WorkerStatus::Idle
|
runtime_api::WorkerStatus::Idle
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -3810,6 +3893,47 @@ mod tests {
|
|||||||
.await;
|
.await;
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
coordination.preparation = None;
|
||||||
|
let response = authed_json_request(
|
||||||
|
app.clone(),
|
||||||
|
Method::POST,
|
||||||
|
&coordinate_uri,
|
||||||
|
token,
|
||||||
|
&coordination,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let recovered: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await;
|
||||||
|
assert!(recovered.preparation.is_none());
|
||||||
|
assert_eq!(recovered.result, Some(restored.clone()));
|
||||||
|
|
||||||
|
restore_request.preparation = Some(serde_json::from_value(serde_json::json!({
|
||||||
|
"workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" },
|
||||||
|
"workdir_attachments": [],
|
||||||
|
"repository_access": []
|
||||||
|
})).unwrap());
|
||||||
|
let response = authed_json_request(
|
||||||
|
app.clone(),
|
||||||
|
Method::POST,
|
||||||
|
&restore_uri,
|
||||||
|
token,
|
||||||
|
&restore_request,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let replayed: runtime_api::WorkerRestoreResponse = read_json(response).await;
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(replayed).unwrap(),
|
||||||
|
serde_json::to_value(restored).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.worker_detail(&created.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.status,
|
||||||
|
crate::catalog::WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
|
||||||
let response = authed_empty_request(
|
let response = authed_empty_request(
|
||||||
app.clone(),
|
app.clone(),
|
||||||
Method::POST,
|
Method::POST,
|
||||||
@@ -3892,11 +4016,12 @@ mod tests {
|
|||||||
assert_eq!(listed.workers[0].status, crate::catalog::WorkerStatus::Idle);
|
assert_eq!(listed.workers[0].status, crate::catalog::WorkerStatus::Idle);
|
||||||
assert!(listed.workers[0].worker_state.is_none());
|
assert!(listed.workers[0].worker_state.is_none());
|
||||||
|
|
||||||
let response = authed_empty_request(
|
let response = authed_json_request(
|
||||||
app.clone(),
|
app.clone(),
|
||||||
Method::POST,
|
Method::POST,
|
||||||
&format!("{worker_uri}/restore"),
|
&format!("{worker_uri}/restore"),
|
||||||
token,
|
token,
|
||||||
|
&runtime.test_restore_request(&created.worker_ref),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
@@ -4864,7 +4989,9 @@ mod ws_tests {
|
|||||||
let (mut old_stream, _) = connect_async(authed_ws_request(&url)).await.unwrap();
|
let (mut old_stream, _) = connect_async(authed_ws_request(&url)).await.unwrap();
|
||||||
let _ = next_frame(&mut old_stream).await;
|
let _ = next_frame(&mut old_stream).await;
|
||||||
runtime.stop_worker(&worker_ref, None).unwrap();
|
runtime.stop_worker(&worker_ref, None).unwrap();
|
||||||
runtime.restore_worker(&worker_ref).unwrap();
|
runtime
|
||||||
|
.restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref))
|
||||||
|
.unwrap();
|
||||||
backend.publish(
|
backend.publish(
|
||||||
&worker_ref,
|
&worker_ref,
|
||||||
protocol::Event::TextDelta {
|
protocol::Event::TextDelta {
|
||||||
@@ -4946,7 +5073,9 @@ mod ws_tests {
|
|||||||
);
|
);
|
||||||
backend.publish(&worker_ref, protocol::Event::Shutdown);
|
backend.publish(&worker_ref, protocol::Event::Shutdown);
|
||||||
runtime.stop_worker(&worker_ref, None).unwrap();
|
runtime.stop_worker(&worker_ref, None).unwrap();
|
||||||
runtime.restore_worker(&worker_ref).unwrap();
|
runtime
|
||||||
|
.restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref))
|
||||||
|
.unwrap();
|
||||||
let resume_url = format!("{url}?cursor={}", old.cursor);
|
let resume_url = format!("{url}?cursor={}", old.cursor);
|
||||||
let (mut stream, _) = connect_async(authed_ws_request(&resume_url)).await.unwrap();
|
let (mut stream, _) = connect_async(authed_ws_request(&resume_url)).await.unwrap();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
|
|||||||
@@ -462,21 +462,74 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
Ok(runtime_api::WorkerLifecycleResponse { ack })
|
Ok(runtime_api::WorkerLifecycleResponse { ack })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn coordinate_worker_restore(
|
||||||
|
&self,
|
||||||
|
worker_id: String,
|
||||||
|
request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||||
|
) -> Result<runtime_api::WorkerRestoreCoordinationResponse, runtime_api::RuntimeApiError> {
|
||||||
|
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||||
|
let scope =
|
||||||
|
auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?;
|
||||||
|
let runtime = self.state.runtime.clone();
|
||||||
|
let (result, preparation) = tokio::task::spawn_blocking(move || {
|
||||||
|
runtime.coordinate_worker_restore_operation(&worker_ref, request, scope.as_ref())
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
runtime_api::RuntimeApiError::new(500, "runtime_restore_task_failed", error.to_string())
|
||||||
|
})?
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)?;
|
||||||
|
let result = result
|
||||||
|
.map(|result| -> Result<_, runtime_api::RuntimeApiError> {
|
||||||
|
Ok(runtime_api::WorkerRestoreResponse {
|
||||||
|
state: match result.state {
|
||||||
|
server_api::WorkerRestoreState::Accepted => {
|
||||||
|
runtime_api::WorkerRestoreState::Accepted
|
||||||
|
}
|
||||||
|
server_api::WorkerRestoreState::Rejected => {
|
||||||
|
runtime_api::WorkerRestoreState::Rejected
|
||||||
|
}
|
||||||
|
server_api::WorkerRestoreState::RolledBack => {
|
||||||
|
runtime_api::WorkerRestoreState::RolledBack
|
||||||
|
}
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
|
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
}
|
||||||
|
},
|
||||||
|
worker: result.worker.map(response).transpose()?,
|
||||||
|
reason_code: result.reason_code,
|
||||||
|
message: result.message,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
|
Ok(runtime_api::WorkerRestoreCoordinationResponse {
|
||||||
|
result,
|
||||||
|
preparation,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn restore_worker(
|
async fn restore_worker(
|
||||||
&self,
|
&self,
|
||||||
worker_id: String,
|
worker_id: String,
|
||||||
_request: runtime_api::EmptyObjectRequest,
|
request: runtime_api::WorkerRestoreRequest,
|
||||||
) -> Result<runtime_api::WorkerRestoreResponse, runtime_api::RuntimeApiError> {
|
) -> Result<runtime_api::WorkerRestoreResponse, runtime_api::RuntimeApiError> {
|
||||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||||
let result = match auth_workspace_scope(&self.state, auth_extension().as_ref())
|
let scope =
|
||||||
.map_err(api_error)?
|
auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?;
|
||||||
{
|
let runtime = self.state.runtime.clone();
|
||||||
Some(scope) => self
|
let result = tokio::task::spawn_blocking(move || match scope {
|
||||||
.state
|
Some(scope) => runtime.restore_worker_operation_scoped(&scope, &worker_ref, request),
|
||||||
.runtime
|
None => runtime.restore_worker_operation(&worker_ref, request),
|
||||||
.restore_worker_operation_scoped(&scope, &worker_ref),
|
})
|
||||||
None => self.state.runtime.restore_worker_operation(&worker_ref),
|
.await
|
||||||
}
|
.map_err(|error| {
|
||||||
|
runtime_api::RuntimeApiError::new(
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
|
||||||
|
"runtime_restore_task_failed",
|
||||||
|
error.to_string(),
|
||||||
|
)
|
||||||
|
})?
|
||||||
.map_err(RuntimeHttpRestError::runtime)
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
.map_err(api_error)?;
|
.map_err(api_error)?;
|
||||||
Ok(runtime_api::WorkerRestoreResponse {
|
Ok(runtime_api::WorkerRestoreResponse {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -7394,7 +7394,14 @@ mod tests {
|
|||||||
.create_worker(create_request("restart cleanup"))
|
.create_worker(create_request("restart cleanup"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
assert!(runtime.restore_worker(&worker.worker_ref).is_err());
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_worker(
|
||||||
|
&worker.worker_ref,
|
||||||
|
runtime.test_restore_request(&worker.worker_ref)
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
assert!(runtime.stop_worker(&worker.worker_ref, None).is_err());
|
assert!(runtime.stop_worker(&worker.worker_ref, None).is_err());
|
||||||
let prior_restore_id = restore_id.lock().unwrap().unwrap();
|
let prior_restore_id = restore_id.lock().unwrap().unwrap();
|
||||||
let aggregate = runtime_store
|
let aggregate = runtime_store
|
||||||
@@ -8904,7 +8911,12 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(detached.workdir_attachments.is_empty());
|
assert!(detached.workdir_attachments.is_empty());
|
||||||
runtime.stop_worker(&detail.worker_ref, None).unwrap();
|
runtime.stop_worker(&detail.worker_ref, None).unwrap();
|
||||||
let restored_without_workdir = runtime.restore_worker(&detail.worker_ref).unwrap();
|
let restored_without_workdir = runtime
|
||||||
|
.restore_worker(
|
||||||
|
&detail.worker_ref,
|
||||||
|
runtime.test_restore_request(&detail.worker_ref),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
assert!(restored_without_workdir.workdir_attachments.is_empty());
|
assert!(restored_without_workdir.workdir_attachments.is_empty());
|
||||||
assert!(
|
assert!(
|
||||||
!repo.path().join(".yoi").exists(),
|
!repo.path().join(".yoi").exists(),
|
||||||
@@ -9191,7 +9203,12 @@ mod tests {
|
|||||||
.contains_key(&detail.worker_ref)
|
.contains_key(&detail.worker_ref)
|
||||||
);
|
);
|
||||||
|
|
||||||
runtime.restore_worker(&detail.worker_ref).unwrap();
|
runtime
|
||||||
|
.restore_worker(
|
||||||
|
&detail.worker_ref,
|
||||||
|
runtime.test_restore_request(&detail.worker_ref),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
runtime.worker_detail(&detail.worker_ref).unwrap().status,
|
runtime.worker_detail(&detail.worker_ref).unwrap().status,
|
||||||
crate::catalog::WorkerStatus::Idle
|
crate::catalog::WorkerStatus::Idle
|
||||||
|
|||||||
@@ -555,9 +555,15 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
|
|||||||
WorkspaceServerOperation::WorkerControlRestore {
|
WorkspaceServerOperation::WorkerControlRestore {
|
||||||
runtime_id,
|
runtime_id,
|
||||||
worker_id,
|
worker_id,
|
||||||
|
request,
|
||||||
} => generated_workspace_response(
|
} => generated_workspace_response(
|
||||||
client
|
client
|
||||||
.worker_control_restore(workspace_id, runtime_id, worker_id)
|
.worker_control_restore(
|
||||||
|
workspace_id,
|
||||||
|
runtime_id,
|
||||||
|
worker_id,
|
||||||
|
request,
|
||||||
|
)
|
||||||
.await,
|
.await,
|
||||||
),
|
),
|
||||||
WorkspaceServerOperation::WorkerObservationSessions => {
|
WorkspaceServerOperation::WorkerObservationSessions => {
|
||||||
|
|||||||
@@ -77,8 +77,9 @@ pub trait WorkerControlService: Send + Sync {
|
|||||||
&self,
|
&self,
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
worker_id: String,
|
worker_id: String,
|
||||||
|
request: server_api::WorkerRestoreRequest,
|
||||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||||
let _ = (runtime_id, worker_id);
|
let _ = (runtime_id, worker_id, request);
|
||||||
Err(WorkspaceClientError::Unavailable(
|
Err(WorkspaceClientError::Unavailable(
|
||||||
"Runtime Worker restore is unavailable".to_string(),
|
"Runtime Worker restore is unavailable".to_string(),
|
||||||
))
|
))
|
||||||
@@ -347,6 +348,7 @@ impl WorkerControlService for WorkspaceWorkerControlService {
|
|||||||
&self,
|
&self,
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
worker_id: String,
|
worker_id: String,
|
||||||
|
request: server_api::WorkerRestoreRequest,
|
||||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||||
if !self.runtime_worker_control {
|
if !self.runtime_worker_control {
|
||||||
return Err(WorkspaceClientError::Unavailable(
|
return Err(WorkspaceClientError::Unavailable(
|
||||||
@@ -357,6 +359,7 @@ impl WorkerControlService for WorkspaceWorkerControlService {
|
|||||||
crate::worker::WorkspaceServerOperation::WorkerControlRestore {
|
crate::worker::WorkspaceServerOperation::WorkerControlRestore {
|
||||||
runtime_id,
|
runtime_id,
|
||||||
worker_id,
|
worker_id,
|
||||||
|
request,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -750,8 +753,13 @@ enum WorkerSubjectInput {
|
|||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
struct WorkerTargetInput {
|
struct WorkerRestoreInput {
|
||||||
subject: WorkerSubjectInput,
|
subject: WorkerSubjectInput,
|
||||||
|
expected_observation_token: String,
|
||||||
|
/// Reuse the reported ID for explicit unknown-outcome recovery. A new intent
|
||||||
|
/// defaults to the durable Tool call identity.
|
||||||
|
#[serde(default)]
|
||||||
|
request_id: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
@@ -864,7 +872,7 @@ impl WorkerOperation {
|
|||||||
Self::Cancel => "Cancel the current turn of a known Runtime Worker when allowed.",
|
Self::Cancel => "Cancel the current turn of a known Runtime Worker when allowed.",
|
||||||
Self::Stop => "Stop a known Runtime Worker when allowed.",
|
Self::Stop => "Stop a known Runtime Worker when allowed.",
|
||||||
Self::Restore => {
|
Self::Restore => {
|
||||||
"Restore a stopped Backend/Runtime Worker session in the current Workspace."
|
"Restore a stopped Backend/Runtime Worker session in the current Workspace. Supply expected_observation_token from WorkerList; a conflict requires refreshing and deliberate new intent. New requests default to the durable Tool call ID. For explicit recovery of an unknown/pending outcome, supply the reported request_id and the original token unchanged; never create a new intent or automatically retry."
|
||||||
}
|
}
|
||||||
Self::Remove => {
|
Self::Remove => {
|
||||||
"Remove an eligible stopped, unassigned, non-internal Worker. Supply a bounded reason; Backend validation and retention are authoritative."
|
"Remove an eligible stopped, unassigned, non-internal Worker. Supply a bounded reason; Backend validation and retention are authoritative."
|
||||||
@@ -981,12 +989,41 @@ impl Tool for WorkspaceWorkerTool {
|
|||||||
.map_err(control_tool_error)?
|
.map_err(control_tool_error)?
|
||||||
}
|
}
|
||||||
WorkerOperation::Restore => {
|
WorkerOperation::Restore => {
|
||||||
let input = parse::<WorkerTargetInput>(input_json, "WorkerRestore")?;
|
let input = parse::<WorkerRestoreInput>(input_json, "WorkerRestore")?;
|
||||||
let (runtime_id, worker_id) = runtime_subject_ids(&input.subject, self.operation)?;
|
let (runtime_id, worker_id) = runtime_subject_ids(&input.subject, self.operation)?;
|
||||||
self.control
|
let request_id = match input.request_id {
|
||||||
.restore_worker(runtime_id, worker_id)
|
Some(id) => non_empty(id, "request_id")?,
|
||||||
|
None => format!(
|
||||||
|
"worker-restore:{}",
|
||||||
|
non_empty(ctx.call_id.clone(), "tool call_id")?
|
||||||
|
),
|
||||||
|
};
|
||||||
|
if request_id.len() > 512 {
|
||||||
|
return Err(ToolError::InvalidArgument(
|
||||||
|
"request_id must contain at most 512 bytes".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let recoverable_error = |error| {
|
||||||
|
ToolError::ExecutionFailed(format!(
|
||||||
|
"WorkerRestore request_id={request_id}; reuse this ID with the original observation token only for explicit unknown/pending outcome recovery, not after a stale-intent conflict: {error}"
|
||||||
|
))
|
||||||
|
};
|
||||||
|
let response = self
|
||||||
|
.control
|
||||||
|
.restore_worker(
|
||||||
|
runtime_id,
|
||||||
|
worker_id,
|
||||||
|
server_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: non_empty(
|
||||||
|
input.expected_observation_token,
|
||||||
|
"expected_observation_token",
|
||||||
|
)?,
|
||||||
|
request_id: request_id.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(control_tool_error)?
|
.map_err(|error| recoverable_error(control_tool_error(error)))?;
|
||||||
|
return tool_output(self.operation, response).map_err(recoverable_error);
|
||||||
}
|
}
|
||||||
WorkerOperation::Remove => {
|
WorkerOperation::Remove => {
|
||||||
let input = parse::<WorkerRemoveInput>(input_json, "WorkerRemove")?;
|
let input = parse::<WorkerRemoveInput>(input_json, "WorkerRemove")?;
|
||||||
@@ -1058,6 +1095,18 @@ fn tool_output(
|
|||||||
response.status, response.body
|
response.status, response.body
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
if operation == WorkerOperation::Restore {
|
||||||
|
let restored: server_api::WorkerRestoreResponse = serde_json::from_str(&response.body)
|
||||||
|
.map_err(|error| {
|
||||||
|
ToolError::ExecutionFailed(format!("invalid WorkerRestore response: {error}"))
|
||||||
|
})?;
|
||||||
|
if restored.result.state != server_api::WorkerRestoreState::Accepted {
|
||||||
|
return Err(ToolError::ExecutionFailed(format!(
|
||||||
|
"WorkerRestore returned {:?}: {}",
|
||||||
|
restored.result.state, response.body
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
if operation == WorkerOperation::Stop
|
if operation == WorkerOperation::Stop
|
||||||
&& let Ok(summary) = serde_json::from_str::<SubWorkerStopSummary>(&response.body)
|
&& let Ok(summary) = serde_json::from_str::<SubWorkerStopSummary>(&response.body)
|
||||||
{
|
{
|
||||||
@@ -1150,7 +1199,7 @@ fn worker_tool_contribution(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
WorkerOperation::Restore => {
|
WorkerOperation::Restore => {
|
||||||
definition::<WorkerTargetInput>(operation, control, runtime_worker_control)
|
definition::<WorkerRestoreInput>(operation, control, runtime_worker_control)
|
||||||
}
|
}
|
||||||
WorkerOperation::Remove => {
|
WorkerOperation::Remove => {
|
||||||
definition::<WorkerRemoveInput>(operation, control, runtime_worker_control)
|
definition::<WorkerRemoveInput>(operation, control, runtime_worker_control)
|
||||||
@@ -1410,6 +1459,95 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn worker_restore_forwards_exact_observation_and_request_identity() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::default());
|
||||||
|
let control = test_control(client.clone());
|
||||||
|
for _ in 0..2 {
|
||||||
|
control
|
||||||
|
.restore_worker(
|
||||||
|
"runtime/a".into(),
|
||||||
|
"worker/b".into(),
|
||||||
|
server_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: "same-observation".into(),
|
||||||
|
request_id: "same-request".into(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let requests = client.requests.lock().unwrap();
|
||||||
|
assert_eq!(requests.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
requests[0].path,
|
||||||
|
"/api/w/workspace%2Ftest/worker-control/workers/runtime%2Fa/worker%2Fb/restore"
|
||||||
|
);
|
||||||
|
assert_eq!(requests[0].body, requests[1].body);
|
||||||
|
let body: serde_json::Value =
|
||||||
|
serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
body,
|
||||||
|
serde_json::json!({"expected_observation_token":"same-observation","request_id":"same-request"})
|
||||||
|
);
|
||||||
|
assert!(serde_json::from_str::<WorkerRestoreInput>(r#"{"subject":{"kind":"runtime_worker","runtime_id":"runtime-a","worker_id":"worker-a"}}"#).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn worker_restore_reports_retry_identity_and_reuses_it_across_explicit_tool_calls() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::default());
|
||||||
|
let tool = WorkspaceWorkerTool {
|
||||||
|
operation: WorkerOperation::Restore,
|
||||||
|
control: test_control(client.clone()),
|
||||||
|
runtime_worker_control: true,
|
||||||
|
};
|
||||||
|
let mut input = serde_json::json!({
|
||||||
|
"subject": {"kind":"runtime_worker", "runtime_id":"runtime-a", "worker_id":"worker-b"},
|
||||||
|
"expected_observation_token":"original-observation"
|
||||||
|
});
|
||||||
|
// The fixture deliberately returns malformed JSON shape: delivery may
|
||||||
|
// have succeeded, so retain the exact tuple rather than issuing new intent.
|
||||||
|
let first = tool
|
||||||
|
.execute(
|
||||||
|
&input.to_string(),
|
||||||
|
ToolExecutionContext::new("original-call", "batch", 0),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
first
|
||||||
|
.to_string()
|
||||||
|
.contains("request_id=worker-restore:original-call")
|
||||||
|
);
|
||||||
|
input["request_id"] = "worker-restore:original-call".into();
|
||||||
|
let second = tool
|
||||||
|
.execute(
|
||||||
|
&input.to_string(),
|
||||||
|
ToolExecutionContext::new("explicit-recovery-call", "batch", 1),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
second
|
||||||
|
.to_string()
|
||||||
|
.contains("request_id=worker-restore:original-call")
|
||||||
|
);
|
||||||
|
{
|
||||||
|
let requests = client.requests.lock().unwrap();
|
||||||
|
assert_eq!(requests.len(), 2);
|
||||||
|
assert_eq!(requests[0].body, requests[1].body);
|
||||||
|
}
|
||||||
|
input["request_id"] = "x".repeat(513).into();
|
||||||
|
assert!(
|
||||||
|
tool.execute(
|
||||||
|
&input.to_string(),
|
||||||
|
ToolExecutionContext::new("invalid", "batch", 2)
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(client.requests.lock().unwrap().len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn worker_spawn_forwards_typed_initial_submit_to_workspace_api() {
|
async fn worker_spawn_forwards_typed_initial_submit_to_workspace_api() {
|
||||||
let client = Arc::new(RecordingWorkspaceClient::default());
|
let client = Arc::new(RecordingWorkspaceClient::default());
|
||||||
|
|||||||
@@ -728,6 +728,7 @@ pub enum WorkspaceServerOperation {
|
|||||||
WorkerControlRestore {
|
WorkerControlRestore {
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
worker_id: String,
|
worker_id: String,
|
||||||
|
request: server_api::WorkerRestoreRequest,
|
||||||
},
|
},
|
||||||
WorkerObservationSessions,
|
WorkerObservationSessions,
|
||||||
WorkerObservationCapture(server_api::WorkerObservationSubjectRef),
|
WorkerObservationCapture(server_api::WorkerObservationSubjectRef),
|
||||||
@@ -802,11 +803,15 @@ fn workspace_server_operation_request(
|
|||||||
WorkspaceServerOperation::WorkerControlRestore {
|
WorkspaceServerOperation::WorkerControlRestore {
|
||||||
runtime_id,
|
runtime_id,
|
||||||
worker_id,
|
worker_id,
|
||||||
} => Ok(WorkspaceRequest::json(
|
request,
|
||||||
WorkspaceRequestMethod::Post,
|
} => workspace_server_json_request(
|
||||||
format!("{base}/worker-control/workers/{runtime_id}/{worker_id}/restore"),
|
format!(
|
||||||
"".to_string(),
|
"{base}/worker-control/workers/{}/{}/restore",
|
||||||
)),
|
encode_workspace_path_segment(&runtime_id),
|
||||||
|
encode_workspace_path_segment(&worker_id)
|
||||||
|
),
|
||||||
|
&request,
|
||||||
|
),
|
||||||
WorkspaceServerOperation::WorkerObservationSessions => Ok(WorkspaceRequest::get(format!(
|
WorkspaceServerOperation::WorkerObservationSessions => Ok(WorkspaceRequest::get(format!(
|
||||||
"{base}/worker-observation/sessions"
|
"{base}/worker-observation/sessions"
|
||||||
))),
|
))),
|
||||||
|
|||||||
@@ -242,6 +242,8 @@ pub struct InternalWorkerImplementationSummary {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct InternalWorkerSummary {
|
pub struct InternalWorkerSummary {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub restore_observation_token: Option<String>,
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
pub worker: RuntimeWorkerRef,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub host_id: String,
|
pub host_id: String,
|
||||||
@@ -356,6 +358,7 @@ pub(crate) fn workspace_worker_summary(
|
|||||||
workdir_attachments: Vec<server_api::WorkerWorkdirAttachmentSummary>,
|
workdir_attachments: Vec<server_api::WorkerWorkdirAttachmentSummary>,
|
||||||
) -> server_api::WorkerSummary {
|
) -> server_api::WorkerSummary {
|
||||||
server_api::WorkerSummary {
|
server_api::WorkerSummary {
|
||||||
|
restore_observation_token: summary.restore_observation_token,
|
||||||
runtime_id: summary.worker.runtime_id,
|
runtime_id: summary.worker.runtime_id,
|
||||||
worker_id: summary.worker.worker_id,
|
worker_id: summary.worker.worker_id,
|
||||||
resource_key,
|
resource_key,
|
||||||
@@ -834,11 +837,17 @@ impl RuntimeRegistryError {
|
|||||||
runtime_id,
|
runtime_id,
|
||||||
code,
|
code,
|
||||||
message,
|
message,
|
||||||
} => Error::RuntimeOperationFailed {
|
} => {
|
||||||
runtime_id,
|
if code == "restore_observation_conflict" {
|
||||||
code,
|
Error::RestoreObservationConflict
|
||||||
message,
|
} else {
|
||||||
},
|
Error::RuntimeOperationFailed {
|
||||||
|
runtime_id,
|
||||||
|
code,
|
||||||
|
message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -938,7 +947,29 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
|
|
||||||
fn worker(&self, worker_id: &str) -> WorkerLookupResult;
|
fn worker(&self, worker_id: &str) -> WorkerLookupResult;
|
||||||
|
|
||||||
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
|
fn coordinate_worker_restore(
|
||||||
|
&self,
|
||||||
|
_worker_id: &str,
|
||||||
|
_request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||||
|
) -> Result<
|
||||||
|
(
|
||||||
|
Option<InternalWorkerRestoreResult>,
|
||||||
|
Option<runtime_api::WorkerRestorePreparation>,
|
||||||
|
),
|
||||||
|
RuntimeDiagnostic,
|
||||||
|
> {
|
||||||
|
Err(diagnostic(
|
||||||
|
"worker_restore_coordination_unsupported",
|
||||||
|
HostDiagnosticSeverity::Error,
|
||||||
|
"Runtime does not support guarded Restore coordination",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_worker(
|
||||||
|
&self,
|
||||||
|
worker_id: &str,
|
||||||
|
_request: runtime_api::WorkerRestoreRequest,
|
||||||
|
) -> InternalWorkerRestoreResult {
|
||||||
InternalWorkerRestoreResult {
|
InternalWorkerRestoreResult {
|
||||||
state: server_api::WorkerRestoreState::Rejected,
|
state: server_api::WorkerRestoreState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
@@ -1602,16 +1633,39 @@ impl RuntimeRegistry {
|
|||||||
Ok(worker)
|
Ok(worker)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn coordinate_worker_restore(
|
||||||
|
&self,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||||
|
) -> Result<
|
||||||
|
(
|
||||||
|
Option<InternalWorkerRestoreResult>,
|
||||||
|
Option<runtime_api::WorkerRestorePreparation>,
|
||||||
|
),
|
||||||
|
RuntimeRegistryError,
|
||||||
|
> {
|
||||||
|
validate_backend_identifier("runtime_id", &worker.runtime_id)?;
|
||||||
|
validate_backend_identifier("worker_id", &worker.worker_id)?;
|
||||||
|
self.runtime(&worker.runtime_id)?
|
||||||
|
.coordinate_worker_restore(&worker.worker_id, request)
|
||||||
|
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
|
||||||
|
runtime_id: worker.runtime_id.clone(),
|
||||||
|
code: error.code,
|
||||||
|
message: error.message,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn restore_worker(
|
pub fn restore_worker(
|
||||||
&self,
|
&self,
|
||||||
worker: &RuntimeWorkerRef,
|
worker: &RuntimeWorkerRef,
|
||||||
|
request: runtime_api::WorkerRestoreRequest,
|
||||||
) -> Result<InternalWorkerRestoreResult, RuntimeRegistryError> {
|
) -> Result<InternalWorkerRestoreResult, RuntimeRegistryError> {
|
||||||
let runtime_id = worker.runtime_id.as_str();
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
let worker_id = worker.worker_id.as_str();
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
Ok(runtime.restore_worker(worker_id))
|
Ok(runtime.restore_worker(worker_id, request))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn replace_worker_workspace_api(
|
pub fn replace_worker_workspace_api(
|
||||||
@@ -2372,6 +2426,7 @@ impl EmbeddedWorkerRuntime {
|
|||||||
true,
|
true,
|
||||||
);
|
);
|
||||||
InternalWorkerSummary {
|
InternalWorkerSummary {
|
||||||
|
restore_observation_token: summary.restore_observation_token.clone(),
|
||||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||||
host_id: self.host_id.clone(),
|
host_id: self.host_id.clone(),
|
||||||
display_name: display.display_name.clone(),
|
display_name: display.display_name.clone(),
|
||||||
@@ -2416,6 +2471,7 @@ impl EmbeddedWorkerRuntime {
|
|||||||
true,
|
true,
|
||||||
);
|
);
|
||||||
InternalWorkerSummary {
|
InternalWorkerSummary {
|
||||||
|
restore_observation_token: detail.restore_observation_token.clone(),
|
||||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||||
host_id: self.host_id.clone(),
|
host_id: self.host_id.clone(),
|
||||||
display_name: display.display_name.clone(),
|
display_name: display.display_name.clone(),
|
||||||
@@ -2582,7 +2638,48 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
|
fn coordinate_worker_restore(
|
||||||
|
&self,
|
||||||
|
worker_id: &str,
|
||||||
|
request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||||
|
) -> Result<
|
||||||
|
(
|
||||||
|
Option<InternalWorkerRestoreResult>,
|
||||||
|
Option<runtime_api::WorkerRestorePreparation>,
|
||||||
|
),
|
||||||
|
RuntimeDiagnostic,
|
||||||
|
> {
|
||||||
|
let worker_ref = self.worker_ref(worker_id).ok_or_else(|| {
|
||||||
|
diagnostic(
|
||||||
|
"worker_not_found",
|
||||||
|
HostDiagnosticSeverity::Warning,
|
||||||
|
"Unknown Worker",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let (result, preparation) = self
|
||||||
|
.runtime
|
||||||
|
.coordinate_worker_restore_operation(&worker_ref, request, None)
|
||||||
|
.map_err(|error| embedded_runtime_diagnostic(&error))?;
|
||||||
|
Ok((
|
||||||
|
result.map(|result| InternalWorkerRestoreResult {
|
||||||
|
state: result.state,
|
||||||
|
worker: result.worker.map(|detail| self.map_worker_detail(detail)),
|
||||||
|
diagnostics: match (result.reason_code, result.message) {
|
||||||
|
(Some(code), Some(message)) => {
|
||||||
|
vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)]
|
||||||
|
}
|
||||||
|
_ => Vec::new(),
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
preparation,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_worker(
|
||||||
|
&self,
|
||||||
|
worker_id: &str,
|
||||||
|
request: runtime_api::WorkerRestoreRequest,
|
||||||
|
) -> InternalWorkerRestoreResult {
|
||||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||||
return InternalWorkerRestoreResult {
|
return InternalWorkerRestoreResult {
|
||||||
state: server_api::WorkerRestoreState::Rejected,
|
state: server_api::WorkerRestoreState::Rejected,
|
||||||
@@ -2594,7 +2691,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
)],
|
)],
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
match self.runtime.restore_worker_operation(&worker_ref) {
|
match self.runtime.restore_worker_operation(&worker_ref, request) {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
let diagnostics = match (result.reason_code, result.message) {
|
let diagnostics = match (result.reason_code, result.message) {
|
||||||
(Some(code), Some(message)) => {
|
(Some(code), Some(message)) => {
|
||||||
@@ -2608,6 +2705,17 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
diagnostics,
|
diagnostics,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Err(worker_runtime::error::RuntimeError::RestoreObservationConflict { .. }) => {
|
||||||
|
InternalWorkerRestoreResult {
|
||||||
|
state: server_api::WorkerRestoreState::Rejected,
|
||||||
|
worker: None,
|
||||||
|
diagnostics: vec![diagnostic(
|
||||||
|
"restore_observation_conflict",
|
||||||
|
HostDiagnosticSeverity::Warning,
|
||||||
|
"Restore observation changed; refresh before a new intent".to_string(),
|
||||||
|
)],
|
||||||
|
}
|
||||||
|
}
|
||||||
Err(err) => InternalWorkerRestoreResult {
|
Err(err) => InternalWorkerRestoreResult {
|
||||||
state: server_api::WorkerRestoreState::ReconciliationRequired,
|
state: server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
worker: None,
|
worker: None,
|
||||||
@@ -4375,6 +4483,7 @@ impl RemoteWorkerRuntime {
|
|||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
InternalWorkerSummary {
|
InternalWorkerSummary {
|
||||||
|
restore_observation_token: summary.restore_observation_token.clone(),
|
||||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||||
host_id: self.host_id.clone(),
|
host_id: self.host_id.clone(),
|
||||||
display_name: display.display_name.clone(),
|
display_name: display.display_name.clone(),
|
||||||
@@ -4417,6 +4526,7 @@ impl RemoteWorkerRuntime {
|
|||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
InternalWorkerSummary {
|
InternalWorkerSummary {
|
||||||
|
restore_observation_token: detail.restore_observation_token.clone(),
|
||||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||||
host_id: self.host_id.clone(),
|
host_id: self.host_id.clone(),
|
||||||
display_name: display.display_name.clone(),
|
display_name: display.display_name.clone(),
|
||||||
@@ -4708,17 +4818,58 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
|
fn coordinate_worker_restore(
|
||||||
|
&self,
|
||||||
|
worker_id: &str,
|
||||||
|
request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||||
|
) -> Result<
|
||||||
|
(
|
||||||
|
Option<InternalWorkerRestoreResult>,
|
||||||
|
Option<runtime_api::WorkerRestorePreparation>,
|
||||||
|
),
|
||||||
|
RuntimeDiagnostic,
|
||||||
|
> {
|
||||||
|
let worker_id = worker_id.to_string();
|
||||||
|
let response = self.run_runtime_api(
|
||||||
|
self.request_timeout,
|
||||||
|
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||||
|
move |client| async move { client.coordinate_worker_restore(worker_id, request).await },
|
||||||
|
)?;
|
||||||
|
let result = response
|
||||||
|
.result
|
||||||
|
.map(|result| -> Result<_, RuntimeDiagnostic> {
|
||||||
|
let worker = result
|
||||||
|
.worker
|
||||||
|
.map(runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>)
|
||||||
|
.transpose()?;
|
||||||
|
let state =
|
||||||
|
runtime_contract_convert::<_, server_api::WorkerRestoreState>(result.state)?;
|
||||||
|
Ok(InternalWorkerRestoreResult {
|
||||||
|
state,
|
||||||
|
worker: worker.map(|detail| self.map_worker_detail(detail)),
|
||||||
|
diagnostics: match (result.reason_code, result.message) {
|
||||||
|
(Some(code), Some(message)) => {
|
||||||
|
vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)]
|
||||||
|
}
|
||||||
|
_ => Vec::new(),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
|
Ok((result, response.preparation))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_worker(
|
||||||
|
&self,
|
||||||
|
worker_id: &str,
|
||||||
|
request: runtime_api::WorkerRestoreRequest,
|
||||||
|
) -> InternalWorkerRestoreResult {
|
||||||
let worker_id_owned = worker_id.to_string();
|
let worker_id_owned = worker_id.to_string();
|
||||||
match self
|
match self
|
||||||
.run_runtime_api(
|
.run_runtime_api(
|
||||||
self.request_timeout,
|
self.request_timeout,
|
||||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||||
move |client| async move {
|
move |client| async move { client.restore_worker(worker_id_owned, request).await },
|
||||||
client
|
|
||||||
.restore_worker(worker_id_owned, runtime_api::EmptyObjectRequest::default())
|
|
||||||
.await
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
.and_then(|value| {
|
.and_then(|value| {
|
||||||
let worker = value
|
let worker = value
|
||||||
@@ -5886,6 +6037,11 @@ fn sanitize_embedded_execution_detail(summary: &str, message: &str) -> String {
|
|||||||
|
|
||||||
fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnostic {
|
fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnostic {
|
||||||
match error {
|
match error {
|
||||||
|
EmbeddedRuntimeError::RestoreObservationConflict { .. } => diagnostic(
|
||||||
|
"restore_observation_conflict",
|
||||||
|
HostDiagnosticSeverity::Warning,
|
||||||
|
"Restore observation changed; refresh before a new intent".to_string(),
|
||||||
|
),
|
||||||
EmbeddedRuntimeError::RuntimeStopped => diagnostic(
|
EmbeddedRuntimeError::RuntimeStopped => diagnostic(
|
||||||
"embedded_runtime_stopped",
|
"embedded_runtime_stopped",
|
||||||
HostDiagnosticSeverity::Warning,
|
HostDiagnosticSeverity::Warning,
|
||||||
@@ -6292,6 +6448,7 @@ fn worker_spawn_intent_label(intent: &WorkerSpawnIntent) -> &'static str {
|
|||||||
pub fn placeholder_worker(host_id: impl Into<String>) -> InternalWorkerSummary {
|
pub fn placeholder_worker(host_id: impl Into<String>) -> InternalWorkerSummary {
|
||||||
let host_id = host_id.into();
|
let host_id = host_id.into();
|
||||||
InternalWorkerSummary {
|
InternalWorkerSummary {
|
||||||
|
restore_observation_token: None,
|
||||||
worker: RuntimeWorkerRef::new("placeholder", "worker-placeholder"),
|
worker: RuntimeWorkerRef::new("placeholder", "worker-placeholder"),
|
||||||
host_id,
|
host_id,
|
||||||
display_name: "Worker runtime actions are not implemented".to_string(),
|
display_name: "Worker runtime actions are not implemented".to_string(),
|
||||||
@@ -6369,7 +6526,14 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let result = provider.restore_worker("worker-transport-uncertain");
|
let result = provider.restore_worker(
|
||||||
|
"worker-transport-uncertain",
|
||||||
|
runtime_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: "observation".into(),
|
||||||
|
request_id: "transport-uncertain".into(),
|
||||||
|
preparation: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
result.state,
|
result.state,
|
||||||
@@ -7052,6 +7216,7 @@ mod tests {
|
|||||||
runtime_id: runtime_id.to_string(),
|
runtime_id: runtime_id.to_string(),
|
||||||
host_id: host_id.to_string(),
|
host_id: host_id.to_string(),
|
||||||
workers: vec![InternalWorkerSummary {
|
workers: vec![InternalWorkerSummary {
|
||||||
|
restore_observation_token: None,
|
||||||
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
|
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
|
||||||
host_id: host_id.to_string(),
|
host_id: host_id.to_string(),
|
||||||
display_name: label.to_string(),
|
display_name: label.to_string(),
|
||||||
@@ -7698,7 +7863,16 @@ mod tests {
|
|||||||
|
|
||||||
let detail = registry.worker(&worker.worker).unwrap();
|
let detail = registry.worker(&worker.worker).unwrap();
|
||||||
|
|
||||||
let json = serde_json::to_string(&(embedded_summary, worker, input, detail)).unwrap();
|
// The public observation generation is intentionally exposed, unlike
|
||||||
|
// private authentication tokens. Whitelist its key, not arbitrary token
|
||||||
|
// values or other token-bearing fields.
|
||||||
|
for summary in [&worker, &detail] {
|
||||||
|
let token = summary.restore_observation_token.as_ref().unwrap();
|
||||||
|
assert_eq!(uuid::Uuid::parse_str(token).unwrap().get_version_num(), 7);
|
||||||
|
}
|
||||||
|
let json = serde_json::to_string(&(embedded_summary, worker, input, detail))
|
||||||
|
.unwrap()
|
||||||
|
.replace("\"restore_observation_token\":", "\"restore_observation\":");
|
||||||
for forbidden in [
|
for forbidden in [
|
||||||
"/workspace/project",
|
"/workspace/project",
|
||||||
"metadata.json",
|
"metadata.json",
|
||||||
|
|||||||
@@ -139,6 +139,10 @@ pub enum Error {
|
|||||||
RuntimeBindingFingerprintConflict { fingerprint: String },
|
RuntimeBindingFingerprintConflict { fingerprint: String },
|
||||||
#[error("Runtime binding was not found for {runtime_id}")]
|
#[error("Runtime binding was not found for {runtime_id}")]
|
||||||
RuntimeBindingNotFound { runtime_id: String },
|
RuntimeBindingNotFound { runtime_id: String },
|
||||||
|
#[error(
|
||||||
|
"restore_observation_conflict: Restore observation changed; refresh before a new intent"
|
||||||
|
)]
|
||||||
|
RestoreObservationConflict,
|
||||||
#[error("Repository conflict: {0}")]
|
#[error("Repository conflict: {0}")]
|
||||||
RepositoryConflict(String),
|
RepositoryConflict(String),
|
||||||
#[error("Registry inconsistency: {0}")]
|
#[error("Registry inconsistency: {0}")]
|
||||||
|
|||||||
@@ -61,6 +61,16 @@ impl Drop for StoredResource {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl BackendResourceBroker {
|
impl BackendResourceBroker {
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn repository_ssh_access_handle_count(&self) -> usize {
|
||||||
|
self.resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.values()
|
||||||
|
.filter(|resource| resource.handle.kind == BackendResourceKind::RepositorySshAccess)
|
||||||
|
.count()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn issue_profile_source_archive_handle(
|
pub fn issue_profile_source_archive_handle(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: impl Into<String>,
|
workspace_id: impl Into<String>,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -250,7 +250,13 @@ async fn offline_handle_can_pin_but_deleted_handle_cannot_mutate_or_connect() {
|
|||||||
);
|
);
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
handle
|
handle
|
||||||
.restore(&WorkerOperationContext::Backend)
|
.restore(
|
||||||
|
&WorkerOperationContext::Backend,
|
||||||
|
server_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: "stale".into(),
|
||||||
|
request_id: "removed-handle".into(),
|
||||||
|
}
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap_err()
|
.unwrap_err()
|
||||||
.error,
|
.error,
|
||||||
@@ -374,7 +380,17 @@ async fn worker_control_handle_rechecks_revoked_grants_for_each_operation() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
handle.restore(&context).await.unwrap_err().error,
|
handle
|
||||||
|
.restore(
|
||||||
|
&context,
|
||||||
|
server_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: "stale".into(),
|
||||||
|
request_id: "revoked-grant".into(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err()
|
||||||
|
.error,
|
||||||
Error::UnknownWorker { .. }
|
Error::UnknownWorker { .. }
|
||||||
));
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
@@ -682,7 +698,15 @@ async fn generic_management_aliases_cannot_bypass_worker_grants_or_target_bound_
|
|||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
service
|
service
|
||||||
.runtime_worker_restore_alias(context(), "offline-runtime".into(), "subject".into())
|
.runtime_worker_restore_alias(
|
||||||
|
context(),
|
||||||
|
"offline-runtime".into(),
|
||||||
|
"subject".into(),
|
||||||
|
server_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: "old".into(),
|
||||||
|
request_id: "unauthorized".into(),
|
||||||
|
}
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.is_err()
|
.is_err()
|
||||||
);
|
);
|
||||||
@@ -862,3 +886,696 @@ async fn browser_protocol_connection_and_cleanup_reject_cookie_origin_mismatch()
|
|||||||
assert!(matches!(result.error, Error::WorkspacePermissionDenied(_)));
|
assert!(matches!(result.error, Error::WorkspacePermissionDenied(_)));
|
||||||
assert!(execution.protocol_methods().is_empty());
|
assert!(execution.protocol_methods().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn restore_request_for(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
request_id: &str,
|
||||||
|
) -> server_api::WorkerRestoreRequest {
|
||||||
|
server_api::WorkerRestoreRequest {
|
||||||
|
expected_observation_token: api
|
||||||
|
.runtime
|
||||||
|
.worker(worker)
|
||||||
|
.unwrap()
|
||||||
|
.restore_observation_token
|
||||||
|
.unwrap(),
|
||||||
|
request_id: request_id.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_guard_rejects_stale_list_and_stopped_restored_stopped_aba() {
|
||||||
|
let workspace = tempfile::tempdir().unwrap();
|
||||||
|
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||||
|
execution.accept_restores();
|
||||||
|
let identity = spawn_ticket_check_source(&api, "restore-guard-aba");
|
||||||
|
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||||
|
handle
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let original = restore_request_for(&api, &identity, "restore-original");
|
||||||
|
let stale_list = server_api::WorkerRestoreRequest {
|
||||||
|
request_id: "restore-stale-list".into(),
|
||||||
|
..original.clone()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, original.clone())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
let conflict = handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, stale_list.clone())
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(conflict.error, Error::RestoreObservationConflict));
|
||||||
|
let typed = conflict.into_repository_api_error();
|
||||||
|
assert_eq!(
|
||||||
|
api_error_status(&Error::RestoreObservationConflict),
|
||||||
|
StatusCode::CONFLICT
|
||||||
|
);
|
||||||
|
assert_eq!(typed.error, "restore_observation_conflict");
|
||||||
|
handle
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_ne!(
|
||||||
|
original.expected_observation_token,
|
||||||
|
api.runtime
|
||||||
|
.worker(&identity)
|
||||||
|
.unwrap()
|
||||||
|
.restore_observation_token
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, stale_list)
|
||||||
|
.await
|
||||||
|
.unwrap_err()
|
||||||
|
.error,
|
||||||
|
Error::RestoreObservationConflict
|
||||||
|
));
|
||||||
|
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
|
||||||
|
assert!(
|
||||||
|
execution.contexts.lock().unwrap().is_empty(),
|
||||||
|
"conflict must not install execution"
|
||||||
|
);
|
||||||
|
// The original request is result replay, not a new intent; it stays accepted
|
||||||
|
// without resurrecting the Worker after a subsequent stop.
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, original)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
|
||||||
|
assert!(execution.contexts.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_test_workdir(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
id: &str,
|
||||||
|
source: WorkdirRegistrySource,
|
||||||
|
) -> WorkdirRegistryRecord {
|
||||||
|
WorkdirRegistryRecord {
|
||||||
|
workspace_id: api.workspace_id().into(),
|
||||||
|
workdir_id: id.into(),
|
||||||
|
display_name: None,
|
||||||
|
source,
|
||||||
|
creation_selector: None,
|
||||||
|
creation_ref: None,
|
||||||
|
creation_tree: None,
|
||||||
|
current_selector: None,
|
||||||
|
current_ref: None,
|
||||||
|
current_tree: None,
|
||||||
|
observed_at_epoch_seconds: None,
|
||||||
|
materialization_status: "present".into(),
|
||||||
|
cleanliness: "clean".into(),
|
||||||
|
created_at: TEST_CREATED_AT.into(),
|
||||||
|
updated_at: TEST_CREATED_AT.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn attach_restore_test_workdir(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
workdir: &WorkdirRegistryRecord,
|
||||||
|
) {
|
||||||
|
api.store
|
||||||
|
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
|
||||||
|
connection_id: String::new(),
|
||||||
|
workspace_id: api.workspace_id().into(),
|
||||||
|
worker: worker.clone(),
|
||||||
|
workdir_id: workdir.workdir_id.clone(),
|
||||||
|
alias: workdir.workdir_id.clone(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
linked_at: TEST_CREATED_AT.into(),
|
||||||
|
unlinked_at: None,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn displace_restore_singleton(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
replacement: &RuntimeWorkerRef,
|
||||||
|
) {
|
||||||
|
api.config_store.with_conn(|conn| {
|
||||||
|
conn.execute("UPDATE worker_create_reservations SET singleton_key='restore-test-singleton', singleton_generation=1 WHERE workspace_id=?1 AND worker_id=?2",
|
||||||
|
rusqlite::params![api.workspace_id(), worker.worker_id])?;
|
||||||
|
conn.execute("INSERT INTO worker_singleton_owners(workspace_id,singleton_key,runtime_id,worker_id,generation,created_at,updated_at) VALUES(?1,'restore-test-singleton',?2,?3,2,?4,?4)",
|
||||||
|
rusqlite::params![api.workspace_id(), replacement.runtime_id, replacement.worker_id, TEST_CREATED_AT])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
assert!(
|
||||||
|
api.store
|
||||||
|
.require_current_worker_singleton_owner(api.workspace_id(), worker)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn add_offline_restore_workdir(api: &WorkspaceApi, worker: &RuntimeWorkerRef) {
|
||||||
|
let record = restore_test_workdir(
|
||||||
|
api,
|
||||||
|
"restore-offline",
|
||||||
|
WorkdirRegistrySource::ExternalGrant {
|
||||||
|
grant_id: "restore-offline-grant".into(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
api.store
|
||||||
|
.create_external_workdir_grant(
|
||||||
|
&crate::store::ExternalWorkdirGrantRecord {
|
||||||
|
grant_id: "restore-offline-grant".into(),
|
||||||
|
workspace_id: api.workspace_id().into(),
|
||||||
|
workdir_id: record.workdir_id.clone(),
|
||||||
|
provider_instance_id: "offline-provider".into(),
|
||||||
|
display_name: "Offline grant".into(),
|
||||||
|
permissions: "read_only".into(),
|
||||||
|
created_by: "owner-account".into(),
|
||||||
|
created_at: TEST_CREATED_AT.into(),
|
||||||
|
expires_at: None,
|
||||||
|
generation: 1,
|
||||||
|
status: "offline".into(),
|
||||||
|
updated_at: TEST_CREATED_AT.into(),
|
||||||
|
},
|
||||||
|
&record,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
attach_restore_test_workdir(api, worker, &record);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_guard_stale_ssh_intent_does_not_issue_workspace_resource() {
|
||||||
|
let workspace = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api_with_remote_repository(workspace.path()).await;
|
||||||
|
let identity = spawn_ticket_check_source(&api, "restore-stale-ssh");
|
||||||
|
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||||
|
let stale = restore_request_for(&api, &identity, "stale-ssh-intent");
|
||||||
|
handle
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
api.repository_secrets
|
||||||
|
.generate_credential(
|
||||||
|
api.workspace_id(),
|
||||||
|
GenerateRepositorySshCredentialRequest {
|
||||||
|
operation_id: "restore-default-key".into(),
|
||||||
|
credential_id:
|
||||||
|
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(),
|
||||||
|
name: "Restore test default".into(),
|
||||||
|
},
|
||||||
|
"owner-account",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let public_key = api
|
||||||
|
.repository_secrets
|
||||||
|
.credential_public_key(
|
||||||
|
api.workspace_id(),
|
||||||
|
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID,
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key;
|
||||||
|
api.repository_secrets
|
||||||
|
.put_host_trust(
|
||||||
|
api.workspace_id(),
|
||||||
|
PutRepositorySshHostTrustRequest {
|
||||||
|
operation_id: "restore-ssh-trust".into(),
|
||||||
|
host_trust_id: "restore-ssh-host".into(),
|
||||||
|
hostname: "example.invalid".into(),
|
||||||
|
port: 22,
|
||||||
|
host_key: public_key,
|
||||||
|
expected_revision: None,
|
||||||
|
},
|
||||||
|
"owner-account",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let record = restore_test_workdir(
|
||||||
|
&api,
|
||||||
|
"restore-ssh-workdir",
|
||||||
|
WorkdirRegistrySource::Repository {
|
||||||
|
runtime_id: identity.runtime_id.clone(),
|
||||||
|
repository_id: test_repository_id(&api),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
api.store.upsert_workdir_registry(&record).unwrap();
|
||||||
|
attach_restore_test_workdir(&api, &identity, &record);
|
||||||
|
// Prove this is a fully issuable SSH fixture; the old pre-admission path would mint another handle.
|
||||||
|
assert!(
|
||||||
|
repository_access_request_for_workdir(
|
||||||
|
&api,
|
||||||
|
&identity.runtime_id,
|
||||||
|
&record.workdir_id,
|
||||||
|
"restore-ssh-positive-control"
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
let before = api.resource_broker.repository_ssh_access_handle_count();
|
||||||
|
assert!(before > 0);
|
||||||
|
let conflict = handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, stale)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(conflict.error, Error::RestoreObservationConflict));
|
||||||
|
assert_eq!(
|
||||||
|
api.resource_broker.repository_ssh_access_handle_count(),
|
||||||
|
before
|
||||||
|
);
|
||||||
|
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_guard_historical_replay_ignores_new_eligibility_but_rechecks_auth() {
|
||||||
|
let workspace = tempfile::tempdir().unwrap();
|
||||||
|
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||||
|
execution.accept_restores();
|
||||||
|
let identity = spawn_ticket_check_source(&api, "restore-historical");
|
||||||
|
let replacement = spawn_ticket_check_source(&api, "restore-replacement");
|
||||||
|
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||||
|
handle
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let request = restore_request_for(&api, &identity, "restore-historical-request");
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
handle
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
displace_restore_singleton(&api, &identity, &replacement);
|
||||||
|
add_offline_restore_workdir(&api, &identity);
|
||||||
|
let before = execution.restore_operations.lock().unwrap().len();
|
||||||
|
let replay = handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(replay.state, server_api::WorkerRestoreState::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
replay.worker.unwrap().state,
|
||||||
|
"stopped",
|
||||||
|
"never return the old idle receipt summary"
|
||||||
|
);
|
||||||
|
assert_eq!(execution.restore_operations.lock().unwrap().len(), before);
|
||||||
|
let projected = restore_runtime_worker_with_context(
|
||||||
|
api.clone(),
|
||||||
|
identity.runtime_id.clone(),
|
||||||
|
identity.worker_id.clone(),
|
||||||
|
WorkerOperationContext::Backend,
|
||||||
|
request.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.0;
|
||||||
|
assert_eq!(
|
||||||
|
projected.result.worker.unwrap().state,
|
||||||
|
"unavailable",
|
||||||
|
"current fenced projection, never old idle receipt"
|
||||||
|
);
|
||||||
|
let (mut browser, _) = browser_operation_context(&api).await;
|
||||||
|
let WorkerOperationContext::Browser { headers, .. } = &mut browser else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
headers.insert(ORIGIN, "https://untrusted.example".parse().unwrap());
|
||||||
|
assert!(matches!(
|
||||||
|
handle.restore(&browser, request).await.unwrap_err().error,
|
||||||
|
Error::WorkspacePermissionDenied(_)
|
||||||
|
));
|
||||||
|
assert_eq!(execution.restore_operations.lock().unwrap().len(), before);
|
||||||
|
assert!(
|
||||||
|
handle
|
||||||
|
.restore(
|
||||||
|
&WorkerOperationContext::Backend,
|
||||||
|
restore_request_for(&api, &identity, "new-ineligible-intent")
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_guard_pending_recovery_ignores_new_eligibility_and_reuses_operation() {
|
||||||
|
let workspace = tempfile::tempdir().unwrap();
|
||||||
|
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||||
|
execution.accept_restores();
|
||||||
|
let identity = spawn_ticket_check_source(&api, "restore-pending");
|
||||||
|
let replacement = spawn_ticket_check_source(&api, "restore-pending-replacement");
|
||||||
|
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||||
|
handle
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
*execution.restore_failure.lock().unwrap() = Some(
|
||||||
|
worker_runtime::execution::WorkerExecutionSpawnResult::Errored(
|
||||||
|
worker_runtime::execution::WorkerExecutionResult::rejected(
|
||||||
|
worker_runtime::execution::WorkerExecutionOperation::Restore,
|
||||||
|
"uncertain launch",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let request = restore_request_for(&api, &identity, "restore-pending-request");
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let original = execution.restore_operations.lock().unwrap()[0];
|
||||||
|
displace_restore_singleton(&api, &identity, &replacement);
|
||||||
|
add_offline_restore_workdir(&api, &identity);
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
execution.reconcile_operations.lock().unwrap().as_slice(),
|
||||||
|
&[original]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
execution
|
||||||
|
.restore_operations
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.all(|id| id == &original)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_guard_internal_intent_survives_client_store_restart_without_replacing_token() {
|
||||||
|
let workspace = tempfile::tempdir().unwrap();
|
||||||
|
let (mut api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||||
|
execution.accept_restores();
|
||||||
|
let identity = spawn_ticket_check_source(&api, "restore-internal-restart");
|
||||||
|
WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let observed = api.runtime.worker(&identity).unwrap();
|
||||||
|
*execution.restore_failure.lock().unwrap() = Some(
|
||||||
|
worker_runtime::execution::WorkerExecutionSpawnResult::Errored(
|
||||||
|
worker_runtime::execution::WorkerExecutionResult::rejected(
|
||||||
|
worker_runtime::execution::WorkerExecutionOperation::Restore,
|
||||||
|
"uncertain launch",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
restore_internal_observed_worker(&api, &observed, "subject-restore".into())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let original = execution.restore_operations.lock().unwrap()[0];
|
||||||
|
let current = api.runtime.worker(&identity).unwrap();
|
||||||
|
assert_ne!(
|
||||||
|
observed.restore_observation_token,
|
||||||
|
current.restore_observation_token
|
||||||
|
);
|
||||||
|
// Reopen the durable caller journal, discarding all client-side memory of request identity.
|
||||||
|
api.config_store =
|
||||||
|
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
|
||||||
|
let pinned = api
|
||||||
|
.config_store
|
||||||
|
.pin_internal_worker_restore_intent(
|
||||||
|
api.workspace_id(),
|
||||||
|
&identity,
|
||||||
|
"subject-restore",
|
||||||
|
current.restore_observation_token.as_deref(),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
pinned.expected_observation_token,
|
||||||
|
observed.restore_observation_token.unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
restore_internal_observed_worker(&api, ¤t, "subject-restore".into())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
execution.reconcile_operations.lock().unwrap().as_slice(),
|
||||||
|
&[original]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
execution
|
||||||
|
.restore_operations
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.all(|id| id == &original)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_guard_workspace_interrupted_preparation_recovers_admitted_snapshot() {
|
||||||
|
let workspace = tempfile::tempdir().unwrap();
|
||||||
|
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||||
|
execution.accept_restores();
|
||||||
|
let identity = spawn_ticket_check_source(&api, "restore-prepare-interrupted");
|
||||||
|
let replacement = spawn_ticket_check_source(&api, "restore-prepare-replacement");
|
||||||
|
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||||
|
handle
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let request = restore_request_for(&api, &identity, "restore-prepare-interrupted-request");
|
||||||
|
let preparation = runtime_api::WorkerRestorePreparation {
|
||||||
|
workspace_api: Some(
|
||||||
|
runtime_contract_restore(api.workspace_api_ref(&identity.runtime_id)).unwrap(),
|
||||||
|
),
|
||||||
|
workdir_attachments: Some(Vec::new()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let (result, retained) = api
|
||||||
|
.runtime
|
||||||
|
.coordinate_worker_restore(
|
||||||
|
&identity,
|
||||||
|
runtime_api::WorkerRestoreCoordinationRequest {
|
||||||
|
expected_observation_token: request.expected_observation_token.clone(),
|
||||||
|
request_id: request.request_id.clone(),
|
||||||
|
preparation: Some(preparation.clone()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(result.is_none());
|
||||||
|
assert_eq!(retained, Some(preparation));
|
||||||
|
assert!(execution.restore_operations.lock().unwrap().is_empty());
|
||||||
|
// Model interruption between admission and completion, then changed current eligibility.
|
||||||
|
displace_restore_singleton(&api, &identity, &replacement);
|
||||||
|
add_offline_restore_workdir(&api, &identity);
|
||||||
|
let restored = handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted);
|
||||||
|
assert_eq!(execution.restore_operations.lock().unwrap().len(), 1);
|
||||||
|
let original = execution.restore_operations.lock().unwrap()[0];
|
||||||
|
assert!(
|
||||||
|
execution.reconcile_operations.lock().unwrap().is_empty(),
|
||||||
|
"no execution existed to reconcile yet"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
execution.restore_operations.lock().unwrap().as_slice(),
|
||||||
|
&[original]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_workdir_owner() {
|
||||||
|
let workspace = tempfile::tempdir().unwrap();
|
||||||
|
let mut config = test_server_config(workspace.path());
|
||||||
|
let source = server_api::RepositorySource {
|
||||||
|
kind: server_api::RepositorySourceKind::Ssh,
|
||||||
|
uri: "git@example.invalid:owner/repository.git".into(),
|
||||||
|
};
|
||||||
|
config.repositories[0].source_fingerprint =
|
||||||
|
crate::repository_source::repository_source_fingerprint(&source);
|
||||||
|
config.repositories[0].source = source;
|
||||||
|
config.repositories[0].path = None;
|
||||||
|
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
|
||||||
|
let execution = Arc::new(DeterministicExecutionBackend::default());
|
||||||
|
execution.accept_restores();
|
||||||
|
let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let peer = WorkdirlessFixtureRuntime::default();
|
||||||
|
*peer.reject_repository_access.lock().unwrap() = true;
|
||||||
|
api.runtime.register_or_replace(peer.clone());
|
||||||
|
let identity = spawn_ticket_check_source(&api, "restore-cross-runtime-ssh");
|
||||||
|
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||||
|
handle
|
||||||
|
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let original_key = api
|
||||||
|
.repository_secrets
|
||||||
|
.generate_credential(
|
||||||
|
api.workspace_id(),
|
||||||
|
GenerateRepositorySshCredentialRequest {
|
||||||
|
operation_id: "cross-runtime-default-key".into(),
|
||||||
|
credential_id:
|
||||||
|
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(),
|
||||||
|
name: "Restore default".into(),
|
||||||
|
},
|
||||||
|
"owner-account",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let public_key = api
|
||||||
|
.repository_secrets
|
||||||
|
.credential_public_key(api.workspace_id(), &original_key.credential_id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key;
|
||||||
|
api.repository_secrets
|
||||||
|
.put_host_trust(
|
||||||
|
api.workspace_id(),
|
||||||
|
PutRepositorySshHostTrustRequest {
|
||||||
|
operation_id: "cross-runtime-trust".into(),
|
||||||
|
host_trust_id: "cross-runtime-host".into(),
|
||||||
|
hostname: "example.invalid".into(),
|
||||||
|
port: 22,
|
||||||
|
host_key: public_key,
|
||||||
|
expected_revision: None,
|
||||||
|
},
|
||||||
|
"owner-account",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let record = restore_test_workdir(
|
||||||
|
&api,
|
||||||
|
"cross-runtime-restore-workdir",
|
||||||
|
WorkdirRegistrySource::Repository {
|
||||||
|
runtime_id: WorkdirlessFixtureRuntime::RUNTIME_ID.into(),
|
||||||
|
repository_id: test_repository_id(&api),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
peer.set_workdir_repository(&record.workdir_id, &test_repository_id(&api));
|
||||||
|
api.store.upsert_workdir_registry(&record).unwrap();
|
||||||
|
attach_restore_test_workdir(&api, &identity, &record);
|
||||||
|
let request = restore_request_for(&api, &identity, "cross-runtime-restore-request");
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
execution.restore_operations.lock().unwrap().is_empty(),
|
||||||
|
"credential completion must precede execution"
|
||||||
|
);
|
||||||
|
let first = peer.repository_access_requests.lock().unwrap()[0].clone();
|
||||||
|
assert_eq!(
|
||||||
|
first.materialization.runtime_id,
|
||||||
|
WorkdirlessFixtureRuntime::RUNTIME_ID
|
||||||
|
);
|
||||||
|
assert_ne!(first.materialization.runtime_id, identity.runtime_id);
|
||||||
|
// Rotate current host trust while the accepted operation is uncertain. Its
|
||||||
|
// original credential/trust revisions, source and binding must remain pinned.
|
||||||
|
let alternate = api
|
||||||
|
.repository_secrets
|
||||||
|
.generate_credential(
|
||||||
|
api.workspace_id(),
|
||||||
|
GenerateRepositorySshCredentialRequest {
|
||||||
|
operation_id: "cross-runtime-alternate-key".into(),
|
||||||
|
credential_id: "cross-runtime-alternate".into(),
|
||||||
|
name: "Alternate".into(),
|
||||||
|
},
|
||||||
|
"owner-account",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let alternate_public = api
|
||||||
|
.repository_secrets
|
||||||
|
.credential_public_key(api.workspace_id(), &alternate.credential_id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key;
|
||||||
|
let original_trust_revision = first
|
||||||
|
.materialization
|
||||||
|
.ssh
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.host_trust_revision;
|
||||||
|
api.repository_secrets
|
||||||
|
.put_host_trust(
|
||||||
|
api.workspace_id(),
|
||||||
|
PutRepositorySshHostTrustRequest {
|
||||||
|
operation_id: "cross-runtime-trust-rotate".into(),
|
||||||
|
host_trust_id: "cross-runtime-host".into(),
|
||||||
|
hostname: "example.invalid".into(),
|
||||||
|
port: 22,
|
||||||
|
host_key: alternate_public,
|
||||||
|
expected_revision: Some(original_trust_revision),
|
||||||
|
},
|
||||||
|
"owner-account",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
*peer.reject_repository_access.lock().unwrap() = false;
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
let attempts = peer.repository_access_requests.lock().unwrap();
|
||||||
|
assert_eq!(attempts.len(), 2);
|
||||||
|
let second = &attempts[1];
|
||||||
|
assert_eq!(
|
||||||
|
second.materialization.operation_id,
|
||||||
|
first.materialization.operation_id
|
||||||
|
);
|
||||||
|
let mut normalized = second.clone();
|
||||||
|
let old_ssh = first.materialization.ssh.as_ref().unwrap();
|
||||||
|
let new_ssh = normalized.materialization.ssh.as_mut().unwrap();
|
||||||
|
assert_ne!(new_ssh.secret_resource.nonce, old_ssh.secret_resource.nonce);
|
||||||
|
new_ssh.secret_resource = old_ssh.secret_resource.clone();
|
||||||
|
new_ssh.expires_at_epoch_seconds = old_ssh.expires_at_epoch_seconds;
|
||||||
|
assert_eq!(normalized, first, "only opaque handle and expiry may renew");
|
||||||
|
drop(attempts);
|
||||||
|
assert_eq!(execution.restore_operations.lock().unwrap().len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
handle
|
||||||
|
.restore(&WorkerOperationContext::Backend, request)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
peer.repository_access_requests.lock().unwrap().len(),
|
||||||
|
2,
|
||||||
|
"terminal receipt must not renew access"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -291,10 +291,20 @@ impl WorkspaceWorker {
|
|||||||
pub(super) async fn restore(
|
pub(super) async fn restore(
|
||||||
&self,
|
&self,
|
||||||
context: &WorkerOperationContext,
|
context: &WorkerOperationContext,
|
||||||
|
request: server_api::WorkerRestoreRequest,
|
||||||
) -> ApiResult<InternalWorkerRestoreResult> {
|
) -> ApiResult<InternalWorkerRestoreResult> {
|
||||||
let _guard = self.authorize_operation(context, "restore").await?;
|
let _guard = self.authorize_operation(context, "restore").await?;
|
||||||
// Preserve singleton, Workdir, credentials, bindings and pending operation reconciliation.
|
self.restore_internal(request)
|
||||||
self.api.restore_workspace_worker(&self.identity)
|
}
|
||||||
|
|
||||||
|
/// Trusted Backend callers still supply an explicit observed intent. Runtime
|
||||||
|
/// admission is shared with browser/Tool callers; this is not an unguarded launch.
|
||||||
|
pub(super) fn restore_internal(
|
||||||
|
&self,
|
||||||
|
request: server_api::WorkerRestoreRequest,
|
||||||
|
) -> ApiResult<InternalWorkerRestoreResult> {
|
||||||
|
self.current_record()?;
|
||||||
|
self.api.restore_workspace_worker(&self.identity, request)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn stop(
|
pub(super) async fn stop(
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore;
|
|||||||
use crate::{Error, Result};
|
use crate::{Error, Result};
|
||||||
|
|
||||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||||
const LATEST_SCHEMA_VERSION: i64 = 79;
|
const LATEST_SCHEMA_VERSION: i64 = 80;
|
||||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||||
const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings";
|
const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings";
|
||||||
@@ -79,6 +79,8 @@ const ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME: &str =
|
|||||||
const BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME: &str =
|
const BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME: &str =
|
||||||
"durable terminal Backend Job Worker cleanup";
|
"durable terminal Backend Job Worker cleanup";
|
||||||
const WORKDIR_CONNECTION_ID_MIGRATION_NAME: &str = "durable Workdir attachment connection identity";
|
const WORKDIR_CONNECTION_ID_MIGRATION_NAME: &str = "durable Workdir attachment connection identity";
|
||||||
|
const WORKER_RESTORE_INTENTS_MIGRATION_NAME: &str =
|
||||||
|
"durable internal Worker Restore request identity";
|
||||||
const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7;
|
const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7;
|
||||||
const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline";
|
const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline";
|
||||||
const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#"
|
const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#"
|
||||||
@@ -271,6 +273,11 @@ const MIGRATIONS: &[Migration] = &[
|
|||||||
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME,
|
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME,
|
||||||
apply: migrate_workspace_config_v78_to_v79,
|
apply: migrate_workspace_config_v78_to_v79,
|
||||||
},
|
},
|
||||||
|
Migration {
|
||||||
|
version: 80,
|
||||||
|
name: WORKER_RESTORE_INTENTS_MIGRATION_NAME,
|
||||||
|
apply: migrate_worker_restore_intents_v79_to_v80,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
@@ -2167,6 +2174,48 @@ impl SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Client intent journal, not a lifecycle state machine: pin the public tuple
|
||||||
|
/// before dispatch and keep it unchanged until a terminal receipt is obtained.
|
||||||
|
pub(crate) fn pin_internal_worker_restore_intent(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
domain_key: &str,
|
||||||
|
observed_token: Option<&str>,
|
||||||
|
explicit_request_id: Option<&str>,
|
||||||
|
) -> Result<server_api::WorkerRestoreRequest> {
|
||||||
|
self.with_transaction(|tx| {
|
||||||
|
let existing: Option<(String, String)> = if let Some(id) = explicit_request_id {
|
||||||
|
tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4",
|
||||||
|
params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()?
|
||||||
|
} else {
|
||||||
|
tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND domain_key=?4 AND settled=0",
|
||||||
|
params![workspace_id, worker.runtime_id, worker.worker_id, domain_key], |row| Ok((row.get(0)?, row.get(1)?))).optional()?
|
||||||
|
};
|
||||||
|
if let Some((request_id, expected_observation_token)) = existing {
|
||||||
|
return Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token });
|
||||||
|
}
|
||||||
|
let observed_token = observed_token.ok_or(Error::RestoreObservationConflict)?;
|
||||||
|
let request_id = explicit_request_id.map(str::to_owned).unwrap_or_else(|| format!("{domain_key}:{observed_token}"));
|
||||||
|
tx.execute("INSERT INTO worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key,request_id,expected_token,settled) VALUES(?1,?2,?3,?4,?5,?6,0)",
|
||||||
|
params![workspace_id, worker.runtime_id, worker.worker_id, domain_key, request_id, observed_token])?;
|
||||||
|
Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token: observed_token.to_owned() })
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn settle_internal_worker_restore_intent(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
request_id: &str,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.with_conn(|conn| {
|
||||||
|
conn.execute("UPDATE worker_restore_intents SET settled=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4",
|
||||||
|
params![workspace_id, worker.runtime_id, worker.worker_id, request_id])?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn get_workspace_memory_settings(
|
pub(crate) fn get_workspace_memory_settings(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
@@ -11248,6 +11297,7 @@ fn read_worker_registry_projection(
|
|||||||
})?
|
})?
|
||||||
} else {
|
} else {
|
||||||
SubscriptionWorker {
|
SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string())
|
worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string())
|
||||||
.map_err(|error| Error::InvalidInput(error.to_string()))?,
|
.map_err(|error| Error::InvalidInput(error.to_string()))?,
|
||||||
runtime_id: Some(worker.runtime_id.to_string()),
|
runtime_id: Some(worker.runtime_id.to_string()),
|
||||||
@@ -12013,6 +12063,24 @@ fn read_workdir_registry_record(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn migrate_worker_restore_intents_v79_to_v80(conn: &Connection) -> Result<()> {
|
||||||
|
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
|
||||||
|
tx.execute_batch("CREATE TABLE worker_restore_intents (
|
||||||
|
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
|
||||||
|
domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL,
|
||||||
|
settled INTEGER NOT NULL CHECK(settled IN (0,1)),
|
||||||
|
PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id),
|
||||||
|
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(?1,?2)",
|
||||||
|
params![80_i64, WORKER_RESTORE_INTENTS_MIGRATION_NAME],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn prepare_connection(conn: &Connection) -> Result<()> {
|
fn prepare_connection(conn: &Connection) -> Result<()> {
|
||||||
configure_sqlite(conn)?;
|
configure_sqlite(conn)?;
|
||||||
let workspace_schema_version = current_schema_version(conn)?;
|
let workspace_schema_version = current_schema_version(conn)?;
|
||||||
@@ -15094,6 +15162,15 @@ fn migrate_workdir_connection_id_v77_to_v78(conn: &Connection) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
||||||
|
conn.execute_batch("CREATE TABLE worker_restore_intents (
|
||||||
|
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
|
||||||
|
domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL,
|
||||||
|
settled INTEGER NOT NULL CHECK(settled IN (0,1)),
|
||||||
|
PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id),
|
||||||
|
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
|
||||||
|
|
||||||
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
|
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
|
||||||
conn.execute_batch(include_str!("latest_schema.sql"))?;
|
conn.execute_batch(include_str!("latest_schema.sql"))?;
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
@@ -16202,6 +16279,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn downgrade_schema_66_ticket_and_workdir_authority(conn: &Connection) -> Result<()> {
|
fn downgrade_schema_66_ticket_and_workdir_authority(conn: &Connection) -> Result<()> {
|
||||||
|
conn.execute_batch("DROP TABLE IF EXISTS worker_restore_intents;")?;
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
"DROP TABLE backend_job_deliveries;
|
"DROP TABLE backend_job_deliveries;
|
||||||
DROP TABLE backend_job_attempts;
|
DROP TABLE backend_job_attempts;
|
||||||
@@ -17709,6 +17787,10 @@ mod tests {
|
|||||||
version: 79,
|
version: 79,
|
||||||
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()
|
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()
|
||||||
},
|
},
|
||||||
|
WorkspaceSchemaMigrationStep {
|
||||||
|
version: 80,
|
||||||
|
name: WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()
|
||||||
|
},
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -17798,6 +17880,7 @@ mod tests {
|
|||||||
),
|
),
|
||||||
(78, WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string()),
|
(78, WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string()),
|
||||||
(79, WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()),
|
(79, WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()),
|
||||||
|
(80, WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()),
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||||
@@ -18149,7 +18232,7 @@ mod tests {
|
|||||||
.collect::<Vec<_>>(),
|
.collect::<Vec<_>>(),
|
||||||
vec![
|
vec![
|
||||||
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
|
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
|
||||||
73, 74, 75, 76, 77, 78, 79
|
73, 74, 75, 76, 77, 78, 79, 80
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
SqliteWorkspaceStore::migrate_database(&path).unwrap();
|
SqliteWorkspaceStore::migrate_database(&path).unwrap();
|
||||||
@@ -18158,7 +18241,7 @@ mod tests {
|
|||||||
current_schema_version(&conn).unwrap(),
|
current_schema_version(&conn).unwrap(),
|
||||||
LATEST_SCHEMA_VERSION
|
LATEST_SCHEMA_VERSION
|
||||||
);
|
);
|
||||||
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 30);
|
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 31);
|
||||||
assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap());
|
assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap());
|
||||||
assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap());
|
assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap());
|
||||||
assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap());
|
assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap());
|
||||||
@@ -21860,6 +21943,7 @@ INSERT INTO worker_registry (
|
|||||||
|
|
||||||
for record in [&runtime_a, &runtime_b] {
|
for record in [&runtime_a, &runtime_b] {
|
||||||
let observed = SubscriptionWorker {
|
let observed = SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: SubscriptionWorkerId::new("shared-local-id").unwrap(),
|
worker_id: SubscriptionWorkerId::new("shared-local-id").unwrap(),
|
||||||
runtime_id: Some(record.worker.runtime_id.clone()),
|
runtime_id: Some(record.worker.runtime_id.clone()),
|
||||||
resource_key: None,
|
resource_key: None,
|
||||||
@@ -21996,6 +22080,7 @@ INSERT INTO worker_registry (
|
|||||||
Some("W-1")
|
Some("W-1")
|
||||||
);
|
);
|
||||||
let observed = SubscriptionWorker {
|
let observed = SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: SubscriptionWorkerId::new("known").unwrap(),
|
worker_id: SubscriptionWorkerId::new("known").unwrap(),
|
||||||
runtime_id: Some("embedded".to_string()),
|
runtime_id: Some("embedded".to_string()),
|
||||||
resource_key: None,
|
resource_key: None,
|
||||||
@@ -22016,6 +22101,7 @@ INSERT INTO worker_registry (
|
|||||||
};
|
};
|
||||||
store.upsert_worker_registry(&other_catalog).unwrap();
|
store.upsert_worker_registry(&other_catalog).unwrap();
|
||||||
let other_observed = SubscriptionWorker {
|
let other_observed = SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: SubscriptionWorkerId::new("other").unwrap(),
|
worker_id: SubscriptionWorkerId::new("other").unwrap(),
|
||||||
runtime_id: Some("runtime-other".to_string()),
|
runtime_id: Some("runtime-other".to_string()),
|
||||||
display_name: Some("Other Runtime Worker".to_string()),
|
display_name: Some("Other Runtime Worker".to_string()),
|
||||||
@@ -22025,6 +22111,7 @@ INSERT INTO worker_registry (
|
|||||||
.apply_worker_registry_observation("local-dev", "runtime-other", &other_observed, "2")
|
.apply_worker_registry_observation("local-dev", "runtime-other", &other_observed, "2")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let orphan = SubscriptionWorker {
|
let orphan = SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: SubscriptionWorkerId::new("orphan").unwrap(),
|
worker_id: SubscriptionWorkerId::new("orphan").unwrap(),
|
||||||
..observed.clone()
|
..observed.clone()
|
||||||
};
|
};
|
||||||
@@ -22190,6 +22277,7 @@ INSERT INTO worker_registry (
|
|||||||
.is_none()
|
.is_none()
|
||||||
);
|
);
|
||||||
let event = SubscriptionWorker {
|
let event = SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: SubscriptionWorkerId::new("removed").unwrap(),
|
worker_id: SubscriptionWorkerId::new("removed").unwrap(),
|
||||||
runtime_id: Some("embedded".to_string()),
|
runtime_id: Some("embedded".to_string()),
|
||||||
resource_key: None,
|
resource_key: None,
|
||||||
|
|||||||
@@ -76,6 +76,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
|||||||
"worker_registry_projection_diagnostics",
|
"worker_registry_projection_diagnostics",
|
||||||
"worker_registry_projection_removals",
|
"worker_registry_projection_removals",
|
||||||
"worker_removal_operations",
|
"worker_removal_operations",
|
||||||
|
"worker_restore_intents",
|
||||||
"worker_retention_audit_events",
|
"worker_retention_audit_events",
|
||||||
"worker_session_archive_observe_grants",
|
"worker_session_archive_observe_grants",
|
||||||
"worker_session_archives",
|
"worker_session_archives",
|
||||||
|
|||||||
@@ -425,6 +425,7 @@ fn project_registry_worker(
|
|||||||
worker
|
worker
|
||||||
} else {
|
} else {
|
||||||
SubscriptionWorker {
|
SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: protocol::subscription::SubscriptionWorkerId::new(
|
worker_id: protocol::subscription::SubscriptionWorkerId::new(
|
||||||
record.registry.worker.worker_id.clone(),
|
record.registry.worker.worker_id.clone(),
|
||||||
)
|
)
|
||||||
@@ -600,6 +601,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
||||||
let mut worker = SubscriptionWorker {
|
let mut worker = SubscriptionWorker {
|
||||||
|
restore_observation_token: None,
|
||||||
worker_id: worker_id(),
|
worker_id: worker_id(),
|
||||||
runtime_id: Some("runtime-1".to_string()),
|
runtime_id: Some("runtime-1".to_string()),
|
||||||
resource_key: Some("W-1".to_string()),
|
resource_key: Some("W-1".to_string()),
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# Guarded Worker Restore and Web observations
|
||||||
|
|
||||||
|
T-701 extends the [registry-bound operation reception](workspace-worker-operations.md), not the Worker creation, Resume or Restore state machines.
|
||||||
|
|
||||||
|
## Different tokens protect different things
|
||||||
|
|
||||||
|
| Operation | Observed identity / expected input | Atomic validation and execution owner | Conflict / uncertain outcome | Shared client behavior / deliberate difference |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| Restore | Workspace + Runtime + Worker identity; Runtime-owned `restore_observation_token`; immutable `expected_observation_token` + `request_id` | Existing Runtime per-Worker lifecycle operation lock, durable admission and lifecycle commit. Preparation and execution follow admission, not a frontend GET comparison | Stale new intent is HTTP 409 (`restore_observation_conflict`); operation results remain accepted / rejected / rolled_back / reconciliation_required. A lost reply is unknown, not definite rollback | Both entrances use the same scoped POST/parser/notice classifiers and observation refresh. Never replace a rejected intent's token and automatically retry |
|
||||||
|
| Stop | Identity and reason, **not** a catalog revision CAS | Runtime per-Worker operation lock and existing backend Stop settlement; Workspace session finalization only after accepted | Existing flat accepted / unsupported / rejected result with diagnostics | Same encoded target paths and bounded HTTP-error messages; refresh observations instead of writing a guessed local stopped state. Stop does not gain Restore's generation guarantee |
|
||||||
|
| Pause / Resume / Cancel / Compact | Connected execution identity and `WorkerCommandEnvelope.command_id` | Existing bound execution/controller command admission | Command acknowledgement: Accepted / StaleCommandId / Conflict / InvalidState, with authoritative state | Command IDs order commands; they are not catalog observation generations. Resume is not Restore; connections do not silently move to a replacement execution |
|
||||||
|
| Delete / cleanup | Runtime + selected target IDs; `expected_plan_revision` **and** `expected_plan_digest` | Existing cleanup validation, per-target live/retention revalidation, removal service and catalog commits | Existing cleanup-plan conflicts, per-target status and diagnostics | Preserve plan protection for candidate sets and dependencies. Never substitute a Worker generation. Page plans invalidate at the beginning of every catalog refresh and are unavailable until fresh catalog and plan observations settle |
|
||||||
|
| Pin / Unpin | Registry Worker identity and explicit boolean | Registry retention update and ordered projection publication | Existing retention result/errors | Preserve retention semantics and full catalog metadata; refresh shared observations and cleanup plan |
|
||||||
|
| Submit / Notify | Authenticated source + durable submission / notification request ID | Existing controller/session durable receipt admission | Existing accepted/rejected receipts and unknown delivery diagnostics | These are not Restore requests; Restore never manufactures input, Resume or a new Worker |
|
||||||
|
|
||||||
|
There is no universal Worker command token or shared frontend action lock. Workspace reception resolves a stable target and rechecks caller authorization; Runtime/controller/store owners retain operation-specific atomic boundaries.
|
||||||
|
|
||||||
|
## Browser request and result
|
||||||
|
|
||||||
|
Normal explicit Restore is enabled only for `availability = observed`, catalog `stopped`, and an available backend observation token. Sidebar display state and foreground `worker_state` are not eligibility evidence: sidebar uses `lifecycleState`; the page uses the full catalog's `state`. Neither UI guesses snapshot, singleton, Workdir provider or authorization eligibility.
|
||||||
|
|
||||||
|
Both entrances send the generated two-field `WorkerRestoreRequest` to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore
|
||||||
|
```
|
||||||
|
|
||||||
|
Each ID is independently encoded. No Ticket-assignment query, Submit, Resume or Worker-create call is used. The bounded parser validates wrapper identity, the four states, diagnostics and optional **Workspace WorkerSummary**. A Stop lifecycle result or launch summary is not a Restore response. Unknown states/fields/invalid identities cannot produce a success notice.
|
||||||
|
|
||||||
|
Accepted means the existing Restore operation committed, not that the Worker is necessarily Running or a later run succeeded. A returned summary validates result identity; display is updated from fresh shared observations, not an old POST snapshot. Rejected and rolled_back display diagnostics without success. Reconciliation_required and transport/malformed-response uncertainty retain the original immutable request for an explicit **Retry Restore** action. That action sends the same request; it does not create an operation, exchange its observation token, or automatically loop. A conflict clears this local recovery intent and asks the user to review fresh observations before a new explicit operation. A GET alone does not settle Runtime's pending lifecycle operation.
|
||||||
|
|
||||||
|
## Backend admission, preparation and recovery
|
||||||
|
|
||||||
|
The Workspace `WorkspaceWorker` reception still resolves target identity and rechecks caller authority. Its Restore coordinator first dispatches the public tuple to Runtime without candidate preparation. A terminal owner returns its historical receipt; an admitted pending owner supplies its stored preparation before renewed singleton/Workdir/External eligibility is evaluated. Historical result summaries are read from the current Runtime and are not written back as catalog observations.
|
||||||
|
|
||||||
|
For a new intent, Workspace builds read-only preparation references after the normal eligibility checks. Runtime atomically checks the observation token again under the existing per-Worker lock and persists the owner together with `pending_restore` and its existing lifecycle operation ID. Only this admitted operation can proceed to SSH resource issuance, binding/access preparation and execution. A race between the first observation check and admission therefore cannot issue credentials or launch a new Restore from the old intent.
|
||||||
|
|
||||||
|
Preparation completion is attached to that same pending operation, not a replacement operation. Retained repository-access snapshots and exact request identity allow interrupted preparation to resume without exchanging candidate bindings. Runtime retries use the original owner's payload; a freshly generated preparation cannot rewrite the admitted target or expected observation. Stop and startup reconciliation continue to use the existing pending lifecycle ownership. When startup/legacy persistence has a pending lifecycle operation but no public receipt owner, an explicit request with the current observation may bind recovery to that exact operation ID, without applying candidate preparation. Once bound, a different request cannot take over; terminal replay does not resurrect a later stopped Worker.
|
||||||
|
|
||||||
|
| Caller | Intent identity retained / validation boundary |
|
||||||
|
| --- | --- |
|
||||||
|
| Web sidebar and page | Immutable two-field request retained locally for explicit uncertain-result recovery; no shared in-flight lock |
|
||||||
|
| Workspace alias, scoped and Worker-control HTTP | Mandatory public request; common target/auth reception and the same Runtime admission |
|
||||||
|
| Embedded / remote Runtime providers and management HTTP | Typed Runtime request and coordination transport, including original observation and lifecycle operation owner |
|
||||||
|
| `WorkerRestore` Tool | Observed token in Tool arguments and durable Tool-call-derived request identity |
|
||||||
|
| Existing TUI Restore/resume path | Selected row's observation token and retained request; no new TUI entrance |
|
||||||
|
| Internal Coder, Orchestrator, subject reuse and compensation | Domain intent journal pins the request/token before dispatch and retains it through unknown result and restart; it does not own lifecycle status or bypass Runtime validation |
|
||||||
|
| Runtime startup / pending reconciliation | Private recovery of the existing persisted Restore owner and operation; not an external unguarded new-intent endpoint |
|
||||||
|
|
||||||
|
The Workspace internal intent journal stores client retry identity only. Runtime remains the lifecycle authority; it is not a second Restore state machine, Worker ledger or cross-screen lock.
|
||||||
|
|
||||||
|
## Observation and cleanup-plan ordering
|
||||||
|
|
||||||
|
`workspaceWorkersStore` retains the existing sidebar stream/projection and also fetches the full Workspace Worker catalog for the page. Full catalog metadata preserves retention, attachments, implementation and targets that are not sidebar entries. Stream updates overlay lifecycle/availability/foreground snapshot/token without interpreting absence as a new page exclusion.
|
||||||
|
|
||||||
|
Every stream update or explicit refresh increments a local **observation scheduling version** and publishes `catalogRefreshing` before fetching. This local version fences asynchronous display work; it is **not** Restore admission authority and is never sent to Runtime. A newer event/GET supersedes older GET completion. Old POST summaries and page-loader data cannot overwrite the active catalog. On disposal, requests and subscription callbacks are fenced. Catalog fetch errors have their own stable alert episode rather than clearing transport/Workdir alerts.
|
||||||
|
|
||||||
|
The page invalidates cleanup plans immediately when observations refresh. It does not fetch or install plans while `catalogRefreshing` is true. Successful refresh completion triggers plan reads even when the resulting catalog contents did not change. Workspace/lifetime and cleanup epochs reject late previous plan responses. A failed refresh leaves old candidates unavailable instead of treating old page data as fresh.
|
||||||
|
|
||||||
|
Both components retain local busy state only. Requests pin their starting Workspace/target, and late replies after Workspace switch/disposal cannot update another Workspace's notices, list or busy state. Sidebar and page may issue competing requests: atomic Runtime admission—not a cross-component client lock—decides their outcomes. Console links and existing sidebar filtering remain unchanged.
|
||||||
|
|
||||||
|
## Validation surfaces
|
||||||
|
|
||||||
|
- Runtime admission tests prove stale/ABA and serialization, durable same-request replay and pending reconciliation.
|
||||||
|
- Runtime management, embedded/remote providers, Workspace aliases/scoped/control routes, Tool and existing TUI callers exercise the guarded API rather than a legacy unguarded Restore.
|
||||||
|
- `tests/runtime-workers.test.ts` and `test/sidebar/worker-actions.test.ts` prove response/request and notice contracts; the Runtime parser tests are included in the normal Web test task.
|
||||||
|
- Rendered `worker-stop.browser.spec.ts` and `workers-page.browser.spec.ts` cover menu/action eligibility, busy behavior, four outcomes and errors, explicit retry, Workspace/disposal fences, ordinary Console/Pin/Delete, sidebar-origin updates and cleanup epochs.
|
||||||
|
- `worker-restore-store.browser.spec.ts` tests the real store's full metadata, GET/event ordering, dependent-plan invalidation, no POST recovery loop, catalog-only target preservation and disposal.
|
||||||
|
|
||||||
|
Exact executed validation counts and immutable source/review evidence belong to the repository Merge Request and final approved Ticket handoff.
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# T-701 validation ledger
|
||||||
|
|
||||||
|
This records concrete intermediate failures rather than treating later passes as erasing them. Final source and review evidence are on the repository Merge Request.
|
||||||
|
|
||||||
|
## Validation approach
|
||||||
|
|
||||||
|
Start with Restore/parser/UI/serialization boundary filters, then the eight changed crates and the normal root `cargo check` compile closure. Cargo jobs are bounded to three; full crate test concurrency is bounded to four. No all-workspace test, `--all-targets`, Nix/Docker build, deployment, process restart or live Worker repair is part of this validation.
|
||||||
|
|
||||||
|
The Web production bundle, `svelte-check`, normal Deno tests and actual rendered component suites have passed. Those results alone do not prove Backend admission, replay or preparation recovery. Full Rust suites exercise callers, schema migration and dependent inventories that narrow guard filters cannot prove.
|
||||||
|
|
||||||
|
## Intermediate failures and disposition
|
||||||
|
|
||||||
|
1. Early root checks failed because the Runtime request/preparation DTO integration was not yet present. This was an incomplete implementation, not a baseline defect. A subsequent root check passed after the integration.
|
||||||
|
2. Early Server API full tests reported three stale generated artifacts after the OpenAPI contract changed. Regenerating the affected closure artifacts resolved them; the full default suite passed 70 tests. TypeScript-feature tests were also executed after regeneration.
|
||||||
|
3. Adding full catalog GETs exposed existing browser fixtures that returned Workdir-only shapes to every endpoint. The fixtures now return the exact `/workers` contract, and the Workdir-error fixture fails only its intended endpoint. Catalog and transport/Workdir failures have separate alert identities.
|
||||||
|
4. Parent full `cargo test -p yoi-workspace-server -- --test-threads=4` produced **614 passed / 4 failed** (618 library tests):
|
||||||
|
- `hosts::tests::embedded_runtime_registers_routes_input_without_internal_leaks`: a broad forbidden-word assertion rejected the newly public observation field's word `token`. The private credential/token non-leak contract must be preserved while allowing the explicitly public observation token.
|
||||||
|
- `server::tests::embedded_runtime_api_routes_by_runtime_and_worker_ids_without_leaking_internals`: an old `{}` Restore fixture received 422 because guarded Restore now requires the observed token and request ID. Update the real route fixture, do not restore an unguarded compatibility path.
|
||||||
|
- `workspace_deletion::tests::explicit_purge_inventory_covers_every_workspace_scoped_table`: the new internal Restore intent journal was absent from the explicit purge inventory. Integrate its Workspace deletion lifecycle; do not weaken the exhaustive inventory check.
|
||||||
|
- `server::tests::ticket_checker_suppresses_stale_and_clean_results_and_diagnoses_invalid_output`: the inherited stale-checker `execution.take_inputs().is_empty()` assertion failed. Earlier T-702 evidence reproduced this named instability on an isolated original baseline; see [T-702 validation report](2026-10-06-t702-stale-checker-validation.md). T-701 has not fixed that checker, and this failure is not conflated with the three new integration regressions.
|
||||||
|
|
||||||
|
## Follow-up disposition and final pre-publication results
|
||||||
|
|
||||||
|
The projection fixtures now permit only the explicit `restore_observation_token` key and validate its UUIDv7 value; private token/credential fields remain forbidden. The HTTP fixture sends the real observed token and request ID. The internal intent table is in the exhaustive Workspace deletion inventory.
|
||||||
|
|
||||||
|
The final restart recovery test exposed one obsolete Runtime expectation: an unowned startup/legacy pending operation must be explicitly recoverable, not permanently conflict. Public recovery now binds a receipt to the exact existing lifecycle operation ID, without candidate preparation or a replacement execution. Tests assert a competing request conflicts, recovery preserves the ID, and terminal replay after Stop never launches again. A Tool recovery fixture also used path-shaped IDs rejected by the existing authority-ID validator; it now uses valid authority IDs and proves the two explicit calls send an identical request body.
|
||||||
|
|
||||||
|
After those fixes, a full Server run reported **618 passed / 1 failed**, solely the inherited stale-checker assertion above. Its isolated test subsequently passed. An additional whole-crate run with the same four-thread configuration and **no skips** passed **619 library + 11 binary tests**. This does not erase the earlier failure or claim to fix the checker race.
|
||||||
|
|
||||||
|
| Executed command | Final pre-publication evidence |
|
||||||
|
| --- | --- |
|
||||||
|
| root `cargo check` | Passed normal default-member compile closure, including TUI and Server |
|
||||||
|
| `cargo test -p worker-runtime -- --test-threads=4` | 332 library + 34 binary tests passed |
|
||||||
|
| `cargo test -p yoi-workspace-server -- --test-threads=4` | 619 library + 11 binary tests passed in the additional no-skip run; previous flaky failure retained above |
|
||||||
|
| `cargo test -p worker -- --test-threads=4` | 804 library + 109 integration tests passed |
|
||||||
|
| `cargo test -p runtime-api` | 9 tests passed |
|
||||||
|
| `cargo test -p server-api` / `--features typescript` | 70 / 77 tests passed, including generated artifact freshness |
|
||||||
|
| `cargo test -p protocol` / `--features typescript` | 78 / 83 tests passed |
|
||||||
|
| `cargo test -p client` | 52 library + 2 integration tests passed |
|
||||||
|
| `cargo test -p tui` | 331 tests passed |
|
||||||
|
| Web `deno task check` | 0 errors, 0 warnings |
|
||||||
|
| Web `deno task test` | 504 tests passed |
|
||||||
|
| Web `deno task test:component` | 249 tests in 25 files passed, including the real paired sidebar/page/store suite |
|
||||||
|
| Web `deno task build` | Production build passed; existing chunk-size warning remains |
|
||||||
|
| `cargo fmt --all -- --check`, `git diff --check HEAD` | Passed |
|
||||||
|
|
||||||
|
Additional exploratory Runtime feature checks did **not** pass: `--no-default-features` test/check cannot resolve existing unconditional `workspace_request` / `reqwest` imports; the test target additionally references disabled `fs_store`. HTTP-only check also lacks the existing retention and futures dependencies. Inspection of original HEAD confirms those imports/gates already exist; no isolated baseline build was performed for these configurations. These are recorded as failed exploratory checks, not semantic passes or newly supported configurations. The default Runtime configuration, which contains fs-store and ws-server, is exercised in full above. This Ticket does not redesign unrelated Runtime feature gating.
|
||||||
|
|
||||||
|
Raw session validation logs are retained under ignored `target/t701-validation/`, not shipped as product source. Immutable published source, independent verdict and any subsequent fix/review evidence belong to the explicit repository Merge Request.
|
||||||
+52
-1
@@ -8062,6 +8062,13 @@
|
|||||||
"resource_key": {
|
"resource_key": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"restore_observation_token": {
|
||||||
|
"description": "Opaque Runtime-owned observation fence used by explicit Restore intent.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"retention_state": {
|
"retention_state": {
|
||||||
"default": "",
|
"default": "",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
@@ -12953,6 +12960,23 @@
|
|||||||
],
|
],
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"WorkerRestoreRequest": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "One explicit Restore intent. Retries retain both fields; conflicts require a\nfresh observation and a deliberate new intent, not automatic token replacement.",
|
||||||
|
"properties": {
|
||||||
|
"expected_observation_token": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"request_id": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"expected_observation_token",
|
||||||
|
"request_id"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
"WorkerRestoreResponse": {
|
"WorkerRestoreResponse": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
@@ -13157,6 +13181,13 @@
|
|||||||
"resource_key": {
|
"resource_key": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"restore_observation_token": {
|
||||||
|
"description": "Opaque Runtime-owned observation fence used by explicit Restore intent.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"retention_state": {
|
"retention_state": {
|
||||||
"default": "",
|
"default": "",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
@@ -15443,7 +15474,7 @@
|
|||||||
"info": {
|
"info": {
|
||||||
"title": "Yoi Server API",
|
"title": "Yoi Server API",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"x-yoi-source-digest": "sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779"
|
"x-yoi-source-digest": "sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da"
|
||||||
},
|
},
|
||||||
"openapi": "3.1.0",
|
"openapi": "3.1.0",
|
||||||
"paths": {
|
"paths": {
|
||||||
@@ -18016,6 +18047,16 @@
|
|||||||
"style": "simple"
|
"style": "simple"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
"requestBody": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/WorkerRestoreRequest"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
"responses": {
|
"responses": {
|
||||||
"200": {
|
"200": {
|
||||||
"content": {
|
"content": {
|
||||||
@@ -26433,6 +26474,16 @@
|
|||||||
"style": "form"
|
"style": "form"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
"requestBody": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/WorkerRestoreRequest"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
"responses": {
|
"responses": {
|
||||||
"200": {
|
"200": {
|
||||||
"content": {
|
"content": {
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
"dev": "deno run -A npm:vite@8.3.2 dev",
|
"dev": "deno run -A npm:vite@8.3.2 dev",
|
||||||
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
||||||
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
|
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
|
||||||
"test": "deno test --allow-read=src,test,tests,../../docs --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts",
|
"test": "deno test --allow-read=src,test,tests,../../docs --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts",
|
||||||
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
|
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
|
||||||
"build": "deno run -A npm:vite@8.3.2 build",
|
"build": "deno run -A npm:vite@8.3.2 build",
|
||||||
"preview": "deno run -A npm:vite@8.3.2 preview",
|
"preview": "deno run -A npm:vite@8.3.2 preview",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-companion-openapi-typescript@1. Do not edit.
|
// @generated by yoi-companion-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779)
|
// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da)
|
||||||
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/companion-api.ts
|
// Output: web/workspace/src/lib/generated/companion-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -359,7 +359,11 @@ alias: string, repository_key?: string | null, working_directory_id: Subscriptio
|
|||||||
|
|
||||||
export type SubscriptionWorkerJob = { job_id: string, attempt_id: string, purpose: string, };
|
export type SubscriptionWorkerJob = { job_id: string, attempt_id: string, purpose: string, };
|
||||||
|
|
||||||
export type SubscriptionWorker = { worker_id: SubscriptionWorkerId,
|
export type SubscriptionWorker = {
|
||||||
|
/**
|
||||||
|
* Opaque Runtime-owned Restore observation identity, never a display-state hash.
|
||||||
|
*/
|
||||||
|
restore_observation_token?: string | null, worker_id: SubscriptionWorkerId,
|
||||||
/**
|
/**
|
||||||
* Set by the Workspace Server when projecting a Runtime-owned Worker to clients.
|
* Set by the Workspace Server when projecting a Runtime-owned Worker to clients.
|
||||||
* Runtime producers leave this unset because the connection identifies the Runtime.
|
* Runtime producers leave this unset because the connection identifies the Runtime.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-repository-openapi-typescript@1. Do not edit.
|
// @generated by yoi-repository-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779)
|
// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da)
|
||||||
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
|
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/repository-api.ts
|
// Output: web/workspace/src/lib/generated/repository-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
|
||||||
@@ -71,7 +71,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS
|
|||||||
|
|
||||||
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
|
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
|
||||||
|
|
||||||
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
|
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
|
||||||
|
|
||||||
export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; };
|
export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; };
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-runtime-openapi-typescript@1. Do not edit.
|
// @generated by yoi-runtime-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779)
|
// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da)
|
||||||
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/runtime-api.ts
|
// Output: web/workspace/src/lib/generated/runtime-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
|
||||||
@@ -139,7 +139,7 @@ export type RuntimeWorkerLifecycleResult = { diagnostics?: Array<Diagnostic>; ru
|
|||||||
|
|
||||||
export type RuntimeWorkerListResponse = { diagnostics?: Array<Diagnostic>; items: Array<WorkerSummary>; limit: number; source: string; workspace_id: string; };
|
export type RuntimeWorkerListResponse = { diagnostics?: Array<Diagnostic>; items: Array<WorkerSummary>; limit: number; source: string; workspace_id: string; };
|
||||||
|
|
||||||
export type RuntimeWorkerShowResponse = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; updated_at: string; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
|
export type RuntimeWorkerShowResponse = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; updated_at: string; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
|
||||||
|
|
||||||
export type RuntimeWorkerSpawnAcceptanceEvidence = { detail: string; kind: string; };
|
export type RuntimeWorkerSpawnAcceptanceEvidence = { detail: string; kind: string; };
|
||||||
|
|
||||||
@@ -181,6 +181,8 @@ export type WorkerMaintenanceState = "compacting";
|
|||||||
|
|
||||||
export type WorkerOperationState = "accepted" | "unsupported" | "rejected";
|
export type WorkerOperationState = "accepted" | "unsupported" | "rejected";
|
||||||
|
|
||||||
|
export type WorkerRestoreRequest = { expected_observation_token: string; request_id: string; };
|
||||||
|
|
||||||
export type WorkerRestoreResponse = { result: WorkerRestoreResult; runtime_id: string; worker_id: string; workspace_id: string; };
|
export type WorkerRestoreResponse = { result: WorkerRestoreResult; runtime_id: string; worker_id: string; workspace_id: string; };
|
||||||
|
|
||||||
export type WorkerRestoreResult = { diagnostics?: Array<Diagnostic>; state: WorkerRestoreState; worker?: WorkerSummary | null; };
|
export type WorkerRestoreResult = { diagnostics?: Array<Diagnostic>; state: WorkerRestoreState; worker?: WorkerSummary | null; };
|
||||||
@@ -195,7 +197,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS
|
|||||||
|
|
||||||
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
|
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
|
||||||
|
|
||||||
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
|
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
|
||||||
|
|
||||||
export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; };
|
export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; };
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
|
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779)
|
// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da)
|
||||||
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/ticket-api.ts
|
// Output: web/workspace/src/lib/generated/ticket-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-workdir-openapi-typescript@1. Do not edit.
|
// @generated by yoi-workdir-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779)
|
// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da)
|
||||||
// Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/workdir-api.ts
|
// Output: web/workspace/src/lib/generated/workdir-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// @generated by yoi-worker-openapi-typescript@1. Do not edit.
|
// @generated by yoi-worker-openapi-typescript@1. Do not edit.
|
||||||
// Input: openapi/server-api.json
|
// Input: openapi/server-api.json
|
||||||
// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779)
|
// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da)
|
||||||
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
|
||||||
// Output: web/workspace/src/lib/generated/worker-launch-api.ts
|
// Output: web/workspace/src/lib/generated/worker-launch-api.ts
|
||||||
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
|
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
|
||||||
@@ -77,7 +77,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS
|
|||||||
|
|
||||||
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
|
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
|
||||||
|
|
||||||
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
|
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
|
||||||
|
|
||||||
export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; };
|
export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; };
|
||||||
|
|
||||||
|
|||||||
@@ -10,9 +10,13 @@ import type {
|
|||||||
RuntimeCleanupPlanResponse,
|
RuntimeCleanupPlanResponse,
|
||||||
RuntimeWorkerLifecycleResult,
|
RuntimeWorkerLifecycleResult,
|
||||||
WorkerOperationState,
|
WorkerOperationState,
|
||||||
|
WorkerRestoreResponse,
|
||||||
|
WorkerRestoreState,
|
||||||
WorkerRetentionResponse,
|
WorkerRetentionResponse,
|
||||||
} from "#lib/generated/runtime-api.ts";
|
} from "#lib/generated/runtime-api.ts";
|
||||||
|
|
||||||
|
import { parseWorkerSummary } from "#lib/workspace/api/workers.ts";
|
||||||
|
|
||||||
const MAX_STRING_BYTES = 4_096;
|
const MAX_STRING_BYTES = 4_096;
|
||||||
const MAX_CANDIDATES = 1_000;
|
const MAX_CANDIDATES = 1_000;
|
||||||
const MAX_RESULTS = 2_000;
|
const MAX_RESULTS = 2_000;
|
||||||
@@ -68,14 +72,20 @@ function exactKeys(
|
|||||||
): void {
|
): void {
|
||||||
const allowed = new Set([...required, ...optional]);
|
const allowed = new Set([...required, ...optional]);
|
||||||
for (const key of Object.keys(value)) {
|
for (const key of Object.keys(value)) {
|
||||||
if (!allowed.has(key)) fail(`${path}.${key}`, "is not part of the wire contract");
|
if (!allowed.has(key)) {
|
||||||
|
fail(`${path}.${key}`, "is not part of the wire contract");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for (const key of required) {
|
for (const key of required) {
|
||||||
if (!Object.hasOwn(value, key)) fail(`${path}.${key}`, "is required");
|
if (!Object.hasOwn(value, key)) fail(`${path}.${key}`, "is required");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function boundedString(value: unknown, path: string, allowEmpty = false): string {
|
function boundedString(
|
||||||
|
value: unknown,
|
||||||
|
path: string,
|
||||||
|
allowEmpty = false,
|
||||||
|
): string {
|
||||||
if (typeof value !== "string") return fail(path, "must be a string");
|
if (typeof value !== "string") return fail(path, "must be a string");
|
||||||
if (!allowEmpty && value.length === 0) return fail(path, "must not be empty");
|
if (!allowEmpty && value.length === 0) return fail(path, "must not be empty");
|
||||||
if (encoder.encode(value).byteLength > MAX_STRING_BYTES) {
|
if (encoder.encode(value).byteLength > MAX_STRING_BYTES) {
|
||||||
@@ -91,7 +101,9 @@ function boolean(value: unknown, path: string): boolean {
|
|||||||
|
|
||||||
function array(value: unknown, path: string, max: number): unknown[] {
|
function array(value: unknown, path: string, max: number): unknown[] {
|
||||||
if (!Array.isArray(value)) return fail(path, "must be an array");
|
if (!Array.isArray(value)) return fail(path, "must be an array");
|
||||||
if (value.length > max) return fail(path, `must contain at most ${max} items`);
|
if (value.length > max) {
|
||||||
|
return fail(path, `must contain at most ${max} items`);
|
||||||
|
}
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,12 +113,18 @@ function strings(value: unknown, path: string): string[] {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function optionalNullableString(value: unknown, path: string): string | null | undefined {
|
function optionalNullableString(
|
||||||
|
value: unknown,
|
||||||
|
path: string,
|
||||||
|
): string | null | undefined {
|
||||||
if (value === undefined || value === null) return value;
|
if (value === undefined || value === null) return value;
|
||||||
return boundedString(value, path);
|
return boundedString(value, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
function optionalNullableSafeInteger(value: unknown, path: string): number | null | undefined {
|
function optionalNullableSafeInteger(
|
||||||
|
value: unknown,
|
||||||
|
path: string,
|
||||||
|
): number | null | undefined {
|
||||||
if (value === undefined || value === null) return value;
|
if (value === undefined || value === null) return value;
|
||||||
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
||||||
return fail(path, "must be a non-negative safe integer");
|
return fail(path, "must be a non-negative safe integer");
|
||||||
@@ -115,7 +133,9 @@ function optionalNullableSafeInteger(value: unknown, path: string): number | nul
|
|||||||
}
|
}
|
||||||
|
|
||||||
function targetKind(value: unknown, path: string): CleanupTargetKind {
|
function targetKind(value: unknown, path: string): CleanupTargetKind {
|
||||||
if (typeof value !== "string" || !TARGET_KINDS.has(value as CleanupTargetKind)) {
|
if (
|
||||||
|
typeof value !== "string" || !TARGET_KINDS.has(value as CleanupTargetKind)
|
||||||
|
) {
|
||||||
return fail(path, "has an unknown cleanup action");
|
return fail(path, "has an unknown cleanup action");
|
||||||
}
|
}
|
||||||
return value as CleanupTargetKind;
|
return value as CleanupTargetKind;
|
||||||
@@ -164,13 +184,25 @@ function workerCandidate(value: unknown, path: string): CleanupWorkerCandidate {
|
|||||||
target_id: boundedString(item.target_id, `${path}.target_id`),
|
target_id: boundedString(item.target_id, `${path}.target_id`),
|
||||||
action: targetKind(item.action, `${path}.action`),
|
action: targetKind(item.action, `${path}.action`),
|
||||||
worker_id: boundedString(item.worker_id, `${path}.worker_id`),
|
worker_id: boundedString(item.worker_id, `${path}.worker_id`),
|
||||||
runtime_worker_id: boundedString(item.runtime_worker_id, `${path}.runtime_worker_id`),
|
runtime_worker_id: boundedString(
|
||||||
|
item.runtime_worker_id,
|
||||||
|
`${path}.runtime_worker_id`,
|
||||||
|
),
|
||||||
runtime_id: boundedString(item.runtime_id, `${path}.runtime_id`),
|
runtime_id: boundedString(item.runtime_id, `${path}.runtime_id`),
|
||||||
reason: boundedString(item.reason, `${path}.reason`, true),
|
reason: boundedString(item.reason, `${path}.reason`, true),
|
||||||
blocking_reason: optionalNullableString(item.blocking_reason, `${path}.blocking_reason`),
|
blocking_reason: optionalNullableString(
|
||||||
|
item.blocking_reason,
|
||||||
|
`${path}.blocking_reason`,
|
||||||
|
),
|
||||||
pinned: boolean(item.pinned, `${path}.pinned`),
|
pinned: boolean(item.pinned, `${path}.pinned`),
|
||||||
retention_state: boundedString(item.retention_state, `${path}.retention_state`),
|
retention_state: boundedString(
|
||||||
linked_workdir_ids: strings(item.linked_workdir_ids, `${path}.linked_workdir_ids`),
|
item.retention_state,
|
||||||
|
`${path}.retention_state`,
|
||||||
|
),
|
||||||
|
linked_workdir_ids: strings(
|
||||||
|
item.linked_workdir_ids,
|
||||||
|
`${path}.linked_workdir_ids`,
|
||||||
|
),
|
||||||
running_linked: boolean(item.running_linked, `${path}.running_linked`),
|
running_linked: boolean(item.running_linked, `${path}.running_linked`),
|
||||||
estimated_reclaim_bytes: optionalNullableSafeInteger(
|
estimated_reclaim_bytes: optionalNullableSafeInteger(
|
||||||
item.estimated_reclaim_bytes,
|
item.estimated_reclaim_bytes,
|
||||||
@@ -179,7 +211,10 @@ function workerCandidate(value: unknown, path: string): CleanupWorkerCandidate {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate {
|
function workdirCandidate(
|
||||||
|
value: unknown,
|
||||||
|
path: string,
|
||||||
|
): CleanupWorkdirCandidate {
|
||||||
const item = object(value, path);
|
const item = object(value, path);
|
||||||
exactKeys(
|
exactKeys(
|
||||||
item,
|
item,
|
||||||
@@ -213,10 +248,19 @@ function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate
|
|||||||
action: targetKind(item.action, `${path}.action`),
|
action: targetKind(item.action, `${path}.action`),
|
||||||
workdir_id: boundedString(item.workdir_id, `${path}.workdir_id`),
|
workdir_id: boundedString(item.workdir_id, `${path}.workdir_id`),
|
||||||
runtime_id: boundedString(item.runtime_id, `${path}.runtime_id`),
|
runtime_id: boundedString(item.runtime_id, `${path}.runtime_id`),
|
||||||
repository_key: boundedString(item.repository_key, `${path}.repository_key`),
|
repository_key: boundedString(
|
||||||
|
item.repository_key,
|
||||||
|
`${path}.repository_key`,
|
||||||
|
),
|
||||||
reason: boundedString(item.reason, `${path}.reason`, true),
|
reason: boundedString(item.reason, `${path}.reason`, true),
|
||||||
blocking_reason: optionalNullableString(item.blocking_reason, `${path}.blocking_reason`),
|
blocking_reason: optionalNullableString(
|
||||||
linked_worker_ids: strings(item.linked_worker_ids, `${path}.linked_worker_ids`),
|
item.blocking_reason,
|
||||||
|
`${path}.blocking_reason`,
|
||||||
|
),
|
||||||
|
linked_worker_ids: strings(
|
||||||
|
item.linked_worker_ids,
|
||||||
|
`${path}.linked_worker_ids`,
|
||||||
|
),
|
||||||
linked_running_worker_ids: strings(
|
linked_running_worker_ids: strings(
|
||||||
item.linked_running_worker_ids,
|
item.linked_running_worker_ids,
|
||||||
`${path}.linked_running_worker_ids`,
|
`${path}.linked_running_worker_ids`,
|
||||||
@@ -232,7 +276,9 @@ function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function parseWorkerRetentionResponse(value: unknown): WorkerRetentionResponse {
|
export function parseWorkerRetentionResponse(
|
||||||
|
value: unknown,
|
||||||
|
): WorkerRetentionResponse {
|
||||||
const item = object(value, "Worker retention response");
|
const item = object(value, "Worker retention response");
|
||||||
exactKeys(
|
exactKeys(
|
||||||
item,
|
item,
|
||||||
@@ -241,9 +287,18 @@ export function parseWorkerRetentionResponse(value: unknown): WorkerRetentionRes
|
|||||||
"Worker retention response",
|
"Worker retention response",
|
||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
workspace_id: boundedString(item.workspace_id, "Worker retention response.workspace_id"),
|
workspace_id: boundedString(
|
||||||
runtime_id: boundedString(item.runtime_id, "Worker retention response.runtime_id"),
|
item.workspace_id,
|
||||||
worker_id: boundedString(item.worker_id, "Worker retention response.worker_id"),
|
"Worker retention response.workspace_id",
|
||||||
|
),
|
||||||
|
runtime_id: boundedString(
|
||||||
|
item.runtime_id,
|
||||||
|
"Worker retention response.runtime_id",
|
||||||
|
),
|
||||||
|
worker_id: boundedString(
|
||||||
|
item.worker_id,
|
||||||
|
"Worker retention response.worker_id",
|
||||||
|
),
|
||||||
pinned: boolean(item.pinned, "Worker retention response.pinned"),
|
pinned: boolean(item.pinned, "Worker retention response.pinned"),
|
||||||
retention_state: boundedString(
|
retention_state: boundedString(
|
||||||
item.retention_state,
|
item.retention_state,
|
||||||
@@ -268,7 +323,10 @@ export function parseRuntimeWorkerLifecycleResult(
|
|||||||
[],
|
[],
|
||||||
"Runtime Worker lifecycle result",
|
"Runtime Worker lifecycle result",
|
||||||
);
|
);
|
||||||
const state = boundedString(item.state, "Runtime Worker lifecycle result.state");
|
const state = boundedString(
|
||||||
|
item.state,
|
||||||
|
"Runtime Worker lifecycle result.state",
|
||||||
|
);
|
||||||
if (!WORKER_OPERATION_STATES.has(state as WorkerOperationState)) {
|
if (!WORKER_OPERATION_STATES.has(state as WorkerOperationState)) {
|
||||||
fail("Runtime Worker lifecycle result.state", "has an unknown value");
|
fail("Runtime Worker lifecycle result.state", "has an unknown value");
|
||||||
}
|
}
|
||||||
@@ -289,7 +347,58 @@ export function parseRuntimeWorkerLifecycleResult(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function parseRuntimeCleanupPlan(value: unknown): RuntimeCleanupPlanResponse {
|
/** Restore is a wrapped Workspace response, not the flat Stop lifecycle result. */
|
||||||
|
export function parseWorkerRestoreResponse(
|
||||||
|
value: unknown,
|
||||||
|
): WorkerRestoreResponse {
|
||||||
|
const path = "Worker Restore response";
|
||||||
|
const item = object(value, path);
|
||||||
|
exactKeys(
|
||||||
|
item,
|
||||||
|
["workspace_id", "runtime_id", "worker_id", "result"],
|
||||||
|
[],
|
||||||
|
path,
|
||||||
|
);
|
||||||
|
const result = object(item.result, `${path}.result`);
|
||||||
|
exactKeys(result, ["state"], ["diagnostics", "worker"], `${path}.result`);
|
||||||
|
const state = boundedString(result.state, `${path}.result.state`);
|
||||||
|
if (
|
||||||
|
!["accepted", "rejected", "rolled_back", "reconciliation_required"]
|
||||||
|
.includes(state)
|
||||||
|
) {
|
||||||
|
fail(`${path}.result.state`, "has an unknown value");
|
||||||
|
}
|
||||||
|
const workspace_id = boundedString(item.workspace_id, `${path}.workspace_id`);
|
||||||
|
const runtime_id = boundedString(item.runtime_id, `${path}.runtime_id`);
|
||||||
|
const worker_id = boundedString(item.worker_id, `${path}.worker_id`);
|
||||||
|
const worker = result.worker == null
|
||||||
|
? result.worker as null | undefined
|
||||||
|
: parseWorkerSummary(result.worker);
|
||||||
|
if (
|
||||||
|
worker &&
|
||||||
|
(worker.runtime_id !== runtime_id || worker.worker_id !== worker_id ||
|
||||||
|
(worker.workspace.workspace_id != null &&
|
||||||
|
worker.workspace.workspace_id !== workspace_id))
|
||||||
|
) {
|
||||||
|
fail(`${path}.result.worker`, "does not match the response identity");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
workspace_id,
|
||||||
|
runtime_id,
|
||||||
|
worker_id,
|
||||||
|
result: {
|
||||||
|
state: state as WorkerRestoreState,
|
||||||
|
diagnostics: result.diagnostics === undefined
|
||||||
|
? []
|
||||||
|
: diagnostics(result.diagnostics, `${path}.result.diagnostics`),
|
||||||
|
worker,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseRuntimeCleanupPlan(
|
||||||
|
value: unknown,
|
||||||
|
): RuntimeCleanupPlanResponse {
|
||||||
const item = object(value, "Runtime cleanup plan");
|
const item = object(value, "Runtime cleanup plan");
|
||||||
exactKeys(
|
exactKeys(
|
||||||
item,
|
item,
|
||||||
@@ -307,22 +416,44 @@ export function parseRuntimeCleanupPlan(value: unknown): RuntimeCleanupPlanRespo
|
|||||||
"Runtime cleanup plan",
|
"Runtime cleanup plan",
|
||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
workspace_id: boundedString(item.workspace_id, "Runtime cleanup plan.workspace_id"),
|
workspace_id: boundedString(
|
||||||
runtime_id: boundedString(item.runtime_id, "Runtime cleanup plan.runtime_id"),
|
item.workspace_id,
|
||||||
generated_at: boundedString(item.generated_at, "Runtime cleanup plan.generated_at"),
|
"Runtime cleanup plan.workspace_id",
|
||||||
|
),
|
||||||
|
runtime_id: boundedString(
|
||||||
|
item.runtime_id,
|
||||||
|
"Runtime cleanup plan.runtime_id",
|
||||||
|
),
|
||||||
|
generated_at: boundedString(
|
||||||
|
item.generated_at,
|
||||||
|
"Runtime cleanup plan.generated_at",
|
||||||
|
),
|
||||||
revision: boundedString(item.revision, "Runtime cleanup plan.revision"),
|
revision: boundedString(item.revision, "Runtime cleanup plan.revision"),
|
||||||
digest: boundedString(item.digest, "Runtime cleanup plan.digest"),
|
digest: boundedString(item.digest, "Runtime cleanup plan.digest"),
|
||||||
workers: array(item.workers, "Runtime cleanup plan.workers", MAX_CANDIDATES).map(
|
workers: array(item.workers, "Runtime cleanup plan.workers", MAX_CANDIDATES)
|
||||||
(entry, index) => workerCandidate(entry, `Runtime cleanup plan.workers[${index}]`),
|
.map(
|
||||||
|
(entry, index) =>
|
||||||
|
workerCandidate(entry, `Runtime cleanup plan.workers[${index}]`),
|
||||||
|
),
|
||||||
|
workdirs: array(
|
||||||
|
item.workdirs,
|
||||||
|
"Runtime cleanup plan.workdirs",
|
||||||
|
MAX_CANDIDATES,
|
||||||
|
).map(
|
||||||
|
(entry, index) =>
|
||||||
|
workdirCandidate(entry, `Runtime cleanup plan.workdirs[${index}]`),
|
||||||
),
|
),
|
||||||
workdirs: array(item.workdirs, "Runtime cleanup plan.workdirs", MAX_CANDIDATES).map(
|
diagnostics: diagnostics(
|
||||||
(entry, index) => workdirCandidate(entry, `Runtime cleanup plan.workdirs[${index}]`),
|
item.diagnostics,
|
||||||
|
"Runtime cleanup plan.diagnostics",
|
||||||
),
|
),
|
||||||
diagnostics: diagnostics(item.diagnostics, "Runtime cleanup plan.diagnostics"),
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function executionResult(value: unknown, path: string): RuntimeCleanupExecutionResult {
|
function executionResult(
|
||||||
|
value: unknown,
|
||||||
|
path: string,
|
||||||
|
): RuntimeCleanupExecutionResult {
|
||||||
const item = object(value, path);
|
const item = object(value, path);
|
||||||
exactKeys(item, ["target_id", "action", "status", "message"], [], path);
|
exactKeys(item, ["target_id", "action", "status", "message"], [], path);
|
||||||
return {
|
return {
|
||||||
@@ -339,18 +470,42 @@ export function parseRuntimeCleanupExecution(
|
|||||||
const item = object(value, "Runtime cleanup execution");
|
const item = object(value, "Runtime cleanup execution");
|
||||||
exactKeys(
|
exactKeys(
|
||||||
item,
|
item,
|
||||||
["workspace_id", "runtime_id", "executed_at", "results", "plan_after", "diagnostics"],
|
[
|
||||||
|
"workspace_id",
|
||||||
|
"runtime_id",
|
||||||
|
"executed_at",
|
||||||
|
"results",
|
||||||
|
"plan_after",
|
||||||
|
"diagnostics",
|
||||||
|
],
|
||||||
[],
|
[],
|
||||||
"Runtime cleanup execution",
|
"Runtime cleanup execution",
|
||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
workspace_id: boundedString(item.workspace_id, "Runtime cleanup execution.workspace_id"),
|
workspace_id: boundedString(
|
||||||
runtime_id: boundedString(item.runtime_id, "Runtime cleanup execution.runtime_id"),
|
item.workspace_id,
|
||||||
executed_at: boundedString(item.executed_at, "Runtime cleanup execution.executed_at"),
|
"Runtime cleanup execution.workspace_id",
|
||||||
results: array(item.results, "Runtime cleanup execution.results", MAX_RESULTS).map(
|
),
|
||||||
(entry, index) => executionResult(entry, `Runtime cleanup execution.results[${index}]`),
|
runtime_id: boundedString(
|
||||||
|
item.runtime_id,
|
||||||
|
"Runtime cleanup execution.runtime_id",
|
||||||
|
),
|
||||||
|
executed_at: boundedString(
|
||||||
|
item.executed_at,
|
||||||
|
"Runtime cleanup execution.executed_at",
|
||||||
|
),
|
||||||
|
results: array(
|
||||||
|
item.results,
|
||||||
|
"Runtime cleanup execution.results",
|
||||||
|
MAX_RESULTS,
|
||||||
|
).map(
|
||||||
|
(entry, index) =>
|
||||||
|
executionResult(entry, `Runtime cleanup execution.results[${index}]`),
|
||||||
),
|
),
|
||||||
plan_after: parseRuntimeCleanupPlan(item.plan_after),
|
plan_after: parseRuntimeCleanupPlan(item.plan_after),
|
||||||
diagnostics: diagnostics(item.diagnostics, "Runtime cleanup execution.diagnostics"),
|
diagnostics: diagnostics(
|
||||||
|
item.diagnostics,
|
||||||
|
"Runtime cleanup execution.diagnostics",
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,7 +97,11 @@ function optional<T>(
|
|||||||
label: string,
|
label: string,
|
||||||
parse: (item: unknown, label: string) => T,
|
parse: (item: unknown, label: string) => T,
|
||||||
): T | null | undefined {
|
): T | null | undefined {
|
||||||
return value === undefined ? undefined : value === null ? null : parse(value, label);
|
return value === undefined
|
||||||
|
? undefined
|
||||||
|
: value === null
|
||||||
|
? null
|
||||||
|
: parse(value, label);
|
||||||
}
|
}
|
||||||
|
|
||||||
function diagnostic(value: unknown, label: string): Diagnostic {
|
function diagnostic(value: unknown, label: string): Diagnostic {
|
||||||
@@ -372,6 +376,7 @@ export function parseWorkerSummary(
|
|||||||
"availability",
|
"availability",
|
||||||
"state",
|
"state",
|
||||||
"worker_state",
|
"worker_state",
|
||||||
|
"restore_observation_token",
|
||||||
"last_seen_at",
|
"last_seen_at",
|
||||||
"pinned",
|
"pinned",
|
||||||
"retention_state",
|
"retention_state",
|
||||||
@@ -406,13 +411,28 @@ export function parseWorkerSummary(
|
|||||||
`${label}.singleton_key`,
|
`${label}.singleton_key`,
|
||||||
string,
|
string,
|
||||||
),
|
),
|
||||||
tags: item.tags === undefined ? [] : array(item.tags, `${label}.tags`, string),
|
tags: item.tags === undefined
|
||||||
|
? []
|
||||||
|
: array(item.tags, `${label}.tags`, string),
|
||||||
workspace: workspaceSummary(item.workspace, `${label}.workspace`),
|
workspace: workspaceSummary(item.workspace, `${label}.workspace`),
|
||||||
availability,
|
availability,
|
||||||
state: string(item.state, `${label}.state`),
|
state: string(item.state, `${label}.state`),
|
||||||
worker_state: workerState,
|
worker_state: workerState,
|
||||||
|
restore_observation_token: optional(
|
||||||
|
item.restore_observation_token,
|
||||||
|
`${label}.restore_observation_token`,
|
||||||
|
(value, path) => {
|
||||||
|
const token = string(value, path);
|
||||||
|
if (new TextEncoder().encode(token).byteLength > 256) {
|
||||||
|
throw new Error(`${path} exceeds its 256 UTF-8 byte limit`);
|
||||||
|
}
|
||||||
|
return token;
|
||||||
|
},
|
||||||
|
),
|
||||||
last_seen_at: optional(item.last_seen_at, `${label}.last_seen_at`, string),
|
last_seen_at: optional(item.last_seen_at, `${label}.last_seen_at`, string),
|
||||||
pinned: item.pinned === undefined ? false : boolean(item.pinned, `${label}.pinned`),
|
pinned: item.pinned === undefined
|
||||||
|
? false
|
||||||
|
: boolean(item.pinned, `${label}.pinned`),
|
||||||
retention_state: item.retention_state === undefined
|
retention_state: item.retention_state === undefined
|
||||||
? ""
|
? ""
|
||||||
: string(item.retention_state, `${label}.retention_state`),
|
: string(item.retention_state, `${label}.retention_state`),
|
||||||
@@ -420,11 +440,13 @@ export function parseWorkerSummary(
|
|||||||
item.implementation,
|
item.implementation,
|
||||||
`${label}.implementation`,
|
`${label}.implementation`,
|
||||||
),
|
),
|
||||||
workdir_attachments: item.workdir_attachments === undefined ? undefined : array(
|
workdir_attachments: item.workdir_attachments === undefined
|
||||||
item.workdir_attachments,
|
? undefined
|
||||||
`${label}.workdir_attachments`,
|
: array(
|
||||||
workerWorkdirAttachment,
|
item.workdir_attachments,
|
||||||
),
|
`${label}.workdir_attachments`,
|
||||||
|
workerWorkdirAttachment,
|
||||||
|
),
|
||||||
diagnostics: item.diagnostics === undefined
|
diagnostics: item.diagnostics === undefined
|
||||||
? []
|
? []
|
||||||
: array(item.diagnostics, `${label}.diagnostics`, diagnostic),
|
: array(item.diagnostics, `${label}.diagnostics`, diagnostic),
|
||||||
@@ -634,11 +656,13 @@ function workerSummary(
|
|||||||
item.implementation,
|
item.implementation,
|
||||||
`${label}.implementation`,
|
`${label}.implementation`,
|
||||||
),
|
),
|
||||||
workdir_attachments: item.workdir_attachments === undefined ? undefined : array(
|
workdir_attachments: item.workdir_attachments === undefined
|
||||||
item.workdir_attachments,
|
? undefined
|
||||||
`${label}.workdir_attachments`,
|
: array(
|
||||||
runtimeWorkdirAttachment,
|
item.workdir_attachments,
|
||||||
),
|
`${label}.workdir_attachments`,
|
||||||
|
runtimeWorkdirAttachment,
|
||||||
|
),
|
||||||
diagnostics: array(item.diagnostics, `${label}.diagnostics`, diagnostic),
|
diagnostics: array(item.diagnostics, `${label}.diagnostics`, diagnostic),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -728,11 +752,17 @@ function featureConnectionsRequest(
|
|||||||
): WorkspaceWorkerFeatureConnectionsRequest {
|
): WorkspaceWorkerFeatureConnectionsRequest {
|
||||||
const item = record(value, label);
|
const item = record(value, label);
|
||||||
exact(item, ["subjektiv"], label);
|
exact(item, ["subjektiv"], label);
|
||||||
const subjektiv = optional(item.subjektiv, `${label}.subjektiv`, (value, itemLabel) => {
|
const subjektiv = optional(
|
||||||
const connection = record(value, itemLabel);
|
item.subjektiv,
|
||||||
exact(connection, ["subject_id"], itemLabel);
|
`${label}.subjektiv`,
|
||||||
return { subject_id: string(connection.subject_id, `${itemLabel}.subject_id`) };
|
(value, itemLabel) => {
|
||||||
});
|
const connection = record(value, itemLabel);
|
||||||
|
exact(connection, ["subject_id"], itemLabel);
|
||||||
|
return {
|
||||||
|
subject_id: string(connection.subject_id, `${itemLabel}.subject_id`),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
return subjektiv === undefined ? {} : { subjektiv };
|
return subjektiv === undefined ? {} : { subjektiv };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -841,18 +871,32 @@ function uploadedFile(
|
|||||||
function invocationValue(
|
function invocationValue(
|
||||||
value: unknown,
|
value: unknown,
|
||||||
label: string,
|
label: string,
|
||||||
): Extract<Extract<Segment, { kind: "feature_invoke" }>["invocation"]["arguments"][number], { name: string }>["value"] {
|
): Extract<
|
||||||
|
Extract<
|
||||||
|
Segment,
|
||||||
|
{ kind: "feature_invoke" }
|
||||||
|
>["invocation"]["arguments"][number],
|
||||||
|
{ name: string }
|
||||||
|
>["value"] {
|
||||||
const item = record(value, label);
|
const item = record(value, label);
|
||||||
const kind = string(item.kind, `${label}.kind`);
|
const kind = string(item.kind, `${label}.kind`);
|
||||||
exact(item, ["kind", "value"], label);
|
exact(item, ["kind", "value"], label);
|
||||||
if (kind === "string") return { kind, value: string(item.value, `${label}.value`) };
|
if (kind === "string") {
|
||||||
|
return { kind, value: string(item.value, `${label}.value`) };
|
||||||
|
}
|
||||||
if (kind === "integer") {
|
if (kind === "integer") {
|
||||||
const parsed = number(item.value, `${label}.value`);
|
const parsed = number(item.value, `${label}.value`);
|
||||||
if (!Number.isSafeInteger(parsed)) throw new Error(`${label}.value must be a safe integer`);
|
if (!Number.isSafeInteger(parsed)) {
|
||||||
if (parsed < -2147483648 || parsed > 2147483647) throw new Error(`${label}.value must be a signed 32-bit integer`);
|
throw new Error(`${label}.value must be a safe integer`);
|
||||||
|
}
|
||||||
|
if (parsed < -2147483648 || parsed > 2147483647) {
|
||||||
|
throw new Error(`${label}.value must be a signed 32-bit integer`);
|
||||||
|
}
|
||||||
return { kind, value: parsed };
|
return { kind, value: parsed };
|
||||||
}
|
}
|
||||||
if (kind === "boolean") return { kind, value: boolean(item.value, `${label}.value`) };
|
if (kind === "boolean") {
|
||||||
|
return { kind, value: boolean(item.value, `${label}.value`) };
|
||||||
|
}
|
||||||
throw new Error(`${label}.kind is invalid`);
|
throw new Error(`${label}.kind is invalid`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -866,14 +910,18 @@ function featureInvocation(
|
|||||||
invocation_id: string(item.invocation_id, `${label}.invocation_id`),
|
invocation_id: string(item.invocation_id, `${label}.invocation_id`),
|
||||||
identity: string(item.identity, `${label}.identity`),
|
identity: string(item.identity, `${label}.identity`),
|
||||||
name: string(item.name, `${label}.name`),
|
name: string(item.name, `${label}.name`),
|
||||||
arguments: array(item.arguments, `${label}.arguments`, (argument, argumentLabel) => {
|
arguments: array(
|
||||||
const entry = record(argument, argumentLabel);
|
item.arguments,
|
||||||
exact(entry, ["name", "value"], argumentLabel);
|
`${label}.arguments`,
|
||||||
return {
|
(argument, argumentLabel) => {
|
||||||
name: string(entry.name, `${argumentLabel}.name`),
|
const entry = record(argument, argumentLabel);
|
||||||
value: invocationValue(entry.value, `${argumentLabel}.value`),
|
exact(entry, ["name", "value"], argumentLabel);
|
||||||
};
|
return {
|
||||||
}),
|
name: string(entry.name, `${argumentLabel}.name`),
|
||||||
|
value: invocationValue(entry.value, `${argumentLabel}.value`),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,15 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
|
import { untrack } from 'svelte';
|
||||||
import Spinner from '#lib/workspace/console/Spinner.svelte';
|
import Spinner from '#lib/workspace/console/Spinner.svelte';
|
||||||
import { workerConsoleHref } from '#lib/workspace/resource-links.ts';
|
import { workerConsoleHref } from '#lib/workspace/resource-links.ts';
|
||||||
import { pushWorkspaceAlert } from '#lib/workspace/alerts/store.ts';
|
import { pushWorkspaceAlert } from '#lib/workspace/alerts/store.ts';
|
||||||
import {
|
import {
|
||||||
|
canRestoreWorker,
|
||||||
|
createRestoreRequest,
|
||||||
|
restoreWorkspaceWorker,
|
||||||
|
restoreNotice,
|
||||||
|
restoreErrorNotice,
|
||||||
|
type RestoreRequest,
|
||||||
canStopSidebarWorker,
|
canStopSidebarWorker,
|
||||||
canDeleteSidebarWorker,
|
canDeleteSidebarWorker,
|
||||||
deleteSidebarWorker,
|
deleteSidebarWorker,
|
||||||
@@ -10,6 +17,7 @@
|
|||||||
} from './worker-actions';
|
} from './worker-actions';
|
||||||
import {
|
import {
|
||||||
workspaceWorkersStore,
|
workspaceWorkersStore,
|
||||||
|
refreshWorkspaceWorkers,
|
||||||
type SidebarWorker,
|
type SidebarWorker,
|
||||||
} from './worker-subscription';
|
} from './worker-subscription';
|
||||||
import {
|
import {
|
||||||
@@ -21,7 +29,7 @@
|
|||||||
import { sidebarWorkdirMeta } from './worker-workdir-meta';
|
import { sidebarWorkdirMeta } from './worker-workdir-meta';
|
||||||
|
|
||||||
const COLLAPSED_WORKER_COUNT = 6;
|
const COLLAPSED_WORKER_COUNT = 6;
|
||||||
type WorkerActionKind = 'stop' | 'delete';
|
type WorkerActionKind = 'restore' | 'stop' | 'delete';
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
currentPath?: string;
|
currentPath?: string;
|
||||||
@@ -35,6 +43,7 @@
|
|||||||
let openWorkerKey = $state<string | null>(null);
|
let openWorkerKey = $state<string | null>(null);
|
||||||
let menuElement = $state<HTMLElement | null>(null);
|
let menuElement = $state<HTMLElement | null>(null);
|
||||||
let menuTrigger = $state<HTMLButtonElement | null>(null);
|
let menuTrigger = $state<HTMLButtonElement | null>(null);
|
||||||
|
let restoreRetries = $state<Record<string, RestoreRequest>>({});
|
||||||
let busyAction = $state<{ workerKey: string; kind: WorkerActionKind } | null>(null);
|
let busyAction = $state<{ workerKey: string; kind: WorkerActionKind } | null>(null);
|
||||||
let visibleWorkers = $derived(
|
let visibleWorkers = $derived(
|
||||||
visibleWorkersForSidebar(workers, {
|
visibleWorkersForSidebar(workers, {
|
||||||
@@ -90,58 +99,104 @@
|
|||||||
closeWorkerMenu(true);
|
closeWorkerMenu(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let lifetime = 0;
|
||||||
|
|
||||||
|
function actionScope() {
|
||||||
|
const id = workspaceId;
|
||||||
|
const epoch = lifetime;
|
||||||
|
return { workspaceId: id, isCurrent: () => lifetime === epoch && workspaceId === id };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function restoreWorker(worker: SidebarWorker, retry = false) {
|
||||||
|
const key = workerKey(worker);
|
||||||
|
if (busyAction || (retry ? !restoreRetries[key] : restoreRetries[key] || !canRestoreWorker(worker))) return;
|
||||||
|
const scope = actionScope();
|
||||||
|
const request = retry ? restoreRetries[key] : createRestoreRequest(worker);
|
||||||
|
closeWorkerMenu();
|
||||||
|
busyAction = { workerKey: key, kind: 'restore' };
|
||||||
|
try {
|
||||||
|
let notice;
|
||||||
|
try {
|
||||||
|
notice = restoreNotice(await restoreWorkspaceWorker(scope.workspaceId, worker, request));
|
||||||
|
} catch (cause) {
|
||||||
|
notice = restoreErrorNotice(cause);
|
||||||
|
}
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
|
const next = { ...restoreRetries };
|
||||||
|
if (notice.retry) next[key] = request;
|
||||||
|
else delete next[key];
|
||||||
|
restoreRetries = next;
|
||||||
|
pushWorkspaceAlert(notice.level, notice.message, { title: notice.title });
|
||||||
|
await refreshWorkspaceWorkers(scope.workspaceId).catch(() => {});
|
||||||
|
} finally {
|
||||||
|
if (scope.isCurrent()) busyAction = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function stopWorker(worker: SidebarWorker) {
|
async function stopWorker(worker: SidebarWorker) {
|
||||||
if (busyAction || !canStopSidebarWorker(worker)) return;
|
if (busyAction || !canStopSidebarWorker(worker)) return;
|
||||||
|
const scope = actionScope();
|
||||||
closeWorkerMenu();
|
closeWorkerMenu();
|
||||||
busyAction = { workerKey: workerKey(worker), kind: 'stop' };
|
busyAction = { workerKey: workerKey(worker), kind: 'stop' };
|
||||||
try {
|
try {
|
||||||
await stopSidebarWorker(workspaceId, worker);
|
await stopSidebarWorker(scope.workspaceId, worker);
|
||||||
workers = workers.map((item) =>
|
if (!scope.isCurrent()) return;
|
||||||
workerKey(item) === workerKey(worker)
|
await refreshWorkspaceWorkers(scope.workspaceId);
|
||||||
? { ...item, state: 'stopped', lifecycleState: 'stopped' }
|
if (!scope.isCurrent()) return;
|
||||||
: item
|
|
||||||
);
|
|
||||||
pushWorkspaceAlert('info', `${worker.display_name || worker.label} stopped`, {
|
pushWorkspaceAlert('info', `${worker.display_name || worker.label} stopped`, {
|
||||||
title: 'Worker stopped',
|
title: 'Worker stopped',
|
||||||
});
|
});
|
||||||
} catch (cause) {
|
} catch (cause) {
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
pushWorkspaceAlert('error', cause instanceof Error ? cause.message : 'Worker stop failed', {
|
pushWorkspaceAlert('error', cause instanceof Error ? cause.message : 'Worker stop failed', {
|
||||||
title: 'Worker stop failed',
|
title: 'Worker stop failed',
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
busyAction = null;
|
if (scope.isCurrent()) busyAction = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function deleteWorker(worker: SidebarWorker) {
|
async function deleteWorker(worker: SidebarWorker) {
|
||||||
if (busyAction || !canDeleteSidebarWorker(worker)) return;
|
if (busyAction || !canDeleteSidebarWorker(worker)) return;
|
||||||
|
const scope = actionScope();
|
||||||
closeWorkerMenu();
|
closeWorkerMenu();
|
||||||
busyAction = { workerKey: workerKey(worker), kind: 'delete' };
|
busyAction = { workerKey: workerKey(worker), kind: 'delete' };
|
||||||
try {
|
try {
|
||||||
await deleteSidebarWorker(workspaceId, worker);
|
await deleteSidebarWorker(scope.workspaceId, worker, (input, init) => {
|
||||||
workers = workers.filter((item) => workerKey(item) !== workerKey(worker));
|
if (!scope.isCurrent()) throw new Error('Worker action is no longer current');
|
||||||
|
return fetch(input, init);
|
||||||
|
});
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
|
await refreshWorkspaceWorkers(scope.workspaceId);
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
pushWorkspaceAlert('info', `${worker.display_name || worker.label} deleted`, {
|
pushWorkspaceAlert('info', `${worker.display_name || worker.label} deleted`, {
|
||||||
title: 'Worker deleted',
|
title: 'Worker deleted',
|
||||||
});
|
});
|
||||||
} catch (cause) {
|
} catch (cause) {
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
pushWorkspaceAlert('error', cause instanceof Error ? cause.message : 'Worker deletion failed', {
|
pushWorkspaceAlert('error', cause instanceof Error ? cause.message : 'Worker deletion failed', {
|
||||||
title: 'Worker deletion failed',
|
title: 'Worker deletion failed',
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
busyAction = null;
|
if (scope.isCurrent()) busyAction = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
|
const id = workspaceId;
|
||||||
|
lifetime++;
|
||||||
|
busyAction = null;
|
||||||
|
restoreRetries = {};
|
||||||
expanded = false;
|
expanded = false;
|
||||||
openWorkerKey = null;
|
untrack(closeWorkerMenu);
|
||||||
menuElement = null;
|
const unsubscribe = workspaceWorkersStore(id).subscribe((state) => {
|
||||||
menuTrigger = null;
|
|
||||||
const subscription = workspaceWorkersStore(workspaceId);
|
|
||||||
return subscription.subscribe((state) => {
|
|
||||||
loading = state.loading;
|
loading = state.loading;
|
||||||
workers = state.workers.filter(canShowWorkerInSidebar);
|
workers = state.workers.filter(canShowWorkerInSidebar);
|
||||||
});
|
});
|
||||||
|
return () => {
|
||||||
|
lifetime++;
|
||||||
|
unsubscribe();
|
||||||
|
};
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
@@ -221,6 +276,23 @@
|
|||||||
</button>
|
</button>
|
||||||
{#if openWorkerKey === key}
|
{#if openWorkerKey === key}
|
||||||
<div class="worker-actions-menu" role="menu" aria-label={`Actions for ${label}`} bind:this={menuElement}>
|
<div class="worker-actions-menu" role="menu" aria-label={`Actions for ${label}`} bind:this={menuElement}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
role="menuitem"
|
||||||
|
disabled={busyAction !== null || !canRestoreWorker(worker) || Boolean(restoreRetries[key])}
|
||||||
|
onclick={() => restoreWorker(worker)}
|
||||||
|
>
|
||||||
|
{isBusy(worker, 'restore') ? 'Restoring…' : 'Restore'}
|
||||||
|
</button>
|
||||||
|
{#if restoreRetries[key]}
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
role="menuitem"
|
||||||
|
disabled={busyAction !== null}
|
||||||
|
onclick={() => restoreWorker(worker, true)}
|
||||||
|
>Retry Restore</button>
|
||||||
|
<p role="status">Restore outcome unresolved. Retry the same request to reconcile.</p>
|
||||||
|
{/if}
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
role="menuitem"
|
role="menuitem"
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import type {
|
|||||||
CleanupWorkerCandidate,
|
CleanupWorkerCandidate,
|
||||||
RuntimeCleanupExecutionResponse,
|
RuntimeCleanupExecutionResponse,
|
||||||
RuntimeCleanupPlanResponse,
|
RuntimeCleanupPlanResponse,
|
||||||
|
WorkerRestoreResult,
|
||||||
|
WorkerRestoreState,
|
||||||
} from "#lib/generated/runtime-api.ts";
|
} from "#lib/generated/runtime-api.ts";
|
||||||
import type {
|
import type {
|
||||||
BrowserCreateWorkerResponse as SharedBrowserCreateWorkerResponse,
|
BrowserCreateWorkerResponse as SharedBrowserCreateWorkerResponse,
|
||||||
@@ -13,7 +15,6 @@ import type {
|
|||||||
WorkerLaunchOptionsResponse as SharedWorkerLaunchOptionsResponse,
|
WorkerLaunchOptionsResponse as SharedWorkerLaunchOptionsResponse,
|
||||||
WorkerLaunchProfileCandidate as SharedWorkerLaunchProfileCandidate,
|
WorkerLaunchProfileCandidate as SharedWorkerLaunchProfileCandidate,
|
||||||
WorkerLaunchRuntimeOption as SharedWorkerLaunchRuntimeOption,
|
WorkerLaunchRuntimeOption as SharedWorkerLaunchRuntimeOption,
|
||||||
WorkerLaunchWorkerSummary,
|
|
||||||
WorkerSummary as SharedWorkerSummary,
|
WorkerSummary as SharedWorkerSummary,
|
||||||
WorkerWorkdirAttachmentSummary as SharedWorkerWorkdirAttachmentSummary,
|
WorkerWorkdirAttachmentSummary as SharedWorkerWorkdirAttachmentSummary,
|
||||||
WorkingDirectoryRepositoryOption as SharedWorkingDirectoryRepositoryOption,
|
WorkingDirectoryRepositoryOption as SharedWorkingDirectoryRepositoryOption,
|
||||||
@@ -95,18 +96,7 @@ export type Worker =
|
|||||||
|
|
||||||
export type WorkerOperationState = "accepted" | "unsupported" | "rejected";
|
export type WorkerOperationState = "accepted" | "unsupported" | "rejected";
|
||||||
|
|
||||||
/** Typed result of restore across Runtime, Workspace, Web, and TUI clients. */
|
export type { WorkerRestoreResult, WorkerRestoreState };
|
||||||
export type WorkerRestoreState =
|
|
||||||
| "accepted"
|
|
||||||
| "rejected"
|
|
||||||
| "rolled_back"
|
|
||||||
| "reconciliation_required";
|
|
||||||
|
|
||||||
export type WorkerRestoreResult = {
|
|
||||||
state: WorkerRestoreState;
|
|
||||||
worker?: WorkerLaunchWorkerSummary | null;
|
|
||||||
diagnostics: Diagnostic[];
|
|
||||||
};
|
|
||||||
|
|
||||||
export type WorkerLaunchRuntimeOption = SharedWorkerLaunchRuntimeOption;
|
export type WorkerLaunchRuntimeOption = SharedWorkerLaunchRuntimeOption;
|
||||||
export type WorkerLaunchProfileCandidate = SharedWorkerLaunchProfileCandidate;
|
export type WorkerLaunchProfileCandidate = SharedWorkerLaunchProfileCandidate;
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
import { workspaceApiPath } from "#lib/workspace/api/http.ts";
|
import type { WorkerRestoreRequest } from "#lib/generated/runtime-api.ts";
|
||||||
|
import { readBoundedJson, workspaceApiPath } from "#lib/workspace/api/http.ts";
|
||||||
import {
|
import {
|
||||||
parseRuntimeCleanupExecution,
|
parseRuntimeCleanupExecution,
|
||||||
parseRuntimeCleanupPlan,
|
parseRuntimeCleanupPlan,
|
||||||
parseRuntimeWorkerLifecycleResult,
|
parseRuntimeWorkerLifecycleResult,
|
||||||
|
parseWorkerRestoreResponse,
|
||||||
} from "#lib/workspace/api/runtime-workers.ts";
|
} from "#lib/workspace/api/runtime-workers.ts";
|
||||||
import type { Diagnostic, Worker } from "./types";
|
import type { Diagnostic, Worker } from "./types";
|
||||||
|
|
||||||
@@ -21,7 +23,7 @@ function workerPath(workspaceId: string, worker: WorkerActionTarget): string {
|
|||||||
async function responseError(response: Response): Promise<string> {
|
async function responseError(response: Response): Promise<string> {
|
||||||
const fallback = `${response.status} ${response.statusText}`.trim();
|
const fallback = `${response.status} ${response.statusText}`.trim();
|
||||||
try {
|
try {
|
||||||
const payload: unknown = await response.json();
|
const payload: unknown = await readBoundedJson(response, 64 * 1024);
|
||||||
if (
|
if (
|
||||||
typeof payload !== "object" || payload === null || Array.isArray(payload)
|
typeof payload !== "object" || payload === null || Array.isArray(payload)
|
||||||
) {
|
) {
|
||||||
@@ -52,6 +54,134 @@ function diagnosticMessage(
|
|||||||
fallback;
|
fallback;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function canRestoreWorker(worker: {
|
||||||
|
availability: Worker["availability"];
|
||||||
|
state?: string;
|
||||||
|
lifecycleState?: string;
|
||||||
|
restore_observation_token?: string | null;
|
||||||
|
}): boolean {
|
||||||
|
return worker.availability === "observed" &&
|
||||||
|
(worker.lifecycleState ?? worker.state) === "stopped" &&
|
||||||
|
Boolean(worker.restore_observation_token);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Immutable intent: retained unchanged for explicit uncertain-result recovery. */
|
||||||
|
export type RestoreRequest = WorkerRestoreRequest;
|
||||||
|
|
||||||
|
export function createRestoreRequest(
|
||||||
|
worker: { restore_observation_token?: string | null },
|
||||||
|
): RestoreRequest {
|
||||||
|
if (!worker.restore_observation_token) {
|
||||||
|
throw new Error(
|
||||||
|
"Worker Restore observation is unavailable; refresh Workers",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
expected_observation_token: worker.restore_observation_token,
|
||||||
|
request_id: crypto.randomUUID(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export class WorkerActionHttpError extends Error {
|
||||||
|
constructor(public readonly status: number, message: string) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function restoreWorkspaceWorker(
|
||||||
|
workspaceId: string,
|
||||||
|
worker: WorkerActionTarget,
|
||||||
|
request: RestoreRequest,
|
||||||
|
fetchFn: FetchFn = fetch,
|
||||||
|
) {
|
||||||
|
const response = await fetchFn(`${workerPath(workspaceId, worker)}/restore`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify(request),
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new WorkerActionHttpError(
|
||||||
|
response.status,
|
||||||
|
await responseError(response),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const result = parseWorkerRestoreResponse(
|
||||||
|
await readBoundedJson(response, 8 * 1024 * 1024),
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
result.workspace_id !== workspaceId ||
|
||||||
|
result.runtime_id !== worker.runtime_id ||
|
||||||
|
result.worker_id !== worker.worker_id
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
"Worker Restore response does not match the requested target",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RestoreNotice = {
|
||||||
|
level: "info" | "warning" | "error";
|
||||||
|
title: string;
|
||||||
|
message: string;
|
||||||
|
retry: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Shared classification: neither an HTTP 200 nor a GET settles an unknown operation. */
|
||||||
|
export function restoreNotice(
|
||||||
|
result: Awaited<ReturnType<typeof restoreWorkspaceWorker>>,
|
||||||
|
): RestoreNotice {
|
||||||
|
const state = result.result.state;
|
||||||
|
if (state === "accepted") {
|
||||||
|
return {
|
||||||
|
level: "info",
|
||||||
|
title: "Worker restored",
|
||||||
|
message: "Worker restored. Open Console to continue.",
|
||||||
|
retry: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const pending = state === "reconciliation_required";
|
||||||
|
return {
|
||||||
|
level: pending ? "warning" : "error",
|
||||||
|
title: pending
|
||||||
|
? "Worker Restore needs reconciliation"
|
||||||
|
: "Worker Restore failed",
|
||||||
|
message: diagnosticMessage(
|
||||||
|
result.result.diagnostics,
|
||||||
|
`Worker Restore was ${state}`,
|
||||||
|
) +
|
||||||
|
(pending
|
||||||
|
? ". Outcome is unresolved. Refresh observations and explicitly retry the same Restore to reconcile; refreshing alone does not settle it."
|
||||||
|
: ""),
|
||||||
|
retry: pending,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function restoreErrorNotice(cause: unknown): RestoreNotice {
|
||||||
|
const conflict = cause instanceof WorkerActionHttpError &&
|
||||||
|
cause.status === 409;
|
||||||
|
const uncertain = !(cause instanceof WorkerActionHttpError) ||
|
||||||
|
cause.status >= 500;
|
||||||
|
return {
|
||||||
|
level: conflict || uncertain ? "warning" : "error",
|
||||||
|
title: conflict
|
||||||
|
? "Worker observation changed"
|
||||||
|
: uncertain
|
||||||
|
? "Worker Restore outcome unknown"
|
||||||
|
: "Worker Restore failed",
|
||||||
|
message:
|
||||||
|
(cause instanceof Error
|
||||||
|
? cause.message
|
||||||
|
: "Worker Restore request failed") +
|
||||||
|
(conflict
|
||||||
|
? ". Workers refreshed; review the latest state and explicitly try again."
|
||||||
|
: uncertain
|
||||||
|
? ". Outcome is unknown. Check the latest state and explicitly retry the same Restore; no automatic retry was made."
|
||||||
|
: ""),
|
||||||
|
retry: uncertain,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function canStopSidebarWorker(worker: {
|
export function canStopSidebarWorker(worker: {
|
||||||
availability: Worker["availability"];
|
availability: Worker["availability"];
|
||||||
lifecycleState: string;
|
lifecycleState: string;
|
||||||
|
|||||||
@@ -0,0 +1,233 @@
|
|||||||
|
// @vitest-environment happy-dom
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type {
|
||||||
|
SubscriptionFrame,
|
||||||
|
SubscriptionWorker,
|
||||||
|
} from "#lib/generated/protocol.ts";
|
||||||
|
import type { WorkspaceWorkersState } from "./worker-subscription";
|
||||||
|
|
||||||
|
const mock = vi.hoisted(() => ({ handlers: null as any }));
|
||||||
|
vi.mock("#lib/workspace/multiplexer.ts", () => ({
|
||||||
|
workspaceMultiplexer: () => ({
|
||||||
|
subscribe: (_: unknown, handlers: unknown) => {
|
||||||
|
mock.handlers = handlers;
|
||||||
|
return { close: vi.fn() };
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
vi.mock(
|
||||||
|
"#lib/workspace/alerts/store.ts",
|
||||||
|
() => ({ pushWorkspaceAlert: vi.fn(), dismissWorkspaceAlert: vi.fn() }),
|
||||||
|
);
|
||||||
|
import {
|
||||||
|
disposeWorkspaceWorkersStore,
|
||||||
|
refreshWorkspaceWorkers,
|
||||||
|
workspaceWorkersStore,
|
||||||
|
} from "./worker-subscription";
|
||||||
|
|
||||||
|
function summary(state = "stopped", token = "generation-1") {
|
||||||
|
return {
|
||||||
|
runtime_id: "runtime",
|
||||||
|
worker_id: "worker",
|
||||||
|
resource_key: "W-1",
|
||||||
|
host_id: "runtime",
|
||||||
|
display_name: "test",
|
||||||
|
label: "test",
|
||||||
|
availability: "observed",
|
||||||
|
state,
|
||||||
|
restore_observation_token: token,
|
||||||
|
workspace: {
|
||||||
|
visibility: "workspace",
|
||||||
|
identity: "registered",
|
||||||
|
workspace_id: "team",
|
||||||
|
},
|
||||||
|
implementation: { kind: "runtime", display_hint: "Runtime" },
|
||||||
|
pinned: true,
|
||||||
|
retention_state: "pinned",
|
||||||
|
workdir_attachments: [],
|
||||||
|
diagnostics: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function response(state = "stopped", token = "generation-1") {
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
workspace_id: "team",
|
||||||
|
limit: 100,
|
||||||
|
items: [summary(state, token)],
|
||||||
|
source: "catalog",
|
||||||
|
diagnostics: [],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
function snapshot(
|
||||||
|
state = "stopped",
|
||||||
|
token = "generation-1",
|
||||||
|
): SubscriptionFrame {
|
||||||
|
const worker = {
|
||||||
|
worker_id: "worker",
|
||||||
|
runtime_id: "runtime",
|
||||||
|
resource_key: "W-1",
|
||||||
|
availability: "observed",
|
||||||
|
state,
|
||||||
|
restore_observation_token: token,
|
||||||
|
has_running_internal_workers: false,
|
||||||
|
workspace_id: "team",
|
||||||
|
workdir_attachments: [],
|
||||||
|
} as SubscriptionWorker;
|
||||||
|
return {
|
||||||
|
protocol_version: 2,
|
||||||
|
frame: "response",
|
||||||
|
message: {
|
||||||
|
result: "subscribed",
|
||||||
|
payload: {
|
||||||
|
request_id: "request",
|
||||||
|
subscription_id: "sub",
|
||||||
|
selector: { topic: "workspace_workers" },
|
||||||
|
snapshot: { topic: "workers", data: { workers: [worker] } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const settle = async () => {
|
||||||
|
for (let i = 0; i < 12; i++) await Promise.resolve();
|
||||||
|
};
|
||||||
|
let unsubscribe: (() => void) | null;
|
||||||
|
beforeEach(() => {
|
||||||
|
mock.handlers = null;
|
||||||
|
unsubscribe = null;
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
unsubscribe?.();
|
||||||
|
disposeWorkspaceWorkersStore("team");
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("shared catalog observation refresh", () => {
|
||||||
|
it("keeps full catalog metadata while exposing catalog lifecycle separately from foreground display", async () => {
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn(async (url: string) =>
|
||||||
|
url.endsWith("/workers")
|
||||||
|
? response()
|
||||||
|
: new Response(JSON.stringify({ items: [] }))
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let latest!: WorkspaceWorkersState;
|
||||||
|
unsubscribe = workspaceWorkersStore("team").subscribe((state) =>
|
||||||
|
latest = state
|
||||||
|
);
|
||||||
|
mock.handlers.onFrame(snapshot());
|
||||||
|
await settle();
|
||||||
|
expect(latest.catalogWorkers?.[0].pinned).toBe(true);
|
||||||
|
expect(latest.catalogWorkers?.[0].retention_state).toBe("pinned");
|
||||||
|
expect(latest.workers[0].lifecycleState).toBe("stopped");
|
||||||
|
expect(latest.workers[0].restore_observation_token).toBe("generation-1");
|
||||||
|
});
|
||||||
|
it("invalidates dependent plans before refresh and discards a GET superseded by a newer catalog event", async () => {
|
||||||
|
const pending: Array<(response: Response) => void> = [];
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn((url: string) =>
|
||||||
|
url.endsWith("/workers")
|
||||||
|
? new Promise<Response>((resolve) => pending.push(resolve))
|
||||||
|
: Promise.resolve(new Response(JSON.stringify({ items: [] })))
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let latest!: WorkspaceWorkersState;
|
||||||
|
unsubscribe = workspaceWorkersStore("team").subscribe((state) =>
|
||||||
|
latest = state
|
||||||
|
);
|
||||||
|
mock.handlers.onFrame(snapshot());
|
||||||
|
const before = latest.observationVersion;
|
||||||
|
const old = pending.at(-1)!;
|
||||||
|
mock.handlers.onFrame(snapshot("idle", "generation-2"));
|
||||||
|
expect(latest.observationVersion).toBeGreaterThan(before);
|
||||||
|
expect(latest.workers[0].lifecycleState).toBe("idle");
|
||||||
|
pending.at(-1)!(response("idle", "generation-2"));
|
||||||
|
await settle();
|
||||||
|
old(response("stopped", "generation-1"));
|
||||||
|
await settle();
|
||||||
|
expect(latest.catalogWorkers?.[0].state).toBe("idle");
|
||||||
|
expect(latest.workers[0].restore_observation_token).toBe("generation-2");
|
||||||
|
});
|
||||||
|
it("explicit uncertain-outcome refresh is only GET and cannot settle or replay Restore", async () => {
|
||||||
|
const fetchMock = vi.fn(async (url: string) =>
|
||||||
|
url.endsWith("/workers")
|
||||||
|
? response()
|
||||||
|
: new Response(JSON.stringify({ items: [] }))
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", fetchMock);
|
||||||
|
let latest!: WorkspaceWorkersState;
|
||||||
|
unsubscribe = workspaceWorkersStore("team").subscribe((state) =>
|
||||||
|
latest = state
|
||||||
|
);
|
||||||
|
mock.handlers.onFrame(snapshot());
|
||||||
|
await settle();
|
||||||
|
const before = latest.observationVersion;
|
||||||
|
await refreshWorkspaceWorkers("team");
|
||||||
|
expect(latest.observationVersion).toBeGreaterThan(before);
|
||||||
|
expect(fetchMock.mock.calls.every(([url]) => !url.endsWith("/restore")))
|
||||||
|
.toBe(true);
|
||||||
|
});
|
||||||
|
it("does not filter catalog-only targets on a stream update while its metadata refresh is pending", async () => {
|
||||||
|
let hold = false;
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn(async (url: string) => {
|
||||||
|
if (!url.endsWith("/workers")) {
|
||||||
|
return new Response(JSON.stringify({ items: [] }));
|
||||||
|
}
|
||||||
|
if (hold) return new Promise<Response>(() => {});
|
||||||
|
return Response.json({
|
||||||
|
workspace_id: "team",
|
||||||
|
limit: 100,
|
||||||
|
source: "catalog",
|
||||||
|
diagnostics: [],
|
||||||
|
items: [summary(), {
|
||||||
|
...summary(),
|
||||||
|
worker_id: "registry-only",
|
||||||
|
resource_key: "W-2",
|
||||||
|
implementation: {
|
||||||
|
kind: "backend_worker_registry",
|
||||||
|
display_hint: "registry",
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let latest!: WorkspaceWorkersState;
|
||||||
|
unsubscribe = workspaceWorkersStore("team").subscribe((state) =>
|
||||||
|
latest = state
|
||||||
|
);
|
||||||
|
mock.handlers.onFrame(snapshot());
|
||||||
|
await settle();
|
||||||
|
expect(latest.catalogWorkers).toHaveLength(2);
|
||||||
|
hold = true;
|
||||||
|
mock.handlers.onFrame(snapshot("idle", "generation-2"));
|
||||||
|
expect(latest.catalogRefreshing).toBe(true);
|
||||||
|
expect(latest.catalogWorkers).toHaveLength(2);
|
||||||
|
expect(latest.catalogWorkers?.[1].implementation.kind).toBe(
|
||||||
|
"backend_worker_registry",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
it("does not publish a late catalog GET after disposal", async () => {
|
||||||
|
let resolve!: (response: Response) => void;
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn((url: string) =>
|
||||||
|
url.endsWith("/workers")
|
||||||
|
? new Promise<Response>((r) => resolve = r)
|
||||||
|
: Promise.resolve(new Response(JSON.stringify({ items: [] })))
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const seen: WorkspaceWorkersState[] = [];
|
||||||
|
unsubscribe = workspaceWorkersStore("team").subscribe((state) =>
|
||||||
|
seen.push(state)
|
||||||
|
);
|
||||||
|
const before = seen.length;
|
||||||
|
disposeWorkspaceWorkersStore("team");
|
||||||
|
resolve(response("idle", "generation-2"));
|
||||||
|
await settle();
|
||||||
|
expect(seen.length).toBe(before);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -14,6 +14,10 @@ import type {
|
|||||||
} from "#lib/generated/protocol.ts";
|
} from "#lib/generated/protocol.ts";
|
||||||
import WorkersNavSection from "./WorkersNavSection.svelte";
|
import WorkersNavSection from "./WorkersNavSection.svelte";
|
||||||
import { disposeWorkspaceWorkersStore } from "./worker-subscription";
|
import { disposeWorkspaceWorkersStore } from "./worker-subscription";
|
||||||
|
import {
|
||||||
|
clearWorkspaceAlerts,
|
||||||
|
workspaceAlerts,
|
||||||
|
} from "#lib/workspace/alerts/store.ts";
|
||||||
|
|
||||||
const transport = vi.hoisted(() => ({
|
const transport = vi.hoisted(() => ({
|
||||||
listener: null as { onFrame(frame: SubscriptionFrame): void } | null,
|
listener: null as { onFrame(frame: SubscriptionFrame): void } | null,
|
||||||
@@ -34,6 +38,8 @@ vi.mock("#lib/workspace/multiplexer.ts", () => ({
|
|||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
cleanup();
|
cleanup();
|
||||||
disposeWorkspaceWorkersStore("stop-test");
|
disposeWorkspaceWorkersStore("stop-test");
|
||||||
|
disposeWorkspaceWorkersStore("next-test");
|
||||||
|
clearWorkspaceAlerts();
|
||||||
transport.listener = null;
|
transport.listener = null;
|
||||||
vi.unstubAllGlobals();
|
vi.unstubAllGlobals();
|
||||||
});
|
});
|
||||||
@@ -41,6 +47,7 @@ afterEach(() => {
|
|||||||
function snapshot(
|
function snapshot(
|
||||||
state: SubscriptionWorker["state"],
|
state: SubscriptionWorker["state"],
|
||||||
availability: SubscriptionWorker["availability"],
|
availability: SubscriptionWorker["availability"],
|
||||||
|
workspaceId = "stop-test",
|
||||||
): SubscriptionFrame {
|
): SubscriptionFrame {
|
||||||
return {
|
return {
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
@@ -64,8 +71,9 @@ function snapshot(
|
|||||||
worker_state: null,
|
worker_state: null,
|
||||||
has_running_internal_workers: false,
|
has_running_internal_workers: false,
|
||||||
display_name: "Test worker",
|
display_name: "Test worker",
|
||||||
workspace_id: "stop-test",
|
workspace_id: workspaceId,
|
||||||
workdir_attachments: [],
|
workdir_attachments: [],
|
||||||
|
...{ restore_observation_token: "observation-1" },
|
||||||
}],
|
}],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -107,6 +115,14 @@ test.each(["idle", "running", "paused", "stopped"] as const)(
|
|||||||
await fireEvent.click(
|
await fireEvent.click(
|
||||||
screen.getByRole("button", { name: "Actions for Test worker" }),
|
screen.getByRole("button", { name: "Actions for Test worker" }),
|
||||||
);
|
);
|
||||||
|
await waitFor(() =>
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("menuitem", { name: "Stop" }) as HTMLButtonElement)
|
||||||
|
.disabled,
|
||||||
|
).toBe(false)
|
||||||
|
);
|
||||||
|
// A successful POST is not a lifecycle observation. Only a newer authoritative frame disables Stop.
|
||||||
|
transport.listener!.onFrame(snapshot("stopped", "observed"));
|
||||||
await waitFor(() =>
|
await waitFor(() =>
|
||||||
expect(
|
expect(
|
||||||
(screen.getByRole("menuitem", { name: "Stop" }) as HTMLButtonElement)
|
(screen.getByRole("menuitem", { name: "Stop" }) as HTMLButtonElement)
|
||||||
@@ -132,3 +148,249 @@ test("Unavailable observation disables Stop even when catalog state is running",
|
|||||||
.disabled,
|
.disabled,
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test.each(["switch", "dispose"] as const)(
|
||||||
|
"late Stop completion is ignored after workspace %s",
|
||||||
|
async (mode) => {
|
||||||
|
let resolve!: (response: Response) => void;
|
||||||
|
const pending = new Promise<Response>((done) => {
|
||||||
|
resolve = done;
|
||||||
|
});
|
||||||
|
const request = vi.fn((input: RequestInfo | URL) =>
|
||||||
|
String(input).endsWith("/stop")
|
||||||
|
? pending
|
||||||
|
: Promise.resolve(new Response(null, { status: 404 }))
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
const view = render(WorkersNavSection, { workspaceId: "stop-test" });
|
||||||
|
await waitFor(() => expect(transport.listener).not.toBeNull());
|
||||||
|
transport.listener!.onFrame(snapshot("running", "observed"));
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Actions for Test worker" }),
|
||||||
|
);
|
||||||
|
await fireEvent.click(screen.getByRole("menuitem", { name: "Stop" }));
|
||||||
|
const operation = request.mock.calls.find(([url]) =>
|
||||||
|
String(url).endsWith("/stop")
|
||||||
|
);
|
||||||
|
expect(String(operation?.[0])).toContain("/api/w/stop-test/");
|
||||||
|
if (mode === "switch") {
|
||||||
|
await view.rerender({ workspaceId: "next-test" });
|
||||||
|
transport.listener!.onFrame(snapshot("idle", "observed", "next-test"));
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Actions for Test worker" }),
|
||||||
|
);
|
||||||
|
} else view.unmount();
|
||||||
|
clearWorkspaceAlerts();
|
||||||
|
let alerts: unknown[] = [];
|
||||||
|
const unsubscribe = workspaceAlerts.subscribe((value) => {
|
||||||
|
alerts = value;
|
||||||
|
});
|
||||||
|
const readsBeforeCompletion =
|
||||||
|
request.mock.calls.filter(([url]) => String(url).endsWith("/workers"))
|
||||||
|
.length;
|
||||||
|
resolve(
|
||||||
|
Response.json({
|
||||||
|
state: "accepted",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-a",
|
||||||
|
diagnostics: [],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await pending;
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(alerts).toEqual([]);
|
||||||
|
expect(
|
||||||
|
request.mock.calls.filter(([url]) => String(url).endsWith("/workers"))
|
||||||
|
.length,
|
||||||
|
).toBe(readsBeforeCompletion);
|
||||||
|
if (mode === "switch") {
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("menuitem", { name: "Stop" }) as HTMLButtonElement)
|
||||||
|
.disabled,
|
||||||
|
).toBe(false);
|
||||||
|
}
|
||||||
|
unsubscribe();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test.each(["idle", "running", "paused", "stopped"] as const)(
|
||||||
|
"sidebar Restore eligibility (%s)",
|
||||||
|
async (state) => {
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn(async () => new Response(null, { status: 404 })),
|
||||||
|
);
|
||||||
|
render(WorkersNavSection, { workspaceId: "stop-test" });
|
||||||
|
await waitFor(() => expect(transport.listener).not.toBeNull());
|
||||||
|
transport.listener!.onFrame(snapshot(state, "observed"));
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Actions for Test worker" }),
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("menuitem", { name: "Restore" }) as HTMLButtonElement)
|
||||||
|
.disabled,
|
||||||
|
).toBe(state !== "stopped");
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test("sidebar Restore pins the target and preserves an uncertain request for explicit retry", async () => {
|
||||||
|
let attempts = 0;
|
||||||
|
const request = vi.fn(
|
||||||
|
async (input: RequestInfo | URL, _init?: RequestInit) => {
|
||||||
|
if (!String(input).endsWith("/restore")) {
|
||||||
|
return new Response(null, { status: 404 });
|
||||||
|
}
|
||||||
|
return Response.json({
|
||||||
|
workspace_id: "stop-test",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-a",
|
||||||
|
result: {
|
||||||
|
state: ++attempts === 1 ? "reconciliation_required" : "accepted",
|
||||||
|
diagnostics: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
render(WorkersNavSection, { workspaceId: "stop-test" });
|
||||||
|
await waitFor(() => expect(transport.listener).not.toBeNull());
|
||||||
|
transport.listener!.onFrame(snapshot("stopped", "observed"));
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Actions for Test worker" }),
|
||||||
|
);
|
||||||
|
await fireEvent.click(screen.getByRole("menuitem", { name: "Restore" }));
|
||||||
|
await waitFor(() => expect(attempts).toBe(1));
|
||||||
|
await fireEvent.click(
|
||||||
|
screen.getByRole("button", { name: "Actions for Test worker" }),
|
||||||
|
);
|
||||||
|
const retry = await screen.findByRole("menuitem", { name: "Retry Restore" });
|
||||||
|
await waitFor(() =>
|
||||||
|
expect((retry as HTMLButtonElement).disabled).toBe(false)
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("menuitem", { name: "Restore" }) as HTMLButtonElement)
|
||||||
|
.disabled,
|
||||||
|
).toBe(true);
|
||||||
|
await fireEvent.click(retry);
|
||||||
|
await waitFor(() => expect(attempts).toBe(2));
|
||||||
|
const operations = request.mock.calls.filter(([url]) =>
|
||||||
|
String(url).endsWith("/restore")
|
||||||
|
);
|
||||||
|
expect(String(operations[0][0])).toContain(
|
||||||
|
"/api/w/stop-test/runtimes/runtime-a/workers/worker-a/restore",
|
||||||
|
);
|
||||||
|
expect(operations[1][1]?.body).toBe(operations[0][1]?.body);
|
||||||
|
});
|
||||||
|
|
||||||
|
test.each(["switch", "dispose"] as const)(
|
||||||
|
"late sidebar Restore completion is ignored after %s",
|
||||||
|
async (mode) => {
|
||||||
|
let resolve!: (response: Response) => void;
|
||||||
|
const pending = new Promise<Response>((done) => {
|
||||||
|
resolve = done;
|
||||||
|
});
|
||||||
|
const request = vi.fn((input: RequestInfo | URL) =>
|
||||||
|
String(input).endsWith("/restore")
|
||||||
|
? pending
|
||||||
|
: Promise.resolve(new Response(null, { status: 404 }))
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
const view = render(WorkersNavSection, { workspaceId: "stop-test" });
|
||||||
|
await waitFor(() => expect(transport.listener).not.toBeNull());
|
||||||
|
transport.listener!.onFrame(snapshot("stopped", "observed"));
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Actions for Test worker" }),
|
||||||
|
);
|
||||||
|
await fireEvent.click(screen.getByRole("menuitem", { name: "Restore" }));
|
||||||
|
if (mode === "switch") {
|
||||||
|
await view.rerender({ workspaceId: "next-test" });
|
||||||
|
transport.listener!.onFrame(snapshot("stopped", "observed", "next-test"));
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Actions for Test worker" }),
|
||||||
|
);
|
||||||
|
} else view.unmount();
|
||||||
|
clearWorkspaceAlerts();
|
||||||
|
let alerts: unknown[] = [];
|
||||||
|
const unsubscribe = workspaceAlerts.subscribe((value) => {
|
||||||
|
alerts = value;
|
||||||
|
});
|
||||||
|
const readsBeforeCompletion =
|
||||||
|
request.mock.calls.filter(([url]) => String(url).endsWith("/workers"))
|
||||||
|
.length;
|
||||||
|
resolve(
|
||||||
|
Response.json({
|
||||||
|
workspace_id: "stop-test",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-a",
|
||||||
|
result: { state: "accepted", diagnostics: [] },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await pending;
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(alerts).toEqual([]);
|
||||||
|
expect(
|
||||||
|
request.mock.calls.filter(([url]) => String(url).endsWith("/workers"))
|
||||||
|
.length,
|
||||||
|
).toBe(readsBeforeCompletion);
|
||||||
|
if (mode === "switch") {
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("menuitem", { name: "Restore" }) as HTMLButtonElement)
|
||||||
|
.disabled,
|
||||||
|
).toBe(false);
|
||||||
|
}
|
||||||
|
unsubscribe();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test("sidebar Delete cannot proceed to execution when its cleanup plan arrives after workspace switch", async () => {
|
||||||
|
let resolve!: (response: Response) => void;
|
||||||
|
const pending = new Promise<Response>((done) => {
|
||||||
|
resolve = done;
|
||||||
|
});
|
||||||
|
const request = vi.fn((input: RequestInfo | URL) =>
|
||||||
|
String(input).endsWith("/cleanup-plan")
|
||||||
|
? pending
|
||||||
|
: Promise.resolve(new Response(null, { status: 404 }))
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
const view = render(WorkersNavSection, { workspaceId: "stop-test" });
|
||||||
|
await waitFor(() => expect(transport.listener).not.toBeNull());
|
||||||
|
transport.listener!.onFrame(snapshot("stopped", "observed"));
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Actions for Test worker" }),
|
||||||
|
);
|
||||||
|
await fireEvent.click(screen.getByRole("menuitem", { name: "Delete" }));
|
||||||
|
expect(
|
||||||
|
request.mock.calls.some(([url]) => String(url).endsWith("/cleanup-plan")),
|
||||||
|
).toBe(true);
|
||||||
|
await view.rerender({ workspaceId: "next-test" });
|
||||||
|
clearWorkspaceAlerts();
|
||||||
|
resolve(Response.json({
|
||||||
|
workspace_id: "stop-test",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
generated_at: "2026-10-06T00:00:00Z",
|
||||||
|
revision: "old-plan",
|
||||||
|
digest: "old-digest",
|
||||||
|
workdirs: [],
|
||||||
|
diagnostics: [],
|
||||||
|
workers: [{
|
||||||
|
target_id: "worker-target",
|
||||||
|
action: "worker_delete",
|
||||||
|
worker_id: "W-1",
|
||||||
|
runtime_worker_id: "worker-a",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
reason: "stopped",
|
||||||
|
pinned: false,
|
||||||
|
retention_state: "normal",
|
||||||
|
linked_workdir_ids: [],
|
||||||
|
running_linked: false,
|
||||||
|
}],
|
||||||
|
}));
|
||||||
|
await pending;
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(
|
||||||
|
request.mock.calls.some(([url]) =>
|
||||||
|
String(url).endsWith("/cleanup-executions")
|
||||||
|
),
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
|||||||
@@ -65,6 +65,9 @@ beforeEach(() => {
|
|||||||
);
|
);
|
||||||
return Response.json({
|
return Response.json({
|
||||||
workspace_id: workspaceId,
|
workspace_id: workspaceId,
|
||||||
|
...(String(input).endsWith("/workers")
|
||||||
|
? { limit: 100, source: "catalog" }
|
||||||
|
: {}),
|
||||||
items: [],
|
items: [],
|
||||||
diagnostics: [],
|
diagnostics: [],
|
||||||
});
|
});
|
||||||
@@ -283,7 +286,17 @@ test("terminal subscription rejection is shown only through WorkspaceAlerts", as
|
|||||||
test("Workdir fetch failure preserves Worker updates and uses a separate stable alert", async () => {
|
test("Workdir fetch failure preserves Worker updates and uses a separate stable alert", async () => {
|
||||||
vi.stubGlobal(
|
vi.stubGlobal(
|
||||||
"fetch",
|
"fetch",
|
||||||
vi.fn(async () => new Response("Workdir lookup failed", { status: 503 })),
|
vi.fn(async (input: RequestInfo | URL) =>
|
||||||
|
String(input).endsWith("/workers")
|
||||||
|
? Response.json({
|
||||||
|
workspace_id: "workspace-a",
|
||||||
|
limit: 100,
|
||||||
|
source: "catalog",
|
||||||
|
items: [],
|
||||||
|
diagnostics: [],
|
||||||
|
})
|
||||||
|
: new Response("Workdir lookup failed", { status: 503 })
|
||||||
|
),
|
||||||
);
|
);
|
||||||
mount();
|
mount();
|
||||||
await waitFor(() => expect(transport.subscribe).toHaveBeenCalledOnce());
|
await waitFor(() => expect(transport.subscribe).toHaveBeenCalledOnce());
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
pushWorkspaceAlert,
|
pushWorkspaceAlert,
|
||||||
} from "#lib/workspace/alerts/store.ts";
|
} from "#lib/workspace/alerts/store.ts";
|
||||||
import { loadJson, workspaceApiPath } from "#lib/workspace/api/http.ts";
|
import { loadJson, workspaceApiPath } from "#lib/workspace/api/http.ts";
|
||||||
|
import { parseWorkerListResponse } from "#lib/workspace/api/workers.ts";
|
||||||
import { parseWorkingDirectoryListResponse } from "#lib/workspace/api/workdirs.ts";
|
import { parseWorkingDirectoryListResponse } from "#lib/workspace/api/workdirs.ts";
|
||||||
import {
|
import {
|
||||||
workspaceMultiplexer,
|
workspaceMultiplexer,
|
||||||
@@ -31,11 +32,17 @@ export type SidebarWorker = Omit<Worker, "workdir_attachments"> & {
|
|||||||
export type WorkspaceWorkersState = {
|
export type WorkspaceWorkersState = {
|
||||||
loading: boolean;
|
loading: boolean;
|
||||||
workers: SidebarWorker[];
|
workers: SidebarWorker[];
|
||||||
|
/** Full catalog metadata; null until fetched. Never substitutes sidebar display state. */
|
||||||
|
catalogWorkers: Worker[] | null;
|
||||||
|
/** Changes immediately when an observation invalidates dependent cleanup plans. */
|
||||||
|
observationVersion: number;
|
||||||
|
catalogRefreshing: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
type WorkerAlertKind = "subscription" | "workdirs";
|
type WorkerAlertKind = "subscription" | "workdirs" | "catalog";
|
||||||
type WorkspaceWorkersStoreEntry = {
|
type WorkspaceWorkersStoreEntry = {
|
||||||
store: Readable<WorkspaceWorkersState>;
|
store: Readable<WorkspaceWorkersState>;
|
||||||
|
refresh(): Promise<void>;
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -55,6 +62,8 @@ function reportWorkerFailure(
|
|||||||
id: workerAlertId(workspaceId, kind),
|
id: workerAlertId(workspaceId, kind),
|
||||||
title: kind === "workdirs"
|
title: kind === "workdirs"
|
||||||
? "Worker Workdirs unavailable"
|
? "Worker Workdirs unavailable"
|
||||||
|
: kind === "catalog"
|
||||||
|
? "Worker catalog unavailable"
|
||||||
: "Worker updates unavailable",
|
: "Worker updates unavailable",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -66,6 +75,7 @@ function clearWorkerFailure(workspaceId: string, kind: WorkerAlertKind): void {
|
|||||||
function clearWorkerFailures(workspaceId: string): void {
|
function clearWorkerFailures(workspaceId: string): void {
|
||||||
clearWorkerFailure(workspaceId, "subscription");
|
clearWorkerFailure(workspaceId, "subscription");
|
||||||
clearWorkerFailure(workspaceId, "workdirs");
|
clearWorkerFailure(workspaceId, "workdirs");
|
||||||
|
clearWorkerFailure(workspaceId, "catalog");
|
||||||
}
|
}
|
||||||
|
|
||||||
export function disposeWorkspaceWorkersStore(workspaceId: string): void {
|
export function disposeWorkspaceWorkersStore(workspaceId: string): void {
|
||||||
@@ -78,6 +88,13 @@ export function disposeWorkspaceWorkersStore(workspaceId: string): void {
|
|||||||
entry.dispose();
|
entry.dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Refresh observations only; this does not POST or complete pending reconciliation. */
|
||||||
|
export async function refreshWorkspaceWorkers(
|
||||||
|
workspaceId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
await stores.get(workspaceId)?.refresh();
|
||||||
|
}
|
||||||
|
|
||||||
export function workspaceWorkersStore(
|
export function workspaceWorkersStore(
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
): Readable<WorkspaceWorkersState> {
|
): Readable<WorkspaceWorkersState> {
|
||||||
@@ -85,11 +102,24 @@ export function workspaceWorkersStore(
|
|||||||
if (cached) return cached.store;
|
if (cached) return cached.store;
|
||||||
|
|
||||||
let stopActive: (() => void) | null = null;
|
let stopActive: (() => void) | null = null;
|
||||||
|
let refreshActive: (() => Promise<void>) | null = null;
|
||||||
const store = readable<WorkspaceWorkersState>(
|
const store = readable<WorkspaceWorkersState>(
|
||||||
{ loading: true, workers: [] },
|
{
|
||||||
|
loading: true,
|
||||||
|
workers: [],
|
||||||
|
catalogWorkers: null,
|
||||||
|
observationVersion: 0,
|
||||||
|
catalogRefreshing: true,
|
||||||
|
},
|
||||||
(set) => {
|
(set) => {
|
||||||
if (!workspaceId) {
|
if (!workspaceId) {
|
||||||
set({ loading: false, workers: [] });
|
set({
|
||||||
|
loading: false,
|
||||||
|
workers: [],
|
||||||
|
catalogWorkers: [],
|
||||||
|
observationVersion: 0,
|
||||||
|
catalogRefreshing: false,
|
||||||
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const projection = createWorkspaceWorkersProjection();
|
const projection = createWorkspaceWorkersProjection();
|
||||||
@@ -99,13 +129,24 @@ export function workspaceWorkersStore(
|
|||||||
let workers: SidebarWorker[] = [];
|
let workers: SidebarWorker[] = [];
|
||||||
let loading = true;
|
let loading = true;
|
||||||
let disposed = false;
|
let disposed = false;
|
||||||
|
let catalogWorkers: Worker[] | null = null;
|
||||||
|
let observationVersion = 0;
|
||||||
|
let catalogRefreshing = true;
|
||||||
|
let refreshSequence = 0;
|
||||||
|
let catalogAbort: AbortController | null = null;
|
||||||
const publish = (): boolean => {
|
const publish = (): boolean => {
|
||||||
if (disposed) return false;
|
if (disposed) return false;
|
||||||
try {
|
try {
|
||||||
workers = [...projection.workers.values()]
|
workers = [...projection.workers.values()]
|
||||||
.map((worker) => projectWorker(worker, workdirs))
|
.map((worker) => projectWorker(worker, workdirs))
|
||||||
.sort(compareWorkersForSidebar);
|
.sort(compareWorkersForSidebar);
|
||||||
set({ loading, workers });
|
set({
|
||||||
|
loading,
|
||||||
|
workers,
|
||||||
|
catalogWorkers,
|
||||||
|
observationVersion,
|
||||||
|
catalogRefreshing,
|
||||||
|
});
|
||||||
return true;
|
return true;
|
||||||
} catch (cause) {
|
} catch (cause) {
|
||||||
reportWorkerFailure(
|
reportWorkerFailure(
|
||||||
@@ -115,10 +156,86 @@ export function workspaceWorkersStore(
|
|||||||
? cause.message
|
? cause.message
|
||||||
: "Invalid Worker subscription frame",
|
: "Invalid Worker subscription frame",
|
||||||
);
|
);
|
||||||
set({ loading: false, workers });
|
set({
|
||||||
|
loading: false,
|
||||||
|
workers,
|
||||||
|
catalogWorkers,
|
||||||
|
observationVersion,
|
||||||
|
catalogRefreshing,
|
||||||
|
});
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
const refreshCatalog = async () => {
|
||||||
|
if (disposed) return;
|
||||||
|
const sequence = ++refreshSequence;
|
||||||
|
const version = ++observationVersion;
|
||||||
|
catalogAbort?.abort();
|
||||||
|
const abort = new AbortController();
|
||||||
|
catalogAbort = abort;
|
||||||
|
catalogRefreshing = true;
|
||||||
|
publish(); // Invalidate cleanup plans before any request can settle.
|
||||||
|
const result = await loadJson(
|
||||||
|
fetch,
|
||||||
|
workspaceApiPath(workspaceId, "/workers"),
|
||||||
|
{ signal: abort.signal },
|
||||||
|
parseWorkerListResponse,
|
||||||
|
{ diagnosticLabel: "Worker API", maxResponseBytes: 8 * 1024 * 1024 },
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
disposed || sequence !== refreshSequence ||
|
||||||
|
version !== observationVersion
|
||||||
|
) return;
|
||||||
|
if (!result.data || result.data.workspace_id !== workspaceId) {
|
||||||
|
reportWorkerFailure(
|
||||||
|
workspaceId,
|
||||||
|
"catalog",
|
||||||
|
result.error ?? "Worker catalog identity mismatch",
|
||||||
|
"warning",
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
catalogWorkers = result.data.items;
|
||||||
|
catalogRefreshing = false;
|
||||||
|
// GET is a new observation, never a cached POST response. Existing
|
||||||
|
// subscription-only fields (Job/Internal activity) retain their owners.
|
||||||
|
for (const summary of catalogWorkers) {
|
||||||
|
const key = JSON.stringify([summary.runtime_id, summary.worker_id]);
|
||||||
|
const current = projection.workers.get(key);
|
||||||
|
if (current) {
|
||||||
|
projection.workers.set(key, {
|
||||||
|
...current,
|
||||||
|
availability: summary.availability,
|
||||||
|
state: summary.state as SubscriptionWorker["state"],
|
||||||
|
worker_state: summary.worker_state,
|
||||||
|
restore_observation_token: summary.restore_observation_token,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clearWorkerFailure(workspaceId, "catalog");
|
||||||
|
publish();
|
||||||
|
};
|
||||||
|
refreshActive = refreshCatalog;
|
||||||
|
const observeCatalog = () => {
|
||||||
|
if (catalogWorkers !== null) {
|
||||||
|
catalogWorkers = catalogWorkers.map((worker) => {
|
||||||
|
const latest = projection.workers.get(
|
||||||
|
JSON.stringify([worker.runtime_id, worker.worker_id]),
|
||||||
|
);
|
||||||
|
// The catalog can contain targets absent from the sidebar stream;
|
||||||
|
// absence is not a new filtering rule for the Workers page.
|
||||||
|
if (!latest) return worker;
|
||||||
|
return {
|
||||||
|
...worker,
|
||||||
|
availability: latest.availability,
|
||||||
|
state: latest.state,
|
||||||
|
worker_state: latest.worker_state,
|
||||||
|
restore_observation_token: latest.restore_observation_token,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
void refreshCatalog();
|
||||||
|
};
|
||||||
const loadWorkdirs = async () => {
|
const loadWorkdirs = async () => {
|
||||||
const result = await loadJson(
|
const result = await loadJson(
|
||||||
fetch,
|
fetch,
|
||||||
@@ -183,6 +300,13 @@ export function workspaceWorkersStore(
|
|||||||
try {
|
try {
|
||||||
applyWorkspaceWorkersFrame(projection, frame);
|
applyWorkspaceWorkersFrame(projection, frame);
|
||||||
loading = false;
|
loading = false;
|
||||||
|
if (
|
||||||
|
(frame.frame === "response" &&
|
||||||
|
frame.message.result === "subscribed") ||
|
||||||
|
(frame.frame === "event" && frame.message.event === "event" &&
|
||||||
|
frame.message.data.subscription_id ===
|
||||||
|
projection.subscriptionId)
|
||||||
|
) observeCatalog();
|
||||||
if (publish()) clearWorkerFailure(workspaceId, "subscription");
|
if (publish()) clearWorkerFailure(workspaceId, "subscription");
|
||||||
} catch (cause) {
|
} catch (cause) {
|
||||||
loading = false;
|
loading = false;
|
||||||
@@ -216,11 +340,14 @@ export function workspaceWorkersStore(
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
void loadWorkdirs();
|
void loadWorkdirs();
|
||||||
|
void refreshCatalog();
|
||||||
|
|
||||||
const stop = () => {
|
const stop = () => {
|
||||||
if (disposed) return;
|
if (disposed) return;
|
||||||
disposed = true;
|
disposed = true;
|
||||||
workdirRequest.abort();
|
workdirRequest.abort();
|
||||||
|
catalogAbort?.abort();
|
||||||
|
if (refreshActive === refreshCatalog) refreshActive = null;
|
||||||
subscription?.close();
|
subscription?.close();
|
||||||
clearWorkerFailures(workspaceId);
|
clearWorkerFailures(workspaceId);
|
||||||
if (stopActive === stop) stopActive = null;
|
if (stopActive === stop) stopActive = null;
|
||||||
@@ -231,6 +358,9 @@ export function workspaceWorkersStore(
|
|||||||
);
|
);
|
||||||
const entry: WorkspaceWorkersStoreEntry = {
|
const entry: WorkspaceWorkersStoreEntry = {
|
||||||
store,
|
store,
|
||||||
|
refresh: async () => {
|
||||||
|
await refreshActive?.();
|
||||||
|
},
|
||||||
dispose: () => {
|
dispose: () => {
|
||||||
stopActive?.();
|
stopActive?.();
|
||||||
clearWorkerFailures(workspaceId);
|
clearWorkerFailures(workspaceId);
|
||||||
@@ -268,6 +398,7 @@ function projectWorker(
|
|||||||
availability: worker.availability,
|
availability: worker.availability,
|
||||||
state: liveWorkerState(worker),
|
state: liveWorkerState(worker),
|
||||||
lifecycleState: worker.state,
|
lifecycleState: worker.state,
|
||||||
|
restore_observation_token: worker.restore_observation_token,
|
||||||
worker_state: worker.worker_state,
|
worker_state: worker.worker_state,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient",
|
retention_state: "transient",
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
|
import { untrack } from 'svelte';
|
||||||
|
import { canRestoreWorker, createRestoreRequest, restoreWorkspaceWorker, restoreNotice, restoreErrorNotice, type RestoreRequest } from '#lib/workspace/sidebar/worker-actions.ts';
|
||||||
|
import { workspaceWorkersStore, refreshWorkspaceWorkers } from '#lib/workspace/sidebar/worker-subscription.ts';
|
||||||
import { pushWorkspaceAlert } from '#lib/workspace/alerts/store.ts';
|
import { pushWorkspaceAlert } from '#lib/workspace/alerts/store.ts';
|
||||||
import { workspaceApiPath } from '#lib/workspace/api/http.ts';
|
import { workspaceApiPath } from '#lib/workspace/api/http.ts';
|
||||||
import {
|
import {
|
||||||
@@ -14,16 +17,71 @@
|
|||||||
import type { CleanupWorkerCandidate, RuntimeCleanupPlanResponse, Worker } from '#lib/workspace/sidebar/types.ts';
|
import type { CleanupWorkerCandidate, RuntimeCleanupPlanResponse, Worker } from '#lib/workspace/sidebar/types.ts';
|
||||||
import type { PageProps } from './$types';
|
import type { PageProps } from './$types';
|
||||||
|
|
||||||
type WorkerActionKind = 'pin' | 'delete';
|
type WorkerActionKind = 'restore' | 'pin' | 'delete';
|
||||||
|
|
||||||
let { data }: PageProps = $props();
|
let { data }: PageProps = $props();
|
||||||
let cleanupPlans = $state<Record<string, RuntimeCleanupPlanResponse>>({});
|
let cleanupPlans = $state<Record<string, RuntimeCleanupPlanResponse>>({});
|
||||||
let workers = $state<Worker[]>([]);
|
let workers = $state<Worker[]>([]);
|
||||||
|
let restoreRetries = $state<Record<string, RestoreRequest>>({});
|
||||||
let busyAction = $state<{ workerKey: string; kind: WorkerActionKind } | null>(null);
|
let busyAction = $state<{ workerKey: string; kind: WorkerActionKind } | null>(null);
|
||||||
|
|
||||||
|
let lifetime = 0;
|
||||||
|
let cleanupEpoch = 0;
|
||||||
|
let catalogRefreshing = true;
|
||||||
|
let catalogReady = $state(false);
|
||||||
|
|
||||||
|
function actionScope() {
|
||||||
|
const id = data.workspaceId;
|
||||||
|
const epoch = lifetime;
|
||||||
|
return { workspaceId: id, isCurrent: () => lifetime === epoch && data.workspaceId === id };
|
||||||
|
}
|
||||||
|
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
cleanupPlans = data.cleanupPlans;
|
const id = data.workspaceId;
|
||||||
workers = data.workers?.items ?? [];
|
lifetime++;
|
||||||
|
cleanupEpoch++;
|
||||||
|
busyAction = null;
|
||||||
|
restoreRetries = {};
|
||||||
|
catalogReady = false;
|
||||||
|
const initial = untrack(() => data);
|
||||||
|
workers = initial.workers?.items ?? [];
|
||||||
|
cleanupPlans = initial.cleanupPlans;
|
||||||
|
let previousCatalog: string | undefined;
|
||||||
|
let previousObservation: number | undefined;
|
||||||
|
let previousRefreshing: boolean | undefined;
|
||||||
|
const scope = actionScope();
|
||||||
|
const unsubscribe = workspaceWorkersStore(id).subscribe((state) => {
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
|
const observationChanged = previousObservation !== state.observationVersion;
|
||||||
|
const refreshStarted = state.catalogRefreshing && previousRefreshing !== true;
|
||||||
|
const refreshCompleted = !state.catalogRefreshing && previousRefreshing === true;
|
||||||
|
const catalog = state.catalogWorkers === null ? undefined : JSON.stringify(state.catalogWorkers);
|
||||||
|
const catalogChanged = catalog !== previousCatalog;
|
||||||
|
previousObservation = state.observationVersion;
|
||||||
|
previousRefreshing = state.catalogRefreshing;
|
||||||
|
previousCatalog = catalog;
|
||||||
|
catalogRefreshing = state.catalogRefreshing;
|
||||||
|
if (observationChanged || catalogChanged || refreshStarted) {
|
||||||
|
cleanupPlans = {};
|
||||||
|
++cleanupEpoch;
|
||||||
|
}
|
||||||
|
if (state.catalogWorkers === null) return;
|
||||||
|
catalogReady = true;
|
||||||
|
workers = state.catalogWorkers;
|
||||||
|
// Subscription overlays stay visible during the GET, but cleanup must wait
|
||||||
|
// for its authoritative catalog result, even if that result is unchanged.
|
||||||
|
if (catalogRefreshing || (!catalogChanged && !observationChanged && !refreshCompleted)) return;
|
||||||
|
cleanupPlans = {};
|
||||||
|
const epoch = ++cleanupEpoch;
|
||||||
|
for (const runtimeId of new Set(state.catalogWorkers.map((worker) => worker.runtime_id))) {
|
||||||
|
void refreshCleanupPlan(runtimeId, scope, epoch).catch(() => {});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
lifetime++;
|
||||||
|
cleanupEpoch++;
|
||||||
|
unsubscribe();
|
||||||
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
function workerKey(worker: Worker): string {
|
function workerKey(worker: Worker): string {
|
||||||
@@ -56,41 +114,78 @@
|
|||||||
return fallback;
|
return fallback;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function refreshCleanupPlan(runtimeId: string): Promise<void> {
|
async function refreshCleanupPlan(
|
||||||
|
runtimeId: string,
|
||||||
|
scope = actionScope(),
|
||||||
|
epoch = cleanupEpoch,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!scope.isCurrent() || catalogRefreshing || cleanupEpoch !== epoch) return;
|
||||||
const response = await fetch(
|
const response = await fetch(
|
||||||
workspaceApiPath(data.workspaceId, `/runtimes/${encodeURIComponent(runtimeId)}/cleanup-plan`),
|
workspaceApiPath(scope.workspaceId, `/runtimes/${encodeURIComponent(runtimeId)}/cleanup-plan`),
|
||||||
);
|
);
|
||||||
if (!response.ok) return;
|
if (!response.ok) return;
|
||||||
const plan = parseRuntimeCleanupPlan(await response.json());
|
const plan = parseRuntimeCleanupPlan(await response.json());
|
||||||
|
if (!scope.isCurrent() || catalogRefreshing || cleanupEpoch !== epoch) return;
|
||||||
cleanupPlans = { ...cleanupPlans, [runtimeId]: plan };
|
cleanupPlans = { ...cleanupPlans, [runtimeId]: plan };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function restoreWorker(worker: Worker, retry = false): Promise<void> {
|
||||||
|
const key = workerKey(worker);
|
||||||
|
if (busyAction || (retry ? !restoreRetries[key] : restoreRetries[key] || !canRestoreWorker(worker))) return;
|
||||||
|
const scope = actionScope();
|
||||||
|
const request = retry ? restoreRetries[key] : createRestoreRequest(worker);
|
||||||
|
busyAction = { workerKey: key, kind: 'restore' };
|
||||||
|
cleanupPlans = {};
|
||||||
|
++cleanupEpoch;
|
||||||
|
try {
|
||||||
|
let notice;
|
||||||
|
try {
|
||||||
|
notice = restoreNotice(await restoreWorkspaceWorker(scope.workspaceId, worker, request));
|
||||||
|
} catch (cause) {
|
||||||
|
notice = restoreErrorNotice(cause);
|
||||||
|
}
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
|
const next = { ...restoreRetries };
|
||||||
|
if (notice.retry) next[key] = request;
|
||||||
|
else delete next[key];
|
||||||
|
restoreRetries = next;
|
||||||
|
pushWorkspaceAlert(notice.level, notice.message, { title: notice.title });
|
||||||
|
await refreshWorkspaceWorkers(scope.workspaceId).catch(() => {});
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
|
await refreshCleanupPlan(worker.runtime_id, scope).catch(() => {});
|
||||||
|
} finally {
|
||||||
|
if (scope.isCurrent()) busyAction = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function setPinned(worker: Worker, pinned: boolean): Promise<void> {
|
async function setPinned(worker: Worker, pinned: boolean): Promise<void> {
|
||||||
if (busyAction) return;
|
if (busyAction) return;
|
||||||
|
const scope = actionScope();
|
||||||
busyAction = { workerKey: workerKey(worker), kind: 'pin' };
|
busyAction = { workerKey: workerKey(worker), kind: 'pin' };
|
||||||
|
cleanupPlans = {};
|
||||||
|
++cleanupEpoch;
|
||||||
try {
|
try {
|
||||||
const response = await fetch(
|
const response = await fetch(
|
||||||
workspaceApiPath(
|
workspaceApiPath(
|
||||||
data.workspaceId,
|
scope.workspaceId,
|
||||||
`/runtimes/${encodeURIComponent(worker.runtime_id)}/workers/${encodeURIComponent(worker.worker_id)}/pin`,
|
`/runtimes/${encodeURIComponent(worker.runtime_id)}/workers/${encodeURIComponent(worker.worker_id)}/pin`,
|
||||||
),
|
),
|
||||||
{ method: pinned ? 'PUT' : 'DELETE' },
|
{ method: pinned ? 'PUT' : 'DELETE' },
|
||||||
);
|
);
|
||||||
const payload = await response.json().catch(() => null);
|
const payload = await response.json().catch(() => null);
|
||||||
if (!response.ok) {
|
if (!scope.isCurrent()) return;
|
||||||
pushWorkspaceAlert('error', errorMessage(payload, response.statusText), { title: 'Worker pin failed' });
|
if (!response.ok) throw new Error(errorMessage(payload, response.statusText));
|
||||||
return;
|
parseWorkerRetentionResponse(payload);
|
||||||
}
|
await refreshWorkspaceWorkers(scope.workspaceId);
|
||||||
const retention = parseWorkerRetentionResponse(payload);
|
if (!scope.isCurrent()) return;
|
||||||
worker.pinned = retention.pinned;
|
await refreshCleanupPlan(worker.runtime_id, scope);
|
||||||
worker.retention_state = retention.retention_state;
|
|
||||||
await refreshCleanupPlan(worker.runtime_id);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
pushWorkspaceAlert('error', error instanceof Error ? error.message : 'Worker pin failed', {
|
pushWorkspaceAlert('error', error instanceof Error ? error.message : 'Worker pin failed', {
|
||||||
title: 'Worker pin failed',
|
title: 'Worker pin failed',
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
busyAction = null;
|
if (scope.isCurrent()) busyAction = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,12 +196,15 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function deleteWorker(worker: Worker, candidate: CleanupWorkerCandidate): Promise<void> {
|
async function deleteWorker(worker: Worker, candidate: CleanupWorkerCandidate): Promise<void> {
|
||||||
if (!cleanupPlans?.[worker.runtime_id] || busyAction) return;
|
const plan = cleanupPlans?.[worker.runtime_id];
|
||||||
|
if (!plan || busyAction || candidate.blocking_reason || cleanupCandidate(worker) !== candidate) return;
|
||||||
|
const scope = actionScope();
|
||||||
busyAction = { workerKey: workerKey(worker), kind: 'delete' };
|
busyAction = { workerKey: workerKey(worker), kind: 'delete' };
|
||||||
|
cleanupPlans = {};
|
||||||
|
++cleanupEpoch;
|
||||||
try {
|
try {
|
||||||
const plan = cleanupPlans[worker.runtime_id];
|
|
||||||
const response = await fetch(
|
const response = await fetch(
|
||||||
workspaceApiPath(data.workspaceId, `/runtimes/${encodeURIComponent(worker.runtime_id)}/cleanup-executions`),
|
workspaceApiPath(scope.workspaceId, `/runtimes/${encodeURIComponent(worker.runtime_id)}/cleanup-executions`),
|
||||||
{
|
{
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'content-type': 'application/json' },
|
headers: { 'content-type': 'application/json' },
|
||||||
@@ -120,22 +218,23 @@
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
const payload = await response.json().catch(() => null);
|
const payload = await response.json().catch(() => null);
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
if (!response.ok) throw new Error(errorMessage(payload, response.statusText));
|
if (!response.ok) throw new Error(errorMessage(payload, response.statusText));
|
||||||
const execution = parseRuntimeCleanupExecution(payload);
|
const execution = parseRuntimeCleanupExecution(payload);
|
||||||
cleanupPlans = { ...cleanupPlans, [worker.runtime_id]: execution.plan_after };
|
|
||||||
const result = execution.results.find((entry) => entry.target_id === candidate.target_id);
|
const result = execution.results.find((entry) => entry.target_id === candidate.target_id);
|
||||||
if (!result || result.status !== 'deleted') {
|
if (!result || result.status !== 'deleted') {
|
||||||
throw new Error(result?.message ?? 'Runtime did not delete the selected Worker');
|
throw new Error(result?.message ?? 'Runtime did not delete the selected Worker');
|
||||||
}
|
}
|
||||||
workers = workers.filter(
|
await refreshWorkspaceWorkers(scope.workspaceId);
|
||||||
(item) => !(item.runtime_id === worker.runtime_id && item.worker_id === worker.worker_id),
|
if (!scope.isCurrent()) return;
|
||||||
);
|
await refreshCleanupPlan(worker.runtime_id, scope);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (!scope.isCurrent()) return;
|
||||||
pushWorkspaceAlert('error', error instanceof Error ? error.message : 'Worker deletion failed', {
|
pushWorkspaceAlert('error', error instanceof Error ? error.message : 'Worker deletion failed', {
|
||||||
title: 'Worker deletion failed',
|
title: 'Worker deletion failed',
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
busyAction = null;
|
if (scope.isCurrent()) busyAction = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -180,9 +279,9 @@
|
|||||||
<a class="section-action" href={`/w/${data.workspaceId}/workers/new`}>New Worker</a>
|
<a class="section-action" href={`/w/${data.workspaceId}/workers/new`}>New Worker</a>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
{#if data.workersError}
|
{#if !catalogReady && data.workersError}
|
||||||
<p class="section-state error">{data.workersError}</p>
|
<p class="section-state error">{data.workersError}</p>
|
||||||
{:else if !data.workers}
|
{:else if !catalogReady && !data.workers}
|
||||||
<p class="section-state">Loading Workers…</p>
|
<p class="section-state">Loading Workers…</p>
|
||||||
{:else if workers.length === 0}
|
{:else if workers.length === 0}
|
||||||
<p class="section-state">No Workers are visible.</p>
|
<p class="section-state">No Workers are visible.</p>
|
||||||
@@ -222,6 +321,24 @@
|
|||||||
<td>{workerDirectory(worker)}</td>
|
<td>{workerDirectory(worker)}</td>
|
||||||
<td>
|
<td>
|
||||||
<div class="worker-actions" aria-label={`Actions for ${workerDisplayName}`}>
|
<div class="worker-actions" aria-label={`Actions for ${workerDisplayName}`}>
|
||||||
|
<button
|
||||||
|
class="icon-action"
|
||||||
|
type="button"
|
||||||
|
disabled={anyActionDisabled || !canRestoreWorker(worker) || Boolean(restoreRetries[workerKey(worker)])}
|
||||||
|
aria-label={`Restore ${workerDisplayName}`}
|
||||||
|
title="Restore"
|
||||||
|
onclick={() => restoreWorker(worker)}
|
||||||
|
>
|
||||||
|
{#if isActionBusy(worker, 'restore')}
|
||||||
|
<span class="spinner" aria-hidden="true"></span>
|
||||||
|
{:else}
|
||||||
|
<svg class="action-icon" aria-hidden="true" viewBox="0 0 24 24"><path d="M3 11a9 9 0 1 1 2.6 6.4" /><path d="M3 3v8h8" /></svg>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
{#if restoreRetries[workerKey(worker)]}
|
||||||
|
<button type="button" disabled={anyActionDisabled} onclick={() => restoreWorker(worker, true)} aria-label={`Retry Restore ${workerDisplayName}`}>Retry Restore</button>
|
||||||
|
<span role="status">Restore outcome unresolved. Retry the same request to reconcile.</span>
|
||||||
|
{/if}
|
||||||
<button
|
<button
|
||||||
class="icon-action"
|
class="icon-action"
|
||||||
type="button"
|
type="button"
|
||||||
|
|||||||
@@ -0,0 +1,600 @@
|
|||||||
|
// @vitest-environment happy-dom
|
||||||
|
import {
|
||||||
|
cleanup,
|
||||||
|
fireEvent,
|
||||||
|
render,
|
||||||
|
screen,
|
||||||
|
waitFor,
|
||||||
|
} from "@testing-library/svelte";
|
||||||
|
import { afterEach, expect, test, vi } from "vitest";
|
||||||
|
import { writable } from "svelte/store";
|
||||||
|
import type { ComponentProps } from "svelte";
|
||||||
|
import type {
|
||||||
|
RuntimeCleanupPlanResponse,
|
||||||
|
Worker,
|
||||||
|
} from "#lib/workspace/sidebar/types.ts";
|
||||||
|
import type { WorkspaceWorkersState } from "#lib/workspace/sidebar/worker-subscription.ts";
|
||||||
|
import {
|
||||||
|
clearWorkspaceAlerts,
|
||||||
|
workspaceAlerts,
|
||||||
|
} from "#lib/workspace/alerts/store.ts";
|
||||||
|
import WorkersPage from "./+page.svelte";
|
||||||
|
|
||||||
|
const shared = vi.hoisted(() => ({
|
||||||
|
stores: new Map(),
|
||||||
|
refresh: vi.fn(async (_workspaceId: string) => {}),
|
||||||
|
}));
|
||||||
|
vi.mock("#lib/workspace/sidebar/worker-subscription.ts", () => ({
|
||||||
|
workspaceWorkersStore: (id: string) => shared.stores.get(id),
|
||||||
|
refreshWorkspaceWorkers: shared.refresh,
|
||||||
|
}));
|
||||||
|
|
||||||
|
type Data = ComponentProps<typeof WorkersPage>["data"];
|
||||||
|
function worker(overrides: Partial<Worker> = {}): Worker {
|
||||||
|
return {
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-a",
|
||||||
|
resource_key: "W-1",
|
||||||
|
host_id: "host-a",
|
||||||
|
display_name: "Catalog worker",
|
||||||
|
label: "worker-a",
|
||||||
|
profile: "builtin:coder",
|
||||||
|
tags: [],
|
||||||
|
workspace: { visibility: "workspace", identity: "workspace-a" },
|
||||||
|
availability: "observed",
|
||||||
|
state: "stopped",
|
||||||
|
pinned: true,
|
||||||
|
retention_state: "pinned",
|
||||||
|
implementation: { kind: "runtime", display_hint: "Runtime Worker" },
|
||||||
|
workdir_attachments: [],
|
||||||
|
diagnostics: [],
|
||||||
|
...{ restore_observation_token: "observation-1" },
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function plan(revision = "plan-1"): RuntimeCleanupPlanResponse {
|
||||||
|
return {
|
||||||
|
workspace_id: "page-test",
|
||||||
|
generated_at: "2026-10-06T00:00:00Z",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
revision,
|
||||||
|
digest: `digest-${revision}`,
|
||||||
|
diagnostics: [],
|
||||||
|
workdirs: [],
|
||||||
|
workers: [{
|
||||||
|
target_id: `target-${revision}`,
|
||||||
|
action: "worker_delete",
|
||||||
|
worker_id: "W-1",
|
||||||
|
runtime_worker_id: "worker-a",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
reason: "stopped",
|
||||||
|
pinned: false,
|
||||||
|
retention_state: "normal",
|
||||||
|
linked_workdir_ids: [],
|
||||||
|
running_linked: false,
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function catalog(id = "page-test") {
|
||||||
|
const state = writable<WorkspaceWorkersState>({
|
||||||
|
loading: true,
|
||||||
|
workers: [],
|
||||||
|
catalogWorkers: null,
|
||||||
|
observationVersion: 0,
|
||||||
|
catalogRefreshing: true,
|
||||||
|
});
|
||||||
|
shared.stores.set(id, state);
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
function data(id = "page-test", item = worker()): Data {
|
||||||
|
return {
|
||||||
|
workspaceId: id,
|
||||||
|
workers: { items: [item] },
|
||||||
|
workersError: null,
|
||||||
|
cleanupPlans: { "runtime-a": plan() },
|
||||||
|
cleanupPlanErrors: {},
|
||||||
|
} as unknown as Data;
|
||||||
|
}
|
||||||
|
function deferred<T>() {
|
||||||
|
let resolve!: (value: T) => void;
|
||||||
|
const promise = new Promise<T>((done) => {
|
||||||
|
resolve = done;
|
||||||
|
});
|
||||||
|
return { promise, resolve };
|
||||||
|
}
|
||||||
|
function publish(store: ReturnType<typeof catalog>, items: Worker[]) {
|
||||||
|
store.update((previous) => ({
|
||||||
|
loading: false,
|
||||||
|
workers: [],
|
||||||
|
catalogWorkers: items,
|
||||||
|
observationVersion: previous.observationVersion + 1,
|
||||||
|
catalogRefreshing: false,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
shared.stores.clear();
|
||||||
|
vi.clearAllMocks();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
clearWorkspaceAlerts();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("catalog updates preserve retention and workdirs, and stale page data never replaces a ready catalog", async () => {
|
||||||
|
const store = catalog();
|
||||||
|
vi.stubGlobal("fetch", vi.fn(async () => Response.json(plan())));
|
||||||
|
const view = render(WorkersPage, {
|
||||||
|
params: { workspaceId: "page-test" },
|
||||||
|
data: data(),
|
||||||
|
});
|
||||||
|
const attachment: NonNullable<Worker["workdir_attachments"]>[number] = {
|
||||||
|
alias: "source",
|
||||||
|
effective_permissions: { read: true, write: false, command: false },
|
||||||
|
working_directory: {
|
||||||
|
working_directory_id: "dir-a",
|
||||||
|
display_name: "Retained Workdir",
|
||||||
|
status: "active",
|
||||||
|
materializer_kind: "client_hosted_external",
|
||||||
|
source: {
|
||||||
|
kind: "external_grant",
|
||||||
|
grant_id: "grant-a",
|
||||||
|
grant_permissions: { read: true, write: false, command: false },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
publish(store, [
|
||||||
|
worker({
|
||||||
|
display_name: "Live catalog",
|
||||||
|
retention_state: "retained",
|
||||||
|
workdir_attachments: [attachment],
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
expect(await screen.findByText("Live catalog")).toBeTruthy();
|
||||||
|
expect(screen.getByText("retained")).toBeTruthy();
|
||||||
|
expect(screen.getByText(/Retained Workdir/)).toBeTruthy();
|
||||||
|
expect(screen.getByRole("button", { name: "Unpin Live catalog" }))
|
||||||
|
.toBeTruthy();
|
||||||
|
expect(
|
||||||
|
screen.getByRole("link", { name: "Live catalog" }).getAttribute("href"),
|
||||||
|
).toBe("/w/page-test/workers/W-1-live-catalog");
|
||||||
|
await view.rerender({
|
||||||
|
data: data("page-test", worker({ display_name: "Stale loader worker" })),
|
||||||
|
});
|
||||||
|
expect(screen.getByText("Live catalog")).toBeTruthy();
|
||||||
|
expect(screen.queryByText("Stale loader worker")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("catalog change invalidates Delete immediately and an older cleanup response cannot restore a stale action", async () => {
|
||||||
|
const store = catalog();
|
||||||
|
const older = deferred<Response>();
|
||||||
|
const newer = deferred<Response>();
|
||||||
|
const request = vi.fn().mockReturnValueOnce(older.promise)
|
||||||
|
.mockReturnValueOnce(newer.promise);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
render(WorkersPage, { params: { workspaceId: "page-test" }, data: data() });
|
||||||
|
publish(store, [worker({ pinned: false })]);
|
||||||
|
await waitFor(() => expect(request).toHaveBeenCalledTimes(1));
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Catalog worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
publish(store, [worker({ pinned: false, state: "idle" })]);
|
||||||
|
await waitFor(() => expect(request).toHaveBeenCalledTimes(2));
|
||||||
|
newer.resolve(Response.json({ ...plan("new"), workers: [] }));
|
||||||
|
older.resolve(Response.json(plan("old")));
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Catalog worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
expect(
|
||||||
|
request.mock.calls.every(([url]) =>
|
||||||
|
!String(url).endsWith("/cleanup-executions")
|
||||||
|
),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("refresh start invalidates an installed Delete candidate without requesting cleanup from the old catalog", async () => {
|
||||||
|
const store = catalog();
|
||||||
|
const request = vi.fn(async () => Response.json(plan()));
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
render(WorkersPage, { params: { workspaceId: "page-test" }, data: data() });
|
||||||
|
publish(store, [worker({ pinned: false })]);
|
||||||
|
expect(await screen.findByRole("button", { name: "Delete Catalog worker" }))
|
||||||
|
.toBeTruthy();
|
||||||
|
expect(request).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
// The flag itself invalidates plans, independently of the observation version.
|
||||||
|
store.update((previous) => ({ ...previous, catalogRefreshing: true }));
|
||||||
|
await waitFor(() =>
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Catalog worker" }))
|
||||||
|
.toBeNull()
|
||||||
|
);
|
||||||
|
store.update((previous) => ({ ...previous }));
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(request).toHaveBeenCalledTimes(1);
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Catalog worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
|
||||||
|
// No catalog or version change on completion: the transition still reloads.
|
||||||
|
store.update((previous) => ({ ...previous, catalogRefreshing: false }));
|
||||||
|
expect(await screen.findByRole("button", { name: "Delete Catalog worker" }))
|
||||||
|
.toBeTruthy();
|
||||||
|
expect(request).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test.each(["during refresh", "after completion"] as const)(
|
||||||
|
"a previous cleanup GET arriving %s cannot reinstall a stale candidate; completion reschedules cleanup",
|
||||||
|
async (settlement) => {
|
||||||
|
const store = catalog();
|
||||||
|
const older = deferred<Response>();
|
||||||
|
const newer = deferred<Response>();
|
||||||
|
const request = vi.fn().mockReturnValueOnce(older.promise)
|
||||||
|
.mockReturnValueOnce(newer.promise);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
render(WorkersPage, { params: { workspaceId: "page-test" }, data: data() });
|
||||||
|
publish(store, [worker({ pinned: false })]);
|
||||||
|
await waitFor(() => expect(request).toHaveBeenCalledTimes(1));
|
||||||
|
|
||||||
|
store.update((previous) => ({
|
||||||
|
...previous,
|
||||||
|
catalogRefreshing: true,
|
||||||
|
observationVersion: previous.observationVersion + 1,
|
||||||
|
catalogWorkers: [
|
||||||
|
worker({ pinned: false, display_name: "Overlay worker" }),
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
expect(await screen.findByText("Overlay worker")).toBeTruthy();
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Overlay worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
expect(request).toHaveBeenCalledTimes(1);
|
||||||
|
if (settlement === "during refresh") {
|
||||||
|
older.resolve(Response.json(plan("stale")));
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Overlay worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Successful catalog GET retains exactly the same catalog and observationVersion.
|
||||||
|
store.update((previous) => ({ ...previous, catalogRefreshing: false }));
|
||||||
|
await waitFor(() => expect(request).toHaveBeenCalledTimes(2));
|
||||||
|
if (settlement === "after completion") {
|
||||||
|
older.resolve(Response.json(plan("stale")));
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Overlay worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
}
|
||||||
|
newer.resolve(Response.json({ ...plan("fresh"), workers: [] }));
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Overlay worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
expect(request).toHaveBeenCalledTimes(2);
|
||||||
|
expect(
|
||||||
|
request.mock.calls.every(([url]) =>
|
||||||
|
String(url).endsWith("/cleanup-plan")
|
||||||
|
),
|
||||||
|
).toBe(true);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test("restore follow-up cannot request cleanup while the authoritative catalog refresh remains unresolved", async () => {
|
||||||
|
const store = catalog();
|
||||||
|
const request = vi.fn(async (input: RequestInfo | URL) =>
|
||||||
|
String(input).endsWith("/restore")
|
||||||
|
? restoreResponse("accepted")
|
||||||
|
: Response.json(plan())
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
render(WorkersPage, { params: { workspaceId: "page-test" }, data: data() });
|
||||||
|
publish(store, [worker({ pinned: false })]);
|
||||||
|
expect(await screen.findByRole("button", { name: "Delete Catalog worker" }))
|
||||||
|
.toBeTruthy();
|
||||||
|
shared.refresh.mockImplementationOnce(async () => {
|
||||||
|
store.update((previous) => ({
|
||||||
|
...previous,
|
||||||
|
catalogRefreshing: true,
|
||||||
|
observationVersion: previous.observationVersion + 1,
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
await fireEvent.click(
|
||||||
|
screen.getByRole("button", { name: "Restore Catalog worker" }),
|
||||||
|
);
|
||||||
|
await waitFor(() =>
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("button", {
|
||||||
|
name: "Pin Catalog worker",
|
||||||
|
}) as HTMLButtonElement).disabled,
|
||||||
|
).toBe(false)
|
||||||
|
);
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Catalog worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
expect(
|
||||||
|
request.mock.calls.filter(([url]) => String(url).endsWith("/cleanup-plan")),
|
||||||
|
).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test.each(["switch", "dispose"] as const)(
|
||||||
|
"late pin response has no UI, alert, refresh, or cleanup effects after %s",
|
||||||
|
async (mode) => {
|
||||||
|
catalog();
|
||||||
|
catalog("next-workspace");
|
||||||
|
const pending = deferred<Response>();
|
||||||
|
const request = vi.fn((_input: RequestInfo | URL) => pending.promise);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
let alerts: unknown[] = [];
|
||||||
|
const unsubscribe = workspaceAlerts.subscribe((value) => {
|
||||||
|
alerts = value;
|
||||||
|
});
|
||||||
|
const view = render(WorkersPage, {
|
||||||
|
params: { workspaceId: "page-test" },
|
||||||
|
data: data(),
|
||||||
|
});
|
||||||
|
await fireEvent.click(
|
||||||
|
screen.getByRole("button", { name: "Unpin Catalog worker" }),
|
||||||
|
);
|
||||||
|
expect(String(request.mock.calls[0][0])).toContain("/api/w/page-test/");
|
||||||
|
if (mode === "switch") {
|
||||||
|
await view.rerender({
|
||||||
|
data: data("next-workspace", worker({ display_name: "Next worker" })),
|
||||||
|
});
|
||||||
|
} else view.unmount();
|
||||||
|
pending.resolve(
|
||||||
|
Response.json({ message: "old pin failure" }, { status: 409 }),
|
||||||
|
);
|
||||||
|
await pending.promise;
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(shared.refresh).not.toHaveBeenCalled();
|
||||||
|
expect(request).toHaveBeenCalledTimes(1);
|
||||||
|
expect(alerts).toEqual([]);
|
||||||
|
if (mode === "switch") {
|
||||||
|
expect(screen.getByRole("button", { name: "Unpin Next worker" }))
|
||||||
|
.toBeTruthy();
|
||||||
|
}
|
||||||
|
unsubscribe();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test("pin response does not overwrite authoritative catalog retention", async () => {
|
||||||
|
const store = catalog();
|
||||||
|
const pending = deferred<Response>();
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn((input: RequestInfo | URL) =>
|
||||||
|
String(input).endsWith("/pin")
|
||||||
|
? pending.promise
|
||||||
|
: Promise.resolve(Response.json(plan()))
|
||||||
|
),
|
||||||
|
);
|
||||||
|
render(WorkersPage, { params: { workspaceId: "page-test" }, data: data() });
|
||||||
|
publish(store, [worker()]);
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Unpin Catalog worker" }),
|
||||||
|
);
|
||||||
|
publish(store, [worker({ retention_state: "newer catalog", pinned: true })]);
|
||||||
|
pending.resolve(
|
||||||
|
Response.json({
|
||||||
|
workspace_id: "page-test",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-a",
|
||||||
|
pinned: false,
|
||||||
|
retention_state: "old response",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await waitFor(() => expect(shared.refresh).toHaveBeenCalledWith("page-test"));
|
||||||
|
expect(screen.getByText("newer catalog")).toBeTruthy();
|
||||||
|
expect(screen.queryByText("old response")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test.each(["switch", "dispose"] as const)(
|
||||||
|
"late Delete response is ignored after %s",
|
||||||
|
async (mode) => {
|
||||||
|
const store = catalog();
|
||||||
|
catalog("next-workspace");
|
||||||
|
const pending = deferred<Response>();
|
||||||
|
const request = vi.fn((input: RequestInfo | URL) =>
|
||||||
|
String(input).endsWith("/cleanup-executions")
|
||||||
|
? pending.promise
|
||||||
|
: Promise.resolve(Response.json(plan()))
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
const view = render(WorkersPage, {
|
||||||
|
params: { workspaceId: "page-test" },
|
||||||
|
data: data(),
|
||||||
|
});
|
||||||
|
publish(store, [worker({ pinned: false })]);
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Delete Catalog worker" }),
|
||||||
|
);
|
||||||
|
const execution = request.mock.calls.find(([url]) =>
|
||||||
|
String(url).endsWith("/cleanup-executions")
|
||||||
|
);
|
||||||
|
expect(String(execution?.[0])).toContain("/api/w/page-test/");
|
||||||
|
if (mode === "switch") {
|
||||||
|
await view.rerender({
|
||||||
|
data: data("next-workspace", worker({ display_name: "Next worker" })),
|
||||||
|
});
|
||||||
|
} else view.unmount();
|
||||||
|
const countBeforeCompletion = request.mock.calls.length;
|
||||||
|
let alerts: unknown[] = [];
|
||||||
|
const unsubscribe = workspaceAlerts.subscribe((value) => {
|
||||||
|
alerts = value;
|
||||||
|
});
|
||||||
|
pending.resolve(
|
||||||
|
Response.json({ message: "old deletion failure" }, { status: 409 }),
|
||||||
|
);
|
||||||
|
await pending.promise;
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(shared.refresh).not.toHaveBeenCalled();
|
||||||
|
expect(request).toHaveBeenCalledTimes(countBeforeCompletion);
|
||||||
|
expect(alerts).toEqual([]);
|
||||||
|
if (mode === "switch") {
|
||||||
|
expect(screen.getByRole("button", { name: "Unpin Next worker" }))
|
||||||
|
.toBeTruthy();
|
||||||
|
}
|
||||||
|
unsubscribe();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
function restoreResponse(
|
||||||
|
state: "accepted" | "rejected" | "rolled_back" | "reconciliation_required",
|
||||||
|
extra = {},
|
||||||
|
) {
|
||||||
|
return Response.json({
|
||||||
|
workspace_id: "page-test",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-a",
|
||||||
|
result: { state, diagnostics: [], ...extra },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
test.each(["stopped", "idle", "running", "paused", "unavailable"] as const)(
|
||||||
|
"Restore eligibility uses observed stopped catalog lifecycle (%s)",
|
||||||
|
async (state) => {
|
||||||
|
catalog();
|
||||||
|
vi.stubGlobal("fetch", vi.fn(async () => Response.json(plan())));
|
||||||
|
render(WorkersPage, {
|
||||||
|
params: { workspaceId: "page-test" },
|
||||||
|
data: data(
|
||||||
|
"page-test",
|
||||||
|
worker({
|
||||||
|
state,
|
||||||
|
availability: state === "unavailable" ? "unavailable" : "observed",
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("button", {
|
||||||
|
name: "Restore Catalog worker",
|
||||||
|
}) as HTMLButtonElement).disabled,
|
||||||
|
).toBe(state !== "stopped");
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test.each(
|
||||||
|
["accepted", "rejected", "rolled_back", "reconciliation_required"] as const,
|
||||||
|
)(
|
||||||
|
"Restore classifies %s through the shared helper, without treating HTTP 200 as success",
|
||||||
|
async (state) => {
|
||||||
|
catalog();
|
||||||
|
let alerts: { title?: string; level: string }[] = [];
|
||||||
|
const unsubscribe = workspaceAlerts.subscribe((value) => {
|
||||||
|
alerts = value;
|
||||||
|
});
|
||||||
|
const request = vi.fn(async (input: RequestInfo | URL) =>
|
||||||
|
String(input).endsWith("/restore")
|
||||||
|
? restoreResponse(state)
|
||||||
|
: Response.json(plan())
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
render(WorkersPage, { params: { workspaceId: "page-test" }, data: data() });
|
||||||
|
await fireEvent.click(
|
||||||
|
screen.getByRole("button", { name: "Restore Catalog worker" }),
|
||||||
|
);
|
||||||
|
await waitFor(() =>
|
||||||
|
expect(shared.refresh).toHaveBeenCalledWith("page-test")
|
||||||
|
);
|
||||||
|
expect(alerts.at(-1)?.level).toBe(
|
||||||
|
state === "accepted"
|
||||||
|
? "info"
|
||||||
|
: state === "reconciliation_required"
|
||||||
|
? "warning"
|
||||||
|
: "error",
|
||||||
|
);
|
||||||
|
expect(alerts.some((alert) => alert.title === "Worker restored")).toBe(
|
||||||
|
state === "accepted",
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
Boolean(
|
||||||
|
screen.queryByRole("button", { name: "Retry Restore Catalog worker" }),
|
||||||
|
),
|
||||||
|
).toBe(state === "reconciliation_required");
|
||||||
|
// The response is never projected into the catalog by the action.
|
||||||
|
expect(screen.getByRole("cell", { name: "stopped" })).toBeTruthy();
|
||||||
|
unsubscribe();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test.each(["transport", "reconciliation"] as const)(
|
||||||
|
"explicit %s retry reuses the exact guarded request despite newer observations",
|
||||||
|
async (failure) => {
|
||||||
|
const store = catalog();
|
||||||
|
let attempts = 0;
|
||||||
|
const request = vi.fn(
|
||||||
|
async (input: RequestInfo | URL, _init?: RequestInit) => {
|
||||||
|
if (!String(input).endsWith("/restore")) return Response.json(plan());
|
||||||
|
if (++attempts === 1) {
|
||||||
|
if (failure === "transport") throw new TypeError("connection lost");
|
||||||
|
return restoreResponse("reconciliation_required");
|
||||||
|
}
|
||||||
|
return restoreResponse("accepted");
|
||||||
|
},
|
||||||
|
);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
render(WorkersPage, { params: { workspaceId: "page-test" }, data: data() });
|
||||||
|
await fireEvent.click(
|
||||||
|
screen.getByRole("button", { name: "Restore Catalog worker" }),
|
||||||
|
);
|
||||||
|
const retry = await screen.findByRole("button", {
|
||||||
|
name: "Retry Restore Catalog worker",
|
||||||
|
});
|
||||||
|
publish(store, [
|
||||||
|
worker({
|
||||||
|
pinned: true,
|
||||||
|
...{ restore_observation_token: "observation-2" },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
await waitFor(() =>
|
||||||
|
expect((retry as HTMLButtonElement).disabled).toBe(false)
|
||||||
|
);
|
||||||
|
expect(attempts).toBe(1);
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("button", {
|
||||||
|
name: "Restore Catalog worker",
|
||||||
|
}) as HTMLButtonElement).disabled,
|
||||||
|
).toBe(true);
|
||||||
|
await fireEvent.click(retry);
|
||||||
|
await waitFor(() => expect(attempts).toBe(2));
|
||||||
|
const operations = request.mock.calls.filter(([url]) =>
|
||||||
|
String(url).endsWith("/restore")
|
||||||
|
);
|
||||||
|
expect(operations[1][1]?.body).toBe(operations[0][1]?.body);
|
||||||
|
expect(JSON.parse(String(operations[0][1]?.body))).toEqual({
|
||||||
|
expected_observation_token: "observation-1",
|
||||||
|
request_id: expect.any(String),
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test.each(["switch", "dispose"] as const)(
|
||||||
|
"late restore result is ignored after %s",
|
||||||
|
async (mode) => {
|
||||||
|
catalog();
|
||||||
|
catalog("next-workspace");
|
||||||
|
const pending = deferred<Response>();
|
||||||
|
const request = vi.fn((_input: RequestInfo | URL) => pending.promise);
|
||||||
|
vi.stubGlobal("fetch", request);
|
||||||
|
let alerts: unknown[] = [];
|
||||||
|
const unsubscribe = workspaceAlerts.subscribe((value) => {
|
||||||
|
alerts = value;
|
||||||
|
});
|
||||||
|
const view = render(WorkersPage, {
|
||||||
|
params: { workspaceId: "page-test" },
|
||||||
|
data: data(),
|
||||||
|
});
|
||||||
|
await fireEvent.click(
|
||||||
|
screen.getByRole("button", { name: "Restore Catalog worker" }),
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
(screen.getByRole("button", {
|
||||||
|
name: "Unpin Catalog worker",
|
||||||
|
}) as HTMLButtonElement).disabled,
|
||||||
|
).toBe(true);
|
||||||
|
if (mode === "switch") {
|
||||||
|
await view.rerender({
|
||||||
|
data: data("next-workspace", worker({ display_name: "Next worker" })),
|
||||||
|
});
|
||||||
|
} else view.unmount();
|
||||||
|
pending.resolve(restoreResponse("accepted"));
|
||||||
|
await pending.promise;
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
expect(shared.refresh).not.toHaveBeenCalled();
|
||||||
|
expect(request).toHaveBeenCalledTimes(1);
|
||||||
|
expect(alerts).toEqual([]);
|
||||||
|
unsubscribe();
|
||||||
|
},
|
||||||
|
);
|
||||||
+314
@@ -0,0 +1,314 @@
|
|||||||
|
// @vitest-environment happy-dom
|
||||||
|
import {
|
||||||
|
cleanup,
|
||||||
|
fireEvent,
|
||||||
|
render,
|
||||||
|
screen,
|
||||||
|
waitFor,
|
||||||
|
} from "@testing-library/svelte";
|
||||||
|
import { afterEach, beforeEach, expect, test, vi } from "vitest";
|
||||||
|
import type { ComponentProps } from "svelte";
|
||||||
|
import type {
|
||||||
|
SubscriptionFrame,
|
||||||
|
SubscriptionWorker,
|
||||||
|
} from "#lib/generated/protocol.ts";
|
||||||
|
import {
|
||||||
|
clearWorkspaceAlerts,
|
||||||
|
workspaceAlerts,
|
||||||
|
} from "#lib/workspace/alerts/store.ts";
|
||||||
|
import { disposeWorkspaceWorkersStore } from "#lib/workspace/sidebar/worker-subscription.ts";
|
||||||
|
import WorkersNavSection from "#lib/workspace/sidebar/WorkersNavSection.svelte";
|
||||||
|
import WorkersPage from "./+page.svelte";
|
||||||
|
|
||||||
|
const transport = vi.hoisted(() => ({
|
||||||
|
listener: null as null | { onFrame(frame: SubscriptionFrame): void },
|
||||||
|
}));
|
||||||
|
vi.mock("#lib/workspace/multiplexer.ts", () => ({
|
||||||
|
workspaceMultiplexer: () => ({
|
||||||
|
subscribe: (_: unknown, listener: typeof transport.listener) => {
|
||||||
|
transport.listener = listener;
|
||||||
|
return { close: vi.fn() };
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
const workspaceId = "team /";
|
||||||
|
const runtimeId = "runtime /";
|
||||||
|
const workerId = "worker /";
|
||||||
|
const restoreUrl =
|
||||||
|
"/api/w/team%20%2F/runtimes/runtime%20%2F/workers/worker%20%2F/restore";
|
||||||
|
function worker(state = "stopped", token = "generation-1") {
|
||||||
|
return {
|
||||||
|
runtime_id: runtimeId,
|
||||||
|
worker_id: workerId,
|
||||||
|
resource_key: "W-1",
|
||||||
|
host_id: runtimeId,
|
||||||
|
display_name: "Team Worker",
|
||||||
|
label: "Team Worker",
|
||||||
|
availability: "observed" as const,
|
||||||
|
state,
|
||||||
|
restore_observation_token: token,
|
||||||
|
tags: [],
|
||||||
|
diagnostics: [],
|
||||||
|
pinned: false,
|
||||||
|
retention_state: "normal",
|
||||||
|
workspace: {
|
||||||
|
visibility: "workspace",
|
||||||
|
identity: "registered",
|
||||||
|
workspace_id: workspaceId,
|
||||||
|
},
|
||||||
|
implementation: { kind: "runtime", display_hint: "Runtime" },
|
||||||
|
workdir_attachments: [],
|
||||||
|
worker_state: state === "stopped"
|
||||||
|
? null
|
||||||
|
: { last_command_id: 0, state: { kind: "idle" as const } },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function frame(current: ReturnType<typeof worker>): SubscriptionFrame {
|
||||||
|
const subscriptionWorker = {
|
||||||
|
...current,
|
||||||
|
has_running_internal_workers: false,
|
||||||
|
workspace_id: workspaceId,
|
||||||
|
} as SubscriptionWorker;
|
||||||
|
return {
|
||||||
|
protocol_version: 2,
|
||||||
|
frame: "response",
|
||||||
|
message: {
|
||||||
|
result: "subscribed",
|
||||||
|
payload: {
|
||||||
|
request_id: "request",
|
||||||
|
subscription_id: "sub",
|
||||||
|
selector: { topic: "workspace_workers" },
|
||||||
|
snapshot: { topic: "workers", data: { workers: [subscriptionWorker] } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function plan(state: string) {
|
||||||
|
return {
|
||||||
|
workspace_id: workspaceId,
|
||||||
|
runtime_id: runtimeId,
|
||||||
|
generated_at: "2026-10-06T00:00:00Z",
|
||||||
|
revision: state,
|
||||||
|
digest: state,
|
||||||
|
diagnostics: [],
|
||||||
|
workdirs: [],
|
||||||
|
workers: state === "stopped"
|
||||||
|
? [{
|
||||||
|
target_id: "target",
|
||||||
|
action: "worker_delete",
|
||||||
|
worker_id: "W-1",
|
||||||
|
runtime_worker_id: workerId,
|
||||||
|
runtime_id: runtimeId,
|
||||||
|
reason: "stopped",
|
||||||
|
pinned: false,
|
||||||
|
retention_state: "normal",
|
||||||
|
linked_workdir_ids: [],
|
||||||
|
running_linked: false,
|
||||||
|
}]
|
||||||
|
: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function response(state: string, summary?: ReturnType<typeof worker>) {
|
||||||
|
return Response.json({
|
||||||
|
workspace_id: workspaceId,
|
||||||
|
runtime_id: runtimeId,
|
||||||
|
worker_id: workerId,
|
||||||
|
result: { state, worker: summary, diagnostics: [] },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function deferred<T>() {
|
||||||
|
let resolve!: (value: T) => void;
|
||||||
|
const promise = new Promise<T>((r) => resolve = r);
|
||||||
|
return { promise, resolve };
|
||||||
|
}
|
||||||
|
function mount() {
|
||||||
|
const row = worker();
|
||||||
|
render(WorkersNavSection, {
|
||||||
|
workspaceId,
|
||||||
|
currentPath: `/w/${workspaceId}/workers`,
|
||||||
|
});
|
||||||
|
render(WorkersPage, {
|
||||||
|
params: { workspaceId },
|
||||||
|
data: {
|
||||||
|
workspaceId,
|
||||||
|
workers: { items: [row] },
|
||||||
|
workersError: null,
|
||||||
|
cleanupPlans: {},
|
||||||
|
cleanupPlanErrors: {},
|
||||||
|
} as unknown as ComponentProps<typeof WorkersPage>["data"],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async function sidebarRestore() {
|
||||||
|
await fireEvent.click(
|
||||||
|
await screen.findByRole("button", { name: "Actions for Team Worker" }),
|
||||||
|
);
|
||||||
|
return screen.findByRole("menuitem", { name: "Restore" });
|
||||||
|
}
|
||||||
|
function pageRestore() {
|
||||||
|
return screen.getByRole("button", { name: "Restore Team Worker" });
|
||||||
|
}
|
||||||
|
let current = worker();
|
||||||
|
let handleRestore: (init?: RequestInit) => Promise<Response>;
|
||||||
|
let requests: ReturnType<typeof vi.fn>;
|
||||||
|
beforeEach(() => {
|
||||||
|
transport.listener = null;
|
||||||
|
current = worker();
|
||||||
|
handleRestore = async () => response("accepted");
|
||||||
|
requests = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||||
|
const url = String(input);
|
||||||
|
if (url === restoreUrl) return handleRestore(init);
|
||||||
|
if (url.endsWith("/workers")) {
|
||||||
|
return Response.json({
|
||||||
|
workspace_id: workspaceId,
|
||||||
|
limit: 100,
|
||||||
|
source: "catalog",
|
||||||
|
items: [current],
|
||||||
|
diagnostics: [],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (url.endsWith("/cleanup-plan")) {
|
||||||
|
return Response.json(plan(current.state));
|
||||||
|
}
|
||||||
|
if (url.endsWith("/working-directories")) {
|
||||||
|
return Response.json({
|
||||||
|
workspace_id: workspaceId,
|
||||||
|
items: [],
|
||||||
|
diagnostics: [],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw new Error(`Unexpected request ${url}`);
|
||||||
|
});
|
||||||
|
vi.stubGlobal("fetch", requests);
|
||||||
|
clearWorkspaceAlerts();
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
disposeWorkspaceWorkersStore(workspaceId);
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
clearWorkspaceAlerts();
|
||||||
|
});
|
||||||
|
async function ready() {
|
||||||
|
await waitFor(() => expect(transport.listener).not.toBeNull());
|
||||||
|
transport.listener!.onFrame(frame(current));
|
||||||
|
await waitFor(() =>
|
||||||
|
expect((pageRestore() as HTMLButtonElement).disabled).toBe(false)
|
||||||
|
);
|
||||||
|
await screen.findByRole("button", { name: "Delete Team Worker" });
|
||||||
|
}
|
||||||
|
|
||||||
|
test.each(["sidebar", "page"] as const)(
|
||||||
|
"%s Restore updates both actual components through the real store and invalidates cleanup without reload",
|
||||||
|
async (entrance) => {
|
||||||
|
const pending = deferred<Response>();
|
||||||
|
handleRestore = () => pending.promise;
|
||||||
|
mount();
|
||||||
|
await ready();
|
||||||
|
await fireEvent.click(
|
||||||
|
entrance === "sidebar" ? await sidebarRestore() : pageRestore(),
|
||||||
|
);
|
||||||
|
const operations = requests.mock.calls.filter(([url]) =>
|
||||||
|
String(url) === restoreUrl
|
||||||
|
);
|
||||||
|
expect(operations).toHaveLength(1);
|
||||||
|
expect(operations[0][1].method).toBe("POST");
|
||||||
|
expect(JSON.parse(operations[0][1].body)).toEqual({
|
||||||
|
expected_observation_token: "generation-1",
|
||||||
|
request_id: expect.any(String),
|
||||||
|
});
|
||||||
|
current = worker("idle", "generation-2");
|
||||||
|
transport.listener!.onFrame(frame(current));
|
||||||
|
await screen.findByRole("cell", { name: "idle" });
|
||||||
|
await waitFor(() =>
|
||||||
|
expect(screen.queryByRole("button", { name: "Delete Team Worker" }))
|
||||||
|
.toBeNull()
|
||||||
|
);
|
||||||
|
// A late old response cannot put either component back into stopped/running.
|
||||||
|
pending.resolve(response("accepted", worker("stopped", "generation-1")));
|
||||||
|
await waitFor(() =>
|
||||||
|
expect((pageRestore() as HTMLButtonElement).disabled).toBe(true)
|
||||||
|
);
|
||||||
|
expect(screen.queryByRole("cell", { name: "stopped" })).toBeNull();
|
||||||
|
const menuRestore = await sidebarRestore();
|
||||||
|
expect((menuRestore as HTMLButtonElement).disabled).toBe(true);
|
||||||
|
expect(screen.getAllByRole("link", { name: /Team Worker/ })).toHaveLength(
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
expect(requests.mock.calls.filter(([url]) => String(url) === restoreUrl))
|
||||||
|
.toHaveLength(1);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test("separate entrances remain independent while Runtime conflict triggers shared refresh, not automatic replay", async () => {
|
||||||
|
const first = deferred<Response>();
|
||||||
|
let count = 0;
|
||||||
|
handleRestore = async () =>
|
||||||
|
++count === 1 ? first.promise : Response.json({
|
||||||
|
error: "restore_observation_conflict",
|
||||||
|
message: "Observation changed",
|
||||||
|
}, { status: 409 });
|
||||||
|
const alerts: Array<unknown[]> = [];
|
||||||
|
const unsubscribe = workspaceAlerts.subscribe((value) => alerts.push(value));
|
||||||
|
mount();
|
||||||
|
await ready();
|
||||||
|
await fireEvent.click(await sidebarRestore());
|
||||||
|
expect((pageRestore() as HTMLButtonElement).disabled).toBe(false); // not a cross-screen lock
|
||||||
|
await fireEvent.click(pageRestore());
|
||||||
|
await waitFor(() =>
|
||||||
|
expect(JSON.stringify(alerts)).toContain("Worker observation changed")
|
||||||
|
);
|
||||||
|
const operations = requests.mock.calls.filter(([url]) =>
|
||||||
|
String(url) === restoreUrl
|
||||||
|
);
|
||||||
|
expect(operations).toHaveLength(2);
|
||||||
|
const [a, b] = operations.map(([, init]) => JSON.parse(init.body));
|
||||||
|
expect(a.expected_observation_token).toBe(b.expected_observation_token);
|
||||||
|
expect(a.request_id).not.toBe(b.request_id);
|
||||||
|
expect(screen.queryByRole("button", { name: "Retry Restore Team Worker" }))
|
||||||
|
.toBeNull();
|
||||||
|
current = worker("idle", "generation-2");
|
||||||
|
transport.listener!.onFrame(frame(current));
|
||||||
|
first.resolve(response("accepted"));
|
||||||
|
await screen.findByRole("cell", { name: "idle" });
|
||||||
|
await waitFor(() =>
|
||||||
|
expect((pageRestore() as HTMLButtonElement).disabled).toBe(true)
|
||||||
|
);
|
||||||
|
expect(count).toBe(2);
|
||||||
|
unsubscribe();
|
||||||
|
});
|
||||||
|
|
||||||
|
test.each(["http", "malformed"] as const)(
|
||||||
|
"%s Restore response never produces success in either entrance",
|
||||||
|
async (failure) => {
|
||||||
|
handleRestore = async () =>
|
||||||
|
failure === "http"
|
||||||
|
? Response.json({ message: "forbidden restore" }, { status: 403 })
|
||||||
|
: Response.json({
|
||||||
|
workspace_id: workspaceId,
|
||||||
|
runtime_id: runtimeId,
|
||||||
|
worker_id: workerId,
|
||||||
|
result: { state: "new-state" },
|
||||||
|
});
|
||||||
|
const alerts: Array<unknown[]> = [];
|
||||||
|
const unsubscribe = workspaceAlerts.subscribe((value) =>
|
||||||
|
alerts.push(value)
|
||||||
|
);
|
||||||
|
mount();
|
||||||
|
await ready();
|
||||||
|
await fireEvent.click(pageRestore());
|
||||||
|
await waitFor(() =>
|
||||||
|
expect(JSON.stringify(alerts)).toContain(
|
||||||
|
failure === "http" ? "forbidden restore" : "outcome unknown",
|
||||||
|
)
|
||||||
|
);
|
||||||
|
expect(JSON.stringify(alerts)).not.toContain("Worker restored");
|
||||||
|
expect(screen.getByRole("cell", { name: "stopped" })).toBeTruthy();
|
||||||
|
// The sidebar is a second explicit entrance, not an automatic retry.
|
||||||
|
await fireEvent.click(await sidebarRestore());
|
||||||
|
await waitFor(() =>
|
||||||
|
expect(requests.mock.calls.filter(([url]) => String(url) === restoreUrl))
|
||||||
|
.toHaveLength(2)
|
||||||
|
);
|
||||||
|
expect(JSON.stringify(alerts)).not.toContain("Worker restored");
|
||||||
|
unsubscribe();
|
||||||
|
},
|
||||||
|
);
|
||||||
@@ -1,7 +1,12 @@
|
|||||||
import {
|
import {
|
||||||
canDeleteSidebarWorker,
|
canDeleteSidebarWorker,
|
||||||
|
canRestoreWorker,
|
||||||
canStopSidebarWorker,
|
canStopSidebarWorker,
|
||||||
|
createRestoreRequest,
|
||||||
deleteSidebarWorker,
|
deleteSidebarWorker,
|
||||||
|
restoreErrorNotice,
|
||||||
|
restoreNotice,
|
||||||
|
restoreWorkspaceWorker,
|
||||||
stopSidebarWorker,
|
stopSidebarWorker,
|
||||||
} from "../../src/lib/workspace/sidebar/worker-actions.ts";
|
} from "../../src/lib/workspace/sidebar/worker-actions.ts";
|
||||||
import type { Worker } from "../../src/lib/workspace/sidebar/types.ts";
|
import type { Worker } from "../../src/lib/workspace/sidebar/types.ts";
|
||||||
@@ -243,7 +248,176 @@ Deno.test("Worker navigation exposes an accessible hover action menu", async ()
|
|||||||
|
|
||||||
assert(source.includes('aria-haspopup="menu"'));
|
assert(source.includes('aria-haspopup="menu"'));
|
||||||
assert(source.includes('role="menuitem"'));
|
assert(source.includes('role="menuitem"'));
|
||||||
assert(source.includes("stopSidebarWorker(workspaceId, worker)"));
|
assert(source.includes("stopSidebarWorker"));
|
||||||
assert(source.includes("deleteSidebarWorker(workspaceId, worker)"));
|
assert(source.includes("deleteSidebarWorker"));
|
||||||
assert(styles.includes(".worker-nav-item:hover .worker-actions-trigger"));
|
assert(styles.includes(".worker-nav-item:hover .worker-actions-trigger"));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("Restore eligibility uses observed catalog state and backend token, not foreground state", () => {
|
||||||
|
const stopped = {
|
||||||
|
availability: "observed" as const,
|
||||||
|
state: "unknown",
|
||||||
|
lifecycleState: "stopped",
|
||||||
|
restore_observation_token: "generation",
|
||||||
|
};
|
||||||
|
assert(canRestoreWorker(stopped));
|
||||||
|
assert(!canRestoreWorker({ ...stopped, availability: "unavailable" }));
|
||||||
|
assert(
|
||||||
|
!canRestoreWorker({ ...stopped, lifecycleState: "idle", state: "stopped" }),
|
||||||
|
);
|
||||||
|
assert(!canRestoreWorker({ ...stopped, restore_observation_token: null }));
|
||||||
|
assert(
|
||||||
|
canRestoreWorker({
|
||||||
|
availability: "observed",
|
||||||
|
state: "stopped",
|
||||||
|
restore_observation_token: "generation",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
Deno.test("Restore scoped POST encodes identities and sends immutable intent without assignment query", async () => {
|
||||||
|
const request = createRestoreRequest({
|
||||||
|
restore_observation_token: "generation",
|
||||||
|
});
|
||||||
|
const requests: Array<{ url: string; init?: RequestInit }> = [];
|
||||||
|
const fetchFn = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||||
|
requests.push({ url: input.toString(), init });
|
||||||
|
return jsonResponse({
|
||||||
|
workspace_id: "team space",
|
||||||
|
runtime_id: worker.runtime_id,
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
result: { state: "accepted" },
|
||||||
|
});
|
||||||
|
}) as typeof fetch;
|
||||||
|
await restoreWorkspaceWorker("team space", worker, request, fetchFn);
|
||||||
|
await restoreWorkspaceWorker("team space", worker, request, fetchFn);
|
||||||
|
assertEquals(
|
||||||
|
requests[0].url,
|
||||||
|
"/api/w/team%20space/runtimes/runtime%20%2F/workers/worker%20%2F/restore",
|
||||||
|
);
|
||||||
|
assertEquals(requests[0].init?.method, "POST");
|
||||||
|
assertEquals(JSON.parse(String(requests[0].init?.body)), request);
|
||||||
|
assertEquals(requests[1].init?.body, requests[0].init?.body);
|
||||||
|
assert(request.request_id.length > 0);
|
||||||
|
});
|
||||||
|
Deno.test("Restore classification never promotes nonaccepted results and preserves unknown outcome recovery", async () => {
|
||||||
|
for (
|
||||||
|
const state of [
|
||||||
|
"accepted",
|
||||||
|
"rejected",
|
||||||
|
"rolled_back",
|
||||||
|
"reconciliation_required",
|
||||||
|
] as const
|
||||||
|
) {
|
||||||
|
const response = await restoreWorkspaceWorker(
|
||||||
|
"team",
|
||||||
|
worker,
|
||||||
|
{ expected_observation_token: "g", request_id: "id" },
|
||||||
|
(() =>
|
||||||
|
Promise.resolve(jsonResponse({
|
||||||
|
workspace_id: "team",
|
||||||
|
runtime_id: worker.runtime_id,
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
result: {
|
||||||
|
state,
|
||||||
|
diagnostics: [{
|
||||||
|
code: "result",
|
||||||
|
severity: "warning",
|
||||||
|
message: "backend reason",
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
}))) as typeof fetch,
|
||||||
|
);
|
||||||
|
const notice = restoreNotice(response);
|
||||||
|
assertEquals(notice.level === "info", state === "accepted");
|
||||||
|
assertEquals(notice.retry, state === "reconciliation_required");
|
||||||
|
if (state !== "accepted") assert(notice.message.includes("backend reason"));
|
||||||
|
}
|
||||||
|
for (const status of [400, 403, 409, 500]) {
|
||||||
|
try {
|
||||||
|
await restoreWorkspaceWorker(
|
||||||
|
"team",
|
||||||
|
worker,
|
||||||
|
{ expected_observation_token: "g", request_id: "id" },
|
||||||
|
(() =>
|
||||||
|
Promise.resolve(
|
||||||
|
jsonResponse({ error: "conflict", message: "changed" }, status),
|
||||||
|
)) as typeof fetch,
|
||||||
|
);
|
||||||
|
throw new Error("must reject");
|
||||||
|
} catch (error) {
|
||||||
|
const notice = restoreErrorNotice(error);
|
||||||
|
assertEquals(notice.retry, status >= 500);
|
||||||
|
if (status === 409) {
|
||||||
|
assertEquals(notice.title, "Worker observation changed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert(restoreErrorNotice(new TypeError("network failed")).retry);
|
||||||
|
await assertRejects(
|
||||||
|
() =>
|
||||||
|
restoreWorkspaceWorker(
|
||||||
|
"team",
|
||||||
|
worker,
|
||||||
|
{ expected_observation_token: "g", request_id: "id" },
|
||||||
|
(() =>
|
||||||
|
Promise.resolve(
|
||||||
|
jsonResponse({
|
||||||
|
workspace_id: "other",
|
||||||
|
runtime_id: worker.runtime_id,
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
result: { state: "accepted" },
|
||||||
|
}),
|
||||||
|
)) as typeof fetch,
|
||||||
|
),
|
||||||
|
"requested target",
|
||||||
|
);
|
||||||
|
await assertRejects(
|
||||||
|
() =>
|
||||||
|
restoreWorkspaceWorker(
|
||||||
|
"team",
|
||||||
|
worker,
|
||||||
|
{ expected_observation_token: "g", request_id: "id" },
|
||||||
|
(() =>
|
||||||
|
Promise.resolve(
|
||||||
|
jsonResponse({
|
||||||
|
workspace_id: "team",
|
||||||
|
runtime_id: worker.runtime_id,
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
result: { state: "future" },
|
||||||
|
}),
|
||||||
|
)) as typeof fetch,
|
||||||
|
),
|
||||||
|
"unknown value",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
Deno.test("Restore bounds the response before decoding and retains uncertain outcome instead of success", async () => {
|
||||||
|
let response = new Response("{}", {
|
||||||
|
headers: { "content-length": String(8 * 1024 * 1024 + 1) },
|
||||||
|
});
|
||||||
|
let caught: unknown;
|
||||||
|
try {
|
||||||
|
await restoreWorkspaceWorker("team", worker, {
|
||||||
|
expected_observation_token: "g",
|
||||||
|
request_id: "id",
|
||||||
|
}, (() => Promise.resolve(response)) as typeof fetch);
|
||||||
|
} catch (error) {
|
||||||
|
caught = error;
|
||||||
|
}
|
||||||
|
assert(caught instanceof Error);
|
||||||
|
assert(restoreErrorNotice(caught).retry);
|
||||||
|
response = new Response("{}", {
|
||||||
|
status: 409,
|
||||||
|
headers: { "content-length": String(64 * 1024 + 1) },
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await restoreWorkspaceWorker("team", worker, {
|
||||||
|
expected_observation_token: "g",
|
||||||
|
request_id: "id",
|
||||||
|
}, (() => Promise.resolve(response)) as typeof fetch);
|
||||||
|
} catch (error) {
|
||||||
|
caught = error;
|
||||||
|
}
|
||||||
|
assertEquals(restoreErrorNotice(caught).title, "Worker observation changed");
|
||||||
|
assertEquals(restoreErrorNotice(caught).retry, false);
|
||||||
|
});
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
parseRuntimeCleanupExecution,
|
parseRuntimeCleanupExecution,
|
||||||
parseRuntimeCleanupPlan,
|
parseRuntimeCleanupPlan,
|
||||||
parseRuntimeWorkerLifecycleResult,
|
parseRuntimeWorkerLifecycleResult,
|
||||||
|
parseWorkerRestoreResponse,
|
||||||
parseWorkerRetentionResponse,
|
parseWorkerRetentionResponse,
|
||||||
} from "../src/lib/workspace/api/runtime-workers.ts";
|
} from "../src/lib/workspace/api/runtime-workers.ts";
|
||||||
|
|
||||||
@@ -19,7 +20,9 @@ function assertThrows(operation: () => unknown, expected: string): void {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : String(error);
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
if (message.includes(expected)) return;
|
if (message.includes(expected)) return;
|
||||||
throw new Error(`expected error containing ${expected}, received ${message}`);
|
throw new Error(
|
||||||
|
`expected error containing ${expected}, received ${message}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
throw new Error("expected operation to throw");
|
throw new Error("expected operation to throw");
|
||||||
}
|
}
|
||||||
@@ -67,7 +70,10 @@ function plan() {
|
|||||||
|
|
||||||
Deno.test("Runtime cleanup parsers accept the generated exact response shapes", () => {
|
Deno.test("Runtime cleanup parsers accept the generated exact response shapes", () => {
|
||||||
const parsedPlan = parseRuntimeCleanupPlan(plan());
|
const parsedPlan = parseRuntimeCleanupPlan(plan());
|
||||||
assert(parsedPlan.workers[0]?.runtime_worker_id === "worker-1", "worker candidate missing");
|
assert(
|
||||||
|
parsedPlan.workers[0]?.runtime_worker_id === "worker-1",
|
||||||
|
"worker candidate missing",
|
||||||
|
);
|
||||||
|
|
||||||
const execution = parseRuntimeCleanupExecution({
|
const execution = parseRuntimeCleanupExecution({
|
||||||
workspace_id: "workspace-a",
|
workspace_id: "workspace-a",
|
||||||
@@ -82,7 +88,10 @@ Deno.test("Runtime cleanup parsers accept the generated exact response shapes",
|
|||||||
plan_after: plan(),
|
plan_after: plan(),
|
||||||
diagnostics: [],
|
diagnostics: [],
|
||||||
});
|
});
|
||||||
assert(execution.results[0]?.status === "deleted", "execution result missing");
|
assert(
|
||||||
|
execution.results[0]?.status === "deleted",
|
||||||
|
"execution result missing",
|
||||||
|
);
|
||||||
|
|
||||||
const lifecycle = parseRuntimeWorkerLifecycleResult({
|
const lifecycle = parseRuntimeWorkerLifecycleResult({
|
||||||
state: "accepted",
|
state: "accepted",
|
||||||
@@ -108,15 +117,182 @@ Deno.test("Runtime cleanup parsers reject unknown fields and unsafe values", ()
|
|||||||
"not part of the wire contract",
|
"not part of the wire contract",
|
||||||
);
|
);
|
||||||
assertThrows(
|
assertThrows(
|
||||||
() => parseRuntimeCleanupPlan({ ...plan(), workers: new Array(1_001).fill(plan().workers[0]) }),
|
() =>
|
||||||
|
parseRuntimeCleanupPlan({
|
||||||
|
...plan(),
|
||||||
|
workers: new Array(1_001).fill(plan().workers[0]),
|
||||||
|
}),
|
||||||
"at most 1000",
|
"at most 1000",
|
||||||
);
|
);
|
||||||
assertThrows(
|
assertThrows(
|
||||||
() => parseRuntimeCleanupPlan({ ...plan(), workers: [{ ...plan().workers[0], estimated_reclaim_bytes: Number.MAX_SAFE_INTEGER + 1 }] }),
|
() =>
|
||||||
|
parseRuntimeCleanupPlan({
|
||||||
|
...plan(),
|
||||||
|
workers: [{
|
||||||
|
...plan().workers[0],
|
||||||
|
estimated_reclaim_bytes: Number.MAX_SAFE_INTEGER + 1,
|
||||||
|
}],
|
||||||
|
}),
|
||||||
"safe integer",
|
"safe integer",
|
||||||
);
|
);
|
||||||
assertThrows(
|
assertThrows(
|
||||||
() => parseRuntimeWorkerLifecycleResult({ state: "future", runtime_id: "runtime-a", worker_id: "worker-1", diagnostics: [] }),
|
() =>
|
||||||
|
parseRuntimeWorkerLifecycleResult({
|
||||||
|
state: "future",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-1",
|
||||||
|
diagnostics: [],
|
||||||
|
}),
|
||||||
"unknown value",
|
"unknown value",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function restore(state = "accepted", worker?: unknown) {
|
||||||
|
return {
|
||||||
|
workspace_id: "workspace-a",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-1",
|
||||||
|
result: { state, diagnostics: [], worker },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function restoredWorker() {
|
||||||
|
return {
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-1",
|
||||||
|
resource_key: "W-1",
|
||||||
|
host_id: "runtime-a",
|
||||||
|
label: "Worker",
|
||||||
|
availability: "observed",
|
||||||
|
state: "idle",
|
||||||
|
workspace: {
|
||||||
|
visibility: "workspace",
|
||||||
|
identity: "registered",
|
||||||
|
workspace_id: "workspace-a",
|
||||||
|
},
|
||||||
|
implementation: { kind: "runtime", display_hint: "Runtime" },
|
||||||
|
restore_observation_token: "generation-b",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Deno.test("Restore parser preserves all four states and optional Workspace summary", () => {
|
||||||
|
for (
|
||||||
|
const state of [
|
||||||
|
"accepted",
|
||||||
|
"rejected",
|
||||||
|
"rolled_back",
|
||||||
|
"reconciliation_required",
|
||||||
|
]
|
||||||
|
) {
|
||||||
|
assert(
|
||||||
|
parseWorkerRestoreResponse(restore(state)).result.state === state,
|
||||||
|
"state lost",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
parseWorkerRestoreResponse(restore(state, null)).result.worker === null,
|
||||||
|
"null lost",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const parsed = parseWorkerRestoreResponse(
|
||||||
|
restore("accepted", restoredWorker()),
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
parsed.result.worker?.resource_key === "W-1",
|
||||||
|
"Workspace summary lost",
|
||||||
|
);
|
||||||
|
assert(parsed.result.worker?.state === "idle", "must not force running");
|
||||||
|
});
|
||||||
|
Deno.test("Restore parser bounds diagnostics, validates wrapper/summary identity and rejects malformed results", () => {
|
||||||
|
assertThrows(() => parseWorkerRestoreResponse(restore("accepted", {
|
||||||
|
...restoredWorker(), restore_observation_token: "😀".repeat(65),
|
||||||
|
})), "256 UTF-8 byte limit");
|
||||||
|
assertThrows(
|
||||||
|
() => parseWorkerRestoreResponse(restore("future")),
|
||||||
|
"unknown value",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() => parseWorkerRestoreResponse({ ...restore(), runtime_id: null }),
|
||||||
|
"must be a string",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() => parseWorkerRestoreResponse({ ...restore(), extra: true }),
|
||||||
|
"not part of the wire contract",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() =>
|
||||||
|
parseWorkerRestoreResponse({
|
||||||
|
...restore(),
|
||||||
|
result: {
|
||||||
|
state: "accepted",
|
||||||
|
diagnostics: new Array(65).fill({
|
||||||
|
code: "e",
|
||||||
|
severity: "error",
|
||||||
|
message: "e",
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
"at most 64",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() =>
|
||||||
|
parseWorkerRestoreResponse({
|
||||||
|
...restore(),
|
||||||
|
result: {
|
||||||
|
state: "rejected",
|
||||||
|
diagnostics: [{
|
||||||
|
code: "e",
|
||||||
|
severity: "error",
|
||||||
|
message: "x".repeat(4097),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
"4096",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() =>
|
||||||
|
parseWorkerRestoreResponse({
|
||||||
|
...restore(),
|
||||||
|
result: {
|
||||||
|
state: "rejected",
|
||||||
|
diagnostics: [{ code: "e", severity: "invalid", message: "e" }],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
"unknown value",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() =>
|
||||||
|
parseWorkerRestoreResponse(
|
||||||
|
restore("accepted", { ...restoredWorker(), worker_id: "other" }),
|
||||||
|
),
|
||||||
|
"identity",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() =>
|
||||||
|
parseWorkerRestoreResponse(
|
||||||
|
restore("accepted", {
|
||||||
|
...restoredWorker(),
|
||||||
|
workspace: {
|
||||||
|
visibility: "workspace",
|
||||||
|
identity: "registered",
|
||||||
|
workspace_id: "other",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
"identity",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() =>
|
||||||
|
parseWorkerRestoreResponse(
|
||||||
|
restore("accepted", { worker_id: "worker-1", state: "idle" }),
|
||||||
|
),
|
||||||
|
"availability",
|
||||||
|
);
|
||||||
|
assertThrows(
|
||||||
|
() =>
|
||||||
|
parseWorkerRestoreResponse({
|
||||||
|
state: "accepted",
|
||||||
|
runtime_id: "runtime-a",
|
||||||
|
worker_id: "worker-1",
|
||||||
|
diagnostics: [],
|
||||||
|
}),
|
||||||
|
"wire contract",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user