feat: guarded Worker Restore from shared Web actions (T-701)
This commit is contained in:
@@ -10,6 +10,7 @@ pub use server_api::{
|
||||
WorkerLaunchProfileCandidate as BackendWorkerLaunchProfileCandidate,
|
||||
WorkerLaunchRuntimeOption as BackendWorkerLaunchRuntimeOption,
|
||||
WorkerOperationState as BackendWorkerOperationState,
|
||||
WorkerRestoreRequest as BackendWorkerRestoreRequest,
|
||||
WorkerRestoreResponse as BackendWorkerRestoreResponse,
|
||||
WorkerRestoreResult as BackendWorkerRestoreResult,
|
||||
WorkerRestoreState as BackendWorkerRestoreState, WorkerSummary as BackendWorkerSummary,
|
||||
@@ -259,6 +260,15 @@ impl fmt::Display for BackendRuntimeClientError {
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendRuntimeClientError {
|
||||
/// A stale intent is not a rejected Restore operation and must not be retried
|
||||
/// with a silently refreshed token.
|
||||
pub fn is_restore_observation_conflict(&self) -> bool {
|
||||
matches!(self, Self::ContractApi(server_api::client_support::ClientError::Public { status, error })
|
||||
if *status == reqwest::StatusCode::CONFLICT && error.error == "restore_observation_conflict")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for BackendRuntimeClientError {}
|
||||
|
||||
impl From<server_api::ServerApiClientError> for BackendRuntimeClientError {
|
||||
@@ -502,6 +512,7 @@ pub async fn list_backend_stopped_workers(
|
||||
|
||||
pub async fn restore_backend_worker(
|
||||
target: &BackendRuntimeTarget,
|
||||
request: BackendWorkerRestoreRequest,
|
||||
) -> Result<BackendWorkerRestoreResponse, BackendRuntimeClientError> {
|
||||
validate_target(target)?;
|
||||
let api = BackendApiClient::from_stored_token(&target.base_url)?;
|
||||
@@ -514,6 +525,7 @@ pub async fn restore_backend_worker(
|
||||
ticket_id: None,
|
||||
assignment_operation_id: None,
|
||||
},
|
||||
request,
|
||||
)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
|
||||
@@ -26,12 +26,12 @@ pub use backend_runtime::{
|
||||
BackendRuntimeListTarget, BackendRuntimeSummary, BackendRuntimeTarget,
|
||||
BackendWorkerImplementationSummary, BackendWorkerLaunchOptions,
|
||||
BackendWorkerLaunchProfileCandidate, BackendWorkerLaunchRuntimeOption,
|
||||
BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreResponse,
|
||||
BackendWorkerRestoreResult, BackendWorkerRestoreState, BackendWorkerSummary,
|
||||
BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource, BackendWorkingDirectorySummary,
|
||||
connect_backend_runtime, create_backend_worker, get_backend_worker_launch_options,
|
||||
list_backend_stopped_workers, list_backend_workers, observe_backend_worker_session,
|
||||
restore_backend_worker,
|
||||
BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreRequest,
|
||||
BackendWorkerRestoreResponse, BackendWorkerRestoreResult, BackendWorkerRestoreState,
|
||||
BackendWorkerSummary, BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource,
|
||||
BackendWorkingDirectorySummary, connect_backend_runtime, create_backend_worker,
|
||||
get_backend_worker_launch_options, list_backend_stopped_workers, list_backend_workers,
|
||||
observe_backend_worker_session, restore_backend_worker,
|
||||
};
|
||||
pub use backend_workspace::{
|
||||
BackendWorkspace, BackendWorkspaceCatalogTarget, BackendWorkspaceClientError,
|
||||
|
||||
@@ -651,6 +651,10 @@ impl SubscriptionWorkerJob {
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
pub struct SubscriptionWorker {
|
||||
/// Opaque Runtime-owned Restore observation identity, never a display-state hash.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||
pub restore_observation_token: Option<String>,
|
||||
pub worker_id: SubscriptionWorkerId,
|
||||
/// Set by the Workspace Server when projecting a Runtime-owned Worker to clients.
|
||||
/// Runtime producers leave this unset because the connection identifies the Runtime.
|
||||
@@ -689,6 +693,9 @@ pub struct SubscriptionWorker {
|
||||
impl SubscriptionWorker {
|
||||
pub fn validate(&self) -> Result<(), SubscriptionValidationError> {
|
||||
self.worker_id.validate()?;
|
||||
if let Some(token) = &self.restore_observation_token {
|
||||
validate_identifier("restore_observation_token", token, MAX_RESOURCE_ID_BYTES)?;
|
||||
}
|
||||
if let Some(runtime_id) = &self.runtime_id {
|
||||
validate_identifier("runtime_id", runtime_id, MAX_RESOURCE_ID_BYTES)?;
|
||||
}
|
||||
@@ -962,6 +969,7 @@ mod tests {
|
||||
|
||||
fn worker(value: &str) -> SubscriptionWorker {
|
||||
SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: worker_id(value),
|
||||
runtime_id: None,
|
||||
resource_key: None,
|
||||
|
||||
@@ -246,6 +246,7 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec<SubscriptionFrame> {
|
||||
let external_workdir_id = SubscriptionWorkdirId::new("fixture-external-workdir")
|
||||
.expect("fixture external Workdir id must be valid");
|
||||
let worker = SubscriptionWorker {
|
||||
restore_observation_token: Some("fixture-restore-observation".into()),
|
||||
worker_id: worker_id.clone(),
|
||||
runtime_id: Some("fixture-runtime".to_string()),
|
||||
resource_key: Some("fixture-worker-resource".to_string()),
|
||||
|
||||
@@ -378,6 +378,7 @@ function validateWorkers(workers: SubscriptionWorker[]): boolean {
|
||||
}
|
||||
|
||||
function validateWorker(worker: SubscriptionWorker): boolean {
|
||||
if (worker.restore_observation_token != null && !validateIdentifier(worker.restore_observation_token, MAX_RESOURCE_ID_BYTES)) return false;
|
||||
if (!validateIdentifier(worker.worker_id, MAX_RESOURCE_ID_BYTES)) return false;
|
||||
if (worker.runtime_id != null && !validateIdentifier(worker.runtime_id, MAX_RESOURCE_ID_BYTES)) {
|
||||
return false;
|
||||
|
||||
@@ -452,6 +452,8 @@ pub struct CreateWorkerRequest {
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub struct WorkerSummary {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub restore_observation_token: Option<String>,
|
||||
pub worker_ref: WorkerRef,
|
||||
pub worker_id: WorkerId,
|
||||
pub status: WorkerStatus,
|
||||
@@ -484,6 +486,62 @@ pub struct WorkerResponse {
|
||||
pub worker: WorkerDetail,
|
||||
}
|
||||
|
||||
/// Explicit observed Restore intent. Preparation is authored by the Workspace
|
||||
/// Backend, not browser/Tool JSON, and applied only after Runtime admission.
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkerRestoreRequest {
|
||||
pub expected_observation_token: String,
|
||||
pub request_id: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub preparation: Option<WorkerRestorePreparation>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkerRestorePreparation {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub workspace_api: Option<WorkspaceApiRef>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub workdir_attachments: Option<Vec<LogicalWorkdirAttachment>>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub repository_access: Vec<WorkingDirectoryRepositoryAccessRequest>,
|
||||
/// Read-only references resolved by Workspace only after this operation owns admission.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub repository_access_workdirs: Vec<RepositoryAccessWorkdirReference>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RepositoryAccessWorkdirReference {
|
||||
pub runtime_id: String,
|
||||
pub working_directory_id: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkerRestoreCoordinationRequest {
|
||||
pub expected_observation_token: String,
|
||||
pub request_id: String,
|
||||
/// None means owner/result recovery only; it can never admit a new intent.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub preparation: Option<WorkerRestorePreparation>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub struct WorkerRestoreCoordinationResponse {
|
||||
pub result: Option<WorkerRestoreResponse>,
|
||||
/// Original read-only preparation, only when the existing operation still needs it.
|
||||
pub preparation: Option<WorkerRestorePreparation>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkingDirectoryRepositoryAccessRequest {
|
||||
pub working_directory_id: String,
|
||||
pub materialization: RepositoryMaterializationContext,
|
||||
}
|
||||
|
||||
/// Explicit restore outcome returned for every reachable Runtime restore attempt,
|
||||
/// including preflight rejection and uncertain post-side-effect reconciliation.
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
@@ -1000,11 +1058,18 @@ pub trait RuntimeApi {
|
||||
#[body] request: WorkerLifecycleRequest,
|
||||
) -> Result<WorkerLifecycleResponse, RuntimeApiError>;
|
||||
|
||||
#[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400)]
|
||||
#[post("/v1/workers/{worker_id}/restore/coordinate", status = 200, error_status = 400, additional_error_statuses = [409])]
|
||||
async fn coordinate_worker_restore(
|
||||
&self,
|
||||
#[path] worker_id: String,
|
||||
#[body] request: WorkerRestoreCoordinationRequest,
|
||||
) -> Result<WorkerRestoreCoordinationResponse, RuntimeApiError>;
|
||||
|
||||
#[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400, additional_error_statuses = [409])]
|
||||
async fn restore_worker(
|
||||
&self,
|
||||
#[path] worker_id: String,
|
||||
#[body] request: EmptyObjectRequest,
|
||||
#[body] request: WorkerRestoreRequest,
|
||||
) -> Result<WorkerRestoreResponse, RuntimeApiError>;
|
||||
|
||||
#[post(
|
||||
@@ -1210,6 +1275,46 @@ pub const REMAINING_RUNTIME_ROUTES: &[RemainingRuntimeRoute] = &[
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn restore_request_is_mandatory_and_conflict_is_not_an_operation_state() {
|
||||
for invalid in [
|
||||
serde_json::json!({}),
|
||||
serde_json::json!({"request_id":"r"}),
|
||||
serde_json::json!({"expected_observation_token":"t"}),
|
||||
] {
|
||||
assert!(serde_json::from_value::<WorkerRestoreRequest>(invalid).is_err());
|
||||
}
|
||||
let request: WorkerRestoreRequest = serde_json::from_value(
|
||||
serde_json::json!({"expected_observation_token":"t", "request_id":"r"}),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(request.preparation.is_none());
|
||||
for state in [
|
||||
"accepted",
|
||||
"rejected",
|
||||
"rolled_back",
|
||||
"reconciliation_required",
|
||||
] {
|
||||
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
|
||||
}
|
||||
assert!(
|
||||
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
||||
"restore_observation_conflict"
|
||||
))
|
||||
.is_err()
|
||||
);
|
||||
let conflict = RuntimeApiError::new(
|
||||
409,
|
||||
"restore_observation_conflict",
|
||||
"refresh before a new intent",
|
||||
);
|
||||
assert_eq!(conflict.status(), 409);
|
||||
assert_eq!(
|
||||
serde_json::to_value(conflict).unwrap()["error"]["code"],
|
||||
"restore_observation_conflict"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feature_argument_completion_wire_preserves_source_and_argument() {
|
||||
let json = serde_json::json!({"kind":"feature_argument", "prefix":"資料/", "context": {"invocation":"builtin:test/prepare", "argument":"path"}});
|
||||
@@ -1388,10 +1493,17 @@ mod tests {
|
||||
) -> Result<WorkerLifecycleResponse, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
async fn coordinate_worker_restore(
|
||||
&self,
|
||||
_worker_id: String,
|
||||
_request: WorkerRestoreCoordinationRequest,
|
||||
) -> Result<WorkerRestoreCoordinationResponse, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
async fn restore_worker(
|
||||
&self,
|
||||
_worker_id: String,
|
||||
_request: EmptyObjectRequest,
|
||||
_request: WorkerRestoreRequest,
|
||||
) -> Result<WorkerRestoreResponse, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
@@ -1491,7 +1603,7 @@ mod tests {
|
||||
#[test]
|
||||
fn contract_inventory_is_complete_and_unique() {
|
||||
let operations = RuntimeApiMetadata::OPERATIONS;
|
||||
assert_eq!(operations.len(), 19);
|
||||
assert_eq!(operations.len(), 20);
|
||||
let mut routes = operations
|
||||
.iter()
|
||||
.map(|operation| (format!("{:?}", operation.method), operation.path))
|
||||
|
||||
@@ -389,6 +389,7 @@ impl_openapi_schema!(
|
||||
WorkspaceRuntimeListResponse,
|
||||
RuntimeWorkerListResponse,
|
||||
WorkerRestoreResponse,
|
||||
WorkerRestoreRequest,
|
||||
WorkerControlSubject,
|
||||
WorkerControlRecord,
|
||||
WorkerControlListResponse,
|
||||
@@ -1552,6 +1553,7 @@ pub trait ServerApi {
|
||||
#[path] workspace_id: String,
|
||||
#[path] runtime_id: String,
|
||||
#[path] worker_id: String,
|
||||
#[body] request: WorkerRestoreRequest,
|
||||
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
||||
#[get(
|
||||
"/api/w/{workspace_id}/worker-observation/sessions",
|
||||
@@ -2080,6 +2082,7 @@ pub trait ServerApi {
|
||||
#[extension] context: ServerRequestContext,
|
||||
#[path] runtime_id: String,
|
||||
#[path] worker_id: String,
|
||||
#[body] request: WorkerRestoreRequest,
|
||||
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
||||
|
||||
#[post(
|
||||
@@ -2097,6 +2100,7 @@ pub trait ServerApi {
|
||||
#[path] runtime_id: String,
|
||||
#[path] worker_id: String,
|
||||
#[query] query: RestoreTicketAssignmentQuery,
|
||||
#[body] request: WorkerRestoreRequest,
|
||||
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
|
||||
|
||||
#[put(
|
||||
@@ -9594,6 +9598,10 @@ pub struct RuntimeWorkerWorkdirAttachmentSummary {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
pub struct WorkerSummary {
|
||||
/// Opaque Runtime-owned observation fence used by explicit Restore intent.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||
pub restore_observation_token: Option<String>,
|
||||
pub runtime_id: String,
|
||||
pub worker_id: String,
|
||||
pub resource_key: String,
|
||||
@@ -9830,6 +9838,16 @@ pub enum WorkerRestoreState {
|
||||
ReconciliationRequired,
|
||||
}
|
||||
|
||||
/// One explicit Restore intent. Retries retain both fields; conflicts require a
|
||||
/// fresh observation and a deliberate new intent, not automatic token replacement.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkerRestoreRequest {
|
||||
pub expected_observation_token: String,
|
||||
pub request_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -11694,6 +11712,28 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_restore_request_requires_observation_and_request_identity() {
|
||||
let valid = serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request"});
|
||||
let request: WorkerRestoreRequest = serde_json::from_value(valid.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(request).unwrap(), valid);
|
||||
for invalid in [
|
||||
serde_json::json!({}),
|
||||
serde_json::json!({"request_id":"restore-request"}),
|
||||
serde_json::json!({"expected_observation_token":"opaque-generation"}),
|
||||
serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request", "command_id":701}),
|
||||
] {
|
||||
assert!(serde_json::from_value::<WorkerRestoreRequest>(invalid).is_err());
|
||||
}
|
||||
let contract = canonical_openapi_document().unwrap().to_json().unwrap();
|
||||
let contract: serde_json::Value = serde_json::from_str(&contract).unwrap();
|
||||
let restore = &contract["paths"]["/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore"]
|
||||
["post"];
|
||||
assert_eq!(restore["requestBody"]["required"], true);
|
||||
assert!(restore["responses"]["409"].is_object());
|
||||
assert!(contract["components"]["schemas"]["WorkerRestoreRequest"].is_object());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_restore_result_rejects_unknown_fields() {
|
||||
let result = serde_json::from_value::<WorkerRestoreResult>(serde_json::json!({
|
||||
|
||||
@@ -872,14 +872,14 @@ fn validate_runtime_operations(
|
||||
"/api/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
||||
"post",
|
||||
"runtime_worker_restore_alias",
|
||||
None,
|
||||
Some("WorkerRestoreRequest"),
|
||||
&[("200", "WorkerRestoreResponse")],
|
||||
),
|
||||
runtime_operation(
|
||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
||||
"post",
|
||||
"runtime_worker_restore",
|
||||
None,
|
||||
Some("WorkerRestoreRequest"),
|
||||
&[("200", "WorkerRestoreResponse")],
|
||||
),
|
||||
runtime_operation(
|
||||
|
||||
@@ -29,6 +29,7 @@ trait BackendWorkerLifecycle {
|
||||
async fn restore(
|
||||
&self,
|
||||
target: &client::BackendRuntimeTarget,
|
||||
request: client::BackendWorkerRestoreRequest,
|
||||
) -> Result<BackendWorkerRestoreResponse, io::Error>;
|
||||
|
||||
async fn observe(
|
||||
@@ -44,13 +45,19 @@ impl BackendWorkerLifecycle for LiveBackendWorkerLifecycle {
|
||||
async fn restore(
|
||||
&self,
|
||||
target: &client::BackendRuntimeTarget,
|
||||
request: client::BackendWorkerRestoreRequest,
|
||||
) -> Result<BackendWorkerRestoreResponse, io::Error> {
|
||||
restore_backend_worker(target).await.map_err(|error| {
|
||||
io::Error::other(format!(
|
||||
"failed to restore Backend Worker {}: {error}",
|
||||
target.display_label()
|
||||
))
|
||||
})
|
||||
restore_backend_worker(target, request)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if error.is_restore_observation_conflict() {
|
||||
return io::Error::other("Restore observation changed; refresh the Worker list and select again. No new Restore was started.");
|
||||
}
|
||||
io::Error::other(format!(
|
||||
"failed to restore Backend Worker {}: {error}",
|
||||
target.display_label()
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
async fn observe(
|
||||
@@ -133,8 +140,20 @@ pub(crate) async fn run(
|
||||
let attach_target = target
|
||||
.runtime_target(selected.runtime_id.clone(), selected.worker_id.clone())
|
||||
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||
let attach_target =
|
||||
prepare_selected_worker(&LiveBackendWorkerLifecycle, attach_target, intent).await?;
|
||||
let restore_request =
|
||||
selected
|
||||
.restore_observation_token
|
||||
.map(|token| client::BackendWorkerRestoreRequest {
|
||||
expected_observation_token: token,
|
||||
request_id: uuid::Uuid::now_v7().to_string(),
|
||||
});
|
||||
let attach_target = prepare_selected_worker(
|
||||
&LiveBackendWorkerLifecycle,
|
||||
attach_target,
|
||||
intent,
|
||||
restore_request,
|
||||
)
|
||||
.await?;
|
||||
return console::run_backend_runtime(attach_target).await;
|
||||
}
|
||||
}
|
||||
@@ -143,9 +162,13 @@ async fn prepare_selected_worker(
|
||||
lifecycle: &impl BackendWorkerLifecycle,
|
||||
mut target: client::BackendRuntimeTarget,
|
||||
intent: BackendWorkerPickerIntent,
|
||||
restore_request: Option<client::BackendWorkerRestoreRequest>,
|
||||
) -> Result<client::BackendRuntimeTarget, io::Error> {
|
||||
if intent == BackendWorkerPickerIntent::Resume {
|
||||
let response = lifecycle.restore(&target).await?;
|
||||
let request = restore_request.ok_or_else(|| {
|
||||
io::Error::other("Worker has no current Restore observation; refresh and select again")
|
||||
})?;
|
||||
let response = lifecycle.restore(&target, request).await?;
|
||||
ensure_restore_accepted(&response)?;
|
||||
let observation = lifecycle.observe(&target).await?;
|
||||
require_live_resume_observation(observation.observation)?;
|
||||
@@ -158,7 +181,10 @@ async fn prepare_selected_worker(
|
||||
}
|
||||
|
||||
fn is_resume_candidate(worker: &BackendWorkerSummary) -> bool {
|
||||
worker.worker_state.is_none()
|
||||
worker.availability == protocol::subscription::SubscriptionWorkerAvailability::Observed
|
||||
&& worker.state == "stopped"
|
||||
&& worker.worker_state.is_none()
|
||||
&& worker.restore_observation_token.is_some()
|
||||
}
|
||||
|
||||
fn ensure_restore_accepted(response: &BackendWorkerRestoreResponse) -> Result<(), io::Error> {
|
||||
@@ -570,6 +596,7 @@ mod tests {
|
||||
|
||||
fn worker(runtime_id: &str, worker_id: &str, profile: Option<&str>) -> BackendWorkerSummary {
|
||||
BackendWorkerSummary {
|
||||
restore_observation_token: Some("observed-worker-generation".into()),
|
||||
runtime_id: runtime_id.to_string(),
|
||||
worker_id: worker_id.to_string(),
|
||||
resource_key: "W-1".to_string(),
|
||||
@@ -642,6 +669,7 @@ mod tests {
|
||||
async fn restore(
|
||||
&self,
|
||||
_target: &client::BackendRuntimeTarget,
|
||||
_request: client::BackendWorkerRestoreRequest,
|
||||
) -> Result<BackendWorkerRestoreResponse, io::Error> {
|
||||
self.calls.lock().unwrap().push("restore");
|
||||
Ok(self.restore_response.clone())
|
||||
@@ -689,10 +717,17 @@ mod tests {
|
||||
"worker-a",
|
||||
);
|
||||
|
||||
let prepared =
|
||||
prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume)
|
||||
.await
|
||||
.unwrap();
|
||||
let prepared = prepare_selected_worker(
|
||||
&lifecycle,
|
||||
target,
|
||||
BackendWorkerPickerIntent::Resume,
|
||||
Some(client::BackendWorkerRestoreRequest {
|
||||
expected_observation_token: "observed-worker-generation".into(),
|
||||
request_id: "resume-operation".into(),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore", "observe"]);
|
||||
assert!(prepared.initial_snapshot.is_none());
|
||||
@@ -716,10 +751,18 @@ mod tests {
|
||||
"worker-a",
|
||||
);
|
||||
|
||||
let error = prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
let error = prepare_selected_worker(
|
||||
&lifecycle,
|
||||
target,
|
||||
BackendWorkerPickerIntent::Resume,
|
||||
Some(client::BackendWorkerRestoreRequest {
|
||||
expected_observation_token: "observed-worker-generation".into(),
|
||||
request_id: "resume-operation".into(),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
|
||||
assert!(error.contains("Rejected"));
|
||||
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
||||
@@ -736,7 +779,7 @@ mod tests {
|
||||
|
||||
assert!(!is_resume_candidate(&live));
|
||||
assert!(is_resume_candidate(&stopped));
|
||||
assert!(is_resume_candidate(&unknown));
|
||||
assert!(!is_resume_candidate(&unknown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -334,6 +334,8 @@ pub(crate) enum WorkerRestoreIntent {
|
||||
/// Lightweight persisted Worker identity projection.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WorkerSummary {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub restore_observation_token: Option<String>,
|
||||
pub worker_ref: WorkerRef,
|
||||
pub worker_id: WorkerId,
|
||||
pub status: WorkerStatus,
|
||||
@@ -359,6 +361,8 @@ pub struct WorkerSummary {
|
||||
/// Full persisted Worker identity and lifecycle detail.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WorkerDetail {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub restore_observation_token: Option<String>,
|
||||
pub worker_ref: WorkerRef,
|
||||
pub worker_id: WorkerId,
|
||||
pub status: WorkerStatus,
|
||||
|
||||
@@ -12,6 +12,9 @@ pub enum RuntimeError {
|
||||
#[error("initial worker input must be user input, got {kind}")]
|
||||
InvalidInitialInputKind { kind: String },
|
||||
|
||||
#[error("worker {worker_id} Restore observation conflicts with the current operation")]
|
||||
RestoreObservationConflict { worker_id: WorkerId },
|
||||
|
||||
#[error("worker {worker_id} was not found")]
|
||||
WorkerNotFound { worker_id: WorkerId },
|
||||
|
||||
|
||||
@@ -520,6 +520,7 @@ pub(crate) enum PersistedWorkerExecutionState {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct PersistedWorkerRecord {
|
||||
pub(crate) restore_guard: crate::runtime::RestoreGuard,
|
||||
pub(crate) worker_ref: WorkerRef,
|
||||
pub(crate) worker_id: WorkerId,
|
||||
pub(crate) profile: ProfileSelector,
|
||||
@@ -1819,6 +1820,8 @@ impl RuntimeSnapshot {
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct WorkerAggregateRecord {
|
||||
#[serde(default)]
|
||||
restore_guard: crate::runtime::RestoreGuard,
|
||||
schema_version: u32,
|
||||
worker_ref: WorkerRef,
|
||||
worker_id: WorkerId,
|
||||
@@ -1874,6 +1877,7 @@ impl WorkerAggregateRecord {
|
||||
};
|
||||
Self {
|
||||
schema_version: SCHEMA_VERSION,
|
||||
restore_guard: worker.restore_guard.clone(),
|
||||
worker_ref: worker.worker_ref.clone(),
|
||||
worker_id: worker.worker_id,
|
||||
profile: worker.profile.clone(),
|
||||
@@ -1902,6 +1906,7 @@ impl WorkerAggregateRecord {
|
||||
}
|
||||
|
||||
fn validate(self, path: &Path) -> Result<PersistedWorkerRecord, RuntimeError> {
|
||||
let restore_guard = self.restore_guard;
|
||||
let last_finished_submission_request_id = self.last_finished_submission_request_id;
|
||||
let identity = WorkerIdentityRecord {
|
||||
schema_version: self.schema_version,
|
||||
@@ -2007,6 +2012,7 @@ impl WorkerAggregateRecord {
|
||||
WorkerExecutionStateRecord::Unavailable => PersistedWorkerExecutionState::Unavailable,
|
||||
};
|
||||
let mut persisted = identity.into_persisted(execution_state);
|
||||
persisted.restore_guard = restore_guard;
|
||||
persisted.last_finished_submission_request_id = last_finished_submission_request_id;
|
||||
Ok(persisted)
|
||||
}
|
||||
@@ -2180,6 +2186,7 @@ impl WorkerIdentityRecord {
|
||||
self.workdir_attachments
|
||||
};
|
||||
PersistedWorkerRecord {
|
||||
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||
worker_ref: self.worker_ref,
|
||||
worker_id: self.worker_id,
|
||||
profile: self.profile,
|
||||
|
||||
@@ -2302,6 +2302,7 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
||||
}
|
||||
if path.ends_with("/input")
|
||||
|| path.ends_with("/restore")
|
||||
|| path.ends_with("/restore/coordinate")
|
||||
|| (path.contains("/attachments") && *method != Method::GET)
|
||||
|| path.contains("/workdir-attachments")
|
||||
{
|
||||
@@ -2514,6 +2515,7 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
|
||||
StatusCode::CONFLICT
|
||||
}
|
||||
RuntimeError::RuntimeStopped
|
||||
| RuntimeError::RestoreObservationConflict { .. }
|
||||
| RuntimeError::RuntimeStoreAlreadyOpen { .. }
|
||||
| RuntimeError::WorkerExecutionUnavailable { .. }
|
||||
| RuntimeError::ExecutionBackendUnavailable { .. }
|
||||
@@ -2538,6 +2540,9 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
|
||||
fn code_for_runtime_error(error: &RuntimeError) -> String {
|
||||
match error {
|
||||
RuntimeError::RuntimeStopped => "runtime_stopped".to_string(),
|
||||
RuntimeError::RestoreObservationConflict { .. } => {
|
||||
"restore_observation_conflict".to_string()
|
||||
}
|
||||
RuntimeError::RuntimeStoreAlreadyOpen { .. } => "runtime_store_already_open".to_string(),
|
||||
RuntimeError::WorkerNotFound { .. } => "worker_not_found".to_string(),
|
||||
RuntimeError::WorkerExecutionUnavailable { .. } => {
|
||||
@@ -3642,7 +3647,17 @@ mod tests {
|
||||
client
|
||||
.restore_worker(
|
||||
generated_worker_id.clone(),
|
||||
runtime_api::EmptyObjectRequest::default(),
|
||||
runtime_api::WorkerRestoreRequest {
|
||||
expected_observation_token: client
|
||||
.get_worker(generated_worker_id.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.worker
|
||||
.restore_observation_token
|
||||
.unwrap(),
|
||||
request_id: "generated-restore".into(),
|
||||
preparation: Some(runtime_api::WorkerRestorePreparation::default()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -3786,18 +3801,86 @@ mod tests {
|
||||
let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await;
|
||||
assert_eq!(stop.ack.worker_ref, created.worker.worker_ref);
|
||||
|
||||
let response = authed_empty_request(
|
||||
let restore_uri = format!("/v1/workers/{}/restore", created.worker.worker_id);
|
||||
let missing = authed_empty_request(app.clone(), Method::POST, &restore_uri, token).await;
|
||||
assert_eq!(missing.status(), StatusCode::BAD_REQUEST);
|
||||
let stale = runtime_api::WorkerRestoreRequest {
|
||||
expected_observation_token: created.worker.restore_observation_token.clone().unwrap(),
|
||||
request_id: "stale-http-restore".into(),
|
||||
preparation: None,
|
||||
};
|
||||
let response =
|
||||
authed_json_request(app.clone(), Method::POST, &restore_uri, token, &stale).await;
|
||||
assert_eq!(response.status(), StatusCode::CONFLICT);
|
||||
let conflict: RuntimeHttpErrorResponse = read_json(response).await;
|
||||
assert_eq!(conflict.error.code, "restore_observation_conflict");
|
||||
|
||||
let mut restore_request = runtime.test_restore_request(&created.worker.worker_ref);
|
||||
let coordinate_uri = format!("{restore_uri}/coordinate");
|
||||
let mut coordination = runtime_api::WorkerRestoreCoordinationRequest {
|
||||
expected_observation_token: restore_request.expected_observation_token.clone(),
|
||||
request_id: restore_request.request_id.clone(),
|
||||
preparation: None,
|
||||
};
|
||||
let before_lookup = runtime.worker_detail(&created.worker.worker_ref).unwrap();
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&format!("/v1/workers/{}/restore", created.worker.worker_id),
|
||||
&coordinate_uri,
|
||||
token,
|
||||
&coordination,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let missing_owner: runtime_api::WorkerRestoreCoordinationResponse =
|
||||
read_json(response).await;
|
||||
assert!(missing_owner.result.is_none() && missing_owner.preparation.is_none());
|
||||
assert_eq!(
|
||||
runtime.worker_detail(&created.worker.worker_ref).unwrap(),
|
||||
before_lookup
|
||||
);
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&coordinate_uri,
|
||||
"wrong-token",
|
||||
&coordination,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
coordination.preparation = Some(runtime_api::WorkerRestorePreparation::default());
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&coordinate_uri,
|
||||
token,
|
||||
&coordination,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let admitted: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await;
|
||||
assert!(admitted.result.is_none() && admitted.preparation.is_some());
|
||||
assert_eq!(
|
||||
runtime
|
||||
.worker_detail(&created.worker.worker_ref)
|
||||
.unwrap()
|
||||
.status,
|
||||
crate::catalog::WorkerStatus::Stopped
|
||||
);
|
||||
restore_request.preparation = admitted.preparation;
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&restore_uri,
|
||||
token,
|
||||
&restore_request,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let restored: runtime_api::WorkerRestoreResponse = read_json(response).await;
|
||||
assert_eq!(restored.state, WorkerRestoreState::Accepted);
|
||||
assert_eq!(
|
||||
restored.worker.unwrap().status,
|
||||
restored.worker.as_ref().unwrap().status,
|
||||
runtime_api::WorkerStatus::Idle
|
||||
);
|
||||
|
||||
@@ -3810,6 +3893,47 @@ mod tests {
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
coordination.preparation = None;
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&coordinate_uri,
|
||||
token,
|
||||
&coordination,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let recovered: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await;
|
||||
assert!(recovered.preparation.is_none());
|
||||
assert_eq!(recovered.result, Some(restored.clone()));
|
||||
|
||||
restore_request.preparation = Some(serde_json::from_value(serde_json::json!({
|
||||
"workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" },
|
||||
"workdir_attachments": [],
|
||||
"repository_access": []
|
||||
})).unwrap());
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&restore_uri,
|
||||
token,
|
||||
&restore_request,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let replayed: runtime_api::WorkerRestoreResponse = read_json(response).await;
|
||||
assert_eq!(
|
||||
serde_json::to_value(replayed).unwrap(),
|
||||
serde_json::to_value(restored).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.worker_detail(&created.worker.worker_ref)
|
||||
.unwrap()
|
||||
.status,
|
||||
crate::catalog::WorkerStatus::Stopped
|
||||
);
|
||||
|
||||
let response = authed_empty_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
@@ -3892,11 +4016,12 @@ mod tests {
|
||||
assert_eq!(listed.workers[0].status, crate::catalog::WorkerStatus::Idle);
|
||||
assert!(listed.workers[0].worker_state.is_none());
|
||||
|
||||
let response = authed_empty_request(
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&format!("{worker_uri}/restore"),
|
||||
token,
|
||||
&runtime.test_restore_request(&created.worker_ref),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
@@ -4864,7 +4989,9 @@ mod ws_tests {
|
||||
let (mut old_stream, _) = connect_async(authed_ws_request(&url)).await.unwrap();
|
||||
let _ = next_frame(&mut old_stream).await;
|
||||
runtime.stop_worker(&worker_ref, None).unwrap();
|
||||
runtime.restore_worker(&worker_ref).unwrap();
|
||||
runtime
|
||||
.restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref))
|
||||
.unwrap();
|
||||
backend.publish(
|
||||
&worker_ref,
|
||||
protocol::Event::TextDelta {
|
||||
@@ -4946,7 +5073,9 @@ mod ws_tests {
|
||||
);
|
||||
backend.publish(&worker_ref, protocol::Event::Shutdown);
|
||||
runtime.stop_worker(&worker_ref, None).unwrap();
|
||||
runtime.restore_worker(&worker_ref).unwrap();
|
||||
runtime
|
||||
.restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref))
|
||||
.unwrap();
|
||||
let resume_url = format!("{url}?cursor={}", old.cursor);
|
||||
let (mut stream, _) = connect_async(authed_ws_request(&resume_url)).await.unwrap();
|
||||
assert!(matches!(
|
||||
|
||||
@@ -462,21 +462,74 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
Ok(runtime_api::WorkerLifecycleResponse { ack })
|
||||
}
|
||||
|
||||
async fn coordinate_worker_restore(
|
||||
&self,
|
||||
worker_id: String,
|
||||
request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||
) -> Result<runtime_api::WorkerRestoreCoordinationResponse, runtime_api::RuntimeApiError> {
|
||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||
let scope =
|
||||
auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?;
|
||||
let runtime = self.state.runtime.clone();
|
||||
let (result, preparation) = tokio::task::spawn_blocking(move || {
|
||||
runtime.coordinate_worker_restore_operation(&worker_ref, request, scope.as_ref())
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
runtime_api::RuntimeApiError::new(500, "runtime_restore_task_failed", error.to_string())
|
||||
})?
|
||||
.map_err(RuntimeHttpRestError::runtime)
|
||||
.map_err(api_error)?;
|
||||
let result = result
|
||||
.map(|result| -> Result<_, runtime_api::RuntimeApiError> {
|
||||
Ok(runtime_api::WorkerRestoreResponse {
|
||||
state: match result.state {
|
||||
server_api::WorkerRestoreState::Accepted => {
|
||||
runtime_api::WorkerRestoreState::Accepted
|
||||
}
|
||||
server_api::WorkerRestoreState::Rejected => {
|
||||
runtime_api::WorkerRestoreState::Rejected
|
||||
}
|
||||
server_api::WorkerRestoreState::RolledBack => {
|
||||
runtime_api::WorkerRestoreState::RolledBack
|
||||
}
|
||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||
}
|
||||
},
|
||||
worker: result.worker.map(response).transpose()?,
|
||||
reason_code: result.reason_code,
|
||||
message: result.message,
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
Ok(runtime_api::WorkerRestoreCoordinationResponse {
|
||||
result,
|
||||
preparation,
|
||||
})
|
||||
}
|
||||
|
||||
async fn restore_worker(
|
||||
&self,
|
||||
worker_id: String,
|
||||
_request: runtime_api::EmptyObjectRequest,
|
||||
request: runtime_api::WorkerRestoreRequest,
|
||||
) -> Result<runtime_api::WorkerRestoreResponse, runtime_api::RuntimeApiError> {
|
||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||
let result = match auth_workspace_scope(&self.state, auth_extension().as_ref())
|
||||
.map_err(api_error)?
|
||||
{
|
||||
Some(scope) => self
|
||||
.state
|
||||
.runtime
|
||||
.restore_worker_operation_scoped(&scope, &worker_ref),
|
||||
None => self.state.runtime.restore_worker_operation(&worker_ref),
|
||||
}
|
||||
let scope =
|
||||
auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?;
|
||||
let runtime = self.state.runtime.clone();
|
||||
let result = tokio::task::spawn_blocking(move || match scope {
|
||||
Some(scope) => runtime.restore_worker_operation_scoped(&scope, &worker_ref, request),
|
||||
None => runtime.restore_worker_operation(&worker_ref, request),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
runtime_api::RuntimeApiError::new(
|
||||
StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
|
||||
"runtime_restore_task_failed",
|
||||
error.to_string(),
|
||||
)
|
||||
})?
|
||||
.map_err(RuntimeHttpRestError::runtime)
|
||||
.map_err(api_error)?;
|
||||
Ok(runtime_api::WorkerRestoreResponse {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7394,7 +7394,14 @@ mod tests {
|
||||
.create_worker(create_request("restart cleanup"))
|
||||
.unwrap();
|
||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||
assert!(runtime.restore_worker(&worker.worker_ref).is_err());
|
||||
assert!(
|
||||
runtime
|
||||
.restore_worker(
|
||||
&worker.worker_ref,
|
||||
runtime.test_restore_request(&worker.worker_ref)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(runtime.stop_worker(&worker.worker_ref, None).is_err());
|
||||
let prior_restore_id = restore_id.lock().unwrap().unwrap();
|
||||
let aggregate = runtime_store
|
||||
@@ -8904,7 +8911,12 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(detached.workdir_attachments.is_empty());
|
||||
runtime.stop_worker(&detail.worker_ref, None).unwrap();
|
||||
let restored_without_workdir = runtime.restore_worker(&detail.worker_ref).unwrap();
|
||||
let restored_without_workdir = runtime
|
||||
.restore_worker(
|
||||
&detail.worker_ref,
|
||||
runtime.test_restore_request(&detail.worker_ref),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(restored_without_workdir.workdir_attachments.is_empty());
|
||||
assert!(
|
||||
!repo.path().join(".yoi").exists(),
|
||||
@@ -9191,7 +9203,12 @@ mod tests {
|
||||
.contains_key(&detail.worker_ref)
|
||||
);
|
||||
|
||||
runtime.restore_worker(&detail.worker_ref).unwrap();
|
||||
runtime
|
||||
.restore_worker(
|
||||
&detail.worker_ref,
|
||||
runtime.test_restore_request(&detail.worker_ref),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
runtime.worker_detail(&detail.worker_ref).unwrap().status,
|
||||
crate::catalog::WorkerStatus::Idle
|
||||
|
||||
@@ -555,9 +555,15 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
|
||||
WorkspaceServerOperation::WorkerControlRestore {
|
||||
runtime_id,
|
||||
worker_id,
|
||||
request,
|
||||
} => generated_workspace_response(
|
||||
client
|
||||
.worker_control_restore(workspace_id, runtime_id, worker_id)
|
||||
.worker_control_restore(
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
worker_id,
|
||||
request,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
WorkspaceServerOperation::WorkerObservationSessions => {
|
||||
|
||||
@@ -77,8 +77,9 @@ pub trait WorkerControlService: Send + Sync {
|
||||
&self,
|
||||
runtime_id: String,
|
||||
worker_id: String,
|
||||
request: server_api::WorkerRestoreRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
let _ = (runtime_id, worker_id);
|
||||
let _ = (runtime_id, worker_id, request);
|
||||
Err(WorkspaceClientError::Unavailable(
|
||||
"Runtime Worker restore is unavailable".to_string(),
|
||||
))
|
||||
@@ -347,6 +348,7 @@ impl WorkerControlService for WorkspaceWorkerControlService {
|
||||
&self,
|
||||
runtime_id: String,
|
||||
worker_id: String,
|
||||
request: server_api::WorkerRestoreRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
if !self.runtime_worker_control {
|
||||
return Err(WorkspaceClientError::Unavailable(
|
||||
@@ -357,6 +359,7 @@ impl WorkerControlService for WorkspaceWorkerControlService {
|
||||
crate::worker::WorkspaceServerOperation::WorkerControlRestore {
|
||||
runtime_id,
|
||||
worker_id,
|
||||
request,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -750,8 +753,13 @@ enum WorkerSubjectInput {
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct WorkerTargetInput {
|
||||
struct WorkerRestoreInput {
|
||||
subject: WorkerSubjectInput,
|
||||
expected_observation_token: String,
|
||||
/// Reuse the reported ID for explicit unknown-outcome recovery. A new intent
|
||||
/// defaults to the durable Tool call identity.
|
||||
#[serde(default)]
|
||||
request_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
@@ -864,7 +872,7 @@ impl WorkerOperation {
|
||||
Self::Cancel => "Cancel the current turn of a known Runtime Worker when allowed.",
|
||||
Self::Stop => "Stop a known Runtime Worker when allowed.",
|
||||
Self::Restore => {
|
||||
"Restore a stopped Backend/Runtime Worker session in the current Workspace."
|
||||
"Restore a stopped Backend/Runtime Worker session in the current Workspace. Supply expected_observation_token from WorkerList; a conflict requires refreshing and deliberate new intent. New requests default to the durable Tool call ID. For explicit recovery of an unknown/pending outcome, supply the reported request_id and the original token unchanged; never create a new intent or automatically retry."
|
||||
}
|
||||
Self::Remove => {
|
||||
"Remove an eligible stopped, unassigned, non-internal Worker. Supply a bounded reason; Backend validation and retention are authoritative."
|
||||
@@ -981,12 +989,41 @@ impl Tool for WorkspaceWorkerTool {
|
||||
.map_err(control_tool_error)?
|
||||
}
|
||||
WorkerOperation::Restore => {
|
||||
let input = parse::<WorkerTargetInput>(input_json, "WorkerRestore")?;
|
||||
let input = parse::<WorkerRestoreInput>(input_json, "WorkerRestore")?;
|
||||
let (runtime_id, worker_id) = runtime_subject_ids(&input.subject, self.operation)?;
|
||||
self.control
|
||||
.restore_worker(runtime_id, worker_id)
|
||||
let request_id = match input.request_id {
|
||||
Some(id) => non_empty(id, "request_id")?,
|
||||
None => format!(
|
||||
"worker-restore:{}",
|
||||
non_empty(ctx.call_id.clone(), "tool call_id")?
|
||||
),
|
||||
};
|
||||
if request_id.len() > 512 {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"request_id must contain at most 512 bytes".into(),
|
||||
));
|
||||
}
|
||||
let recoverable_error = |error| {
|
||||
ToolError::ExecutionFailed(format!(
|
||||
"WorkerRestore request_id={request_id}; reuse this ID with the original observation token only for explicit unknown/pending outcome recovery, not after a stale-intent conflict: {error}"
|
||||
))
|
||||
};
|
||||
let response = self
|
||||
.control
|
||||
.restore_worker(
|
||||
runtime_id,
|
||||
worker_id,
|
||||
server_api::WorkerRestoreRequest {
|
||||
expected_observation_token: non_empty(
|
||||
input.expected_observation_token,
|
||||
"expected_observation_token",
|
||||
)?,
|
||||
request_id: request_id.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(control_tool_error)?
|
||||
.map_err(|error| recoverable_error(control_tool_error(error)))?;
|
||||
return tool_output(self.operation, response).map_err(recoverable_error);
|
||||
}
|
||||
WorkerOperation::Remove => {
|
||||
let input = parse::<WorkerRemoveInput>(input_json, "WorkerRemove")?;
|
||||
@@ -1058,6 +1095,18 @@ fn tool_output(
|
||||
response.status, response.body
|
||||
)));
|
||||
}
|
||||
if operation == WorkerOperation::Restore {
|
||||
let restored: server_api::WorkerRestoreResponse = serde_json::from_str(&response.body)
|
||||
.map_err(|error| {
|
||||
ToolError::ExecutionFailed(format!("invalid WorkerRestore response: {error}"))
|
||||
})?;
|
||||
if restored.result.state != server_api::WorkerRestoreState::Accepted {
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"WorkerRestore returned {:?}: {}",
|
||||
restored.result.state, response.body
|
||||
)));
|
||||
}
|
||||
}
|
||||
if operation == WorkerOperation::Stop
|
||||
&& let Ok(summary) = serde_json::from_str::<SubWorkerStopSummary>(&response.body)
|
||||
{
|
||||
@@ -1150,7 +1199,7 @@ fn worker_tool_contribution(
|
||||
}
|
||||
}
|
||||
WorkerOperation::Restore => {
|
||||
definition::<WorkerTargetInput>(operation, control, runtime_worker_control)
|
||||
definition::<WorkerRestoreInput>(operation, control, runtime_worker_control)
|
||||
}
|
||||
WorkerOperation::Remove => {
|
||||
definition::<WorkerRemoveInput>(operation, control, runtime_worker_control)
|
||||
@@ -1410,6 +1459,95 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_restore_forwards_exact_observation_and_request_identity() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::default());
|
||||
let control = test_control(client.clone());
|
||||
for _ in 0..2 {
|
||||
control
|
||||
.restore_worker(
|
||||
"runtime/a".into(),
|
||||
"worker/b".into(),
|
||||
server_api::WorkerRestoreRequest {
|
||||
expected_observation_token: "same-observation".into(),
|
||||
request_id: "same-request".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(
|
||||
requests[0].path,
|
||||
"/api/w/workspace%2Ftest/worker-control/workers/runtime%2Fa/worker%2Fb/restore"
|
||||
);
|
||||
assert_eq!(requests[0].body, requests[1].body);
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
body,
|
||||
serde_json::json!({"expected_observation_token":"same-observation","request_id":"same-request"})
|
||||
);
|
||||
assert!(serde_json::from_str::<WorkerRestoreInput>(r#"{"subject":{"kind":"runtime_worker","runtime_id":"runtime-a","worker_id":"worker-a"}}"#).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_restore_reports_retry_identity_and_reuses_it_across_explicit_tool_calls() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::default());
|
||||
let tool = WorkspaceWorkerTool {
|
||||
operation: WorkerOperation::Restore,
|
||||
control: test_control(client.clone()),
|
||||
runtime_worker_control: true,
|
||||
};
|
||||
let mut input = serde_json::json!({
|
||||
"subject": {"kind":"runtime_worker", "runtime_id":"runtime-a", "worker_id":"worker-b"},
|
||||
"expected_observation_token":"original-observation"
|
||||
});
|
||||
// The fixture deliberately returns malformed JSON shape: delivery may
|
||||
// have succeeded, so retain the exact tuple rather than issuing new intent.
|
||||
let first = tool
|
||||
.execute(
|
||||
&input.to_string(),
|
||||
ToolExecutionContext::new("original-call", "batch", 0),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
first
|
||||
.to_string()
|
||||
.contains("request_id=worker-restore:original-call")
|
||||
);
|
||||
input["request_id"] = "worker-restore:original-call".into();
|
||||
let second = tool
|
||||
.execute(
|
||||
&input.to_string(),
|
||||
ToolExecutionContext::new("explicit-recovery-call", "batch", 1),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
second
|
||||
.to_string()
|
||||
.contains("request_id=worker-restore:original-call")
|
||||
);
|
||||
{
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(requests[0].body, requests[1].body);
|
||||
}
|
||||
input["request_id"] = "x".repeat(513).into();
|
||||
assert!(
|
||||
tool.execute(
|
||||
&input.to_string(),
|
||||
ToolExecutionContext::new("invalid", "batch", 2)
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(client.requests.lock().unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_spawn_forwards_typed_initial_submit_to_workspace_api() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::default());
|
||||
|
||||
@@ -728,6 +728,7 @@ pub enum WorkspaceServerOperation {
|
||||
WorkerControlRestore {
|
||||
runtime_id: String,
|
||||
worker_id: String,
|
||||
request: server_api::WorkerRestoreRequest,
|
||||
},
|
||||
WorkerObservationSessions,
|
||||
WorkerObservationCapture(server_api::WorkerObservationSubjectRef),
|
||||
@@ -802,11 +803,15 @@ fn workspace_server_operation_request(
|
||||
WorkspaceServerOperation::WorkerControlRestore {
|
||||
runtime_id,
|
||||
worker_id,
|
||||
} => Ok(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
format!("{base}/worker-control/workers/{runtime_id}/{worker_id}/restore"),
|
||||
"".to_string(),
|
||||
)),
|
||||
request,
|
||||
} => workspace_server_json_request(
|
||||
format!(
|
||||
"{base}/worker-control/workers/{}/{}/restore",
|
||||
encode_workspace_path_segment(&runtime_id),
|
||||
encode_workspace_path_segment(&worker_id)
|
||||
),
|
||||
&request,
|
||||
),
|
||||
WorkspaceServerOperation::WorkerObservationSessions => Ok(WorkspaceRequest::get(format!(
|
||||
"{base}/worker-observation/sessions"
|
||||
))),
|
||||
|
||||
@@ -242,6 +242,8 @@ pub struct InternalWorkerImplementationSummary {
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct InternalWorkerSummary {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub restore_observation_token: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub worker: RuntimeWorkerRef,
|
||||
pub host_id: String,
|
||||
@@ -356,6 +358,7 @@ pub(crate) fn workspace_worker_summary(
|
||||
workdir_attachments: Vec<server_api::WorkerWorkdirAttachmentSummary>,
|
||||
) -> server_api::WorkerSummary {
|
||||
server_api::WorkerSummary {
|
||||
restore_observation_token: summary.restore_observation_token,
|
||||
runtime_id: summary.worker.runtime_id,
|
||||
worker_id: summary.worker.worker_id,
|
||||
resource_key,
|
||||
@@ -834,11 +837,17 @@ impl RuntimeRegistryError {
|
||||
runtime_id,
|
||||
code,
|
||||
message,
|
||||
} => Error::RuntimeOperationFailed {
|
||||
runtime_id,
|
||||
code,
|
||||
message,
|
||||
},
|
||||
} => {
|
||||
if code == "restore_observation_conflict" {
|
||||
Error::RestoreObservationConflict
|
||||
} else {
|
||||
Error::RuntimeOperationFailed {
|
||||
runtime_id,
|
||||
code,
|
||||
message,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -938,7 +947,29 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
|
||||
fn worker(&self, worker_id: &str) -> WorkerLookupResult;
|
||||
|
||||
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
|
||||
fn coordinate_worker_restore(
|
||||
&self,
|
||||
_worker_id: &str,
|
||||
_request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||
) -> Result<
|
||||
(
|
||||
Option<InternalWorkerRestoreResult>,
|
||||
Option<runtime_api::WorkerRestorePreparation>,
|
||||
),
|
||||
RuntimeDiagnostic,
|
||||
> {
|
||||
Err(diagnostic(
|
||||
"worker_restore_coordination_unsupported",
|
||||
HostDiagnosticSeverity::Error,
|
||||
"Runtime does not support guarded Restore coordination",
|
||||
))
|
||||
}
|
||||
|
||||
fn restore_worker(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
_request: runtime_api::WorkerRestoreRequest,
|
||||
) -> InternalWorkerRestoreResult {
|
||||
InternalWorkerRestoreResult {
|
||||
state: server_api::WorkerRestoreState::Rejected,
|
||||
worker: None,
|
||||
@@ -1602,16 +1633,39 @@ impl RuntimeRegistry {
|
||||
Ok(worker)
|
||||
}
|
||||
|
||||
pub fn coordinate_worker_restore(
|
||||
&self,
|
||||
worker: &RuntimeWorkerRef,
|
||||
request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||
) -> Result<
|
||||
(
|
||||
Option<InternalWorkerRestoreResult>,
|
||||
Option<runtime_api::WorkerRestorePreparation>,
|
||||
),
|
||||
RuntimeRegistryError,
|
||||
> {
|
||||
validate_backend_identifier("runtime_id", &worker.runtime_id)?;
|
||||
validate_backend_identifier("worker_id", &worker.worker_id)?;
|
||||
self.runtime(&worker.runtime_id)?
|
||||
.coordinate_worker_restore(&worker.worker_id, request)
|
||||
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: worker.runtime_id.clone(),
|
||||
code: error.code,
|
||||
message: error.message,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn restore_worker(
|
||||
&self,
|
||||
worker: &RuntimeWorkerRef,
|
||||
request: runtime_api::WorkerRestoreRequest,
|
||||
) -> Result<InternalWorkerRestoreResult, RuntimeRegistryError> {
|
||||
let runtime_id = worker.runtime_id.as_str();
|
||||
let worker_id = worker.worker_id.as_str();
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
validate_backend_identifier("worker_id", worker_id)?;
|
||||
let runtime = self.runtime(runtime_id)?;
|
||||
Ok(runtime.restore_worker(worker_id))
|
||||
Ok(runtime.restore_worker(worker_id, request))
|
||||
}
|
||||
|
||||
pub fn replace_worker_workspace_api(
|
||||
@@ -2372,6 +2426,7 @@ impl EmbeddedWorkerRuntime {
|
||||
true,
|
||||
);
|
||||
InternalWorkerSummary {
|
||||
restore_observation_token: summary.restore_observation_token.clone(),
|
||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||
host_id: self.host_id.clone(),
|
||||
display_name: display.display_name.clone(),
|
||||
@@ -2416,6 +2471,7 @@ impl EmbeddedWorkerRuntime {
|
||||
true,
|
||||
);
|
||||
InternalWorkerSummary {
|
||||
restore_observation_token: detail.restore_observation_token.clone(),
|
||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||
host_id: self.host_id.clone(),
|
||||
display_name: display.display_name.clone(),
|
||||
@@ -2582,7 +2638,48 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
|
||||
fn coordinate_worker_restore(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||
) -> Result<
|
||||
(
|
||||
Option<InternalWorkerRestoreResult>,
|
||||
Option<runtime_api::WorkerRestorePreparation>,
|
||||
),
|
||||
RuntimeDiagnostic,
|
||||
> {
|
||||
let worker_ref = self.worker_ref(worker_id).ok_or_else(|| {
|
||||
diagnostic(
|
||||
"worker_not_found",
|
||||
HostDiagnosticSeverity::Warning,
|
||||
"Unknown Worker",
|
||||
)
|
||||
})?;
|
||||
let (result, preparation) = self
|
||||
.runtime
|
||||
.coordinate_worker_restore_operation(&worker_ref, request, None)
|
||||
.map_err(|error| embedded_runtime_diagnostic(&error))?;
|
||||
Ok((
|
||||
result.map(|result| InternalWorkerRestoreResult {
|
||||
state: result.state,
|
||||
worker: result.worker.map(|detail| self.map_worker_detail(detail)),
|
||||
diagnostics: match (result.reason_code, result.message) {
|
||||
(Some(code), Some(message)) => {
|
||||
vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)]
|
||||
}
|
||||
_ => Vec::new(),
|
||||
},
|
||||
}),
|
||||
preparation,
|
||||
))
|
||||
}
|
||||
|
||||
fn restore_worker(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
request: runtime_api::WorkerRestoreRequest,
|
||||
) -> InternalWorkerRestoreResult {
|
||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||
return InternalWorkerRestoreResult {
|
||||
state: server_api::WorkerRestoreState::Rejected,
|
||||
@@ -2594,7 +2691,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
)],
|
||||
};
|
||||
};
|
||||
match self.runtime.restore_worker_operation(&worker_ref) {
|
||||
match self.runtime.restore_worker_operation(&worker_ref, request) {
|
||||
Ok(result) => {
|
||||
let diagnostics = match (result.reason_code, result.message) {
|
||||
(Some(code), Some(message)) => {
|
||||
@@ -2608,6 +2705,17 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
diagnostics,
|
||||
}
|
||||
}
|
||||
Err(worker_runtime::error::RuntimeError::RestoreObservationConflict { .. }) => {
|
||||
InternalWorkerRestoreResult {
|
||||
state: server_api::WorkerRestoreState::Rejected,
|
||||
worker: None,
|
||||
diagnostics: vec![diagnostic(
|
||||
"restore_observation_conflict",
|
||||
HostDiagnosticSeverity::Warning,
|
||||
"Restore observation changed; refresh before a new intent".to_string(),
|
||||
)],
|
||||
}
|
||||
}
|
||||
Err(err) => InternalWorkerRestoreResult {
|
||||
state: server_api::WorkerRestoreState::ReconciliationRequired,
|
||||
worker: None,
|
||||
@@ -4375,6 +4483,7 @@ impl RemoteWorkerRuntime {
|
||||
false,
|
||||
);
|
||||
InternalWorkerSummary {
|
||||
restore_observation_token: summary.restore_observation_token.clone(),
|
||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||
host_id: self.host_id.clone(),
|
||||
display_name: display.display_name.clone(),
|
||||
@@ -4417,6 +4526,7 @@ impl RemoteWorkerRuntime {
|
||||
false,
|
||||
);
|
||||
InternalWorkerSummary {
|
||||
restore_observation_token: detail.restore_observation_token.clone(),
|
||||
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||
host_id: self.host_id.clone(),
|
||||
display_name: display.display_name.clone(),
|
||||
@@ -4708,17 +4818,58 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
|
||||
fn coordinate_worker_restore(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
request: runtime_api::WorkerRestoreCoordinationRequest,
|
||||
) -> Result<
|
||||
(
|
||||
Option<InternalWorkerRestoreResult>,
|
||||
Option<runtime_api::WorkerRestorePreparation>,
|
||||
),
|
||||
RuntimeDiagnostic,
|
||||
> {
|
||||
let worker_id = worker_id.to_string();
|
||||
let response = self.run_runtime_api(
|
||||
self.request_timeout,
|
||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||
move |client| async move { client.coordinate_worker_restore(worker_id, request).await },
|
||||
)?;
|
||||
let result = response
|
||||
.result
|
||||
.map(|result| -> Result<_, RuntimeDiagnostic> {
|
||||
let worker = result
|
||||
.worker
|
||||
.map(runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>)
|
||||
.transpose()?;
|
||||
let state =
|
||||
runtime_contract_convert::<_, server_api::WorkerRestoreState>(result.state)?;
|
||||
Ok(InternalWorkerRestoreResult {
|
||||
state,
|
||||
worker: worker.map(|detail| self.map_worker_detail(detail)),
|
||||
diagnostics: match (result.reason_code, result.message) {
|
||||
(Some(code), Some(message)) => {
|
||||
vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)]
|
||||
}
|
||||
_ => Vec::new(),
|
||||
},
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
Ok((result, response.preparation))
|
||||
}
|
||||
|
||||
fn restore_worker(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
request: runtime_api::WorkerRestoreRequest,
|
||||
) -> InternalWorkerRestoreResult {
|
||||
let worker_id_owned = worker_id.to_string();
|
||||
match self
|
||||
.run_runtime_api(
|
||||
self.request_timeout,
|
||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||
move |client| async move {
|
||||
client
|
||||
.restore_worker(worker_id_owned, runtime_api::EmptyObjectRequest::default())
|
||||
.await
|
||||
},
|
||||
move |client| async move { client.restore_worker(worker_id_owned, request).await },
|
||||
)
|
||||
.and_then(|value| {
|
||||
let worker = value
|
||||
@@ -5886,6 +6037,11 @@ fn sanitize_embedded_execution_detail(summary: &str, message: &str) -> String {
|
||||
|
||||
fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnostic {
|
||||
match error {
|
||||
EmbeddedRuntimeError::RestoreObservationConflict { .. } => diagnostic(
|
||||
"restore_observation_conflict",
|
||||
HostDiagnosticSeverity::Warning,
|
||||
"Restore observation changed; refresh before a new intent".to_string(),
|
||||
),
|
||||
EmbeddedRuntimeError::RuntimeStopped => diagnostic(
|
||||
"embedded_runtime_stopped",
|
||||
HostDiagnosticSeverity::Warning,
|
||||
@@ -6292,6 +6448,7 @@ fn worker_spawn_intent_label(intent: &WorkerSpawnIntent) -> &'static str {
|
||||
pub fn placeholder_worker(host_id: impl Into<String>) -> InternalWorkerSummary {
|
||||
let host_id = host_id.into();
|
||||
InternalWorkerSummary {
|
||||
restore_observation_token: None,
|
||||
worker: RuntimeWorkerRef::new("placeholder", "worker-placeholder"),
|
||||
host_id,
|
||||
display_name: "Worker runtime actions are not implemented".to_string(),
|
||||
@@ -6369,7 +6526,14 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = provider.restore_worker("worker-transport-uncertain");
|
||||
let result = provider.restore_worker(
|
||||
"worker-transport-uncertain",
|
||||
runtime_api::WorkerRestoreRequest {
|
||||
expected_observation_token: "observation".into(),
|
||||
request_id: "transport-uncertain".into(),
|
||||
preparation: None,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
result.state,
|
||||
@@ -7052,6 +7216,7 @@ mod tests {
|
||||
runtime_id: runtime_id.to_string(),
|
||||
host_id: host_id.to_string(),
|
||||
workers: vec![InternalWorkerSummary {
|
||||
restore_observation_token: None,
|
||||
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
|
||||
host_id: host_id.to_string(),
|
||||
display_name: label.to_string(),
|
||||
@@ -7698,7 +7863,16 @@ mod tests {
|
||||
|
||||
let detail = registry.worker(&worker.worker).unwrap();
|
||||
|
||||
let json = serde_json::to_string(&(embedded_summary, worker, input, detail)).unwrap();
|
||||
// The public observation generation is intentionally exposed, unlike
|
||||
// private authentication tokens. Whitelist its key, not arbitrary token
|
||||
// values or other token-bearing fields.
|
||||
for summary in [&worker, &detail] {
|
||||
let token = summary.restore_observation_token.as_ref().unwrap();
|
||||
assert_eq!(uuid::Uuid::parse_str(token).unwrap().get_version_num(), 7);
|
||||
}
|
||||
let json = serde_json::to_string(&(embedded_summary, worker, input, detail))
|
||||
.unwrap()
|
||||
.replace("\"restore_observation_token\":", "\"restore_observation\":");
|
||||
for forbidden in [
|
||||
"/workspace/project",
|
||||
"metadata.json",
|
||||
|
||||
@@ -139,6 +139,10 @@ pub enum Error {
|
||||
RuntimeBindingFingerprintConflict { fingerprint: String },
|
||||
#[error("Runtime binding was not found for {runtime_id}")]
|
||||
RuntimeBindingNotFound { runtime_id: String },
|
||||
#[error(
|
||||
"restore_observation_conflict: Restore observation changed; refresh before a new intent"
|
||||
)]
|
||||
RestoreObservationConflict,
|
||||
#[error("Repository conflict: {0}")]
|
||||
RepositoryConflict(String),
|
||||
#[error("Registry inconsistency: {0}")]
|
||||
|
||||
@@ -61,6 +61,16 @@ impl Drop for StoredResource {
|
||||
}
|
||||
|
||||
impl BackendResourceBroker {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn repository_ssh_access_handle_count(&self) -> usize {
|
||||
self.resources
|
||||
.lock()
|
||||
.unwrap()
|
||||
.values()
|
||||
.filter(|resource| resource.handle.kind == BackendResourceKind::RepositorySshAccess)
|
||||
.count()
|
||||
}
|
||||
|
||||
pub fn issue_profile_source_archive_handle(
|
||||
&self,
|
||||
workspace_id: impl Into<String>,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -250,7 +250,13 @@ async fn offline_handle_can_pin_but_deleted_handle_cannot_mutate_or_connect() {
|
||||
);
|
||||
assert!(matches!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend)
|
||||
.restore(
|
||||
&WorkerOperationContext::Backend,
|
||||
server_api::WorkerRestoreRequest {
|
||||
expected_observation_token: "stale".into(),
|
||||
request_id: "removed-handle".into(),
|
||||
}
|
||||
)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.error,
|
||||
@@ -374,7 +380,17 @@ async fn worker_control_handle_rechecks_revoked_grants_for_each_operation() {
|
||||
);
|
||||
}
|
||||
assert!(matches!(
|
||||
handle.restore(&context).await.unwrap_err().error,
|
||||
handle
|
||||
.restore(
|
||||
&context,
|
||||
server_api::WorkerRestoreRequest {
|
||||
expected_observation_token: "stale".into(),
|
||||
request_id: "revoked-grant".into(),
|
||||
}
|
||||
)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.error,
|
||||
Error::UnknownWorker { .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
@@ -682,7 +698,15 @@ async fn generic_management_aliases_cannot_bypass_worker_grants_or_target_bound_
|
||||
);
|
||||
assert!(
|
||||
service
|
||||
.runtime_worker_restore_alias(context(), "offline-runtime".into(), "subject".into())
|
||||
.runtime_worker_restore_alias(
|
||||
context(),
|
||||
"offline-runtime".into(),
|
||||
"subject".into(),
|
||||
server_api::WorkerRestoreRequest {
|
||||
expected_observation_token: "old".into(),
|
||||
request_id: "unauthorized".into(),
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
@@ -862,3 +886,696 @@ async fn browser_protocol_connection_and_cleanup_reject_cookie_origin_mismatch()
|
||||
assert!(matches!(result.error, Error::WorkspacePermissionDenied(_)));
|
||||
assert!(execution.protocol_methods().is_empty());
|
||||
}
|
||||
|
||||
fn restore_request_for(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
request_id: &str,
|
||||
) -> server_api::WorkerRestoreRequest {
|
||||
server_api::WorkerRestoreRequest {
|
||||
expected_observation_token: api
|
||||
.runtime
|
||||
.worker(worker)
|
||||
.unwrap()
|
||||
.restore_observation_token
|
||||
.unwrap(),
|
||||
request_id: request_id.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_guard_rejects_stale_list_and_stopped_restored_stopped_aba() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||
execution.accept_restores();
|
||||
let identity = spawn_ticket_check_source(&api, "restore-guard-aba");
|
||||
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||
handle
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
let original = restore_request_for(&api, &identity, "restore-original");
|
||||
let stale_list = server_api::WorkerRestoreRequest {
|
||||
request_id: "restore-stale-list".into(),
|
||||
..original.clone()
|
||||
};
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, original.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
let conflict = handle
|
||||
.restore(&WorkerOperationContext::Backend, stale_list.clone())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(conflict.error, Error::RestoreObservationConflict));
|
||||
let typed = conflict.into_repository_api_error();
|
||||
assert_eq!(
|
||||
api_error_status(&Error::RestoreObservationConflict),
|
||||
StatusCode::CONFLICT
|
||||
);
|
||||
assert_eq!(typed.error, "restore_observation_conflict");
|
||||
handle
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_ne!(
|
||||
original.expected_observation_token,
|
||||
api.runtime
|
||||
.worker(&identity)
|
||||
.unwrap()
|
||||
.restore_observation_token
|
||||
.unwrap()
|
||||
);
|
||||
assert!(matches!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, stale_list)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.error,
|
||||
Error::RestoreObservationConflict
|
||||
));
|
||||
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
|
||||
assert!(
|
||||
execution.contexts.lock().unwrap().is_empty(),
|
||||
"conflict must not install execution"
|
||||
);
|
||||
// The original request is result replay, not a new intent; it stays accepted
|
||||
// without resurrecting the Worker after a subsequent stop.
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, original)
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
|
||||
assert!(execution.contexts.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
fn restore_test_workdir(
|
||||
api: &WorkspaceApi,
|
||||
id: &str,
|
||||
source: WorkdirRegistrySource,
|
||||
) -> WorkdirRegistryRecord {
|
||||
WorkdirRegistryRecord {
|
||||
workspace_id: api.workspace_id().into(),
|
||||
workdir_id: id.into(),
|
||||
display_name: None,
|
||||
source,
|
||||
creation_selector: None,
|
||||
creation_ref: None,
|
||||
creation_tree: None,
|
||||
current_selector: None,
|
||||
current_ref: None,
|
||||
current_tree: None,
|
||||
observed_at_epoch_seconds: None,
|
||||
materialization_status: "present".into(),
|
||||
cleanliness: "clean".into(),
|
||||
created_at: TEST_CREATED_AT.into(),
|
||||
updated_at: TEST_CREATED_AT.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn attach_restore_test_workdir(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
workdir: &WorkdirRegistryRecord,
|
||||
) {
|
||||
api.store
|
||||
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
|
||||
connection_id: String::new(),
|
||||
workspace_id: api.workspace_id().into(),
|
||||
worker: worker.clone(),
|
||||
workdir_id: workdir.workdir_id.clone(),
|
||||
alias: workdir.workdir_id.clone(),
|
||||
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||
linked_at: TEST_CREATED_AT.into(),
|
||||
unlinked_at: None,
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn displace_restore_singleton(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
replacement: &RuntimeWorkerRef,
|
||||
) {
|
||||
api.config_store.with_conn(|conn| {
|
||||
conn.execute("UPDATE worker_create_reservations SET singleton_key='restore-test-singleton', singleton_generation=1 WHERE workspace_id=?1 AND worker_id=?2",
|
||||
rusqlite::params![api.workspace_id(), worker.worker_id])?;
|
||||
conn.execute("INSERT INTO worker_singleton_owners(workspace_id,singleton_key,runtime_id,worker_id,generation,created_at,updated_at) VALUES(?1,'restore-test-singleton',?2,?3,2,?4,?4)",
|
||||
rusqlite::params![api.workspace_id(), replacement.runtime_id, replacement.worker_id, TEST_CREATED_AT])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert!(
|
||||
api.store
|
||||
.require_current_worker_singleton_owner(api.workspace_id(), worker)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
fn add_offline_restore_workdir(api: &WorkspaceApi, worker: &RuntimeWorkerRef) {
|
||||
let record = restore_test_workdir(
|
||||
api,
|
||||
"restore-offline",
|
||||
WorkdirRegistrySource::ExternalGrant {
|
||||
grant_id: "restore-offline-grant".into(),
|
||||
},
|
||||
);
|
||||
api.store
|
||||
.create_external_workdir_grant(
|
||||
&crate::store::ExternalWorkdirGrantRecord {
|
||||
grant_id: "restore-offline-grant".into(),
|
||||
workspace_id: api.workspace_id().into(),
|
||||
workdir_id: record.workdir_id.clone(),
|
||||
provider_instance_id: "offline-provider".into(),
|
||||
display_name: "Offline grant".into(),
|
||||
permissions: "read_only".into(),
|
||||
created_by: "owner-account".into(),
|
||||
created_at: TEST_CREATED_AT.into(),
|
||||
expires_at: None,
|
||||
generation: 1,
|
||||
status: "offline".into(),
|
||||
updated_at: TEST_CREATED_AT.into(),
|
||||
},
|
||||
&record,
|
||||
)
|
||||
.unwrap();
|
||||
attach_restore_test_workdir(api, worker, &record);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_guard_stale_ssh_intent_does_not_issue_workspace_resource() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let api = test_api_with_remote_repository(workspace.path()).await;
|
||||
let identity = spawn_ticket_check_source(&api, "restore-stale-ssh");
|
||||
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||
let stale = restore_request_for(&api, &identity, "stale-ssh-intent");
|
||||
handle
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
api.repository_secrets
|
||||
.generate_credential(
|
||||
api.workspace_id(),
|
||||
GenerateRepositorySshCredentialRequest {
|
||||
operation_id: "restore-default-key".into(),
|
||||
credential_id:
|
||||
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(),
|
||||
name: "Restore test default".into(),
|
||||
},
|
||||
"owner-account",
|
||||
)
|
||||
.unwrap();
|
||||
let public_key = api
|
||||
.repository_secrets
|
||||
.credential_public_key(
|
||||
api.workspace_id(),
|
||||
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.public_key;
|
||||
api.repository_secrets
|
||||
.put_host_trust(
|
||||
api.workspace_id(),
|
||||
PutRepositorySshHostTrustRequest {
|
||||
operation_id: "restore-ssh-trust".into(),
|
||||
host_trust_id: "restore-ssh-host".into(),
|
||||
hostname: "example.invalid".into(),
|
||||
port: 22,
|
||||
host_key: public_key,
|
||||
expected_revision: None,
|
||||
},
|
||||
"owner-account",
|
||||
)
|
||||
.unwrap();
|
||||
let record = restore_test_workdir(
|
||||
&api,
|
||||
"restore-ssh-workdir",
|
||||
WorkdirRegistrySource::Repository {
|
||||
runtime_id: identity.runtime_id.clone(),
|
||||
repository_id: test_repository_id(&api),
|
||||
},
|
||||
);
|
||||
api.store.upsert_workdir_registry(&record).unwrap();
|
||||
attach_restore_test_workdir(&api, &identity, &record);
|
||||
// Prove this is a fully issuable SSH fixture; the old pre-admission path would mint another handle.
|
||||
assert!(
|
||||
repository_access_request_for_workdir(
|
||||
&api,
|
||||
&identity.runtime_id,
|
||||
&record.workdir_id,
|
||||
"restore-ssh-positive-control"
|
||||
)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
let before = api.resource_broker.repository_ssh_access_handle_count();
|
||||
assert!(before > 0);
|
||||
let conflict = handle
|
||||
.restore(&WorkerOperationContext::Backend, stale)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(conflict.error, Error::RestoreObservationConflict));
|
||||
assert_eq!(
|
||||
api.resource_broker.repository_ssh_access_handle_count(),
|
||||
before
|
||||
);
|
||||
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_guard_historical_replay_ignores_new_eligibility_but_rechecks_auth() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||
execution.accept_restores();
|
||||
let identity = spawn_ticket_check_source(&api, "restore-historical");
|
||||
let replacement = spawn_ticket_check_source(&api, "restore-replacement");
|
||||
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||
handle
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
let request = restore_request_for(&api, &identity, "restore-historical-request");
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
handle
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
displace_restore_singleton(&api, &identity, &replacement);
|
||||
add_offline_restore_workdir(&api, &identity);
|
||||
let before = execution.restore_operations.lock().unwrap().len();
|
||||
let replay = handle
|
||||
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replay.state, server_api::WorkerRestoreState::Accepted);
|
||||
assert_eq!(
|
||||
replay.worker.unwrap().state,
|
||||
"stopped",
|
||||
"never return the old idle receipt summary"
|
||||
);
|
||||
assert_eq!(execution.restore_operations.lock().unwrap().len(), before);
|
||||
let projected = restore_runtime_worker_with_context(
|
||||
api.clone(),
|
||||
identity.runtime_id.clone(),
|
||||
identity.worker_id.clone(),
|
||||
WorkerOperationContext::Backend,
|
||||
request.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.0;
|
||||
assert_eq!(
|
||||
projected.result.worker.unwrap().state,
|
||||
"unavailable",
|
||||
"current fenced projection, never old idle receipt"
|
||||
);
|
||||
let (mut browser, _) = browser_operation_context(&api).await;
|
||||
let WorkerOperationContext::Browser { headers, .. } = &mut browser else {
|
||||
unreachable!()
|
||||
};
|
||||
headers.insert(ORIGIN, "https://untrusted.example".parse().unwrap());
|
||||
assert!(matches!(
|
||||
handle.restore(&browser, request).await.unwrap_err().error,
|
||||
Error::WorkspacePermissionDenied(_)
|
||||
));
|
||||
assert_eq!(execution.restore_operations.lock().unwrap().len(), before);
|
||||
assert!(
|
||||
handle
|
||||
.restore(
|
||||
&WorkerOperationContext::Backend,
|
||||
restore_request_for(&api, &identity, "new-ineligible-intent")
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_guard_pending_recovery_ignores_new_eligibility_and_reuses_operation() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||
execution.accept_restores();
|
||||
let identity = spawn_ticket_check_source(&api, "restore-pending");
|
||||
let replacement = spawn_ticket_check_source(&api, "restore-pending-replacement");
|
||||
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||
handle
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
*execution.restore_failure.lock().unwrap() = Some(
|
||||
worker_runtime::execution::WorkerExecutionSpawnResult::Errored(
|
||||
worker_runtime::execution::WorkerExecutionResult::rejected(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Restore,
|
||||
"uncertain launch",
|
||||
),
|
||||
),
|
||||
);
|
||||
let request = restore_request_for(&api, &identity, "restore-pending-request");
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired
|
||||
);
|
||||
let original = execution.restore_operations.lock().unwrap()[0];
|
||||
displace_restore_singleton(&api, &identity, &replacement);
|
||||
add_offline_restore_workdir(&api, &identity);
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, request)
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
execution.reconcile_operations.lock().unwrap().as_slice(),
|
||||
&[original]
|
||||
);
|
||||
assert!(
|
||||
execution
|
||||
.restore_operations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|id| id == &original)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_guard_internal_intent_survives_client_store_restart_without_replacing_token() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let (mut api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||
execution.accept_restores();
|
||||
let identity = spawn_ticket_check_source(&api, "restore-internal-restart");
|
||||
WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id)
|
||||
.unwrap()
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
let observed = api.runtime.worker(&identity).unwrap();
|
||||
*execution.restore_failure.lock().unwrap() = Some(
|
||||
worker_runtime::execution::WorkerExecutionSpawnResult::Errored(
|
||||
worker_runtime::execution::WorkerExecutionResult::rejected(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Restore,
|
||||
"uncertain launch",
|
||||
),
|
||||
),
|
||||
);
|
||||
assert_eq!(
|
||||
restore_internal_observed_worker(&api, &observed, "subject-restore".into())
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired
|
||||
);
|
||||
let original = execution.restore_operations.lock().unwrap()[0];
|
||||
let current = api.runtime.worker(&identity).unwrap();
|
||||
assert_ne!(
|
||||
observed.restore_observation_token,
|
||||
current.restore_observation_token
|
||||
);
|
||||
// Reopen the durable caller journal, discarding all client-side memory of request identity.
|
||||
api.config_store =
|
||||
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
|
||||
let pinned = api
|
||||
.config_store
|
||||
.pin_internal_worker_restore_intent(
|
||||
api.workspace_id(),
|
||||
&identity,
|
||||
"subject-restore",
|
||||
current.restore_observation_token.as_deref(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
pinned.expected_observation_token,
|
||||
observed.restore_observation_token.unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
restore_internal_observed_worker(&api, ¤t, "subject-restore".into())
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
execution.reconcile_operations.lock().unwrap().as_slice(),
|
||||
&[original]
|
||||
);
|
||||
assert!(
|
||||
execution
|
||||
.restore_operations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|id| id == &original)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_guard_workspace_interrupted_preparation_recovers_admitted_snapshot() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
|
||||
execution.accept_restores();
|
||||
let identity = spawn_ticket_check_source(&api, "restore-prepare-interrupted");
|
||||
let replacement = spawn_ticket_check_source(&api, "restore-prepare-replacement");
|
||||
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||
handle
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
let request = restore_request_for(&api, &identity, "restore-prepare-interrupted-request");
|
||||
let preparation = runtime_api::WorkerRestorePreparation {
|
||||
workspace_api: Some(
|
||||
runtime_contract_restore(api.workspace_api_ref(&identity.runtime_id)).unwrap(),
|
||||
),
|
||||
workdir_attachments: Some(Vec::new()),
|
||||
..Default::default()
|
||||
};
|
||||
let (result, retained) = api
|
||||
.runtime
|
||||
.coordinate_worker_restore(
|
||||
&identity,
|
||||
runtime_api::WorkerRestoreCoordinationRequest {
|
||||
expected_observation_token: request.expected_observation_token.clone(),
|
||||
request_id: request.request_id.clone(),
|
||||
preparation: Some(preparation.clone()),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(result.is_none());
|
||||
assert_eq!(retained, Some(preparation));
|
||||
assert!(execution.restore_operations.lock().unwrap().is_empty());
|
||||
// Model interruption between admission and completion, then changed current eligibility.
|
||||
displace_restore_singleton(&api, &identity, &replacement);
|
||||
add_offline_restore_workdir(&api, &identity);
|
||||
let restored = handle
|
||||
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted);
|
||||
assert_eq!(execution.restore_operations.lock().unwrap().len(), 1);
|
||||
let original = execution.restore_operations.lock().unwrap()[0];
|
||||
assert!(
|
||||
execution.reconcile_operations.lock().unwrap().is_empty(),
|
||||
"no execution existed to reconcile yet"
|
||||
);
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, request)
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
execution.restore_operations.lock().unwrap().as_slice(),
|
||||
&[original]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_workdir_owner() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let mut config = test_server_config(workspace.path());
|
||||
let source = server_api::RepositorySource {
|
||||
kind: server_api::RepositorySourceKind::Ssh,
|
||||
uri: "git@example.invalid:owner/repository.git".into(),
|
||||
};
|
||||
config.repositories[0].source_fingerprint =
|
||||
crate::repository_source::repository_source_fingerprint(&source);
|
||||
config.repositories[0].source = source;
|
||||
config.repositories[0].path = None;
|
||||
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
|
||||
let execution = Arc::new(DeterministicExecutionBackend::default());
|
||||
execution.accept_restores();
|
||||
let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let peer = WorkdirlessFixtureRuntime::default();
|
||||
*peer.reject_repository_access.lock().unwrap() = true;
|
||||
api.runtime.register_or_replace(peer.clone());
|
||||
let identity = spawn_ticket_check_source(&api, "restore-cross-runtime-ssh");
|
||||
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
|
||||
handle
|
||||
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
|
||||
.await
|
||||
.unwrap();
|
||||
let original_key = api
|
||||
.repository_secrets
|
||||
.generate_credential(
|
||||
api.workspace_id(),
|
||||
GenerateRepositorySshCredentialRequest {
|
||||
operation_id: "cross-runtime-default-key".into(),
|
||||
credential_id:
|
||||
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(),
|
||||
name: "Restore default".into(),
|
||||
},
|
||||
"owner-account",
|
||||
)
|
||||
.unwrap();
|
||||
let public_key = api
|
||||
.repository_secrets
|
||||
.credential_public_key(api.workspace_id(), &original_key.credential_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.public_key;
|
||||
api.repository_secrets
|
||||
.put_host_trust(
|
||||
api.workspace_id(),
|
||||
PutRepositorySshHostTrustRequest {
|
||||
operation_id: "cross-runtime-trust".into(),
|
||||
host_trust_id: "cross-runtime-host".into(),
|
||||
hostname: "example.invalid".into(),
|
||||
port: 22,
|
||||
host_key: public_key,
|
||||
expected_revision: None,
|
||||
},
|
||||
"owner-account",
|
||||
)
|
||||
.unwrap();
|
||||
let record = restore_test_workdir(
|
||||
&api,
|
||||
"cross-runtime-restore-workdir",
|
||||
WorkdirRegistrySource::Repository {
|
||||
runtime_id: WorkdirlessFixtureRuntime::RUNTIME_ID.into(),
|
||||
repository_id: test_repository_id(&api),
|
||||
},
|
||||
);
|
||||
peer.set_workdir_repository(&record.workdir_id, &test_repository_id(&api));
|
||||
api.store.upsert_workdir_registry(&record).unwrap();
|
||||
attach_restore_test_workdir(&api, &identity, &record);
|
||||
let request = restore_request_for(&api, &identity, "cross-runtime-restore-request");
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired
|
||||
);
|
||||
assert!(
|
||||
execution.restore_operations.lock().unwrap().is_empty(),
|
||||
"credential completion must precede execution"
|
||||
);
|
||||
let first = peer.repository_access_requests.lock().unwrap()[0].clone();
|
||||
assert_eq!(
|
||||
first.materialization.runtime_id,
|
||||
WorkdirlessFixtureRuntime::RUNTIME_ID
|
||||
);
|
||||
assert_ne!(first.materialization.runtime_id, identity.runtime_id);
|
||||
// Rotate current host trust while the accepted operation is uncertain. Its
|
||||
// original credential/trust revisions, source and binding must remain pinned.
|
||||
let alternate = api
|
||||
.repository_secrets
|
||||
.generate_credential(
|
||||
api.workspace_id(),
|
||||
GenerateRepositorySshCredentialRequest {
|
||||
operation_id: "cross-runtime-alternate-key".into(),
|
||||
credential_id: "cross-runtime-alternate".into(),
|
||||
name: "Alternate".into(),
|
||||
},
|
||||
"owner-account",
|
||||
)
|
||||
.unwrap();
|
||||
let alternate_public = api
|
||||
.repository_secrets
|
||||
.credential_public_key(api.workspace_id(), &alternate.credential_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.public_key;
|
||||
let original_trust_revision = first
|
||||
.materialization
|
||||
.ssh
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.host_trust_revision;
|
||||
api.repository_secrets
|
||||
.put_host_trust(
|
||||
api.workspace_id(),
|
||||
PutRepositorySshHostTrustRequest {
|
||||
operation_id: "cross-runtime-trust-rotate".into(),
|
||||
host_trust_id: "cross-runtime-host".into(),
|
||||
hostname: "example.invalid".into(),
|
||||
port: 22,
|
||||
host_key: alternate_public,
|
||||
expected_revision: Some(original_trust_revision),
|
||||
},
|
||||
"owner-account",
|
||||
)
|
||||
.unwrap();
|
||||
*peer.reject_repository_access.lock().unwrap() = false;
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, request.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
let attempts = peer.repository_access_requests.lock().unwrap();
|
||||
assert_eq!(attempts.len(), 2);
|
||||
let second = &attempts[1];
|
||||
assert_eq!(
|
||||
second.materialization.operation_id,
|
||||
first.materialization.operation_id
|
||||
);
|
||||
let mut normalized = second.clone();
|
||||
let old_ssh = first.materialization.ssh.as_ref().unwrap();
|
||||
let new_ssh = normalized.materialization.ssh.as_mut().unwrap();
|
||||
assert_ne!(new_ssh.secret_resource.nonce, old_ssh.secret_resource.nonce);
|
||||
new_ssh.secret_resource = old_ssh.secret_resource.clone();
|
||||
new_ssh.expires_at_epoch_seconds = old_ssh.expires_at_epoch_seconds;
|
||||
assert_eq!(normalized, first, "only opaque handle and expiry may renew");
|
||||
drop(attempts);
|
||||
assert_eq!(execution.restore_operations.lock().unwrap().len(), 1);
|
||||
assert_eq!(
|
||||
handle
|
||||
.restore(&WorkerOperationContext::Backend, request)
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
peer.repository_access_requests.lock().unwrap().len(),
|
||||
2,
|
||||
"terminal receipt must not renew access"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -291,10 +291,20 @@ impl WorkspaceWorker {
|
||||
pub(super) async fn restore(
|
||||
&self,
|
||||
context: &WorkerOperationContext,
|
||||
request: server_api::WorkerRestoreRequest,
|
||||
) -> ApiResult<InternalWorkerRestoreResult> {
|
||||
let _guard = self.authorize_operation(context, "restore").await?;
|
||||
// Preserve singleton, Workdir, credentials, bindings and pending operation reconciliation.
|
||||
self.api.restore_workspace_worker(&self.identity)
|
||||
self.restore_internal(request)
|
||||
}
|
||||
|
||||
/// Trusted Backend callers still supply an explicit observed intent. Runtime
|
||||
/// admission is shared with browser/Tool callers; this is not an unguarded launch.
|
||||
pub(super) fn restore_internal(
|
||||
&self,
|
||||
request: server_api::WorkerRestoreRequest,
|
||||
) -> ApiResult<InternalWorkerRestoreResult> {
|
||||
self.current_record()?;
|
||||
self.api.restore_workspace_worker(&self.identity, request)
|
||||
}
|
||||
|
||||
pub(super) async fn stop(
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore;
|
||||
use crate::{Error, Result};
|
||||
|
||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 79;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 80;
|
||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||
const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings";
|
||||
@@ -79,6 +79,8 @@ const ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME: &str =
|
||||
const BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME: &str =
|
||||
"durable terminal Backend Job Worker cleanup";
|
||||
const WORKDIR_CONNECTION_ID_MIGRATION_NAME: &str = "durable Workdir attachment connection identity";
|
||||
const WORKER_RESTORE_INTENTS_MIGRATION_NAME: &str =
|
||||
"durable internal Worker Restore request identity";
|
||||
const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7;
|
||||
const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline";
|
||||
const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#"
|
||||
@@ -271,6 +273,11 @@ const MIGRATIONS: &[Migration] = &[
|
||||
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME,
|
||||
apply: migrate_workspace_config_v78_to_v79,
|
||||
},
|
||||
Migration {
|
||||
version: 80,
|
||||
name: WORKER_RESTORE_INTENTS_MIGRATION_NAME,
|
||||
apply: migrate_worker_restore_intents_v79_to_v80,
|
||||
},
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -2167,6 +2174,48 @@ impl SqliteWorkspaceStore {
|
||||
})
|
||||
}
|
||||
|
||||
/// Client intent journal, not a lifecycle state machine: pin the public tuple
|
||||
/// before dispatch and keep it unchanged until a terminal receipt is obtained.
|
||||
pub(crate) fn pin_internal_worker_restore_intent(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
domain_key: &str,
|
||||
observed_token: Option<&str>,
|
||||
explicit_request_id: Option<&str>,
|
||||
) -> Result<server_api::WorkerRestoreRequest> {
|
||||
self.with_transaction(|tx| {
|
||||
let existing: Option<(String, String)> = if let Some(id) = explicit_request_id {
|
||||
tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4",
|
||||
params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()?
|
||||
} else {
|
||||
tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND domain_key=?4 AND settled=0",
|
||||
params![workspace_id, worker.runtime_id, worker.worker_id, domain_key], |row| Ok((row.get(0)?, row.get(1)?))).optional()?
|
||||
};
|
||||
if let Some((request_id, expected_observation_token)) = existing {
|
||||
return Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token });
|
||||
}
|
||||
let observed_token = observed_token.ok_or(Error::RestoreObservationConflict)?;
|
||||
let request_id = explicit_request_id.map(str::to_owned).unwrap_or_else(|| format!("{domain_key}:{observed_token}"));
|
||||
tx.execute("INSERT INTO worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key,request_id,expected_token,settled) VALUES(?1,?2,?3,?4,?5,?6,0)",
|
||||
params![workspace_id, worker.runtime_id, worker.worker_id, domain_key, request_id, observed_token])?;
|
||||
Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token: observed_token.to_owned() })
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn settle_internal_worker_restore_intent(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
request_id: &str,
|
||||
) -> Result<()> {
|
||||
self.with_conn(|conn| {
|
||||
conn.execute("UPDATE worker_restore_intents SET settled=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4",
|
||||
params![workspace_id, worker.runtime_id, worker.worker_id, request_id])?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn get_workspace_memory_settings(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -11248,6 +11297,7 @@ fn read_worker_registry_projection(
|
||||
})?
|
||||
} else {
|
||||
SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string())
|
||||
.map_err(|error| Error::InvalidInput(error.to_string()))?,
|
||||
runtime_id: Some(worker.runtime_id.to_string()),
|
||||
@@ -12013,6 +12063,24 @@ fn read_workdir_registry_record(
|
||||
})
|
||||
}
|
||||
|
||||
fn migrate_worker_restore_intents_v79_to_v80(conn: &Connection) -> Result<()> {
|
||||
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
|
||||
tx.execute_batch("CREATE TABLE worker_restore_intents (
|
||||
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
|
||||
domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL,
|
||||
settled INTEGER NOT NULL CHECK(settled IN (0,1)),
|
||||
PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id),
|
||||
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
|
||||
tx.execute(
|
||||
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(?1,?2)",
|
||||
params![80_i64, WORKER_RESTORE_INTENTS_MIGRATION_NAME],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn prepare_connection(conn: &Connection) -> Result<()> {
|
||||
configure_sqlite(conn)?;
|
||||
let workspace_schema_version = current_schema_version(conn)?;
|
||||
@@ -15094,6 +15162,15 @@ fn migrate_workdir_connection_id_v77_to_v78(conn: &Connection) -> Result<()> {
|
||||
}
|
||||
|
||||
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
||||
conn.execute_batch("CREATE TABLE worker_restore_intents (
|
||||
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
|
||||
domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL,
|
||||
settled INTEGER NOT NULL CHECK(settled IN (0,1)),
|
||||
PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id),
|
||||
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
|
||||
|
||||
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
|
||||
conn.execute_batch(include_str!("latest_schema.sql"))?;
|
||||
conn.execute_batch(
|
||||
@@ -16202,6 +16279,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn downgrade_schema_66_ticket_and_workdir_authority(conn: &Connection) -> Result<()> {
|
||||
conn.execute_batch("DROP TABLE IF EXISTS worker_restore_intents;")?;
|
||||
conn.execute_batch(
|
||||
"DROP TABLE backend_job_deliveries;
|
||||
DROP TABLE backend_job_attempts;
|
||||
@@ -17709,6 +17787,10 @@ mod tests {
|
||||
version: 79,
|
||||
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()
|
||||
},
|
||||
WorkspaceSchemaMigrationStep {
|
||||
version: 80,
|
||||
name: WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -17798,6 +17880,7 @@ mod tests {
|
||||
),
|
||||
(78, WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string()),
|
||||
(79, WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()),
|
||||
(80, WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()),
|
||||
]
|
||||
);
|
||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||
@@ -18149,7 +18232,7 @@ mod tests {
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
|
||||
73, 74, 75, 76, 77, 78, 79
|
||||
73, 74, 75, 76, 77, 78, 79, 80
|
||||
]
|
||||
);
|
||||
SqliteWorkspaceStore::migrate_database(&path).unwrap();
|
||||
@@ -18158,7 +18241,7 @@ mod tests {
|
||||
current_schema_version(&conn).unwrap(),
|
||||
LATEST_SCHEMA_VERSION
|
||||
);
|
||||
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 30);
|
||||
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 31);
|
||||
assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap());
|
||||
assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap());
|
||||
assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap());
|
||||
@@ -21860,6 +21943,7 @@ INSERT INTO worker_registry (
|
||||
|
||||
for record in [&runtime_a, &runtime_b] {
|
||||
let observed = SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: SubscriptionWorkerId::new("shared-local-id").unwrap(),
|
||||
runtime_id: Some(record.worker.runtime_id.clone()),
|
||||
resource_key: None,
|
||||
@@ -21996,6 +22080,7 @@ INSERT INTO worker_registry (
|
||||
Some("W-1")
|
||||
);
|
||||
let observed = SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: SubscriptionWorkerId::new("known").unwrap(),
|
||||
runtime_id: Some("embedded".to_string()),
|
||||
resource_key: None,
|
||||
@@ -22016,6 +22101,7 @@ INSERT INTO worker_registry (
|
||||
};
|
||||
store.upsert_worker_registry(&other_catalog).unwrap();
|
||||
let other_observed = SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: SubscriptionWorkerId::new("other").unwrap(),
|
||||
runtime_id: Some("runtime-other".to_string()),
|
||||
display_name: Some("Other Runtime Worker".to_string()),
|
||||
@@ -22025,6 +22111,7 @@ INSERT INTO worker_registry (
|
||||
.apply_worker_registry_observation("local-dev", "runtime-other", &other_observed, "2")
|
||||
.unwrap();
|
||||
let orphan = SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: SubscriptionWorkerId::new("orphan").unwrap(),
|
||||
..observed.clone()
|
||||
};
|
||||
@@ -22190,6 +22277,7 @@ INSERT INTO worker_registry (
|
||||
.is_none()
|
||||
);
|
||||
let event = SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: SubscriptionWorkerId::new("removed").unwrap(),
|
||||
runtime_id: Some("embedded".to_string()),
|
||||
resource_key: None,
|
||||
|
||||
@@ -76,6 +76,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
||||
"worker_registry_projection_diagnostics",
|
||||
"worker_registry_projection_removals",
|
||||
"worker_removal_operations",
|
||||
"worker_restore_intents",
|
||||
"worker_retention_audit_events",
|
||||
"worker_session_archive_observe_grants",
|
||||
"worker_session_archives",
|
||||
|
||||
@@ -425,6 +425,7 @@ fn project_registry_worker(
|
||||
worker
|
||||
} else {
|
||||
SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: protocol::subscription::SubscriptionWorkerId::new(
|
||||
record.registry.worker.worker_id.clone(),
|
||||
)
|
||||
@@ -600,6 +601,7 @@ mod tests {
|
||||
#[test]
|
||||
fn external_workdir_attachment_does_not_remove_workspace_worker() {
|
||||
let mut worker = SubscriptionWorker {
|
||||
restore_observation_token: None,
|
||||
worker_id: worker_id(),
|
||||
runtime_id: Some("runtime-1".to_string()),
|
||||
resource_key: Some("W-1".to_string()),
|
||||
|
||||
Reference in New Issue
Block a user