feat: guarded Worker Restore from shared Web actions (T-701)

This commit is contained in:
2026-10-06 17:45:54 +09:00
parent 1a6d391f83
commit fdce12a47b
54 changed files with 7214 additions and 527 deletions
+12
View File
@@ -10,6 +10,7 @@ pub use server_api::{
WorkerLaunchProfileCandidate as BackendWorkerLaunchProfileCandidate,
WorkerLaunchRuntimeOption as BackendWorkerLaunchRuntimeOption,
WorkerOperationState as BackendWorkerOperationState,
WorkerRestoreRequest as BackendWorkerRestoreRequest,
WorkerRestoreResponse as BackendWorkerRestoreResponse,
WorkerRestoreResult as BackendWorkerRestoreResult,
WorkerRestoreState as BackendWorkerRestoreState, WorkerSummary as BackendWorkerSummary,
@@ -259,6 +260,15 @@ impl fmt::Display for BackendRuntimeClientError {
}
}
impl BackendRuntimeClientError {
/// A stale intent is not a rejected Restore operation and must not be retried
/// with a silently refreshed token.
pub fn is_restore_observation_conflict(&self) -> bool {
matches!(self, Self::ContractApi(server_api::client_support::ClientError::Public { status, error })
if *status == reqwest::StatusCode::CONFLICT && error.error == "restore_observation_conflict")
}
}
impl std::error::Error for BackendRuntimeClientError {}
impl From<server_api::ServerApiClientError> for BackendRuntimeClientError {
@@ -502,6 +512,7 @@ pub async fn list_backend_stopped_workers(
pub async fn restore_backend_worker(
target: &BackendRuntimeTarget,
request: BackendWorkerRestoreRequest,
) -> Result<BackendWorkerRestoreResponse, BackendRuntimeClientError> {
validate_target(target)?;
let api = BackendApiClient::from_stored_token(&target.base_url)?;
@@ -514,6 +525,7 @@ pub async fn restore_backend_worker(
ticket_id: None,
assignment_operation_id: None,
},
request,
)
.await
.map_err(Into::into)
+6 -6
View File
@@ -26,12 +26,12 @@ pub use backend_runtime::{
BackendRuntimeListTarget, BackendRuntimeSummary, BackendRuntimeTarget,
BackendWorkerImplementationSummary, BackendWorkerLaunchOptions,
BackendWorkerLaunchProfileCandidate, BackendWorkerLaunchRuntimeOption,
BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreResponse,
BackendWorkerRestoreResult, BackendWorkerRestoreState, BackendWorkerSummary,
BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource, BackendWorkingDirectorySummary,
connect_backend_runtime, create_backend_worker, get_backend_worker_launch_options,
list_backend_stopped_workers, list_backend_workers, observe_backend_worker_session,
restore_backend_worker,
BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreRequest,
BackendWorkerRestoreResponse, BackendWorkerRestoreResult, BackendWorkerRestoreState,
BackendWorkerSummary, BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource,
BackendWorkingDirectorySummary, connect_backend_runtime, create_backend_worker,
get_backend_worker_launch_options, list_backend_stopped_workers, list_backend_workers,
observe_backend_worker_session, restore_backend_worker,
};
pub use backend_workspace::{
BackendWorkspace, BackendWorkspaceCatalogTarget, BackendWorkspaceClientError,
+8
View File
@@ -651,6 +651,10 @@ impl SubscriptionWorkerJob {
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
pub struct SubscriptionWorker {
/// Opaque Runtime-owned Restore observation identity, never a display-state hash.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
pub restore_observation_token: Option<String>,
pub worker_id: SubscriptionWorkerId,
/// Set by the Workspace Server when projecting a Runtime-owned Worker to clients.
/// Runtime producers leave this unset because the connection identifies the Runtime.
@@ -689,6 +693,9 @@ pub struct SubscriptionWorker {
impl SubscriptionWorker {
pub fn validate(&self) -> Result<(), SubscriptionValidationError> {
self.worker_id.validate()?;
if let Some(token) = &self.restore_observation_token {
validate_identifier("restore_observation_token", token, MAX_RESOURCE_ID_BYTES)?;
}
if let Some(runtime_id) = &self.runtime_id {
validate_identifier("runtime_id", runtime_id, MAX_RESOURCE_ID_BYTES)?;
}
@@ -962,6 +969,7 @@ mod tests {
fn worker(value: &str) -> SubscriptionWorker {
SubscriptionWorker {
restore_observation_token: None,
worker_id: worker_id(value),
runtime_id: None,
resource_key: None,
+1
View File
@@ -246,6 +246,7 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec<SubscriptionFrame> {
let external_workdir_id = SubscriptionWorkdirId::new("fixture-external-workdir")
.expect("fixture external Workdir id must be valid");
let worker = SubscriptionWorker {
restore_observation_token: Some("fixture-restore-observation".into()),
worker_id: worker_id.clone(),
runtime_id: Some("fixture-runtime".to_string()),
resource_key: Some("fixture-worker-resource".to_string()),
@@ -378,6 +378,7 @@ function validateWorkers(workers: SubscriptionWorker[]): boolean {
}
function validateWorker(worker: SubscriptionWorker): boolean {
if (worker.restore_observation_token != null && !validateIdentifier(worker.restore_observation_token, MAX_RESOURCE_ID_BYTES)) return false;
if (!validateIdentifier(worker.worker_id, MAX_RESOURCE_ID_BYTES)) return false;
if (worker.runtime_id != null && !validateIdentifier(worker.runtime_id, MAX_RESOURCE_ID_BYTES)) {
return false;
+116 -4
View File
@@ -452,6 +452,8 @@ pub struct CreateWorkerRequest {
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct WorkerSummary {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub restore_observation_token: Option<String>,
pub worker_ref: WorkerRef,
pub worker_id: WorkerId,
pub status: WorkerStatus,
@@ -484,6 +486,62 @@ pub struct WorkerResponse {
pub worker: WorkerDetail,
}
/// Explicit observed Restore intent. Preparation is authored by the Workspace
/// Backend, not browser/Tool JSON, and applied only after Runtime admission.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct WorkerRestoreRequest {
pub expected_observation_token: String,
pub request_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub preparation: Option<WorkerRestorePreparation>,
}
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct WorkerRestorePreparation {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workspace_api: Option<WorkspaceApiRef>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workdir_attachments: Option<Vec<LogicalWorkdirAttachment>>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub repository_access: Vec<WorkingDirectoryRepositoryAccessRequest>,
/// Read-only references resolved by Workspace only after this operation owns admission.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub repository_access_workdirs: Vec<RepositoryAccessWorkdirReference>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RepositoryAccessWorkdirReference {
pub runtime_id: String,
pub working_directory_id: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct WorkerRestoreCoordinationRequest {
pub expected_observation_token: String,
pub request_id: String,
/// None means owner/result recovery only; it can never admit a new intent.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub preparation: Option<WorkerRestorePreparation>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct WorkerRestoreCoordinationResponse {
pub result: Option<WorkerRestoreResponse>,
/// Original read-only preparation, only when the existing operation still needs it.
pub preparation: Option<WorkerRestorePreparation>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct WorkingDirectoryRepositoryAccessRequest {
pub working_directory_id: String,
pub materialization: RepositoryMaterializationContext,
}
/// Explicit restore outcome returned for every reachable Runtime restore attempt,
/// including preflight rejection and uncertain post-side-effect reconciliation.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
@@ -1000,11 +1058,18 @@ pub trait RuntimeApi {
#[body] request: WorkerLifecycleRequest,
) -> Result<WorkerLifecycleResponse, RuntimeApiError>;
#[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400)]
#[post("/v1/workers/{worker_id}/restore/coordinate", status = 200, error_status = 400, additional_error_statuses = [409])]
async fn coordinate_worker_restore(
&self,
#[path] worker_id: String,
#[body] request: WorkerRestoreCoordinationRequest,
) -> Result<WorkerRestoreCoordinationResponse, RuntimeApiError>;
#[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400, additional_error_statuses = [409])]
async fn restore_worker(
&self,
#[path] worker_id: String,
#[body] request: EmptyObjectRequest,
#[body] request: WorkerRestoreRequest,
) -> Result<WorkerRestoreResponse, RuntimeApiError>;
#[post(
@@ -1210,6 +1275,46 @@ pub const REMAINING_RUNTIME_ROUTES: &[RemainingRuntimeRoute] = &[
mod tests {
use super::*;
#[test]
fn restore_request_is_mandatory_and_conflict_is_not_an_operation_state() {
for invalid in [
serde_json::json!({}),
serde_json::json!({"request_id":"r"}),
serde_json::json!({"expected_observation_token":"t"}),
] {
assert!(serde_json::from_value::<WorkerRestoreRequest>(invalid).is_err());
}
let request: WorkerRestoreRequest = serde_json::from_value(
serde_json::json!({"expected_observation_token":"t", "request_id":"r"}),
)
.unwrap();
assert!(request.preparation.is_none());
for state in [
"accepted",
"rejected",
"rolled_back",
"reconciliation_required",
] {
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
}
assert!(
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
"restore_observation_conflict"
))
.is_err()
);
let conflict = RuntimeApiError::new(
409,
"restore_observation_conflict",
"refresh before a new intent",
);
assert_eq!(conflict.status(), 409);
assert_eq!(
serde_json::to_value(conflict).unwrap()["error"]["code"],
"restore_observation_conflict"
);
}
#[test]
fn feature_argument_completion_wire_preserves_source_and_argument() {
let json = serde_json::json!({"kind":"feature_argument", "prefix":"資料/", "context": {"invocation":"builtin:test/prepare", "argument":"path"}});
@@ -1388,10 +1493,17 @@ mod tests {
) -> Result<WorkerLifecycleResponse, RuntimeApiError> {
Err(test_error(501))
}
async fn coordinate_worker_restore(
&self,
_worker_id: String,
_request: WorkerRestoreCoordinationRequest,
) -> Result<WorkerRestoreCoordinationResponse, RuntimeApiError> {
Err(test_error(501))
}
async fn restore_worker(
&self,
_worker_id: String,
_request: EmptyObjectRequest,
_request: WorkerRestoreRequest,
) -> Result<WorkerRestoreResponse, RuntimeApiError> {
Err(test_error(501))
}
@@ -1491,7 +1603,7 @@ mod tests {
#[test]
fn contract_inventory_is_complete_and_unique() {
let operations = RuntimeApiMetadata::OPERATIONS;
assert_eq!(operations.len(), 19);
assert_eq!(operations.len(), 20);
let mut routes = operations
.iter()
.map(|operation| (format!("{:?}", operation.method), operation.path))
+40
View File
@@ -389,6 +389,7 @@ impl_openapi_schema!(
WorkspaceRuntimeListResponse,
RuntimeWorkerListResponse,
WorkerRestoreResponse,
WorkerRestoreRequest,
WorkerControlSubject,
WorkerControlRecord,
WorkerControlListResponse,
@@ -1552,6 +1553,7 @@ pub trait ServerApi {
#[path] workspace_id: String,
#[path] runtime_id: String,
#[path] worker_id: String,
#[body] request: WorkerRestoreRequest,
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
#[get(
"/api/w/{workspace_id}/worker-observation/sessions",
@@ -2080,6 +2082,7 @@ pub trait ServerApi {
#[extension] context: ServerRequestContext,
#[path] runtime_id: String,
#[path] worker_id: String,
#[body] request: WorkerRestoreRequest,
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
#[post(
@@ -2097,6 +2100,7 @@ pub trait ServerApi {
#[path] runtime_id: String,
#[path] worker_id: String,
#[query] query: RestoreTicketAssignmentQuery,
#[body] request: WorkerRestoreRequest,
) -> Result<WorkerRestoreResponse, RepositoryApiError>;
#[put(
@@ -9594,6 +9598,10 @@ pub struct RuntimeWorkerWorkdirAttachmentSummary {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct WorkerSummary {
/// Opaque Runtime-owned observation fence used by explicit Restore intent.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
pub restore_observation_token: Option<String>,
pub runtime_id: String,
pub worker_id: String,
pub resource_key: String,
@@ -9830,6 +9838,16 @@ pub enum WorkerRestoreState {
ReconciliationRequired,
}
/// One explicit Restore intent. Retries retain both fields; conflicts require a
/// fresh observation and a deliberate new intent, not automatic token replacement.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
pub struct WorkerRestoreRequest {
pub expected_observation_token: String,
pub request_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
@@ -11694,6 +11712,28 @@ mod tests {
);
}
#[test]
fn worker_restore_request_requires_observation_and_request_identity() {
let valid = serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request"});
let request: WorkerRestoreRequest = serde_json::from_value(valid.clone()).unwrap();
assert_eq!(serde_json::to_value(request).unwrap(), valid);
for invalid in [
serde_json::json!({}),
serde_json::json!({"request_id":"restore-request"}),
serde_json::json!({"expected_observation_token":"opaque-generation"}),
serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request", "command_id":701}),
] {
assert!(serde_json::from_value::<WorkerRestoreRequest>(invalid).is_err());
}
let contract = canonical_openapi_document().unwrap().to_json().unwrap();
let contract: serde_json::Value = serde_json::from_str(&contract).unwrap();
let restore = &contract["paths"]["/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore"]
["post"];
assert_eq!(restore["requestBody"]["required"], true);
assert!(restore["responses"]["409"].is_object());
assert!(contract["components"]["schemas"]["WorkerRestoreRequest"].is_object());
}
#[test]
fn worker_restore_result_rejects_unknown_fields() {
let result = serde_json::from_value::<WorkerRestoreResult>(serde_json::json!({
@@ -872,14 +872,14 @@ fn validate_runtime_operations(
"/api/runtimes/{runtime_id}/workers/{worker_id}/restore",
"post",
"runtime_worker_restore_alias",
None,
Some("WorkerRestoreRequest"),
&[("200", "WorkerRestoreResponse")],
),
runtime_operation(
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore",
"post",
"runtime_worker_restore",
None,
Some("WorkerRestoreRequest"),
&[("200", "WorkerRestoreResponse")],
),
runtime_operation(
+62 -19
View File
@@ -29,6 +29,7 @@ trait BackendWorkerLifecycle {
async fn restore(
&self,
target: &client::BackendRuntimeTarget,
request: client::BackendWorkerRestoreRequest,
) -> Result<BackendWorkerRestoreResponse, io::Error>;
async fn observe(
@@ -44,13 +45,19 @@ impl BackendWorkerLifecycle for LiveBackendWorkerLifecycle {
async fn restore(
&self,
target: &client::BackendRuntimeTarget,
request: client::BackendWorkerRestoreRequest,
) -> Result<BackendWorkerRestoreResponse, io::Error> {
restore_backend_worker(target).await.map_err(|error| {
io::Error::other(format!(
"failed to restore Backend Worker {}: {error}",
target.display_label()
))
})
restore_backend_worker(target, request)
.await
.map_err(|error| {
if error.is_restore_observation_conflict() {
return io::Error::other("Restore observation changed; refresh the Worker list and select again. No new Restore was started.");
}
io::Error::other(format!(
"failed to restore Backend Worker {}: {error}",
target.display_label()
))
})
}
async fn observe(
@@ -133,8 +140,20 @@ pub(crate) async fn run(
let attach_target = target
.runtime_target(selected.runtime_id.clone(), selected.worker_id.clone())
.map_err(|error| io::Error::other(error.to_string()))?;
let attach_target =
prepare_selected_worker(&LiveBackendWorkerLifecycle, attach_target, intent).await?;
let restore_request =
selected
.restore_observation_token
.map(|token| client::BackendWorkerRestoreRequest {
expected_observation_token: token,
request_id: uuid::Uuid::now_v7().to_string(),
});
let attach_target = prepare_selected_worker(
&LiveBackendWorkerLifecycle,
attach_target,
intent,
restore_request,
)
.await?;
return console::run_backend_runtime(attach_target).await;
}
}
@@ -143,9 +162,13 @@ async fn prepare_selected_worker(
lifecycle: &impl BackendWorkerLifecycle,
mut target: client::BackendRuntimeTarget,
intent: BackendWorkerPickerIntent,
restore_request: Option<client::BackendWorkerRestoreRequest>,
) -> Result<client::BackendRuntimeTarget, io::Error> {
if intent == BackendWorkerPickerIntent::Resume {
let response = lifecycle.restore(&target).await?;
let request = restore_request.ok_or_else(|| {
io::Error::other("Worker has no current Restore observation; refresh and select again")
})?;
let response = lifecycle.restore(&target, request).await?;
ensure_restore_accepted(&response)?;
let observation = lifecycle.observe(&target).await?;
require_live_resume_observation(observation.observation)?;
@@ -158,7 +181,10 @@ async fn prepare_selected_worker(
}
fn is_resume_candidate(worker: &BackendWorkerSummary) -> bool {
worker.worker_state.is_none()
worker.availability == protocol::subscription::SubscriptionWorkerAvailability::Observed
&& worker.state == "stopped"
&& worker.worker_state.is_none()
&& worker.restore_observation_token.is_some()
}
fn ensure_restore_accepted(response: &BackendWorkerRestoreResponse) -> Result<(), io::Error> {
@@ -570,6 +596,7 @@ mod tests {
fn worker(runtime_id: &str, worker_id: &str, profile: Option<&str>) -> BackendWorkerSummary {
BackendWorkerSummary {
restore_observation_token: Some("observed-worker-generation".into()),
runtime_id: runtime_id.to_string(),
worker_id: worker_id.to_string(),
resource_key: "W-1".to_string(),
@@ -642,6 +669,7 @@ mod tests {
async fn restore(
&self,
_target: &client::BackendRuntimeTarget,
_request: client::BackendWorkerRestoreRequest,
) -> Result<BackendWorkerRestoreResponse, io::Error> {
self.calls.lock().unwrap().push("restore");
Ok(self.restore_response.clone())
@@ -689,10 +717,17 @@ mod tests {
"worker-a",
);
let prepared =
prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume)
.await
.unwrap();
let prepared = prepare_selected_worker(
&lifecycle,
target,
BackendWorkerPickerIntent::Resume,
Some(client::BackendWorkerRestoreRequest {
expected_observation_token: "observed-worker-generation".into(),
request_id: "resume-operation".into(),
}),
)
.await
.unwrap();
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore", "observe"]);
assert!(prepared.initial_snapshot.is_none());
@@ -716,10 +751,18 @@ mod tests {
"worker-a",
);
let error = prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume)
.await
.unwrap_err()
.to_string();
let error = prepare_selected_worker(
&lifecycle,
target,
BackendWorkerPickerIntent::Resume,
Some(client::BackendWorkerRestoreRequest {
expected_observation_token: "observed-worker-generation".into(),
request_id: "resume-operation".into(),
}),
)
.await
.unwrap_err()
.to_string();
assert!(error.contains("Rejected"));
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
@@ -736,7 +779,7 @@ mod tests {
assert!(!is_resume_candidate(&live));
assert!(is_resume_candidate(&stopped));
assert!(is_resume_candidate(&unknown));
assert!(!is_resume_candidate(&unknown));
}
#[test]
+4
View File
@@ -334,6 +334,8 @@ pub(crate) enum WorkerRestoreIntent {
/// Lightweight persisted Worker identity projection.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerSummary {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub restore_observation_token: Option<String>,
pub worker_ref: WorkerRef,
pub worker_id: WorkerId,
pub status: WorkerStatus,
@@ -359,6 +361,8 @@ pub struct WorkerSummary {
/// Full persisted Worker identity and lifecycle detail.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerDetail {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub restore_observation_token: Option<String>,
pub worker_ref: WorkerRef,
pub worker_id: WorkerId,
pub status: WorkerStatus,
+3
View File
@@ -12,6 +12,9 @@ pub enum RuntimeError {
#[error("initial worker input must be user input, got {kind}")]
InvalidInitialInputKind { kind: String },
#[error("worker {worker_id} Restore observation conflicts with the current operation")]
RestoreObservationConflict { worker_id: WorkerId },
#[error("worker {worker_id} was not found")]
WorkerNotFound { worker_id: WorkerId },
+7
View File
@@ -520,6 +520,7 @@ pub(crate) enum PersistedWorkerExecutionState {
#[derive(Clone, Debug)]
pub(crate) struct PersistedWorkerRecord {
pub(crate) restore_guard: crate::runtime::RestoreGuard,
pub(crate) worker_ref: WorkerRef,
pub(crate) worker_id: WorkerId,
pub(crate) profile: ProfileSelector,
@@ -1819,6 +1820,8 @@ impl RuntimeSnapshot {
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct WorkerAggregateRecord {
#[serde(default)]
restore_guard: crate::runtime::RestoreGuard,
schema_version: u32,
worker_ref: WorkerRef,
worker_id: WorkerId,
@@ -1874,6 +1877,7 @@ impl WorkerAggregateRecord {
};
Self {
schema_version: SCHEMA_VERSION,
restore_guard: worker.restore_guard.clone(),
worker_ref: worker.worker_ref.clone(),
worker_id: worker.worker_id,
profile: worker.profile.clone(),
@@ -1902,6 +1906,7 @@ impl WorkerAggregateRecord {
}
fn validate(self, path: &Path) -> Result<PersistedWorkerRecord, RuntimeError> {
let restore_guard = self.restore_guard;
let last_finished_submission_request_id = self.last_finished_submission_request_id;
let identity = WorkerIdentityRecord {
schema_version: self.schema_version,
@@ -2007,6 +2012,7 @@ impl WorkerAggregateRecord {
WorkerExecutionStateRecord::Unavailable => PersistedWorkerExecutionState::Unavailable,
};
let mut persisted = identity.into_persisted(execution_state);
persisted.restore_guard = restore_guard;
persisted.last_finished_submission_request_id = last_finished_submission_request_id;
Ok(persisted)
}
@@ -2180,6 +2186,7 @@ impl WorkerIdentityRecord {
self.workdir_attachments
};
PersistedWorkerRecord {
restore_guard: crate::runtime::RestoreGuard::default(),
worker_ref: self.worker_ref,
worker_id: self.worker_id,
profile: self.profile,
+136 -7
View File
@@ -2302,6 +2302,7 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
}
if path.ends_with("/input")
|| path.ends_with("/restore")
|| path.ends_with("/restore/coordinate")
|| (path.contains("/attachments") && *method != Method::GET)
|| path.contains("/workdir-attachments")
{
@@ -2514,6 +2515,7 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
StatusCode::CONFLICT
}
RuntimeError::RuntimeStopped
| RuntimeError::RestoreObservationConflict { .. }
| RuntimeError::RuntimeStoreAlreadyOpen { .. }
| RuntimeError::WorkerExecutionUnavailable { .. }
| RuntimeError::ExecutionBackendUnavailable { .. }
@@ -2538,6 +2540,9 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
fn code_for_runtime_error(error: &RuntimeError) -> String {
match error {
RuntimeError::RuntimeStopped => "runtime_stopped".to_string(),
RuntimeError::RestoreObservationConflict { .. } => {
"restore_observation_conflict".to_string()
}
RuntimeError::RuntimeStoreAlreadyOpen { .. } => "runtime_store_already_open".to_string(),
RuntimeError::WorkerNotFound { .. } => "worker_not_found".to_string(),
RuntimeError::WorkerExecutionUnavailable { .. } => {
@@ -3642,7 +3647,17 @@ mod tests {
client
.restore_worker(
generated_worker_id.clone(),
runtime_api::EmptyObjectRequest::default(),
runtime_api::WorkerRestoreRequest {
expected_observation_token: client
.get_worker(generated_worker_id.clone())
.await
.unwrap()
.worker
.restore_observation_token
.unwrap(),
request_id: "generated-restore".into(),
preparation: Some(runtime_api::WorkerRestorePreparation::default()),
},
)
.await
.unwrap();
@@ -3786,18 +3801,86 @@ mod tests {
let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await;
assert_eq!(stop.ack.worker_ref, created.worker.worker_ref);
let response = authed_empty_request(
let restore_uri = format!("/v1/workers/{}/restore", created.worker.worker_id);
let missing = authed_empty_request(app.clone(), Method::POST, &restore_uri, token).await;
assert_eq!(missing.status(), StatusCode::BAD_REQUEST);
let stale = runtime_api::WorkerRestoreRequest {
expected_observation_token: created.worker.restore_observation_token.clone().unwrap(),
request_id: "stale-http-restore".into(),
preparation: None,
};
let response =
authed_json_request(app.clone(), Method::POST, &restore_uri, token, &stale).await;
assert_eq!(response.status(), StatusCode::CONFLICT);
let conflict: RuntimeHttpErrorResponse = read_json(response).await;
assert_eq!(conflict.error.code, "restore_observation_conflict");
let mut restore_request = runtime.test_restore_request(&created.worker.worker_ref);
let coordinate_uri = format!("{restore_uri}/coordinate");
let mut coordination = runtime_api::WorkerRestoreCoordinationRequest {
expected_observation_token: restore_request.expected_observation_token.clone(),
request_id: restore_request.request_id.clone(),
preparation: None,
};
let before_lookup = runtime.worker_detail(&created.worker.worker_ref).unwrap();
let response = authed_json_request(
app.clone(),
Method::POST,
&format!("/v1/workers/{}/restore", created.worker.worker_id),
&coordinate_uri,
token,
&coordination,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let missing_owner: runtime_api::WorkerRestoreCoordinationResponse =
read_json(response).await;
assert!(missing_owner.result.is_none() && missing_owner.preparation.is_none());
assert_eq!(
runtime.worker_detail(&created.worker.worker_ref).unwrap(),
before_lookup
);
let response = authed_json_request(
app.clone(),
Method::POST,
&coordinate_uri,
"wrong-token",
&coordination,
)
.await;
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
coordination.preparation = Some(runtime_api::WorkerRestorePreparation::default());
let response = authed_json_request(
app.clone(),
Method::POST,
&coordinate_uri,
token,
&coordination,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let admitted: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await;
assert!(admitted.result.is_none() && admitted.preparation.is_some());
assert_eq!(
runtime
.worker_detail(&created.worker.worker_ref)
.unwrap()
.status,
crate::catalog::WorkerStatus::Stopped
);
restore_request.preparation = admitted.preparation;
let response = authed_json_request(
app.clone(),
Method::POST,
&restore_uri,
token,
&restore_request,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let restored: runtime_api::WorkerRestoreResponse = read_json(response).await;
assert_eq!(restored.state, WorkerRestoreState::Accepted);
assert_eq!(
restored.worker.unwrap().status,
restored.worker.as_ref().unwrap().status,
runtime_api::WorkerStatus::Idle
);
@@ -3810,6 +3893,47 @@ mod tests {
.await;
assert_eq!(response.status(), StatusCode::OK);
coordination.preparation = None;
let response = authed_json_request(
app.clone(),
Method::POST,
&coordinate_uri,
token,
&coordination,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let recovered: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await;
assert!(recovered.preparation.is_none());
assert_eq!(recovered.result, Some(restored.clone()));
restore_request.preparation = Some(serde_json::from_value(serde_json::json!({
"workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" },
"workdir_attachments": [],
"repository_access": []
})).unwrap());
let response = authed_json_request(
app.clone(),
Method::POST,
&restore_uri,
token,
&restore_request,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let replayed: runtime_api::WorkerRestoreResponse = read_json(response).await;
assert_eq!(
serde_json::to_value(replayed).unwrap(),
serde_json::to_value(restored).unwrap()
);
assert_eq!(
runtime
.worker_detail(&created.worker.worker_ref)
.unwrap()
.status,
crate::catalog::WorkerStatus::Stopped
);
let response = authed_empty_request(
app.clone(),
Method::POST,
@@ -3892,11 +4016,12 @@ mod tests {
assert_eq!(listed.workers[0].status, crate::catalog::WorkerStatus::Idle);
assert!(listed.workers[0].worker_state.is_none());
let response = authed_empty_request(
let response = authed_json_request(
app.clone(),
Method::POST,
&format!("{worker_uri}/restore"),
token,
&runtime.test_restore_request(&created.worker_ref),
)
.await;
assert_eq!(response.status(), StatusCode::OK);
@@ -4864,7 +4989,9 @@ mod ws_tests {
let (mut old_stream, _) = connect_async(authed_ws_request(&url)).await.unwrap();
let _ = next_frame(&mut old_stream).await;
runtime.stop_worker(&worker_ref, None).unwrap();
runtime.restore_worker(&worker_ref).unwrap();
runtime
.restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref))
.unwrap();
backend.publish(
&worker_ref,
protocol::Event::TextDelta {
@@ -4946,7 +5073,9 @@ mod ws_tests {
);
backend.publish(&worker_ref, protocol::Event::Shutdown);
runtime.stop_worker(&worker_ref, None).unwrap();
runtime.restore_worker(&worker_ref).unwrap();
runtime
.restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref))
.unwrap();
let resume_url = format!("{url}?cursor={}", old.cursor);
let (mut stream, _) = connect_async(authed_ws_request(&resume_url)).await.unwrap();
assert!(matches!(
@@ -462,21 +462,74 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
Ok(runtime_api::WorkerLifecycleResponse { ack })
}
async fn coordinate_worker_restore(
&self,
worker_id: String,
request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<runtime_api::WorkerRestoreCoordinationResponse, runtime_api::RuntimeApiError> {
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
let scope =
auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?;
let runtime = self.state.runtime.clone();
let (result, preparation) = tokio::task::spawn_blocking(move || {
runtime.coordinate_worker_restore_operation(&worker_ref, request, scope.as_ref())
})
.await
.map_err(|error| {
runtime_api::RuntimeApiError::new(500, "runtime_restore_task_failed", error.to_string())
})?
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)?;
let result = result
.map(|result| -> Result<_, runtime_api::RuntimeApiError> {
Ok(runtime_api::WorkerRestoreResponse {
state: match result.state {
server_api::WorkerRestoreState::Accepted => {
runtime_api::WorkerRestoreState::Accepted
}
server_api::WorkerRestoreState::Rejected => {
runtime_api::WorkerRestoreState::Rejected
}
server_api::WorkerRestoreState::RolledBack => {
runtime_api::WorkerRestoreState::RolledBack
}
server_api::WorkerRestoreState::ReconciliationRequired => {
runtime_api::WorkerRestoreState::ReconciliationRequired
}
},
worker: result.worker.map(response).transpose()?,
reason_code: result.reason_code,
message: result.message,
})
})
.transpose()?;
Ok(runtime_api::WorkerRestoreCoordinationResponse {
result,
preparation,
})
}
async fn restore_worker(
&self,
worker_id: String,
_request: runtime_api::EmptyObjectRequest,
request: runtime_api::WorkerRestoreRequest,
) -> Result<runtime_api::WorkerRestoreResponse, runtime_api::RuntimeApiError> {
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
let result = match auth_workspace_scope(&self.state, auth_extension().as_ref())
.map_err(api_error)?
{
Some(scope) => self
.state
.runtime
.restore_worker_operation_scoped(&scope, &worker_ref),
None => self.state.runtime.restore_worker_operation(&worker_ref),
}
let scope =
auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?;
let runtime = self.state.runtime.clone();
let result = tokio::task::spawn_blocking(move || match scope {
Some(scope) => runtime.restore_worker_operation_scoped(&scope, &worker_ref, request),
None => runtime.restore_worker_operation(&worker_ref, request),
})
.await
.map_err(|error| {
runtime_api::RuntimeApiError::new(
StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
"runtime_restore_task_failed",
error.to_string(),
)
})?
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)?;
Ok(runtime_api::WorkerRestoreResponse {
File diff suppressed because it is too large Load Diff
+20 -3
View File
@@ -7394,7 +7394,14 @@ mod tests {
.create_worker(create_request("restart cleanup"))
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
assert!(runtime.restore_worker(&worker.worker_ref).is_err());
assert!(
runtime
.restore_worker(
&worker.worker_ref,
runtime.test_restore_request(&worker.worker_ref)
)
.is_err()
);
assert!(runtime.stop_worker(&worker.worker_ref, None).is_err());
let prior_restore_id = restore_id.lock().unwrap().unwrap();
let aggregate = runtime_store
@@ -8904,7 +8911,12 @@ mod tests {
.unwrap();
assert!(detached.workdir_attachments.is_empty());
runtime.stop_worker(&detail.worker_ref, None).unwrap();
let restored_without_workdir = runtime.restore_worker(&detail.worker_ref).unwrap();
let restored_without_workdir = runtime
.restore_worker(
&detail.worker_ref,
runtime.test_restore_request(&detail.worker_ref),
)
.unwrap();
assert!(restored_without_workdir.workdir_attachments.is_empty());
assert!(
!repo.path().join(".yoi").exists(),
@@ -9191,7 +9203,12 @@ mod tests {
.contains_key(&detail.worker_ref)
);
runtime.restore_worker(&detail.worker_ref).unwrap();
runtime
.restore_worker(
&detail.worker_ref,
runtime.test_restore_request(&detail.worker_ref),
)
.unwrap();
assert_eq!(
runtime.worker_detail(&detail.worker_ref).unwrap().status,
crate::catalog::WorkerStatus::Idle
+7 -1
View File
@@ -555,9 +555,15 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
WorkspaceServerOperation::WorkerControlRestore {
runtime_id,
worker_id,
request,
} => generated_workspace_response(
client
.worker_control_restore(workspace_id, runtime_id, worker_id)
.worker_control_restore(
workspace_id,
runtime_id,
worker_id,
request,
)
.await,
),
WorkspaceServerOperation::WorkerObservationSessions => {
@@ -77,8 +77,9 @@ pub trait WorkerControlService: Send + Sync {
&self,
runtime_id: String,
worker_id: String,
request: server_api::WorkerRestoreRequest,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
let _ = (runtime_id, worker_id);
let _ = (runtime_id, worker_id, request);
Err(WorkspaceClientError::Unavailable(
"Runtime Worker restore is unavailable".to_string(),
))
@@ -347,6 +348,7 @@ impl WorkerControlService for WorkspaceWorkerControlService {
&self,
runtime_id: String,
worker_id: String,
request: server_api::WorkerRestoreRequest,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
if !self.runtime_worker_control {
return Err(WorkspaceClientError::Unavailable(
@@ -357,6 +359,7 @@ impl WorkerControlService for WorkspaceWorkerControlService {
crate::worker::WorkspaceServerOperation::WorkerControlRestore {
runtime_id,
worker_id,
request,
},
)
}
@@ -750,8 +753,13 @@ enum WorkerSubjectInput {
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
struct WorkerTargetInput {
struct WorkerRestoreInput {
subject: WorkerSubjectInput,
expected_observation_token: String,
/// Reuse the reported ID for explicit unknown-outcome recovery. A new intent
/// defaults to the durable Tool call identity.
#[serde(default)]
request_id: Option<String>,
}
#[derive(Debug, Deserialize, JsonSchema)]
@@ -864,7 +872,7 @@ impl WorkerOperation {
Self::Cancel => "Cancel the current turn of a known Runtime Worker when allowed.",
Self::Stop => "Stop a known Runtime Worker when allowed.",
Self::Restore => {
"Restore a stopped Backend/Runtime Worker session in the current Workspace."
"Restore a stopped Backend/Runtime Worker session in the current Workspace. Supply expected_observation_token from WorkerList; a conflict requires refreshing and deliberate new intent. New requests default to the durable Tool call ID. For explicit recovery of an unknown/pending outcome, supply the reported request_id and the original token unchanged; never create a new intent or automatically retry."
}
Self::Remove => {
"Remove an eligible stopped, unassigned, non-internal Worker. Supply a bounded reason; Backend validation and retention are authoritative."
@@ -981,12 +989,41 @@ impl Tool for WorkspaceWorkerTool {
.map_err(control_tool_error)?
}
WorkerOperation::Restore => {
let input = parse::<WorkerTargetInput>(input_json, "WorkerRestore")?;
let input = parse::<WorkerRestoreInput>(input_json, "WorkerRestore")?;
let (runtime_id, worker_id) = runtime_subject_ids(&input.subject, self.operation)?;
self.control
.restore_worker(runtime_id, worker_id)
let request_id = match input.request_id {
Some(id) => non_empty(id, "request_id")?,
None => format!(
"worker-restore:{}",
non_empty(ctx.call_id.clone(), "tool call_id")?
),
};
if request_id.len() > 512 {
return Err(ToolError::InvalidArgument(
"request_id must contain at most 512 bytes".into(),
));
}
let recoverable_error = |error| {
ToolError::ExecutionFailed(format!(
"WorkerRestore request_id={request_id}; reuse this ID with the original observation token only for explicit unknown/pending outcome recovery, not after a stale-intent conflict: {error}"
))
};
let response = self
.control
.restore_worker(
runtime_id,
worker_id,
server_api::WorkerRestoreRequest {
expected_observation_token: non_empty(
input.expected_observation_token,
"expected_observation_token",
)?,
request_id: request_id.clone(),
},
)
.await
.map_err(control_tool_error)?
.map_err(|error| recoverable_error(control_tool_error(error)))?;
return tool_output(self.operation, response).map_err(recoverable_error);
}
WorkerOperation::Remove => {
let input = parse::<WorkerRemoveInput>(input_json, "WorkerRemove")?;
@@ -1058,6 +1095,18 @@ fn tool_output(
response.status, response.body
)));
}
if operation == WorkerOperation::Restore {
let restored: server_api::WorkerRestoreResponse = serde_json::from_str(&response.body)
.map_err(|error| {
ToolError::ExecutionFailed(format!("invalid WorkerRestore response: {error}"))
})?;
if restored.result.state != server_api::WorkerRestoreState::Accepted {
return Err(ToolError::ExecutionFailed(format!(
"WorkerRestore returned {:?}: {}",
restored.result.state, response.body
)));
}
}
if operation == WorkerOperation::Stop
&& let Ok(summary) = serde_json::from_str::<SubWorkerStopSummary>(&response.body)
{
@@ -1150,7 +1199,7 @@ fn worker_tool_contribution(
}
}
WorkerOperation::Restore => {
definition::<WorkerTargetInput>(operation, control, runtime_worker_control)
definition::<WorkerRestoreInput>(operation, control, runtime_worker_control)
}
WorkerOperation::Remove => {
definition::<WorkerRemoveInput>(operation, control, runtime_worker_control)
@@ -1410,6 +1459,95 @@ mod tests {
})
}
#[tokio::test]
async fn worker_restore_forwards_exact_observation_and_request_identity() {
let client = Arc::new(RecordingWorkspaceClient::default());
let control = test_control(client.clone());
for _ in 0..2 {
control
.restore_worker(
"runtime/a".into(),
"worker/b".into(),
server_api::WorkerRestoreRequest {
expected_observation_token: "same-observation".into(),
request_id: "same-request".into(),
},
)
.await
.unwrap();
}
let requests = client.requests.lock().unwrap();
assert_eq!(requests.len(), 2);
assert_eq!(
requests[0].path,
"/api/w/workspace%2Ftest/worker-control/workers/runtime%2Fa/worker%2Fb/restore"
);
assert_eq!(requests[0].body, requests[1].body);
let body: serde_json::Value =
serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap();
assert_eq!(
body,
serde_json::json!({"expected_observation_token":"same-observation","request_id":"same-request"})
);
assert!(serde_json::from_str::<WorkerRestoreInput>(r#"{"subject":{"kind":"runtime_worker","runtime_id":"runtime-a","worker_id":"worker-a"}}"#).is_err());
}
#[tokio::test]
async fn worker_restore_reports_retry_identity_and_reuses_it_across_explicit_tool_calls() {
let client = Arc::new(RecordingWorkspaceClient::default());
let tool = WorkspaceWorkerTool {
operation: WorkerOperation::Restore,
control: test_control(client.clone()),
runtime_worker_control: true,
};
let mut input = serde_json::json!({
"subject": {"kind":"runtime_worker", "runtime_id":"runtime-a", "worker_id":"worker-b"},
"expected_observation_token":"original-observation"
});
// The fixture deliberately returns malformed JSON shape: delivery may
// have succeeded, so retain the exact tuple rather than issuing new intent.
let first = tool
.execute(
&input.to_string(),
ToolExecutionContext::new("original-call", "batch", 0),
)
.await
.unwrap_err();
assert!(
first
.to_string()
.contains("request_id=worker-restore:original-call")
);
input["request_id"] = "worker-restore:original-call".into();
let second = tool
.execute(
&input.to_string(),
ToolExecutionContext::new("explicit-recovery-call", "batch", 1),
)
.await
.unwrap_err();
assert!(
second
.to_string()
.contains("request_id=worker-restore:original-call")
);
{
let requests = client.requests.lock().unwrap();
assert_eq!(requests.len(), 2);
assert_eq!(requests[0].body, requests[1].body);
}
input["request_id"] = "x".repeat(513).into();
assert!(
tool.execute(
&input.to_string(),
ToolExecutionContext::new("invalid", "batch", 2)
)
.await
.is_err()
);
assert_eq!(client.requests.lock().unwrap().len(), 2);
}
#[tokio::test]
async fn worker_spawn_forwards_typed_initial_submit_to_workspace_api() {
let client = Arc::new(RecordingWorkspaceClient::default());
+10 -5
View File
@@ -728,6 +728,7 @@ pub enum WorkspaceServerOperation {
WorkerControlRestore {
runtime_id: String,
worker_id: String,
request: server_api::WorkerRestoreRequest,
},
WorkerObservationSessions,
WorkerObservationCapture(server_api::WorkerObservationSubjectRef),
@@ -802,11 +803,15 @@ fn workspace_server_operation_request(
WorkspaceServerOperation::WorkerControlRestore {
runtime_id,
worker_id,
} => Ok(WorkspaceRequest::json(
WorkspaceRequestMethod::Post,
format!("{base}/worker-control/workers/{runtime_id}/{worker_id}/restore"),
"".to_string(),
)),
request,
} => workspace_server_json_request(
format!(
"{base}/worker-control/workers/{}/{}/restore",
encode_workspace_path_segment(&runtime_id),
encode_workspace_path_segment(&worker_id)
),
&request,
),
WorkspaceServerOperation::WorkerObservationSessions => Ok(WorkspaceRequest::get(format!(
"{base}/worker-observation/sessions"
))),
+191 -17
View File
@@ -242,6 +242,8 @@ pub struct InternalWorkerImplementationSummary {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct InternalWorkerSummary {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub restore_observation_token: Option<String>,
#[serde(flatten)]
pub worker: RuntimeWorkerRef,
pub host_id: String,
@@ -356,6 +358,7 @@ pub(crate) fn workspace_worker_summary(
workdir_attachments: Vec<server_api::WorkerWorkdirAttachmentSummary>,
) -> server_api::WorkerSummary {
server_api::WorkerSummary {
restore_observation_token: summary.restore_observation_token,
runtime_id: summary.worker.runtime_id,
worker_id: summary.worker.worker_id,
resource_key,
@@ -834,11 +837,17 @@ impl RuntimeRegistryError {
runtime_id,
code,
message,
} => Error::RuntimeOperationFailed {
runtime_id,
code,
message,
},
} => {
if code == "restore_observation_conflict" {
Error::RestoreObservationConflict
} else {
Error::RuntimeOperationFailed {
runtime_id,
code,
message,
}
}
}
}
}
}
@@ -938,7 +947,29 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
fn worker(&self, worker_id: &str) -> WorkerLookupResult;
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
fn coordinate_worker_restore(
&self,
_worker_id: &str,
_request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<
(
Option<InternalWorkerRestoreResult>,
Option<runtime_api::WorkerRestorePreparation>,
),
RuntimeDiagnostic,
> {
Err(diagnostic(
"worker_restore_coordination_unsupported",
HostDiagnosticSeverity::Error,
"Runtime does not support guarded Restore coordination",
))
}
fn restore_worker(
&self,
worker_id: &str,
_request: runtime_api::WorkerRestoreRequest,
) -> InternalWorkerRestoreResult {
InternalWorkerRestoreResult {
state: server_api::WorkerRestoreState::Rejected,
worker: None,
@@ -1602,16 +1633,39 @@ impl RuntimeRegistry {
Ok(worker)
}
pub fn coordinate_worker_restore(
&self,
worker: &RuntimeWorkerRef,
request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<
(
Option<InternalWorkerRestoreResult>,
Option<runtime_api::WorkerRestorePreparation>,
),
RuntimeRegistryError,
> {
validate_backend_identifier("runtime_id", &worker.runtime_id)?;
validate_backend_identifier("worker_id", &worker.worker_id)?;
self.runtime(&worker.runtime_id)?
.coordinate_worker_restore(&worker.worker_id, request)
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: worker.runtime_id.clone(),
code: error.code,
message: error.message,
})
}
pub fn restore_worker(
&self,
worker: &RuntimeWorkerRef,
request: runtime_api::WorkerRestoreRequest,
) -> Result<InternalWorkerRestoreResult, RuntimeRegistryError> {
let runtime_id = worker.runtime_id.as_str();
let worker_id = worker.worker_id.as_str();
validate_backend_identifier("runtime_id", runtime_id)?;
validate_backend_identifier("worker_id", worker_id)?;
let runtime = self.runtime(runtime_id)?;
Ok(runtime.restore_worker(worker_id))
Ok(runtime.restore_worker(worker_id, request))
}
pub fn replace_worker_workspace_api(
@@ -2372,6 +2426,7 @@ impl EmbeddedWorkerRuntime {
true,
);
InternalWorkerSummary {
restore_observation_token: summary.restore_observation_token.clone(),
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
host_id: self.host_id.clone(),
display_name: display.display_name.clone(),
@@ -2416,6 +2471,7 @@ impl EmbeddedWorkerRuntime {
true,
);
InternalWorkerSummary {
restore_observation_token: detail.restore_observation_token.clone(),
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
host_id: self.host_id.clone(),
display_name: display.display_name.clone(),
@@ -2582,7 +2638,48 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
}
}
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
fn coordinate_worker_restore(
&self,
worker_id: &str,
request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<
(
Option<InternalWorkerRestoreResult>,
Option<runtime_api::WorkerRestorePreparation>,
),
RuntimeDiagnostic,
> {
let worker_ref = self.worker_ref(worker_id).ok_or_else(|| {
diagnostic(
"worker_not_found",
HostDiagnosticSeverity::Warning,
"Unknown Worker",
)
})?;
let (result, preparation) = self
.runtime
.coordinate_worker_restore_operation(&worker_ref, request, None)
.map_err(|error| embedded_runtime_diagnostic(&error))?;
Ok((
result.map(|result| InternalWorkerRestoreResult {
state: result.state,
worker: result.worker.map(|detail| self.map_worker_detail(detail)),
diagnostics: match (result.reason_code, result.message) {
(Some(code), Some(message)) => {
vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)]
}
_ => Vec::new(),
},
}),
preparation,
))
}
fn restore_worker(
&self,
worker_id: &str,
request: runtime_api::WorkerRestoreRequest,
) -> InternalWorkerRestoreResult {
let Some(worker_ref) = self.worker_ref(worker_id) else {
return InternalWorkerRestoreResult {
state: server_api::WorkerRestoreState::Rejected,
@@ -2594,7 +2691,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
)],
};
};
match self.runtime.restore_worker_operation(&worker_ref) {
match self.runtime.restore_worker_operation(&worker_ref, request) {
Ok(result) => {
let diagnostics = match (result.reason_code, result.message) {
(Some(code), Some(message)) => {
@@ -2608,6 +2705,17 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
diagnostics,
}
}
Err(worker_runtime::error::RuntimeError::RestoreObservationConflict { .. }) => {
InternalWorkerRestoreResult {
state: server_api::WorkerRestoreState::Rejected,
worker: None,
diagnostics: vec![diagnostic(
"restore_observation_conflict",
HostDiagnosticSeverity::Warning,
"Restore observation changed; refresh before a new intent".to_string(),
)],
}
}
Err(err) => InternalWorkerRestoreResult {
state: server_api::WorkerRestoreState::ReconciliationRequired,
worker: None,
@@ -4375,6 +4483,7 @@ impl RemoteWorkerRuntime {
false,
);
InternalWorkerSummary {
restore_observation_token: summary.restore_observation_token.clone(),
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
host_id: self.host_id.clone(),
display_name: display.display_name.clone(),
@@ -4417,6 +4526,7 @@ impl RemoteWorkerRuntime {
false,
);
InternalWorkerSummary {
restore_observation_token: detail.restore_observation_token.clone(),
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
host_id: self.host_id.clone(),
display_name: display.display_name.clone(),
@@ -4708,17 +4818,58 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
}
}
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
fn coordinate_worker_restore(
&self,
worker_id: &str,
request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<
(
Option<InternalWorkerRestoreResult>,
Option<runtime_api::WorkerRestorePreparation>,
),
RuntimeDiagnostic,
> {
let worker_id = worker_id.to_string();
let response = self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.coordinate_worker_restore(worker_id, request).await },
)?;
let result = response
.result
.map(|result| -> Result<_, RuntimeDiagnostic> {
let worker = result
.worker
.map(runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>)
.transpose()?;
let state =
runtime_contract_convert::<_, server_api::WorkerRestoreState>(result.state)?;
Ok(InternalWorkerRestoreResult {
state,
worker: worker.map(|detail| self.map_worker_detail(detail)),
diagnostics: match (result.reason_code, result.message) {
(Some(code), Some(message)) => {
vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)]
}
_ => Vec::new(),
},
})
})
.transpose()?;
Ok((result, response.preparation))
}
fn restore_worker(
&self,
worker_id: &str,
request: runtime_api::WorkerRestoreRequest,
) -> InternalWorkerRestoreResult {
let worker_id_owned = worker_id.to_string();
match self
.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move {
client
.restore_worker(worker_id_owned, runtime_api::EmptyObjectRequest::default())
.await
},
move |client| async move { client.restore_worker(worker_id_owned, request).await },
)
.and_then(|value| {
let worker = value
@@ -5886,6 +6037,11 @@ fn sanitize_embedded_execution_detail(summary: &str, message: &str) -> String {
fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnostic {
match error {
EmbeddedRuntimeError::RestoreObservationConflict { .. } => diagnostic(
"restore_observation_conflict",
HostDiagnosticSeverity::Warning,
"Restore observation changed; refresh before a new intent".to_string(),
),
EmbeddedRuntimeError::RuntimeStopped => diagnostic(
"embedded_runtime_stopped",
HostDiagnosticSeverity::Warning,
@@ -6292,6 +6448,7 @@ fn worker_spawn_intent_label(intent: &WorkerSpawnIntent) -> &'static str {
pub fn placeholder_worker(host_id: impl Into<String>) -> InternalWorkerSummary {
let host_id = host_id.into();
InternalWorkerSummary {
restore_observation_token: None,
worker: RuntimeWorkerRef::new("placeholder", "worker-placeholder"),
host_id,
display_name: "Worker runtime actions are not implemented".to_string(),
@@ -6369,7 +6526,14 @@ mod tests {
)
.unwrap();
let result = provider.restore_worker("worker-transport-uncertain");
let result = provider.restore_worker(
"worker-transport-uncertain",
runtime_api::WorkerRestoreRequest {
expected_observation_token: "observation".into(),
request_id: "transport-uncertain".into(),
preparation: None,
},
);
assert_eq!(
result.state,
@@ -7052,6 +7216,7 @@ mod tests {
runtime_id: runtime_id.to_string(),
host_id: host_id.to_string(),
workers: vec![InternalWorkerSummary {
restore_observation_token: None,
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
host_id: host_id.to_string(),
display_name: label.to_string(),
@@ -7698,7 +7863,16 @@ mod tests {
let detail = registry.worker(&worker.worker).unwrap();
let json = serde_json::to_string(&(embedded_summary, worker, input, detail)).unwrap();
// The public observation generation is intentionally exposed, unlike
// private authentication tokens. Whitelist its key, not arbitrary token
// values or other token-bearing fields.
for summary in [&worker, &detail] {
let token = summary.restore_observation_token.as_ref().unwrap();
assert_eq!(uuid::Uuid::parse_str(token).unwrap().get_version_num(), 7);
}
let json = serde_json::to_string(&(embedded_summary, worker, input, detail))
.unwrap()
.replace("\"restore_observation_token\":", "\"restore_observation\":");
for forbidden in [
"/workspace/project",
"metadata.json",
+4
View File
@@ -139,6 +139,10 @@ pub enum Error {
RuntimeBindingFingerprintConflict { fingerprint: String },
#[error("Runtime binding was not found for {runtime_id}")]
RuntimeBindingNotFound { runtime_id: String },
#[error(
"restore_observation_conflict: Restore observation changed; refresh before a new intent"
)]
RestoreObservationConflict,
#[error("Repository conflict: {0}")]
RepositoryConflict(String),
#[error("Registry inconsistency: {0}")]
@@ -61,6 +61,16 @@ impl Drop for StoredResource {
}
impl BackendResourceBroker {
#[cfg(test)]
pub(crate) fn repository_ssh_access_handle_count(&self) -> usize {
self.resources
.lock()
.unwrap()
.values()
.filter(|resource| resource.handle.kind == BackendResourceKind::RepositorySshAccess)
.count()
}
pub fn issue_profile_source_archive_handle(
&self,
workspace_id: impl Into<String>,
File diff suppressed because it is too large Load Diff
@@ -250,7 +250,13 @@ async fn offline_handle_can_pin_but_deleted_handle_cannot_mutate_or_connect() {
);
assert!(matches!(
handle
.restore(&WorkerOperationContext::Backend)
.restore(
&WorkerOperationContext::Backend,
server_api::WorkerRestoreRequest {
expected_observation_token: "stale".into(),
request_id: "removed-handle".into(),
}
)
.await
.unwrap_err()
.error,
@@ -374,7 +380,17 @@ async fn worker_control_handle_rechecks_revoked_grants_for_each_operation() {
);
}
assert!(matches!(
handle.restore(&context).await.unwrap_err().error,
handle
.restore(
&context,
server_api::WorkerRestoreRequest {
expected_observation_token: "stale".into(),
request_id: "revoked-grant".into(),
}
)
.await
.unwrap_err()
.error,
Error::UnknownWorker { .. }
));
assert!(matches!(
@@ -682,7 +698,15 @@ async fn generic_management_aliases_cannot_bypass_worker_grants_or_target_bound_
);
assert!(
service
.runtime_worker_restore_alias(context(), "offline-runtime".into(), "subject".into())
.runtime_worker_restore_alias(
context(),
"offline-runtime".into(),
"subject".into(),
server_api::WorkerRestoreRequest {
expected_observation_token: "old".into(),
request_id: "unauthorized".into(),
}
)
.await
.is_err()
);
@@ -862,3 +886,696 @@ async fn browser_protocol_connection_and_cleanup_reject_cookie_origin_mismatch()
assert!(matches!(result.error, Error::WorkspacePermissionDenied(_)));
assert!(execution.protocol_methods().is_empty());
}
fn restore_request_for(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
request_id: &str,
) -> server_api::WorkerRestoreRequest {
server_api::WorkerRestoreRequest {
expected_observation_token: api
.runtime
.worker(worker)
.unwrap()
.restore_observation_token
.unwrap(),
request_id: request_id.into(),
}
}
#[tokio::test]
async fn restore_guard_rejects_stale_list_and_stopped_restored_stopped_aba() {
let workspace = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-guard-aba");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let original = restore_request_for(&api, &identity, "restore-original");
let stale_list = server_api::WorkerRestoreRequest {
request_id: "restore-stale-list".into(),
..original.clone()
};
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, original.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
let conflict = handle
.restore(&WorkerOperationContext::Backend, stale_list.clone())
.await
.unwrap_err();
assert!(matches!(conflict.error, Error::RestoreObservationConflict));
let typed = conflict.into_repository_api_error();
assert_eq!(
api_error_status(&Error::RestoreObservationConflict),
StatusCode::CONFLICT
);
assert_eq!(typed.error, "restore_observation_conflict");
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
assert_ne!(
original.expected_observation_token,
api.runtime
.worker(&identity)
.unwrap()
.restore_observation_token
.unwrap()
);
assert!(matches!(
handle
.restore(&WorkerOperationContext::Backend, stale_list)
.await
.unwrap_err()
.error,
Error::RestoreObservationConflict
));
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
assert!(
execution.contexts.lock().unwrap().is_empty(),
"conflict must not install execution"
);
// The original request is result replay, not a new intent; it stays accepted
// without resurrecting the Worker after a subsequent stop.
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, original)
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
assert!(execution.contexts.lock().unwrap().is_empty());
}
fn restore_test_workdir(
api: &WorkspaceApi,
id: &str,
source: WorkdirRegistrySource,
) -> WorkdirRegistryRecord {
WorkdirRegistryRecord {
workspace_id: api.workspace_id().into(),
workdir_id: id.into(),
display_name: None,
source,
creation_selector: None,
creation_ref: None,
creation_tree: None,
current_selector: None,
current_ref: None,
current_tree: None,
observed_at_epoch_seconds: None,
materialization_status: "present".into(),
cleanliness: "clean".into(),
created_at: TEST_CREATED_AT.into(),
updated_at: TEST_CREATED_AT.into(),
}
}
fn attach_restore_test_workdir(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
workdir: &WorkdirRegistryRecord,
) {
api.store
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
connection_id: String::new(),
workspace_id: api.workspace_id().into(),
worker: worker.clone(),
workdir_id: workdir.workdir_id.clone(),
alias: workdir.workdir_id.clone(),
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
linked_at: TEST_CREATED_AT.into(),
unlinked_at: None,
})
.unwrap();
}
fn displace_restore_singleton(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
replacement: &RuntimeWorkerRef,
) {
api.config_store.with_conn(|conn| {
conn.execute("UPDATE worker_create_reservations SET singleton_key='restore-test-singleton', singleton_generation=1 WHERE workspace_id=?1 AND worker_id=?2",
rusqlite::params![api.workspace_id(), worker.worker_id])?;
conn.execute("INSERT INTO worker_singleton_owners(workspace_id,singleton_key,runtime_id,worker_id,generation,created_at,updated_at) VALUES(?1,'restore-test-singleton',?2,?3,2,?4,?4)",
rusqlite::params![api.workspace_id(), replacement.runtime_id, replacement.worker_id, TEST_CREATED_AT])?;
Ok(())
}).unwrap();
assert!(
api.store
.require_current_worker_singleton_owner(api.workspace_id(), worker)
.is_err()
);
}
fn add_offline_restore_workdir(api: &WorkspaceApi, worker: &RuntimeWorkerRef) {
let record = restore_test_workdir(
api,
"restore-offline",
WorkdirRegistrySource::ExternalGrant {
grant_id: "restore-offline-grant".into(),
},
);
api.store
.create_external_workdir_grant(
&crate::store::ExternalWorkdirGrantRecord {
grant_id: "restore-offline-grant".into(),
workspace_id: api.workspace_id().into(),
workdir_id: record.workdir_id.clone(),
provider_instance_id: "offline-provider".into(),
display_name: "Offline grant".into(),
permissions: "read_only".into(),
created_by: "owner-account".into(),
created_at: TEST_CREATED_AT.into(),
expires_at: None,
generation: 1,
status: "offline".into(),
updated_at: TEST_CREATED_AT.into(),
},
&record,
)
.unwrap();
attach_restore_test_workdir(api, worker, &record);
}
#[tokio::test]
async fn restore_guard_stale_ssh_intent_does_not_issue_workspace_resource() {
let workspace = tempfile::tempdir().unwrap();
let api = test_api_with_remote_repository(workspace.path()).await;
let identity = spawn_ticket_check_source(&api, "restore-stale-ssh");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
let stale = restore_request_for(&api, &identity, "stale-ssh-intent");
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
api.repository_secrets
.generate_credential(
api.workspace_id(),
GenerateRepositorySshCredentialRequest {
operation_id: "restore-default-key".into(),
credential_id:
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(),
name: "Restore test default".into(),
},
"owner-account",
)
.unwrap();
let public_key = api
.repository_secrets
.credential_public_key(
api.workspace_id(),
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID,
)
.unwrap()
.unwrap()
.public_key;
api.repository_secrets
.put_host_trust(
api.workspace_id(),
PutRepositorySshHostTrustRequest {
operation_id: "restore-ssh-trust".into(),
host_trust_id: "restore-ssh-host".into(),
hostname: "example.invalid".into(),
port: 22,
host_key: public_key,
expected_revision: None,
},
"owner-account",
)
.unwrap();
let record = restore_test_workdir(
&api,
"restore-ssh-workdir",
WorkdirRegistrySource::Repository {
runtime_id: identity.runtime_id.clone(),
repository_id: test_repository_id(&api),
},
);
api.store.upsert_workdir_registry(&record).unwrap();
attach_restore_test_workdir(&api, &identity, &record);
// Prove this is a fully issuable SSH fixture; the old pre-admission path would mint another handle.
assert!(
repository_access_request_for_workdir(
&api,
&identity.runtime_id,
&record.workdir_id,
"restore-ssh-positive-control"
)
.unwrap()
.is_some()
);
let before = api.resource_broker.repository_ssh_access_handle_count();
assert!(before > 0);
let conflict = handle
.restore(&WorkerOperationContext::Backend, stale)
.await
.unwrap_err();
assert!(matches!(conflict.error, Error::RestoreObservationConflict));
assert_eq!(
api.resource_broker.repository_ssh_access_handle_count(),
before
);
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
}
#[tokio::test]
async fn restore_guard_historical_replay_ignores_new_eligibility_but_rechecks_auth() {
let workspace = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-historical");
let replacement = spawn_ticket_check_source(&api, "restore-replacement");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let request = restore_request_for(&api, &identity, "restore-historical-request");
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
displace_restore_singleton(&api, &identity, &replacement);
add_offline_restore_workdir(&api, &identity);
let before = execution.restore_operations.lock().unwrap().len();
let replay = handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap();
assert_eq!(replay.state, server_api::WorkerRestoreState::Accepted);
assert_eq!(
replay.worker.unwrap().state,
"stopped",
"never return the old idle receipt summary"
);
assert_eq!(execution.restore_operations.lock().unwrap().len(), before);
let projected = restore_runtime_worker_with_context(
api.clone(),
identity.runtime_id.clone(),
identity.worker_id.clone(),
WorkerOperationContext::Backend,
request.clone(),
)
.await
.unwrap()
.0;
assert_eq!(
projected.result.worker.unwrap().state,
"unavailable",
"current fenced projection, never old idle receipt"
);
let (mut browser, _) = browser_operation_context(&api).await;
let WorkerOperationContext::Browser { headers, .. } = &mut browser else {
unreachable!()
};
headers.insert(ORIGIN, "https://untrusted.example".parse().unwrap());
assert!(matches!(
handle.restore(&browser, request).await.unwrap_err().error,
Error::WorkspacePermissionDenied(_)
));
assert_eq!(execution.restore_operations.lock().unwrap().len(), before);
assert!(
handle
.restore(
&WorkerOperationContext::Backend,
restore_request_for(&api, &identity, "new-ineligible-intent")
)
.await
.is_err()
);
}
#[tokio::test]
async fn restore_guard_pending_recovery_ignores_new_eligibility_and_reuses_operation() {
let workspace = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-pending");
let replacement = spawn_ticket_check_source(&api, "restore-pending-replacement");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
*execution.restore_failure.lock().unwrap() = Some(
worker_runtime::execution::WorkerExecutionSpawnResult::Errored(
worker_runtime::execution::WorkerExecutionResult::rejected(
worker_runtime::execution::WorkerExecutionOperation::Restore,
"uncertain launch",
),
),
);
let request = restore_request_for(&api, &identity, "restore-pending-request");
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
let original = execution.restore_operations.lock().unwrap()[0];
displace_restore_singleton(&api, &identity, &replacement);
add_offline_restore_workdir(&api, &identity);
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request)
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(
execution.reconcile_operations.lock().unwrap().as_slice(),
&[original]
);
assert!(
execution
.restore_operations
.lock()
.unwrap()
.iter()
.all(|id| id == &original)
);
}
#[tokio::test]
async fn restore_guard_internal_intent_survives_client_store_restart_without_replacing_token() {
let workspace = tempfile::tempdir().unwrap();
let (mut api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-internal-restart");
WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id)
.unwrap()
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let observed = api.runtime.worker(&identity).unwrap();
*execution.restore_failure.lock().unwrap() = Some(
worker_runtime::execution::WorkerExecutionSpawnResult::Errored(
worker_runtime::execution::WorkerExecutionResult::rejected(
worker_runtime::execution::WorkerExecutionOperation::Restore,
"uncertain launch",
),
),
);
assert_eq!(
restore_internal_observed_worker(&api, &observed, "subject-restore".into())
.unwrap()
.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
let original = execution.restore_operations.lock().unwrap()[0];
let current = api.runtime.worker(&identity).unwrap();
assert_ne!(
observed.restore_observation_token,
current.restore_observation_token
);
// Reopen the durable caller journal, discarding all client-side memory of request identity.
api.config_store =
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
let pinned = api
.config_store
.pin_internal_worker_restore_intent(
api.workspace_id(),
&identity,
"subject-restore",
current.restore_observation_token.as_deref(),
None,
)
.unwrap();
assert_eq!(
pinned.expected_observation_token,
observed.restore_observation_token.unwrap()
);
assert_eq!(
restore_internal_observed_worker(&api, &current, "subject-restore".into())
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(
execution.reconcile_operations.lock().unwrap().as_slice(),
&[original]
);
assert!(
execution
.restore_operations
.lock()
.unwrap()
.iter()
.all(|id| id == &original)
);
}
#[tokio::test]
async fn restore_guard_workspace_interrupted_preparation_recovers_admitted_snapshot() {
let workspace = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-prepare-interrupted");
let replacement = spawn_ticket_check_source(&api, "restore-prepare-replacement");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let request = restore_request_for(&api, &identity, "restore-prepare-interrupted-request");
let preparation = runtime_api::WorkerRestorePreparation {
workspace_api: Some(
runtime_contract_restore(api.workspace_api_ref(&identity.runtime_id)).unwrap(),
),
workdir_attachments: Some(Vec::new()),
..Default::default()
};
let (result, retained) = api
.runtime
.coordinate_worker_restore(
&identity,
runtime_api::WorkerRestoreCoordinationRequest {
expected_observation_token: request.expected_observation_token.clone(),
request_id: request.request_id.clone(),
preparation: Some(preparation.clone()),
},
)
.unwrap();
assert!(result.is_none());
assert_eq!(retained, Some(preparation));
assert!(execution.restore_operations.lock().unwrap().is_empty());
// Model interruption between admission and completion, then changed current eligibility.
displace_restore_singleton(&api, &identity, &replacement);
add_offline_restore_workdir(&api, &identity);
let restored = handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap();
assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted);
assert_eq!(execution.restore_operations.lock().unwrap().len(), 1);
let original = execution.restore_operations.lock().unwrap()[0];
assert!(
execution.reconcile_operations.lock().unwrap().is_empty(),
"no execution existed to reconcile yet"
);
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request)
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(
execution.restore_operations.lock().unwrap().as_slice(),
&[original]
);
}
#[tokio::test]
async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_workdir_owner() {
let workspace = tempfile::tempdir().unwrap();
let mut config = test_server_config(workspace.path());
let source = server_api::RepositorySource {
kind: server_api::RepositorySourceKind::Ssh,
uri: "git@example.invalid:owner/repository.git".into(),
};
config.repositories[0].source_fingerprint =
crate::repository_source::repository_source_fingerprint(&source);
config.repositories[0].source = source;
config.repositories[0].path = None;
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
let execution = Arc::new(DeterministicExecutionBackend::default());
execution.accept_restores();
let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone())
.await
.unwrap();
let peer = WorkdirlessFixtureRuntime::default();
*peer.reject_repository_access.lock().unwrap() = true;
api.runtime.register_or_replace(peer.clone());
let identity = spawn_ticket_check_source(&api, "restore-cross-runtime-ssh");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let original_key = api
.repository_secrets
.generate_credential(
api.workspace_id(),
GenerateRepositorySshCredentialRequest {
operation_id: "cross-runtime-default-key".into(),
credential_id:
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(),
name: "Restore default".into(),
},
"owner-account",
)
.unwrap();
let public_key = api
.repository_secrets
.credential_public_key(api.workspace_id(), &original_key.credential_id)
.unwrap()
.unwrap()
.public_key;
api.repository_secrets
.put_host_trust(
api.workspace_id(),
PutRepositorySshHostTrustRequest {
operation_id: "cross-runtime-trust".into(),
host_trust_id: "cross-runtime-host".into(),
hostname: "example.invalid".into(),
port: 22,
host_key: public_key,
expected_revision: None,
},
"owner-account",
)
.unwrap();
let record = restore_test_workdir(
&api,
"cross-runtime-restore-workdir",
WorkdirRegistrySource::Repository {
runtime_id: WorkdirlessFixtureRuntime::RUNTIME_ID.into(),
repository_id: test_repository_id(&api),
},
);
peer.set_workdir_repository(&record.workdir_id, &test_repository_id(&api));
api.store.upsert_workdir_registry(&record).unwrap();
attach_restore_test_workdir(&api, &identity, &record);
let request = restore_request_for(&api, &identity, "cross-runtime-restore-request");
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
assert!(
execution.restore_operations.lock().unwrap().is_empty(),
"credential completion must precede execution"
);
let first = peer.repository_access_requests.lock().unwrap()[0].clone();
assert_eq!(
first.materialization.runtime_id,
WorkdirlessFixtureRuntime::RUNTIME_ID
);
assert_ne!(first.materialization.runtime_id, identity.runtime_id);
// Rotate current host trust while the accepted operation is uncertain. Its
// original credential/trust revisions, source and binding must remain pinned.
let alternate = api
.repository_secrets
.generate_credential(
api.workspace_id(),
GenerateRepositorySshCredentialRequest {
operation_id: "cross-runtime-alternate-key".into(),
credential_id: "cross-runtime-alternate".into(),
name: "Alternate".into(),
},
"owner-account",
)
.unwrap();
let alternate_public = api
.repository_secrets
.credential_public_key(api.workspace_id(), &alternate.credential_id)
.unwrap()
.unwrap()
.public_key;
let original_trust_revision = first
.materialization
.ssh
.as_ref()
.unwrap()
.host_trust_revision;
api.repository_secrets
.put_host_trust(
api.workspace_id(),
PutRepositorySshHostTrustRequest {
operation_id: "cross-runtime-trust-rotate".into(),
host_trust_id: "cross-runtime-host".into(),
hostname: "example.invalid".into(),
port: 22,
host_key: alternate_public,
expected_revision: Some(original_trust_revision),
},
"owner-account",
)
.unwrap();
*peer.reject_repository_access.lock().unwrap() = false;
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
let attempts = peer.repository_access_requests.lock().unwrap();
assert_eq!(attempts.len(), 2);
let second = &attempts[1];
assert_eq!(
second.materialization.operation_id,
first.materialization.operation_id
);
let mut normalized = second.clone();
let old_ssh = first.materialization.ssh.as_ref().unwrap();
let new_ssh = normalized.materialization.ssh.as_mut().unwrap();
assert_ne!(new_ssh.secret_resource.nonce, old_ssh.secret_resource.nonce);
new_ssh.secret_resource = old_ssh.secret_resource.clone();
new_ssh.expires_at_epoch_seconds = old_ssh.expires_at_epoch_seconds;
assert_eq!(normalized, first, "only opaque handle and expiry may renew");
drop(attempts);
assert_eq!(execution.restore_operations.lock().unwrap().len(), 1);
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request)
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(
peer.repository_access_requests.lock().unwrap().len(),
2,
"terminal receipt must not renew access"
);
}
@@ -291,10 +291,20 @@ impl WorkspaceWorker {
pub(super) async fn restore(
&self,
context: &WorkerOperationContext,
request: server_api::WorkerRestoreRequest,
) -> ApiResult<InternalWorkerRestoreResult> {
let _guard = self.authorize_operation(context, "restore").await?;
// Preserve singleton, Workdir, credentials, bindings and pending operation reconciliation.
self.api.restore_workspace_worker(&self.identity)
self.restore_internal(request)
}
/// Trusted Backend callers still supply an explicit observed intent. Runtime
/// admission is shared with browser/Tool callers; this is not an unguarded launch.
pub(super) fn restore_internal(
&self,
request: server_api::WorkerRestoreRequest,
) -> ApiResult<InternalWorkerRestoreResult> {
self.current_record()?;
self.api.restore_workspace_worker(&self.identity, request)
}
pub(super) async fn stop(
+91 -3
View File
@@ -28,7 +28,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore;
use crate::{Error, Result};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 79;
const LATEST_SCHEMA_VERSION: i64 = 80;
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings";
@@ -79,6 +79,8 @@ const ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME: &str =
const BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME: &str =
"durable terminal Backend Job Worker cleanup";
const WORKDIR_CONNECTION_ID_MIGRATION_NAME: &str = "durable Workdir attachment connection identity";
const WORKER_RESTORE_INTENTS_MIGRATION_NAME: &str =
"durable internal Worker Restore request identity";
const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7;
const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline";
const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#"
@@ -271,6 +273,11 @@ const MIGRATIONS: &[Migration] = &[
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME,
apply: migrate_workspace_config_v78_to_v79,
},
Migration {
version: 80,
name: WORKER_RESTORE_INTENTS_MIGRATION_NAME,
apply: migrate_worker_restore_intents_v79_to_v80,
},
];
#[derive(Clone, Copy)]
@@ -2167,6 +2174,48 @@ impl SqliteWorkspaceStore {
})
}
/// Client intent journal, not a lifecycle state machine: pin the public tuple
/// before dispatch and keep it unchanged until a terminal receipt is obtained.
pub(crate) fn pin_internal_worker_restore_intent(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
domain_key: &str,
observed_token: Option<&str>,
explicit_request_id: Option<&str>,
) -> Result<server_api::WorkerRestoreRequest> {
self.with_transaction(|tx| {
let existing: Option<(String, String)> = if let Some(id) = explicit_request_id {
tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4",
params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()?
} else {
tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND domain_key=?4 AND settled=0",
params![workspace_id, worker.runtime_id, worker.worker_id, domain_key], |row| Ok((row.get(0)?, row.get(1)?))).optional()?
};
if let Some((request_id, expected_observation_token)) = existing {
return Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token });
}
let observed_token = observed_token.ok_or(Error::RestoreObservationConflict)?;
let request_id = explicit_request_id.map(str::to_owned).unwrap_or_else(|| format!("{domain_key}:{observed_token}"));
tx.execute("INSERT INTO worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key,request_id,expected_token,settled) VALUES(?1,?2,?3,?4,?5,?6,0)",
params![workspace_id, worker.runtime_id, worker.worker_id, domain_key, request_id, observed_token])?;
Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token: observed_token.to_owned() })
})
}
pub(crate) fn settle_internal_worker_restore_intent(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
request_id: &str,
) -> Result<()> {
self.with_conn(|conn| {
conn.execute("UPDATE worker_restore_intents SET settled=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4",
params![workspace_id, worker.runtime_id, worker.worker_id, request_id])?;
Ok(())
})
}
pub(crate) fn get_workspace_memory_settings(
&self,
workspace_id: &str,
@@ -11248,6 +11297,7 @@ fn read_worker_registry_projection(
})?
} else {
SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string())
.map_err(|error| Error::InvalidInput(error.to_string()))?,
runtime_id: Some(worker.runtime_id.to_string()),
@@ -12013,6 +12063,24 @@ fn read_workdir_registry_record(
})
}
fn migrate_worker_restore_intents_v79_to_v80(conn: &Connection) -> Result<()> {
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
tx.execute_batch("CREATE TABLE worker_restore_intents (
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL,
settled INTEGER NOT NULL CHECK(settled IN (0,1)),
PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id),
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
tx.execute(
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(?1,?2)",
params![80_i64, WORKER_RESTORE_INTENTS_MIGRATION_NAME],
)?;
tx.commit()?;
Ok(())
}
fn prepare_connection(conn: &Connection) -> Result<()> {
configure_sqlite(conn)?;
let workspace_schema_version = current_schema_version(conn)?;
@@ -15094,6 +15162,15 @@ fn migrate_workdir_connection_id_v77_to_v78(conn: &Connection) -> Result<()> {
}
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
conn.execute_batch("CREATE TABLE worker_restore_intents (
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL,
settled INTEGER NOT NULL CHECK(settled IN (0,1)),
PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id),
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
conn.execute_batch(include_str!("latest_schema.sql"))?;
conn.execute_batch(
@@ -16202,6 +16279,7 @@ mod tests {
}
fn downgrade_schema_66_ticket_and_workdir_authority(conn: &Connection) -> Result<()> {
conn.execute_batch("DROP TABLE IF EXISTS worker_restore_intents;")?;
conn.execute_batch(
"DROP TABLE backend_job_deliveries;
DROP TABLE backend_job_attempts;
@@ -17709,6 +17787,10 @@ mod tests {
version: 79,
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()
},
WorkspaceSchemaMigrationStep {
version: 80,
name: WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()
},
]
);
@@ -17798,6 +17880,7 @@ mod tests {
),
(78, WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string()),
(79, WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()),
(80, WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -18149,7 +18232,7 @@ mod tests {
.collect::<Vec<_>>(),
vec![
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
73, 74, 75, 76, 77, 78, 79
73, 74, 75, 76, 77, 78, 79, 80
]
);
SqliteWorkspaceStore::migrate_database(&path).unwrap();
@@ -18158,7 +18241,7 @@ mod tests {
current_schema_version(&conn).unwrap(),
LATEST_SCHEMA_VERSION
);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 30);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 31);
assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap());
assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap());
assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap());
@@ -21860,6 +21943,7 @@ INSERT INTO worker_registry (
for record in [&runtime_a, &runtime_b] {
let observed = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("shared-local-id").unwrap(),
runtime_id: Some(record.worker.runtime_id.clone()),
resource_key: None,
@@ -21996,6 +22080,7 @@ INSERT INTO worker_registry (
Some("W-1")
);
let observed = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("known").unwrap(),
runtime_id: Some("embedded".to_string()),
resource_key: None,
@@ -22016,6 +22101,7 @@ INSERT INTO worker_registry (
};
store.upsert_worker_registry(&other_catalog).unwrap();
let other_observed = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("other").unwrap(),
runtime_id: Some("runtime-other".to_string()),
display_name: Some("Other Runtime Worker".to_string()),
@@ -22025,6 +22111,7 @@ INSERT INTO worker_registry (
.apply_worker_registry_observation("local-dev", "runtime-other", &other_observed, "2")
.unwrap();
let orphan = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("orphan").unwrap(),
..observed.clone()
};
@@ -22190,6 +22277,7 @@ INSERT INTO worker_registry (
.is_none()
);
let event = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("removed").unwrap(),
runtime_id: Some("embedded".to_string()),
resource_key: None,
@@ -76,6 +76,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"worker_registry_projection_diagnostics",
"worker_registry_projection_removals",
"worker_removal_operations",
"worker_restore_intents",
"worker_retention_audit_events",
"worker_session_archive_observe_grants",
"worker_session_archives",
@@ -425,6 +425,7 @@ fn project_registry_worker(
worker
} else {
SubscriptionWorker {
restore_observation_token: None,
worker_id: protocol::subscription::SubscriptionWorkerId::new(
record.registry.worker.worker_id.clone(),
)
@@ -600,6 +601,7 @@ mod tests {
#[test]
fn external_workdir_attachment_does_not_remove_workspace_worker() {
let mut worker = SubscriptionWorker {
restore_observation_token: None,
worker_id: worker_id(),
runtime_id: Some("runtime-1".to_string()),
resource_key: Some("W-1".to_string()),