Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: guarded Worker Restore from shared Web actions (T-701)

Browse Source
This commit is contained in:
Hare
2026-10-06 17:45:54 +09:00
parent 1a6d391f83
commit fdce12a47b
54 changed files with 7214 additions and 527 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/hosts.rs
+191 -17
View File
@@ -242,6 +242,8 @@ pub struct InternalWorkerImplementationSummary {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct InternalWorkerSummary {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub restore_observation_token: Option<String>,
#[serde(flatten)]
pub worker: RuntimeWorkerRef,
pub host_id: String,
@@ -356,6 +358,7 @@ pub(crate) fn workspace_worker_summary(
workdir_attachments: Vec<server_api::WorkerWorkdirAttachmentSummary>,
) -> server_api::WorkerSummary {
server_api::WorkerSummary {
restore_observation_token: summary.restore_observation_token,
runtime_id: summary.worker.runtime_id,
worker_id: summary.worker.worker_id,
resource_key,
@@ -834,11 +837,17 @@ impl RuntimeRegistryError {
runtime_id,
code,
message,
} => Error::RuntimeOperationFailed {
runtime_id,
code,
message,
},
} => {
if code == "restore_observation_conflict" {
Error::RestoreObservationConflict
} else {
Error::RuntimeOperationFailed {
runtime_id,
code,
message,
}
}
}
}
}
}
@@ -938,7 +947,29 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
fn worker(&self, worker_id: &str) -> WorkerLookupResult;
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
fn coordinate_worker_restore(
&self,
_worker_id: &str,
_request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<
(
Option<InternalWorkerRestoreResult>,
Option<runtime_api::WorkerRestorePreparation>,
),
RuntimeDiagnostic,
> {
Err(diagnostic(
"worker_restore_coordination_unsupported",
HostDiagnosticSeverity::Error,
"Runtime does not support guarded Restore coordination",
))
}
fn restore_worker(
&self,
worker_id: &str,
_request: runtime_api::WorkerRestoreRequest,
) -> InternalWorkerRestoreResult {
InternalWorkerRestoreResult {
state: server_api::WorkerRestoreState::Rejected,
worker: None,
@@ -1602,16 +1633,39 @@ impl RuntimeRegistry {
Ok(worker)
}
pub fn coordinate_worker_restore(
&self,
worker: &RuntimeWorkerRef,
request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<
(
Option<InternalWorkerRestoreResult>,
Option<runtime_api::WorkerRestorePreparation>,
),
RuntimeRegistryError,
> {
validate_backend_identifier("runtime_id", &worker.runtime_id)?;
validate_backend_identifier("worker_id", &worker.worker_id)?;
self.runtime(&worker.runtime_id)?
.coordinate_worker_restore(&worker.worker_id, request)
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: worker.runtime_id.clone(),
code: error.code,
message: error.message,
})
}
pub fn restore_worker(
&self,
worker: &RuntimeWorkerRef,
request: runtime_api::WorkerRestoreRequest,
) -> Result<InternalWorkerRestoreResult, RuntimeRegistryError> {
let runtime_id = worker.runtime_id.as_str();
let worker_id = worker.worker_id.as_str();
validate_backend_identifier("runtime_id", runtime_id)?;
validate_backend_identifier("worker_id", worker_id)?;
let runtime = self.runtime(runtime_id)?;
Ok(runtime.restore_worker(worker_id))
Ok(runtime.restore_worker(worker_id, request))
}
pub fn replace_worker_workspace_api(
@@ -2372,6 +2426,7 @@ impl EmbeddedWorkerRuntime {
true,
);
InternalWorkerSummary {
restore_observation_token: summary.restore_observation_token.clone(),
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
host_id: self.host_id.clone(),
display_name: display.display_name.clone(),
@@ -2416,6 +2471,7 @@ impl EmbeddedWorkerRuntime {
true,
);
InternalWorkerSummary {
restore_observation_token: detail.restore_observation_token.clone(),
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
host_id: self.host_id.clone(),
display_name: display.display_name.clone(),
@@ -2582,7 +2638,48 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
}
}
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
fn coordinate_worker_restore(
&self,
worker_id: &str,
request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<
(
Option<InternalWorkerRestoreResult>,
Option<runtime_api::WorkerRestorePreparation>,
),
RuntimeDiagnostic,
> {
let worker_ref = self.worker_ref(worker_id).ok_or_else(|| {
diagnostic(
"worker_not_found",
HostDiagnosticSeverity::Warning,
"Unknown Worker",
)
})?;
let (result, preparation) = self
.runtime
.coordinate_worker_restore_operation(&worker_ref, request, None)
.map_err(|error| embedded_runtime_diagnostic(&error))?;
Ok((
result.map(|result| InternalWorkerRestoreResult {
state: result.state,
worker: result.worker.map(|detail| self.map_worker_detail(detail)),
diagnostics: match (result.reason_code, result.message) {
(Some(code), Some(message)) => {
vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)]
}
_ => Vec::new(),
},
}),
preparation,
))
}
fn restore_worker(
&self,
worker_id: &str,
request: runtime_api::WorkerRestoreRequest,
) -> InternalWorkerRestoreResult {
let Some(worker_ref) = self.worker_ref(worker_id) else {
return InternalWorkerRestoreResult {
state: server_api::WorkerRestoreState::Rejected,
@@ -2594,7 +2691,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
)],
};
};
match self.runtime.restore_worker_operation(&worker_ref) {
match self.runtime.restore_worker_operation(&worker_ref, request) {
Ok(result) => {
let diagnostics = match (result.reason_code, result.message) {
(Some(code), Some(message)) => {
@@ -2608,6 +2705,17 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
diagnostics,
}
}
Err(worker_runtime::error::RuntimeError::RestoreObservationConflict { .. }) => {
InternalWorkerRestoreResult {
state: server_api::WorkerRestoreState::Rejected,
worker: None,
diagnostics: vec![diagnostic(
"restore_observation_conflict",
HostDiagnosticSeverity::Warning,
"Restore observation changed; refresh before a new intent".to_string(),
)],
}
}
Err(err) => InternalWorkerRestoreResult {
state: server_api::WorkerRestoreState::ReconciliationRequired,
worker: None,
@@ -4375,6 +4483,7 @@ impl RemoteWorkerRuntime {
false,
);
InternalWorkerSummary {
restore_observation_token: summary.restore_observation_token.clone(),
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
host_id: self.host_id.clone(),
display_name: display.display_name.clone(),
@@ -4417,6 +4526,7 @@ impl RemoteWorkerRuntime {
false,
);
InternalWorkerSummary {
restore_observation_token: detail.restore_observation_token.clone(),
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
host_id: self.host_id.clone(),
display_name: display.display_name.clone(),
@@ -4708,17 +4818,58 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
}
}
fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult {
fn coordinate_worker_restore(
&self,
worker_id: &str,
request: runtime_api::WorkerRestoreCoordinationRequest,
) -> Result<
(
Option<InternalWorkerRestoreResult>,
Option<runtime_api::WorkerRestorePreparation>,
),
RuntimeDiagnostic,
> {
let worker_id = worker_id.to_string();
let response = self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.coordinate_worker_restore(worker_id, request).await },
)?;
let result = response
.result
.map(|result| -> Result<_, RuntimeDiagnostic> {
let worker = result
.worker
.map(runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>)
.transpose()?;
let state =
runtime_contract_convert::<_, server_api::WorkerRestoreState>(result.state)?;
Ok(InternalWorkerRestoreResult {
state,
worker: worker.map(|detail| self.map_worker_detail(detail)),
diagnostics: match (result.reason_code, result.message) {
(Some(code), Some(message)) => {
vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)]
}
_ => Vec::new(),
},
})
})
.transpose()?;
Ok((result, response.preparation))
}
fn restore_worker(
&self,
worker_id: &str,
request: runtime_api::WorkerRestoreRequest,
) -> InternalWorkerRestoreResult {
let worker_id_owned = worker_id.to_string();
match self
.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move {
client
.restore_worker(worker_id_owned, runtime_api::EmptyObjectRequest::default())
.await
},
move |client| async move { client.restore_worker(worker_id_owned, request).await },
)
.and_then(|value| {
let worker = value
@@ -5886,6 +6037,11 @@ fn sanitize_embedded_execution_detail(summary: &str, message: &str) -> String {
fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnostic {
match error {
EmbeddedRuntimeError::RestoreObservationConflict { .. } => diagnostic(
"restore_observation_conflict",
HostDiagnosticSeverity::Warning,
"Restore observation changed; refresh before a new intent".to_string(),
),
EmbeddedRuntimeError::RuntimeStopped => diagnostic(
"embedded_runtime_stopped",
HostDiagnosticSeverity::Warning,
@@ -6292,6 +6448,7 @@ fn worker_spawn_intent_label(intent: &WorkerSpawnIntent) -> &'static str {
pub fn placeholder_worker(host_id: impl Into<String>) -> InternalWorkerSummary {
let host_id = host_id.into();
InternalWorkerSummary {
restore_observation_token: None,
worker: RuntimeWorkerRef::new("placeholder", "worker-placeholder"),
host_id,
display_name: "Worker runtime actions are not implemented".to_string(),
@@ -6369,7 +6526,14 @@ mod tests {
)
.unwrap();
let result = provider.restore_worker("worker-transport-uncertain");
let result = provider.restore_worker(
"worker-transport-uncertain",
runtime_api::WorkerRestoreRequest {
expected_observation_token: "observation".into(),
request_id: "transport-uncertain".into(),
preparation: None,
},
);
assert_eq!(
result.state,
@@ -7052,6 +7216,7 @@ mod tests {
runtime_id: runtime_id.to_string(),
host_id: host_id.to_string(),
workers: vec![InternalWorkerSummary {
restore_observation_token: None,
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
host_id: host_id.to_string(),
display_name: label.to_string(),
@@ -7698,7 +7863,16 @@ mod tests {
let detail = registry.worker(&worker.worker).unwrap();
let json = serde_json::to_string(&(embedded_summary, worker, input, detail)).unwrap();
// The public observation generation is intentionally exposed, unlike
// private authentication tokens. Whitelist its key, not arbitrary token
// values or other token-bearing fields.
for summary in [&worker, &detail] {
let token = summary.restore_observation_token.as_ref().unwrap();
assert_eq!(uuid::Uuid::parse_str(token).unwrap().get_version_num(), 7);
}
let json = serde_json::to_string(&(embedded_summary, worker, input, detail))
.unwrap()
.replace("\"restore_observation_token\":", "\"restore_observation\":");
for forbidden in [
"/workspace/project",
"metadata.json",
crates/workspace-server/src/lib.rs
+4
View File
@@ -139,6 +139,10 @@ pub enum Error {
RuntimeBindingFingerprintConflict { fingerprint: String },
#[error("Runtime binding was not found for {runtime_id}")]
RuntimeBindingNotFound { runtime_id: String },
#[error(
"restore_observation_conflict: Restore observation changed; refresh before a new intent"
)]
RestoreObservationConflict,
#[error("Repository conflict: {0}")]
RepositoryConflict(String),
#[error("Registry inconsistency: {0}")]
crates/workspace-server/src/resource_broker.rs
+10
View File
@@ -61,6 +61,16 @@ impl Drop for StoredResource {
}
impl BackendResourceBroker {
#[cfg(test)]
pub(crate) fn repository_ssh_access_handle_count(&self) -> usize {
self.resources
.lock()
.unwrap()
.values()
.filter(|resource| resource.handle.kind == BackendResourceKind::RepositorySshAccess)
.count()
}
pub fn issue_profile_source_archive_handle(
&self,
workspace_id: impl Into<String>,
crates/workspace-server/src/server.rs
+642 -185
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/server/tests/worker_operations_tests.rs
+720 -3
View File
@@ -250,7 +250,13 @@ async fn offline_handle_can_pin_but_deleted_handle_cannot_mutate_or_connect() {
);
assert!(matches!(
handle
.restore(&WorkerOperationContext::Backend)
.restore(
&WorkerOperationContext::Backend,
server_api::WorkerRestoreRequest {
expected_observation_token: "stale".into(),
request_id: "removed-handle".into(),
}
)
.await
.unwrap_err()
.error,
@@ -374,7 +380,17 @@ async fn worker_control_handle_rechecks_revoked_grants_for_each_operation() {
);
}
assert!(matches!(
handle.restore(&context).await.unwrap_err().error,
handle
.restore(
&context,
server_api::WorkerRestoreRequest {
expected_observation_token: "stale".into(),
request_id: "revoked-grant".into(),
}
)
.await
.unwrap_err()
.error,
Error::UnknownWorker { .. }
));
assert!(matches!(
@@ -682,7 +698,15 @@ async fn generic_management_aliases_cannot_bypass_worker_grants_or_target_bound_
);
assert!(
service
.runtime_worker_restore_alias(context(), "offline-runtime".into(), "subject".into())
.runtime_worker_restore_alias(
context(),
"offline-runtime".into(),
"subject".into(),
server_api::WorkerRestoreRequest {
expected_observation_token: "old".into(),
request_id: "unauthorized".into(),
}
)
.await
.is_err()
);
@@ -862,3 +886,696 @@ async fn browser_protocol_connection_and_cleanup_reject_cookie_origin_mismatch()
assert!(matches!(result.error, Error::WorkspacePermissionDenied(_)));
assert!(execution.protocol_methods().is_empty());
}
fn restore_request_for(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
request_id: &str,
) -> server_api::WorkerRestoreRequest {
server_api::WorkerRestoreRequest {
expected_observation_token: api
.runtime
.worker(worker)
.unwrap()
.restore_observation_token
.unwrap(),
request_id: request_id.into(),
}
}
#[tokio::test]
async fn restore_guard_rejects_stale_list_and_stopped_restored_stopped_aba() {
let workspace = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-guard-aba");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let original = restore_request_for(&api, &identity, "restore-original");
let stale_list = server_api::WorkerRestoreRequest {
request_id: "restore-stale-list".into(),
..original.clone()
};
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, original.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
let conflict = handle
.restore(&WorkerOperationContext::Backend, stale_list.clone())
.await
.unwrap_err();
assert!(matches!(conflict.error, Error::RestoreObservationConflict));
let typed = conflict.into_repository_api_error();
assert_eq!(
api_error_status(&Error::RestoreObservationConflict),
StatusCode::CONFLICT
);
assert_eq!(typed.error, "restore_observation_conflict");
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
assert_ne!(
original.expected_observation_token,
api.runtime
.worker(&identity)
.unwrap()
.restore_observation_token
.unwrap()
);
assert!(matches!(
handle
.restore(&WorkerOperationContext::Backend, stale_list)
.await
.unwrap_err()
.error,
Error::RestoreObservationConflict
));
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
assert!(
execution.contexts.lock().unwrap().is_empty(),
"conflict must not install execution"
);
// The original request is result replay, not a new intent; it stays accepted
// without resurrecting the Worker after a subsequent stop.
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, original)
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
assert!(execution.contexts.lock().unwrap().is_empty());
}
fn restore_test_workdir(
api: &WorkspaceApi,
id: &str,
source: WorkdirRegistrySource,
) -> WorkdirRegistryRecord {
WorkdirRegistryRecord {
workspace_id: api.workspace_id().into(),
workdir_id: id.into(),
display_name: None,
source,
creation_selector: None,
creation_ref: None,
creation_tree: None,
current_selector: None,
current_ref: None,
current_tree: None,
observed_at_epoch_seconds: None,
materialization_status: "present".into(),
cleanliness: "clean".into(),
created_at: TEST_CREATED_AT.into(),
updated_at: TEST_CREATED_AT.into(),
}
}
fn attach_restore_test_workdir(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
workdir: &WorkdirRegistryRecord,
) {
api.store
.attach_worker_workdir(&WorkerWorkdirLinkRecord {
connection_id: String::new(),
workspace_id: api.workspace_id().into(),
worker: worker.clone(),
workdir_id: workdir.workdir_id.clone(),
alias: workdir.workdir_id.clone(),
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
linked_at: TEST_CREATED_AT.into(),
unlinked_at: None,
})
.unwrap();
}
fn displace_restore_singleton(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
replacement: &RuntimeWorkerRef,
) {
api.config_store.with_conn(|conn| {
conn.execute("UPDATE worker_create_reservations SET singleton_key='restore-test-singleton', singleton_generation=1 WHERE workspace_id=?1 AND worker_id=?2",
rusqlite::params![api.workspace_id(), worker.worker_id])?;
conn.execute("INSERT INTO worker_singleton_owners(workspace_id,singleton_key,runtime_id,worker_id,generation,created_at,updated_at) VALUES(?1,'restore-test-singleton',?2,?3,2,?4,?4)",
rusqlite::params![api.workspace_id(), replacement.runtime_id, replacement.worker_id, TEST_CREATED_AT])?;
Ok(())
}).unwrap();
assert!(
api.store
.require_current_worker_singleton_owner(api.workspace_id(), worker)
.is_err()
);
}
fn add_offline_restore_workdir(api: &WorkspaceApi, worker: &RuntimeWorkerRef) {
let record = restore_test_workdir(
api,
"restore-offline",
WorkdirRegistrySource::ExternalGrant {
grant_id: "restore-offline-grant".into(),
},
);
api.store
.create_external_workdir_grant(
&crate::store::ExternalWorkdirGrantRecord {
grant_id: "restore-offline-grant".into(),
workspace_id: api.workspace_id().into(),
workdir_id: record.workdir_id.clone(),
provider_instance_id: "offline-provider".into(),
display_name: "Offline grant".into(),
permissions: "read_only".into(),
created_by: "owner-account".into(),
created_at: TEST_CREATED_AT.into(),
expires_at: None,
generation: 1,
status: "offline".into(),
updated_at: TEST_CREATED_AT.into(),
},
&record,
)
.unwrap();
attach_restore_test_workdir(api, worker, &record);
}
#[tokio::test]
async fn restore_guard_stale_ssh_intent_does_not_issue_workspace_resource() {
let workspace = tempfile::tempdir().unwrap();
let api = test_api_with_remote_repository(workspace.path()).await;
let identity = spawn_ticket_check_source(&api, "restore-stale-ssh");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
let stale = restore_request_for(&api, &identity, "stale-ssh-intent");
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
api.repository_secrets
.generate_credential(
api.workspace_id(),
GenerateRepositorySshCredentialRequest {
operation_id: "restore-default-key".into(),
credential_id:
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(),
name: "Restore test default".into(),
},
"owner-account",
)
.unwrap();
let public_key = api
.repository_secrets
.credential_public_key(
api.workspace_id(),
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID,
)
.unwrap()
.unwrap()
.public_key;
api.repository_secrets
.put_host_trust(
api.workspace_id(),
PutRepositorySshHostTrustRequest {
operation_id: "restore-ssh-trust".into(),
host_trust_id: "restore-ssh-host".into(),
hostname: "example.invalid".into(),
port: 22,
host_key: public_key,
expected_revision: None,
},
"owner-account",
)
.unwrap();
let record = restore_test_workdir(
&api,
"restore-ssh-workdir",
WorkdirRegistrySource::Repository {
runtime_id: identity.runtime_id.clone(),
repository_id: test_repository_id(&api),
},
);
api.store.upsert_workdir_registry(&record).unwrap();
attach_restore_test_workdir(&api, &identity, &record);
// Prove this is a fully issuable SSH fixture; the old pre-admission path would mint another handle.
assert!(
repository_access_request_for_workdir(
&api,
&identity.runtime_id,
&record.workdir_id,
"restore-ssh-positive-control"
)
.unwrap()
.is_some()
);
let before = api.resource_broker.repository_ssh_access_handle_count();
assert!(before > 0);
let conflict = handle
.restore(&WorkerOperationContext::Backend, stale)
.await
.unwrap_err();
assert!(matches!(conflict.error, Error::RestoreObservationConflict));
assert_eq!(
api.resource_broker.repository_ssh_access_handle_count(),
before
);
assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped");
}
#[tokio::test]
async fn restore_guard_historical_replay_ignores_new_eligibility_but_rechecks_auth() {
let workspace = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-historical");
let replacement = spawn_ticket_check_source(&api, "restore-replacement");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let request = restore_request_for(&api, &identity, "restore-historical-request");
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
displace_restore_singleton(&api, &identity, &replacement);
add_offline_restore_workdir(&api, &identity);
let before = execution.restore_operations.lock().unwrap().len();
let replay = handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap();
assert_eq!(replay.state, server_api::WorkerRestoreState::Accepted);
assert_eq!(
replay.worker.unwrap().state,
"stopped",
"never return the old idle receipt summary"
);
assert_eq!(execution.restore_operations.lock().unwrap().len(), before);
let projected = restore_runtime_worker_with_context(
api.clone(),
identity.runtime_id.clone(),
identity.worker_id.clone(),
WorkerOperationContext::Backend,
request.clone(),
)
.await
.unwrap()
.0;
assert_eq!(
projected.result.worker.unwrap().state,
"unavailable",
"current fenced projection, never old idle receipt"
);
let (mut browser, _) = browser_operation_context(&api).await;
let WorkerOperationContext::Browser { headers, .. } = &mut browser else {
unreachable!()
};
headers.insert(ORIGIN, "https://untrusted.example".parse().unwrap());
assert!(matches!(
handle.restore(&browser, request).await.unwrap_err().error,
Error::WorkspacePermissionDenied(_)
));
assert_eq!(execution.restore_operations.lock().unwrap().len(), before);
assert!(
handle
.restore(
&WorkerOperationContext::Backend,
restore_request_for(&api, &identity, "new-ineligible-intent")
)
.await
.is_err()
);
}
#[tokio::test]
async fn restore_guard_pending_recovery_ignores_new_eligibility_and_reuses_operation() {
let workspace = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-pending");
let replacement = spawn_ticket_check_source(&api, "restore-pending-replacement");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
*execution.restore_failure.lock().unwrap() = Some(
worker_runtime::execution::WorkerExecutionSpawnResult::Errored(
worker_runtime::execution::WorkerExecutionResult::rejected(
worker_runtime::execution::WorkerExecutionOperation::Restore,
"uncertain launch",
),
),
);
let request = restore_request_for(&api, &identity, "restore-pending-request");
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
let original = execution.restore_operations.lock().unwrap()[0];
displace_restore_singleton(&api, &identity, &replacement);
add_offline_restore_workdir(&api, &identity);
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request)
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(
execution.reconcile_operations.lock().unwrap().as_slice(),
&[original]
);
assert!(
execution
.restore_operations
.lock()
.unwrap()
.iter()
.all(|id| id == &original)
);
}
#[tokio::test]
async fn restore_guard_internal_intent_survives_client_store_restart_without_replacing_token() {
let workspace = tempfile::tempdir().unwrap();
let (mut api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-internal-restart");
WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id)
.unwrap()
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let observed = api.runtime.worker(&identity).unwrap();
*execution.restore_failure.lock().unwrap() = Some(
worker_runtime::execution::WorkerExecutionSpawnResult::Errored(
worker_runtime::execution::WorkerExecutionResult::rejected(
worker_runtime::execution::WorkerExecutionOperation::Restore,
"uncertain launch",
),
),
);
assert_eq!(
restore_internal_observed_worker(&api, &observed, "subject-restore".into())
.unwrap()
.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
let original = execution.restore_operations.lock().unwrap()[0];
let current = api.runtime.worker(&identity).unwrap();
assert_ne!(
observed.restore_observation_token,
current.restore_observation_token
);
// Reopen the durable caller journal, discarding all client-side memory of request identity.
api.config_store =
Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap());
let pinned = api
.config_store
.pin_internal_worker_restore_intent(
api.workspace_id(),
&identity,
"subject-restore",
current.restore_observation_token.as_deref(),
None,
)
.unwrap();
assert_eq!(
pinned.expected_observation_token,
observed.restore_observation_token.unwrap()
);
assert_eq!(
restore_internal_observed_worker(&api, &current, "subject-restore".into())
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(
execution.reconcile_operations.lock().unwrap().as_slice(),
&[original]
);
assert!(
execution
.restore_operations
.lock()
.unwrap()
.iter()
.all(|id| id == &original)
);
}
#[tokio::test]
async fn restore_guard_workspace_interrupted_preparation_recovers_admitted_snapshot() {
let workspace = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(workspace.path()).await;
execution.accept_restores();
let identity = spawn_ticket_check_source(&api, "restore-prepare-interrupted");
let replacement = spawn_ticket_check_source(&api, "restore-prepare-replacement");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let request = restore_request_for(&api, &identity, "restore-prepare-interrupted-request");
let preparation = runtime_api::WorkerRestorePreparation {
workspace_api: Some(
runtime_contract_restore(api.workspace_api_ref(&identity.runtime_id)).unwrap(),
),
workdir_attachments: Some(Vec::new()),
..Default::default()
};
let (result, retained) = api
.runtime
.coordinate_worker_restore(
&identity,
runtime_api::WorkerRestoreCoordinationRequest {
expected_observation_token: request.expected_observation_token.clone(),
request_id: request.request_id.clone(),
preparation: Some(preparation.clone()),
},
)
.unwrap();
assert!(result.is_none());
assert_eq!(retained, Some(preparation));
assert!(execution.restore_operations.lock().unwrap().is_empty());
// Model interruption between admission and completion, then changed current eligibility.
displace_restore_singleton(&api, &identity, &replacement);
add_offline_restore_workdir(&api, &identity);
let restored = handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap();
assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted);
assert_eq!(execution.restore_operations.lock().unwrap().len(), 1);
let original = execution.restore_operations.lock().unwrap()[0];
assert!(
execution.reconcile_operations.lock().unwrap().is_empty(),
"no execution existed to reconcile yet"
);
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request)
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(
execution.restore_operations.lock().unwrap().as_slice(),
&[original]
);
}
#[tokio::test]
async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_workdir_owner() {
let workspace = tempfile::tempdir().unwrap();
let mut config = test_server_config(workspace.path());
let source = server_api::RepositorySource {
kind: server_api::RepositorySourceKind::Ssh,
uri: "git@example.invalid:owner/repository.git".into(),
};
config.repositories[0].source_fingerprint =
crate::repository_source::repository_source_fingerprint(&source);
config.repositories[0].source = source;
config.repositories[0].path = None;
let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap();
let execution = Arc::new(DeterministicExecutionBackend::default());
execution.accept_restores();
let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone())
.await
.unwrap();
let peer = WorkdirlessFixtureRuntime::default();
*peer.reject_repository_access.lock().unwrap() = true;
api.runtime.register_or_replace(peer.clone());
let identity = spawn_ticket_check_source(&api, "restore-cross-runtime-ssh");
let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap();
handle
.stop(&WorkerOperationContext::Backend, operation_lifecycle())
.await
.unwrap();
let original_key = api
.repository_secrets
.generate_credential(
api.workspace_id(),
GenerateRepositorySshCredentialRequest {
operation_id: "cross-runtime-default-key".into(),
credential_id:
crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(),
name: "Restore default".into(),
},
"owner-account",
)
.unwrap();
let public_key = api
.repository_secrets
.credential_public_key(api.workspace_id(), &original_key.credential_id)
.unwrap()
.unwrap()
.public_key;
api.repository_secrets
.put_host_trust(
api.workspace_id(),
PutRepositorySshHostTrustRequest {
operation_id: "cross-runtime-trust".into(),
host_trust_id: "cross-runtime-host".into(),
hostname: "example.invalid".into(),
port: 22,
host_key: public_key,
expected_revision: None,
},
"owner-account",
)
.unwrap();
let record = restore_test_workdir(
&api,
"cross-runtime-restore-workdir",
WorkdirRegistrySource::Repository {
runtime_id: WorkdirlessFixtureRuntime::RUNTIME_ID.into(),
repository_id: test_repository_id(&api),
},
);
peer.set_workdir_repository(&record.workdir_id, &test_repository_id(&api));
api.store.upsert_workdir_registry(&record).unwrap();
attach_restore_test_workdir(&api, &identity, &record);
let request = restore_request_for(&api, &identity, "cross-runtime-restore-request");
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
assert!(
execution.restore_operations.lock().unwrap().is_empty(),
"credential completion must precede execution"
);
let first = peer.repository_access_requests.lock().unwrap()[0].clone();
assert_eq!(
first.materialization.runtime_id,
WorkdirlessFixtureRuntime::RUNTIME_ID
);
assert_ne!(first.materialization.runtime_id, identity.runtime_id);
// Rotate current host trust while the accepted operation is uncertain. Its
// original credential/trust revisions, source and binding must remain pinned.
let alternate = api
.repository_secrets
.generate_credential(
api.workspace_id(),
GenerateRepositorySshCredentialRequest {
operation_id: "cross-runtime-alternate-key".into(),
credential_id: "cross-runtime-alternate".into(),
name: "Alternate".into(),
},
"owner-account",
)
.unwrap();
let alternate_public = api
.repository_secrets
.credential_public_key(api.workspace_id(), &alternate.credential_id)
.unwrap()
.unwrap()
.public_key;
let original_trust_revision = first
.materialization
.ssh
.as_ref()
.unwrap()
.host_trust_revision;
api.repository_secrets
.put_host_trust(
api.workspace_id(),
PutRepositorySshHostTrustRequest {
operation_id: "cross-runtime-trust-rotate".into(),
host_trust_id: "cross-runtime-host".into(),
hostname: "example.invalid".into(),
port: 22,
host_key: alternate_public,
expected_revision: Some(original_trust_revision),
},
"owner-account",
)
.unwrap();
*peer.reject_repository_access.lock().unwrap() = false;
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request.clone())
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
let attempts = peer.repository_access_requests.lock().unwrap();
assert_eq!(attempts.len(), 2);
let second = &attempts[1];
assert_eq!(
second.materialization.operation_id,
first.materialization.operation_id
);
let mut normalized = second.clone();
let old_ssh = first.materialization.ssh.as_ref().unwrap();
let new_ssh = normalized.materialization.ssh.as_mut().unwrap();
assert_ne!(new_ssh.secret_resource.nonce, old_ssh.secret_resource.nonce);
new_ssh.secret_resource = old_ssh.secret_resource.clone();
new_ssh.expires_at_epoch_seconds = old_ssh.expires_at_epoch_seconds;
assert_eq!(normalized, first, "only opaque handle and expiry may renew");
drop(attempts);
assert_eq!(execution.restore_operations.lock().unwrap().len(), 1);
assert_eq!(
handle
.restore(&WorkerOperationContext::Backend, request)
.await
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(
peer.repository_access_requests.lock().unwrap().len(),
2,
"terminal receipt must not renew access"
);
}
crates/workspace-server/src/server/worker_operations.rs
+12 -2
View File
@@ -291,10 +291,20 @@ impl WorkspaceWorker {
pub(super) async fn restore(
&self,
context: &WorkerOperationContext,
request: server_api::WorkerRestoreRequest,
) -> ApiResult<InternalWorkerRestoreResult> {
let _guard = self.authorize_operation(context, "restore").await?;
// Preserve singleton, Workdir, credentials, bindings and pending operation reconciliation.
self.api.restore_workspace_worker(&self.identity)
self.restore_internal(request)
}
/// Trusted Backend callers still supply an explicit observed intent. Runtime
/// admission is shared with browser/Tool callers; this is not an unguarded launch.
pub(super) fn restore_internal(
&self,
request: server_api::WorkerRestoreRequest,
) -> ApiResult<InternalWorkerRestoreResult> {
self.current_record()?;
self.api.restore_workspace_worker(&self.identity, request)
}
pub(super) async fn stop(
crates/workspace-server/src/store.rs
+91 -3
View File
@@ -28,7 +28,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore;
use crate::{Error, Result};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 79;
const LATEST_SCHEMA_VERSION: i64 = 80;
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings";
@@ -79,6 +79,8 @@ const ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME: &str =
const BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME: &str =
"durable terminal Backend Job Worker cleanup";
const WORKDIR_CONNECTION_ID_MIGRATION_NAME: &str = "durable Workdir attachment connection identity";
const WORKER_RESTORE_INTENTS_MIGRATION_NAME: &str =
"durable internal Worker Restore request identity";
const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7;
const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline";
const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#"
@@ -271,6 +273,11 @@ const MIGRATIONS: &[Migration] = &[
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME,
apply: migrate_workspace_config_v78_to_v79,
},
Migration {
version: 80,
name: WORKER_RESTORE_INTENTS_MIGRATION_NAME,
apply: migrate_worker_restore_intents_v79_to_v80,
},
];
#[derive(Clone, Copy)]
@@ -2167,6 +2174,48 @@ impl SqliteWorkspaceStore {
})
}
/// Client intent journal, not a lifecycle state machine: pin the public tuple
/// before dispatch and keep it unchanged until a terminal receipt is obtained.
pub(crate) fn pin_internal_worker_restore_intent(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
domain_key: &str,
observed_token: Option<&str>,
explicit_request_id: Option<&str>,
) -> Result<server_api::WorkerRestoreRequest> {
self.with_transaction(|tx| {
let existing: Option<(String, String)> = if let Some(id) = explicit_request_id {
tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4",
params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()?
} else {
tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND domain_key=?4 AND settled=0",
params![workspace_id, worker.runtime_id, worker.worker_id, domain_key], |row| Ok((row.get(0)?, row.get(1)?))).optional()?
};
if let Some((request_id, expected_observation_token)) = existing {
return Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token });
}
let observed_token = observed_token.ok_or(Error::RestoreObservationConflict)?;
let request_id = explicit_request_id.map(str::to_owned).unwrap_or_else(|| format!("{domain_key}:{observed_token}"));
tx.execute("INSERT INTO worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key,request_id,expected_token,settled) VALUES(?1,?2,?3,?4,?5,?6,0)",
params![workspace_id, worker.runtime_id, worker.worker_id, domain_key, request_id, observed_token])?;
Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token: observed_token.to_owned() })
})
}
pub(crate) fn settle_internal_worker_restore_intent(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
request_id: &str,
) -> Result<()> {
self.with_conn(|conn| {
conn.execute("UPDATE worker_restore_intents SET settled=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4",
params![workspace_id, worker.runtime_id, worker.worker_id, request_id])?;
Ok(())
})
}
pub(crate) fn get_workspace_memory_settings(
&self,
workspace_id: &str,
@@ -11248,6 +11297,7 @@ fn read_worker_registry_projection(
})?
} else {
SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string())
.map_err(|error| Error::InvalidInput(error.to_string()))?,
runtime_id: Some(worker.runtime_id.to_string()),
@@ -12013,6 +12063,24 @@ fn read_workdir_registry_record(
})
}
fn migrate_worker_restore_intents_v79_to_v80(conn: &Connection) -> Result<()> {
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
tx.execute_batch("CREATE TABLE worker_restore_intents (
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL,
settled INTEGER NOT NULL CHECK(settled IN (0,1)),
PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id),
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
tx.execute(
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(?1,?2)",
params![80_i64, WORKER_RESTORE_INTENTS_MIGRATION_NAME],
)?;
tx.commit()?;
Ok(())
}
fn prepare_connection(conn: &Connection) -> Result<()> {
configure_sqlite(conn)?;
let workspace_schema_version = current_schema_version(conn)?;
@@ -15094,6 +15162,15 @@ fn migrate_workdir_connection_id_v77_to_v78(conn: &Connection) -> Result<()> {
}
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
conn.execute_batch("CREATE TABLE worker_restore_intents (
workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL,
domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL,
settled INTEGER NOT NULL CHECK(settled IN (0,1)),
PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id),
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
conn.execute_batch(include_str!("latest_schema.sql"))?;
conn.execute_batch(
@@ -16202,6 +16279,7 @@ mod tests {
}
fn downgrade_schema_66_ticket_and_workdir_authority(conn: &Connection) -> Result<()> {
conn.execute_batch("DROP TABLE IF EXISTS worker_restore_intents;")?;
conn.execute_batch(
"DROP TABLE backend_job_deliveries;
DROP TABLE backend_job_attempts;
@@ -17709,6 +17787,10 @@ mod tests {
version: 79,
name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()
},
WorkspaceSchemaMigrationStep {
version: 80,
name: WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()
},
]
);
@@ -17798,6 +17880,7 @@ mod tests {
),
(78, WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string()),
(79, WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()),
(80, WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -18149,7 +18232,7 @@ mod tests {
.collect::<Vec<_>>(),
vec![
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
73, 74, 75, 76, 77, 78, 79
73, 74, 75, 76, 77, 78, 79, 80
]
);
SqliteWorkspaceStore::migrate_database(&path).unwrap();
@@ -18158,7 +18241,7 @@ mod tests {
current_schema_version(&conn).unwrap(),
LATEST_SCHEMA_VERSION
);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 30);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 31);
assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap());
assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap());
assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap());
@@ -21860,6 +21943,7 @@ INSERT INTO worker_registry (
for record in [&runtime_a, &runtime_b] {
let observed = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("shared-local-id").unwrap(),
runtime_id: Some(record.worker.runtime_id.clone()),
resource_key: None,
@@ -21996,6 +22080,7 @@ INSERT INTO worker_registry (
Some("W-1")
);
let observed = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("known").unwrap(),
runtime_id: Some("embedded".to_string()),
resource_key: None,
@@ -22016,6 +22101,7 @@ INSERT INTO worker_registry (
};
store.upsert_worker_registry(&other_catalog).unwrap();
let other_observed = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("other").unwrap(),
runtime_id: Some("runtime-other".to_string()),
display_name: Some("Other Runtime Worker".to_string()),
@@ -22025,6 +22111,7 @@ INSERT INTO worker_registry (
.apply_worker_registry_observation("local-dev", "runtime-other", &other_observed, "2")
.unwrap();
let orphan = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("orphan").unwrap(),
..observed.clone()
};
@@ -22190,6 +22277,7 @@ INSERT INTO worker_registry (
.is_none()
);
let event = SubscriptionWorker {
restore_observation_token: None,
worker_id: SubscriptionWorkerId::new("removed").unwrap(),
runtime_id: Some("embedded".to_string()),
resource_key: None,
crates/workspace-server/src/workspace_deletion.rs
+1
View File
@@ -76,6 +76,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"worker_registry_projection_diagnostics",
"worker_registry_projection_removals",
"worker_removal_operations",
"worker_restore_intents",
"worker_retention_audit_events",
"worker_session_archive_observe_grants",
"worker_session_archives",
crates/workspace-server/src/workspace_subscription.rs
+2
View File
@@ -425,6 +425,7 @@ fn project_registry_worker(
worker
} else {
SubscriptionWorker {
restore_observation_token: None,
worker_id: protocol::subscription::SubscriptionWorkerId::new(
record.registry.worker.worker_id.clone(),
)
@@ -600,6 +601,7 @@ mod tests {
#[test]
fn external_workdir_attachment_does_not_remove_workspace_worker() {
let mut worker = SubscriptionWorker {
restore_observation_token: None,
worker_id: worker_id(),
runtime_id: Some("runtime-1".to_string()),
resource_key: Some("W-1".to_string()),