fix: merge worker restore into develop host lifecycle
Integrate integration/worker-restore without reviving removed Worker socket/discovery paths. Keep restore candidate allocation and canonical effects deferred until activation, with registry reclamation and observer publication fences on the in-process controller. Preserve develop BackendJob/provenance, immutable Journal body capture and erasure, and Workdir diagnostics. Adapt obsolete socket fault tests to in-process publication and registry drift boundaries. Validation: root cargo check; ordinary worker, worker-runtime and standalone package tests; Server restore (43) and Journal (49) filters; mcp/server-api/runtime-api/session-store tests; optional Runtime feature check; OpenAPI and six generated TypeScript checks; Web restore unit tests (15); runner tests (21); fmt, naming and diff checks. Full workspace/Server suites, E2E, live/model and fixture generation not run.
This commit is contained in:
@@ -10,6 +10,10 @@ autobins = false
|
||||
name = "yoi-server"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
# Offline fixture preparation only; build product binaries without this feature.
|
||||
test-support = []
|
||||
|
||||
[dependencies]
|
||||
api-macros = { workspace = true, features = ["axum"] }
|
||||
async-trait.workspace = true
|
||||
|
||||
@@ -5113,6 +5113,9 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
runtime_api::WorkerRestoreState::RolledBack => {
|
||||
server_api::WorkerRestoreState::RolledBack
|
||||
}
|
||||
runtime_api::WorkerRestoreState::Superseded => {
|
||||
server_api::WorkerRestoreState::Superseded
|
||||
}
|
||||
runtime_api::WorkerRestoreState::ReconciliationRequired => {
|
||||
server_api::WorkerRestoreState::ReconciliationRequired
|
||||
}
|
||||
|
||||
@@ -19,12 +19,17 @@ pub mod repositories;
|
||||
pub mod repository_access;
|
||||
pub mod repository_source;
|
||||
pub mod resource_broker;
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
pub mod retention;
|
||||
pub mod runtime_settings;
|
||||
pub mod runtime_subscription;
|
||||
pub mod server;
|
||||
pub mod skills;
|
||||
pub mod store;
|
||||
#[cfg(feature = "test-support")]
|
||||
pub use store::fixture_support as test_support;
|
||||
pub mod subjektiv;
|
||||
#[cfg(test)]
|
||||
mod ticket_browse_performance_tests;
|
||||
|
||||
@@ -48742,6 +48742,12 @@ mod tests {
|
||||
std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
||||
reconcile_operations:
|
||||
std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
||||
restore_owners: std::sync::Mutex<
|
||||
std::collections::HashMap<
|
||||
worker_runtime::execution::WorkerLifecycleOperationId,
|
||||
worker_runtime::identity::WorkerRef,
|
||||
>,
|
||||
>,
|
||||
inputs: std::sync::Mutex<Vec<(worker_runtime::identity::WorkerRef, String)>>,
|
||||
protocol_methods:
|
||||
Arc<std::sync::Mutex<Vec<(worker_runtime::identity::WorkerRef, protocol::Method)>>>,
|
||||
@@ -48775,6 +48781,7 @@ mod tests {
|
||||
restore_failure: std::sync::Mutex::new(None),
|
||||
restore_operations: std::sync::Mutex::new(Vec::new()),
|
||||
reconcile_operations: std::sync::Mutex::new(Vec::new()),
|
||||
restore_owners: Default::default(),
|
||||
inputs: std::sync::Mutex::new(Vec::new()),
|
||||
protocol_methods: Arc::new(std::sync::Mutex::new(Vec::new())),
|
||||
stops: std::sync::Mutex::new(Vec::new()),
|
||||
@@ -48979,6 +48986,10 @@ mod tests {
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(request.operation_id);
|
||||
self.restore_owners
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(request.operation_id, request.worker_ref.clone());
|
||||
if let Some(result) = self.restore_failure.lock().unwrap().take() {
|
||||
return result;
|
||||
}
|
||||
@@ -49014,6 +49025,21 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_recovery_disposition(
|
||||
&self,
|
||||
operation_id: worker_runtime::execution::WorkerLifecycleOperationId,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
) -> Option<worker_runtime::execution::RestoreRecoveryDisposition> {
|
||||
use worker_runtime::execution::RestoreRecoveryDisposition as D;
|
||||
Some(
|
||||
if self.restore_owners.lock().unwrap().get(&operation_id) == Some(worker_ref) {
|
||||
D::Continue
|
||||
} else {
|
||||
D::Absent
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn reconcile_restore(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
|
||||
@@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{
|
||||
WorkerRuntimeExecutionBackend,
|
||||
};
|
||||
|
||||
pub(super) fn connected_restore_tempdir() -> tempfile::TempDir {
|
||||
crate::restore_test_environment::require_restore_test_environment();
|
||||
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap();
|
||||
tempfile::Builder::new()
|
||||
.prefix("c")
|
||||
.rand_bytes(3)
|
||||
.tempdir_in(checkout.join("target"))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
const WATCHDOG: Duration = Duration::from_secs(45);
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -82,24 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory {
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner.preflight_restore(request).await
|
||||
}
|
||||
async fn restore_controller(
|
||||
async fn recover_restore_source(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||
self.inner.recover_restore_source(request).await
|
||||
}
|
||||
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<RuntimeWorkerController, String> {
|
||||
) -> std::result::Result<
|
||||
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let worker_ref = request.worker_ref.clone();
|
||||
let controller = self.inner.restore_controller(request).await?;
|
||||
let inner = self.inner.prepare_restore(request).await?;
|
||||
Ok(Box::new(CapturingCandidate {
|
||||
inner,
|
||||
worker_ref,
|
||||
captures: self.captures.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cleanup_failed_restore(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inner.cleanup_failed_restore(request).await
|
||||
}
|
||||
|
||||
async fn reconcile_stopped_worker(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||
}
|
||||
|
||||
fn activate_restored_controller(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
workspace_id: Option<&str>,
|
||||
handle: &worker::WorkerHandle,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner
|
||||
.activate_restored_controller(worker_ref, workspace_id, handle)
|
||||
}
|
||||
}
|
||||
|
||||
struct CapturingCandidate {
|
||||
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_ref: worker_runtime::identity::WorkerRef,
|
||||
captures: Arc<Captures>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate {
|
||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||
self.inner.initial_state()
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
&mut self,
|
||||
) -> std::result::Result<
|
||||
RuntimeWorkerController,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let controller = self.inner.activate().await?;
|
||||
self.captures
|
||||
.handles
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((worker_ref.clone(), controller.handle.clone()));
|
||||
.push((self.worker_ref.clone(), controller.handle.clone()));
|
||||
self.captures
|
||||
.restored_clients
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((worker_ref, controller.workspace_client.clone()));
|
||||
.push((self.worker_ref.clone(), controller.workspace_client.clone()));
|
||||
Ok(controller)
|
||||
}
|
||||
|
||||
async fn discard(
|
||||
&mut self,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inner.discard().await
|
||||
}
|
||||
}
|
||||
|
||||
struct Resources {
|
||||
@@ -259,7 +340,7 @@ async fn call(tools: &BTreeMap<String, Arc<dyn Tool>>, name: &str, args: Value)
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let root = connected_restore_tempdir();
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let mut config = test_server_config(root.path());
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -476,6 +476,7 @@ impl AttentionWorkerFactory {
|
||||
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
||||
controller_task: started.controller_task,
|
||||
workspace_client,
|
||||
restore_streams: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -489,12 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac
|
||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||
.await
|
||||
}
|
||||
async fn restore_controller(
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
||||
.await
|
||||
) -> std::result::Result<
|
||||
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
Ok(Box::new(AttentionRestoreCandidate {
|
||||
inputs: Some((
|
||||
AttentionWorkerFactory {
|
||||
root: self.root.clone(),
|
||||
},
|
||||
request.worker_ref.worker_id.to_string(),
|
||||
)),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
// Preparation owns only fixture inputs; reconstruction and Controller startup
|
||||
// remain lazy so discard never needs to shut down a started Worker.
|
||||
struct AttentionRestoreCandidate {
|
||||
inputs: Option<(AttentionWorkerFactory, String)>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate {
|
||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||
protocol::WorkerStateSnapshot::initial()
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
&mut self,
|
||||
) -> std::result::Result<
|
||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||
"attention fixture candidate already consumed",
|
||||
)
|
||||
})?;
|
||||
factory.controller(worker_id, true).await.map_err(|_| {
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||
"attention fixture reconstruction failed",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn discard(
|
||||
&mut self,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inputs.take();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -601,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() {
|
||||
|
||||
async fn real_attention_worker_start_restore(lose_response: bool) {
|
||||
use session_store::{LogEntry, Store, WorkerSessionStore};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dir = restored_parent_control_tests::connected_restore_tempdir();
|
||||
init_clean_git_workspace(dir.path());
|
||||
let config = test_server_config(dir.path());
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
|
||||
@@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() {
|
||||
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
|
||||
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
|
||||
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str());
|
||||
let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
||||
assert_eq!(page["items"].as_array().unwrap().len(),1);
|
||||
assert_eq!(page["items"][0]["model_processing_confirmed"],false);
|
||||
let path = format!(
|
||||
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",
|
||||
ticket.id.as_str()
|
||||
);
|
||||
let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||
assert_eq!(page["items"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(page["items"][0]["model_processing_confirmed"], false);
|
||||
assert!(page["items"][0].get("content").is_none());
|
||||
let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
||||
assert_eq!(again,page);
|
||||
let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap();
|
||||
assert_eq!(response.status(),StatusCode::UNAUTHORIZED);
|
||||
delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await;
|
||||
delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await;
|
||||
let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||
assert_eq!(again, page);
|
||||
let response = build_router(api.clone())
|
||||
.oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
delivery_http(
|
||||
&api,
|
||||
"GET",
|
||||
"/api/w/other-workspace/notification-diagnostics",
|
||||
None,
|
||||
StatusCode::NOT_FOUND,
|
||||
)
|
||||
.await;
|
||||
delivery_http(
|
||||
&api,
|
||||
"GET",
|
||||
&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),
|
||||
None,
|
||||
StatusCode::BAD_REQUEST,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
fn delivery_assignment_path(ticket: &str) -> String {
|
||||
@@ -814,6 +834,7 @@ impl DeliveryWorkerFactory {
|
||||
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
||||
controller_task: started.controller_task,
|
||||
workspace_client,
|
||||
restore_streams: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -827,12 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact
|
||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||
.await
|
||||
}
|
||||
async fn restore_controller(
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
request: delivery_execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
||||
.await
|
||||
) -> std::result::Result<
|
||||
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
Ok(Box::new(DeliveryRestoreCandidate {
|
||||
inputs: Some((
|
||||
DeliveryWorkerFactory {
|
||||
root: self.root.clone(),
|
||||
client: self.client.clone(),
|
||||
},
|
||||
request.worker_ref.worker_id.to_string(),
|
||||
)),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
// Retain the same scripted client/receipt store without opening either store or
|
||||
// starting a Controller until activation. Discard drops only these owned inputs.
|
||||
struct DeliveryRestoreCandidate {
|
||||
inputs: Option<(DeliveryWorkerFactory, String)>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate {
|
||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||
protocol::WorkerStateSnapshot::initial()
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
&mut self,
|
||||
) -> std::result::Result<
|
||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||
"delivery fixture candidate already consumed",
|
||||
)
|
||||
})?;
|
||||
factory.controller(worker_id, true).await.map_err(|_| {
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||
"delivery fixture reconstruction failed",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn discard(
|
||||
&mut self,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inputs.take();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -931,6 +1002,7 @@ async fn delivery_real_fixture(
|
||||
Arc<DeliveryResponseLossBackend>,
|
||||
DeliveryScriptedClient,
|
||||
) {
|
||||
let _environment = restored_parent_control_tests::connected_restore_tempdir();
|
||||
let config = test_server_config(root);
|
||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||
seed_test_registered_workspace(store.as_ref(), &config)
|
||||
@@ -1136,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
||||
"uncertain delivery changed request id or payload: {attempts:?}"
|
||||
);
|
||||
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
|
||||
let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery {
|
||||
assignment_id: Some(assignment.into()), ..Default::default()
|
||||
}).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt");
|
||||
let unknown = api
|
||||
.store
|
||||
.notification_diagnostics(
|
||||
TEST_WORKSPACE_ID,
|
||||
&server_api::NotificationDiagnosticsQuery {
|
||||
assignment_id: Some(assignment.into()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.items
|
||||
.into_iter()
|
||||
.find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown)
|
||||
.expect(
|
||||
"saved unknown acceptance must remain visible while Runtime already has its receipt",
|
||||
);
|
||||
assert!(!unknown.model_processing_confirmed);
|
||||
// Change mutable rendering context while the acknowledgement is unknown.
|
||||
// A new self-report is not a new input and cannot change the frozen retry.
|
||||
@@ -1199,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
||||
}
|
||||
delivery_wait_idle(&reopened, &worker, 2).await;
|
||||
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
|
||||
let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await;
|
||||
assert_eq!(evidence["items"][0]["stage"],"runtime_accepted");
|
||||
assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id));
|
||||
assert_eq!(evidence["items"][0]["model_processing_confirmed"],false);
|
||||
assert_eq!(evidence["items"][0]["retry_candidate"],false);
|
||||
let evidence = delivery_http(
|
||||
&reopened,
|
||||
"GET",
|
||||
&format!(
|
||||
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",
|
||||
unknown.source_id
|
||||
),
|
||||
None,
|
||||
StatusCode::OK,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(evidence["items"][0]["stage"], "runtime_accepted");
|
||||
assert_eq!(
|
||||
evidence["items"][0]["delivery_id"],
|
||||
json!(unknown.delivery_id)
|
||||
);
|
||||
assert_eq!(evidence["items"][0]["model_processing_confirmed"], false);
|
||||
assert_eq!(evidence["items"][0]["retry_candidate"], false);
|
||||
assert_eq!(
|
||||
delivery_commits(&reopened, &worker)
|
||||
.iter()
|
||||
|
||||
@@ -187,7 +187,14 @@ async fn config_call(
|
||||
runtime
|
||||
.invoke(
|
||||
path.into(),
|
||||
if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)},
|
||||
if matches!(operation, "edit" | "write") {
|
||||
wip_protocol::InterfaceReference {
|
||||
scope: "/".into(),
|
||||
name: format!("yoi.file/{operation}-text/v1"),
|
||||
}
|
||||
} else {
|
||||
config_interface(path)
|
||||
},
|
||||
operation.into(),
|
||||
args,
|
||||
agen::tool::ToolExecutionContext::direct(),
|
||||
@@ -581,11 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto
|
||||
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||
Ok(controller)
|
||||
}
|
||||
async fn restore_controller(
|
||||
async fn preflight_restore(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner.preflight_restore(request).await
|
||||
}
|
||||
|
||||
async fn recover_restore_source(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||
self.inner.recover_restore_source(request).await
|
||||
}
|
||||
|
||||
async fn prepare_restore(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
||||
self.inner.restore_controller(request).await
|
||||
) -> std::result::Result<
|
||||
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let inner = self.inner.prepare_restore(request).await?;
|
||||
Ok(Box::new(ConfigCaptureCandidate {
|
||||
inner,
|
||||
client: self.client.clone(),
|
||||
controller_abort: self.controller_abort.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cleanup_failed_restore(
|
||||
&self,
|
||||
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inner.cleanup_failed_restore(request).await
|
||||
}
|
||||
|
||||
async fn reconcile_stopped_worker(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||
}
|
||||
|
||||
fn activate_restored_controller(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
workspace_id: Option<&str>,
|
||||
handle: &worker::WorkerHandle,
|
||||
) -> std::result::Result<(), String> {
|
||||
self.inner
|
||||
.activate_restored_controller(worker_ref, workspace_id, handle)
|
||||
}
|
||||
}
|
||||
|
||||
struct ConfigCaptureCandidate {
|
||||
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||
client: Arc<Mutex<Option<Arc<dyn worker::WorkspaceClient>>>>,
|
||||
controller_abort: Arc<Mutex<Option<tokio::task::AbortHandle>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate {
|
||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||
self.inner.initial_state()
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
&mut self,
|
||||
) -> std::result::Result<
|
||||
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||
> {
|
||||
let controller = self.inner.activate().await?;
|
||||
*self.client.lock().unwrap() = Some(controller.workspace_client.clone());
|
||||
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||
Ok(controller)
|
||||
}
|
||||
|
||||
async fn discard(
|
||||
&mut self,
|
||||
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||
self.inner.discard().await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -714,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
|
||||
decode_runtime_request_source_claims,
|
||||
};
|
||||
let root = restored_parent_control_tests::connected_restore_tempdir();
|
||||
// Remote Runtime-owned HTTP adapter, including its real request signer.
|
||||
let mut fixture = manual_worker_assignment_fixture().await;
|
||||
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
|
||||
@@ -793,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
||||
|
||||
// Actual embedded Runtime factory -> real Worker controller -> captured
|
||||
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(root.path());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
|
||||
@@ -2603,6 +2603,10 @@ pub struct SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-support")]
|
||||
#[path = "test_support.rs"]
|
||||
pub mod fixture_support;
|
||||
|
||||
impl SqliteWorkspaceStore {
|
||||
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
|
||||
let conn = Connection::open(path)?;
|
||||
@@ -2914,6 +2918,33 @@ impl SqliteWorkspaceStore {
|
||||
request_fingerprint: &str,
|
||||
singleton_key: Option<&str>,
|
||||
current_memory_settings: &WorkspaceMemorySettingsRecord,
|
||||
) -> Result<WorkerCreateReservation> {
|
||||
self.reserve_worker_create_with(
|
||||
workspace_id,
|
||||
runtime_id,
|
||||
allocation_key,
|
||||
request_fingerprint,
|
||||
singleton_key,
|
||||
current_memory_settings,
|
||||
|_| Ok(None),
|
||||
|_, _| Ok(()),
|
||||
)
|
||||
}
|
||||
|
||||
// Keep reservation validation, Memory binding and writes in one transaction. The
|
||||
// nondefault test-support module supplies fixed identity / offline catalog hooks;
|
||||
// ordinary creation always supplies a newly generated ID and no completion hook.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn reserve_worker_create_with(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
runtime_id: &str,
|
||||
allocation_key: &str,
|
||||
request_fingerprint: &str,
|
||||
singleton_key: Option<&str>,
|
||||
current_memory_settings: &WorkspaceMemorySettingsRecord,
|
||||
prepare: impl FnOnce(&rusqlite::Transaction<'_>) -> Result<Option<WorkerId>>,
|
||||
complete: impl FnOnce(&rusqlite::Transaction<'_>, &WorkerCreateReservation) -> Result<()>,
|
||||
) -> Result<WorkerCreateReservation> {
|
||||
if allocation_key.trim().is_empty() || request_fingerprint.trim().is_empty() {
|
||||
return Err(Error::InvalidInput(
|
||||
@@ -2926,6 +2957,7 @@ impl SqliteWorkspaceStore {
|
||||
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let fixed_worker_id = prepare(&tx)?;
|
||||
let existing = tx
|
||||
.query_row(
|
||||
"SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \
|
||||
@@ -3108,7 +3140,7 @@ impl SqliteWorkspaceStore {
|
||||
validate_workspace_memory_settings_snapshot(&snapshot, workspace_id)?;
|
||||
let create_fingerprint =
|
||||
bound_worker_create_fingerprint(request_fingerprint, &snapshot);
|
||||
let worker_id = WorkerId::now_v7();
|
||||
let worker_id = fixed_worker_id.unwrap_or_else(WorkerId::now_v7);
|
||||
let worker_id_text = worker_id.to_string();
|
||||
let now = chrono::Utc::now().to_rfc3339();
|
||||
let singleton_generation = match singleton_owner.as_ref() {
|
||||
@@ -3197,13 +3229,15 @@ impl SqliteWorkspaceStore {
|
||||
));
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(WorkerCreateReservation {
|
||||
let reservation = WorkerCreateReservation {
|
||||
worker_id,
|
||||
create_fingerprint,
|
||||
memory_settings: snapshot,
|
||||
singleton,
|
||||
})
|
||||
};
|
||||
complete(&tx, &reservation)?;
|
||||
tx.commit()?;
|
||||
Ok(reservation)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,688 @@
|
||||
//! Offline Backend fixture preparation, available only with `test-support`.
|
||||
//!
|
||||
//! Use a new isolated database and the existing account, Workspace signing-identity /
|
||||
//! `ControlPlaneStore::create_workspace_bootstrap`, and Runtime binding APIs first.
|
||||
//! Generate fresh account credentials and signing keys for each environment; no fixture
|
||||
//! credentials are accepted here. Install Profile configuration through the existing
|
||||
//! config-source mechanisms before starting the product processes.
|
||||
//!
|
||||
//! This module seeds Backend ownership only. It neither writes Runtime storage nor
|
||||
//! launches a Worker, fabricates Runtime observations, changes authorization, or sends
|
||||
//! initial input. The caller must separately install the matching stopped Runtime
|
||||
//! fixture. Build product binaries in a separate Cargo invocation **without** this
|
||||
//! feature (Cargo features unify within an invocation).
|
||||
|
||||
use super::*;
|
||||
use crate::hosts::{WorkerSpawnIntent, WorkerSpawnRequest, worker_spawn_create_fingerprint};
|
||||
use worker_runtime::catalog::ProfileSelector;
|
||||
|
||||
/// One ordinary, attachment-free Worker from an externally validated fixture.
|
||||
///
|
||||
/// IDs must match the Runtime fixture. `request` is the unresolved canonical create
|
||||
/// intent, not a Runtime launch request containing credentials or execution grants.
|
||||
/// The Profile selector must already be installed by the harness. `initial_submit`
|
||||
/// must be empty: saved Session contents belong to the Runtime fixture, not new input.
|
||||
/// Singleton/service, Ticket-assigned and attachment-bearing Workers are deliberately
|
||||
/// unsupported.
|
||||
pub struct FixedWorkerSeed<'a> {
|
||||
pub workspace_id: &'a str,
|
||||
pub runtime_id: &'a str,
|
||||
pub worker_id: WorkerId,
|
||||
pub allocation_key: &'a str,
|
||||
pub request: &'a WorkerSpawnRequest,
|
||||
}
|
||||
|
||||
/// Insert a fixed Worker catalog entry and completed creation reservation atomically.
|
||||
///
|
||||
/// Uses the normal reservation validators, authoritative Memory snapshot, fingerprint
|
||||
/// binding, resource-key allocator and immediate transaction. Even an exact repeat is
|
||||
/// rejected: this is not an upsert/import/recovery API. Existing allocations, Worker
|
||||
/// identities and residual lifecycle/attachment records are never adopted or repaired.
|
||||
/// Returns the creation binding needed to check the corresponding Runtime fixture.
|
||||
pub fn seed_fixed_worker(
|
||||
store: &SqliteWorkspaceStore,
|
||||
seed: FixedWorkerSeed<'_>,
|
||||
) -> Result<WorkerCreateReservation> {
|
||||
validate_identifier("workspace_id", seed.workspace_id)?;
|
||||
validate_identifier("runtime_id", seed.runtime_id)?;
|
||||
let request = seed.request;
|
||||
if !request.initial_submit.is_empty() {
|
||||
return Err(Error::InvalidInput(
|
||||
"fixture Worker cannot contain initial_submit; offline seeding accepts no new input"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
if !matches!(request.intent, WorkerSpawnIntent::WorkspaceCoding)
|
||||
|| request.singleton_key.is_some()
|
||||
|| request.ticket_assignment.is_some()
|
||||
|| request.resolved_subjektiv_attached
|
||||
|| !request.workdir_attachment_requests.is_empty()
|
||||
|| !request.resolved_workdir_attachment_requests.is_empty()
|
||||
|| !request.resolved_workdir_attachments.is_empty()
|
||||
|| request.resolved_config_bundle.is_some()
|
||||
|| request.resolved_workspace_api.is_some()
|
||||
|| request.resolved_memory_settings.is_some()
|
||||
|| request.resolved_worker_observation_enabled
|
||||
|| !request.resolved_worker_observation_grants.is_empty()
|
||||
|| request.resolved_control_operation.is_some()
|
||||
|| request.resolved_execution_grant.is_some()
|
||||
{
|
||||
return Err(Error::InvalidInput(
|
||||
"fixture Worker must be ordinary unresolved create intent without singleton, assignments, attachments or grants".into(),
|
||||
));
|
||||
}
|
||||
// Builtin selectors already carry their canonical `builtin:` prefix.
|
||||
let profile = match &request.profile {
|
||||
ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => name,
|
||||
};
|
||||
validate_non_empty("fixture Profile selector", profile)?;
|
||||
let worker_id = seed.worker_id.to_string();
|
||||
let display_name = request
|
||||
.requested_worker_name
|
||||
.as_deref()
|
||||
.unwrap_or(&worker_id);
|
||||
validate_non_empty("fixture Worker display name", display_name)?;
|
||||
let fingerprint = worker_spawn_create_fingerprint(request).map_err(Error::InvalidInput)?;
|
||||
let memory = store.get_workspace_memory_settings(seed.workspace_id)?;
|
||||
store.reserve_worker_create_with(
|
||||
seed.workspace_id,
|
||||
seed.runtime_id,
|
||||
seed.allocation_key,
|
||||
&fingerprint,
|
||||
None,
|
||||
&memory,
|
||||
|tx| {
|
||||
require_seed_target(tx, &seed)?;
|
||||
Ok(Some(seed.worker_id))
|
||||
},
|
||||
|tx, reservation| {
|
||||
let now = chrono::Utc::now().to_rfc3339();
|
||||
tx.execute(
|
||||
"INSERT INTO worker_registry (
|
||||
workspace_id, runtime_id, worker_id, display_name, profile,
|
||||
retention_state, created_at, updated_at
|
||||
) VALUES (?1, ?2, ?3, ?4, ?5, 'normal', ?6, ?6)",
|
||||
params![seed.workspace_id, seed.runtime_id, worker_id, display_name, profile, now],
|
||||
)?;
|
||||
let changed = tx.execute(
|
||||
"UPDATE worker_create_reservations SET state = 'created', updated_at = ?4
|
||||
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND state = 'reserved'",
|
||||
params![seed.workspace_id, seed.runtime_id, reservation.worker_id.to_string(), now],
|
||||
)?;
|
||||
if changed != 1 {
|
||||
return Err(Error::Store("fixture creation reservation disappeared".into()));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn require_seed_target(tx: &rusqlite::Transaction<'_>, seed: &FixedWorkerSeed<'_>) -> Result<()> {
|
||||
let active: bool = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM workspaces w
|
||||
JOIN workspace_runtime_bindings r ON r.workspace_id = w.workspace_id
|
||||
JOIN workspace_signing_identities s ON s.workspace_id = w.workspace_id
|
||||
WHERE w.workspace_id = ?1 AND w.state = 'active'
|
||||
AND r.runtime_id = ?2 AND r.revoked_at IS NULL
|
||||
AND r.authentication_mode = 'workspace_identity' AND s.state = 'active')",
|
||||
params![seed.workspace_id, seed.runtime_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if !active {
|
||||
return Err(Error::InvalidInput(
|
||||
"fixture requires an active bootstrapped Workspace and authenticated Runtime binding"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
let worker_id = seed.worker_id.to_string();
|
||||
let allocation_exists: bool = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations
|
||||
WHERE workspace_id = ?1 AND allocation_key = ?2)",
|
||||
params![seed.workspace_id, seed.allocation_key],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if allocation_exists {
|
||||
return Err(Error::InvalidInput(
|
||||
"fixture Worker allocation already exists".into(),
|
||||
));
|
||||
}
|
||||
// Check by WorkerId, not just the proposed Runtime tuple: orphaned and terminal
|
||||
// ownership is still a collision. These tables are owned by this store schema.
|
||||
for table in [
|
||||
"worker_create_reservations",
|
||||
"worker_registry",
|
||||
"worker_registry_observations",
|
||||
"worker_registry_projection_removals",
|
||||
"worker_removal_operations",
|
||||
"worker_singleton_owners",
|
||||
"worker_workdir_links",
|
||||
"worker_restore_intents",
|
||||
"worker_tombstones",
|
||||
"worker_orphan_diagnostics",
|
||||
"worker_registry_projection_diagnostics",
|
||||
"worker_session_archives",
|
||||
"worker_diagnostics_archives",
|
||||
"ticket_assignment_worker_tombstones",
|
||||
] {
|
||||
let exists: bool = tx.query_row(
|
||||
&format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE worker_id = ?1)"),
|
||||
[&worker_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if exists {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"fixture Worker identity already exists in {table}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
// A resource key without a catalog/reservation must not be silently reused.
|
||||
let mut statement = tx.prepare(
|
||||
"SELECT resource_id FROM workspace_resource_keys WHERE resource_kind = 'worker'",
|
||||
)?;
|
||||
let resources = statement.query_map([], |row| row.get::<_, String>(0))?;
|
||||
for resource in resources {
|
||||
let resource = resource?;
|
||||
let existing = resource
|
||||
.is_ascii()
|
||||
.then(|| parse_worker_resource_id(&resource))
|
||||
.flatten()
|
||||
.ok_or_else(|| {
|
||||
Error::InvalidInput("unknown fixture Worker resource identity collision".into())
|
||||
})?;
|
||||
if existing.worker_id == worker_id {
|
||||
return Err(Error::InvalidInput(
|
||||
"fixture Worker resource identity already exists".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::hosts::{
|
||||
WorkerSpawnAcceptanceRequirement, WorkerSpawnWorkingDirectoryAttachmentRequest,
|
||||
WorkerSpawnWorkingDirectoryRequest,
|
||||
};
|
||||
use crate::workspace_signing_identity::{
|
||||
InMemoryWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService,
|
||||
};
|
||||
|
||||
const WORKSPACE: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c501";
|
||||
const RUNTIME: &str = "fixture-runtime";
|
||||
const WORKER: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c502";
|
||||
|
||||
// Exercises the same public bootstrap APIs the offline harness uses. All account
|
||||
// and key identities are newly generated, even across stores with fixed fixture IDs.
|
||||
fn bootstrap() -> SqliteWorkspaceStore {
|
||||
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
let account = AccountRecord {
|
||||
account_id: format!("fixture-account-{}", Uuid::now_v7()),
|
||||
kind: "user".into(),
|
||||
handle: format!("fixture-{}", Uuid::now_v7()),
|
||||
display_name: "Fixture owner".into(),
|
||||
created_at: "2026-10-24T00:00:00Z".into(),
|
||||
updated_at: "2026-10-24T00:00:00Z".into(),
|
||||
};
|
||||
store.upsert_account(&account).unwrap();
|
||||
let identities = WorkspaceSigningIdentityService::new(
|
||||
Arc::new(store.clone()),
|
||||
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
|
||||
);
|
||||
let (identity, operation) = identities
|
||||
.prepare_workspace_creation(
|
||||
"fixture-bootstrap",
|
||||
"fixture-bootstrap",
|
||||
WORKSPACE,
|
||||
&account.account_id,
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.create_workspace_bootstrap(
|
||||
&WorkspaceBootstrapRecord {
|
||||
operation_key: "fixture-bootstrap".into(),
|
||||
request_fingerprint: "fixture-bootstrap".into(),
|
||||
workspace: WorkspaceRecord {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
owner_account_id: account.account_id,
|
||||
display_name: "Fixture".into(),
|
||||
state: "active".into(),
|
||||
created_at: account.created_at,
|
||||
updated_at: account.updated_at,
|
||||
},
|
||||
repository: None,
|
||||
},
|
||||
&identity,
|
||||
&operation,
|
||||
)
|
||||
.unwrap();
|
||||
let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate(RUNTIME).unwrap();
|
||||
store
|
||||
.upsert_workspace_runtime_binding(
|
||||
WorkspaceRuntimeBinding {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
runtime_id: RUNTIME.into(),
|
||||
display_name: "Fixture Runtime".into(),
|
||||
base_url: "http://127.0.0.1:12345".into(),
|
||||
public_key: runtime_key.public_key,
|
||||
public_key_fingerprint: String::new(),
|
||||
binding_id: String::new(),
|
||||
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
|
||||
created_at: "2026-10-24T00:00:00Z".into(),
|
||||
updated_at: "2026-10-24T00:00:00Z".into(),
|
||||
revoked_at: None,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
}
|
||||
|
||||
fn request() -> WorkerSpawnRequest {
|
||||
WorkerSpawnRequest {
|
||||
intent: WorkerSpawnIntent::WorkspaceCoding,
|
||||
requested_worker_name: Some("Fixture Worker".into()),
|
||||
singleton_key: None,
|
||||
acceptance: WorkerSpawnAcceptanceRequirement::SocketReady,
|
||||
profile: ProfileSelector::Builtin("builtin:coder".into()),
|
||||
ticket_assignment: None,
|
||||
initial_submit: Vec::new(),
|
||||
workdir_attachment_requests: Vec::new(),
|
||||
resolved_workdir_attachment_requests: Vec::new(),
|
||||
resolved_workdir_attachments: Vec::new(),
|
||||
resolved_config_bundle: None,
|
||||
resolved_workspace_api: None,
|
||||
resolved_memory_settings: None,
|
||||
resolved_subjektiv_attached: false,
|
||||
resolved_worker_observation_enabled: false,
|
||||
resolved_worker_observation_grants: Vec::new(),
|
||||
resolved_control_operation: None,
|
||||
resolved_execution_grant: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn seed(request: &WorkerSpawnRequest) -> FixedWorkerSeed<'_> {
|
||||
FixedWorkerSeed {
|
||||
workspace_id: WORKSPACE,
|
||||
runtime_id: RUNTIME,
|
||||
worker_id: WORKER.parse().unwrap(),
|
||||
allocation_key: "fixture-allocation",
|
||||
request,
|
||||
}
|
||||
}
|
||||
|
||||
fn worker() -> RuntimeWorkerRef {
|
||||
RuntimeWorkerRef::new(RUNTIME, WORKER)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_ids_are_isolated_and_seed_has_no_runtime_observation() {
|
||||
let a = bootstrap();
|
||||
let b = bootstrap();
|
||||
let request = request();
|
||||
let first = seed_fixed_worker(&a, seed(&request)).unwrap();
|
||||
let second = seed_fixed_worker(&b, seed(&request)).unwrap();
|
||||
assert_eq!(first, second);
|
||||
assert_eq!(first.worker_id.to_string(), WORKER);
|
||||
assert_eq!(first.memory_settings.workspace_id, WORKSPACE);
|
||||
assert_eq!(first.memory_settings.language, "English");
|
||||
assert!(first.singleton.is_none());
|
||||
assert_ne!(
|
||||
a.get_workspace_signing_identity(WORKSPACE)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.public_key,
|
||||
b.get_workspace_signing_identity(WORKSPACE)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.public_key
|
||||
);
|
||||
assert_ne!(
|
||||
a.get_workspace_runtime_binding(WORKSPACE, RUNTIME)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.public_key,
|
||||
b.get_workspace_runtime_binding(WORKSPACE, RUNTIME)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.public_key
|
||||
);
|
||||
for store in [&a, &b] {
|
||||
assert_eq!(
|
||||
store
|
||||
.worker_resource_key(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("W-1")
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.resolve_worker_resource_reference(WORKSPACE, "W-1")
|
||||
.unwrap(),
|
||||
Some(worker())
|
||||
);
|
||||
assert!(
|
||||
!store
|
||||
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
);
|
||||
let registry = store
|
||||
.get_worker_registry(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(registry.profile.as_deref(), Some("builtin:coder"));
|
||||
assert_eq!(registry.display_name, "Fixture Worker");
|
||||
assert_eq!(registry.retention_state, "normal");
|
||||
let projection = store
|
||||
.with_conn(|conn| read_worker_registry_projection(conn, WORKSPACE, &worker()))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(projection.observation.is_none());
|
||||
assert!(projection.job.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initial_submit_is_refused_without_registration_or_allocation() {
|
||||
let store = bootstrap();
|
||||
let mut request = request();
|
||||
// Even an empty text segment is new input; the vector itself must be empty.
|
||||
for text in ["new input must not be seeded", ""] {
|
||||
request.initial_submit = vec![protocol::Segment::text(text)];
|
||||
let error = seed_fixed_worker(&store, seed(&request)).unwrap_err();
|
||||
assert!(
|
||||
matches!(error, Error::InvalidInput(ref message) if message.contains("initial_submit"))
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_worker_registry(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.worker_resource_key(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
!store
|
||||
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
// The rejected attempts must not consume the allocation or resource counter.
|
||||
request.initial_submit.clear();
|
||||
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.worker_resource_key(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("W-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_replay_changed_input_and_cross_runtime_identity_are_refused() {
|
||||
let store = bootstrap();
|
||||
let mut request = request();
|
||||
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||
let before = store.get_worker_registry(WORKSPACE, &worker()).unwrap();
|
||||
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||
request.requested_worker_name = Some("Must not overwrite".into());
|
||||
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||
let mut new_allocation = seed(&request);
|
||||
new_allocation.allocation_key = "another-allocation";
|
||||
assert!(seed_fixed_worker(&store, new_allocation).is_err());
|
||||
let mut same_allocation = seed(&request);
|
||||
same_allocation.worker_id = WorkerId::now_v7();
|
||||
assert!(seed_fixed_worker(&store, same_allocation).is_err());
|
||||
assert_eq!(
|
||||
store.get_worker_registry(WORKSPACE, &worker()).unwrap(),
|
||||
before
|
||||
);
|
||||
|
||||
let other = bootstrap();
|
||||
let mut record = before.unwrap();
|
||||
record.worker.runtime_id = "different-runtime".into();
|
||||
other.upsert_worker_registry(&record).unwrap();
|
||||
assert!(seed_fixed_worker(&other, seed(&request)).is_err());
|
||||
assert!(
|
||||
other
|
||||
.get_worker_registry(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reserved_and_orphaned_resource_identities_are_not_adopted() {
|
||||
let store = bootstrap();
|
||||
let request = request();
|
||||
let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap();
|
||||
let ordinary = store
|
||||
.reserve_worker_create(WORKSPACE, RUNTIME, "ordinary", "fingerprint", None, &memory)
|
||||
.unwrap();
|
||||
let mut collision = seed(&request);
|
||||
collision.worker_id = ordinary.worker_id;
|
||||
assert!(seed_fixed_worker(&store, collision).is_err());
|
||||
// Store-level fault setup only: the external harness never writes SQL.
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
allocate_resource_key(
|
||||
conn,
|
||||
WORKSPACE,
|
||||
WorkspaceResourceKind::Worker,
|
||||
&worker_resource_id(&worker()),
|
||||
"now",
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||
assert!(
|
||||
store
|
||||
.get_worker_registry(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_resource_identity_and_retained_removal_fence_are_refused() {
|
||||
let request = request();
|
||||
let malformed = bootstrap();
|
||||
malformed
|
||||
.with_conn(|conn| {
|
||||
allocate_resource_key(
|
||||
conn,
|
||||
WORKSPACE,
|
||||
WorkspaceResourceKind::Worker,
|
||||
"unknown-identity",
|
||||
"now",
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
seed_fixed_worker(&malformed, seed(&request)),
|
||||
Err(Error::InvalidInput(_))
|
||||
));
|
||||
assert!(
|
||||
malformed
|
||||
.get_worker_registry(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let removed = bootstrap();
|
||||
removed.with_conn(|conn| {
|
||||
conn.execute(
|
||||
"INSERT INTO worker_registry_projection_removals(workspace_id, runtime_id, worker_id) VALUES (?1, ?2, ?3)",
|
||||
params![WORKSPACE, RUNTIME, WORKER],
|
||||
)?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert!(matches!(
|
||||
seed_fixed_worker(&removed, seed(&request)),
|
||||
Err(Error::InvalidInput(_))
|
||||
));
|
||||
assert!(
|
||||
removed
|
||||
.worker_resource_key(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_workspace_runtime_and_revoked_binding_are_refused() {
|
||||
let store = bootstrap();
|
||||
let request = request();
|
||||
let mut unknown = seed(&request);
|
||||
unknown.workspace_id = "unknown";
|
||||
assert!(seed_fixed_worker(&store, unknown).is_err());
|
||||
let mut unknown = seed(&request);
|
||||
unknown.runtime_id = "unknown";
|
||||
assert!(seed_fixed_worker(&store, unknown).is_err());
|
||||
store
|
||||
.revoke_workspace_runtime_binding(WORKSPACE, RUNTIME, "2026-10-24T01:00:00Z")
|
||||
.unwrap();
|
||||
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||
assert!(
|
||||
store
|
||||
.worker_resource_key(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_intent_is_refused_without_partial_registration() {
|
||||
let store = bootstrap();
|
||||
let base = request();
|
||||
let mut requests = Vec::new();
|
||||
let mut request = base.clone();
|
||||
request.singleton_key = Some("fixture-singleton".into());
|
||||
requests.push(request);
|
||||
let mut request = base.clone();
|
||||
request.intent = WorkerSpawnIntent::WorkspaceOrchestrator;
|
||||
requests.push(request);
|
||||
let mut request = base.clone();
|
||||
request.resolved_subjektiv_attached = true;
|
||||
requests.push(request);
|
||||
let mut request = base.clone();
|
||||
request
|
||||
.workdir_attachment_requests
|
||||
.push(WorkerSpawnWorkingDirectoryAttachmentRequest {
|
||||
alias: workdir::WorkdirAttachmentAlias::new("repo").unwrap(),
|
||||
working_directory: WorkerSpawnWorkingDirectoryRequest {
|
||||
repository_key: "repo".into(),
|
||||
selector: None,
|
||||
},
|
||||
});
|
||||
requests.push(request);
|
||||
for request in requests {
|
||||
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.get_worker_registry(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.worker_resource_key(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
seed_fixed_worker(&store, seed(&base))
|
||||
.unwrap()
|
||||
.worker_id
|
||||
.to_string(),
|
||||
WORKER
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn late_catalog_failure_rolls_back_reservation_and_resource_allocation() {
|
||||
let store = bootstrap();
|
||||
let request = request();
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch(
|
||||
"CREATE TEMP TRIGGER fail_fixture_catalog BEFORE INSERT ON worker_registry
|
||||
BEGIN SELECT RAISE(ABORT, 'injected catalog failure'); END",
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||
assert!(
|
||||
!store
|
||||
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.worker_resource_key(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER fail_fixture_catalog")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.worker_resource_key(WORKSPACE, &worker())
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some("W-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_creation_still_allocates_and_replays_without_catalog_insertion() {
|
||||
let store = bootstrap();
|
||||
let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap();
|
||||
let reserve = |key| {
|
||||
store
|
||||
.reserve_worker_create(WORKSPACE, RUNTIME, key, "fingerprint", None, &memory)
|
||||
.unwrap()
|
||||
};
|
||||
let first = reserve("ordinary-one");
|
||||
assert_eq!(first, reserve("ordinary-one"));
|
||||
let second = reserve("ordinary-two");
|
||||
assert_ne!(first.worker_id, second.worker_id);
|
||||
assert_ne!(first.worker_id.to_string(), WORKER);
|
||||
assert_eq!(
|
||||
first.worker_id.as_uuid().get_version(),
|
||||
Some(uuid::Version::SortRand)
|
||||
);
|
||||
let worker = RuntimeWorkerRef::new(RUNTIME, first.worker_id.to_string());
|
||||
assert!(
|
||||
store
|
||||
.has_active_worker_create_reservation(WORKSPACE, &worker)
|
||||
.unwrap()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_worker_registry(WORKSPACE, &worker)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.reserve_worker_create(WORKSPACE, RUNTIME, "ordinary-one", "changed", None, &memory)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user