fix: preserve restore effects and fence prepared attachment replay
This commit is contained in:
@@ -2150,6 +2150,16 @@ impl Runtime {
|
|||||||
{
|
{
|
||||||
return Err(conflict());
|
return Err(conflict());
|
||||||
}
|
}
|
||||||
|
if let Err(error) = owner.validate_workdir_preparation_authority(&candidate) {
|
||||||
|
return Ok((
|
||||||
|
Some(RuntimeWorkerRestoreResult::failed(
|
||||||
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"worker_restore_authority_changed",
|
||||||
|
sanitize_worker_create_failure_message(&error.to_string()),
|
||||||
|
)),
|
||||||
|
None,
|
||||||
|
));
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
// Startup/older persisted pending operations have a lifecycle
|
// Startup/older persisted pending operations have a lifecycle
|
||||||
// owner but no public request receipt. Bind recovery to that
|
// owner but no public request receipt. Bind recovery to that
|
||||||
@@ -2215,12 +2225,18 @@ impl Runtime {
|
|||||||
candidate.pending_restore = Some(pending);
|
candidate.pending_restore = Some(pending);
|
||||||
candidate.restore_guard.rotate();
|
candidate.restore_guard.rotate();
|
||||||
candidate.restore_guard.active_request_id = Some(request.request_id.clone());
|
candidate.restore_guard.active_request_id = Some(request.request_id.clone());
|
||||||
|
let workdir_attachment_preparation_base = request
|
||||||
|
.preparation
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|preparation| preparation.workdir_attachments.as_ref())
|
||||||
|
.map(|_| candidate.logical_workdir_attachments.clone());
|
||||||
candidate.restore_guard.owners.insert(
|
candidate.restore_guard.owners.insert(
|
||||||
request.request_id.clone(),
|
request.request_id.clone(),
|
||||||
RestoreRequestOwner {
|
RestoreRequestOwner {
|
||||||
fingerprint,
|
fingerprint,
|
||||||
operation_id,
|
operation_id,
|
||||||
request,
|
request,
|
||||||
|
workdir_attachment_preparation_base,
|
||||||
preparation_applied: recovering_pending,
|
preparation_applied: recovering_pending,
|
||||||
execution_started: recovering_pending,
|
execution_started: recovering_pending,
|
||||||
receipt: None,
|
receipt: None,
|
||||||
@@ -2433,14 +2449,15 @@ impl Runtime {
|
|||||||
{
|
{
|
||||||
return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail()));
|
return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail()));
|
||||||
}
|
}
|
||||||
let preparation = worker
|
let owner = worker
|
||||||
.restore_guard
|
.restore_guard
|
||||||
.active_request_id
|
.active_request_id
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|id| worker.restore_guard.owners.get(id))
|
.and_then(|id| worker.restore_guard.owners.get(id));
|
||||||
.filter(|owner| !owner.preparation_applied)
|
let prepared_worker = match owner.map_or_else(
|
||||||
.and_then(|owner| owner.request.preparation.as_ref());
|
|| Ok(worker.clone()),
|
||||||
let prepared_worker = match worker.prepared_restore_record(preparation) {
|
|owner| owner.prepared_worker_record(worker),
|
||||||
|
) {
|
||||||
Ok(prepared) => prepared,
|
Ok(prepared) => prepared,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
@@ -4398,8 +4415,7 @@ impl Runtime {
|
|||||||
worker_id: worker_ref.worker_id,
|
worker_id: worker_ref.worker_id,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let mut candidate =
|
let mut candidate = owner.prepared_worker_record(previous)?;
|
||||||
previous.prepared_restore_record(owner.request.preparation.as_ref())?;
|
|
||||||
candidate
|
candidate
|
||||||
.restore_guard
|
.restore_guard
|
||||||
.owners
|
.owners
|
||||||
@@ -6277,12 +6293,48 @@ pub(crate) struct RestoreRequestOwner {
|
|||||||
fingerprint: String,
|
fingerprint: String,
|
||||||
operation_id: WorkerLifecycleOperationId,
|
operation_id: WorkerLifecycleOperationId,
|
||||||
request: runtime_api::WorkerRestoreRequest,
|
request: runtime_api::WorkerRestoreRequest,
|
||||||
|
// This is the admission comparison input, not a second attachment authority.
|
||||||
|
// Without it an old preparation could hide a later canonical revocation.
|
||||||
|
// Older owners lacking this fact cannot safely reapply attachment overrides.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
workdir_attachment_preparation_base: Option<Vec<LogicalWorkdirAttachment>>,
|
||||||
preparation_applied: bool,
|
preparation_applied: bool,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
execution_started: bool,
|
execution_started: bool,
|
||||||
receipt: Option<RuntimeWorkerRestoreResult>,
|
receipt: Option<RuntimeWorkerRestoreResult>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl RestoreRequestOwner {
|
||||||
|
fn validate_workdir_preparation_authority(
|
||||||
|
&self,
|
||||||
|
worker: &WorkerRecord,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
if !self.preparation_applied
|
||||||
|
&& self
|
||||||
|
.request
|
||||||
|
.preparation
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|preparation| preparation.workdir_attachments.is_some())
|
||||||
|
&& self.workdir_attachment_preparation_base.as_ref()
|
||||||
|
!= Some(&worker.logical_workdir_attachments)
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"current Worker authority differs from admitted Restore attachment preparation"
|
||||||
|
.into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prepared_worker_record(&self, worker: &WorkerRecord) -> Result<WorkerRecord, RuntimeError> {
|
||||||
|
if self.preparation_applied {
|
||||||
|
return Ok(worker.clone());
|
||||||
|
}
|
||||||
|
self.validate_workdir_preparation_authority(worker)?;
|
||||||
|
worker.prepared_restore_record(self.request.preparation.as_ref())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub(crate) struct RestoreGuard {
|
pub(crate) struct RestoreGuard {
|
||||||
@@ -11995,7 +12047,7 @@ mod tests {
|
|||||||
let mut request = runtime.test_restore_request(&worker.worker_ref);
|
let mut request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
request.preparation = Some(serde_json::from_value(serde_json::json!({
|
request.preparation = Some(serde_json::from_value(serde_json::json!({
|
||||||
"workspace_api": { "workspace_id": "workspace-a", "base_url": "https://prepared.example" },
|
"workspace_api": { "workspace_id": "workspace-a", "base_url": "https://prepared.example" },
|
||||||
"workdir_attachments": [], "repository_access": []
|
"workdir_attachments": [{ "alias": "checkout", "working_directory_id": "prepared-logical-workdir" }], "repository_access": []
|
||||||
})).unwrap());
|
})).unwrap());
|
||||||
let gate = Arc::new(RestoreGate::default());
|
let gate = Arc::new(RestoreGate::default());
|
||||||
*backend.restore_gate.lock().unwrap() = Some(gate.clone());
|
*backend.restore_gate.lock().unwrap() = Some(gate.clone());
|
||||||
@@ -12014,6 +12066,26 @@ mod tests {
|
|||||||
let state = runtime.lock().unwrap();
|
let state = runtime.lock().unwrap();
|
||||||
let record = state.worker(&worker.worker_ref).unwrap();
|
let record = state.worker(&worker.worker_ref).unwrap();
|
||||||
assert!(record.request.as_ref().unwrap().workspace_api.is_none());
|
assert!(record.request.as_ref().unwrap().workspace_api.is_none());
|
||||||
|
assert!(record.logical_workdir_attachments.is_empty());
|
||||||
|
let owner = record.restore_guard.owners.values().next().unwrap();
|
||||||
|
let mut saved_owner = serde_json::to_value(owner).unwrap();
|
||||||
|
let decoded: RestoreRequestOwner = serde_json::from_value(saved_owner.clone()).unwrap();
|
||||||
|
assert!(
|
||||||
|
decoded
|
||||||
|
.validate_workdir_preparation_authority(record)
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
saved_owner
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.remove("workdir_attachment_preparation_base");
|
||||||
|
let legacy: RestoreRequestOwner = serde_json::from_value(saved_owner).unwrap();
|
||||||
|
assert!(
|
||||||
|
legacy
|
||||||
|
.validate_workdir_preparation_authority(record)
|
||||||
|
.is_err(),
|
||||||
|
"a missing historical comparison input must not be invented from current authority"
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
!record
|
!record
|
||||||
.restore_guard
|
.restore_guard
|
||||||
@@ -12033,6 +12105,11 @@ mod tests {
|
|||||||
.base_url,
|
.base_url,
|
||||||
"https://prepared.example"
|
"https://prepared.example"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend.restore_requests.lock().unwrap()[0].logical_workdir_attachments[0]
|
||||||
|
.working_directory_id,
|
||||||
|
"prepared-logical-workdir"
|
||||||
|
);
|
||||||
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
|
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
|
||||||
gate.release();
|
gate.release();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -12053,6 +12130,25 @@ mod tests {
|
|||||||
.base_url,
|
.base_url,
|
||||||
"https://prepared.example"
|
"https://prepared.example"
|
||||||
);
|
);
|
||||||
|
let record = state.worker(&worker.worker_ref).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
record.logical_workdir_attachments[0].working_directory_id,
|
||||||
|
"prepared-logical-workdir"
|
||||||
|
);
|
||||||
|
let owner = record.restore_guard.owners.values().next().unwrap();
|
||||||
|
assert!(owner.preparation_applied);
|
||||||
|
let mut revoked = record.clone();
|
||||||
|
revoked
|
||||||
|
.apply_logical_workdir_attachments(Vec::new())
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
owner
|
||||||
|
.prepared_worker_record(&revoked)
|
||||||
|
.unwrap()
|
||||||
|
.logical_workdir_attachments
|
||||||
|
.is_empty(),
|
||||||
|
"a committed preparation is not a perpetual grant"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
|
|||||||
@@ -360,20 +360,21 @@ async fn prepare_restore_operation<F: RuntimeWorkerFactory + ?Sized>(
|
|||||||
failure
|
failure
|
||||||
})?;
|
})?;
|
||||||
if repaired {
|
if repaired {
|
||||||
Ok(Box::new(RepairedRestoreCandidate { candidate }))
|
Ok(Box::new(EffectRetainingRestoreCandidate { candidate }))
|
||||||
} else {
|
} else {
|
||||||
Ok(candidate)
|
Ok(candidate)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A completed repair is a canonical effect even when later activation fails
|
// Completed repair or an earlier activation may have changed canonical state.
|
||||||
// before its own mutation boundary. Carry that evidence with the resource owner.
|
// Keep that operation-owned evidence through reconstruction and even an
|
||||||
struct RepairedRestoreCandidate {
|
// activation failure before the replacement candidate's own mutation boundary.
|
||||||
|
struct EffectRetainingRestoreCandidate {
|
||||||
candidate: Box<dyn RuntimeRestoreCandidate>,
|
candidate: Box<dyn RuntimeRestoreCandidate>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl RuntimeRestoreCandidate for RepairedRestoreCandidate {
|
impl RuntimeRestoreCandidate for EffectRetainingRestoreCandidate {
|
||||||
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
self.candidate.initial_state()
|
self.candidate.initial_state()
|
||||||
}
|
}
|
||||||
@@ -2953,6 +2954,9 @@ struct PendingRuntimeRestore {
|
|||||||
task: tokio::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
task: tokio::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||||
result: Mutex<Option<Result<Box<dyn RuntimeRestoreCandidate>, RuntimeRestoreFailure>>>,
|
result: Mutex<Option<Result<Box<dyn RuntimeRestoreCandidate>, RuntimeRestoreFailure>>>,
|
||||||
activated: Mutex<Option<Result<RuntimeWorkerController, RuntimeRestoreFailure>>>,
|
activated: Mutex<Option<Result<RuntimeWorkerController, RuntimeRestoreFailure>>>,
|
||||||
|
// Monotonic for this operation, not a claim that its current candidate has
|
||||||
|
// started. A reconstructed candidate has a new result and no completion in
|
||||||
|
// `activated`; prior activation/effect evidence must survive that handoff.
|
||||||
activation_started: AtomicBool,
|
activation_started: AtomicBool,
|
||||||
discard_started: AtomicBool,
|
discard_started: AtomicBool,
|
||||||
discard_result: Mutex<Option<Result<(), RuntimeRestoreCleanupFailure>>>,
|
discard_result: Mutex<Option<Result<(), RuntimeRestoreCleanupFailure>>>,
|
||||||
@@ -2960,6 +2964,16 @@ struct PendingRuntimeRestore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl PendingRuntimeRestore {
|
impl PendingRuntimeRestore {
|
||||||
|
fn owns_prepared_candidate(&self) -> bool {
|
||||||
|
matches!(
|
||||||
|
self.result
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.as_ref(),
|
||||||
|
Some(Ok(_))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
async fn wait(&self, timeout: Duration) -> Result<(), String> {
|
async fn wait(&self, timeout: Duration) -> Result<(), String> {
|
||||||
let mut task = self.task.lock().await;
|
let mut task = self.task.lock().await;
|
||||||
if let Some(message) = self
|
if let Some(message) = self
|
||||||
@@ -3279,23 +3293,40 @@ where
|
|||||||
&& (failure.durable_effects_possible
|
&& (failure.durable_effects_possible
|
||||||
|| owner.activation_started.load(Ordering::Acquire))
|
|| owner.activation_started.load(Ordering::Acquire))
|
||||||
{
|
{
|
||||||
|
let prior_effects = failure.durable_effects_possible
|
||||||
|
|| owner.activation_started.load(Ordering::Acquire);
|
||||||
let result =
|
let result =
|
||||||
match prepare_restore_operation(factory.as_ref(), owner.request.clone())
|
match prepare_restore_operation(factory.as_ref(), owner.request.clone())
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(candidate) => Ok(candidate),
|
Ok(candidate) => {
|
||||||
Err(failure) => Err(finish_restore_failure(
|
Ok(Box::new(EffectRetainingRestoreCandidate { candidate })
|
||||||
|
as Box<dyn RuntimeRestoreCandidate>)
|
||||||
|
}
|
||||||
|
Err(mut failure) => {
|
||||||
|
failure.durable_effects_possible |= prior_effects;
|
||||||
|
Err(finish_restore_failure(
|
||||||
factory.as_ref(),
|
factory.as_ref(),
|
||||||
&owner.request,
|
&owner.request,
|
||||||
failure,
|
failure,
|
||||||
)
|
)
|
||||||
.await),
|
.await)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// A completed reprepare replaces attempt resources, never
|
||||||
|
// the operation's effect history. Only a ready candidate
|
||||||
|
// may clear the old activation completion for a new attempt.
|
||||||
|
let activated = match &result {
|
||||||
|
Ok(_) => None,
|
||||||
|
Err(failure) if owner.activation_started.load(Ordering::Acquire) => {
|
||||||
|
Some(Err(failure.clone()))
|
||||||
|
}
|
||||||
|
Err(_) => None,
|
||||||
};
|
};
|
||||||
owner.activation_started.store(false, Ordering::Release);
|
|
||||||
*owner
|
*owner
|
||||||
.activated
|
.activated
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
.unwrap_or_else(std::sync::PoisonError::into_inner) = activated;
|
||||||
*owner
|
*owner
|
||||||
.result
|
.result
|
||||||
.lock()
|
.lock()
|
||||||
@@ -4019,7 +4050,9 @@ where
|
|||||||
failure.to_string(),
|
failure.to_string(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if pending.activation_started.load(Ordering::Acquire) {
|
if pending.activation_started.load(Ordering::Acquire)
|
||||||
|
&& !pending.owns_prepared_candidate()
|
||||||
|
{
|
||||||
let mut activated = match pending.activated.lock() {
|
let mut activated = match pending.activated.lock() {
|
||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
@@ -5205,7 +5238,8 @@ where
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if pending.activation_started.load(Ordering::Acquire) {
|
if pending.activation_started.load(Ordering::Acquire) && !pending.owns_prepared_candidate()
|
||||||
|
{
|
||||||
let activated = pending
|
let activated = pending
|
||||||
.activated
|
.activated
|
||||||
.lock()
|
.lock()
|
||||||
@@ -5330,7 +5364,8 @@ where
|
|||||||
return Err("restored Worker operation identity does not match".into());
|
return Err("restored Worker operation identity does not match".into());
|
||||||
}
|
}
|
||||||
self.wait_pending_restore(&pending, self.spawn_restore_timeout)?;
|
self.wait_pending_restore(&pending, self.spawn_restore_timeout)?;
|
||||||
if !pending.activation_started.swap(true, Ordering::AcqRel) {
|
if pending.owns_prepared_candidate() {
|
||||||
|
pending.activation_started.store(true, Ordering::Release);
|
||||||
let owner = pending.clone();
|
let owner = pending.clone();
|
||||||
let factory = self.factory.clone();
|
let factory = self.factory.clone();
|
||||||
let task = self.spawn_on_adapter_runtime(async move {
|
let task = self.spawn_on_adapter_runtime(async move {
|
||||||
|
|||||||
@@ -895,4 +895,210 @@ mod restore_candidate {
|
|||||||
assert_eq!(model.call_count.load(Ordering::SeqCst), 0);
|
assert_eq!(model.call_count.load(Ordering::SeqCst), 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn attachment_preparation_replay_cannot_hide_logical_authority_revocation() {
|
||||||
|
use crate::fs_store::{
|
||||||
|
FsRuntimeStoreOptions, PersistedWorkerExecution, PersistedWorkerExecutionState,
|
||||||
|
PersistedWorkerRecord,
|
||||||
|
};
|
||||||
|
let mut fixture = Fixture::new();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_secs(2);
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.clone().unwrap();
|
||||||
|
let options = FsRuntimeStoreOptions {
|
||||||
|
root: fixture._root.path().join("catalog"),
|
||||||
|
runtime_id: "candidate-runtime".into(),
|
||||||
|
display_name: None,
|
||||||
|
};
|
||||||
|
let attachment = crate::catalog::LogicalWorkdirAttachment {
|
||||||
|
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||||
|
working_directory_id: "workspace-only-workdir".into(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
};
|
||||||
|
// Only the initial Runtime catalog is seeded. The saved Profile, queue,
|
||||||
|
// restore admission, revocation, replay and Stop use their real owners.
|
||||||
|
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||||
|
options.clone(),
|
||||||
|
fixture.backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let create = fixture.request.request.clone();
|
||||||
|
runtime
|
||||||
|
.test_runtime_store()
|
||||||
|
.write_worker_record(&PersistedWorkerRecord {
|
||||||
|
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||||
|
worker_ref: worker_ref.clone(),
|
||||||
|
worker_id: worker_ref.worker_id,
|
||||||
|
profile: create.profile.clone(),
|
||||||
|
display_name: None,
|
||||||
|
profile_source: create.profile_source.reference(),
|
||||||
|
config_bundle: create.config_bundle.clone(),
|
||||||
|
created_at_ms: None,
|
||||||
|
status: crate::catalog::WorkerStatus::Stopped,
|
||||||
|
last_finished_submission_request_id: None,
|
||||||
|
execution_state: PersistedWorkerExecutionState::Available(
|
||||||
|
PersistedWorkerExecution {
|
||||||
|
request: create,
|
||||||
|
binding: None,
|
||||||
|
restore_intent: crate::catalog::WorkerRestoreIntent::Explicit,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
workspace_id: Some(scope.workspace_id.clone()),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
logical_workdir_attachments: vec![attachment.clone()],
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
drop(runtime);
|
||||||
|
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||||
|
options.clone(),
|
||||||
|
fixture.backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.worker_detail(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.workdir_attachments
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
let before = fixture.canonical_files();
|
||||||
|
let authority = || -> serde_json::Value {
|
||||||
|
serde_json::from_slice(
|
||||||
|
&fs::read(
|
||||||
|
options
|
||||||
|
.root
|
||||||
|
.join("workers")
|
||||||
|
.join(worker_ref.worker_id.to_string())
|
||||||
|
.join("worker.json"),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let (entered_tx, entered) = mpsc::channel();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(release.clone());
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_entered
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(entered_tx);
|
||||||
|
let mut request = runtime.test_restore_request(&worker_ref);
|
||||||
|
request.preparation.as_mut().unwrap().workdir_attachments =
|
||||||
|
Some(serde_json::from_value(serde_json::to_value(vec![attachment]).unwrap()).unwrap());
|
||||||
|
let first = runtime
|
||||||
|
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
entered.recv_timeout(WAIT).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"{first:?}"
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(!pending.request.logical_workdir_attachments.is_empty());
|
||||||
|
assert!(pending.request.previous_workdir_attachments.is_empty());
|
||||||
|
assert!(pending.request.deferred_workdir_sessions.is_empty());
|
||||||
|
runtime
|
||||||
|
.replace_worker_workdir_attachments_scoped(&scope, &worker_ref, Vec::new())
|
||||||
|
.unwrap();
|
||||||
|
assert!(authority().get("logical_workdir_attachments").is_none());
|
||||||
|
release.notify_one();
|
||||||
|
let replay = runtime
|
||||||
|
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
replay.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"revocation hidden by saved preparation: {replay:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
replay
|
||||||
|
.message
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.contains("current Worker authority differs"),
|
||||||
|
"{replay:?}"
|
||||||
|
);
|
||||||
|
let record = authority();
|
||||||
|
assert!(record.get("logical_workdir_attachments").is_none());
|
||||||
|
let owner = &record["restore_guard"]["owners"][&request.request_id];
|
||||||
|
assert_eq!(
|
||||||
|
owner["operation_id"],
|
||||||
|
serde_json::to_value(pending.request.operation_id).unwrap()
|
||||||
|
);
|
||||||
|
assert!(owner["receipt"].is_null());
|
||||||
|
assert_eq!(
|
||||||
|
owner["workdir_attachment_preparation_base"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
let loaded = runtime.test_runtime_store().load_runtime_state().unwrap();
|
||||||
|
let durable_guard =
|
||||||
|
serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap();
|
||||||
|
assert_eq!(durable_guard["owners"][&request.request_id], *owner);
|
||||||
|
let (coordinated, preparation) = runtime
|
||||||
|
.coordinate_worker_restore_operation(
|
||||||
|
&worker_ref,
|
||||||
|
runtime_api::WorkerRestoreCoordinationRequest {
|
||||||
|
expected_observation_token: request.expected_observation_token.clone(),
|
||||||
|
request_id: request.request_id.clone(),
|
||||||
|
preparation: None,
|
||||||
|
},
|
||||||
|
Some(&scope),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
coordinated.unwrap().state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
preparation.is_none(),
|
||||||
|
"coordinator must not export the revoked preparation again"
|
||||||
|
);
|
||||||
|
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&pending,
|
||||||
|
&fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
));
|
||||||
|
assert!(!pending.activation_started.load(Ordering::SeqCst));
|
||||||
|
fixture.assert_quiet();
|
||||||
|
assert_eq!(fixture.canonical_files(), before);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(&scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_released();
|
||||||
|
fixture.assert_quiet();
|
||||||
|
assert_eq!(fixture.canonical_files(), before);
|
||||||
|
assert!(authority().get("logical_workdir_attachments").is_none());
|
||||||
|
assert_ne!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(&scope, &worker_ref, request)
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,6 +100,148 @@ impl WorkingDirectoryMaterializer for RestoreOpenCountingMaterializer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Substitute only initial creation: a new aggregate/Session is left pending
|
||||||
|
// before its first head, without ever materializing or rewinding a saved head.
|
||||||
|
// All Restore preflight, repair, retry, activation and Stop use the real backend.
|
||||||
|
struct PendingProfileSeedBackend {
|
||||||
|
inner: Arc<WorkerRuntimeExecutionBackend>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerExecutionBackend for PendingProfileSeedBackend {
|
||||||
|
fn backend_id(&self) -> &str {
|
||||||
|
self.inner.backend_id()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||||
|
let aggregate = self
|
||||||
|
.inner
|
||||||
|
.factory
|
||||||
|
.worker_aggregate_dir(&request.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref);
|
||||||
|
self.inner
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
assert!(
|
||||||
|
!aggregate.join("session").exists(),
|
||||||
|
"pending fixture requires a new Session aggregate"
|
||||||
|
);
|
||||||
|
let manifest = manifest::WorkerManifest::from_toml(&format!(
|
||||||
|
r#"
|
||||||
|
[worker]
|
||||||
|
name = "{name}"
|
||||||
|
pwd = "./"
|
||||||
|
[scope]
|
||||||
|
allow = []
|
||||||
|
[model]
|
||||||
|
scheme = "anthropic"
|
||||||
|
model_id = "test-model"
|
||||||
|
auth = {{ kind = "none" }}
|
||||||
|
[engine]
|
||||||
|
max_tokens = 100
|
||||||
|
"#
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
let sessions = WorkerSessionStore::new(aggregate.join("session")).unwrap();
|
||||||
|
let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap();
|
||||||
|
assert!(metadata.read_by_name(&name).unwrap().is_none());
|
||||||
|
let engine =
|
||||||
|
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||||
|
MockClient::new(Vec::new()),
|
||||||
|
);
|
||||||
|
let mut head = Worker::new(
|
||||||
|
manifest.clone(),
|
||||||
|
engine,
|
||||||
|
CombinedStore::new(sessions.clone(), metadata.clone()),
|
||||||
|
WorkerWorkspaceContext::unavailable(
|
||||||
|
None,
|
||||||
|
"initial creation interrupted before first head",
|
||||||
|
),
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
Scope::empty(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
head.enable_worker_metadata_write_through().unwrap();
|
||||||
|
let mut saved = metadata.read_by_name(&name).unwrap().unwrap();
|
||||||
|
let active = saved.active.as_ref().unwrap();
|
||||||
|
assert!(active.segment_id.is_none());
|
||||||
|
assert!(sessions.list_segments(active.session_id).unwrap().is_empty());
|
||||||
|
saved.resolved_manifest_snapshot =
|
||||||
|
Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap());
|
||||||
|
metadata.write(&saved).unwrap();
|
||||||
|
drop(head);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
WorkerExecutionSpawnResult::Connected {
|
||||||
|
worker_state: protocol::WorkerStateSnapshot::initial(),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fetch_workspace_config(
|
||||||
|
&self,
|
||||||
|
request: WorkspaceConfigFetchRequest,
|
||||||
|
) -> Result<WorkspaceConfigFetchResult, String> {
|
||||||
|
self.inner.fetch_workspace_config(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn observe_workspace_prompt_projection(
|
||||||
|
&self,
|
||||||
|
projection: worker::WorkspacePromptProjection,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.inner.observe_workspace_prompt_projection(projection)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn preflight_restore(
|
||||||
|
&self,
|
||||||
|
request: &WorkerExecutionRestoreRequest,
|
||||||
|
) -> Result<(), WorkerExecutionResult> {
|
||||||
|
self.inner.preflight_restore(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
operation_id: crate::execution::WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
self.inner
|
||||||
|
.completed_restore_failure(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_worker(&self, request: WorkerExecutionRestoreRequest) -> WorkerExecutionSpawnResult {
|
||||||
|
self.inner.restore_worker(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reconcile_restore(
|
||||||
|
&self,
|
||||||
|
request: WorkerExecutionRestoreRequest,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
self.inner.reconcile_restore(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activate_restored_worker(
|
||||||
|
&self,
|
||||||
|
operation_id: crate::execution::WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.inner
|
||||||
|
.activate_restored_worker(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult {
|
||||||
|
self.inner.dispatch_input(worker_ref, input)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dispatch_method(&self, worker_ref: &WorkerRef, method: Method) -> WorkerExecutionResult {
|
||||||
|
self.inner.dispatch_method(worker_ref, method)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stop_worker_operation(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult {
|
||||||
|
self.inner.stop_worker_operation(request)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
struct ProfileFailureFixture {
|
struct ProfileFailureFixture {
|
||||||
runtime: EmbeddedRuntime,
|
runtime: EmbeddedRuntime,
|
||||||
backend: Arc<WorkerRuntimeExecutionBackend>,
|
backend: Arc<WorkerRuntimeExecutionBackend>,
|
||||||
@@ -115,6 +257,14 @@ impl ProfileFailureFixture {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn with_saved_inputs(timeout: Duration, model: Option<MockClient>) -> Self {
|
fn with_saved_inputs(timeout: Duration, model: Option<MockClient>) -> Self {
|
||||||
|
Self::with_seed(timeout, model, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_pending_head(timeout: Duration) -> Self {
|
||||||
|
Self::with_seed(timeout, None, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_seed(timeout: Duration, model: Option<MockClient>, pending_head: bool) -> Self {
|
||||||
let root = restore_failure_tempdir();
|
let root = restore_failure_tempdir();
|
||||||
let repo = root.path().join("g");
|
let repo = root.path().join("g");
|
||||||
fs::create_dir(&repo).unwrap();
|
fs::create_dir(&repo).unwrap();
|
||||||
@@ -141,22 +291,31 @@ impl ProfileFailureFixture {
|
|||||||
opens: provider_opens.clone(),
|
opens: provider_opens.clone(),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
let execution: Arc<dyn WorkerExecutionBackend> = if pending_head {
|
||||||
|
Arc::new(PendingProfileSeedBackend {
|
||||||
|
inner: backend.clone(),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
backend.clone()
|
||||||
|
};
|
||||||
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||||
crate::fs_store::FsRuntimeStoreOptions {
|
crate::fs_store::FsRuntimeStoreOptions {
|
||||||
root: store,
|
root: store,
|
||||||
runtime_id: "r".into(),
|
runtime_id: "r".into(),
|
||||||
display_name: None,
|
display_name: None,
|
||||||
},
|
},
|
||||||
backend.clone(),
|
execution,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
let mut request = create_request("restore failure recovery");
|
let mut request = create_request("restore failure recovery");
|
||||||
request.profile = ProfileSelector::Builtin("default".into());
|
request.profile = ProfileSelector::Builtin("default".into());
|
||||||
|
if !pending_head {
|
||||||
request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest {
|
request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest {
|
||||||
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
|
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||||
working_directory: working_directory_request(&repo),
|
working_directory: working_directory_request(&repo),
|
||||||
}];
|
}];
|
||||||
|
}
|
||||||
let worker = runtime.create_worker(request).unwrap();
|
let worker = runtime.create_worker(request).unwrap();
|
||||||
let seed_handle = queue_inputs.then(|| {
|
let seed_handle = queue_inputs.then(|| {
|
||||||
backend.workers.lock().unwrap()[&worker.worker_ref]
|
backend.workers.lock().unwrap()[&worker.worker_ref]
|
||||||
@@ -203,6 +362,127 @@ impl ProfileFailureFixture {
|
|||||||
*self.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(stage);
|
*self.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(stage);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn pin_unclaimed_restore_upload(&self) -> (PathBuf, BTreeMap<PathBuf, Vec<u8>>) {
|
||||||
|
let aggregate = self
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_aggregate_dir(&self.worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let name =
|
||||||
|
ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&self.worker.worker_ref);
|
||||||
|
let metadata = WorkerAggregateStore::open_read_only(&aggregate, name.clone()).unwrap();
|
||||||
|
let session_id = metadata
|
||||||
|
.read_by_name(&name)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.active
|
||||||
|
.unwrap()
|
||||||
|
.session_id;
|
||||||
|
let session = WorkerSessionStore::new(aggregate.join("session")).unwrap();
|
||||||
|
let orphan = session
|
||||||
|
.write_uploaded_file(
|
||||||
|
session_id,
|
||||||
|
"orphan.txt",
|
||||||
|
"text/plain",
|
||||||
|
b"restore effect evidence",
|
||||||
|
session_store::UploadedFileLimits::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
session
|
||||||
|
.pin_uploaded_file(session_id, &orphan, "uncommitted-restore-input")
|
||||||
|
.unwrap();
|
||||||
|
let artifacts = aggregate.join("session/artifacts/paste");
|
||||||
|
let pinned = persisted_files(&artifacts);
|
||||||
|
assert!(!pinned.is_empty());
|
||||||
|
(artifacts, pinned)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_retained_restore_effects(
|
||||||
|
&self,
|
||||||
|
request: &runtime_api::WorkerRestoreRequest,
|
||||||
|
pending: &Arc<PendingRuntimeRestore>,
|
||||||
|
stage: RuntimeRestoreStage,
|
||||||
|
) {
|
||||||
|
let retained = self
|
||||||
|
.backend
|
||||||
|
.pending_restore(&self.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.expect("resource cleanup must not remove the operation's effect owner");
|
||||||
|
assert!(Arc::ptr_eq(pending, &retained));
|
||||||
|
{
|
||||||
|
let result = retained.result.lock().unwrap();
|
||||||
|
let failure = result.as_ref().unwrap().as_ref().err().unwrap();
|
||||||
|
assert_eq!(failure.stage, stage);
|
||||||
|
assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed);
|
||||||
|
assert!(
|
||||||
|
failure.durable_effects_possible,
|
||||||
|
"resource cleanup cannot erase prior canonical effects: {failure:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let record: serde_json::Value = serde_json::from_slice(
|
||||||
|
&fs::read(
|
||||||
|
self.root
|
||||||
|
.path()
|
||||||
|
.join("s/workers")
|
||||||
|
.join(self.worker.worker_ref.worker_id.to_string())
|
||||||
|
.join("worker.json"),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
record["restore_guard"]["active_request_id"],
|
||||||
|
request.request_id
|
||||||
|
);
|
||||||
|
let operation = serde_json::to_value(pending.request.operation_id).unwrap();
|
||||||
|
let owner = &record["restore_guard"]["owners"][&request.request_id];
|
||||||
|
assert_eq!(owner["operation_id"], operation);
|
||||||
|
assert!(
|
||||||
|
owner["receipt"].is_null(),
|
||||||
|
"false terminal receipt: {owner:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
record["execution_state"]["state"],
|
||||||
|
"reconciliation_required"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
record["execution_state"]["execution"]["operation"],
|
||||||
|
"restore"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
record["execution_state"]["execution"]["intent"]["operation_id"],
|
||||||
|
operation
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.completed_restore_failures
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert!(self.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty(),
|
||||||
|
"cleanup confirmed resources, not rollback of canonical effects"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&self.worker.worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
if !pending.request.deferred_workdir_sessions.is_empty() {
|
||||||
|
self.assert_closed_deferred_sessions(pending);
|
||||||
|
}
|
||||||
|
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
|
||||||
fn assert_released(&self) {
|
fn assert_released(&self) {
|
||||||
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
|
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
|
||||||
assert!(self.backend.pending_restores.lock().unwrap().is_empty());
|
assert!(self.backend.pending_restores.lock().unwrap().is_empty());
|
||||||
@@ -422,6 +702,311 @@ impl ProfileFailureFixture {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_prior_effects_survive_read_only_reprepare_failure() {
|
||||||
|
for (activation_stage, read_stage) in [
|
||||||
|
(RuntimeRestoreStage::Feature, RuntimeRestoreStage::Metadata),
|
||||||
|
(
|
||||||
|
RuntimeRestoreStage::Controller,
|
||||||
|
RuntimeRestoreStage::Reconstruction,
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
||||||
|
let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload();
|
||||||
|
fixture.fail(activation_stage);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let first = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"{activation_stage:?}: {first:?}"
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(pending.activation_started.load(Ordering::SeqCst));
|
||||||
|
fixture.assert_retained_restore_effects(&request, &pending, activation_stage);
|
||||||
|
let effected = persisted_files(&artifacts);
|
||||||
|
assert_ne!(
|
||||||
|
effected, pinned,
|
||||||
|
"activation must actually reconcile the orphan pin"
|
||||||
|
);
|
||||||
|
|
||||||
|
// The next attempt needs no repair, and fails before any new mutation.
|
||||||
|
// Resource cleanup is confirmed, but the first attempt's pin change
|
||||||
|
// was not rolled back. Repeated sends must retain the same owner.
|
||||||
|
for _ in 0..2 {
|
||||||
|
fixture.fail(read_stage);
|
||||||
|
let replay = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
replay.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"{activation_stage:?} then {read_stage:?}: {replay:?}"
|
||||||
|
);
|
||||||
|
fixture.assert_retained_restore_effects(&request, &pending, read_stage);
|
||||||
|
assert!(pending.activation_started.load(Ordering::SeqCst));
|
||||||
|
assert_eq!(persisted_files(&artifacts), effected);
|
||||||
|
}
|
||||||
|
fixture.resume_same_operation(request, pending.request.operation_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_prior_effects_from_pending_head_repair_survive_read_only_reprepare_failure() {
|
||||||
|
let fixture = ProfileFailureFixture::with_pending_head(Duration::from_secs(5));
|
||||||
|
let aggregate = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_aggregate_dir(&fixture.worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref);
|
||||||
|
let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap();
|
||||||
|
let initial = metadata.read_by_name(&name).unwrap().unwrap();
|
||||||
|
let pending_session = initial.active.as_ref().unwrap().session_id;
|
||||||
|
assert!(initial.active.as_ref().unwrap().segment_id.is_none());
|
||||||
|
{
|
||||||
|
let before = WorkerSessionStore::new(aggregate.join("session")).unwrap();
|
||||||
|
assert!(before.list_sessions().unwrap().is_empty());
|
||||||
|
assert!(before.list_segments(pending_session).unwrap().is_empty());
|
||||||
|
}
|
||||||
|
// The independent new Worker has never published a head. Its first segment
|
||||||
|
// and active pointer must be created by production repair, not fixture
|
||||||
|
// rewinding/deletion of a materialized Session.
|
||||||
|
fixture.fail(RuntimeRestoreStage::Metadata);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let first = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"{first:?}"
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(!pending.activation_started.load(Ordering::SeqCst));
|
||||||
|
assert!(pending.request.deferred_workdir_sessions.is_empty());
|
||||||
|
fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Metadata);
|
||||||
|
let failure = pending
|
||||||
|
.result
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.err()
|
||||||
|
.unwrap()
|
||||||
|
.clone();
|
||||||
|
let repaired = WorkerAggregateStore::open_read_only(&aggregate, &name)
|
||||||
|
.unwrap()
|
||||||
|
.read_by_name(&name)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
eprintln!(
|
||||||
|
"pending-head repair first={first:?} failure={failure:?} active={:?}",
|
||||||
|
repaired.active
|
||||||
|
);
|
||||||
|
let active = repaired
|
||||||
|
.active
|
||||||
|
.as_ref()
|
||||||
|
.expect("real repair must retain the pending Session's active pointer");
|
||||||
|
assert_eq!(active.session_id, pending_session);
|
||||||
|
// Store handles opened before the first Manifest retain session_id=None;
|
||||||
|
// they cannot observe that another owner has since published this Session.
|
||||||
|
// Reopen read-only to require the canonical Manifest as well as its logs.
|
||||||
|
let session = WorkerSessionStore::open_read_only(aggregate.join("session"))
|
||||||
|
.unwrap_or_else(|error| {
|
||||||
|
panic!(
|
||||||
|
"repair Manifest reread failed: {error}; first={first:?}; failure={failure:?}; active={active:?}"
|
||||||
|
)
|
||||||
|
});
|
||||||
|
assert_eq!(session.list_sessions().unwrap(), vec![pending_session]);
|
||||||
|
let segments = session.list_segments(pending_session).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
segments.len(),
|
||||||
|
1,
|
||||||
|
"real repair must commit exactly one seed; failure={failure:?}; active={active:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(active.segment_id, Some(segments[0]));
|
||||||
|
let entries = session
|
||||||
|
.read_all_read_only(pending_session, segments[0])
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
entries.first(),
|
||||||
|
Some(LogEntry::AnnotatedSegmentStart { .. })
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
session_store::collect_state(&entries)
|
||||||
|
.system_prompt
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
repaired, initial,
|
||||||
|
"repair must publish the first active head"
|
||||||
|
);
|
||||||
|
let repaired_session = persisted_files(&aggregate.join("session"));
|
||||||
|
|
||||||
|
for _ in 0..2 {
|
||||||
|
fixture.fail(RuntimeRestoreStage::Reconstruction);
|
||||||
|
let replay = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
replay.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"repair effect cannot become a rollback on the next read failure: {replay:?}"
|
||||||
|
);
|
||||||
|
fixture.assert_retained_restore_effects(
|
||||||
|
&request,
|
||||||
|
&pending,
|
||||||
|
RuntimeRestoreStage::Reconstruction,
|
||||||
|
);
|
||||||
|
let current_metadata = WorkerAggregateStore::open_read_only(&aggregate, &name).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
current_metadata.read_by_name(&name).unwrap().unwrap(),
|
||||||
|
repaired
|
||||||
|
);
|
||||||
|
let current_session =
|
||||||
|
WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
current_session.list_sessions().unwrap(),
|
||||||
|
vec![pending_session]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
current_session.list_segments(pending_session).unwrap(),
|
||||||
|
segments
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
persisted_files(&aggregate.join("session")),
|
||||||
|
repaired_session
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fixture.resume_same_operation(request, pending.request.operation_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation_failure() {
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
||||||
|
let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload();
|
||||||
|
fixture.fail(RuntimeRestoreStage::Feature);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let first = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"{first:?}"
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Feature);
|
||||||
|
let effected = persisted_files(&artifacts);
|
||||||
|
assert_ne!(effected, pinned);
|
||||||
|
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let (entered_tx, entered) = mpsc::channel();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(release.clone());
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_entered
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(entered_tx);
|
||||||
|
let runtime = fixture.runtime.clone();
|
||||||
|
let worker_ref = fixture.worker.worker_ref.clone();
|
||||||
|
let resend = request.clone();
|
||||||
|
let retry = std::thread::spawn(move || runtime.restore_worker_operation(&worker_ref, resend));
|
||||||
|
let reached = entered.recv_timeout(Duration::from_secs(5));
|
||||||
|
// This gate is after real read-only reconstruction. Change only saved
|
||||||
|
// metadata, so the rebuilt candidate's first activation revalidation fails
|
||||||
|
// before allocation/pin/metadata/delegation mutation. No factory or repair
|
||||||
|
// implementation is replaced by a fake.
|
||||||
|
let aggregate = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_aggregate_dir(&fixture.worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref);
|
||||||
|
let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap();
|
||||||
|
let mut changed = metadata.read_by_name(&name).unwrap().unwrap();
|
||||||
|
changed.workspace_root = Some(fixture.root.path().join("changed-after-reconstruction"));
|
||||||
|
metadata.write(&changed).unwrap();
|
||||||
|
release.notify_one();
|
||||||
|
let rebuilt = retry.join().unwrap().unwrap();
|
||||||
|
reached.expect("same-operation retry must reach the real reconstructed candidate");
|
||||||
|
assert_eq!(
|
||||||
|
rebuilt.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"{rebuilt:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
assert!(pending.activation_started.load(Ordering::SeqCst));
|
||||||
|
fixture.assert_retained_restore_effects(
|
||||||
|
&request,
|
||||||
|
&pending,
|
||||||
|
RuntimeRestoreStage::Reconstruction,
|
||||||
|
);
|
||||||
|
assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), changed);
|
||||||
|
assert_eq!(persisted_files(&artifacts), effected);
|
||||||
|
|
||||||
|
fixture.fail(RuntimeRestoreStage::Metadata);
|
||||||
|
let replay = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
replay.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"successful reconstruction must not erase the older effect: {replay:?}"
|
||||||
|
);
|
||||||
|
fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Metadata);
|
||||||
|
assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), changed);
|
||||||
|
assert_eq!(persisted_files(&artifacts), effected);
|
||||||
|
fixture.resume_same_operation(request, pending.request.operation_id);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial_test::serial(worker_allocation)]
|
#[serial_test::serial(worker_allocation)]
|
||||||
fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() {
|
fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() {
|
||||||
|
|||||||
Reference in New Issue
Block a user