fix: acknowledge durable submissions before run preparation
This commit is contained in:
@@ -929,15 +929,15 @@ impl Runtime {
|
||||
result: dispatch_result,
|
||||
});
|
||||
}
|
||||
let has_commit_ack = dispatch_result
|
||||
let has_durable_acceptance = dispatch_result
|
||||
.submission
|
||||
.as_ref()
|
||||
.is_some_and(|ack| ack.submission_request_id == expected_submission_id);
|
||||
if !has_commit_ack {
|
||||
if !has_durable_acceptance {
|
||||
self.cleanup_connected_failed_create(&backend, &worker_ref, &handle)?;
|
||||
let result = WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::Input,
|
||||
"execution backend accepted initial input without a durable session commit acknowledgement",
|
||||
"execution backend accepted initial input without a durable submission acknowledgement",
|
||||
);
|
||||
return Err(RuntimeError::WorkerExecutionRejected {
|
||||
worker_id: worker_ref.worker_id.clone(),
|
||||
@@ -4222,7 +4222,7 @@ mod tests {
|
||||
repository_accesses: Mutex<Vec<WorkingDirectoryRepositoryAccessRequest>>,
|
||||
repository_access_available: AtomicBool,
|
||||
working_directory_requests: Mutex<Vec<WorkingDirectoryRequest>>,
|
||||
preserve_commit_ack_submission_id: AtomicBool,
|
||||
preserve_submission_acknowledgement_id: AtomicBool,
|
||||
#[cfg(feature = "ws-server")]
|
||||
snapshots: Mutex<BTreeMap<WorkerId, protocol::Event>>,
|
||||
}
|
||||
@@ -4236,8 +4236,8 @@ mod tests {
|
||||
*self.stop_result.lock().unwrap() = Some(result);
|
||||
}
|
||||
|
||||
fn preserve_commit_ack_submission_id(&self) {
|
||||
self.preserve_commit_ack_submission_id
|
||||
fn preserve_submission_acknowledgement_id(&self) {
|
||||
self.preserve_submission_acknowledgement_id
|
||||
.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
@@ -4382,7 +4382,7 @@ mod tests {
|
||||
)
|
||||
});
|
||||
if !self
|
||||
.preserve_commit_ack_submission_id
|
||||
.preserve_submission_acknowledgement_id
|
||||
.load(Ordering::SeqCst)
|
||||
&& let (Some(ack), Some(submission_id)) =
|
||||
(result.submission.as_mut(), submission_id)
|
||||
@@ -5202,7 +5202,7 @@ mod tests {
|
||||
#[test]
|
||||
fn create_worker_rejects_mismatched_submission_acknowledgement() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
backend.preserve_commit_ack_submission_id();
|
||||
backend.preserve_submission_acknowledgement_id();
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
"request-test",
|
||||
@@ -5225,7 +5225,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_worker_rejects_initial_input_without_commit_acknowledgement() {
|
||||
fn create_worker_rejects_initial_input_without_durable_submission_acknowledgement() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Input,
|
||||
|
||||
@@ -90,11 +90,11 @@ use worker::{
|
||||
const DEFAULT_BACKEND_ID: &str = "worker-crate";
|
||||
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const SPAWN_RESTORE_TASK_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const USER_INPUT_TASK_TIMEOUT: Duration = Duration::from_secs(125);
|
||||
const USER_INPUT_TASK_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const WORKSPACE_CONFIG_HTTP_TIMEOUT: Duration = Duration::from_secs(8);
|
||||
const MAX_WORKSPACE_CONFIG_RESPONSE_BYTES: usize = 72 * 1024 * 1024;
|
||||
// Leave adapter cancellation margin after the durable submission deadline.
|
||||
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9);
|
||||
|
||||
pub struct RuntimeWorkerController {
|
||||
pub handle: WorkerHandle,
|
||||
@@ -2546,7 +2546,7 @@ mod tests {
|
||||
assert!(USER_INPUT_TASK_TIMEOUT > USER_INPUT_COMMIT_TIMEOUT);
|
||||
assert_eq!(
|
||||
USER_INPUT_TASK_TIMEOUT - USER_INPUT_COMMIT_TIMEOUT,
|
||||
Duration::from_secs(5)
|
||||
Duration::from_secs(1)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3562,7 +3562,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_with_initial_input_returns_after_session_commit() {
|
||||
fn create_with_initial_input_returns_after_durable_submission_acceptance() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
let cwd = tempfile::tempdir().unwrap();
|
||||
@@ -3586,21 +3586,32 @@ mod tests {
|
||||
|
||||
let detail = runtime.create_worker(request).unwrap();
|
||||
|
||||
let entries = backend
|
||||
let handle = backend
|
||||
.workers
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&detail.worker_ref)
|
||||
.expect("live Worker execution")
|
||||
.handle
|
||||
.committed_entries();
|
||||
assert!(entries.iter().any(|entry| {
|
||||
matches!(
|
||||
entry,
|
||||
LogEntry::AnnotatedUserInput { segments, .. }
|
||||
if segments == &vec![Segment::text("start the ticket")]
|
||||
)
|
||||
}));
|
||||
.clone();
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(2);
|
||||
let entries = loop {
|
||||
let entries = handle.committed_entries();
|
||||
if entries.iter().any(|entry| {
|
||||
matches!(
|
||||
entry,
|
||||
LogEntry::AnnotatedUserInput { segments, .. }
|
||||
if segments == &vec![Segment::text("start the ticket")]
|
||||
)
|
||||
}) {
|
||||
break entries;
|
||||
}
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"durably accepted initial input must eventually commit to history"
|
||||
);
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
};
|
||||
let submission_id = entries
|
||||
.iter()
|
||||
.find_map(|entry| {
|
||||
|
||||
Reference in New Issue
Block a user