Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: persist worker submit activation queue

Browse Source
This commit is contained in:
Hare
2026-09-05 22:06:59 +09:00
parent aa96bbedbc
commit bb56283063
41 changed files with 2336 additions and 811 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/worker-runtime/src/execution.rs
+21 -16
View File
@@ -41,14 +41,12 @@ pub enum WorkerExecutionOperation {
Cancel,
}
/// Evidence that a user input reached the durable Worker session boundary.
///
/// This is intentionally distinct from accepting a method on the Worker's
/// in-memory channel. For Flow submissions, the committed UserInput entry also
/// carries the initial Flow runtime-state extension.
/// Evidence that a Submit request reached the durable Worker session boundary.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerInputCommitAck {
pub struct WorkerSubmissionAck {
pub submission_request_id: String,
pub submission_id: String,
pub disposition: protocol::SubmissionDisposition,
}
/// Typed execution result class. Results are transient operation outcomes and
@@ -61,7 +59,7 @@ pub struct WorkerExecutionResult {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_commit: Option<WorkerInputCommitAck>,
pub submission: Option<WorkerSubmissionAck>,
}
/// Backend result class for a Worker execution operation.
@@ -85,22 +83,26 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Accepted,
run_state,
message: None,
input_commit: None,
submission: None,
}
}
pub fn accepted_input_committed(
pub fn accepted_submission(
operation: WorkerExecutionOperation,
run_state: WorkerExecutionRunState,
submission_request_id: impl Into<String>,
submission_id: impl Into<String>,
disposition: protocol::SubmissionDisposition,
) -> Self {
Self {
operation,
outcome: WorkerExecutionOutcome::Accepted,
run_state,
message: None,
input_commit: Some(WorkerInputCommitAck {
submission: Some(WorkerSubmissionAck {
submission_request_id: submission_request_id.into(),
submission_id: submission_id.into(),
disposition,
}),
}
}
@@ -111,7 +113,7 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Busy,
run_state: WorkerExecutionRunState::Busy,
message: Some(message.into()),
input_commit: None,
submission: None,
}
}
@@ -121,7 +123,7 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Rejected,
run_state: WorkerExecutionRunState::Stopped,
message: Some(message.into()),
input_commit: None,
submission: None,
}
}
@@ -131,7 +133,7 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Errored,
run_state: WorkerExecutionRunState::Errored,
message: Some(message.into()),
input_commit: None,
submission: None,
}
}
@@ -141,7 +143,7 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Unsupported,
run_state: WorkerExecutionRunState::Stopped,
message: Some(message.into()),
input_commit: None,
submission: None,
}
}
@@ -618,14 +620,17 @@ mod tests {
use super::*;
#[test]
fn input_commit_ack_survives_json_round_trip() {
let result = WorkerExecutionResult::accepted_input_committed(
fn submission_ack_survives_json_round_trip() {
let result = WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
"request-1",
"submission-1",
protocol::SubmissionDisposition::Started,
);
let json = serde_json::to_string(&result).unwrap();
assert!(json.contains("\"submission_request_id\":\"request-1\""));
assert!(json.contains("\"submission_id\":\"submission-1\""));
assert_eq!(
serde_json::from_str::<WorkerExecutionResult>(&json).unwrap(),
crates/worker-runtime/src/http_server.rs
+8 -4
View File
@@ -2735,11 +2735,13 @@ mod tests {
_handle: &WorkerExecutionHandle,
input: WorkerInput,
) -> WorkerExecutionResult {
if let Some(submission_id) = input.submission_id {
WorkerExecutionResult::accepted_input_committed(
if let Some(submission_id) = input.submission_request_id {
WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
submission_id.clone(),
submission_id,
protocol::SubmissionDisposition::Started,
)
} else {
WorkerExecutionResult::accepted(
@@ -3059,11 +3061,13 @@ mod ws_tests {
_handle: &WorkerExecutionHandle,
input: WorkerInput,
) -> WorkerExecutionResult {
if let Some(submission_id) = input.submission_id {
WorkerExecutionResult::accepted_input_committed(
if let Some(submission_id) = input.submission_request_id {
WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
submission_id.clone(),
submission_id,
protocol::SubmissionDisposition::Started,
)
} else {
WorkerExecutionResult::accepted(
crates/worker-runtime/src/interaction.rs
+24 -6
View File
@@ -25,10 +25,10 @@ impl WorkerInputKind {
pub struct WorkerInput {
pub kind: WorkerInputKind,
pub content: String,
/// Runtime-generated correlation id. This is never accepted from public
/// JSON input and is consumed only by the execution backend.
#[serde(skip)]
pub submission_id: Option<String>,
/// Authenticated client-generated idempotency key. Runtime generates one
/// only for trusted internal callers that omit it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub submission_request_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub segments: Option<Vec<Segment>>,
}
@@ -38,7 +38,7 @@ impl WorkerInput {
Self {
kind: WorkerInputKind::User,
content: content.into(),
submission_id: None,
submission_request_id: None,
segments: None,
}
}
@@ -47,7 +47,7 @@ impl WorkerInput {
Self {
kind: WorkerInputKind::Notify,
content: content.into(),
submission_id: None,
submission_request_id: None,
segments: None,
}
}
@@ -57,6 +57,21 @@ impl WorkerInput {
mod tests {
use super::WorkerInput;
#[test]
fn submission_request_id_round_trips_for_authenticated_client_retry() {
let input: WorkerInput = serde_json::from_value(serde_json::json!({
"kind": "user",
"content": "message",
"submission_request_id": "request-1"
}))
.unwrap();
assert_eq!(input.submission_request_id.as_deref(), Some("request-1"));
assert_eq!(
serde_json::to_value(input).unwrap()["submission_request_id"],
"request-1"
);
}
#[test]
fn notify_is_an_operation_and_legacy_system_kind_is_rejected() {
assert_eq!(
@@ -78,4 +93,7 @@ mod tests {
pub struct WorkerInteractionAck {
pub worker_ref: WorkerRef,
pub status: WorkerStatus,
/// Present for User Submit and absent for non-Submit interactions.
#[serde(skip_serializing_if = "Option::is_none")]
pub submission: Option<crate::execution::WorkerSubmissionAck>,
}
crates/worker-runtime/src/runtime.rs
+62 -50
View File
@@ -748,8 +748,12 @@ impl Runtime {
let state = self.lock()?;
state.worker(&worker_ref)?.request.initial_input.clone()
} {
let expected_submission_id = Uuid::now_v7().to_string();
initial_input.submission_id = Some(expected_submission_id.clone());
let expected_submission_id = initial_input
.submission_request_id
.clone()
.filter(|request_id| !request_id.trim().is_empty())
.unwrap_or_else(|| Uuid::now_v7().to_string());
initial_input.submission_request_id = Some(expected_submission_id.clone());
let dispatch_result = backend.dispatch_input(&handle, initial_input.clone());
if !dispatch_result.is_accepted() {
let _ = backend.stop_worker(&handle);
@@ -763,9 +767,9 @@ impl Runtime {
});
}
let has_commit_ack = dispatch_result
.input_commit
.submission
.as_ref()
.is_some_and(|ack| ack.submission_id == expected_submission_id);
.is_some_and(|ack| ack.submission_request_id == expected_submission_id);
if !has_commit_ack {
let _ = backend.stop_worker(&handle);
self.rollback_failed_create(&worker_ref)?;
@@ -1146,13 +1150,18 @@ impl Runtime {
mut input: WorkerInput,
) -> Result<WorkerInteractionAck, RuntimeError> {
validate_worker_input(&input)?;
let expected_submission_id = if input.kind == WorkerInputKind::User {
let submission_id = Uuid::now_v7().to_string();
input.submission_id = Some(submission_id.clone());
Some(submission_id)
} else {
None
};
let expected_submission_id =
if matches!(input.kind, WorkerInputKind::User | WorkerInputKind::Notify) {
let submission_id = input
.submission_request_id
.clone()
.filter(|request_id| !request_id.trim().is_empty())
.unwrap_or_else(|| Uuid::now_v7().to_string());
input.submission_request_id = Some(submission_id.clone());
Some(submission_id)
} else {
None
};
self.ensure_worker_execution(worker_ref)?;
let (backend, handle) = {
let state = self.lock()?;
@@ -1191,13 +1200,13 @@ impl Runtime {
}
if let Some(expected_submission_id) = expected_submission_id
&& dispatch_result
.input_commit
.submission
.as_ref()
.is_none_or(|ack| ack.submission_id != expected_submission_id)
.is_none_or(|ack| ack.submission_request_id != expected_submission_id)
{
let result = WorkerExecutionResult::rejected(
WorkerExecutionOperation::Input,
"execution backend did not acknowledge the committed Runtime submission id",
"execution backend did not acknowledge the committed Runtime submission request id",
);
self.record_execution_result(worker_ref, result.clone())?;
return Err(RuntimeError::WorkerExecutionRejected {
@@ -1209,6 +1218,7 @@ impl Runtime {
});
}
let submission = dispatch_result.submission.clone();
let mut state = self.lock()?;
state.ensure_running()?;
let worker = state.worker_mut(worker_ref)?;
@@ -1225,6 +1235,7 @@ impl Runtime {
Ok(WorkerInteractionAck {
worker_ref: worker_ref.clone(),
status,
submission,
})
}
@@ -1706,6 +1717,7 @@ impl Runtime {
}
Ok(protocol::Event::Snapshot {
session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(),
},
greeting: protocol::Greeting {
@@ -3250,17 +3262,9 @@ fn validate_worker_input(input: &WorkerInput) -> Result<(), RuntimeError> {
#[cfg(feature = "ws-server")]
fn input_protocol_event(input: &WorkerInput) -> Option<protocol::Event> {
match input.kind {
WorkerInputKind::User => Some(protocol::Event::UserMessage {
segments: input.segments.clone().unwrap_or_else(|| {
vec![protocol::Segment::Text {
content: input.content.clone(),
}]
}),
}),
// The committed `SystemItem::Notification` is the sole agent-visible
// and Console-visible authority for Notify. A synthetic observation
// here would display the same notification twice.
WorkerInputKind::Notify => None,
// Submit is projected only after the Worker commits UserInput. Queued
// payloads must never become model- or client-visible history early.
WorkerInputKind::User | WorkerInputKind::Notify => None,
WorkerInputKind::Compact
| WorkerInputKind::ListRewindTargets
| WorkerInputKind::RegisterPeer => Some(protocol::Event::SystemItem {
@@ -3435,6 +3439,7 @@ mod tests {
);
let snapshot = protocol::Event::Snapshot {
session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(),
},
greeting: protocol::Greeting {
@@ -3475,7 +3480,7 @@ mod tests {
let input = WorkerInput {
kind: WorkerInputKind::User,
content: String::new(),
submission_id: None,
submission_request_id: None,
segments: Some(vec![protocol::Segment::Flow {
selector: "builtin:coder-review".to_string(),
}]),
@@ -3488,7 +3493,7 @@ mod tests {
let input = WorkerInput {
kind: WorkerInputKind::User,
content: String::new(),
submission_id: None,
submission_request_id: None,
segments: Some(Vec::new()),
};
assert!(matches!(
@@ -3503,7 +3508,7 @@ mod tests {
request.initial_input = Some(WorkerInput {
kind: WorkerInputKind::User,
content: String::new(),
submission_id: None,
submission_request_id: None,
segments: Some(vec![protocol::Segment::Flow {
selector: "builtin:coder-review".to_string(),
}]),
@@ -4005,7 +4010,7 @@ mod tests {
_handle: &WorkerExecutionHandle,
input: WorkerInput,
) -> WorkerExecutionResult {
let submission_id = input.submission_id.clone();
let submission_id = input.submission_request_id.clone();
self.dispatched_inputs.lock().unwrap().push(input);
let mut result = self
.dispatch_result
@@ -4013,19 +4018,21 @@ mod tests {
.unwrap()
.clone()
.unwrap_or_else(|| {
WorkerExecutionResult::accepted_input_committed(
WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
"request-test",
"test-submission",
protocol::SubmissionDisposition::Started,
)
});
if !self
.preserve_commit_ack_submission_id
.load(Ordering::SeqCst)
&& let (Some(ack), Some(submission_id)) =
(result.input_commit.as_mut(), submission_id)
(result.submission.as_mut(), submission_id)
{
ack.submission_id = submission_id;
ack.submission_request_id = submission_id;
}
result
}
@@ -4717,10 +4724,12 @@ mod tests {
#[test]
fn create_worker_uses_committed_input_ack_run_state() {
let (runtime, backend) = runtime_and_backend();
backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed(
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
"request-test",
"test-submission",
protocol::SubmissionDisposition::Started,
));
let mut request = task_request("committed initial input is already idle");
request.initial_input = Some(WorkerInput::user("start the ticket"));
@@ -4731,13 +4740,15 @@ mod tests {
}
#[test]
fn create_worker_rejects_mismatched_input_commit_acknowledgement() {
fn create_worker_rejects_mismatched_submission_acknowledgement() {
let (runtime, backend) = runtime_and_backend();
backend.preserve_commit_ack_submission_id();
backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed(
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
"request-test",
"forged-submission",
protocol::SubmissionDisposition::Started,
));
let mut request = task_request("mismatched initial input commit ack");
request.initial_input = Some(WorkerInput::user("start the ticket"));
@@ -4866,6 +4877,7 @@ mod tests {
&detail.worker_ref,
protocol::Event::Snapshot {
session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![protocol::SessionSnapshotEntry {
entry_id: "restored-log-entry".to_owned(),
timestamp: 1,
@@ -4937,10 +4949,14 @@ mod tests {
_handle: &WorkerExecutionHandle,
input: WorkerInput,
) -> WorkerExecutionResult {
WorkerExecutionResult::accepted_input_committed(
WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
input.submission_id.expect("Runtime submission id"),
"request-test",
input
.submission_request_id
.expect("Runtime submission request id"),
protocol::SubmissionDisposition::Started,
)
}
}
@@ -5030,7 +5046,7 @@ mod tests {
request.initial_input = Some(WorkerInput {
kind: WorkerInputKind::User,
content: String::new(),
submission_id: None,
submission_request_id: None,
segments: Some(vec![
protocol::Segment::Flow {
selector: "builtin:coder-review".to_string(),
@@ -5068,7 +5084,7 @@ mod tests {
let input = WorkerInput {
kind: WorkerInputKind::User,
content: String::new(),
submission_id: None,
submission_request_id: None,
segments: Some(vec![protocol::Segment::Flow {
selector: "builtin:coder-review".to_string(),
}]),
@@ -5083,11 +5099,11 @@ mod tests {
assert_eq!(dispatched[0].kind, input.kind);
assert_eq!(dispatched[0].content, input.content);
assert_eq!(dispatched[0].segments, input.segments);
let submission_id = dispatched[0]
.submission_id
let submission_request_id = dispatched[0]
.submission_request_id
.as_deref()
.expect("Runtime submission id");
Uuid::parse_str(submission_id).expect("submission id UUID");
.expect("Runtime submission request id");
Uuid::parse_str(submission_request_id).expect("submission request id UUID");
}
#[cfg(feature = "ws-server")]
@@ -5113,11 +5129,7 @@ mod tests {
let observations = runtime
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
.unwrap();
assert_eq!(observations.len(), 1);
assert!(matches!(
observations[0].payload,
protocol::Event::UserMessage { .. }
));
assert!(observations.is_empty());
runtime
.observe_worker_event(
@@ -5135,8 +5147,8 @@ mod tests {
let observations = runtime
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
.unwrap();
assert_eq!(observations.len(), 2);
let protocol::Event::SystemItem { item } = &observations[1].payload else {
assert_eq!(observations.len(), 1);
let protocol::Event::SystemItem { item } = &observations[0].payload else {
panic!("committed notification observation must be a system item");
};
assert_eq!(item["kind"], "notification");
crates/worker-runtime/src/worker_backend.rs
+137 -137
View File
@@ -39,7 +39,7 @@ use crate::working_directory::{
};
use async_trait::async_trait;
use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus};
use session_store::{CombinedStore, LogEntry, WorkerAggregateStore, WorkerSessionStore};
use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore};
#[cfg(test)]
use session_store::{FsStore, FsWorkerStore};
use tokio::runtime::Runtime;
@@ -57,11 +57,10 @@ use worker::feature::builtin::{
#[cfg(feature = "ws-server")]
use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session};
use worker::{
PreparedWorker, PromptCatalogSource, SegmentLogSink, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN,
Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout,
WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle,
WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId,
bash_output_dir_for_worker_id,
PreparedWorker, PromptCatalogSource, SegmentLogSink, Worker, WorkerBootstrap,
WorkerBootstrapError, WorkerBootstrapLayout, WorkerControllerTransport, WorkerError,
WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, WorkerWorkspaceContext,
WorkspaceClient, WorkspaceId, bash_output_dir_for_worker_id,
};
const DEFAULT_BACKEND_ID: &str = "worker-crate";
@@ -70,17 +69,6 @@ const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
// returns a typed execution error instead of leaving the outer waiter to time out.
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9);
fn user_input_has_submission(entry: &LogEntry, submission_id: &str) -> bool {
let extensions = match entry {
LogEntry::AnnotatedUserInput { extensions, .. } => extensions,
_ => return false,
};
extensions.iter().any(|extension| {
extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN
&& extension.payload["submission_id"].as_str() == Some(submission_id)
})
}
pub struct RuntimeWorkerController {
pub handle: WorkerHandle,
pub shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>,
@@ -1342,126 +1330,75 @@ where
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
}
fn send_user_input_and_wait_for_commit(
fn send_submit_and_wait_for_acceptance(
&self,
operation: WorkerExecutionOperation,
worker: WorkerHandle,
method: Method,
submission_id: String,
submission_request_id: String,
accepted_run_state: WorkerExecutionRunState,
) -> WorkerExecutionResult {
let acknowledged_submission_id = submission_id.clone();
let request_id = submission_request_id.clone();
self.run_on_adapter_runtime(async move {
// Subscribe before enqueueing the input so the acknowledgement cannot
// race with a fast Worker commit. The opaque submission id is stored in
// the same UserInput entry as the transformed Flow input and its state.
let (_, mut committed_entries) = worker.sink.subscribe_with_snapshot();
let committed_probe = worker.clone();
// Subscribe before enqueueing so a fast durable acceptance cannot
// race the Runtime acknowledgement.
let mut events = worker.subscribe();
worker
.send(method)
.await
.map_err(|err| format!("failed to send Worker method: {err}"))?;
let timeout_probe = committed_probe.clone();
let timeout_submission_id = submission_id.clone();
let acknowledgement = tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move {
let input_was_committed = || {
committed_probe
.committed_entries()
.iter()
.any(|entry| user_input_has_submission(entry, &submission_id))
};
tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move {
loop {
tokio::select! {
entry = committed_entries.recv() => {
match entry {
Ok(entry) if user_input_has_submission(&entry, &submission_id) => {
return Ok(());
}
Ok(_) => {}
Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
if input_was_committed() {
return Ok(());
}
return Err(format!(
"worker input commit acknowledgement lagged by {skipped} entry event(s)"
));
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
if input_was_committed() {
return Ok(());
}
return Err(
"worker entry stream closed before user input was committed"
.to_string(),
);
}
}
match events.recv().await {
Ok(Event::SubmissionAccepted {
submission_request_id,
submission_id,
disposition,
}) if submission_request_id == request_id => {
return Ok((submission_id, disposition));
}
event = events.recv() => {
match event {
Ok(Event::Error { message, .. }) => {
if input_was_committed() {
return Ok(());
}
return Err(format!(
"worker rejected user input before session commit: {message}"
));
}
Ok(Event::Shutdown) => {
if input_was_committed() {
return Ok(());
}
return Err(
"worker shut down before user input was committed".to_string()
);
}
Ok(_) => {}
Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
if input_was_committed() {
return Ok(());
}
return Err(format!(
"worker input commit acknowledgement lagged by {skipped} protocol event(s)"
));
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
if input_was_committed() {
return Ok(());
}
return Err(
"worker event stream closed before user input was committed"
.to_string(),
);
}
}
Ok(Event::SubmissionRejected {
submission_request_id,
message,
}) if submission_request_id == request_id => {
return Err(format!("worker rejected Submit: {message}"));
}
Ok(Event::Error { message, .. }) => {
return Err(format!(
"worker rejected Submit before durable acceptance: {message}"
));
}
Ok(Event::Shutdown) => {
return Err(
"worker shut down before Submit was durably accepted".to_string()
);
}
Ok(_) => {}
Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
return Err(format!(
"worker Submit acknowledgement lagged by {skipped} protocol event(s)"
));
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
return Err(
"worker event stream closed before Submit was durably accepted"
.to_string(),
);
}
}
}
})
.await;
match acknowledgement {
Ok(result) => result,
Err(_) => {
if timeout_probe
.committed_entries()
.iter()
.any(|entry| user_input_has_submission(entry, &timeout_submission_id))
{
Ok(())
} else {
Err("timed out waiting for worker user input commit".to_string())
}
}
}
.await
.map_err(|_| "timed out waiting for durable Worker Submit acceptance".to_string())?
})
.map(|_| {
WorkerExecutionResult::accepted_input_committed(
.map(|(submission_id, disposition)| {
WorkerExecutionResult::accepted_submission(
operation,
accepted_run_state,
acknowledged_submission_id,
submission_request_id,
submission_id,
disposition,
)
})
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
@@ -1582,8 +1519,8 @@ impl<F> Drop for WorkerRuntimeExecutionBackend<F> {
fn method_starts_turn(method: &Method) -> bool {
matches!(
method,
Method::Run { .. }
| Method::RunTracked { .. }
Method::Submit { .. }
| Method::SubmitTracked { .. }
| Method::Notify { auto_run: true, .. }
| Method::Resume
| Method::Compact
@@ -1609,8 +1546,8 @@ fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExec
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
match method {
Method::Run { .. }
| Method::RunTracked { .. }
Method::Submit { .. }
| Method::SubmitTracked { .. }
| Method::Notify { auto_run: true, .. }
| Method::Resume
| Method::Compact => WorkerExecutionRunState::Busy,
@@ -1963,6 +1900,9 @@ where
WorkerExecutionOperation::Input,
worker,
Method::Notify {
notification_request_id: input
.submission_request_id
.unwrap_or_else(protocol::new_submission_request_id),
message: input.content,
auto_run: true,
},
@@ -1975,21 +1915,23 @@ where
return result;
}
if worker.shared_state.get_status() != WorkerStatus::Idle
|| busy
let is_user_submit = input.kind == WorkerInputKind::User;
let status = worker.shared_state.get_status();
let claimed_here = status == WorkerStatus::Idle
&& busy
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_err()
{
.is_ok();
if !is_user_submit && !claimed_here {
return WorkerExecutionResult::busy(
WorkerExecutionOperation::Input,
"Worker is already running; runtime adapter v0 does not queue input",
"Worker is already running",
);
}
let (method, submission_id) = match input.kind {
let (method, submission_request_id) = match input.kind {
WorkerInputKind::User => {
let Some(submission_id) = input
.submission_id
.submission_request_id
.filter(|submission_id| !submission_id.trim().is_empty())
else {
busy.store(false, Ordering::SeqCst);
@@ -1999,11 +1941,11 @@ where
);
};
(
Method::RunTracked {
Method::SubmitTracked {
submission_request_id: submission_id.clone(),
input: input.segments.unwrap_or_else(|| {
vec![Segment::text(input.content.trim().to_string())]
}),
submission_id: submission_id.clone(),
},
Some(submission_id),
)
@@ -2021,21 +1963,21 @@ where
),
};
let accepted_run_state = match method {
Method::Run { .. }
| Method::RunTracked { .. }
Method::Submit { .. }
| Method::SubmitTracked { .. }
| Method::Notify { .. }
| Method::Compact => WorkerExecutionRunState::Busy,
_ => WorkerExecutionRunState::Idle,
};
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
let waits_for_user_input_commit = submission_id.is_some();
let waits_for_submission_acceptance = submission_request_id.is_some();
let result = if waits_for_user_input_commit {
self.send_user_input_and_wait_for_commit(
let result = if waits_for_submission_acceptance {
self.send_submit_and_wait_for_acceptance(
WorkerExecutionOperation::Input,
worker,
method,
submission_id.expect("tracked Run has submission id"),
submission_request_id.expect("Submit must have a submission request id"),
accepted_run_state,
)
} else {
@@ -2046,7 +1988,9 @@ where
accepted_run_state,
)
};
if accepted_is_idle || result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
if accepted_is_idle
|| (claimed_here
&& result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
{
busy.store(false, Ordering::SeqCst);
}
@@ -3400,6 +3344,60 @@ mod tests {
);
}
#[test]
fn running_worker_accepts_a_second_submit_as_queued() {
let client = MockClient::sequential(vec![MockResponse::Hang(vec![])]);
let runtime_base = tempfile::tempdir().unwrap();
let cwd = tempfile::tempdir().unwrap();
let store = tempfile::tempdir().unwrap();
let factory = MockFactory {
client,
runtime_base: runtime_base.path().to_path_buf(),
cwd: cwd.path().to_path_buf(),
store_dir: store.path().join("sessions"),
worker_metadata_dir: store.path().join("workers"),
observed_cwds: Arc::new(Mutex::new(Vec::new())),
observed_workspace_clients: Arc::new(Mutex::new(Vec::new())),
};
let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
let runtime =
EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend).unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let detail = runtime
.create_worker(create_request("queued-submit"))
.unwrap();
let mut first_input = WorkerInput::user("first");
first_input.submission_request_id = Some("request-first".into());
let first = runtime
.send_input(&detail.worker_ref, first_input.clone())
.unwrap();
assert_eq!(
first.submission.as_ref().map(|ack| ack.disposition),
Some(protocol::SubmissionDisposition::Started)
);
let retry = runtime.send_input(&detail.worker_ref, first_input).unwrap();
assert_eq!(retry.submission, first.submission);
let mut conflicting_retry = WorkerInput::user("different");
conflicting_retry.submission_request_id = Some("request-first".into());
assert!(
runtime
.send_input(&detail.worker_ref, conflicting_retry)
.is_err(),
"same request id with a different payload must fail"
);
let mut second_input = WorkerInput::user("second");
second_input.submission_request_id = Some("request-second".into());
let second = runtime
.send_input(&detail.worker_ref, second_input)
.unwrap();
assert_eq!(
second.submission.as_ref().map(|ack| ack.disposition),
Some(protocol::SubmissionDisposition::Queued)
);
}
#[test]
fn create_with_initial_input_returns_after_session_commit() {
let client = MockClient::new(simple_text_events());
@@ -3449,8 +3447,10 @@ mod tests {
};
extensions
.iter()
.find(|extension| extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN)
.and_then(|extension| extension.payload["submission_id"].as_str())
.find(|extension| extension.domain == "worker.pending_activations.v1")
.and_then(|extension| {
extension.payload["receipts"][0]["submission_id"].as_str()
})
})
.expect("committed input submission id");
uuid::Uuid::parse_str(submission_id).expect("opaque submission id is a UUID");