Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: fence cleanup on tracked run completion

Browse Source
This commit is contained in:
Hare
2026-10-04 13:40:10 +09:00
parent 1fbf93c8c1
commit 20109b88df
18 changed files with 123 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/protocol/src/lib.rs
+11
View File
@@ -175,6 +175,11 @@ pub struct WorkerStateSnapshot {
schemars(range(min = 0, max = 9_007_199_254_740_991_u64))
)]
pub last_command_id: u64,
/// Most recent Submit request whose controller Run has returned and whose
/// post-Run state transition has been published. This is a teardown fence,
/// not a successful-result claim; cancelled and failed Runs may also finish.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub last_finished_submission_request_id: Option<String>,
pub state: WorkerState,
}
@@ -182,6 +187,7 @@ impl WorkerStateSnapshot {
pub fn initial() -> Self {
Self {
last_command_id: 0,
last_finished_submission_request_id: None,
state: WorkerState::Idle,
}
}
@@ -209,6 +215,7 @@ impl From<WorkerStatus> for WorkerStateSnapshot {
};
Self {
last_command_id: 0,
last_finished_submission_request_id: None,
state,
}
}
@@ -1925,6 +1932,7 @@ mod tests {
fn worker_state_snapshot_wire_shape_has_one_authoritative_state() {
let snapshot = WorkerStateSnapshot {
last_command_id: 7,
last_finished_submission_request_id: Some("request-7".to_string()),
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
};
let value = serde_json::to_value(&snapshot).unwrap();
@@ -1932,6 +1940,7 @@ mod tests {
value,
serde_json::json!({
"last_command_id": 7,
"last_finished_submission_request_id": "request-7",
"state": {
"kind": "busy",
"state": { "kind": "run", "state": "running" }
@@ -2648,6 +2657,7 @@ mod tests {
let event = Event::WorkerState {
snapshot: WorkerStateSnapshot {
last_command_id: 9,
last_finished_submission_request_id: None,
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
},
};
@@ -2669,6 +2679,7 @@ mod tests {
snapshot: WorkerStateSnapshot {
last_command_id: 9,
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
..
}
}
));
crates/tui/src/console/mod.rs
+1
View File
@@ -508,6 +508,7 @@ impl RetainedSnapshotSocket {
},
state: protocol::WorkerStateSnapshot {
last_command_id: 0,
last_finished_submission_request_id: None,
state: protocol::WorkerState::Idle,
},
in_flight: protocol::InFlightSnapshot::default(),
crates/worker-runtime/src/runtime.rs
+2
View File
@@ -8028,6 +8028,7 @@ mod tests {
protocol::WorkerRunState::Running,
)),
last_command_id: 2,
last_finished_submission_request_id: Some("request-old".to_string()),
};
{
let mut state = runtime.lock().unwrap();
@@ -8049,6 +8050,7 @@ mod tests {
let fresh = protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
last_command_id: 0,
last_finished_submission_request_id: None,
};
{
let mut state = runtime.lock().unwrap();
crates/worker-runtime/src/worker_backend.rs
+3
View File
@@ -3824,11 +3824,13 @@ mod tests {
protocol::WorkerRunState::Running,
)),
last_command_id: 2,
last_finished_submission_request_id: Some("request-old".to_string()),
};
let current = Arc::new(RwLock::new(running));
let idle = protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
last_command_id: 0,
last_finished_submission_request_id: None,
};
let mut replacement = Event::WorkerState {
snapshot: idle.clone(),
@@ -3841,6 +3843,7 @@ mod tests {
protocol::WorkerRunState::Paused,
)),
last_command_id: 3,
last_finished_submission_request_id: None,
};
let mut acknowledgement = Event::CommandAcknowledged {
acknowledgement: protocol::WorkerCommandAcknowledgement {
crates/worker/src/controller.rs
+22
View File
@@ -300,6 +300,7 @@ async fn finish_controller_run<C, St>(
runtime_dir: &RuntimeDir,
working_event_tx: &broadcast::Sender<Event>,
new_status: WorkerStatus,
finished_submission_request_id: Option<String>,
) where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + 'static,
@@ -320,7 +321,20 @@ async fn finish_controller_run<C, St>(
Some(worker.total_tokens().tokens),
protocol::ContextTokenSource::Estimated,
);
if let Some(submission_request_id) = finished_submission_request_id {
let state = match new_status {
WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle,
WorkerStatus::Running => {
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
}
WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
};
let snapshot = shared_state.finish_submission_run(state, submission_request_id);
let _ = runtime_dir.write_status(shared_state).await;
let _ = working_event_tx.send(Event::WorkerState { snapshot });
} else {
set_controller_status(shared_state, runtime_dir, working_event_tx, new_status).await;
}
}
/// Pending turn launch staged by an event handler for the next outer-loop
@@ -2023,6 +2037,10 @@ async fn controller_loop<C, St>(
// interrupted/error turn from being carried into the next snapshot.
worker.clear_in_flight_events();
let parent_originated = run.is_parent_originated();
let finished_submission_request_id = match &run {
PendingRun::Submit(submission) => Some(submission.submission_request_id.clone()),
PendingRun::RunForNotification { .. } | PendingRun::Resume => None,
};
let user_input_submit = matches!(&run, PendingRun::Submit(_));
if !user_input_submit {
set_controller_status(
@@ -2121,12 +2139,16 @@ async fn controller_loop<C, St>(
}
}
}
let finished_submission_request_id = (new_status != WorkerStatus::Paused)
.then_some(finished_submission_request_id)
.flatten();
finish_controller_run(
&mut worker,
&shared_state,
&runtime_dir,
&working_event_tx,
new_status,
finished_submission_request_id,
)
.await;
if shutdown {
crates/worker/src/internal_worker.rs
+1
View File
@@ -335,6 +335,7 @@ fn send_internal_worker_state(
let _ = event_tx.send(Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
last_command_id: 0,
last_finished_submission_request_id: None,
state,
},
});
crates/worker/src/shared_state.rs
+26
View File
@@ -117,6 +117,23 @@ impl WorkerSharedState {
snapshot.clone()
}
/// Publish the post-Run state and the exact Submit request whose controller
/// invocation has returned. Updating both in one snapshot prevents observers
/// from treating an earlier Idle state as evidence for a later Run.
pub(crate) fn finish_submission_run(
&self,
state: WorkerState,
submission_request_id: String,
) -> WorkerStateSnapshot {
let mut snapshot = self
.state
.write()
.expect("worker state lock poisoned; refusing an inferred fallback state");
snapshot.state = state;
snapshot.last_finished_submission_request_id = Some(submission_request_id);
snapshot.clone()
}
pub(crate) fn admit_command(
&self,
envelope: WorkerCommandEnvelope,
@@ -305,6 +322,14 @@ mod tests {
assert_eq!(snapshot.state, paused);
assert_eq!(snapshot.last_command_id, 0);
assert_eq!(state.catalog_status(), WorkerStatus::Paused);
let finished = state.finish_submission_run(WorkerState::Idle, "request-1".to_string());
assert_eq!(finished.state, WorkerState::Idle);
assert_eq!(
finished.last_finished_submission_request_id.as_deref(),
Some("request-1")
);
assert_eq!(finished.last_command_id, 0);
}
#[test]
@@ -319,6 +344,7 @@ mod tests {
state.snapshot(),
WorkerStateSnapshot {
last_command_id: 9,
last_finished_submission_request_id: None,
state: WorkerState::Idle,
}
);
crates/worker/tests/controller_test.rs
+8 -6
View File
@@ -1299,12 +1299,10 @@ async fn run_end_returns_to_idle_without_busy_status() {
let worker = make_worker(client).await;
let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe();
let submission_request_id = protocol::new_submission_request_id();
handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.send(Method::submit_text(submission_request_id.clone(), "Hello"))
.await
.unwrap();
@@ -1319,7 +1317,11 @@ async fn run_end_returns_to_idle_without_busy_status() {
saw_run_end = true;
}
Ok(Event::WorkerState { snapshot })
if saw_run_end && snapshot.catalog_status() == WorkerStatus::Idle => {
if saw_run_end
&& snapshot.catalog_status() == WorkerStatus::Idle
&& snapshot.last_finished_submission_request_id.as_deref()
== Some(submission_request_id.as_str()) =>
{
saw_idle_status = true;
break;
}
@@ -1334,7 +1336,7 @@ async fn run_end_returns_to_idle_without_busy_status() {
assert!(saw_run_end, "expected RunEnd::Finished");
assert!(
saw_idle_status,
"expected idle status immediately after RunEnd"
"expected exact Submit teardown fence in idle status immediately after RunEnd"
);
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
}
crates/workspace-server/src/server.rs
+23 -10
View File
@@ -3774,21 +3774,30 @@ impl WorkspaceApi {
let worker = attempt.worker.as_ref().ok_or_else(|| {
Error::Store("claimed Backend Job Worker cleanup lost its binding".to_string())
})?;
let Some(runtime_run_id) = attempt.runtime_run_id.as_deref() else {
// Attempts created before tracked Runtime submissions can only be
// recovered after a Backend restart, which has already ended their
// in-process result response and Run. Canonical stop/remove remains
// the safety boundary for those legacy leftovers.
return Ok(());
};
loop {
match self.runtime.worker(worker) {
Ok(detail)
if detail.state.eq_ignore_ascii_case("stopped")
|| detail.worker_state.as_ref().is_some_and(|snapshot| {
snapshot.last_command_id > 0
&& snapshot.state == protocol::WorkerState::Idle
snapshot.state == protocol::WorkerState::Idle
&& snapshot.last_finished_submission_request_id.as_deref()
== Some(runtime_run_id)
}) =>
{
return Ok(());
}
Ok(_) => {
// The result capability runs inside this Worker Run. Idle
// with a committed command is the Runtime-owned fence that
// the HTTP response and Run post-processing have completed.
// The result capability runs inside this tracked Submit. The
// Worker controller publishes this exact request id together
// with its post-Run state only after the invocation returns,
// so an unrelated initial Idle snapshot cannot release cleanup.
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
Err(RuntimeRegistryError::UnknownWorker { .. }) => return Ok(()),
@@ -41796,12 +41805,13 @@ mod tests {
} else if let Some(submission_request_id) = submission_request_id {
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
worker_runtime::execution::WorkerExecutionOperation::Input,
submission_request_id,
submission_request_id.clone(),
uuid::Uuid::now_v7().to_string(),
protocol::SubmissionDisposition::Started,
)
.with_worker_state(protocol::WorkerStateSnapshot {
last_command_id: 1,
last_command_id: 0,
last_finished_submission_request_id: Some(submission_request_id),
state: protocol::WorkerState::Idle,
})
} else {
@@ -43175,7 +43185,7 @@ mod tests {
}
#[tokio::test]
async fn backend_job_result_http_response_precedes_cleanup_until_run_idle() {
async fn backend_job_result_http_response_precedes_cleanup_until_tracked_run_finishes() {
let temp = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(temp.path()).await;
let request = BackendJobRequest {
@@ -43192,11 +43202,13 @@ mod tests {
};
let dispatched = api.dispatch_backend_job(&request).unwrap();
let worker = dispatched.attempt.worker.clone().unwrap();
let runtime_run_id = dispatched.attempt.runtime_run_id.clone().unwrap();
seed_worker_session_for_cleanup(temp.path(), &worker);
execution.publish_worker_state(
&worker,
protocol::WorkerStateSnapshot {
last_command_id: 1,
last_command_id: 0,
last_finished_submission_request_id: None,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
@@ -43249,7 +43261,8 @@ mod tests {
execution.publish_worker_state(
&worker,
protocol::WorkerStateSnapshot {
last_command_id: 2,
last_command_id: 0,
last_finished_submission_request_id: Some(runtime_run_id),
state: protocol::WorkerState::Idle,
},
);
openapi/server-api.json
+8 -1
View File
@@ -12568,6 +12568,13 @@
"minimum": 0,
"type": "integer"
},
"last_finished_submission_request_id": {
"description": "Most recent Submit request whose controller Run has returned and whose\npost-Run state transition has been published. This is a teardown fence,\nnot a successful-result claim; cancelled and failed Runs may also finish.",
"type": [
"string",
"null"
]
},
"state": {
"$ref": "#/components/schemas/WorkerState"
}
@@ -14787,7 +14794,7 @@
"info": {
"title": "Yoi Server API",
"version": "0.1.0",
"x-yoi-source-digest": "sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6"
"x-yoi-source-digest": "sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870"
},
"openapi": "3.1.0",
"paths": {
web/workspace/src/lib/generated/companion-api.ts
+1 -1
View File
@@ -1,6 +1,6 @@
// @generated by yoi-companion-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/companion-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
web/workspace/src/lib/generated/protocol-validator.ts
+1 -1
View File
File diff suppressed because one or more lines are too long
web/workspace/src/lib/generated/protocol.ts
+7 -1
View File
@@ -34,7 +34,13 @@ export type WorkerStateSnapshot = {
/**
* Highest lifecycle command id observed by this controller instance.
*/
last_command_id: number, state: WorkerState, };
last_command_id: number,
/**
* Most recent Submit request whose controller Run has returned and whose
* post-Run state transition has been published. This is a teardown fence,
* not a successful-result claim; cancelled and failed Runs may also finish.
*/
last_finished_submission_request_id?: string | null, state: WorkerState, };
export type TurnResult = "finished" | "paused";
web/workspace/src/lib/generated/repository-api.ts
+2 -2
View File
@@ -1,6 +1,6 @@
// @generated by yoi-repository-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
// Output: web/workspace/src/lib/generated/repository-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
@@ -69,7 +69,7 @@ export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyState; };
export type WorkerStateSnapshot = { last_command_id: number; state: WorkerState; };
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
web/workspace/src/lib/generated/runtime-api.ts
+2 -2
View File
@@ -1,6 +1,6 @@
// @generated by yoi-runtime-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/runtime-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
@@ -159,7 +159,7 @@ export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyState; };
export type WorkerStateSnapshot = { last_command_id: number; state: WorkerState; };
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
web/workspace/src/lib/generated/ticket-api.ts
+1 -1
View File
@@ -1,6 +1,6 @@
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/ticket-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
web/workspace/src/lib/generated/workdir-api.ts
+1 -1
View File
@@ -1,6 +1,6 @@
// @generated by yoi-workdir-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
// Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/workdir-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check
web/workspace/src/lib/generated/worker-launch-api.ts
+2 -2
View File
@@ -1,6 +1,6 @@
// @generated by yoi-worker-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/worker-launch-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
@@ -67,7 +67,7 @@ export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyState; };
export type WorkerStateSnapshot = { last_command_id: number; state: WorkerState; };
export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };