fix: fence cleanup on tracked run completion

crates/protocol/src/lib.rs

@@ -175,6 +175,11 @@ pub struct WorkerStateSnapshot {
         schemars(range(min = 0, max = 9_007_199_254_740_991_u64))
     )]
     pub last_command_id: u64,
+    /// Most recent Submit request whose controller Run has returned and whose
+    /// post-Run state transition has been published. This is a teardown fence,
+    /// not a successful-result claim; cancelled and failed Runs may also finish.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub last_finished_submission_request_id: Option<String>,
     pub state: WorkerState,
 }
 
@@ -182,6 +187,7 @@ impl WorkerStateSnapshot {
     pub fn initial() -> Self {
         Self {
             last_command_id: 0,
+            last_finished_submission_request_id: None,
             state: WorkerState::Idle,
         }
     }
@@ -209,6 +215,7 @@ impl From<WorkerStatus> for WorkerStateSnapshot {
         };
         Self {
             last_command_id: 0,
+            last_finished_submission_request_id: None,
             state,
         }
     }
@@ -1925,6 +1932,7 @@ mod tests {
     fn worker_state_snapshot_wire_shape_has_one_authoritative_state() {
         let snapshot = WorkerStateSnapshot {
             last_command_id: 7,
+            last_finished_submission_request_id: Some("request-7".to_string()),
             state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
         };
         let value = serde_json::to_value(&snapshot).unwrap();
@@ -1932,6 +1940,7 @@ mod tests {
             value,
             serde_json::json!({
                 "last_command_id": 7,
+                "last_finished_submission_request_id": "request-7",
                 "state": {
                     "kind": "busy",
                     "state": { "kind": "run", "state": "running" }
@@ -2648,6 +2657,7 @@ mod tests {
         let event = Event::WorkerState {
             snapshot: WorkerStateSnapshot {
                 last_command_id: 9,
+                last_finished_submission_request_id: None,
                 state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
             },
         };
@@ -2669,6 +2679,7 @@ mod tests {
                 snapshot: WorkerStateSnapshot {
                     last_command_id: 9,
                     state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
+                    ..
                 }
             }
         ));

crates/tui/src/console/mod.rs

@@ -508,6 +508,7 @@ impl RetainedSnapshotSocket {
                 },
                 state: protocol::WorkerStateSnapshot {
                     last_command_id: 0,
+                    last_finished_submission_request_id: None,
                     state: protocol::WorkerState::Idle,
                 },
                 in_flight: protocol::InFlightSnapshot::default(),

crates/worker-runtime/src/runtime.rs

@@ -8028,6 +8028,7 @@ mod tests {
                 protocol::WorkerRunState::Running,
             )),
             last_command_id: 2,
+            last_finished_submission_request_id: Some("request-old".to_string()),
         };
         {
             let mut state = runtime.lock().unwrap();
@@ -8049,6 +8050,7 @@ mod tests {
         let fresh = protocol::WorkerStateSnapshot {
             state: protocol::WorkerState::Idle,
             last_command_id: 0,
+            last_finished_submission_request_id: None,
         };
         {
             let mut state = runtime.lock().unwrap();

crates/worker-runtime/src/worker_backend.rs

@@ -3824,11 +3824,13 @@ mod tests {
                 protocol::WorkerRunState::Running,
             )),
             last_command_id: 2,
+            last_finished_submission_request_id: Some("request-old".to_string()),
         };
         let current = Arc::new(RwLock::new(running));
         let idle = protocol::WorkerStateSnapshot {
             state: protocol::WorkerState::Idle,
             last_command_id: 0,
+            last_finished_submission_request_id: None,
         };
         let mut replacement = Event::WorkerState {
             snapshot: idle.clone(),
@@ -3841,6 +3843,7 @@ mod tests {
                 protocol::WorkerRunState::Paused,
             )),
             last_command_id: 3,
+            last_finished_submission_request_id: None,
         };
         let mut acknowledgement = Event::CommandAcknowledged {
             acknowledgement: protocol::WorkerCommandAcknowledgement {

crates/worker/src/controller.rs

@@ -300,6 +300,7 @@ async fn finish_controller_run<C, St>(
     runtime_dir: &RuntimeDir,
     working_event_tx: &broadcast::Sender<Event>,
     new_status: WorkerStatus,
+    finished_submission_request_id: Option<String>,
 ) where
     C: LlmClient + Clone + 'static,
     St: Store + WorkerMetadataStore + Clone + 'static,
@@ -320,7 +321,20 @@ async fn finish_controller_run<C, St>(
         Some(worker.total_tokens().tokens),
         protocol::ContextTokenSource::Estimated,
     );
+    if let Some(submission_request_id) = finished_submission_request_id {
+        let state = match new_status {
+            WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle,
+            WorkerStatus::Running => {
+                WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
+            }
+            WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
+        };
+        let snapshot = shared_state.finish_submission_run(state, submission_request_id);
+        let _ = runtime_dir.write_status(shared_state).await;
+        let _ = working_event_tx.send(Event::WorkerState { snapshot });
+    } else {
         set_controller_status(shared_state, runtime_dir, working_event_tx, new_status).await;
+    }
 }
 
 /// Pending turn launch staged by an event handler for the next outer-loop
@@ -2023,6 +2037,10 @@ async fn controller_loop<C, St>(
             // interrupted/error turn from being carried into the next snapshot.
             worker.clear_in_flight_events();
             let parent_originated = run.is_parent_originated();
+            let finished_submission_request_id = match &run {
+                PendingRun::Submit(submission) => Some(submission.submission_request_id.clone()),
+                PendingRun::RunForNotification { .. } | PendingRun::Resume => None,
+            };
             let user_input_submit = matches!(&run, PendingRun::Submit(_));
             if !user_input_submit {
                 set_controller_status(
@@ -2121,12 +2139,16 @@ async fn controller_loop<C, St>(
                     }
                 }
             }
+            let finished_submission_request_id = (new_status != WorkerStatus::Paused)
+                .then_some(finished_submission_request_id)
+                .flatten();
             finish_controller_run(
                 &mut worker,
                 &shared_state,
                 &runtime_dir,
                 &working_event_tx,
                 new_status,
+                finished_submission_request_id,
             )
             .await;
             if shutdown {

crates/worker/src/internal_worker.rs

@@ -335,6 +335,7 @@ fn send_internal_worker_state(
     let _ = event_tx.send(Event::WorkerState {
         snapshot: protocol::WorkerStateSnapshot {
             last_command_id: 0,
+            last_finished_submission_request_id: None,
             state,
         },
     });

crates/worker/src/shared_state.rs

@@ -117,6 +117,23 @@ impl WorkerSharedState {
         snapshot.clone()
     }
 
+    /// Publish the post-Run state and the exact Submit request whose controller
+    /// invocation has returned. Updating both in one snapshot prevents observers
+    /// from treating an earlier Idle state as evidence for a later Run.
+    pub(crate) fn finish_submission_run(
+        &self,
+        state: WorkerState,
+        submission_request_id: String,
+    ) -> WorkerStateSnapshot {
+        let mut snapshot = self
+            .state
+            .write()
+            .expect("worker state lock poisoned; refusing an inferred fallback state");
+        snapshot.state = state;
+        snapshot.last_finished_submission_request_id = Some(submission_request_id);
+        snapshot.clone()
+    }
+
     pub(crate) fn admit_command(
         &self,
         envelope: WorkerCommandEnvelope,
@@ -305,6 +322,14 @@ mod tests {
         assert_eq!(snapshot.state, paused);
         assert_eq!(snapshot.last_command_id, 0);
         assert_eq!(state.catalog_status(), WorkerStatus::Paused);
+
+        let finished = state.finish_submission_run(WorkerState::Idle, "request-1".to_string());
+        assert_eq!(finished.state, WorkerState::Idle);
+        assert_eq!(
+            finished.last_finished_submission_request_id.as_deref(),
+            Some("request-1")
+        );
+        assert_eq!(finished.last_command_id, 0);
     }
 
     #[test]
@@ -319,6 +344,7 @@ mod tests {
             state.snapshot(),
             WorkerStateSnapshot {
                 last_command_id: 9,
+                last_finished_submission_request_id: None,
                 state: WorkerState::Idle,
             }
         );

crates/worker/tests/controller_test.rs

@@ -1299,12 +1299,10 @@ async fn run_end_returns_to_idle_without_busy_status() {
     let worker = make_worker(client).await;
     let handle = spawn_controller(worker).await;
     let mut rx = handle.subscribe();
+    let submission_request_id = protocol::new_submission_request_id();
 
     handle
-        .send(Method::submit_text(
+        .send(Method::submit_text(submission_request_id.clone(), "Hello"))
-            protocol::new_submission_request_id(),
-            "Hello",
-        ))
         .await
         .unwrap();
 
@@ -1319,7 +1317,11 @@ async fn run_end_returns_to_idle_without_busy_status() {
                         saw_run_end = true;
                     }
                     Ok(Event::WorkerState { snapshot })
-                        if saw_run_end && snapshot.catalog_status() == WorkerStatus::Idle => {
+                        if saw_run_end
+                            && snapshot.catalog_status() == WorkerStatus::Idle
+                            && snapshot.last_finished_submission_request_id.as_deref()
+                                == Some(submission_request_id.as_str()) =>
+                    {
                         saw_idle_status = true;
                         break;
                     }
@@ -1334,7 +1336,7 @@ async fn run_end_returns_to_idle_without_busy_status() {
     assert!(saw_run_end, "expected RunEnd::Finished");
     assert!(
         saw_idle_status,
-        "expected idle status immediately after RunEnd"
+        "expected exact Submit teardown fence in idle status immediately after RunEnd"
     );
     assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
 }

crates/workspace-server/src/server.rs

@@ -3774,21 +3774,30 @@ impl WorkspaceApi {
         let worker = attempt.worker.as_ref().ok_or_else(|| {
             Error::Store("claimed Backend Job Worker cleanup lost its binding".to_string())
         })?;
+        let Some(runtime_run_id) = attempt.runtime_run_id.as_deref() else {
+            // Attempts created before tracked Runtime submissions can only be
+            // recovered after a Backend restart, which has already ended their
+            // in-process result response and Run. Canonical stop/remove remains
+            // the safety boundary for those legacy leftovers.
+            return Ok(());
+        };
         loop {
             match self.runtime.worker(worker) {
                 Ok(detail)
                     if detail.state.eq_ignore_ascii_case("stopped")
                         || detail.worker_state.as_ref().is_some_and(|snapshot| {
-                            snapshot.last_command_id > 0
+                            snapshot.state == protocol::WorkerState::Idle
-                                && snapshot.state == protocol::WorkerState::Idle
+                                && snapshot.last_finished_submission_request_id.as_deref()
+                                    == Some(runtime_run_id)
                         }) =>
                 {
                     return Ok(());
                 }
                 Ok(_) => {
-                    // The result capability runs inside this Worker Run. Idle
+                    // The result capability runs inside this tracked Submit. The
-                    // with a committed command is the Runtime-owned fence that
+                    // Worker controller publishes this exact request id together
-                    // the HTTP response and Run post-processing have completed.
+                    // with its post-Run state only after the invocation returns,
+                    // so an unrelated initial Idle snapshot cannot release cleanup.
                     tokio::time::sleep(std::time::Duration::from_millis(10)).await;
                 }
                 Err(RuntimeRegistryError::UnknownWorker { .. }) => return Ok(()),
@@ -41796,12 +41805,13 @@ mod tests {
             } else if let Some(submission_request_id) = submission_request_id {
                 worker_runtime::execution::WorkerExecutionResult::accepted_submission(
                     worker_runtime::execution::WorkerExecutionOperation::Input,
-                    submission_request_id,
+                    submission_request_id.clone(),
                     uuid::Uuid::now_v7().to_string(),
                     protocol::SubmissionDisposition::Started,
                 )
                 .with_worker_state(protocol::WorkerStateSnapshot {
-                    last_command_id: 1,
+                    last_command_id: 0,
+                    last_finished_submission_request_id: Some(submission_request_id),
                     state: protocol::WorkerState::Idle,
                 })
             } else {
@@ -43175,7 +43185,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn backend_job_result_http_response_precedes_cleanup_until_run_idle() {
+    async fn backend_job_result_http_response_precedes_cleanup_until_tracked_run_finishes() {
         let temp = tempfile::tempdir().unwrap();
         let (api, execution) = test_api_with_recording_backend(temp.path()).await;
         let request = BackendJobRequest {
@@ -43192,11 +43202,13 @@ mod tests {
         };
         let dispatched = api.dispatch_backend_job(&request).unwrap();
         let worker = dispatched.attempt.worker.clone().unwrap();
+        let runtime_run_id = dispatched.attempt.runtime_run_id.clone().unwrap();
         seed_worker_session_for_cleanup(temp.path(), &worker);
         execution.publish_worker_state(
             &worker,
             protocol::WorkerStateSnapshot {
-                last_command_id: 1,
+                last_command_id: 0,
+                last_finished_submission_request_id: None,
                 state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
                     protocol::WorkerRunState::Running,
                 )),
@@ -43249,7 +43261,8 @@ mod tests {
         execution.publish_worker_state(
             &worker,
             protocol::WorkerStateSnapshot {
-                last_command_id: 2,
+                last_command_id: 0,
+                last_finished_submission_request_id: Some(runtime_run_id),
                 state: protocol::WorkerState::Idle,
             },
         );

openapi/server-api.json

@@ -12568,6 +12568,13 @@
             "minimum": 0,
             "type": "integer"
           },
+          "last_finished_submission_request_id": {
+            "description": "Most recent Submit request whose controller Run has returned and whose\npost-Run state transition has been published. This is a teardown fence,\nnot a successful-result claim; cancelled and failed Runs may also finish.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
           "state": {
             "$ref": "#/components/schemas/WorkerState"
           }
@@ -14787,7 +14794,7 @@
   "info": {
     "title": "Yoi Server API",
     "version": "0.1.0",
-    "x-yoi-source-digest": "sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6"
+    "x-yoi-source-digest": "sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870"
   },
   "openapi": "3.1.0",
   "paths": {

web/workspace/src/lib/generated/companion-api.ts

@@ -1,6 +1,6 @@
 // @generated by yoi-companion-openapi-typescript@1. Do not edit.
 // Input: openapi/server-api.json
-// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
+// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
 // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
 // Output: web/workspace/src/lib/generated/companion-api.ts
 // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check

web/workspace/src/lib/generated/protocol.ts

@@ -34,7 +34,13 @@ export type WorkerStateSnapshot = {
 /**
  * Highest lifecycle command id observed by this controller instance.
  */
-last_command_id: number, state: WorkerState, };
+last_command_id: number,
+/**
+ * Most recent Submit request whose controller Run has returned and whose
+ * post-Run state transition has been published. This is a teardown fence,
+ * not a successful-result claim; cancelled and failed Runs may also finish.
+ */
+last_finished_submission_request_id?: string | null, state: WorkerState, };
 
 export type TurnResult = "finished" | "paused";
 

web/workspace/src/lib/generated/repository-api.ts

@@ -1,6 +1,6 @@
 // @generated by yoi-repository-openapi-typescript@1. Do not edit.
 // Input: openapi/server-api.json
-// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
+// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
 // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
 // Output: web/workspace/src/lib/generated/repository-api.ts
 // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
@@ -69,7 +69,7 @@ export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
 
 export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyState; };
 
-export type WorkerStateSnapshot = { last_command_id: number; state: WorkerState; };
+export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
 
 export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
 

web/workspace/src/lib/generated/runtime-api.ts

@@ -1,6 +1,6 @@
 // @generated by yoi-runtime-openapi-typescript@1. Do not edit.
 // Input: openapi/server-api.json
-// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
+// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
 // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
 // Output: web/workspace/src/lib/generated/runtime-api.ts
 // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
@@ -159,7 +159,7 @@ export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
 
 export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyState; };
 
-export type WorkerStateSnapshot = { last_command_id: number; state: WorkerState; };
+export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
 
 export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };
 

web/workspace/src/lib/generated/ticket-api.ts

@@ -1,6 +1,6 @@
 // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
 // Input: openapi/server-api.json
-// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
+// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
 // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
 // Output: web/workspace/src/lib/generated/ticket-api.ts
 // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check

web/workspace/src/lib/generated/workdir-api.ts

@@ -1,6 +1,6 @@
 // @generated by yoi-workdir-openapi-typescript@1. Do not edit.
 // Input: openapi/server-api.json
-// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
+// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
 // Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
 // Output: web/workspace/src/lib/generated/workdir-api.ts
 // Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check

web/workspace/src/lib/generated/worker-launch-api.ts

@@ -1,6 +1,6 @@
 // @generated by yoi-worker-openapi-typescript@1. Do not edit.
 // Input: openapi/server-api.json
-// Input contract: 0.1.0 (sha256:cba261b44424ef69fde6339246ed891300b456ac3f8cc3b99103cb90e5fa32d6)
+// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870)
 // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
 // Output: web/workspace/src/lib/generated/worker-launch-api.ts
 // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
@@ -67,7 +67,7 @@ export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
 
 export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyState; };
 
-export type WorkerStateSnapshot = { last_command_id: number; state: WorkerState; };
+export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; };
 
 export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array<Diagnostic>; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array<string>; workdir_attachments?: Array<WorkerWorkdirAttachmentSummary>; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; };