fix: order SubWorker cleanup before Workdir release

crates/worker/src/controller.rs

@@ -1101,8 +1101,15 @@ where
                 "manage Workdir tools require Backend Workspace API authority",
             ));
         }
+        let child_registry = spawned_registry.clone();
         feature_registry.add_module(
-            crate::feature::builtin::manage_workdir::manage_workdir_feature(workspace_client),
+            crate::feature::builtin::manage_workdir::ManageWorkdirFeature::with_before_workdir_release(
+                workspace_client,
+                Arc::new(move || {
+                    let child_registry = child_registry.clone();
+                    Box::pin(async move { child_registry.shutdown_internal().await })
+                }),
+            ),
         );
     }
     if feature_config.workspace_worker_discovery.enabled {
@@ -1726,7 +1733,16 @@ async fn controller_loop<C, St>(
     // Memory/Workdir teardown so they cannot observe a partially closed Worker.
     worker.stop_feature_runtime("controller shutdown").await;
 
-    if let Some(session) = worker.workdir_session()
+    let child_cleanup_succeeded = match spawned_registry.shutdown_internal().await {
+        Ok(()) => true,
+        Err(error) => {
+            tracing::warn!(%error, "Internal SubWorker cleanup failed before Workdir shutdown");
+            false
+        }
+    };
+
+    if child_cleanup_succeeded
+        && let Some(session) = worker.workdir_session()
         && let Err(error) = session.close().await
     {
         tracing::warn!(%error, "Workdir session close failed");
@@ -2604,4 +2620,21 @@ mod tests {
             other => panic!("expected compact rejection error, got {other:?}"),
         }
     }
+
+    #[test]
+    fn controller_shutdown_orders_child_cleanup_before_workdir_close() {
+        let source = include_str!("controller.rs");
+        let shutdown_start = source
+            .rfind("worker.stop_feature_runtime(\"controller shutdown\")")
+            .expect("controller shutdown block");
+        let shutdown = &source[shutdown_start..];
+        let children = shutdown
+            .find("spawned_registry.shutdown_internal().await")
+            .expect("Internal SubWorker cleanup");
+        let workdir = shutdown
+            .find("session.close().await")
+            .expect("parent Workdir close");
+        assert!(children < workdir);
+        assert!(shutdown.contains("if child_cleanup_succeeded"));
+    }
 }

crates/worker/src/feature/builtin/manage_workdir.rs

@@ -5,6 +5,8 @@
 //! endpoints, credentials, materializer handles, and operation sessions stay
 //! behind [`WorkspaceClient`].
 
+use std::future::Future;
+use std::pin::Pin;
 use std::sync::Arc;
 
 use agen::tool::{Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
@@ -52,16 +54,43 @@ const LIST_DESCRIPTION: &str = "List persistent Workdirs in the current Workspac
 const CREATE_DESCRIPTION: &str = "Materialize a persistent Workdir on a selected Runtime from a Workspace repository and optional selector. This does not change this Worker's attachment; use WorkdirAttach explicitly after creation.";
 const ATTACH_DESCRIPTION: &str = "Attach this Worker to one existing Workdir. The Backend enforces one active Workdir per Worker and one active Worker per Workdir, then opens an ephemeral operation session.";
 const DETACH_DESCRIPTION: &str = "Detach this Worker from its active Workdir and release Workdir occupancy. Any ephemeral operation session is closed.";
+pub(crate) type BeforeWorkdirRelease =
+    Arc<dyn Fn() -> Pin<Box<dyn Future<Output = std::io::Result<()>> + Send>> + Send + Sync>;
+
 const DELETE_DESCRIPTION: &str = "Request removal of one persistent Workdir by id through durable Backend Workspace authority. The input includes only the Workdir id and a bounded reason. The result reports removed, retained, or attention_required without exposing operation-table or provider internals.";
 
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct ManageWorkdirFeature {
     client: Arc<dyn WorkspaceClient>,
+    before_workdir_release: Option<BeforeWorkdirRelease>,
+}
+
+impl std::fmt::Debug for ManageWorkdirFeature {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter
+            .debug_struct("ManageWorkdirFeature")
+            .field("client_kind", &self.client.kind())
+            .field("release_guard", &self.before_workdir_release.is_some())
+            .finish()
+    }
 }
 
 impl ManageWorkdirFeature {
     pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
-        Self { client }
+        Self {
+            client,
+            before_workdir_release: None,
+        }
+    }
+
+    pub(crate) fn with_before_workdir_release(
+        client: Arc<dyn WorkspaceClient>,
+        before_workdir_release: BeforeWorkdirRelease,
+    ) -> Self {
+        Self {
+            client,
+            before_workdir_release: Some(before_workdir_release),
+        }
     }
 }
 
@@ -81,7 +110,8 @@ impl FeatureModule for ManageWorkdirFeature {
     }
 
     fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> {
-        let backend = WorkspaceHttpWorkdirBackend::new(self.client.clone());
+        let backend = WorkspaceHttpWorkdirBackend::new(self.client.clone())
+            .with_before_workdir_release(self.before_workdir_release.clone());
         for (name, definition) in [
             (
                 LIST_TOOL,
@@ -142,9 +172,20 @@ impl FeatureModule for ManageWorkdirFeature {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 struct WorkspaceHttpWorkdirBackend {
     client: Arc<dyn WorkspaceClient>,
+    before_workdir_release: Option<BeforeWorkdirRelease>,
+}
+
+impl std::fmt::Debug for WorkspaceHttpWorkdirBackend {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter
+            .debug_struct("WorkspaceHttpWorkdirBackend")
+            .field("client_kind", &self.client.kind())
+            .field("release_guard", &self.before_workdir_release.is_some())
+            .finish()
+    }
 }
 
 /// Worker-local Workdir handle whose operation authority remains in the Workspace Backend.
@@ -327,7 +368,18 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession {
 
 impl WorkspaceHttpWorkdirBackend {
     fn new(client: Arc<dyn WorkspaceClient>) -> Self {
-        Self { client }
+        Self {
+            client,
+            before_workdir_release: None,
+        }
+    }
+
+    fn with_before_workdir_release(
+        mut self,
+        before_workdir_release: Option<BeforeWorkdirRelease>,
+    ) -> Self {
+        self.before_workdir_release = before_workdir_release;
+        self
     }
 
     fn workspace_id(&self) -> Result<&str, ToolError> {
@@ -510,6 +562,13 @@ impl Tool for WorkspaceHttpWorkdirTool {
                 .attach(parse_input::<WorkdirAttachInput>(input_json)?),
             WorkdirOperation::Detach => {
                 let _input = parse_input::<WorkdirDetachInput>(input_json)?;
+                if let Some(before_release) = &self.backend.before_workdir_release {
+                    before_release().await.map_err(|error| {
+                        ToolError::ExecutionFailed(format!(
+                            "stop Internal SubWorkers before Workdir detach: {error}"
+                        ))
+                    })?;
+                }
                 self.backend.detach()
             }
             WorkdirOperation::Delete => self
@@ -705,6 +764,7 @@ struct WorkdirDeleteInput {
 #[cfg(test)]
 mod tests {
     use std::sync::Mutex;
+    use std::sync::atomic::{AtomicUsize, Ordering};
 
     use super::*;
     use crate::feature::{FeatureModule, FeatureRegistryBuilder};
@@ -1259,4 +1319,58 @@ mod tests {
         assert!(client.requests().is_empty());
         assert!(parse_input::<WorkdirListInput>(r#"{"path":"/tmp"}"#).is_err());
     }
+
+    #[tokio::test]
+    async fn detach_stops_internal_subworkers_before_backend_release() {
+        let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
+            "workspace_id": "workspace/test",
+            "workdir_id": "wd-attached",
+            "attached": false
+        }))]));
+        let cleanup_calls = Arc::new(AtomicUsize::new(0));
+        let cleanup_calls_for_guard = cleanup_calls.clone();
+        let before_release: BeforeWorkdirRelease = Arc::new(move || {
+            let cleanup_calls = cleanup_calls_for_guard.clone();
+            Box::pin(async move {
+                cleanup_calls.fetch_add(1, Ordering::SeqCst);
+                Ok(())
+            })
+        });
+        let tool = WorkspaceHttpWorkdirTool {
+            backend: WorkspaceHttpWorkdirBackend::new(client.clone())
+                .with_before_workdir_release(Some(before_release)),
+            operation: WorkdirOperation::Detach,
+        };
+
+        tool.execute("{}", ToolExecutionContext::default())
+            .await
+            .unwrap();
+
+        assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(client.requests().len(), 1);
+        assert_eq!(
+            client.requests()[0].path,
+            "/api/w/workspace%2Ftest/workers/self/workdir-attachment"
+        );
+    }
+
+    #[tokio::test]
+    async fn detach_does_not_release_backend_when_child_cleanup_fails() {
+        let client = Arc::new(RecordingWorkspaceClient::new(Vec::new()));
+        let before_release: BeforeWorkdirRelease =
+            Arc::new(|| Box::pin(async { Err(std::io::Error::other("child cleanup failed")) }));
+        let tool = WorkspaceHttpWorkdirTool {
+            backend: WorkspaceHttpWorkdirBackend::new(client.clone())
+                .with_before_workdir_release(Some(before_release)),
+            operation: WorkdirOperation::Detach,
+        };
+
+        let error = tool
+            .execute("{}", ToolExecutionContext::default())
+            .await
+            .unwrap_err();
+
+        assert!(error.to_string().contains("stop Internal SubWorkers"));
+        assert!(client.requests().is_empty());
+    }
 }

crates/worker/src/spawn/registry.rs

@@ -679,13 +679,6 @@ impl SpawnedWorkerRegistry {
             .unwrap_or_default()
     }
 
-    pub(crate) fn reclaim_internal_scope(&self, worker_name: &str) -> io::Result<bool> {
-        let record = self.get_internal(worker_name).ok_or_else(|| {
-            io::Error::new(io::ErrorKind::NotFound, "internal SubWorker not found")
-        })?;
-        self.reclaim_record_scope(&record)
-    }
-
     fn reclaim_record_scope(&self, record: &InternalSpawnedWorkerRecord) -> io::Result<bool> {
         if !record.claim_scope_reclaim() {
             return Ok(false);
@@ -705,6 +698,35 @@ impl SpawnedWorkerRegistry {
         result
     }
 
+    pub(crate) async fn close_internal_scope(&self, name: &str) -> io::Result<bool> {
+        let Some(record) = self.get_internal(name) else {
+            return Ok(false);
+        };
+        record
+            .workdir_tool_scope
+            .close()
+            .await
+            .map_err(|error| io::Error::other(error.to_string()))?;
+        self.reclaim_record_scope(&record)
+    }
+
+    pub(crate) async fn shutdown_internal(&self) -> io::Result<()> {
+        let names = self
+            .internal_records
+            .lock()
+            .expect("internal Worker registry lock poisoned")
+            .iter()
+            .map(|record| record.worker_name.clone())
+            .collect::<Vec<_>>();
+        let mut first_error = None;
+        for name in names {
+            if let Err(error) = self.remove_internal(&name).await {
+                first_error.get_or_insert(error);
+            }
+        }
+        first_error.map_or(Ok(()), Err)
+    }
+
     /// Stop one direct Internal SubWorker and discard its registry/scope state.
     ///
     /// The child actor must acknowledge its stop before the registry is removed.
@@ -1236,6 +1258,24 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn parent_shutdown_stops_all_internal_workers_before_returning() {
+        let registry = registry();
+        for name in ["first", "second"] {
+            let (record, _events) = record(name, InternalWorkerVisibility::ParentClient).await;
+            record
+                .session
+                .force_status(InternalWorkerSessionStatus::Running);
+            install_record(&registry, record);
+        }
+
+        registry.shutdown_internal().await.unwrap();
+
+        assert!(registry.list_internal().is_empty());
+        assert!(registry.get_internal("first").is_none());
+        assert!(registry.get_internal("second").is_none());
+    }
+
     #[tokio::test]
     async fn running_worker_is_stopped_before_removal() {
         let registry = registry();

crates/worker/src/spawn/tool.rs

@@ -532,13 +532,16 @@ impl Tool for SubWorkerSpawnTool {
                     InternalWorkerSessionStatus::Failed | InternalWorkerSessionStatus::Stopped
                 ) {
                     if let Some(registry) = registry.upgrade() {
-                        if let Err(error) = registry.reclaim_internal_scope(&child_name) {
-                            tracing::warn!(
-                                child_name,
-                                %error,
-                                "failed to reclaim delegated scope after Internal SubWorker failure"
-                            );
-                        }
+                        let child_name = child_name.clone();
+                        tokio::spawn(async move {
+                            if let Err(error) = registry.close_internal_scope(&child_name).await {
+                                tracing::warn!(
+                                    child_name,
+                                    %error,
+                                    "failed to close parent-owned Workdir tools after Internal SubWorker failure"
+                                );
+                            }
+                        });
                     }
                 }
                 let message = format!(