worker: add guarded WorkerRemove lifecycle

crates/worker-runtime/src/worker_source.rs

@@ -124,6 +124,8 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync {
         proof: InProcessWorkerMutationProof,
         target_runtime_id: &str,
         target_worker_id: &str,
+        expected_worker_revision: &str,
+        reason: &str,
     ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError>;
 }
 
@@ -186,6 +188,8 @@ impl RuntimeWorkerMutationForwarder {
         &self,
         target_runtime_id: &str,
         target_worker_id: &str,
+        expected_worker_revision: &str,
+        reason: &str,
     ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
         let proof = self.authority.issue_worker_remove(
             &self.scope,
@@ -205,6 +209,8 @@ impl RuntimeWorkerMutationForwarder {
                 let body = serde_json::json!({
                     "target_runtime_id": target_runtime_id,
                     "target_worker_id": target_worker_id,
+                    "expected_worker_revision": expected_worker_revision,
+                    "reason": reason,
                 });
                 let response = client
                     .post(url)
@@ -223,7 +229,13 @@ impl RuntimeWorkerMutationForwarder {
             (
                 RuntimeWorkerMutationTransport::Embedded { dispatcher },
                 RuntimeOwnedWorkerMutationProof::InProcess(claims),
-            ) => dispatcher.execute_worker_remove(claims, target_runtime_id, target_worker_id),
+            ) => dispatcher.execute_worker_remove(
+                claims,
+                target_runtime_id,
+                target_worker_id,
+                expected_worker_revision,
+                reason,
+            ),
             _ => Err(RuntimeWorkerMutationForwardError::AuthorityTransportMismatch),
         }
     }
@@ -316,6 +328,8 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
         &self,
         target_runtime_id: &str,
         target_worker_id: &str,
+        expected_worker_revision: &str,
+        reason: &str,
     ) -> Result<WorkspaceResponse, WorkspaceClientError> {
         self.worker_remove
             .as_ref()
@@ -324,7 +338,12 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
                     "Runtime-owned WorkerRemove forwarding is unavailable".to_string(),
                 )
             })?
-            .execute_worker_remove(target_runtime_id, target_worker_id)
+            .execute_worker_remove(
+                target_runtime_id,
+                target_worker_id,
+                expected_worker_revision,
+                reason,
+            )
             .map_err(|error| WorkspaceClientError::Request(error.to_string()))
     }
 }
@@ -500,16 +519,22 @@ mod tests {
             format!("http://{address}"),
         );
         let response = forwarder
-            .execute_worker_remove("runtime-target", "worker-target")
+            .execute_worker_remove(
+                "runtime-target",
+                "worker-target",
+                "revision-7",
+                "retire obsolete Worker",
+            )
             .unwrap();
         assert_eq!(response.status, 204);
         server.join().unwrap();
 
         let request = received.lock().unwrap().clone();
         assert!(request.starts_with("POST /api/w/workspace-a/workers/remove HTTP/1.1"));
-        assert!(request.contains(
-            r#"{"target_runtime_id":"runtime-target","target_worker_id":"worker-target"}"#
-        ));
+        assert!(request.contains("\"target_runtime_id\":\"runtime-target\""));
+        assert!(request.contains("\"target_worker_id\":\"worker-target\""));
+        assert!(request.contains("\"expected_worker_revision\":\"revision-7\""));
+        assert!(request.contains("\"reason\":\"retire obsolete Worker\""));
         let token = request
             .lines()
             .find_map(|line| {
@@ -541,7 +566,7 @@ mod tests {
 
         #[derive(Default)]
         struct RecordingDispatcher {
-            seen: Mutex<Option<(WorkerMutationSourceClaims, String, String)>>,
+            seen: Mutex<Option<(WorkerMutationSourceClaims, String, String, String, String)>>,
         }
         impl EmbeddedWorkerMutationDispatcher for RecordingDispatcher {
             fn execute_worker_remove(
@@ -549,11 +574,15 @@ mod tests {
                 proof: InProcessWorkerMutationProof,
                 target_runtime_id: &str,
                 target_worker_id: &str,
+                expected_worker_revision: &str,
+                reason: &str,
             ) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
                 *self.seen.lock().unwrap() = Some((
                     proof.into_claims(),
                     target_runtime_id.to_string(),
                     target_worker_id.to_string(),
+                    expected_worker_revision.to_string(),
+                    reason.to_string(),
                 ));
                 Ok(WorkspaceResponse {
                     status: 202,
@@ -571,10 +600,15 @@ mod tests {
             dispatcher.clone(),
         );
         let response = forwarder
-            .execute_worker_remove("runtime-target", "worker-target")
+            .execute_worker_remove(
+                "runtime-target",
+                "worker-target",
+                "revision-7",
+                "retire obsolete Worker",
+            )
             .unwrap();
         assert_eq!(response.status, 202);
-        let (claims, target_runtime_id, target_worker_id) =
+        let (claims, target_runtime_id, target_worker_id, expected_revision, reason) =
             dispatcher.seen.lock().unwrap().take().unwrap();
         assert_eq!(claims.iss, "runtime-embedded");
         assert_eq!(claims.worker_id, "worker-source");
@@ -582,6 +616,8 @@ mod tests {
         assert_eq!(claims.target_worker_id, "worker-target");
         assert_eq!(target_runtime_id, "runtime-target");
         assert_eq!(target_worker_id, "worker-target");
+        assert_eq!(expected_revision, "revision-7");
+        assert_eq!(reason, "retire obsolete Worker");
     }
 
     #[test]