fix: remove speculative worker control delegation
This commit is contained in:
@@ -435,12 +435,6 @@ impl FeatureModule for ManageWorkerFeature {
|
|||||||
WorkerOperation::Remove => {
|
WorkerOperation::Remove => {
|
||||||
definition::<WorkerRemoveInput>(operation, self.control.clone())
|
definition::<WorkerRemoveInput>(operation, self.control.clone())
|
||||||
}
|
}
|
||||||
WorkerOperation::Share | WorkerOperation::Transfer => {
|
|
||||||
definition::<WorkerDelegateInput>(operation, self.control.clone())
|
|
||||||
}
|
|
||||||
WorkerOperation::Revoke => {
|
|
||||||
definition::<WorkerRevokeInput>(operation, self.control.clone())
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
context
|
context
|
||||||
.tools()
|
.tools()
|
||||||
@@ -540,19 +534,6 @@ struct WorkerRemoveInput {
|
|||||||
reason: String,
|
reason: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
|
||||||
#[serde(deny_unknown_fields)]
|
|
||||||
struct WorkerRevokeInput {
|
|
||||||
grant_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
|
||||||
#[serde(deny_unknown_fields)]
|
|
||||||
struct WorkerDelegateInput {
|
|
||||||
grant_id: String,
|
|
||||||
target_controller: WorkerSubjectInput,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct WorkspaceWorkerTool {
|
struct WorkspaceWorkerTool {
|
||||||
operation: WorkerOperation,
|
operation: WorkerOperation,
|
||||||
control: Arc<dyn WorkerControlService>,
|
control: Arc<dyn WorkerControlService>,
|
||||||
@@ -568,13 +549,10 @@ enum WorkerOperation {
|
|||||||
Stop,
|
Stop,
|
||||||
Restore,
|
Restore,
|
||||||
Remove,
|
Remove,
|
||||||
Share,
|
|
||||||
Transfer,
|
|
||||||
Revoke,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerOperation {
|
impl WorkerOperation {
|
||||||
const ALL: [Self; 11] = [
|
const ALL: [Self; 8] = [
|
||||||
Self::List,
|
Self::List,
|
||||||
Self::Spawn,
|
Self::Spawn,
|
||||||
Self::SendInput,
|
Self::SendInput,
|
||||||
@@ -583,9 +561,6 @@ impl WorkerOperation {
|
|||||||
Self::Stop,
|
Self::Stop,
|
||||||
Self::Restore,
|
Self::Restore,
|
||||||
Self::Remove,
|
Self::Remove,
|
||||||
Self::Share,
|
|
||||||
Self::Transfer,
|
|
||||||
Self::Revoke,
|
|
||||||
];
|
];
|
||||||
|
|
||||||
fn tool_name(self) -> &'static str {
|
fn tool_name(self) -> &'static str {
|
||||||
@@ -598,9 +573,6 @@ impl WorkerOperation {
|
|||||||
Self::Stop => "WorkerStop",
|
Self::Stop => "WorkerStop",
|
||||||
Self::Restore => "WorkerRestore",
|
Self::Restore => "WorkerRestore",
|
||||||
Self::Remove => "WorkerRemove",
|
Self::Remove => "WorkerRemove",
|
||||||
Self::Share => "WorkerShare",
|
|
||||||
Self::Transfer => "WorkerTransfer",
|
|
||||||
Self::Revoke => "WorkerRevoke",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -622,11 +594,6 @@ impl WorkerOperation {
|
|||||||
Self::Remove => {
|
Self::Remove => {
|
||||||
"Remove an eligible stopped, unassigned, non-internal Worker. Supply the current Worker revision and a bounded reason; Backend validation and retention are authoritative."
|
"Remove an eligible stopped, unassigned, non-internal Worker. Supply the current Worker revision and a bounded reason; Backend validation and retention are authoritative."
|
||||||
}
|
}
|
||||||
Self::Share => "Share one controlled Runtime Worker with another known Runtime Worker.",
|
|
||||||
Self::Transfer => {
|
|
||||||
"Transfer one controlled Runtime Worker to another known Runtime Worker."
|
|
||||||
}
|
|
||||||
Self::Revoke => "Revoke one durable Runtime Worker control grant owned by this Worker.",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -792,59 +759,6 @@ impl Tool for WorkspaceWorkerTool {
|
|||||||
)
|
)
|
||||||
.map_err(control_tool_error)?
|
.map_err(control_tool_error)?
|
||||||
}
|
}
|
||||||
WorkerOperation::Share | WorkerOperation::Transfer => {
|
|
||||||
let input = parse::<WorkerDelegateInput>(input_json, self.operation.tool_name())?;
|
|
||||||
let grant_id = authority_id(&input.grant_id, "grant_id")?;
|
|
||||||
let (runtime_id, worker_id) =
|
|
||||||
runtime_subject_ids(&input.target_controller, self.operation)?;
|
|
||||||
let operation_id = format!(
|
|
||||||
"worker-control-{}:{}",
|
|
||||||
if self.operation == WorkerOperation::Transfer {
|
|
||||||
"transfer"
|
|
||||||
} else {
|
|
||||||
"share"
|
|
||||||
},
|
|
||||||
non_empty(ctx.call_id.clone(), "tool call_id")?
|
|
||||||
);
|
|
||||||
let action = if self.operation == WorkerOperation::Transfer {
|
|
||||||
"transfer"
|
|
||||||
} else {
|
|
||||||
"share"
|
|
||||||
};
|
|
||||||
self.control
|
|
||||||
.execute_runtime(WorkspaceRequest::json(
|
|
||||||
WorkspaceRequestMethod::Post,
|
|
||||||
format!(
|
|
||||||
"/api/w/{}/worker-control/grants/{grant_id}/{action}",
|
|
||||||
self.control.workspace_id()
|
|
||||||
),
|
|
||||||
serde_json::json!({
|
|
||||||
"target_controller": {
|
|
||||||
"runtime_id": runtime_id,
|
|
||||||
"worker_id": worker_id,
|
|
||||||
},
|
|
||||||
"operation_id": operation_id,
|
|
||||||
})
|
|
||||||
.to_string(),
|
|
||||||
))
|
|
||||||
.await
|
|
||||||
.map_err(control_tool_error)?
|
|
||||||
}
|
|
||||||
WorkerOperation::Revoke => {
|
|
||||||
let input = parse::<WorkerRevokeInput>(input_json, "WorkerRevoke")?;
|
|
||||||
let grant_id = authority_id(&input.grant_id, "grant_id")?;
|
|
||||||
self.control
|
|
||||||
.execute_runtime(WorkspaceRequest::json(
|
|
||||||
WorkspaceRequestMethod::Post,
|
|
||||||
format!(
|
|
||||||
"/api/w/{}/worker-control/grants/{grant_id}/revoke",
|
|
||||||
self.control.workspace_id()
|
|
||||||
),
|
|
||||||
"{}",
|
|
||||||
))
|
|
||||||
.await
|
|
||||||
.map_err(control_tool_error)?
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
tool_output(self.operation, response)
|
tool_output(self.operation, response)
|
||||||
}
|
}
|
||||||
@@ -1131,9 +1045,6 @@ mod tests {
|
|||||||
"WorkerStop",
|
"WorkerStop",
|
||||||
"WorkerRestore",
|
"WorkerRestore",
|
||||||
"WorkerRemove",
|
"WorkerRemove",
|
||||||
"WorkerShare",
|
|
||||||
"WorkerTransfer",
|
|
||||||
"WorkerRevoke",
|
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1248,45 +1159,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn worker_share_and_transfer_use_typed_runtime_subjects_and_operation_ids() {
|
|
||||||
let client = Arc::new(RecordingWorkspaceClient::default());
|
|
||||||
for (operation, action) in [
|
|
||||||
(WorkerOperation::Share, "share"),
|
|
||||||
(WorkerOperation::Transfer, "transfer"),
|
|
||||||
] {
|
|
||||||
WorkspaceWorkerTool {
|
|
||||||
operation,
|
|
||||||
control: test_control(client.clone()),
|
|
||||||
}
|
|
||||||
.execute(
|
|
||||||
&serde_json::json!({
|
|
||||||
"grant_id": "grant-1",
|
|
||||||
"target_controller": {
|
|
||||||
"kind": "runtime_worker",
|
|
||||||
"runtime_id": "runtime-2",
|
|
||||||
"worker_id": "worker-9",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
.to_string(),
|
|
||||||
ToolExecutionContext::new("call-delegate", "batch-delegate", 0),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let request = client.requests.lock().unwrap().last().cloned().unwrap();
|
|
||||||
assert!(request.path.ends_with(&format!("/grant-1/{action}")));
|
|
||||||
let body: serde_json::Value =
|
|
||||||
serde_json::from_str(request.body.as_deref().unwrap()).unwrap();
|
|
||||||
assert_eq!(body["target_controller"]["runtime_id"], "runtime-2");
|
|
||||||
assert!(
|
|
||||||
body["operation_id"]
|
|
||||||
.as_str()
|
|
||||||
.unwrap()
|
|
||||||
.contains("call-delegate")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn worker_remove_forwards_only_target_revision_and_bounded_reason() {
|
async fn worker_remove_forwards_only_target_revision_and_bounded_reason() {
|
||||||
let client = Arc::new(RecordingWorkspaceClient::default());
|
let client = Arc::new(RecordingWorkspaceClient::default());
|
||||||
|
|||||||
@@ -103,9 +103,8 @@ use crate::skills;
|
|||||||
use crate::store::{
|
use crate::store::{
|
||||||
AccountRecord, ApiTokenRecord, AuthChallengeRecord, BrowserSessionRecord, ControlPlaneStore,
|
AccountRecord, ApiTokenRecord, AuthChallengeRecord, BrowserSessionRecord, ControlPlaneStore,
|
||||||
DeviceLoginFlowRecord, FlowSourceRecord, PasskeyCredentialRecord, RepositoryRecord,
|
DeviceLoginFlowRecord, FlowSourceRecord, PasskeyCredentialRecord, RepositoryRecord,
|
||||||
TicketWorkerAssignmentRecord, UserRecord, WorkdirRegistryRecord,
|
TicketWorkerAssignmentRecord, UserRecord, WorkdirRegistryRecord, WorkerControlGrantRecord,
|
||||||
WorkerControlDelegationOperationRecord, WorkerControlGrantRecord, WorkerRegistryRecord,
|
WorkerRegistryRecord, WorkerWorkdirLinkRecord, WorkspaceRecord,
|
||||||
WorkerWorkdirLinkRecord, WorkspaceRecord,
|
|
||||||
};
|
};
|
||||||
use crate::{Error, Result};
|
use crate::{Error, Result};
|
||||||
use worker_runtime::catalog::{
|
use worker_runtime::catalog::{
|
||||||
@@ -1477,18 +1476,6 @@ pub fn build_router(api: WorkspaceApi) -> Router {
|
|||||||
"/api/w/{workspace_id}/worker-control/workers",
|
"/api/w/{workspace_id}/worker-control/workers",
|
||||||
get(list_known_workers).post(spawn_known_worker),
|
get(list_known_workers).post(spawn_known_worker),
|
||||||
)
|
)
|
||||||
.route(
|
|
||||||
"/api/w/{workspace_id}/worker-control/grants/{grant_id}/share",
|
|
||||||
post(share_worker_control_grant),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/api/w/{workspace_id}/worker-control/grants/{grant_id}/transfer",
|
|
||||||
post(transfer_worker_control_grant),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/api/w/{workspace_id}/worker-control/grants/{grant_id}/revoke",
|
|
||||||
post(revoke_worker_control_grant),
|
|
||||||
)
|
|
||||||
.route(
|
.route(
|
||||||
"/api/w/{workspace_id}/worker-control/workers/{runtime_id}/{worker_id}/input",
|
"/api/w/{workspace_id}/worker-control/workers/{runtime_id}/{worker_id}/input",
|
||||||
post(send_known_worker_input),
|
post(send_known_worker_input),
|
||||||
@@ -2258,12 +2245,6 @@ struct ScopedWorkspacePath {
|
|||||||
workspace_id: String,
|
workspace_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
struct ScopedWorkerControlGrantPath {
|
|
||||||
workspace_id: String,
|
|
||||||
grant_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct ScopedFlowPath {
|
struct ScopedFlowPath {
|
||||||
workspace_id: String,
|
workspace_id: String,
|
||||||
@@ -5917,7 +5898,6 @@ async fn scoped_workspace_orchestrator_status(
|
|||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
struct KnownWorkerRecord {
|
struct KnownWorkerRecord {
|
||||||
grant_id: String,
|
|
||||||
subject: RuntimeWorkerRef,
|
subject: RuntimeWorkerRef,
|
||||||
relation: String,
|
relation: String,
|
||||||
origin: String,
|
origin: String,
|
||||||
@@ -5952,7 +5932,6 @@ async fn list_known_workers(
|
|||||||
.worker(&grant.subject)
|
.worker(&grant.subject)
|
||||||
.map_err(|error| error.into_error())?;
|
.map_err(|error| error.into_error())?;
|
||||||
items.push(KnownWorkerRecord {
|
items.push(KnownWorkerRecord {
|
||||||
grant_id: grant.grant_id,
|
|
||||||
subject: grant.subject,
|
subject: grant.subject,
|
||||||
relation: grant.relation,
|
relation: grant.relation,
|
||||||
origin: grant.origin,
|
origin: grant.origin,
|
||||||
@@ -6028,8 +6007,6 @@ async fn spawn_known_worker(
|
|||||||
"stop".to_string(),
|
"stop".to_string(),
|
||||||
"restore".to_string(),
|
"restore".to_string(),
|
||||||
"remove".to_string(),
|
"remove".to_string(),
|
||||||
"share".to_string(),
|
|
||||||
"transfer".to_string(),
|
|
||||||
"observe".to_string(),
|
"observe".to_string(),
|
||||||
],
|
],
|
||||||
operation_id,
|
operation_id,
|
||||||
@@ -6083,250 +6060,6 @@ fn authorize_known_worker_permission(
|
|||||||
Ok(grant)
|
Ok(grant)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(deny_unknown_fields)]
|
|
||||||
struct DelegateWorkerControlGrantRequest {
|
|
||||||
target_controller: RuntimeWorkerRef,
|
|
||||||
operation_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn share_worker_control_grant(
|
|
||||||
State(api): State<WorkspaceApi>,
|
|
||||||
AxumPath(path): AxumPath<ScopedWorkerControlGrantPath>,
|
|
||||||
headers: HeaderMap,
|
|
||||||
Json(request): Json<DelegateWorkerControlGrantRequest>,
|
|
||||||
) -> ApiResult<Json<WorkerControlGrantRecord>> {
|
|
||||||
delegate_worker_control_grant(api, path, headers, request, false).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn transfer_worker_control_grant(
|
|
||||||
State(api): State<WorkspaceApi>,
|
|
||||||
AxumPath(path): AxumPath<ScopedWorkerControlGrantPath>,
|
|
||||||
headers: HeaderMap,
|
|
||||||
Json(request): Json<DelegateWorkerControlGrantRequest>,
|
|
||||||
) -> ApiResult<Json<WorkerControlGrantRecord>> {
|
|
||||||
delegate_worker_control_grant(api, path, headers, request, true).await
|
|
||||||
}
|
|
||||||
|
|
||||||
fn worker_control_delegation_input_fingerprint(
|
|
||||||
controller: &RuntimeWorkerRef,
|
|
||||||
grant: &WorkerControlGrantRecord,
|
|
||||||
action: &str,
|
|
||||||
target_controller: &RuntimeWorkerRef,
|
|
||||||
) -> Result<String> {
|
|
||||||
let operation_input = serde_json::json!({
|
|
||||||
"source_controller": controller,
|
|
||||||
"source_grant_id": &grant.grant_id,
|
|
||||||
"action": action,
|
|
||||||
"target_controller": target_controller,
|
|
||||||
"subject": &grant.subject,
|
|
||||||
"permissions": &grant.permissions,
|
|
||||||
});
|
|
||||||
let operation_bytes = serde_json::to_vec(&operation_input).map_err(|error| {
|
|
||||||
Error::InvalidInput(format!("invalid Worker delegation input: {error}"))
|
|
||||||
})?;
|
|
||||||
Ok(format!(
|
|
||||||
"sha256:{}",
|
|
||||||
Sha256::digest(&operation_bytes)
|
|
||||||
.iter()
|
|
||||||
.map(|byte| format!("{byte:02x}"))
|
|
||||||
.collect::<String>()
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn delegate_worker_control_grant(
|
|
||||||
api: WorkspaceApi,
|
|
||||||
path: ScopedWorkerControlGrantPath,
|
|
||||||
headers: HeaderMap,
|
|
||||||
request: DelegateWorkerControlGrantRequest,
|
|
||||||
transfer: bool,
|
|
||||||
) -> ApiResult<Json<WorkerControlGrantRecord>> {
|
|
||||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
|
||||||
let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?;
|
|
||||||
let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id);
|
|
||||||
let permission = if transfer { "transfer" } else { "share" };
|
|
||||||
let grant = api
|
|
||||||
.store
|
|
||||||
.get_worker_control_grant(&path.workspace_id, &path.grant_id)?
|
|
||||||
.filter(|grant| {
|
|
||||||
grant.controller == controller
|
|
||||||
&& grant.permissions.iter().any(|value| value == permission)
|
|
||||||
})
|
|
||||||
.ok_or_else(|| Error::UnknownWorker {
|
|
||||||
worker: controller.clone(),
|
|
||||||
})?;
|
|
||||||
if request.target_controller == controller {
|
|
||||||
return Err(ApiError::from(Error::InvalidInput(
|
|
||||||
"target_controller must differ from the current Worker".to_string(),
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let operation_id = request.operation_id.trim();
|
|
||||||
if operation_id.is_empty() || operation_id.len() > 200 {
|
|
||||||
return Err(ApiError::from(Error::InvalidInput(
|
|
||||||
"operation_id must contain 1..=200 bytes".to_string(),
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let input_fingerprint = worker_control_delegation_input_fingerprint(
|
|
||||||
&controller,
|
|
||||||
&grant,
|
|
||||||
permission,
|
|
||||||
&request.target_controller,
|
|
||||||
)?;
|
|
||||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
|
|
||||||
let operation = api.store.reserve_worker_control_delegation_operation(
|
|
||||||
&WorkerControlDelegationOperationRecord {
|
|
||||||
workspace_id: path.workspace_id.clone(),
|
|
||||||
source_controller: controller.clone(),
|
|
||||||
source_grant_id: grant.grant_id.clone(),
|
|
||||||
operation_id: operation_id.to_string(),
|
|
||||||
input_fingerprint,
|
|
||||||
delegated_grant_id: None,
|
|
||||||
created_at: now.clone(),
|
|
||||||
completed_at: None,
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
if let Some(delegated_grant_id) = operation.delegated_grant_id.as_deref() {
|
|
||||||
let delegated = api
|
|
||||||
.store
|
|
||||||
.get_worker_control_grant(&path.workspace_id, delegated_grant_id)?
|
|
||||||
.ok_or_else(|| {
|
|
||||||
Error::Store("completed Worker delegation references a missing grant".to_string())
|
|
||||||
})?;
|
|
||||||
if transfer && grant.revoked_at.is_none() {
|
|
||||||
let lock = worker_control_lock(&api, &grant.grant_id);
|
|
||||||
let _guard = lock.lock().await;
|
|
||||||
if api
|
|
||||||
.store
|
|
||||||
.get_worker_control_grant(&path.workspace_id, &grant.grant_id)?
|
|
||||||
.is_some_and(|current| current.revoked_at.is_none())
|
|
||||||
{
|
|
||||||
api.store
|
|
||||||
.revoke_worker_control_grant(&path.workspace_id, &grant.grant_id, &now)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Ok(Json(delegated));
|
|
||||||
}
|
|
||||||
if grant.revoked_at.is_some() {
|
|
||||||
return Err(ApiError::from(Error::UnknownWorker {
|
|
||||||
worker: grant.subject,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
api.store
|
|
||||||
.get_active_worker_control_grant(
|
|
||||||
&path.workspace_id,
|
|
||||||
&controller,
|
|
||||||
&request.target_controller,
|
|
||||||
)?
|
|
||||||
.ok_or_else(|| Error::UnknownWorker {
|
|
||||||
worker: request.target_controller.clone(),
|
|
||||||
})?;
|
|
||||||
api.store
|
|
||||||
.get_worker_registry(&path.workspace_id, &request.target_controller)?
|
|
||||||
.ok_or_else(|| Error::UnknownWorker {
|
|
||||||
worker: request.target_controller.clone(),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let lock = worker_control_lock(&api, &grant.grant_id);
|
|
||||||
let _guard = lock.lock().await;
|
|
||||||
let current = api
|
|
||||||
.store
|
|
||||||
.get_worker_control_grant(&path.workspace_id, &path.grant_id)?
|
|
||||||
.filter(|candidate| {
|
|
||||||
candidate.controller == controller
|
|
||||||
&& candidate.revoked_at.is_none()
|
|
||||||
&& candidate
|
|
||||||
.permissions
|
|
||||||
.iter()
|
|
||||||
.any(|value| value == permission)
|
|
||||||
})
|
|
||||||
.ok_or_else(|| Error::UnknownWorker {
|
|
||||||
worker: grant.subject.clone(),
|
|
||||||
})?;
|
|
||||||
api.store
|
|
||||||
.get_active_worker_control_grant(
|
|
||||||
&path.workspace_id,
|
|
||||||
&controller,
|
|
||||||
&request.target_controller,
|
|
||||||
)?
|
|
||||||
.ok_or_else(|| Error::UnknownWorker {
|
|
||||||
worker: request.target_controller.clone(),
|
|
||||||
})?;
|
|
||||||
let delegated_operation_id = format!(
|
|
||||||
"worker-control-delegate:{}:{}:{}:{}:{}",
|
|
||||||
controller.runtime_id, controller.worker_id, current.grant_id, permission, operation_id
|
|
||||||
);
|
|
||||||
let expected_origin = format!("worker_control_{permission}:{}", current.grant_id);
|
|
||||||
let delegated = api
|
|
||||||
.store
|
|
||||||
.create_worker_control_grant(&WorkerControlGrantRecord {
|
|
||||||
workspace_id: path.workspace_id.clone(),
|
|
||||||
grant_id: new_id("wcg"),
|
|
||||||
controller: request.target_controller,
|
|
||||||
subject: current.subject.clone(),
|
|
||||||
relation: if transfer { "transferred" } else { "shared" }.to_string(),
|
|
||||||
origin: expected_origin,
|
|
||||||
permissions: current.permissions.clone(),
|
|
||||||
operation_id: delegated_operation_id,
|
|
||||||
created_at: now.clone(),
|
|
||||||
revoked_at: None,
|
|
||||||
})?;
|
|
||||||
api.store.complete_worker_control_delegation_operation(
|
|
||||||
&path.workspace_id,
|
|
||||||
&controller,
|
|
||||||
operation_id,
|
|
||||||
&delegated.grant_id,
|
|
||||||
&now,
|
|
||||||
)?;
|
|
||||||
if transfer
|
|
||||||
&& !api
|
|
||||||
.store
|
|
||||||
.revoke_worker_control_grant(&path.workspace_id, ¤t.grant_id, &now)?
|
|
||||||
{
|
|
||||||
return Err(ApiError::from(Error::UnknownWorker {
|
|
||||||
worker: current.subject,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
Ok(Json(delegated))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn revoke_worker_control_grant(
|
|
||||||
State(api): State<WorkspaceApi>,
|
|
||||||
AxumPath(path): AxumPath<ScopedWorkerControlGrantPath>,
|
|
||||||
headers: HeaderMap,
|
|
||||||
) -> ApiResult<Json<WorkerControlGrantRecord>> {
|
|
||||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
|
||||||
let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?;
|
|
||||||
let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id);
|
|
||||||
let grant = api
|
|
||||||
.store
|
|
||||||
.get_worker_control_grant(&path.workspace_id, &path.grant_id)?
|
|
||||||
.filter(|grant| grant.controller == controller && grant.revoked_at.is_none())
|
|
||||||
.ok_or_else(|| Error::UnknownWorker {
|
|
||||||
worker: controller.clone(),
|
|
||||||
})?;
|
|
||||||
let lock = worker_control_lock(&api, &grant.grant_id);
|
|
||||||
let _guard = lock.lock().await;
|
|
||||||
let current = api
|
|
||||||
.store
|
|
||||||
.get_worker_control_grant(&path.workspace_id, &path.grant_id)?
|
|
||||||
.filter(|candidate| candidate.controller == controller && candidate.revoked_at.is_none())
|
|
||||||
.ok_or_else(|| Error::UnknownWorker {
|
|
||||||
worker: grant.subject.clone(),
|
|
||||||
})?;
|
|
||||||
let revoked_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
|
|
||||||
if !api
|
|
||||||
.store
|
|
||||||
.revoke_worker_control_grant(&path.workspace_id, &path.grant_id, &revoked_at)?
|
|
||||||
{
|
|
||||||
return Err(ApiError::from(Error::UnknownWorker {
|
|
||||||
worker: current.subject,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
let mut revoked = current;
|
|
||||||
revoked.revoked_at = Some(revoked_at);
|
|
||||||
Ok(Json(revoked))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn send_known_worker_input(
|
async fn send_known_worker_input(
|
||||||
State(api): State<WorkspaceApi>,
|
State(api): State<WorkspaceApi>,
|
||||||
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
|
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
|
||||||
@@ -13532,7 +13265,6 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(known.items.len(), 1);
|
assert_eq!(known.items.len(), 1);
|
||||||
assert_eq!(known.items[0].grant_id, "orchestrator-controls-generic");
|
|
||||||
assert_eq!(known.items[0].subject, generic.worker_ref);
|
assert_eq!(known.items[0].subject, generic.worker_ref);
|
||||||
assert_eq!(known.items[0].permissions, ["observe"]);
|
assert_eq!(known.items[0].permissions, ["observe"]);
|
||||||
|
|
||||||
@@ -13571,17 +13303,15 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(capture["entries"].is_array());
|
assert!(capture["entries"].is_array());
|
||||||
|
|
||||||
let Json(revoked) = revoke_worker_control_grant(
|
let revoked = api
|
||||||
State(api.clone()),
|
.store
|
||||||
AxumPath(ScopedWorkerControlGrantPath {
|
.revoke_worker_control_grant(
|
||||||
workspace_id: workspace_id.clone(),
|
&workspace_id,
|
||||||
grant_id: "orchestrator-controls-generic".to_string(),
|
"orchestrator-controls-generic",
|
||||||
}),
|
"2026-07-27T00:00:02Z",
|
||||||
observation_headers.clone(),
|
)
|
||||||
)
|
.unwrap();
|
||||||
.await
|
assert!(revoked);
|
||||||
.unwrap();
|
|
||||||
assert!(revoked.revoked_at.is_some());
|
|
||||||
let revoked_capture = scoped_capture_worker_observation_session(
|
let revoked_capture = scoped_capture_worker_observation_session(
|
||||||
State(api.clone()),
|
State(api.clone()),
|
||||||
AxumPath(ScopedWorkspacePath {
|
AxumPath(ScopedWorkspacePath {
|
||||||
@@ -13620,270 +13350,6 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(unauthorized["sessions"].as_array().unwrap().is_empty());
|
assert!(unauthorized["sessions"].as_array().unwrap().is_empty());
|
||||||
|
|
||||||
for (grant_id, permission) in [
|
|
||||||
("orchestrator-share-source", "share"),
|
|
||||||
("orchestrator-transfer-source", "transfer"),
|
|
||||||
] {
|
|
||||||
api.store
|
|
||||||
.create_worker_control_grant(&WorkerControlGrantRecord {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: grant_id.to_string(),
|
|
||||||
controller: dedicated.worker.clone(),
|
|
||||||
subject: generic.worker_ref.clone(),
|
|
||||||
relation: "spawned".to_string(),
|
|
||||||
origin: "test-delegation".to_string(),
|
|
||||||
permissions: vec!["observe".to_string(), permission.to_string()],
|
|
||||||
operation_id: format!("seed-{permission}"),
|
|
||||||
created_at: "2026-07-27T00:00:01Z".to_string(),
|
|
||||||
revoked_at: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
let target_controller = generic.worker_ref.clone();
|
|
||||||
let Json(shared) = share_worker_control_grant(
|
|
||||||
State(api.clone()),
|
|
||||||
AxumPath(ScopedWorkerControlGrantPath {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: "orchestrator-share-source".to_string(),
|
|
||||||
}),
|
|
||||||
observation_headers.clone(),
|
|
||||||
Json(DelegateWorkerControlGrantRequest {
|
|
||||||
target_controller: target_controller.clone(),
|
|
||||||
operation_id: "share-operation".to_string(),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(shared.controller, target_controller);
|
|
||||||
assert_eq!(shared.relation, "shared");
|
|
||||||
|
|
||||||
let alternate_target = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, "1000");
|
|
||||||
let now = now_registry_timestamp();
|
|
||||||
api.store
|
|
||||||
.upsert_worker_registry(&WorkerRegistryRecord {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
worker: alternate_target.clone(),
|
|
||||||
display_name: "Alternate known target".to_string(),
|
|
||||||
profile: None,
|
|
||||||
retention_state: "normal".to_string(),
|
|
||||||
transcript_ref: None,
|
|
||||||
session_ref: None,
|
|
||||||
summary_ref: None,
|
|
||||||
diagnostics_ref: None,
|
|
||||||
created_at: now.clone(),
|
|
||||||
updated_at: now.clone(),
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
let registry_only_target = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, "1001");
|
|
||||||
api.store
|
|
||||||
.upsert_worker_registry(&WorkerRegistryRecord {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
worker: registry_only_target.clone(),
|
|
||||||
display_name: "Registry-only target".to_string(),
|
|
||||||
profile: None,
|
|
||||||
retention_state: "normal".to_string(),
|
|
||||||
transcript_ref: None,
|
|
||||||
session_ref: None,
|
|
||||||
summary_ref: None,
|
|
||||||
diagnostics_ref: None,
|
|
||||||
created_at: now.clone(),
|
|
||||||
updated_at: now.clone(),
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
let unknown_target = share_worker_control_grant(
|
|
||||||
State(api.clone()),
|
|
||||||
AxumPath(ScopedWorkerControlGrantPath {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: "orchestrator-share-source".to_string(),
|
|
||||||
}),
|
|
||||||
observation_headers.clone(),
|
|
||||||
Json(DelegateWorkerControlGrantRequest {
|
|
||||||
target_controller: registry_only_target,
|
|
||||||
operation_id: "share-registry-only".to_string(),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap_err();
|
|
||||||
assert_eq!(
|
|
||||||
unknown_target.into_response().status(),
|
|
||||||
StatusCode::NOT_FOUND
|
|
||||||
);
|
|
||||||
for (grant_id, operation_id, subject) in [
|
|
||||||
(
|
|
||||||
"orchestrator-knows-alternate",
|
|
||||||
"seed-known-alternate",
|
|
||||||
alternate_target.clone(),
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"orchestrator-second-share-source",
|
|
||||||
"seed-second-share",
|
|
||||||
generic.worker_ref.clone(),
|
|
||||||
),
|
|
||||||
] {
|
|
||||||
api.store
|
|
||||||
.create_worker_control_grant(&WorkerControlGrantRecord {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: grant_id.to_string(),
|
|
||||||
controller: dedicated.worker.clone(),
|
|
||||||
subject,
|
|
||||||
relation: "spawned".to_string(),
|
|
||||||
origin: "test-delegation-conflict".to_string(),
|
|
||||||
permissions: vec!["share".to_string()],
|
|
||||||
operation_id: operation_id.to_string(),
|
|
||||||
created_at: now.clone(),
|
|
||||||
revoked_at: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
let changed_target = share_worker_control_grant(
|
|
||||||
State(api.clone()),
|
|
||||||
AxumPath(ScopedWorkerControlGrantPath {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: "orchestrator-share-source".to_string(),
|
|
||||||
}),
|
|
||||||
observation_headers.clone(),
|
|
||||||
Json(DelegateWorkerControlGrantRequest {
|
|
||||||
target_controller: alternate_target,
|
|
||||||
operation_id: "share-operation".to_string(),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap_err();
|
|
||||||
assert_eq!(
|
|
||||||
changed_target.into_response().status(),
|
|
||||||
StatusCode::BAD_REQUEST
|
|
||||||
);
|
|
||||||
let changed_source_grant = share_worker_control_grant(
|
|
||||||
State(api.clone()),
|
|
||||||
AxumPath(ScopedWorkerControlGrantPath {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: "orchestrator-second-share-source".to_string(),
|
|
||||||
}),
|
|
||||||
observation_headers.clone(),
|
|
||||||
Json(DelegateWorkerControlGrantRequest {
|
|
||||||
target_controller: generic.worker_ref.clone(),
|
|
||||||
operation_id: "share-operation".to_string(),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap_err();
|
|
||||||
assert_eq!(
|
|
||||||
changed_source_grant.into_response().status(),
|
|
||||||
StatusCode::BAD_REQUEST
|
|
||||||
);
|
|
||||||
|
|
||||||
let recovery_source = api
|
|
||||||
.store
|
|
||||||
.get_worker_control_grant(&workspace_id, "orchestrator-transfer-source")
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
let recovery_operation_id = "transfer-operation";
|
|
||||||
let recovery_fingerprint = worker_control_delegation_input_fingerprint(
|
|
||||||
&dedicated.worker,
|
|
||||||
&recovery_source,
|
|
||||||
"transfer",
|
|
||||||
&generic.worker_ref,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let recovery_now = now_registry_timestamp();
|
|
||||||
api.store
|
|
||||||
.reserve_worker_control_delegation_operation(&WorkerControlDelegationOperationRecord {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
source_controller: dedicated.worker.clone(),
|
|
||||||
source_grant_id: recovery_source.grant_id.clone(),
|
|
||||||
operation_id: recovery_operation_id.to_string(),
|
|
||||||
input_fingerprint: recovery_fingerprint,
|
|
||||||
delegated_grant_id: None,
|
|
||||||
created_at: recovery_now.clone(),
|
|
||||||
completed_at: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
let precompleted_transfer = api
|
|
||||||
.store
|
|
||||||
.create_worker_control_grant(&WorkerControlGrantRecord {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: "precompleted-transfer-grant".to_string(),
|
|
||||||
controller: generic.worker_ref.clone(),
|
|
||||||
subject: recovery_source.subject.clone(),
|
|
||||||
relation: "transferred".to_string(),
|
|
||||||
origin: format!("worker_control_transfer:{}", recovery_source.grant_id),
|
|
||||||
permissions: recovery_source.permissions.clone(),
|
|
||||||
operation_id: format!(
|
|
||||||
"worker-control-delegate:{}:{}:{}:transfer:{}",
|
|
||||||
dedicated.worker.runtime_id,
|
|
||||||
dedicated.worker.worker_id,
|
|
||||||
recovery_source.grant_id,
|
|
||||||
recovery_operation_id,
|
|
||||||
),
|
|
||||||
created_at: recovery_now.clone(),
|
|
||||||
revoked_at: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
api.store
|
|
||||||
.complete_worker_control_delegation_operation(
|
|
||||||
&workspace_id,
|
|
||||||
&dedicated.worker,
|
|
||||||
recovery_operation_id,
|
|
||||||
&precompleted_transfer.grant_id,
|
|
||||||
&recovery_now,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
assert!(
|
|
||||||
api.store
|
|
||||||
.get_worker_control_grant(&workspace_id, &recovery_source.grant_id)
|
|
||||||
.unwrap()
|
|
||||||
.unwrap()
|
|
||||||
.revoked_at
|
|
||||||
.is_none()
|
|
||||||
);
|
|
||||||
|
|
||||||
let Json(transferred) = transfer_worker_control_grant(
|
|
||||||
State(api.clone()),
|
|
||||||
AxumPath(ScopedWorkerControlGrantPath {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: "orchestrator-transfer-source".to_string(),
|
|
||||||
}),
|
|
||||||
observation_headers.clone(),
|
|
||||||
Json(DelegateWorkerControlGrantRequest {
|
|
||||||
target_controller: generic.worker_ref.clone(),
|
|
||||||
operation_id: "transfer-operation".to_string(),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(transferred.grant_id, precompleted_transfer.grant_id);
|
|
||||||
assert!(
|
|
||||||
api.store
|
|
||||||
.get_worker_control_grant(&workspace_id, &recovery_source.grant_id)
|
|
||||||
.unwrap()
|
|
||||||
.unwrap()
|
|
||||||
.revoked_at
|
|
||||||
.is_some()
|
|
||||||
);
|
|
||||||
let Json(transfer_replay) = transfer_worker_control_grant(
|
|
||||||
State(api.clone()),
|
|
||||||
AxumPath(ScopedWorkerControlGrantPath {
|
|
||||||
workspace_id: workspace_id.clone(),
|
|
||||||
grant_id: "orchestrator-transfer-source".to_string(),
|
|
||||||
}),
|
|
||||||
observation_headers,
|
|
||||||
Json(DelegateWorkerControlGrantRequest {
|
|
||||||
target_controller: generic.worker_ref.clone(),
|
|
||||||
operation_id: "transfer-operation".to_string(),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(transfer_replay.grant_id, transferred.grant_id);
|
|
||||||
assert!(
|
|
||||||
api.store
|
|
||||||
.get_worker_control_grant(&workspace_id, "orchestrator-transfer-source")
|
|
||||||
.unwrap()
|
|
||||||
.unwrap()
|
|
||||||
.revoked_at
|
|
||||||
.is_some()
|
|
||||||
);
|
|
||||||
|
|
||||||
let Json(existing) = scoped_start_workspace_orchestrator(
|
let Json(existing) = scoped_start_workspace_orchestrator(
|
||||||
State(api.clone()),
|
State(api.clone()),
|
||||||
AxumPath(ScopedWorkspacePath {
|
AxumPath(ScopedWorkspacePath {
|
||||||
|
|||||||
@@ -191,6 +191,11 @@ const MIGRATIONS: &[Migration] = &[
|
|||||||
name: "create Worker control delegation operation authority",
|
name: "create Worker control delegation operation authority",
|
||||||
apply: create_worker_control_delegation_operation_authority,
|
apply: create_worker_control_delegation_operation_authority,
|
||||||
},
|
},
|
||||||
|
Migration {
|
||||||
|
version: 35,
|
||||||
|
name: "remove Worker control delegation authority",
|
||||||
|
apply: remove_worker_control_delegation_authority,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
struct Migration {
|
struct Migration {
|
||||||
@@ -352,18 +357,6 @@ pub struct WorkerControlGrantRecord {
|
|||||||
pub revoked_at: Option<String>,
|
pub revoked_at: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
|
||||||
pub struct WorkerControlDelegationOperationRecord {
|
|
||||||
pub workspace_id: String,
|
|
||||||
pub source_controller: RuntimeWorkerRef,
|
|
||||||
pub source_grant_id: String,
|
|
||||||
pub operation_id: String,
|
|
||||||
pub input_fingerprint: String,
|
|
||||||
pub delegated_grant_id: Option<String>,
|
|
||||||
pub created_at: String,
|
|
||||||
pub completed_at: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct TicketWorkerAssignmentRecord {
|
pub struct TicketWorkerAssignmentRecord {
|
||||||
pub workspace_id: String,
|
pub workspace_id: String,
|
||||||
@@ -804,19 +797,6 @@ pub trait ControlPlaneStore: Send + Sync {
|
|||||||
grant_id: &str,
|
grant_id: &str,
|
||||||
revoked_at: &str,
|
revoked_at: &str,
|
||||||
) -> Result<bool>;
|
) -> Result<bool>;
|
||||||
fn reserve_worker_control_delegation_operation(
|
|
||||||
&self,
|
|
||||||
record: &WorkerControlDelegationOperationRecord,
|
|
||||||
) -> Result<WorkerControlDelegationOperationRecord>;
|
|
||||||
fn complete_worker_control_delegation_operation(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
source_controller: &RuntimeWorkerRef,
|
|
||||||
operation_id: &str,
|
|
||||||
delegated_grant_id: &str,
|
|
||||||
completed_at: &str,
|
|
||||||
) -> Result<WorkerControlDelegationOperationRecord>;
|
|
||||||
|
|
||||||
fn get_ticket_assignment_operation(
|
fn get_ticket_assignment_operation(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
@@ -2578,95 +2558,6 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reserve_worker_control_delegation_operation(
|
|
||||||
&self,
|
|
||||||
record: &WorkerControlDelegationOperationRecord,
|
|
||||||
) -> Result<WorkerControlDelegationOperationRecord> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
conn.execute(
|
|
||||||
r#"INSERT INTO worker_control_delegation_operations (
|
|
||||||
workspace_id, source_controller_runtime_id, source_controller_worker_id,
|
|
||||||
source_grant_id, operation_id, input_fingerprint,
|
|
||||||
delegated_grant_id, created_at, completed_at
|
|
||||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
|
||||||
ON CONFLICT (
|
|
||||||
workspace_id, source_controller_runtime_id,
|
|
||||||
source_controller_worker_id, operation_id
|
|
||||||
) DO NOTHING"#,
|
|
||||||
params![
|
|
||||||
record.workspace_id,
|
|
||||||
record.source_controller.runtime_id,
|
|
||||||
record.source_controller.worker_id,
|
|
||||||
record.source_grant_id,
|
|
||||||
record.operation_id,
|
|
||||||
record.input_fingerprint,
|
|
||||||
record.delegated_grant_id,
|
|
||||||
record.created_at,
|
|
||||||
record.completed_at,
|
|
||||||
],
|
|
||||||
)?;
|
|
||||||
let persisted = read_worker_control_delegation_operation_by_key(
|
|
||||||
conn,
|
|
||||||
&record.workspace_id,
|
|
||||||
&record.source_controller,
|
|
||||||
&record.operation_id,
|
|
||||||
)?
|
|
||||||
.ok_or_else(|| {
|
|
||||||
Error::Store("worker control delegation operation was not persisted".to_string())
|
|
||||||
})?;
|
|
||||||
if persisted.source_grant_id != record.source_grant_id
|
|
||||||
|| persisted.input_fingerprint != record.input_fingerprint
|
|
||||||
{
|
|
||||||
return Err(Error::InvalidInput(format!(
|
|
||||||
"worker control delegation operation `{}` was already used with different input",
|
|
||||||
record.operation_id
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
Ok(persisted)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn complete_worker_control_delegation_operation(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
source_controller: &RuntimeWorkerRef,
|
|
||||||
operation_id: &str,
|
|
||||||
delegated_grant_id: &str,
|
|
||||||
completed_at: &str,
|
|
||||||
) -> Result<WorkerControlDelegationOperationRecord> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
conn.execute(
|
|
||||||
r#"UPDATE worker_control_delegation_operations
|
|
||||||
SET delegated_grant_id = ?5, completed_at = ?6
|
|
||||||
WHERE workspace_id = ?1
|
|
||||||
AND source_controller_runtime_id = ?2
|
|
||||||
AND source_controller_worker_id = ?3
|
|
||||||
AND operation_id = ?4
|
|
||||||
AND (delegated_grant_id IS NULL OR delegated_grant_id = ?5)"#,
|
|
||||||
params![
|
|
||||||
workspace_id,
|
|
||||||
source_controller.runtime_id,
|
|
||||||
source_controller.worker_id,
|
|
||||||
operation_id,
|
|
||||||
delegated_grant_id,
|
|
||||||
completed_at,
|
|
||||||
],
|
|
||||||
)?;
|
|
||||||
read_worker_control_delegation_operation_by_key(
|
|
||||||
conn,
|
|
||||||
workspace_id,
|
|
||||||
source_controller,
|
|
||||||
operation_id,
|
|
||||||
)?
|
|
||||||
.filter(|record| record.delegated_grant_id.as_deref() == Some(delegated_grant_id))
|
|
||||||
.ok_or_else(|| {
|
|
||||||
Error::InvalidInput(format!(
|
|
||||||
"worker control delegation operation `{operation_id}` completed with a different grant"
|
|
||||||
))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_ticket_assignment_operation(
|
fn get_ticket_assignment_operation(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
@@ -4024,52 +3915,6 @@ fn read_worker_control_grant_by_operation(
|
|||||||
.map_err(Error::from)
|
.map_err(Error::from)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read_worker_control_delegation_operation_record(
|
|
||||||
row: &rusqlite::Row<'_>,
|
|
||||||
) -> rusqlite::Result<WorkerControlDelegationOperationRecord> {
|
|
||||||
Ok(WorkerControlDelegationOperationRecord {
|
|
||||||
workspace_id: row.get(0)?,
|
|
||||||
source_controller: RuntimeWorkerRef::new(
|
|
||||||
row.get::<_, String>(1)?,
|
|
||||||
row.get::<_, u64>(2)?.to_string(),
|
|
||||||
),
|
|
||||||
source_grant_id: row.get(3)?,
|
|
||||||
operation_id: row.get(4)?,
|
|
||||||
input_fingerprint: row.get(5)?,
|
|
||||||
delegated_grant_id: row.get(6)?,
|
|
||||||
created_at: row.get(7)?,
|
|
||||||
completed_at: row.get(8)?,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read_worker_control_delegation_operation_by_key(
|
|
||||||
conn: &Connection,
|
|
||||||
workspace_id: &str,
|
|
||||||
source_controller: &RuntimeWorkerRef,
|
|
||||||
operation_id: &str,
|
|
||||||
) -> Result<Option<WorkerControlDelegationOperationRecord>> {
|
|
||||||
conn.query_row(
|
|
||||||
r#"SELECT workspace_id,
|
|
||||||
source_controller_runtime_id, source_controller_worker_id,
|
|
||||||
source_grant_id, operation_id, input_fingerprint,
|
|
||||||
delegated_grant_id, created_at, completed_at
|
|
||||||
FROM worker_control_delegation_operations
|
|
||||||
WHERE workspace_id = ?1
|
|
||||||
AND source_controller_runtime_id = ?2
|
|
||||||
AND source_controller_worker_id = ?3
|
|
||||||
AND operation_id = ?4"#,
|
|
||||||
params![
|
|
||||||
workspace_id,
|
|
||||||
source_controller.runtime_id,
|
|
||||||
source_controller.worker_id,
|
|
||||||
operation_id,
|
|
||||||
],
|
|
||||||
read_worker_control_delegation_operation_record,
|
|
||||||
)
|
|
||||||
.optional()
|
|
||||||
.map_err(Error::from)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn current_ticket_worker_assignment_select_sql() -> String {
|
fn current_ticket_worker_assignment_select_sql() -> String {
|
||||||
"SELECT a.workspace_id, a.ticket_id, a.assignment_id, a.runtime_id, a.worker_id, \
|
"SELECT a.workspace_id, a.ticket_id, a.assignment_id, a.runtime_id, a.worker_id, \
|
||||||
a.assigned_by, a.assigned_at \
|
a.assigned_by, a.assigned_at \
|
||||||
@@ -5168,6 +5013,58 @@ fn create_worker_control_delegation_operation_authority(conn: &Connection) -> Re
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn remove_worker_control_delegation_authority(conn: &Connection) -> Result<()> {
|
||||||
|
let mut statement =
|
||||||
|
conn.prepare("SELECT workspace_id, grant_id, permissions_json FROM worker_control_grants")?;
|
||||||
|
let rows = statement.query_map([], |row| {
|
||||||
|
Ok((
|
||||||
|
row.get::<_, String>(0)?,
|
||||||
|
row.get::<_, String>(1)?,
|
||||||
|
row.get::<_, String>(2)?,
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let mut permission_updates = Vec::new();
|
||||||
|
for row in rows {
|
||||||
|
let (workspace_id, grant_id, permissions_json) = row?;
|
||||||
|
let mut permissions: Vec<String> =
|
||||||
|
serde_json::from_str(&permissions_json).map_err(|error| {
|
||||||
|
Error::Store(format!(
|
||||||
|
"failed to decode Worker control grant `{grant_id}` permissions during delegation removal: {error}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let previous_len = permissions.len();
|
||||||
|
permissions
|
||||||
|
.retain(|permission| !matches!(permission.as_str(), "share" | "transfer" | "revoke"));
|
||||||
|
if permissions.len() != previous_len {
|
||||||
|
permission_updates.push((
|
||||||
|
workspace_id,
|
||||||
|
grant_id,
|
||||||
|
serde_json::to_string(&permissions).map_err(|error| {
|
||||||
|
Error::Store(format!(
|
||||||
|
"failed to encode Worker control grant permissions during delegation removal: {error}"
|
||||||
|
))
|
||||||
|
})?,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(statement);
|
||||||
|
|
||||||
|
for (workspace_id, grant_id, permissions_json) in permission_updates {
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE worker_control_grants SET permissions_json = ?3 WHERE workspace_id = ?1 AND grant_id = ?2",
|
||||||
|
params![workspace_id, grant_id, permissions_json],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
conn.execute(
|
||||||
|
r#"UPDATE worker_control_grants
|
||||||
|
SET revoked_at = COALESCE(revoked_at, strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
|
||||||
|
WHERE relation IN ('shared', 'transferred')"#,
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
conn.execute_batch("DROP TABLE IF EXISTS worker_control_delegation_operations;")?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn create_worker_mutation_source_proof_replay_guard(conn: &Connection) -> Result<()> {
|
fn create_worker_mutation_source_proof_replay_guard(conn: &Connection) -> Result<()> {
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
r#"
|
r#"
|
||||||
@@ -5819,6 +5716,82 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_v35_removes_worker_control_delegation_authority() {
|
||||||
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
|
configure_sqlite(&conn).unwrap();
|
||||||
|
for migration in MIGRATIONS
|
||||||
|
.iter()
|
||||||
|
.filter(|migration| migration.version <= 34)
|
||||||
|
{
|
||||||
|
let tx = conn.unchecked_transaction().unwrap();
|
||||||
|
(migration.apply)(&tx).unwrap();
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)",
|
||||||
|
params![migration.version, migration.name],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
tx.commit().unwrap();
|
||||||
|
}
|
||||||
|
conn.execute_batch(
|
||||||
|
r#"
|
||||||
|
INSERT INTO workspaces (
|
||||||
|
workspace_id, display_name, state, created_at, updated_at
|
||||||
|
) VALUES ('workspace-a', 'Workspace A', 'active', '1', '1');
|
||||||
|
INSERT INTO worker_registry (
|
||||||
|
workspace_id, runtime_id, runtime_worker_id, display_name,
|
||||||
|
retention_state, created_at, updated_at
|
||||||
|
) VALUES
|
||||||
|
('workspace-a', 'runtime-a', 1, 'Controller', 'normal', '1', '1'),
|
||||||
|
('workspace-a', 'runtime-a', 2, 'Spawned Worker', 'normal', '1', '1'),
|
||||||
|
('workspace-a', 'runtime-a', 3, 'Shared Worker', 'normal', '1', '1'),
|
||||||
|
('workspace-a', 'runtime-a', 4, 'Transferred Worker', 'normal', '1', '1');
|
||||||
|
INSERT INTO worker_control_grants (
|
||||||
|
workspace_id, grant_id,
|
||||||
|
controller_runtime_id, controller_worker_id,
|
||||||
|
subject_runtime_id, subject_worker_id,
|
||||||
|
relation, origin, permissions_json, operation_id, created_at, revoked_at
|
||||||
|
) VALUES
|
||||||
|
('workspace-a', 'spawned', 'runtime-a', 1, 'runtime-a', 2,
|
||||||
|
'spawned', 'spawn', '["observe","share","transfer","revoke","stop"]', 'spawn-op', '1', NULL),
|
||||||
|
('workspace-a', 'shared', 'runtime-a', 1, 'runtime-a', 3,
|
||||||
|
'shared', 'share', '["observe"]', 'share-op', '1', NULL),
|
||||||
|
('workspace-a', 'transferred', 'runtime-a', 1, 'runtime-a', 4,
|
||||||
|
'transferred', 'transfer', '["observe"]', 'transfer-op', '1', NULL);
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(current_schema_version(&conn).unwrap(), 34);
|
||||||
|
assert!(table_exists(&conn, "worker_control_delegation_operations").unwrap());
|
||||||
|
|
||||||
|
apply_migrations(&conn).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(current_schema_version(&conn).unwrap(), 35);
|
||||||
|
assert!(!table_exists(&conn, "worker_control_delegation_operations").unwrap());
|
||||||
|
let (permissions_json, revoked_at): (String, Option<String>) = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT permissions_json, revoked_at FROM worker_control_grants WHERE grant_id = 'spawned'",
|
||||||
|
[],
|
||||||
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_str::<Vec<String>>(&permissions_json).unwrap(),
|
||||||
|
vec!["observe", "stop"]
|
||||||
|
);
|
||||||
|
assert!(revoked_at.is_none());
|
||||||
|
for grant_id in ["shared", "transferred"] {
|
||||||
|
let revoked_at: Option<String> = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT revoked_at FROM worker_control_grants WHERE grant_id = ?1",
|
||||||
|
[grant_id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(revoked_at.is_some(), "{grant_id} grant remained active");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn schema_v24_adds_attachment_reservations_to_already_applied_v23() {
|
fn schema_v24_adds_attachment_reservations_to_already_applied_v23() {
|
||||||
let conn = Connection::open_in_memory().unwrap();
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
@@ -5841,7 +5814,7 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
|
|
||||||
apply_migrations(&conn).unwrap();
|
apply_migrations(&conn).unwrap();
|
||||||
|
|
||||||
assert_eq!(current_schema_version(&conn).unwrap(), 34);
|
assert_eq!(current_schema_version(&conn).unwrap(), 35);
|
||||||
assert!(table_exists(&conn, "worker_workdir_attachment_reservations").unwrap());
|
assert!(table_exists(&conn, "worker_workdir_attachment_reservations").unwrap());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5874,7 +5847,7 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY);
|
|||||||
|
|
||||||
apply_migrations(&conn).unwrap();
|
apply_migrations(&conn).unwrap();
|
||||||
|
|
||||||
assert_eq!(current_schema_version(&conn).unwrap(), 34);
|
assert_eq!(current_schema_version(&conn).unwrap(), 35);
|
||||||
assert!(table_exists(&conn, "flow_sources").unwrap());
|
assert!(table_exists(&conn, "flow_sources").unwrap());
|
||||||
assert!(table_exists(&conn, "flow_source_revisions").unwrap());
|
assert!(table_exists(&conn, "flow_source_revisions").unwrap());
|
||||||
assert!(!table_exists(&conn, "flow_instances").unwrap());
|
assert!(!table_exists(&conn, "flow_instances").unwrap());
|
||||||
@@ -5941,7 +5914,7 @@ INSERT INTO worker_workdir_attachment_reservations (
|
|||||||
|
|
||||||
apply_migrations(&conn).unwrap();
|
apply_migrations(&conn).unwrap();
|
||||||
|
|
||||||
assert_eq!(current_schema_version(&conn).unwrap(), 34);
|
assert_eq!(current_schema_version(&conn).unwrap(), 35);
|
||||||
let repositories_sql: String = conn
|
let repositories_sql: String = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'repositories'",
|
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'repositories'",
|
||||||
@@ -6121,7 +6094,7 @@ INSERT INTO workdir_registry (
|
|||||||
let db = dir.path().join("control-plane.sqlite");
|
let db = dir.path().join("control-plane.sqlite");
|
||||||
let store = SqliteWorkspaceStore::open(&db).unwrap();
|
let store = SqliteWorkspaceStore::open(&db).unwrap();
|
||||||
|
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 34);
|
assert_eq!(store.schema_version().await.unwrap(), 35);
|
||||||
assert!(
|
assert!(
|
||||||
!store
|
!store
|
||||||
.with_conn(|conn| table_exists(conn, "worker_workspace_credentials"))
|
.with_conn(|conn| table_exists(conn, "worker_workspace_credentials"))
|
||||||
@@ -6138,7 +6111,7 @@ INSERT INTO workdir_registry (
|
|||||||
store.upsert_workspace(&record).await.unwrap();
|
store.upsert_workspace(&record).await.unwrap();
|
||||||
|
|
||||||
let reopened = SqliteWorkspaceStore::open(&db).unwrap();
|
let reopened = SqliteWorkspaceStore::open(&db).unwrap();
|
||||||
assert_eq!(reopened.schema_version().await.unwrap(), 34);
|
assert_eq!(reopened.schema_version().await.unwrap(), 35);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
reopened.get_workspace("local-dev").await.unwrap(),
|
reopened.get_workspace("local-dev").await.unwrap(),
|
||||||
Some(record)
|
Some(record)
|
||||||
@@ -6685,7 +6658,7 @@ INSERT INTO workdir_registry (
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let store = SqliteWorkspaceStore::from_connection(conn).unwrap();
|
let store = SqliteWorkspaceStore::from_connection(conn).unwrap();
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 34);
|
assert_eq!(store.schema_version().await.unwrap(), 35);
|
||||||
|
|
||||||
store
|
store
|
||||||
.with_conn(|conn| {
|
.with_conn(|conn| {
|
||||||
@@ -6874,7 +6847,7 @@ CREATE TABLE ticket_assignment_operations (
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn repository_records_round_trip() {
|
async fn repository_records_round_trip() {
|
||||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 34);
|
assert_eq!(store.schema_version().await.unwrap(), 35);
|
||||||
let workspace = WorkspaceRecord {
|
let workspace = WorkspaceRecord {
|
||||||
workspace_id: "local-dev".to_string(),
|
workspace_id: "local-dev".to_string(),
|
||||||
owner_account_id: None,
|
owner_account_id: None,
|
||||||
@@ -6940,7 +6913,7 @@ CREATE TABLE ticket_assignment_operations (
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn memory_authority_records_round_trip_and_close_staging() {
|
async fn memory_authority_records_round_trip_and_close_staging() {
|
||||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 34);
|
assert_eq!(store.schema_version().await.unwrap(), 35);
|
||||||
let workspace = WorkspaceRecord {
|
let workspace = WorkspaceRecord {
|
||||||
workspace_id: "local-dev".to_string(),
|
workspace_id: "local-dev".to_string(),
|
||||||
owner_account_id: None,
|
owner_account_id: None,
|
||||||
@@ -7201,7 +7174,7 @@ CREATE TABLE ticket_assignment_operations (
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn worker_control_grants_are_idempotent_scoped_and_revocable() {
|
async fn worker_control_grants_are_idempotent_scoped_and_support_internal_invalidation() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
let database = dir.path().join("control-grants.db");
|
let database = dir.path().join("control-grants.db");
|
||||||
let store = SqliteWorkspaceStore::open(&database).unwrap();
|
let store = SqliteWorkspaceStore::open(&database).unwrap();
|
||||||
@@ -7331,7 +7304,7 @@ CREATE TABLE ticket_assignment_operations (
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn account_and_login_records_round_trip() {
|
async fn account_and_login_records_round_trip() {
|
||||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 34);
|
assert_eq!(store.schema_version().await.unwrap(), 35);
|
||||||
let now = "2026-07-22T00:00:00Z".to_string();
|
let now = "2026-07-22T00:00:00Z".to_string();
|
||||||
let account = AccountRecord {
|
let account = AccountRecord {
|
||||||
account_id: "acct-user-alice".to_string(),
|
account_id: "acct-user-alice".to_string(),
|
||||||
|
|||||||
Reference in New Issue
Block a user