auth: enforce workspace worker credentials over ticket REST
This commit is contained in:
@@ -31,6 +31,7 @@ pub enum WorkerExecutionOperation {
|
||||
Restore,
|
||||
Input,
|
||||
ProtocolMethod,
|
||||
ReplaceWorkspaceAccessToken,
|
||||
Stop,
|
||||
Cancel,
|
||||
}
|
||||
@@ -331,6 +332,17 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn replace_workspace_access_token(
|
||||
&self,
|
||||
_handle: &WorkerExecutionHandle,
|
||||
_access_token: String,
|
||||
) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::unsupported(
|
||||
WorkerExecutionOperation::ReplaceWorkspaceAccessToken,
|
||||
"execution backend does not support replacing Workspace access tokens",
|
||||
)
|
||||
}
|
||||
|
||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::unsupported(
|
||||
WorkerExecutionOperation::Stop,
|
||||
@@ -443,6 +455,15 @@ impl WorkerExecutionBackendRef {
|
||||
self.backend.worker_completions(handle, kind, prefix)
|
||||
}
|
||||
|
||||
pub(crate) fn replace_workspace_access_token(
|
||||
&self,
|
||||
handle: &WorkerExecutionHandle,
|
||||
access_token: String,
|
||||
) -> WorkerExecutionResult {
|
||||
self.backend
|
||||
.replace_workspace_access_token(handle, access_token)
|
||||
}
|
||||
|
||||
pub(crate) fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
self.backend.stop_worker(handle)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::auth::{
|
||||
};
|
||||
use crate::catalog::{
|
||||
ConfigBundleRef, CreateWorkerRequest, WorkerDetail, WorkerLifecycleAck, WorkerSummary,
|
||||
WorkingDirectoryRequest, WorkingDirectoryStatus,
|
||||
WorkingDirectoryRequest, WorkingDirectoryStatus, WorkspaceApiRef,
|
||||
};
|
||||
use crate::config_bundle::{ConfigBundle, ConfigBundleAvailability, ConfigBundleSummary};
|
||||
use crate::error::RuntimeError;
|
||||
@@ -193,6 +193,10 @@ fn runtime_http_router_with_optional_auth(
|
||||
)
|
||||
.route("/v1/workers/{worker_id}/input", post(send_worker_input))
|
||||
.route("/v1/workers/{worker_id}/restore", post(restore_worker))
|
||||
.route(
|
||||
"/v1/workers/{worker_id}/workspace-api",
|
||||
post(replace_worker_workspace_api),
|
||||
)
|
||||
.route(
|
||||
"/v1/workers/{worker_id}/completions",
|
||||
post(worker_completions),
|
||||
@@ -282,6 +286,12 @@ pub struct RuntimeHttpWorkerResponse {
|
||||
pub worker: WorkerDetail,
|
||||
}
|
||||
|
||||
/// Replace the Workspace API binding for an existing Worker.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RuntimeHttpWorkerWorkspaceApiRequest {
|
||||
pub workspace_api: WorkspaceApiRef,
|
||||
}
|
||||
|
||||
/// Worker delete response.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RuntimeHttpWorkerDeleteResponse {
|
||||
@@ -509,6 +519,28 @@ async fn create_worker(
|
||||
Ok(Json(RuntimeHttpWorkerResponse { worker }))
|
||||
}
|
||||
|
||||
async fn replace_worker_workspace_api(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
auth: Option<Extension<RuntimeAuthContext>>,
|
||||
Path(worker_id): Path<String>,
|
||||
body: Result<Json<RuntimeHttpWorkerWorkspaceApiRequest>, JsonRejection>,
|
||||
) -> RestResult<RuntimeHttpWorkerResponse> {
|
||||
let Json(request) = body.map_err(RuntimeHttpRestError::json_rejection)?;
|
||||
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
|
||||
let worker = match auth_workspace_scope(&state, auth.as_ref())? {
|
||||
Some(scope) => state.runtime.replace_worker_workspace_api_scoped(
|
||||
&scope,
|
||||
&worker_ref,
|
||||
request.workspace_api,
|
||||
),
|
||||
None => state
|
||||
.runtime
|
||||
.replace_worker_workspace_api(&worker_ref, request.workspace_api),
|
||||
}
|
||||
.map_err(RuntimeHttpRestError::runtime)?;
|
||||
Ok(Json(RuntimeHttpWorkerResponse { worker }))
|
||||
}
|
||||
|
||||
async fn restore_worker(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
auth: Option<Extension<RuntimeAuthContext>>,
|
||||
@@ -960,6 +992,9 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
||||
if path.starts_with("/v1/config-bundles") || path.starts_with("/v1/working-directories") {
|
||||
return Some("workers:create");
|
||||
}
|
||||
if path.ends_with("/workspace-api") {
|
||||
return Some("workers:create");
|
||||
}
|
||||
if path.ends_with("/input") || path.ends_with("/restore") {
|
||||
return Some("workers:input");
|
||||
}
|
||||
@@ -1484,6 +1519,17 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
fn replace_workspace_access_token(
|
||||
&self,
|
||||
_handle: &WorkerExecutionHandle,
|
||||
_access_token: String,
|
||||
) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::ReplaceWorkspaceAccessToken,
|
||||
WorkerExecutionRunState::Idle,
|
||||
)
|
||||
}
|
||||
|
||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Stop,
|
||||
@@ -1575,6 +1621,23 @@ mod tests {
|
||||
created.worker.worker_id
|
||||
);
|
||||
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&format!("/v1/workers/{}/workspace-api", created.worker.worker_id),
|
||||
token,
|
||||
&RuntimeHttpWorkerWorkspaceApiRequest {
|
||||
workspace_api: WorkspaceApiRef {
|
||||
workspace_id: "local".to_string(),
|
||||
base_url: "http://127.0.0.1:8787".to_string(),
|
||||
runtime_id: None,
|
||||
access_token: Some("workspace-access-token".to_string()),
|
||||
},
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let input = WorkerInput::user("hello from backend");
|
||||
let response = authed_json_request(
|
||||
app.clone(),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use crate::catalog::{
|
||||
ConfigBundleRef, CreateWorkerRequest, WorkerDetail, WorkerLifecycleAck, WorkerStatus,
|
||||
WorkerSummary, WorkingDirectoryRequest,
|
||||
WorkingDirectoryStatus as CatalogWorkingDirectoryStatus,
|
||||
WorkingDirectoryStatus as CatalogWorkingDirectoryStatus, WorkspaceApiRef,
|
||||
};
|
||||
use crate::config_bundle::{
|
||||
ConfigBundle, ConfigBundleAvailability, ConfigBundleSummary, validate_config_bundle,
|
||||
@@ -589,6 +589,94 @@ impl Runtime {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Replace the Workspace API binding persisted for a Worker and update the
|
||||
/// live execution when one is connected.
|
||||
pub fn replace_worker_workspace_api_scoped(
|
||||
&self,
|
||||
scope: &RuntimeWorkspaceScope,
|
||||
worker_ref: &WorkerRef,
|
||||
workspace_api: WorkspaceApiRef,
|
||||
) -> Result<WorkerDetail, RuntimeError> {
|
||||
self.ensure_worker_in_workspace(scope, worker_ref)?;
|
||||
if workspace_api.workspace_id != scope.workspace_id {
|
||||
return Err(RuntimeError::InvalidRequest(format!(
|
||||
"Workspace API scope `{}` does not match authorized workspace `{}`",
|
||||
workspace_api.workspace_id, scope.workspace_id
|
||||
)));
|
||||
}
|
||||
self.replace_worker_workspace_api(worker_ref, workspace_api)
|
||||
}
|
||||
|
||||
pub fn replace_worker_workspace_api(
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
workspace_api: WorkspaceApiRef,
|
||||
) -> Result<WorkerDetail, RuntimeError> {
|
||||
let access_token = workspace_api
|
||||
.access_token
|
||||
.as_ref()
|
||||
.filter(|token| !token.trim().is_empty())
|
||||
.cloned()
|
||||
.ok_or_else(|| {
|
||||
RuntimeError::InvalidRequest(
|
||||
"Workspace API replacement requires an access token".to_string(),
|
||||
)
|
||||
})?;
|
||||
let (previous_workspace_api, live_execution) = {
|
||||
let state = self.lock()?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
if let Some(existing) = worker.request.workspace_api.as_ref()
|
||||
&& (existing.workspace_id != workspace_api.workspace_id
|
||||
|| existing.base_url.trim_end_matches('/')
|
||||
!= workspace_api.base_url.trim_end_matches('/')
|
||||
|| existing.runtime_id.as_ref().is_some_and(|runtime_id| {
|
||||
workspace_api.runtime_id.as_ref() != Some(runtime_id)
|
||||
}))
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Workspace API replacement cannot change Worker Workspace identity, Runtime identity, or base URL"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
let live_execution = match (
|
||||
state.execution_backend.clone(),
|
||||
worker.execution_handle.clone(),
|
||||
) {
|
||||
(Some(backend), Some(handle)) => Some((backend, handle)),
|
||||
_ => None,
|
||||
};
|
||||
(worker.request.workspace_api.clone(), live_execution)
|
||||
};
|
||||
|
||||
{
|
||||
let mut state = self.lock()?;
|
||||
state.worker_mut(worker_ref)?.request.workspace_api = Some(workspace_api);
|
||||
if let Err(error) = state.persist_runtime_snapshot() {
|
||||
state.worker_mut(worker_ref)?.request.workspace_api = previous_workspace_api;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((backend, handle)) = live_execution {
|
||||
let result = backend.replace_workspace_access_token(&handle, access_token);
|
||||
if !result.is_accepted() {
|
||||
let mut state = self.lock()?;
|
||||
state.worker_mut(worker_ref)?.request.workspace_api = previous_workspace_api;
|
||||
state.persist_runtime_snapshot()?;
|
||||
return Err(RuntimeError::WorkerExecutionRejected {
|
||||
worker_id: worker_ref.worker_id.clone(),
|
||||
operation: result.operation,
|
||||
outcome: result.outcome,
|
||||
message: result.message_or_default(),
|
||||
result,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let state = self.lock()?;
|
||||
Ok(state.worker(worker_ref)?.detail())
|
||||
}
|
||||
|
||||
/// Attach a live execution through a workspace-scoped Runtime authorization context.
|
||||
pub fn restore_worker_scoped(
|
||||
&self,
|
||||
@@ -953,7 +1041,8 @@ impl Runtime {
|
||||
WorkerExecutionOperation::Spawn
|
||||
| WorkerExecutionOperation::Restore
|
||||
| WorkerExecutionOperation::Input
|
||||
| WorkerExecutionOperation::ProtocolMethod => return Ok(()),
|
||||
| WorkerExecutionOperation::ProtocolMethod
|
||||
| WorkerExecutionOperation::ReplaceWorkspaceAccessToken => return Ok(()),
|
||||
};
|
||||
if result.is_accepted() {
|
||||
return Ok(());
|
||||
@@ -2228,6 +2317,7 @@ mod tests {
|
||||
restore_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
||||
restore_count: Mutex<u64>,
|
||||
contexts: Mutex<BTreeMap<WorkerId, WorkerExecutionContext>>,
|
||||
workspace_access_tokens: Mutex<BTreeMap<WorkerId, String>>,
|
||||
#[cfg(feature = "ws-server")]
|
||||
snapshots: Mutex<BTreeMap<WorkerId, protocol::Event>>,
|
||||
}
|
||||
@@ -2316,6 +2406,21 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
fn replace_workspace_access_token(
|
||||
&self,
|
||||
handle: &WorkerExecutionHandle,
|
||||
access_token: String,
|
||||
) -> WorkerExecutionResult {
|
||||
self.workspace_access_tokens
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(handle.worker_ref().worker_id.clone(), access_token);
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::ReplaceWorkspaceAccessToken,
|
||||
WorkerExecutionRunState::Idle,
|
||||
)
|
||||
}
|
||||
|
||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Stop,
|
||||
@@ -2442,6 +2547,46 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_api_replacement_updates_live_execution_and_persisted_request() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
let scope = scope("workspace-a", "server-a");
|
||||
let worker = runtime
|
||||
.create_worker_scoped(
|
||||
&scope,
|
||||
scoped_task_request("repair credential", "workspace-a"),
|
||||
)
|
||||
.unwrap();
|
||||
let replacement = WorkspaceApiRef {
|
||||
workspace_id: "workspace-a".to_string(),
|
||||
base_url: "https://workspace.example/workspace-a/".to_string(),
|
||||
runtime_id: Some("runtime-a".to_string()),
|
||||
access_token: Some("replacement-token".to_string()),
|
||||
};
|
||||
|
||||
runtime
|
||||
.replace_worker_workspace_api_scoped(&scope, &worker.worker_ref, replacement.clone())
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
backend
|
||||
.workspace_access_tokens
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&worker.worker_ref.worker_id),
|
||||
Some(&"replacement-token".to_string())
|
||||
);
|
||||
let state = runtime.lock().unwrap();
|
||||
assert_eq!(
|
||||
state
|
||||
.worker(&worker.worker_ref)
|
||||
.unwrap()
|
||||
.request
|
||||
.workspace_api,
|
||||
Some(replacement)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_owner_binding_rejects_other_backend_and_forgets_after_last_worker_delete() {
|
||||
let runtime = runtime_with_backend();
|
||||
|
||||
@@ -1124,6 +1124,34 @@ where
|
||||
result
|
||||
}
|
||||
|
||||
fn replace_workspace_access_token(
|
||||
&self,
|
||||
handle: &WorkerExecutionHandle,
|
||||
access_token: String,
|
||||
) -> WorkerExecutionResult {
|
||||
let (worker, _busy) = match self.get_execution(handle) {
|
||||
Ok(execution) => execution,
|
||||
Err(mut result) => {
|
||||
result.operation = WorkerExecutionOperation::ReplaceWorkspaceAccessToken;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
worker
|
||||
.replace_workspace_access_token(access_token)
|
||||
.map(|_| {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::ReplaceWorkspaceAccessToken,
|
||||
WorkerExecutionRunState::Idle,
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|error| {
|
||||
WorkerExecutionResult::errored(
|
||||
WorkerExecutionOperation::ReplaceWorkspaceAccessToken,
|
||||
error.to_string(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
if handle.backend_id() != self.backend_id() {
|
||||
return WorkerExecutionResult::rejected(
|
||||
|
||||
Reference in New Issue
Block a user