feat: route backend worker controls over protocol
This commit is contained in:
@@ -514,6 +514,21 @@ pub enum RuntimeRegistryError {
|
||||
}
|
||||
|
||||
impl RuntimeRegistryError {
|
||||
pub fn message(&self) -> String {
|
||||
match self {
|
||||
Self::InvalidIdentifier { kind, value } => {
|
||||
format!("invalid {kind} identifier `{value}`")
|
||||
}
|
||||
Self::UnknownRuntime(runtime_id) => format!("unknown runtime `{runtime_id}`"),
|
||||
Self::UnknownHost(host_id) => format!("unknown host `{host_id}`"),
|
||||
Self::UnknownWorker {
|
||||
runtime_id,
|
||||
worker_id,
|
||||
} => format!("unknown worker `{worker_id}` in runtime `{runtime_id}`"),
|
||||
Self::RuntimeOperationFailed { message, .. } => message.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_error(self) -> Error {
|
||||
match self {
|
||||
Self::InvalidIdentifier { kind, value } => Error::InvalidRuntimeIdentifier {
|
||||
@@ -658,6 +673,18 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
}
|
||||
}
|
||||
|
||||
fn send_protocol_method(
|
||||
&self,
|
||||
_worker_id: &str,
|
||||
_method: protocol::Method,
|
||||
) -> Result<Vec<protocol::Event>, RuntimeRegistryError> {
|
||||
Err(RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: self.runtime_id().to_string(),
|
||||
code: "worker_protocol_method_unsupported".to_string(),
|
||||
message: "runtime does not support Worker protocol command transport".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn stop_worker(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
@@ -1062,6 +1089,26 @@ impl RuntimeRegistry {
|
||||
Ok(runtime.list_config_bundles())
|
||||
}
|
||||
|
||||
pub fn send_protocol_method(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
worker_id: &str,
|
||||
method: protocol::Method,
|
||||
) -> Result<Vec<protocol::Event>, RuntimeRegistryError> {
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
validate_backend_identifier("worker_id", worker_id)?;
|
||||
let runtime = self.runtime(runtime_id)?;
|
||||
let lookup = runtime.worker(worker_id);
|
||||
if lookup.worker.is_none() {
|
||||
return Err(operation_failed_or_unknown_worker(
|
||||
runtime_id,
|
||||
worker_id,
|
||||
lookup.diagnostics,
|
||||
));
|
||||
}
|
||||
runtime.send_protocol_method(worker_id, method)
|
||||
}
|
||||
|
||||
pub fn send_input(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -1807,6 +1854,35 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
))
|
||||
}
|
||||
|
||||
fn send_protocol_method(
|
||||
&self,
|
||||
worker_id: &str,
|
||||
method: protocol::Method,
|
||||
) -> Result<Vec<protocol::Event>, RuntimeRegistryError> {
|
||||
if !self.execution_enabled {
|
||||
return Err(RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: self.runtime_id.clone(),
|
||||
code: "embedded_worker_execution_unavailable".to_string(),
|
||||
message: format!(
|
||||
"worker protocol command for '{worker_id}' requires an embedded execution backend"
|
||||
),
|
||||
});
|
||||
}
|
||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||
return Err(RuntimeRegistryError::UnknownWorker {
|
||||
runtime_id: self.runtime_id.clone(),
|
||||
worker_id: worker_id.to_string(),
|
||||
});
|
||||
};
|
||||
self.runtime
|
||||
.send_protocol_method(&worker_ref, method)
|
||||
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: self.runtime_id.clone(),
|
||||
code: "embedded_worker_protocol_command_failed".to_string(),
|
||||
message: error.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn send_input(&self, worker_id: &str, request: WorkerInputRequest) -> WorkerInputResult {
|
||||
if !self.execution_enabled {
|
||||
return embedded_input_rejected(
|
||||
|
||||
@@ -555,6 +555,14 @@ pub fn build_router(api: WorkspaceApi) -> Router {
|
||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/events/ws",
|
||||
get(scoped_worker_observation_ws),
|
||||
)
|
||||
.route(
|
||||
"/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws",
|
||||
get(worker_protocol_ws),
|
||||
)
|
||||
.route(
|
||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws",
|
||||
get(scoped_worker_protocol_ws),
|
||||
)
|
||||
.route("/api/hosts/{host_id}/workers", get(list_host_workers))
|
||||
.route(
|
||||
"/api/w/{workspace_id}/hosts/{host_id}/workers",
|
||||
@@ -2457,6 +2465,19 @@ async fn scoped_worker_observation_ws(
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn scoped_worker_protocol_ws(
|
||||
ws: WebSocketUpgrade,
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
|
||||
) -> Response {
|
||||
if let Err(err) = validate_workspace_scope(&api, &path.workspace_id) {
|
||||
return err.into_response();
|
||||
}
|
||||
worker_protocol_ws(State(api), AxumPath((path.runtime_id, path.worker_id)), ws)
|
||||
.await
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn scoped_list_host_workers(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedHostPath>,
|
||||
@@ -3388,6 +3409,89 @@ async fn cancel_runtime_worker(
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
async fn worker_protocol_ws(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> impl IntoResponse {
|
||||
match api.runtime.worker(&runtime_id, &worker_id) {
|
||||
Ok(_) => ws.on_upgrade(move |socket| {
|
||||
worker_protocol_ws_session(api.runtime, runtime_id, worker_id, socket)
|
||||
}),
|
||||
Err(error) => ApiError::from(error.into_error()).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn worker_protocol_ws_session(
|
||||
runtime: Arc<RuntimeRegistry>,
|
||||
runtime_id: String,
|
||||
worker_id: String,
|
||||
mut socket: WebSocket,
|
||||
) {
|
||||
while let Some(frame) = socket.next().await {
|
||||
match frame {
|
||||
Ok(WsMessage::Text(text)) => match serde_json::from_str::<protocol::Method>(&text) {
|
||||
Ok(method) => match runtime.send_protocol_method(&runtime_id, &worker_id, method) {
|
||||
Ok(events) => {
|
||||
for event in events {
|
||||
if !send_protocol_event(&mut socket, &event).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let event = protocol_error_event(error.message());
|
||||
if !send_protocol_event(&mut socket, &event).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(error) => {
|
||||
let event =
|
||||
protocol_error_event(format!("malformed protocol method frame: {error}"));
|
||||
if !send_protocol_event(&mut socket, &event).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(WsMessage::Close(_)) => return,
|
||||
Ok(WsMessage::Ping(payload)) => {
|
||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Ok(WsMessage::Pong(_)) | Ok(WsMessage::Binary(_)) => {}
|
||||
Err(error) => {
|
||||
let event = protocol_error_event(format!("protocol WebSocket error: {error}"));
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_protocol_event(socket: &mut WebSocket, event: &protocol::Event) -> bool {
|
||||
match serde_json::to_string(event) {
|
||||
Ok(text) => socket.send(WsMessage::Text(text.into())).await.is_ok(),
|
||||
Err(error) => {
|
||||
let fallback = protocol_error_event(format!(
|
||||
"failed to serialize protocol response event: {error}"
|
||||
));
|
||||
let Ok(text) = serde_json::to_string(&fallback) else {
|
||||
return false;
|
||||
};
|
||||
socket.send(WsMessage::Text(text.into())).await.is_ok()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
||||
protocol::Event::Error {
|
||||
code: protocol::ErrorCode::Internal,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn worker_observation_ws(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
||||
|
||||
Reference in New Issue
Block a user