fix: unify worker protocol websocket
This commit is contained in:
parent
70a26a3042
commit
8841d063be
|
|
@ -2,7 +2,7 @@
|
||||||
title: 'Backend runtime経由の操作をprotocol transportへ統一しTUI同等にする'
|
title: 'Backend runtime経由の操作をprotocol transportへ統一しTUI同等にする'
|
||||||
state: 'planning'
|
state: 'planning'
|
||||||
created_at: '2026-07-21T09:20:07Z'
|
created_at: '2026-07-21T09:20:07Z'
|
||||||
updated_at: '2026-07-21T10:33:12Z'
|
updated_at: '2026-07-21T11:29:08Z'
|
||||||
assignee: null
|
assignee: null
|
||||||
readiness: 'draft'
|
readiness: 'draft'
|
||||||
---
|
---
|
||||||
|
|
|
||||||
|
|
@ -95,3 +95,43 @@ Frontend check note:
|
||||||
- The Svelte page was updated against the generated protocol types in `web/workspace/src/lib/generated/protocol.ts`, but a frontend-native check still needs to be run in an environment with the frontend deps installed.
|
- The Svelte page was updated against the generated protocol types in `web/workspace/src/lib/generated/protocol.ts`, but a frontend-native check still needs to be run in an environment with the frontend deps installed.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
<!-- event: implementation_report author: assistant at: 2026-07-21T11:29:08Z -->
|
||||||
|
|
||||||
|
## Implementation report
|
||||||
|
|
||||||
|
Follow-up implementation after design clarification:
|
||||||
|
|
||||||
|
Changed the transport split so `/events/ws` no longer exists. `/protocol/ws` is now the single Worker protocol WebSocket for both directions:
|
||||||
|
|
||||||
|
- client -> runtime/backend: raw `protocol::Method` JSON frames
|
||||||
|
- runtime/backend -> client: raw `protocol::Event` JSON frames
|
||||||
|
|
||||||
|
Runtime changes:
|
||||||
|
- Removed `/v1/workers/{worker_id}/events/ws` route.
|
||||||
|
- Removed runtime-specific event WS envelope/diagnostic frame types.
|
||||||
|
- Extended `/v1/workers/{worker_id}/protocol/ws` to stream snapshot/backlog/live observation events as raw `protocol::Event` while also accepting method frames on the same socket.
|
||||||
|
|
||||||
|
Workspace Backend changes:
|
||||||
|
- Removed public `/api/.../events/ws` and `/api/w/.../events/ws` routes.
|
||||||
|
- Removed Backend client event WS envelope types.
|
||||||
|
- Workspace `/protocol/ws` now proxies/streams raw protocol events and forwards raw protocol methods on the same socket.
|
||||||
|
- Runtime observation source endpoint generation now targets runtime `/protocol/ws` instead of `/events/ws`.
|
||||||
|
|
||||||
|
Client/frontend changes:
|
||||||
|
- TUI `BackendRuntimeClient` no longer opens a separate observation socket. It uses only `/protocol/ws` for send and receive.
|
||||||
|
- Browser Console no longer opens both observation and command sockets. It uses one protocol socket for replay/live events, completions, rewind targets, and controls.
|
||||||
|
- Removed old frontend `ClientWorkerEventWs*` types.
|
||||||
|
- Updated frontend UI tests to assert protocol-only Console behavior and absence of operation HTTP paths.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
- `nix develop -c cargo fmt -- --check`
|
||||||
|
- `nix develop -c cargo check -p worker-runtime -p yoi-workspace-server -p client -p tui`
|
||||||
|
- `nix develop -c cargo test -p client backend_runtime --lib`
|
||||||
|
- `nix develop -c cargo test -p worker-runtime --features ws-server --lib protocol_ws`
|
||||||
|
- `nix develop -c cargo test -p yoi-workspace-server protocol_ws --lib`
|
||||||
|
- `nix develop -c cargo test -p yoi-workspace-server proxy_maps_runtime_worker_not_found_http_404_to_protocol_error_event --lib`
|
||||||
|
- `deno test --allow-read web/workspace/src/lib/workspace/console/worker-console.ui.test.ts` from `web/workspace`
|
||||||
|
- `git diff --check`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,12 @@
|
||||||
use std::collections::VecDeque;
|
|
||||||
use std::fmt;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use futures::{SinkExt, StreamExt};
|
use futures::{SinkExt, StreamExt};
|
||||||
use protocol::{ErrorCode, Event, Method};
|
use protocol::{ErrorCode, Event, Method};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::fmt;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio_tungstenite::connect_async;
|
use tokio_tungstenite::connect_async;
|
||||||
use tokio_tungstenite::tungstenite::Message as TungsteniteMessage;
|
use tokio_tungstenite::tungstenite::Message as TungsteniteMessage;
|
||||||
|
|
||||||
const RECONNECT_DELAY: Duration = Duration::from_millis(500);
|
|
||||||
const MAX_RECONNECT_ATTEMPTS: usize = 3;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct BackendRuntimeTarget {
|
pub struct BackendRuntimeTarget {
|
||||||
/// Workspace Backend API root URL, for example `http://127.0.0.1:8787`.
|
/// Workspace Backend API root URL, for example `http://127.0.0.1:8787`.
|
||||||
|
|
@ -163,8 +158,7 @@ pub struct BackendRuntimeClient {
|
||||||
command_tx: mpsc::UnboundedSender<Method>,
|
command_tx: mpsc::UnboundedSender<Method>,
|
||||||
events: mpsc::UnboundedReceiver<Event>,
|
events: mpsc::UnboundedReceiver<Event>,
|
||||||
diagnostics: VecDeque<Event>,
|
diagnostics: VecDeque<Event>,
|
||||||
_observation_task: tokio::task::JoinHandle<()>,
|
_protocol_task: tokio::task::JoinHandle<()>,
|
||||||
_command_task: tokio::task::JoinHandle<()>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
|
@ -262,16 +256,10 @@ impl BackendRuntimeClient {
|
||||||
let (event_tx, rx) = mpsc::unbounded_channel();
|
let (event_tx, rx) = mpsc::unbounded_channel();
|
||||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||||
|
|
||||||
let observation_target = target.clone();
|
let protocol_target = target.clone();
|
||||||
let observation_tx = event_tx.clone();
|
let protocol_event_tx = event_tx.clone();
|
||||||
let observation_task = tokio::spawn(async move {
|
let protocol_task = tokio::spawn(async move {
|
||||||
observe_worker_events(observation_target, observation_tx).await;
|
run_worker_protocol_transport(protocol_target, command_rx, protocol_event_tx).await;
|
||||||
});
|
|
||||||
|
|
||||||
let command_target = target.clone();
|
|
||||||
let command_event_tx = event_tx.clone();
|
|
||||||
let command_task = tokio::spawn(async move {
|
|
||||||
run_worker_protocol_commands(command_target, command_rx, command_event_tx).await;
|
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
|
@ -279,8 +267,7 @@ impl BackendRuntimeClient {
|
||||||
command_tx,
|
command_tx,
|
||||||
events: rx,
|
events: rx,
|
||||||
diagnostics: VecDeque::new(),
|
diagnostics: VecDeque::new(),
|
||||||
_observation_task: observation_task,
|
_protocol_task: protocol_task,
|
||||||
_command_task: command_task,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -311,12 +298,11 @@ impl BackendRuntimeClient {
|
||||||
|
|
||||||
impl Drop for BackendRuntimeClient {
|
impl Drop for BackendRuntimeClient {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
self._observation_task.abort();
|
self._protocol_task.abort();
|
||||||
self._command_task.abort();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_worker_protocol_commands(
|
async fn run_worker_protocol_transport(
|
||||||
target: BackendRuntimeTarget,
|
target: BackendRuntimeTarget,
|
||||||
mut commands: mpsc::UnboundedReceiver<Method>,
|
mut commands: mpsc::UnboundedReceiver<Method>,
|
||||||
tx: mpsc::UnboundedSender<Event>,
|
tx: mpsc::UnboundedSender<Event>,
|
||||||
|
|
@ -402,81 +388,6 @@ async fn run_worker_protocol_commands(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn observe_worker_events(target: BackendRuntimeTarget, tx: mpsc::UnboundedSender<Event>) {
|
|
||||||
let mut attempts = 0_usize;
|
|
||||||
|
|
||||||
loop {
|
|
||||||
let url = observation_ws_url(&target);
|
|
||||||
match connect_async(&url).await {
|
|
||||||
Ok((mut ws, _)) => {
|
|
||||||
attempts = 0;
|
|
||||||
while let Some(frame) = ws.next().await {
|
|
||||||
match frame {
|
|
||||||
Ok(TungsteniteMessage::Text(text)) => {
|
|
||||||
match serde_json::from_str::<ClientWorkerEventWsFrame>(&text) {
|
|
||||||
Ok(ClientWorkerEventWsFrame::Event { envelope }) => {
|
|
||||||
if envelope.runtime_id != target.runtime_id
|
|
||||||
|| envelope.worker_id != target.worker_id
|
|
||||||
{
|
|
||||||
let _ = tx.send(diagnostic_event(format!(
|
|
||||||
"Backend observation frame target mismatch: got {}:{}, expected {}",
|
|
||||||
envelope.runtime_id,
|
|
||||||
envelope.worker_id,
|
|
||||||
target.display_label()
|
|
||||||
)));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let _ = tx.send(envelope.payload);
|
|
||||||
}
|
|
||||||
Ok(ClientWorkerEventWsFrame::Diagnostic { diagnostic }) => {
|
|
||||||
let message = format!(
|
|
||||||
"Backend observation diagnostic [{}]: {}",
|
|
||||||
diagnostic.code, diagnostic.message
|
|
||||||
);
|
|
||||||
let _ = tx.send(diagnostic_event(message));
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
let _ = tx.send(diagnostic_event(format!(
|
|
||||||
"Backend observation frame was not valid JSON: {error}"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(TungsteniteMessage::Close(_)) => break,
|
|
||||||
Ok(TungsteniteMessage::Ping(_))
|
|
||||||
| Ok(TungsteniteMessage::Pong(_))
|
|
||||||
| Ok(TungsteniteMessage::Binary(_))
|
|
||||||
| Ok(TungsteniteMessage::Frame(_)) => {}
|
|
||||||
Err(error) => {
|
|
||||||
let _ = tx.send(diagnostic_event(format!(
|
|
||||||
"Backend observation WebSocket error for {}: {error}",
|
|
||||||
target.display_label()
|
|
||||||
)));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
let _ = tx.send(diagnostic_event(format!(
|
|
||||||
"Backend observation WebSocket connect failed for {}: {error}",
|
|
||||||
target.display_label()
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
attempts += 1;
|
|
||||||
if attempts > MAX_RECONNECT_ATTEMPTS {
|
|
||||||
let _ = tx.send(diagnostic_event(format!(
|
|
||||||
"Backend observation stream for {} stopped after {MAX_RECONNECT_ATTEMPTS} reconnect attempts",
|
|
||||||
target.display_label()
|
|
||||||
)));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
tokio::time::sleep(RECONNECT_DELAY).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn diagnostic_event(message: impl Into<String>) -> Event {
|
fn diagnostic_event(message: impl Into<String>) -> Event {
|
||||||
Event::Error {
|
Event::Error {
|
||||||
code: ErrorCode::Internal,
|
code: ErrorCode::Internal,
|
||||||
|
|
@ -552,15 +463,6 @@ fn backend_runtime_workers_path(workspace_id: Option<&str>, runtime_id: &str) ->
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn observation_ws_url(target: &BackendRuntimeTarget) -> String {
|
|
||||||
let path = format!(
|
|
||||||
"/api/runtimes/{}/workers/{}/events/ws",
|
|
||||||
path_segment_encode(&target.runtime_id),
|
|
||||||
path_segment_encode(&target.worker_id)
|
|
||||||
);
|
|
||||||
join_base_and_path(&http_base_to_ws(&target.base_url), &path)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn protocol_ws_url(target: &BackendRuntimeTarget) -> String {
|
fn protocol_ws_url(target: &BackendRuntimeTarget) -> String {
|
||||||
let path = format!(
|
let path = format!(
|
||||||
"/api/runtimes/{}/workers/{}/protocol/ws",
|
"/api/runtimes/{}/workers/{}/protocol/ws",
|
||||||
|
|
@ -611,42 +513,14 @@ pub struct BackendDiagnostic {
|
||||||
pub message: String,
|
pub message: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
|
||||||
enum ClientWorkerEventWsFrame {
|
|
||||||
Event {
|
|
||||||
envelope: ClientWorkerEventWsEnvelope,
|
|
||||||
},
|
|
||||||
Diagnostic {
|
|
||||||
diagnostic: ClientWorkerEventWsDiagnostic,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
struct ClientWorkerEventWsEnvelope {
|
|
||||||
runtime_id: String,
|
|
||||||
worker_id: String,
|
|
||||||
payload: Event,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
struct ClientWorkerEventWsDiagnostic {
|
|
||||||
code: String,
|
|
||||||
message: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn command_and_observation_urls_use_backend_protocol_paths() {
|
fn protocol_url_uses_backend_runtime_worker_identity() {
|
||||||
let target =
|
let target =
|
||||||
BackendRuntimeTarget::new("http://127.0.0.1:8787/", "runtime/one", "worker one");
|
BackendRuntimeTarget::new("http://127.0.0.1:8787/", "runtime/one", "worker one");
|
||||||
assert_eq!(
|
|
||||||
observation_ws_url(&target),
|
|
||||||
"ws://127.0.0.1:8787/api/runtimes/runtime%2Fone/workers/worker%20one/events/ws"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
protocol_ws_url(&target),
|
protocol_ws_url(&target),
|
||||||
"ws://127.0.0.1:8787/api/runtimes/runtime%2Fone/workers/worker%20one/protocol/ws"
|
"ws://127.0.0.1:8787/api/runtimes/runtime%2Fone/workers/worker%20one/protocol/ws"
|
||||||
|
|
|
||||||
|
|
@ -154,12 +154,10 @@ pub fn runtime_http_router(runtime: Runtime, local_token: Option<String>) -> Rou
|
||||||
.route("/v1/workers/{worker_id}/cancel", post(cancel_worker));
|
.route("/v1/workers/{worker_id}/cancel", post(cancel_worker));
|
||||||
|
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
let router = router
|
let router = router.route(
|
||||||
.route("/v1/workers/{worker_id}/events/ws", get(worker_events_ws))
|
"/v1/workers/{worker_id}/protocol/ws",
|
||||||
.route(
|
get(worker_protocol_ws),
|
||||||
"/v1/workers/{worker_id}/protocol/ws",
|
);
|
||||||
get(worker_protocol_ws),
|
|
||||||
);
|
|
||||||
|
|
||||||
router
|
router
|
||||||
.with_state(state.clone())
|
.with_state(state.clone())
|
||||||
|
|
@ -277,88 +275,12 @@ pub struct RuntimeHttpErrorDetail {
|
||||||
pub message: String,
|
pub message: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Runtime-owned WebSocket frame for worker-scoped observation.
|
|
||||||
#[cfg(feature = "ws-server")]
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
||||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
|
||||||
pub enum RuntimeWorkerEventWsFrame {
|
|
||||||
Event {
|
|
||||||
envelope: RuntimeWorkerEventWsEnvelope,
|
|
||||||
},
|
|
||||||
Diagnostic {
|
|
||||||
diagnostic: RuntimeWorkerEventWsDiagnostic,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Runtime-local protocol event envelope.
|
|
||||||
#[cfg(feature = "ws-server")]
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct RuntimeWorkerEventWsEnvelope {
|
|
||||||
pub cursor: String,
|
|
||||||
pub event_id: String,
|
|
||||||
pub worker_id: WorkerId,
|
|
||||||
pub payload: protocol::Event,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Runtime-local observation diagnostic.
|
|
||||||
#[cfg(feature = "ws-server")]
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct RuntimeWorkerEventWsDiagnostic {
|
|
||||||
pub code: String,
|
|
||||||
pub message: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
#[derive(Clone, Debug, Default, Deserialize)]
|
#[derive(Clone, Debug, Default, Deserialize)]
|
||||||
struct RuntimeWorkerEventsWsQuery {
|
struct RuntimeWorkerEventsWsQuery {
|
||||||
cursor: Option<String>,
|
cursor: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "ws-server")]
|
|
||||||
impl RuntimeWorkerEventWsFrame {
|
|
||||||
fn event(
|
|
||||||
cursor: String,
|
|
||||||
event_id: String,
|
|
||||||
worker_id: WorkerId,
|
|
||||||
payload: protocol::Event,
|
|
||||||
) -> Self {
|
|
||||||
Self::Event {
|
|
||||||
envelope: RuntimeWorkerEventWsEnvelope {
|
|
||||||
cursor,
|
|
||||||
event_id,
|
|
||||||
worker_id,
|
|
||||||
payload,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn diagnostic(code: impl Into<String>, message: impl Into<String>) -> Self {
|
|
||||||
Self::Diagnostic {
|
|
||||||
diagnostic: RuntimeWorkerEventWsDiagnostic {
|
|
||||||
code: code.into(),
|
|
||||||
message: message.into(),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "ws-server")]
|
|
||||||
async fn send_ws_frame(socket: &mut WebSocket, frame: &RuntimeWorkerEventWsFrame) -> bool {
|
|
||||||
match serde_json::to_string(frame) {
|
|
||||||
Ok(text) => socket.send(WsMessage::Text(text.into())).await.is_ok(),
|
|
||||||
Err(error) => {
|
|
||||||
let fallback = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.serialize_failed",
|
|
||||||
format!("failed to serialize observation frame: {error}"),
|
|
||||||
);
|
|
||||||
let Ok(text) = serde_json::to_string(&fallback) else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
socket.send(WsMessage::Text(text.into())).await.is_ok()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type RestResult<T> = Result<Json<T>, RuntimeHttpRestError>;
|
type RestResult<T> = Result<Json<T>, RuntimeHttpRestError>;
|
||||||
|
|
||||||
async fn get_runtime(
|
async fn get_runtime(
|
||||||
|
|
@ -515,6 +437,7 @@ async fn create_worker(
|
||||||
async fn worker_protocol_ws(
|
async fn worker_protocol_ws(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
Path(worker_id): Path<String>,
|
Path(worker_id): Path<String>,
|
||||||
|
Query(query): Query<RuntimeWorkerEventsWsQuery>,
|
||||||
ws: WebSocketUpgrade,
|
ws: WebSocketUpgrade,
|
||||||
) -> Result<Response, RuntimeHttpRestError> {
|
) -> Result<Response, RuntimeHttpRestError> {
|
||||||
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
|
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
|
||||||
|
|
@ -523,7 +446,9 @@ async fn worker_protocol_ws(
|
||||||
.worker_detail(&worker_ref)
|
.worker_detail(&worker_ref)
|
||||||
.map_err(RuntimeHttpRestError::runtime)?;
|
.map_err(RuntimeHttpRestError::runtime)?;
|
||||||
Ok(ws
|
Ok(ws
|
||||||
.on_upgrade(move |socket| worker_protocol_ws_session(state.runtime, worker_ref, socket))
|
.on_upgrade(move |socket| {
|
||||||
|
worker_protocol_ws_session(state.runtime, worker_ref, query, socket)
|
||||||
|
})
|
||||||
.into_response())
|
.into_response())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -531,45 +456,130 @@ async fn worker_protocol_ws(
|
||||||
async fn worker_protocol_ws_session(
|
async fn worker_protocol_ws_session(
|
||||||
runtime: Runtime,
|
runtime: Runtime,
|
||||||
worker_ref: WorkerRef,
|
worker_ref: WorkerRef,
|
||||||
|
query: RuntimeWorkerEventsWsQuery,
|
||||||
mut socket: WebSocket,
|
mut socket: WebSocket,
|
||||||
) {
|
) {
|
||||||
while let Some(frame) = socket.next().await {
|
let mut cursor = match query.cursor.as_deref() {
|
||||||
match frame {
|
Some(raw) => match WorkerObservationCursor::decode(raw) {
|
||||||
Ok(WsMessage::Text(text)) => match serde_json::from_str::<protocol::Method>(&text) {
|
Some(cursor) => cursor,
|
||||||
Ok(method) => match runtime.send_protocol_method(&worker_ref, method) {
|
None => {
|
||||||
Ok(events) => {
|
let event =
|
||||||
for event in events {
|
protocol_error_event(format!("malformed worker observation cursor: {raw}"));
|
||||||
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
None => match runtime.worker_observation_cursor_now(&worker_ref) {
|
||||||
|
Ok(cursor) => cursor,
|
||||||
|
Err(error) => {
|
||||||
|
let event = protocol_error_event(error.to_string());
|
||||||
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut receiver = match runtime.subscribe_worker_observation() {
|
||||||
|
Ok(receiver) => receiver,
|
||||||
|
Err(error) => {
|
||||||
|
let event =
|
||||||
|
protocol_error_event(format!("runtime observation bus unavailable: {error}"));
|
||||||
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let snapshot = match runtime.worker_observation_snapshot(&worker_ref) {
|
||||||
|
Ok(snapshot) => snapshot,
|
||||||
|
Err(error) => {
|
||||||
|
let event = protocol_error_event(error.to_string());
|
||||||
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if !send_protocol_event(&mut socket, &snapshot).await {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
match runtime.read_worker_observation_events(&worker_ref, cursor) {
|
||||||
|
Ok(backlog) => {
|
||||||
|
for event in backlog {
|
||||||
|
cursor = WorkerObservationCursor::new(event.sequence);
|
||||||
|
if !send_protocol_event(&mut socket, &event.payload).await {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
let event = protocol_error_event(error.to_string());
|
||||||
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
inbound = socket.next() => {
|
||||||
|
match inbound {
|
||||||
|
Some(Ok(WsMessage::Text(text))) => match serde_json::from_str::<protocol::Method>(&text) {
|
||||||
|
Ok(method) => match runtime.send_protocol_method(&worker_ref, method) {
|
||||||
|
Ok(events) => {
|
||||||
|
for event in events {
|
||||||
|
if !send_protocol_event(&mut socket, &event).await {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
let event = protocol_error_event(error.to_string());
|
||||||
|
if !send_protocol_event(&mut socket, &event).await {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(error) => {
|
||||||
|
let event = protocol_error_event(format!(
|
||||||
|
"malformed protocol method frame: {error}"
|
||||||
|
));
|
||||||
if !send_protocol_event(&mut socket, &event).await {
|
if !send_protocol_event(&mut socket, &event).await {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
Err(error) => {
|
Some(Ok(WsMessage::Close(_))) | None => return,
|
||||||
let event = protocol_error_event(error.to_string());
|
Some(Ok(WsMessage::Ping(payload))) => {
|
||||||
if !send_protocol_event(&mut socket, &event).await {
|
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
Some(Ok(WsMessage::Pong(_))) | Some(Ok(WsMessage::Binary(_))) => {}
|
||||||
Err(error) => {
|
Some(Err(error)) => {
|
||||||
let event =
|
let event = protocol_error_event(format!("protocol WebSocket error: {error}"));
|
||||||
protocol_error_event(format!("malformed protocol method frame: {error}"));
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
if !send_protocol_event(&mut socket, &event).await {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
Ok(WsMessage::Close(_)) => return,
|
|
||||||
Ok(WsMessage::Ping(payload)) => {
|
|
||||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Ok(WsMessage::Pong(_)) | Ok(WsMessage::Binary(_)) => {}
|
event = receiver.recv() => {
|
||||||
Err(error) => {
|
match event {
|
||||||
let event = protocol_error_event(format!("protocol WebSocket error: {error}"));
|
Ok(event) if event.worker_ref == worker_ref && event.sequence > cursor.sequence => {
|
||||||
let _ = send_protocol_event(&mut socket, &event).await;
|
cursor = WorkerObservationCursor::new(event.sequence);
|
||||||
return;
|
if !send_protocol_event(&mut socket, &event.payload).await {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
|
||||||
|
let event = protocol_error_event("runtime observation backlog was overrun");
|
||||||
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||||
|
let event = protocol_error_event("runtime observation bus closed");
|
||||||
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -599,182 +609,6 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "ws-server")]
|
|
||||||
async fn worker_events_ws(
|
|
||||||
State(state): State<RuntimeHttpState>,
|
|
||||||
Path(worker_id): Path<String>,
|
|
||||||
Query(query): Query<RuntimeWorkerEventsWsQuery>,
|
|
||||||
ws: WebSocketUpgrade,
|
|
||||||
) -> Result<Response, RuntimeHttpRestError> {
|
|
||||||
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
|
|
||||||
state
|
|
||||||
.runtime
|
|
||||||
.worker_detail(&worker_ref)
|
|
||||||
.map_err(RuntimeHttpRestError::runtime)?;
|
|
||||||
Ok(ws
|
|
||||||
.on_upgrade(move |socket| {
|
|
||||||
worker_events_ws_session(state.runtime, worker_ref, query, socket)
|
|
||||||
})
|
|
||||||
.into_response())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "ws-server")]
|
|
||||||
async fn worker_events_ws_session(
|
|
||||||
runtime: Runtime,
|
|
||||||
worker_ref: WorkerRef,
|
|
||||||
query: RuntimeWorkerEventsWsQuery,
|
|
||||||
mut socket: WebSocket,
|
|
||||||
) {
|
|
||||||
let mut cursor = match query.cursor.as_deref() {
|
|
||||||
Some(raw) => match WorkerObservationCursor::decode(raw) {
|
|
||||||
Some(cursor) => cursor,
|
|
||||||
None => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.cursor_malformed",
|
|
||||||
format!("malformed worker observation cursor: {raw}"),
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
None => match runtime.worker_observation_cursor_now(&worker_ref) {
|
|
||||||
Ok(cursor) => cursor,
|
|
||||||
Err(error) => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.worker_not_found",
|
|
||||||
error.to_string(),
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut receiver = match runtime.subscribe_worker_observation() {
|
|
||||||
Ok(receiver) => receiver,
|
|
||||||
Err(error) => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.unavailable",
|
|
||||||
format!("runtime observation bus unavailable: {error}"),
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let snapshot = match runtime.worker_observation_snapshot(&worker_ref) {
|
|
||||||
Ok(snapshot) => snapshot,
|
|
||||||
Err(error) => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.worker_not_found",
|
|
||||||
error.to_string(),
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let snapshot_cursor = cursor.encode();
|
|
||||||
let snapshot_frame = RuntimeWorkerEventWsFrame::event(
|
|
||||||
snapshot_cursor.clone(),
|
|
||||||
format!("snapshot:{snapshot_cursor}"),
|
|
||||||
worker_ref.worker_id.clone(),
|
|
||||||
snapshot,
|
|
||||||
);
|
|
||||||
if !send_ws_frame(&mut socket, &snapshot_frame).await {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
match runtime.read_worker_observation_events(&worker_ref, cursor) {
|
|
||||||
Ok(backlog) => {
|
|
||||||
for event in backlog {
|
|
||||||
cursor = WorkerObservationCursor::new(event.sequence);
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::event(
|
|
||||||
event.cursor,
|
|
||||||
event.event_id,
|
|
||||||
event.worker_ref.worker_id,
|
|
||||||
event.payload,
|
|
||||||
);
|
|
||||||
if !send_ws_frame(&mut socket, &frame).await {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.cursor_unknown_or_expired",
|
|
||||||
error.to_string(),
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
loop {
|
|
||||||
tokio::select! {
|
|
||||||
inbound = socket.next() => {
|
|
||||||
match inbound {
|
|
||||||
Some(Ok(WsMessage::Close(_))) | None => return,
|
|
||||||
Some(Ok(WsMessage::Ping(payload))) => {
|
|
||||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(Ok(WsMessage::Pong(_))) => {}
|
|
||||||
Some(Ok(_)) => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.observation_only",
|
|
||||||
"runtime worker event WebSocket is observation-only",
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Some(Err(error)) => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.websocket_error",
|
|
||||||
format!("runtime WebSocket receive error: {error}"),
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
event = receiver.recv() => {
|
|
||||||
match event {
|
|
||||||
Ok(event) if event.worker_ref == worker_ref && event.sequence > cursor.sequence => {
|
|
||||||
cursor = WorkerObservationCursor::new(event.sequence);
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::event(
|
|
||||||
event.cursor,
|
|
||||||
event.event_id,
|
|
||||||
event.worker_ref.worker_id,
|
|
||||||
event.payload,
|
|
||||||
);
|
|
||||||
if !send_ws_frame(&mut socket, &frame).await {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(_) => {}
|
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.cursor_expired",
|
|
||||||
"runtime observation backlog was overrun",
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
|
||||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
|
||||||
"runtime.upstream_closed",
|
|
||||||
"runtime observation bus closed",
|
|
||||||
);
|
|
||||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn send_worker_input(
|
async fn send_worker_input(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
Path(worker_id): Path<String>,
|
Path(worker_id): Path<String>,
|
||||||
|
|
@ -1409,7 +1243,7 @@ mod ws_tests {
|
||||||
runtime,
|
runtime,
|
||||||
worker.worker_ref.clone(),
|
worker.worker_ref.clone(),
|
||||||
format!(
|
format!(
|
||||||
"ws://{addr}/v1/workers/{}/events/ws",
|
"ws://{addr}/v1/workers/{}/protocol/ws",
|
||||||
worker.worker_ref.worker_id
|
worker.worker_ref.worker_id
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
@ -1419,7 +1253,7 @@ mod ws_tests {
|
||||||
stream: &mut tokio_tungstenite::WebSocketStream<
|
stream: &mut tokio_tungstenite::WebSocketStream<
|
||||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||||
>,
|
>,
|
||||||
) -> RuntimeWorkerEventWsFrame {
|
) -> protocol::Event {
|
||||||
let message = stream.next().await.unwrap().unwrap();
|
let message = stream.next().await.unwrap().unwrap();
|
||||||
let Message::Text(text) = message else {
|
let Message::Text(text) = message else {
|
||||||
panic!("expected text frame");
|
panic!("expected text frame");
|
||||||
|
|
@ -1428,21 +1262,16 @@ mod ws_tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn runtime_ws_connect_sends_snapshot_and_live_worker_events() {
|
async fn protocol_ws_connect_sends_snapshot_and_live_worker_events() {
|
||||||
let (runtime, worker_ref, url) = spawn_runtime_server().await;
|
let (runtime, worker_ref, url) = spawn_runtime_server().await;
|
||||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||||
|
|
||||||
match next_frame(&mut stream).await {
|
assert!(matches!(
|
||||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
next_frame(&mut stream).await,
|
||||||
assert_eq!(envelope.worker_id, worker_ref.worker_id);
|
protocol::Event::Snapshot { .. }
|
||||||
assert!(matches!(envelope.payload, protocol::Event::Snapshot { .. }));
|
));
|
||||||
}
|
|
||||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
|
||||||
panic!("unexpected diagnostic: {diagnostic:?}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let stored = runtime
|
runtime
|
||||||
.observe_worker_event(
|
.observe_worker_event(
|
||||||
&worker_ref,
|
&worker_ref,
|
||||||
protocol::Event::TextDelta {
|
protocol::Event::TextDelta {
|
||||||
|
|
@ -1450,23 +1279,14 @@ mod ws_tests {
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
match next_frame(&mut stream).await {
|
assert!(matches!(
|
||||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
next_frame(&mut stream).await,
|
||||||
assert_eq!(envelope.worker_id, worker_ref.worker_id);
|
protocol::Event::TextDelta { .. }
|
||||||
assert_eq!(envelope.cursor, stored.cursor);
|
));
|
||||||
assert!(matches!(
|
|
||||||
envelope.payload,
|
|
||||||
protocol::Event::TextDelta { .. }
|
|
||||||
));
|
|
||||||
}
|
|
||||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
|
||||||
panic!("unexpected diagnostic: {diagnostic:?}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn runtime_ws_cursor_resume_is_duplicate_safe_and_filters_workers() {
|
async fn protocol_ws_cursor_resume_is_duplicate_safe_and_filters_workers() {
|
||||||
let (runtime, worker_ref, url) = spawn_runtime_server().await;
|
let (runtime, worker_ref, url) = spawn_runtime_server().await;
|
||||||
let other = runtime.create_worker(ws_create_request()).unwrap();
|
let other = runtime.create_worker(ws_create_request()).unwrap();
|
||||||
let first = runtime
|
let first = runtime
|
||||||
|
|
@ -1481,7 +1301,7 @@ mod ws_tests {
|
||||||
.observe_worker_event(
|
.observe_worker_event(
|
||||||
&other.worker_ref,
|
&other.worker_ref,
|
||||||
protocol::Event::TextDelta {
|
protocol::Event::TextDelta {
|
||||||
text: "started".into(),
|
text: "other".into(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
@ -1491,10 +1311,10 @@ mod ws_tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
next_frame(&mut stream).await,
|
next_frame(&mut stream).await,
|
||||||
RuntimeWorkerEventWsFrame::Event { envelope } if matches!(envelope.payload, protocol::Event::Snapshot { .. })
|
protocol::Event::Snapshot { .. }
|
||||||
));
|
));
|
||||||
|
|
||||||
let second = runtime
|
runtime
|
||||||
.observe_worker_event(
|
.observe_worker_event(
|
||||||
&worker_ref,
|
&worker_ref,
|
||||||
protocol::Event::TextDone {
|
protocol::Event::TextDone {
|
||||||
|
|
@ -1502,41 +1322,27 @@ mod ws_tests {
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
match next_frame(&mut stream).await {
|
assert!(matches!(
|
||||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
next_frame(&mut stream).await,
|
||||||
assert_eq!(envelope.cursor, second.cursor);
|
protocol::Event::TextDone { .. }
|
||||||
assert_ne!(envelope.cursor, first.cursor);
|
));
|
||||||
assert!(matches!(envelope.payload, protocol::Event::TextDone { .. }));
|
|
||||||
}
|
|
||||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
|
||||||
panic!("unexpected diagnostic: {diagnostic:?}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn runtime_ws_reports_malformed_cursor_and_observation_only_input() {
|
async fn protocol_ws_reports_malformed_cursor_and_method_frame() {
|
||||||
let (_runtime, _worker_ref, url) = spawn_runtime_server().await;
|
let (_runtime, _worker_ref, url) = spawn_runtime_server().await;
|
||||||
let (mut malformed, _) = connect_async(format!("{url}?cursor=bad")).await.unwrap();
|
let (mut malformed, _) = connect_async(format!("{url}?cursor=bad")).await.unwrap();
|
||||||
match next_frame(&mut malformed).await {
|
assert!(matches!(
|
||||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
next_frame(&mut malformed).await,
|
||||||
assert_eq!(diagnostic.code, "runtime.cursor_malformed");
|
protocol::Event::Error { .. }
|
||||||
}
|
));
|
||||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
|
||||||
panic!("unexpected event: {envelope:?}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||||
let _ = next_frame(&mut stream).await;
|
let _ = next_frame(&mut stream).await;
|
||||||
stream.send(Message::Text("{}".into())).await.unwrap();
|
stream.send(Message::Text("{}".into())).await.unwrap();
|
||||||
match next_frame(&mut stream).await {
|
assert!(matches!(
|
||||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
next_frame(&mut stream).await,
|
||||||
assert_eq!(diagnostic.code, "runtime.observation_only");
|
protocol::Event::Error { .. }
|
||||||
}
|
));
|
||||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
|
||||||
panic!("unexpected event: {envelope:?}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2128,7 +2128,7 @@ impl RemoteWorkerRuntime {
|
||||||
} else if let Some(rest) = base.strip_prefix("http://") {
|
} else if let Some(rest) = base.strip_prefix("http://") {
|
||||||
base = format!("ws://{rest}");
|
base = format!("ws://{rest}");
|
||||||
}
|
}
|
||||||
format!("{base}/v1/workers/{worker_id}/events/ws")
|
format!("{base}/v1/workers/{worker_id}/protocol/ws")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_json<T>(&self, path: &str) -> Result<T, RuntimeDiagnostic>
|
fn get_json<T>(&self, path: &str) -> Result<T, RuntimeDiagnostic>
|
||||||
|
|
@ -4285,7 +4285,7 @@ mod tests {
|
||||||
panic!("remote runtime should expose a remote WS observation source");
|
panic!("remote runtime should expose a remote WS observation source");
|
||||||
};
|
};
|
||||||
assert!(observation.endpoint.starts_with("ws://127.0.0.1:"));
|
assert!(observation.endpoint.starts_with("ws://127.0.0.1:"));
|
||||||
assert!(observation.endpoint.ends_with("/v1/workers/1/events/ws"));
|
assert!(observation.endpoint.ends_with("/v1/workers/1/protocol/ws"));
|
||||||
assert_eq!(observation.bearer_token.as_deref(), Some(secret.as_str()));
|
assert_eq!(observation.bearer_token.as_deref(), Some(secret.as_str()));
|
||||||
|
|
||||||
let workers = registry.list_workers(10);
|
let workers = registry.list_workers(10);
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,6 @@ use serde::{Deserialize, Serialize};
|
||||||
use tokio_tungstenite::connect_async;
|
use tokio_tungstenite::connect_async;
|
||||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||||
use tokio_tungstenite::tungstenite::{Error as TungsteniteError, Message as TungsteniteMessage};
|
use tokio_tungstenite::tungstenite::{Error as TungsteniteError, Message as TungsteniteMessage};
|
||||||
use worker_runtime::http_server::{RuntimeWorkerEventWsEnvelope, RuntimeWorkerEventWsFrame};
|
|
||||||
|
|
||||||
/// Backend-private source for a runtime worker observation stream.
|
/// Backend-private source for a runtime worker observation stream.
|
||||||
#[derive(Clone, PartialEq, Eq)]
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
|
|
@ -104,42 +103,12 @@ pub struct RuntimeObservationUpstreamEvent {
|
||||||
pub payload: protocol::Event,
|
pub payload: protocol::Event,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Backend-local frame exposed to browser/future-TUI clients.
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
||||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
|
||||||
pub enum ClientWorkerEventWsFrame {
|
|
||||||
Event {
|
|
||||||
envelope: ClientWorkerEventWsEnvelope,
|
|
||||||
},
|
|
||||||
Diagnostic {
|
|
||||||
diagnostic: ClientWorkerEventWsDiagnostic,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Backend-owned event envelope. It intentionally omits Runtime endpoints,
|
|
||||||
/// credentials, sockets and session paths.
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
||||||
pub struct ClientWorkerEventWsEnvelope {
|
|
||||||
pub event_id: String,
|
|
||||||
pub runtime_id: String,
|
|
||||||
pub worker_id: String,
|
|
||||||
pub payload: protocol::Event,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Client-facing typed observation diagnostic.
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ClientWorkerEventWsDiagnostic {
|
|
||||||
pub code: String,
|
|
||||||
pub message: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub enum ObservationProxyError {
|
pub enum ObservationProxyError {
|
||||||
RuntimeUnavailable(String),
|
RuntimeUnavailable(String),
|
||||||
WorkerNotFound(String),
|
WorkerNotFound(String),
|
||||||
UpstreamDisconnect(String),
|
UpstreamDisconnect(String),
|
||||||
MalformedFrame(String),
|
MalformedFrame(String),
|
||||||
ObservationOnly,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ObservationProxyError {
|
impl ObservationProxyError {
|
||||||
|
|
@ -149,7 +118,6 @@ impl ObservationProxyError {
|
||||||
ObservationProxyError::WorkerNotFound(_) => "backend.worker_not_found",
|
ObservationProxyError::WorkerNotFound(_) => "backend.worker_not_found",
|
||||||
ObservationProxyError::UpstreamDisconnect(_) => "backend.upstream_disconnect",
|
ObservationProxyError::UpstreamDisconnect(_) => "backend.upstream_disconnect",
|
||||||
ObservationProxyError::MalformedFrame(_) => "backend.malformed_frame",
|
ObservationProxyError::MalformedFrame(_) => "backend.malformed_frame",
|
||||||
ObservationProxyError::ObservationOnly => "backend.observation_only",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -159,24 +127,6 @@ impl ObservationProxyError {
|
||||||
| ObservationProxyError::WorkerNotFound(message)
|
| ObservationProxyError::WorkerNotFound(message)
|
||||||
| ObservationProxyError::UpstreamDisconnect(message)
|
| ObservationProxyError::UpstreamDisconnect(message)
|
||||||
| ObservationProxyError::MalformedFrame(message) => message,
|
| ObservationProxyError::MalformedFrame(message) => message,
|
||||||
ObservationProxyError::ObservationOnly => {
|
|
||||||
"backend worker event WebSocket is observation-only"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ClientWorkerEventWsFrame {
|
|
||||||
pub fn event(envelope: ClientWorkerEventWsEnvelope) -> Self {
|
|
||||||
Self::Event { envelope }
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn diagnostic(error: ObservationProxyError) -> Self {
|
|
||||||
Self::Diagnostic {
|
|
||||||
diagnostic: ClientWorkerEventWsDiagnostic {
|
|
||||||
code: error.code().to_string(),
|
|
||||||
message: error.message().to_string(),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -238,15 +188,6 @@ impl BackendObservationProxy {
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn map_event(&self, event: RuntimeObservationUpstreamEvent) -> ClientWorkerEventWsEnvelope {
|
|
||||||
ClientWorkerEventWsEnvelope {
|
|
||||||
event_id: event.runtime_event_id,
|
|
||||||
runtime_id: event.runtime_id,
|
|
||||||
worker_id: event.worker_id,
|
|
||||||
payload: event.payload,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_runtime_connect_error(error: TungsteniteError) -> ObservationProxyError {
|
fn map_runtime_connect_error(error: TungsteniteError) -> ObservationProxyError {
|
||||||
|
|
@ -266,24 +207,6 @@ fn map_runtime_connect_error(error: TungsteniteError) -> ObservationProxyError {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_runtime_diagnostic(code: String, message: String) -> ObservationProxyError {
|
|
||||||
match code.as_str() {
|
|
||||||
"runtime.worker_not_found" => ObservationProxyError::WorkerNotFound(message),
|
|
||||||
"runtime.cursor_malformed"
|
|
||||||
| "runtime.cursor_unknown_or_expired"
|
|
||||||
| "runtime.cursor_expired" => ObservationProxyError::RuntimeUnavailable(message),
|
|
||||||
"runtime.unavailable" => ObservationProxyError::RuntimeUnavailable(message),
|
|
||||||
"runtime.upstream_closed" | "runtime.websocket_error" => {
|
|
||||||
ObservationProxyError::UpstreamDisconnect(message)
|
|
||||||
}
|
|
||||||
"runtime.serialize_failed" => ObservationProxyError::MalformedFrame(message),
|
|
||||||
"runtime.observation_only" => ObservationProxyError::ObservationOnly,
|
|
||||||
_ => ObservationProxyError::RuntimeUnavailable(format!(
|
|
||||||
"runtime diagnostic {code}: {message}"
|
|
||||||
)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct RuntimeWsObservationClient {
|
pub struct RuntimeWsObservationClient {
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
worker_id: String,
|
worker_id: String,
|
||||||
|
|
@ -361,32 +284,17 @@ impl RuntimeWsObservationClient {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let frame: RuntimeWorkerEventWsFrame =
|
let payload: protocol::Event = serde_json::from_str(&text).map_err(|error| {
|
||||||
serde_json::from_str(&text).map_err(|error| {
|
ObservationProxyError::MalformedFrame(format!(
|
||||||
ObservationProxyError::MalformedFrame(format!(
|
"failed to decode runtime protocol event frame: {error}"
|
||||||
"failed to decode runtime observation frame: {error}"
|
))
|
||||||
))
|
})?;
|
||||||
})?;
|
return Ok(RuntimeObservationUpstreamEvent {
|
||||||
match frame {
|
runtime_id: self.runtime_id.clone(),
|
||||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
worker_id: self.worker_id.clone(),
|
||||||
return Ok(self.map_envelope(envelope));
|
runtime_event_id: "protocol".to_string(),
|
||||||
}
|
payload,
|
||||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
});
|
||||||
return Err(map_runtime_diagnostic(diagnostic.code, diagnostic.message));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn map_envelope(
|
|
||||||
&self,
|
|
||||||
envelope: RuntimeWorkerEventWsEnvelope,
|
|
||||||
) -> RuntimeObservationUpstreamEvent {
|
|
||||||
RuntimeObservationUpstreamEvent {
|
|
||||||
runtime_id: self.runtime_id.clone(),
|
|
||||||
worker_id: self.worker_id.clone(),
|
|
||||||
runtime_event_id: envelope.event_id,
|
|
||||||
payload: envelope.payload,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -548,7 +456,8 @@ mod tests {
|
||||||
RuntimeObservationSourceConfig {
|
RuntimeObservationSourceConfig {
|
||||||
runtime_id: "remote-runtime".to_string(),
|
runtime_id: "remote-runtime".to_string(),
|
||||||
worker_id: "worker-1".to_string(),
|
worker_id: "worker-1".to_string(),
|
||||||
endpoint: "wss://remote.example.invalid/private/workers/worker-1/events/ws".to_string(),
|
endpoint: "wss://remote.example.invalid/private/workers/worker-1/protocol/ws"
|
||||||
|
.to_string(),
|
||||||
bearer_token: Some("top-secret-bearer-token".to_string()),
|
bearer_token: Some("top-secret-bearer-token".to_string()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -563,7 +472,7 @@ mod tests {
|
||||||
assert!(debug.contains("<redacted>"));
|
assert!(debug.contains("<redacted>"));
|
||||||
for forbidden in [
|
for forbidden in [
|
||||||
"remote.example.invalid",
|
"remote.example.invalid",
|
||||||
"/private/workers/worker-1/events/ws",
|
"/private/workers/worker-1/protocol/ws",
|
||||||
"top-secret-bearer-token",
|
"top-secret-bearer-token",
|
||||||
] {
|
] {
|
||||||
assert!(
|
assert!(
|
||||||
|
|
@ -582,7 +491,7 @@ mod tests {
|
||||||
assert!(debug.contains("source_count"));
|
assert!(debug.contains("source_count"));
|
||||||
for forbidden in [
|
for forbidden in [
|
||||||
"remote.example.invalid",
|
"remote.example.invalid",
|
||||||
"/private/workers/worker-1/events/ws",
|
"/private/workers/worker-1/protocol/ws",
|
||||||
"top-secret-bearer-token",
|
"top-secret-bearer-token",
|
||||||
] {
|
] {
|
||||||
assert!(
|
assert!(
|
||||||
|
|
|
||||||
|
|
@ -41,8 +41,8 @@ use crate::hosts::{
|
||||||
};
|
};
|
||||||
use crate::identity::WorkspaceIdentity;
|
use crate::identity::WorkspaceIdentity;
|
||||||
use crate::observation::{
|
use crate::observation::{
|
||||||
BackendObservationProxy, ClientWorkerEventWsFrame, ObservationProxyError,
|
BackendObservationProxy, ObservationProxyError, RuntimeObservationClient,
|
||||||
RuntimeObservationClient, RuntimeObservationSourceConfig,
|
RuntimeObservationSourceConfig,
|
||||||
};
|
};
|
||||||
use crate::profile_settings::{
|
use crate::profile_settings::{
|
||||||
CreateWorkspaceProfileSourceRequest, DeleteWorkspaceProfileSourceRequest,
|
CreateWorkspaceProfileSourceRequest, DeleteWorkspaceProfileSourceRequest,
|
||||||
|
|
@ -547,14 +547,6 @@ pub fn build_router(api: WorkspaceApi) -> Router {
|
||||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/cancel",
|
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/cancel",
|
||||||
post(scoped_cancel_runtime_worker),
|
post(scoped_cancel_runtime_worker),
|
||||||
)
|
)
|
||||||
.route(
|
|
||||||
"/api/runtimes/{runtime_id}/workers/{worker_id}/events/ws",
|
|
||||||
get(worker_observation_ws),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/events/ws",
|
|
||||||
get(scoped_worker_observation_ws),
|
|
||||||
)
|
|
||||||
.route(
|
.route(
|
||||||
"/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws",
|
"/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws",
|
||||||
get(worker_protocol_ws),
|
get(worker_protocol_ws),
|
||||||
|
|
@ -2452,19 +2444,6 @@ async fn scoped_cancel_runtime_worker(
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn scoped_worker_observation_ws(
|
|
||||||
ws: WebSocketUpgrade,
|
|
||||||
State(api): State<WorkspaceApi>,
|
|
||||||
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
|
|
||||||
) -> Response {
|
|
||||||
if let Err(err) = validate_workspace_scope(&api, &path.workspace_id) {
|
|
||||||
return err.into_response();
|
|
||||||
}
|
|
||||||
worker_observation_ws(State(api), AxumPath((path.runtime_id, path.worker_id)), ws)
|
|
||||||
.await
|
|
||||||
.into_response()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn scoped_worker_protocol_ws(
|
async fn scoped_worker_protocol_ws(
|
||||||
ws: WebSocketUpgrade,
|
ws: WebSocketUpgrade,
|
||||||
State(api): State<WorkspaceApi>,
|
State(api): State<WorkspaceApi>,
|
||||||
|
|
@ -3414,57 +3393,101 @@ async fn worker_protocol_ws(
|
||||||
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
||||||
ws: WebSocketUpgrade,
|
ws: WebSocketUpgrade,
|
||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
match api.runtime.worker(&runtime_id, &worker_id) {
|
let source = match api.observation_proxy.source(&runtime_id, &worker_id) {
|
||||||
Ok(_) => ws.on_upgrade(move |socket| {
|
Ok(source) => source,
|
||||||
worker_protocol_ws_session(api.runtime, runtime_id, worker_id, socket)
|
Err(ObservationProxyError::WorkerNotFound(_)) => {
|
||||||
}),
|
match api.runtime.observation_source(&runtime_id, &worker_id) {
|
||||||
Err(error) => ApiError::from(error.into_error()).into_response(),
|
Ok(source) => source,
|
||||||
}
|
Err(error) => return ApiError::from(error.into_error()).into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({
|
||||||
|
"error": error.code(),
|
||||||
|
"message": error.message(),
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
ws.on_upgrade(move |socket| {
|
||||||
|
worker_protocol_ws_session(api.runtime, source, runtime_id, worker_id, socket)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn worker_protocol_ws_session(
|
async fn worker_protocol_ws_session(
|
||||||
runtime: Arc<RuntimeRegistry>,
|
runtime: Arc<RuntimeRegistry>,
|
||||||
|
source: crate::observation::RuntimeObservationSource,
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
worker_id: String,
|
worker_id: String,
|
||||||
mut socket: WebSocket,
|
mut socket: WebSocket,
|
||||||
) {
|
) {
|
||||||
while let Some(frame) = socket.next().await {
|
let mut upstream = match RuntimeObservationClient::connect(&source).await {
|
||||||
match frame {
|
Ok(client) => client,
|
||||||
Ok(WsMessage::Text(text)) => match serde_json::from_str::<protocol::Method>(&text) {
|
Err(error) => {
|
||||||
Ok(method) => match runtime.send_protocol_method(&runtime_id, &worker_id, method) {
|
let event = protocol_error_event(error.message());
|
||||||
Ok(events) => {
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
for event in events {
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
inbound = socket.next() => {
|
||||||
|
match inbound {
|
||||||
|
Some(Ok(WsMessage::Text(text))) => match serde_json::from_str::<protocol::Method>(&text) {
|
||||||
|
Ok(method) => match runtime.send_protocol_method(&runtime_id, &worker_id, method) {
|
||||||
|
Ok(events) => {
|
||||||
|
for event in events {
|
||||||
|
if !send_protocol_event(&mut socket, &event).await {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
let event = protocol_error_event(error.message());
|
||||||
|
if !send_protocol_event(&mut socket, &event).await {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(error) => {
|
||||||
|
let event =
|
||||||
|
protocol_error_event(format!("malformed protocol method frame: {error}"));
|
||||||
if !send_protocol_event(&mut socket, &event).await {
|
if !send_protocol_event(&mut socket, &event).await {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
Err(error) => {
|
Some(Ok(WsMessage::Close(_))) | None => return,
|
||||||
let event = protocol_error_event(error.message());
|
Some(Ok(WsMessage::Ping(payload))) => {
|
||||||
if !send_protocol_event(&mut socket, &event).await {
|
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
Some(Ok(WsMessage::Pong(_))) | Some(Ok(WsMessage::Binary(_))) => {}
|
||||||
Err(error) => {
|
Some(Err(error)) => {
|
||||||
let event =
|
let event = protocol_error_event(format!("protocol WebSocket error: {error}"));
|
||||||
protocol_error_event(format!("malformed protocol method frame: {error}"));
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
if !send_protocol_event(&mut socket, &event).await {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
Ok(WsMessage::Close(_)) => return,
|
|
||||||
Ok(WsMessage::Ping(payload)) => {
|
|
||||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Ok(WsMessage::Pong(_)) | Ok(WsMessage::Binary(_)) => {}
|
upstream_event = upstream.next_event() => {
|
||||||
Err(error) => {
|
match upstream_event {
|
||||||
let event = protocol_error_event(format!("protocol WebSocket error: {error}"));
|
Ok(event) => {
|
||||||
let _ = send_protocol_event(&mut socket, &event).await;
|
if !send_protocol_event(&mut socket, &event.payload).await {
|
||||||
return;
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
let event = protocol_error_event(error.message());
|
||||||
|
let _ = send_protocol_event(&mut socket, &event).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -3492,116 +3515,6 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn worker_observation_ws(
|
|
||||||
State(api): State<WorkspaceApi>,
|
|
||||||
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
|
||||||
ws: WebSocketUpgrade,
|
|
||||||
) -> impl IntoResponse {
|
|
||||||
match api.observation_proxy.source(&runtime_id, &worker_id) {
|
|
||||||
Ok(source) => ws.on_upgrade(move |socket| {
|
|
||||||
worker_observation_ws_session(api.observation_proxy, source, socket)
|
|
||||||
}),
|
|
||||||
Err(ObservationProxyError::WorkerNotFound(_)) => {
|
|
||||||
match api.runtime.observation_source(&runtime_id, &worker_id) {
|
|
||||||
Ok(source) => ws.on_upgrade(move |socket| {
|
|
||||||
worker_observation_ws_session(api.observation_proxy, source, socket)
|
|
||||||
}),
|
|
||||||
Err(error) => ApiError::from(error.into_error()).into_response(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
let status = StatusCode::BAD_REQUEST;
|
|
||||||
(
|
|
||||||
status,
|
|
||||||
Json(serde_json::json!({
|
|
||||||
"error": error.code(),
|
|
||||||
"message": error.message(),
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
.into_response()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn worker_observation_ws_session(
|
|
||||||
proxy: BackendObservationProxy,
|
|
||||||
source: crate::observation::RuntimeObservationSource,
|
|
||||||
mut socket: WebSocket,
|
|
||||||
) {
|
|
||||||
let mut upstream = match RuntimeObservationClient::connect(&source).await {
|
|
||||||
Ok(client) => client,
|
|
||||||
Err(error) => {
|
|
||||||
let _ = send_client_ws_frame(&mut socket, ClientWorkerEventWsFrame::diagnostic(error))
|
|
||||||
.await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
loop {
|
|
||||||
tokio::select! {
|
|
||||||
inbound = socket.next() => {
|
|
||||||
match inbound {
|
|
||||||
Some(Ok(WsMessage::Close(_))) | None => return,
|
|
||||||
Some(Ok(WsMessage::Ping(payload))) => {
|
|
||||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(Ok(WsMessage::Pong(_))) => {}
|
|
||||||
Some(Ok(_)) => {
|
|
||||||
let _ = send_client_ws_frame(
|
|
||||||
&mut socket,
|
|
||||||
ClientWorkerEventWsFrame::diagnostic(ObservationProxyError::ObservationOnly),
|
|
||||||
).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Some(Err(error)) => {
|
|
||||||
let _ = send_client_ws_frame(
|
|
||||||
&mut socket,
|
|
||||||
ClientWorkerEventWsFrame::diagnostic(
|
|
||||||
ObservationProxyError::MalformedFrame(format!(
|
|
||||||
"client WebSocket receive error: {error}"
|
|
||||||
)),
|
|
||||||
),
|
|
||||||
).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
upstream_event = upstream.next_event() => {
|
|
||||||
match upstream_event {
|
|
||||||
Ok(event) => {
|
|
||||||
let envelope = proxy.map_event(event);
|
|
||||||
if !send_client_ws_frame(&mut socket, ClientWorkerEventWsFrame::event(envelope)).await {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
Err(error) => {
|
|
||||||
let _ = send_client_ws_frame(&mut socket, ClientWorkerEventWsFrame::diagnostic(error)).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn send_client_ws_frame(socket: &mut WebSocket, frame: ClientWorkerEventWsFrame) -> bool {
|
|
||||||
match serde_json::to_string(&frame) {
|
|
||||||
Ok(text) => socket.send(WsMessage::Text(text.into())).await.is_ok(),
|
|
||||||
Err(error) => {
|
|
||||||
let fallback =
|
|
||||||
ClientWorkerEventWsFrame::diagnostic(ObservationProxyError::MalformedFrame(
|
|
||||||
format!("failed to serialize backend observation frame: {error}"),
|
|
||||||
));
|
|
||||||
let Ok(text) = serde_json::to_string(&fallback) else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
socket.send(WsMessage::Text(text.into())).await.is_ok()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn list_host_workers(
|
async fn list_host_workers(
|
||||||
State(api): State<WorkspaceApi>,
|
State(api): State<WorkspaceApi>,
|
||||||
AxumPath(host_id): AxumPath<String>,
|
AxumPath(host_id): AxumPath<String>,
|
||||||
|
|
@ -5436,7 +5349,7 @@ mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use axum::body::{Body, to_bytes};
|
use axum::body::{Body, to_bytes};
|
||||||
use axum::http::Request;
|
use axum::http::Request;
|
||||||
use futures::{SinkExt, StreamExt};
|
use futures::StreamExt;
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use std::{fs, sync::Arc};
|
use std::{fs, sync::Arc};
|
||||||
use tokio_tungstenite::connect_async;
|
use tokio_tungstenite::connect_async;
|
||||||
|
|
@ -5449,7 +5362,6 @@ mod tests {
|
||||||
TicketWorkerRole, WorkerInputKind, WorkerOperationState, WorkerSpawnAcceptanceRequirement,
|
TicketWorkerRole, WorkerInputKind, WorkerOperationState, WorkerSpawnAcceptanceRequirement,
|
||||||
WorkerSpawnIntent,
|
WorkerSpawnIntent,
|
||||||
};
|
};
|
||||||
use crate::observation::ClientWorkerEventWsDiagnostic;
|
|
||||||
use crate::store::SqliteWorkspaceStore;
|
use crate::store::SqliteWorkspaceStore;
|
||||||
|
|
||||||
const TEST_WORKSPACE_ID: &str = "0192f0e8-4d84-7d6e-a000-000000000001";
|
const TEST_WORKSPACE_ID: &str = "0192f0e8-4d84-7d6e-a000-000000000001";
|
||||||
|
|
@ -7861,53 +7773,20 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn proxies_worker_observation_ws_with_backend_cursors_and_diagnostics() {
|
async fn proxies_worker_protocol_ws_as_raw_events() {
|
||||||
let (runtime, worker_ref) = runtime_with_worker();
|
let (runtime, worker_ref, endpoint) = spawn_runtime_worker().await;
|
||||||
let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let source = RuntimeObservationSourceConfig {
|
||||||
let runtime_addr = runtime_listener.local_addr().unwrap();
|
runtime_id: "runtime-a".into(),
|
||||||
tokio::spawn({
|
worker_id: "worker-a".into(),
|
||||||
let runtime = runtime.clone();
|
endpoint,
|
||||||
async move {
|
bearer_token: None,
|
||||||
worker_runtime::http_server::serve_runtime_http(runtime, runtime_listener, None)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let dir = tempfile::tempdir().unwrap();
|
|
||||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
|
||||||
let mut config = test_server_config(dir.path());
|
|
||||||
config
|
|
||||||
.runtime_event_sources
|
|
||||||
.push(RuntimeObservationSourceConfig {
|
|
||||||
runtime_id: "runtime-a".into(),
|
|
||||||
worker_id: "worker-a".into(),
|
|
||||||
endpoint: format!(
|
|
||||||
"ws://{runtime_addr}/v1/workers/{}/events/ws",
|
|
||||||
worker_ref.worker_id
|
|
||||||
),
|
|
||||||
bearer_token: None,
|
|
||||||
});
|
|
||||||
let api = WorkspaceApi::new_with_execution_backend(
|
|
||||||
config,
|
|
||||||
Arc::new(store),
|
|
||||||
Arc::new(DeterministicExecutionBackend::default()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let app_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let app_addr = app_listener.local_addr().unwrap();
|
|
||||||
tokio::spawn(async move { axum::serve(app_listener, build_router(api)).await.unwrap() });
|
|
||||||
|
|
||||||
let url = format!("ws://{app_addr}/api/runtimes/runtime-a/workers/worker-a/events/ws");
|
|
||||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
|
||||||
let snapshot = next_client_frame(&mut stream).await;
|
|
||||||
let ClientWorkerEventWsFrame::Event { envelope: snapshot } = snapshot else {
|
|
||||||
panic!("expected snapshot event");
|
|
||||||
};
|
};
|
||||||
assert_eq!(snapshot.runtime_id, "runtime-a");
|
let (url, _dir) = spawn_workspace_proxy(source).await;
|
||||||
assert_eq!(snapshot.worker_id, "worker-a");
|
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||||
assert!(matches!(snapshot.payload, protocol::Event::Snapshot { .. }));
|
assert!(matches!(
|
||||||
|
next_client_frame(&mut stream).await,
|
||||||
|
protocol::Event::Snapshot { .. }
|
||||||
|
));
|
||||||
|
|
||||||
runtime
|
runtime
|
||||||
.observe_worker_event(
|
.observe_worker_event(
|
||||||
|
|
@ -7917,59 +7796,16 @@ mod tests {
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let live = next_client_frame(&mut stream).await;
|
|
||||||
let ClientWorkerEventWsFrame::Event { envelope: live } = live else {
|
|
||||||
panic!("expected live event");
|
|
||||||
};
|
|
||||||
assert_eq!(live.runtime_id, "runtime-a");
|
|
||||||
assert_eq!(live.worker_id, "worker-a");
|
|
||||||
assert!(matches!(live.payload, protocol::Event::TextDelta { .. }));
|
|
||||||
|
|
||||||
let (mut fresh, _) = connect_async(&url).await.unwrap();
|
|
||||||
let fresh_snapshot = next_client_frame(&mut fresh).await;
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
fresh_snapshot,
|
next_client_frame(&mut stream).await,
|
||||||
ClientWorkerEventWsFrame::Event { envelope } if matches!(envelope.payload, protocol::Event::Snapshot { .. })
|
protocol::Event::TextDelta { .. }
|
||||||
));
|
));
|
||||||
runtime
|
|
||||||
.observe_worker_event(
|
|
||||||
&worker_ref,
|
|
||||||
protocol::Event::TextDone {
|
|
||||||
text: "fresh".into(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let fresh_event = next_client_frame(&mut fresh).await;
|
|
||||||
assert!(matches!(
|
|
||||||
fresh_event,
|
|
||||||
ClientWorkerEventWsFrame::Event { envelope } if matches!(envelope.payload, protocol::Event::TextDone { .. })
|
|
||||||
));
|
|
||||||
|
|
||||||
let (mut query_stream, _) = connect_async(format!("{url}?cursor=bad")).await.unwrap();
|
|
||||||
let query_snapshot = next_client_frame(&mut query_stream).await;
|
|
||||||
assert!(matches!(
|
|
||||||
query_snapshot,
|
|
||||||
ClientWorkerEventWsFrame::Event { envelope } if matches!(envelope.payload, protocol::Event::Snapshot { .. })
|
|
||||||
));
|
|
||||||
|
|
||||||
stream.send(Message::Text("{}".into())).await.unwrap();
|
|
||||||
let mut saw_observation_only = false;
|
|
||||||
for _ in 0..3 {
|
|
||||||
if let ClientWorkerEventWsFrame::Diagnostic { diagnostic } =
|
|
||||||
next_client_frame(&mut stream).await
|
|
||||||
{
|
|
||||||
assert_eq!(diagnostic.code, "backend.observation_only");
|
|
||||||
saw_observation_only = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert!(saw_observation_only, "expected observation-only diagnostic");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn proxy_maps_runtime_worker_not_found_http_404_to_typed_backend_diagnostic() {
|
async fn proxy_maps_runtime_worker_not_found_http_404_to_protocol_error_event() {
|
||||||
let (_runtime, _worker_ref, endpoint) = spawn_runtime_worker().await;
|
let (_runtime, _worker_ref, endpoint) = spawn_runtime_worker().await;
|
||||||
let endpoint = endpoint.replace("/events/ws", "/missing-worker/events/ws");
|
let endpoint = endpoint.replace("/protocol/ws", "/missing-worker/protocol/ws");
|
||||||
let source = RuntimeObservationSourceConfig {
|
let source = RuntimeObservationSourceConfig {
|
||||||
runtime_id: "runtime-a".into(),
|
runtime_id: "runtime-a".into(),
|
||||||
worker_id: "worker-a".into(),
|
worker_id: "worker-a".into(),
|
||||||
|
|
@ -7978,30 +7814,17 @@ mod tests {
|
||||||
};
|
};
|
||||||
let (url, _dir) = spawn_workspace_proxy(source).await;
|
let (url, _dir) = spawn_workspace_proxy(source).await;
|
||||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||||
let diagnostic = next_client_diagnostic(&mut stream).await;
|
assert!(matches!(
|
||||||
assert_eq!(diagnostic.code, "backend.worker_not_found");
|
next_client_frame(&mut stream).await,
|
||||||
}
|
protocol::Event::Error { .. }
|
||||||
|
));
|
||||||
#[tokio::test]
|
|
||||||
async fn proxy_reports_actual_upstream_disconnect_separately() {
|
|
||||||
let endpoint = spawn_closing_runtime_ws().await;
|
|
||||||
let source = RuntimeObservationSourceConfig {
|
|
||||||
runtime_id: "runtime-a".into(),
|
|
||||||
worker_id: "worker-a".into(),
|
|
||||||
endpoint,
|
|
||||||
bearer_token: None,
|
|
||||||
};
|
|
||||||
let (url, _dir) = spawn_workspace_proxy(source).await;
|
|
||||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
|
||||||
let diagnostic = next_client_diagnostic(&mut stream).await;
|
|
||||||
assert_eq!(diagnostic.code, "backend.upstream_disconnect");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_client_frame(
|
async fn next_client_frame(
|
||||||
stream: &mut tokio_tungstenite::WebSocketStream<
|
stream: &mut tokio_tungstenite::WebSocketStream<
|
||||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||||
>,
|
>,
|
||||||
) -> ClientWorkerEventWsFrame {
|
) -> protocol::Event {
|
||||||
let message = stream.next().await.unwrap().unwrap();
|
let message = stream.next().await.unwrap().unwrap();
|
||||||
let Message::Text(text) = message else {
|
let Message::Text(text) = message else {
|
||||||
panic!("expected text frame");
|
panic!("expected text frame");
|
||||||
|
|
@ -8009,19 +7832,6 @@ mod tests {
|
||||||
serde_json::from_str(&text).unwrap()
|
serde_json::from_str(&text).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_client_diagnostic(
|
|
||||||
stream: &mut tokio_tungstenite::WebSocketStream<
|
|
||||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
|
||||||
>,
|
|
||||||
) -> ClientWorkerEventWsDiagnostic {
|
|
||||||
match next_client_frame(stream).await {
|
|
||||||
ClientWorkerEventWsFrame::Diagnostic { diagnostic } => diagnostic,
|
|
||||||
ClientWorkerEventWsFrame::Event { envelope } => {
|
|
||||||
panic!("expected diagnostic, got event: {envelope:?}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn spawn_runtime_worker() -> (
|
async fn spawn_runtime_worker() -> (
|
||||||
worker_runtime::Runtime,
|
worker_runtime::Runtime,
|
||||||
worker_runtime::identity::WorkerRef,
|
worker_runtime::identity::WorkerRef,
|
||||||
|
|
@ -8039,7 +7849,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
let endpoint = format!(
|
let endpoint = format!(
|
||||||
"ws://{runtime_addr}/v1/workers/{}/events/ws",
|
"ws://{runtime_addr}/v1/workers/{}/protocol/ws",
|
||||||
worker_ref.worker_id
|
worker_ref.worker_id
|
||||||
);
|
);
|
||||||
(runtime, worker_ref, endpoint)
|
(runtime, worker_ref, endpoint)
|
||||||
|
|
@ -8065,22 +7875,10 @@ mod tests {
|
||||||
let app_addr = app_listener.local_addr().unwrap();
|
let app_addr = app_listener.local_addr().unwrap();
|
||||||
tokio::spawn(async move { axum::serve(app_listener, build_router(api)).await.unwrap() });
|
tokio::spawn(async move { axum::serve(app_listener, build_router(api)).await.unwrap() });
|
||||||
(
|
(
|
||||||
format!("ws://{app_addr}/api/runtimes/{runtime_id}/workers/{worker_id}/events/ws"),
|
format!("ws://{app_addr}/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws"),
|
||||||
dir,
|
dir,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn spawn_closing_runtime_ws() -> String {
|
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let addr = listener.local_addr().unwrap();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let (stream, _) = listener.accept().await.unwrap();
|
|
||||||
let mut websocket = tokio_tungstenite::accept_async(stream).await.unwrap();
|
|
||||||
let _ = websocket.close(None).await;
|
|
||||||
});
|
|
||||||
format!("ws://{addr}/events/ws")
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn get_json(app: Router, uri: &str) -> Value {
|
async fn get_json(app: Router, uri: &str) -> Value {
|
||||||
let response = app
|
let response = app
|
||||||
.oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
|
.oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
|
||||||
|
|
|
||||||
|
|
@ -65,17 +65,17 @@ Deno.test("Worker Console uses protocol observation events without transcript fe
|
||||||
);
|
);
|
||||||
|
|
||||||
assert(
|
assert(
|
||||||
consolePage.includes("seenObservationEventIds") &&
|
consolePage.includes("connectProtocolTransport") &&
|
||||||
consolePage.includes(
|
consolePage.includes("handleIncomingProtocolEvent") &&
|
||||||
"rememberObservationEvent(frame.envelope.event_id)",
|
consolePage.includes("/protocol/ws") &&
|
||||||
) &&
|
!consolePage.includes("seenObservationEventIds") &&
|
||||||
consolePage.includes("createConsoleProjector") &&
|
consolePage.includes("createConsoleProjector") &&
|
||||||
consolePage.includes("consoleProjector.append(eventBatch)") &&
|
consolePage.includes("consoleProjector.append(eventBatch)") &&
|
||||||
consolePage.includes("{#each lines as item (item.id)}") &&
|
consolePage.includes("{#each lines as item (item.id)}") &&
|
||||||
!consolePage.includes("projectConsole(observedEvents.map") &&
|
!consolePage.includes("projectConsole(observedEvents.map") &&
|
||||||
!consolePage.includes("/transcript") &&
|
!consolePage.includes("/transcript") &&
|
||||||
!consolePage.includes("WorkerTranscriptProjection"),
|
!consolePage.includes("WorkerTranscriptProjection"),
|
||||||
"Console should render protocol observation replay/live events directly and dedupe repeated frames by event id",
|
"Console should render raw protocol replay/live events directly from the unified protocol WS",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -285,8 +285,11 @@ Deno.test("Worker Console page is routed by runtime_id and worker_id through bac
|
||||||
);
|
);
|
||||||
assert(
|
assert(
|
||||||
!consolePage.includes("/transcript") &&
|
!consolePage.includes("/transcript") &&
|
||||||
consolePage.includes("/events/ws") && consolePage.includes("/input"),
|
consolePage.includes("/protocol/ws") &&
|
||||||
"Console should use protocol observation WS and input APIs without a transcript API",
|
!consolePage.includes("/events" + "/ws") &&
|
||||||
|
!consolePage.includes("/input") &&
|
||||||
|
!consolePage.includes("/completions"),
|
||||||
|
"Console should use only the protocol WS for observation and operations",
|
||||||
);
|
);
|
||||||
assert(
|
assert(
|
||||||
!consolePage.includes("/api/companion"),
|
!consolePage.includes("/api/companion"),
|
||||||
|
|
|
||||||
|
|
@ -245,22 +245,6 @@ export type WorkerInputResult = {
|
||||||
diagnostics: Diagnostic[];
|
diagnostics: Diagnostic[];
|
||||||
};
|
};
|
||||||
|
|
||||||
export type ClientWorkerEventWsEnvelope = {
|
|
||||||
event_id: string;
|
|
||||||
runtime_id: string;
|
|
||||||
worker_id: string;
|
|
||||||
payload: PodProtocolEvent;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type ClientWorkerEventWsDiagnostic = {
|
|
||||||
code: string;
|
|
||||||
message: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type ClientWorkerEventWsFrame =
|
|
||||||
| { kind: "event"; envelope: ClientWorkerEventWsEnvelope }
|
|
||||||
| { kind: "diagnostic"; diagnostic: ClientWorkerEventWsDiagnostic };
|
|
||||||
|
|
||||||
export type ListResponse<T> = {
|
export type ListResponse<T> = {
|
||||||
workspace_id: string;
|
workspace_id: string;
|
||||||
limit: number;
|
limit: number;
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,6 @@
|
||||||
import type { Event as ProtocolEvent, Method as ProtocolMethod, RewindTarget, Segment } from "$lib/generated/protocol";
|
import type { Event as ProtocolEvent, Method as ProtocolMethod, RewindTarget, Segment } from "$lib/generated/protocol";
|
||||||
import { workspaceApiPath } from "$lib/workspace/api/http";
|
import { workspaceApiPath } from "$lib/workspace/api/http";
|
||||||
import type {
|
import type {
|
||||||
ClientWorkerEventWsFrame,
|
|
||||||
Diagnostic,
|
Diagnostic,
|
||||||
Worker,
|
Worker,
|
||||||
PodProtocolEvent,
|
PodProtocolEvent,
|
||||||
|
|
@ -98,13 +97,10 @@
|
||||||
let rewindHeadEntries = $state(0);
|
let rewindHeadEntries = $state(0);
|
||||||
let controlNotice = $state<string | null>(null);
|
let controlNotice = $state<string | null>(null);
|
||||||
let composerNotice = $state<string | null>(null);
|
let composerNotice = $state<string | null>(null);
|
||||||
let streamState = $state<"connecting" | "open" | "closed" | "error">(
|
let protocolState = $state<"connecting" | "open" | "closed" | "error">(
|
||||||
"connecting",
|
"connecting",
|
||||||
);
|
);
|
||||||
let commandState = $state<"connecting" | "open" | "closed" | "error">(
|
let protocolSocket: WebSocket | null = null;
|
||||||
"connecting",
|
|
||||||
);
|
|
||||||
let commandSocket: WebSocket | null = null;
|
|
||||||
let pendingCompletionRequest: {
|
let pendingCompletionRequest: {
|
||||||
resolve: (entries: ComposerCompletionEntry[]) => void;
|
resolve: (entries: ComposerCompletionEntry[]) => void;
|
||||||
reject: (error: Error) => void;
|
reject: (error: Error) => void;
|
||||||
|
|
@ -129,8 +125,8 @@
|
||||||
let consoleProjection = $state.raw<ConsoleProjection>(
|
let consoleProjection = $state.raw<ConsoleProjection>(
|
||||||
consoleProjector.snapshot(),
|
consoleProjector.snapshot(),
|
||||||
);
|
);
|
||||||
let seenObservationEventIds = new Set<string>();
|
|
||||||
let pendingObservationEvents: ConsoleEventInput[] = [];
|
let pendingObservationEvents: ConsoleEventInput[] = [];
|
||||||
|
let protocolEventSequence = 0;
|
||||||
let pendingObservedStates: Array<string | null> = [];
|
let pendingObservedStates: Array<string | null> = [];
|
||||||
let pendingStreamDiagnostics: Diagnostic[] = [];
|
let pendingStreamDiagnostics: Diagnostic[] = [];
|
||||||
let observationFlushHandle: number | null = null;
|
let observationFlushHandle: number | null = null;
|
||||||
|
|
@ -208,7 +204,6 @@
|
||||||
consoleProjection = consoleProjector.reset();
|
consoleProjection = consoleProjector.reset();
|
||||||
eventObservedAtById.clear();
|
eventObservedAtById.clear();
|
||||||
advanceEventObservedAtVersion();
|
advanceEventObservedAtVersion();
|
||||||
seenObservationEventIds = new Set();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function cancelObservationFlush() {
|
function cancelObservationFlush() {
|
||||||
|
|
@ -253,23 +248,27 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function queueObservationEvent(
|
function handleIncomingProtocolEvent(payload: ProtocolEvent) {
|
||||||
frame: ClientWorkerEventWsFrame & { kind: "event" },
|
handleProtocolCommandEvent(payload);
|
||||||
) {
|
if (payload.event === "completions" || payload.event === "error") {
|
||||||
if (!rememberObservationEvent(frame.envelope.event_id)) {
|
if (payload.event === "error") {
|
||||||
|
queueObservationDiagnostic({
|
||||||
|
code: payload.data.code,
|
||||||
|
severity: "error",
|
||||||
|
message: payload.data.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const eventId = `protocol-${++protocolEventSequence}`;
|
||||||
const observedAtMs = Date.now();
|
const observedAtMs = Date.now();
|
||||||
eventObservedAtById.set(frame.envelope.event_id, observedAtMs);
|
eventObservedAtById.set(eventId, observedAtMs);
|
||||||
const payload = frame.envelope.payload;
|
|
||||||
pendingObservationEvents.push({
|
pendingObservationEvents.push({
|
||||||
eventId: frame.envelope.event_id,
|
eventId,
|
||||||
event: payload,
|
event: payload,
|
||||||
observedAtMs,
|
observedAtMs,
|
||||||
});
|
});
|
||||||
if (payload.event === "rewind_targets") {
|
|
||||||
handleProtocolCommandEvent(payload as ProtocolEvent);
|
|
||||||
}
|
|
||||||
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
|
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
|
||||||
scheduleObservationFlush();
|
scheduleObservationFlush();
|
||||||
}
|
}
|
||||||
|
|
@ -279,14 +278,6 @@
|
||||||
scheduleObservationFlush();
|
scheduleObservationFlush();
|
||||||
}
|
}
|
||||||
|
|
||||||
function rememberObservationEvent(eventId: string): boolean {
|
|
||||||
if (seenObservationEventIds.has(eventId)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
seenObservationEventIds.add(eventId);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function applyComposerCompletion(event: KeyboardEvent) {
|
async function applyComposerCompletion(event: KeyboardEvent) {
|
||||||
const target = event.currentTarget;
|
const target = event.currentTarget;
|
||||||
if (!(target instanceof HTMLTextAreaElement)) {
|
if (!(target instanceof HTMLTextAreaElement)) {
|
||||||
|
|
@ -502,89 +493,16 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function connectObservation(
|
function connectProtocolTransport(
|
||||||
targetWorker: Worker | null,
|
targetWorker: Worker | null,
|
||||||
token: number,
|
token: number,
|
||||||
target: ConsoleTarget,
|
target: ConsoleTarget,
|
||||||
) {
|
) {
|
||||||
if (!targetWorker) {
|
if (!targetWorker) {
|
||||||
streamState = "closed";
|
protocolState = "closed";
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
streamState = "connecting";
|
protocolState = "connecting";
|
||||||
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
|
|
||||||
const wsPath = workerApiPath(
|
|
||||||
`/runtimes/${encodeURIComponent(target.runtimeId)}/workers/${encodeURIComponent(
|
|
||||||
target.workerId,
|
|
||||||
)}/events/ws`,
|
|
||||||
);
|
|
||||||
const ws = new WebSocket(
|
|
||||||
`${protocol}//${window.location.host}${wsPath}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
ws.onopen = () => {
|
|
||||||
if (token === reloadToken) {
|
|
||||||
streamState = "open";
|
|
||||||
}
|
|
||||||
};
|
|
||||||
ws.onmessage = (message) => {
|
|
||||||
if (token !== reloadToken) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const frame = JSON.parse(
|
|
||||||
String(message.data),
|
|
||||||
) as ClientWorkerEventWsFrame;
|
|
||||||
if (frame.kind === "event") {
|
|
||||||
queueObservationEvent(frame);
|
|
||||||
} else {
|
|
||||||
queueObservationDiagnostic({
|
|
||||||
code: frame.diagnostic.code,
|
|
||||||
severity: "warning",
|
|
||||||
message: frame.diagnostic.message,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
queueObservationDiagnostic({
|
|
||||||
code: "worker_observation_frame_invalid",
|
|
||||||
severity: "warning",
|
|
||||||
message:
|
|
||||||
error instanceof Error ? error.message : String(error),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
ws.onerror = () => {
|
|
||||||
if (token === reloadToken) {
|
|
||||||
streamState = "error";
|
|
||||||
streamDiagnostics = [
|
|
||||||
...streamDiagnostics,
|
|
||||||
{
|
|
||||||
code: "worker_observation_ws_error",
|
|
||||||
severity: "error",
|
|
||||||
message: "Worker observation WebSocket failed.",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
ws.onclose = () => {
|
|
||||||
if (token === reloadToken && streamState !== "error") {
|
|
||||||
streamState = "closed";
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
return () => ws.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
function connectProtocolCommands(
|
|
||||||
targetWorker: Worker | null,
|
|
||||||
token: number,
|
|
||||||
target: ConsoleTarget,
|
|
||||||
) {
|
|
||||||
if (!targetWorker) {
|
|
||||||
commandState = "closed";
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
commandState = "connecting";
|
|
||||||
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
|
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
|
||||||
const wsPath = workerApiPath(
|
const wsPath = workerApiPath(
|
||||||
`/runtimes/${encodeURIComponent(target.runtimeId)}/workers/${encodeURIComponent(
|
`/runtimes/${encodeURIComponent(target.runtimeId)}/workers/${encodeURIComponent(
|
||||||
|
|
@ -594,11 +512,11 @@
|
||||||
const ws = new WebSocket(
|
const ws = new WebSocket(
|
||||||
`${protocol}//${window.location.host}${wsPath}`,
|
`${protocol}//${window.location.host}${wsPath}`,
|
||||||
);
|
);
|
||||||
commandSocket = ws;
|
protocolSocket = ws;
|
||||||
|
|
||||||
ws.onopen = () => {
|
ws.onopen = () => {
|
||||||
if (token === reloadToken) {
|
if (token === reloadToken) {
|
||||||
commandState = "open";
|
protocolState = "open";
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
ws.onmessage = (message) => {
|
ws.onmessage = (message) => {
|
||||||
|
|
@ -606,14 +524,14 @@
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
handleProtocolCommandEvent(
|
handleIncomingProtocolEvent(
|
||||||
JSON.parse(String(message.data)) as ProtocolEvent,
|
JSON.parse(String(message.data)) as ProtocolEvent,
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
streamDiagnostics = [
|
streamDiagnostics = [
|
||||||
...streamDiagnostics,
|
...streamDiagnostics,
|
||||||
{
|
{
|
||||||
code: "worker_protocol_command_frame_invalid",
|
code: "worker_protocol_frame_invalid",
|
||||||
severity: "warning",
|
severity: "warning",
|
||||||
message:
|
message:
|
||||||
error instanceof Error ? error.message : String(error),
|
error instanceof Error ? error.message : String(error),
|
||||||
|
|
@ -623,42 +541,42 @@
|
||||||
};
|
};
|
||||||
ws.onerror = () => {
|
ws.onerror = () => {
|
||||||
if (token === reloadToken) {
|
if (token === reloadToken) {
|
||||||
commandState = "error";
|
protocolState = "error";
|
||||||
streamDiagnostics = [
|
streamDiagnostics = [
|
||||||
...streamDiagnostics,
|
...streamDiagnostics,
|
||||||
{
|
{
|
||||||
code: "worker_protocol_command_ws_error",
|
code: "worker_protocol_ws_error",
|
||||||
severity: "error",
|
severity: "error",
|
||||||
message: "Worker protocol command WebSocket failed.",
|
message: "Worker protocol WebSocket failed.",
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
ws.onclose = () => {
|
ws.onclose = () => {
|
||||||
if (commandSocket === ws) {
|
if (protocolSocket === ws) {
|
||||||
commandSocket = null;
|
protocolSocket = null;
|
||||||
}
|
}
|
||||||
if (token === reloadToken && commandState !== "error") {
|
if (token === reloadToken && protocolState !== "error") {
|
||||||
commandState = "closed";
|
protocolState = "closed";
|
||||||
}
|
}
|
||||||
rejectPendingCompletion(
|
rejectPendingCompletion(
|
||||||
new Error("Worker protocol command WebSocket closed."),
|
new Error("Worker protocol WebSocket closed."),
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
if (commandSocket === ws) {
|
if (protocolSocket === ws) {
|
||||||
commandSocket = null;
|
protocolSocket = null;
|
||||||
}
|
}
|
||||||
ws.close();
|
ws.close();
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function sendProtocolMethod(method: ProtocolMethod) {
|
function sendProtocolMethod(method: ProtocolMethod) {
|
||||||
if (!commandSocket || commandSocket.readyState !== WebSocket.OPEN) {
|
if (!protocolSocket || protocolSocket.readyState !== WebSocket.OPEN) {
|
||||||
throw new Error("Worker protocol command WebSocket is not open.");
|
throw new Error("Worker protocol WebSocket is not open.");
|
||||||
}
|
}
|
||||||
commandSocket.send(JSON.stringify(method));
|
protocolSocket.send(JSON.stringify(method));
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleProtocolCommandEvent(event: ProtocolEvent) {
|
function handleProtocolCommandEvent(event: ProtocolEvent) {
|
||||||
|
|
@ -1162,8 +1080,7 @@
|
||||||
void loadConsoleData(target);
|
void loadConsoleData(target);
|
||||||
});
|
});
|
||||||
|
|
||||||
$effect(() => connectObservation(worker, reloadToken, consoleTarget));
|
$effect(() => connectProtocolTransport(worker, reloadToken, consoleTarget));
|
||||||
$effect(() => connectProtocolCommands(worker, reloadToken, consoleTarget));
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<svelte:head>
|
<svelte:head>
|
||||||
|
|
@ -1182,14 +1099,14 @@
|
||||||
<div class="console-header-actions">
|
<div class="console-header-actions">
|
||||||
<div
|
<div
|
||||||
class="console-status-pill"
|
class="console-status-pill"
|
||||||
class:warn={streamState !== "open"}
|
class:warn={protocolState !== "open"}
|
||||||
>
|
>
|
||||||
{workerState} · stream {streamState} · command {commandState}
|
{workerState} · protocol {protocolState}
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="secondary-button"
|
class="secondary-button"
|
||||||
disabled={commandState !== "open"}
|
disabled={protocolState !== "open"}
|
||||||
onclick={() => sendControl({ method: "cancel" }, "Cancel")}
|
onclick={() => sendControl({ method: "cancel" }, "Cancel")}
|
||||||
>
|
>
|
||||||
Cancel
|
Cancel
|
||||||
|
|
@ -1197,7 +1114,7 @@
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="secondary-button"
|
class="secondary-button"
|
||||||
disabled={commandState !== "open"}
|
disabled={protocolState !== "open"}
|
||||||
onclick={() => sendControl({ method: "pause" }, "Pause")}
|
onclick={() => sendControl({ method: "pause" }, "Pause")}
|
||||||
>
|
>
|
||||||
Pause
|
Pause
|
||||||
|
|
@ -1205,7 +1122,7 @@
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="secondary-button"
|
class="secondary-button"
|
||||||
disabled={commandState !== "open"}
|
disabled={protocolState !== "open"}
|
||||||
onclick={() => sendControl({ method: "resume" }, "Resume")}
|
onclick={() => sendControl({ method: "resume" }, "Resume")}
|
||||||
>
|
>
|
||||||
Resume
|
Resume
|
||||||
|
|
@ -1213,7 +1130,7 @@
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="secondary-button"
|
class="secondary-button"
|
||||||
disabled={commandState !== "open"}
|
disabled={protocolState !== "open"}
|
||||||
onclick={() => sendControl({ method: "compact" }, "Compact")}
|
onclick={() => sendControl({ method: "compact" }, "Compact")}
|
||||||
>
|
>
|
||||||
Compact
|
Compact
|
||||||
|
|
@ -1221,7 +1138,7 @@
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="secondary-button"
|
class="secondary-button"
|
||||||
disabled={commandState !== "open"}
|
disabled={protocolState !== "open"}
|
||||||
onclick={requestRewindTargets}
|
onclick={requestRewindTargets}
|
||||||
>
|
>
|
||||||
Rewind
|
Rewind
|
||||||
|
|
@ -1249,7 +1166,7 @@
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="secondary-button"
|
class="secondary-button"
|
||||||
disabled={commandState !== "open" || !target.eligible}
|
disabled={protocolState !== "open" || !target.eligible}
|
||||||
title={target.disabled_reason ?? target.warning ?? undefined}
|
title={target.disabled_reason ?? target.warning ?? undefined}
|
||||||
onclick={() => rewindTo(target)}
|
onclick={() => rewindTo(target)}
|
||||||
>
|
>
|
||||||
|
|
@ -1377,7 +1294,7 @@
|
||||||
{#if diagnostics.length > 0}
|
{#if diagnostics.length > 0}
|
||||||
<details
|
<details
|
||||||
class="metadata-details"
|
class="metadata-details"
|
||||||
open={streamState === "error"}
|
open={protocolState === "error"}
|
||||||
>
|
>
|
||||||
<summary>Diagnostics ({diagnostics.length})</summary>
|
<summary>Diagnostics ({diagnostics.length})</summary>
|
||||||
<ul>
|
<ul>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user