fix: unify worker protocol websocket
This commit is contained in:
parent
70a26a3042
commit
8841d063be
|
|
@ -2,7 +2,7 @@
|
|||
title: 'Backend runtime経由の操作をprotocol transportへ統一しTUI同等にする'
|
||||
state: 'planning'
|
||||
created_at: '2026-07-21T09:20:07Z'
|
||||
updated_at: '2026-07-21T10:33:12Z'
|
||||
updated_at: '2026-07-21T11:29:08Z'
|
||||
assignee: null
|
||||
readiness: 'draft'
|
||||
---
|
||||
|
|
|
|||
|
|
@ -95,3 +95,43 @@ Frontend check note:
|
|||
- The Svelte page was updated against the generated protocol types in `web/workspace/src/lib/generated/protocol.ts`, but a frontend-native check still needs to be run in an environment with the frontend deps installed.
|
||||
|
||||
---
|
||||
|
||||
<!-- event: implementation_report author: assistant at: 2026-07-21T11:29:08Z -->
|
||||
|
||||
## Implementation report
|
||||
|
||||
Follow-up implementation after design clarification:
|
||||
|
||||
Changed the transport split so `/events/ws` no longer exists. `/protocol/ws` is now the single Worker protocol WebSocket for both directions:
|
||||
|
||||
- client -> runtime/backend: raw `protocol::Method` JSON frames
|
||||
- runtime/backend -> client: raw `protocol::Event` JSON frames
|
||||
|
||||
Runtime changes:
|
||||
- Removed `/v1/workers/{worker_id}/events/ws` route.
|
||||
- Removed runtime-specific event WS envelope/diagnostic frame types.
|
||||
- Extended `/v1/workers/{worker_id}/protocol/ws` to stream snapshot/backlog/live observation events as raw `protocol::Event` while also accepting method frames on the same socket.
|
||||
|
||||
Workspace Backend changes:
|
||||
- Removed public `/api/.../events/ws` and `/api/w/.../events/ws` routes.
|
||||
- Removed Backend client event WS envelope types.
|
||||
- Workspace `/protocol/ws` now proxies/streams raw protocol events and forwards raw protocol methods on the same socket.
|
||||
- Runtime observation source endpoint generation now targets runtime `/protocol/ws` instead of `/events/ws`.
|
||||
|
||||
Client/frontend changes:
|
||||
- TUI `BackendRuntimeClient` no longer opens a separate observation socket. It uses only `/protocol/ws` for send and receive.
|
||||
- Browser Console no longer opens both observation and command sockets. It uses one protocol socket for replay/live events, completions, rewind targets, and controls.
|
||||
- Removed old frontend `ClientWorkerEventWs*` types.
|
||||
- Updated frontend UI tests to assert protocol-only Console behavior and absence of operation HTTP paths.
|
||||
|
||||
Verification:
|
||||
- `nix develop -c cargo fmt -- --check`
|
||||
- `nix develop -c cargo check -p worker-runtime -p yoi-workspace-server -p client -p tui`
|
||||
- `nix develop -c cargo test -p client backend_runtime --lib`
|
||||
- `nix develop -c cargo test -p worker-runtime --features ws-server --lib protocol_ws`
|
||||
- `nix develop -c cargo test -p yoi-workspace-server protocol_ws --lib`
|
||||
- `nix develop -c cargo test -p yoi-workspace-server proxy_maps_runtime_worker_not_found_http_404_to_protocol_error_event --lib`
|
||||
- `deno test --allow-read web/workspace/src/lib/workspace/console/worker-console.ui.test.ts` from `web/workspace`
|
||||
- `git diff --check`
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -1,17 +1,12 @@
|
|||
use std::collections::VecDeque;
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use protocol::{ErrorCode, Event, Method};
|
||||
use serde::Deserialize;
|
||||
use std::collections::VecDeque;
|
||||
use std::fmt;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::Message as TungsteniteMessage;
|
||||
|
||||
const RECONNECT_DELAY: Duration = Duration::from_millis(500);
|
||||
const MAX_RECONNECT_ATTEMPTS: usize = 3;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackendRuntimeTarget {
|
||||
/// Workspace Backend API root URL, for example `http://127.0.0.1:8787`.
|
||||
|
|
@ -163,8 +158,7 @@ pub struct BackendRuntimeClient {
|
|||
command_tx: mpsc::UnboundedSender<Method>,
|
||||
events: mpsc::UnboundedReceiver<Event>,
|
||||
diagnostics: VecDeque<Event>,
|
||||
_observation_task: tokio::task::JoinHandle<()>,
|
||||
_command_task: tokio::task::JoinHandle<()>,
|
||||
_protocol_task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
@ -262,16 +256,10 @@ impl BackendRuntimeClient {
|
|||
let (event_tx, rx) = mpsc::unbounded_channel();
|
||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||
|
||||
let observation_target = target.clone();
|
||||
let observation_tx = event_tx.clone();
|
||||
let observation_task = tokio::spawn(async move {
|
||||
observe_worker_events(observation_target, observation_tx).await;
|
||||
});
|
||||
|
||||
let command_target = target.clone();
|
||||
let command_event_tx = event_tx.clone();
|
||||
let command_task = tokio::spawn(async move {
|
||||
run_worker_protocol_commands(command_target, command_rx, command_event_tx).await;
|
||||
let protocol_target = target.clone();
|
||||
let protocol_event_tx = event_tx.clone();
|
||||
let protocol_task = tokio::spawn(async move {
|
||||
run_worker_protocol_transport(protocol_target, command_rx, protocol_event_tx).await;
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
|
|
@ -279,8 +267,7 @@ impl BackendRuntimeClient {
|
|||
command_tx,
|
||||
events: rx,
|
||||
diagnostics: VecDeque::new(),
|
||||
_observation_task: observation_task,
|
||||
_command_task: command_task,
|
||||
_protocol_task: protocol_task,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -311,12 +298,11 @@ impl BackendRuntimeClient {
|
|||
|
||||
impl Drop for BackendRuntimeClient {
|
||||
fn drop(&mut self) {
|
||||
self._observation_task.abort();
|
||||
self._command_task.abort();
|
||||
self._protocol_task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_worker_protocol_commands(
|
||||
async fn run_worker_protocol_transport(
|
||||
target: BackendRuntimeTarget,
|
||||
mut commands: mpsc::UnboundedReceiver<Method>,
|
||||
tx: mpsc::UnboundedSender<Event>,
|
||||
|
|
@ -402,81 +388,6 @@ async fn run_worker_protocol_commands(
|
|||
}
|
||||
}
|
||||
|
||||
async fn observe_worker_events(target: BackendRuntimeTarget, tx: mpsc::UnboundedSender<Event>) {
|
||||
let mut attempts = 0_usize;
|
||||
|
||||
loop {
|
||||
let url = observation_ws_url(&target);
|
||||
match connect_async(&url).await {
|
||||
Ok((mut ws, _)) => {
|
||||
attempts = 0;
|
||||
while let Some(frame) = ws.next().await {
|
||||
match frame {
|
||||
Ok(TungsteniteMessage::Text(text)) => {
|
||||
match serde_json::from_str::<ClientWorkerEventWsFrame>(&text) {
|
||||
Ok(ClientWorkerEventWsFrame::Event { envelope }) => {
|
||||
if envelope.runtime_id != target.runtime_id
|
||||
|| envelope.worker_id != target.worker_id
|
||||
{
|
||||
let _ = tx.send(diagnostic_event(format!(
|
||||
"Backend observation frame target mismatch: got {}:{}, expected {}",
|
||||
envelope.runtime_id,
|
||||
envelope.worker_id,
|
||||
target.display_label()
|
||||
)));
|
||||
continue;
|
||||
}
|
||||
let _ = tx.send(envelope.payload);
|
||||
}
|
||||
Ok(ClientWorkerEventWsFrame::Diagnostic { diagnostic }) => {
|
||||
let message = format!(
|
||||
"Backend observation diagnostic [{}]: {}",
|
||||
diagnostic.code, diagnostic.message
|
||||
);
|
||||
let _ = tx.send(diagnostic_event(message));
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = tx.send(diagnostic_event(format!(
|
||||
"Backend observation frame was not valid JSON: {error}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(TungsteniteMessage::Close(_)) => break,
|
||||
Ok(TungsteniteMessage::Ping(_))
|
||||
| Ok(TungsteniteMessage::Pong(_))
|
||||
| Ok(TungsteniteMessage::Binary(_))
|
||||
| Ok(TungsteniteMessage::Frame(_)) => {}
|
||||
Err(error) => {
|
||||
let _ = tx.send(diagnostic_event(format!(
|
||||
"Backend observation WebSocket error for {}: {error}",
|
||||
target.display_label()
|
||||
)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = tx.send(diagnostic_event(format!(
|
||||
"Backend observation WebSocket connect failed for {}: {error}",
|
||||
target.display_label()
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
attempts += 1;
|
||||
if attempts > MAX_RECONNECT_ATTEMPTS {
|
||||
let _ = tx.send(diagnostic_event(format!(
|
||||
"Backend observation stream for {} stopped after {MAX_RECONNECT_ATTEMPTS} reconnect attempts",
|
||||
target.display_label()
|
||||
)));
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(RECONNECT_DELAY).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn diagnostic_event(message: impl Into<String>) -> Event {
|
||||
Event::Error {
|
||||
code: ErrorCode::Internal,
|
||||
|
|
@ -552,15 +463,6 @@ fn backend_runtime_workers_path(workspace_id: Option<&str>, runtime_id: &str) ->
|
|||
}
|
||||
}
|
||||
|
||||
fn observation_ws_url(target: &BackendRuntimeTarget) -> String {
|
||||
let path = format!(
|
||||
"/api/runtimes/{}/workers/{}/events/ws",
|
||||
path_segment_encode(&target.runtime_id),
|
||||
path_segment_encode(&target.worker_id)
|
||||
);
|
||||
join_base_and_path(&http_base_to_ws(&target.base_url), &path)
|
||||
}
|
||||
|
||||
fn protocol_ws_url(target: &BackendRuntimeTarget) -> String {
|
||||
let path = format!(
|
||||
"/api/runtimes/{}/workers/{}/protocol/ws",
|
||||
|
|
@ -611,42 +513,14 @@ pub struct BackendDiagnostic {
|
|||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
enum ClientWorkerEventWsFrame {
|
||||
Event {
|
||||
envelope: ClientWorkerEventWsEnvelope,
|
||||
},
|
||||
Diagnostic {
|
||||
diagnostic: ClientWorkerEventWsDiagnostic,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ClientWorkerEventWsEnvelope {
|
||||
runtime_id: String,
|
||||
worker_id: String,
|
||||
payload: Event,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ClientWorkerEventWsDiagnostic {
|
||||
code: String,
|
||||
message: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn command_and_observation_urls_use_backend_protocol_paths() {
|
||||
fn protocol_url_uses_backend_runtime_worker_identity() {
|
||||
let target =
|
||||
BackendRuntimeTarget::new("http://127.0.0.1:8787/", "runtime/one", "worker one");
|
||||
assert_eq!(
|
||||
observation_ws_url(&target),
|
||||
"ws://127.0.0.1:8787/api/runtimes/runtime%2Fone/workers/worker%20one/events/ws"
|
||||
);
|
||||
assert_eq!(
|
||||
protocol_ws_url(&target),
|
||||
"ws://127.0.0.1:8787/api/runtimes/runtime%2Fone/workers/worker%20one/protocol/ws"
|
||||
|
|
|
|||
|
|
@ -154,9 +154,7 @@ pub fn runtime_http_router(runtime: Runtime, local_token: Option<String>) -> Rou
|
|||
.route("/v1/workers/{worker_id}/cancel", post(cancel_worker));
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
let router = router
|
||||
.route("/v1/workers/{worker_id}/events/ws", get(worker_events_ws))
|
||||
.route(
|
||||
let router = router.route(
|
||||
"/v1/workers/{worker_id}/protocol/ws",
|
||||
get(worker_protocol_ws),
|
||||
);
|
||||
|
|
@ -277,88 +275,12 @@ pub struct RuntimeHttpErrorDetail {
|
|||
pub message: String,
|
||||
}
|
||||
|
||||
/// Runtime-owned WebSocket frame for worker-scoped observation.
|
||||
#[cfg(feature = "ws-server")]
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum RuntimeWorkerEventWsFrame {
|
||||
Event {
|
||||
envelope: RuntimeWorkerEventWsEnvelope,
|
||||
},
|
||||
Diagnostic {
|
||||
diagnostic: RuntimeWorkerEventWsDiagnostic,
|
||||
},
|
||||
}
|
||||
|
||||
/// Runtime-local protocol event envelope.
|
||||
#[cfg(feature = "ws-server")]
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct RuntimeWorkerEventWsEnvelope {
|
||||
pub cursor: String,
|
||||
pub event_id: String,
|
||||
pub worker_id: WorkerId,
|
||||
pub payload: protocol::Event,
|
||||
}
|
||||
|
||||
/// Runtime-local observation diagnostic.
|
||||
#[cfg(feature = "ws-server")]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RuntimeWorkerEventWsDiagnostic {
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
struct RuntimeWorkerEventsWsQuery {
|
||||
cursor: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
impl RuntimeWorkerEventWsFrame {
|
||||
fn event(
|
||||
cursor: String,
|
||||
event_id: String,
|
||||
worker_id: WorkerId,
|
||||
payload: protocol::Event,
|
||||
) -> Self {
|
||||
Self::Event {
|
||||
envelope: RuntimeWorkerEventWsEnvelope {
|
||||
cursor,
|
||||
event_id,
|
||||
worker_id,
|
||||
payload,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn diagnostic(code: impl Into<String>, message: impl Into<String>) -> Self {
|
||||
Self::Diagnostic {
|
||||
diagnostic: RuntimeWorkerEventWsDiagnostic {
|
||||
code: code.into(),
|
||||
message: message.into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
async fn send_ws_frame(socket: &mut WebSocket, frame: &RuntimeWorkerEventWsFrame) -> bool {
|
||||
match serde_json::to_string(frame) {
|
||||
Ok(text) => socket.send(WsMessage::Text(text.into())).await.is_ok(),
|
||||
Err(error) => {
|
||||
let fallback = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.serialize_failed",
|
||||
format!("failed to serialize observation frame: {error}"),
|
||||
);
|
||||
let Ok(text) = serde_json::to_string(&fallback) else {
|
||||
return false;
|
||||
};
|
||||
socket.send(WsMessage::Text(text.into())).await.is_ok()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type RestResult<T> = Result<Json<T>, RuntimeHttpRestError>;
|
||||
|
||||
async fn get_runtime(
|
||||
|
|
@ -515,6 +437,7 @@ async fn create_worker(
|
|||
async fn worker_protocol_ws(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
Path(worker_id): Path<String>,
|
||||
Query(query): Query<RuntimeWorkerEventsWsQuery>,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> Result<Response, RuntimeHttpRestError> {
|
||||
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
|
||||
|
|
@ -523,7 +446,9 @@ async fn worker_protocol_ws(
|
|||
.worker_detail(&worker_ref)
|
||||
.map_err(RuntimeHttpRestError::runtime)?;
|
||||
Ok(ws
|
||||
.on_upgrade(move |socket| worker_protocol_ws_session(state.runtime, worker_ref, socket))
|
||||
.on_upgrade(move |socket| {
|
||||
worker_protocol_ws_session(state.runtime, worker_ref, query, socket)
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
|
||||
|
|
@ -531,11 +456,72 @@ async fn worker_protocol_ws(
|
|||
async fn worker_protocol_ws_session(
|
||||
runtime: Runtime,
|
||||
worker_ref: WorkerRef,
|
||||
query: RuntimeWorkerEventsWsQuery,
|
||||
mut socket: WebSocket,
|
||||
) {
|
||||
while let Some(frame) = socket.next().await {
|
||||
match frame {
|
||||
Ok(WsMessage::Text(text)) => match serde_json::from_str::<protocol::Method>(&text) {
|
||||
let mut cursor = match query.cursor.as_deref() {
|
||||
Some(raw) => match WorkerObservationCursor::decode(raw) {
|
||||
Some(cursor) => cursor,
|
||||
None => {
|
||||
let event =
|
||||
protocol_error_event(format!("malformed worker observation cursor: {raw}"));
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
None => match runtime.worker_observation_cursor_now(&worker_ref) {
|
||||
Ok(cursor) => cursor,
|
||||
Err(error) => {
|
||||
let event = protocol_error_event(error.to_string());
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let mut receiver = match runtime.subscribe_worker_observation() {
|
||||
Ok(receiver) => receiver,
|
||||
Err(error) => {
|
||||
let event =
|
||||
protocol_error_event(format!("runtime observation bus unavailable: {error}"));
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let snapshot = match runtime.worker_observation_snapshot(&worker_ref) {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
let event = protocol_error_event(error.to_string());
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
if !send_protocol_event(&mut socket, &snapshot).await {
|
||||
return;
|
||||
}
|
||||
|
||||
match runtime.read_worker_observation_events(&worker_ref, cursor) {
|
||||
Ok(backlog) => {
|
||||
for event in backlog {
|
||||
cursor = WorkerObservationCursor::new(event.sequence);
|
||||
if !send_protocol_event(&mut socket, &event.payload).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let event = protocol_error_event(error.to_string());
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
inbound = socket.next() => {
|
||||
match inbound {
|
||||
Some(Ok(WsMessage::Text(text))) => match serde_json::from_str::<protocol::Method>(&text) {
|
||||
Ok(method) => match runtime.send_protocol_method(&worker_ref, method) {
|
||||
Ok(events) => {
|
||||
for event in events {
|
||||
|
|
@ -552,27 +538,51 @@ async fn worker_protocol_ws_session(
|
|||
}
|
||||
},
|
||||
Err(error) => {
|
||||
let event =
|
||||
protocol_error_event(format!("malformed protocol method frame: {error}"));
|
||||
let event = protocol_error_event(format!(
|
||||
"malformed protocol method frame: {error}"
|
||||
));
|
||||
if !send_protocol_event(&mut socket, &event).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(WsMessage::Close(_)) => return,
|
||||
Ok(WsMessage::Ping(payload)) => {
|
||||
Some(Ok(WsMessage::Close(_))) | None => return,
|
||||
Some(Ok(WsMessage::Ping(payload))) => {
|
||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Ok(WsMessage::Pong(_)) | Ok(WsMessage::Binary(_)) => {}
|
||||
Err(error) => {
|
||||
Some(Ok(WsMessage::Pong(_))) | Some(Ok(WsMessage::Binary(_))) => {}
|
||||
Some(Err(error)) => {
|
||||
let event = protocol_error_event(format!("protocol WebSocket error: {error}"));
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
event = receiver.recv() => {
|
||||
match event {
|
||||
Ok(event) if event.worker_ref == worker_ref && event.sequence > cursor.sequence => {
|
||||
cursor = WorkerObservationCursor::new(event.sequence);
|
||||
if !send_protocol_event(&mut socket, &event.payload).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
|
||||
let event = protocol_error_event("runtime observation backlog was overrun");
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
let event = protocol_error_event("runtime observation bus closed");
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
|
|
@ -599,182 +609,6 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
async fn worker_events_ws(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
Path(worker_id): Path<String>,
|
||||
Query(query): Query<RuntimeWorkerEventsWsQuery>,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> Result<Response, RuntimeHttpRestError> {
|
||||
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
|
||||
state
|
||||
.runtime
|
||||
.worker_detail(&worker_ref)
|
||||
.map_err(RuntimeHttpRestError::runtime)?;
|
||||
Ok(ws
|
||||
.on_upgrade(move |socket| {
|
||||
worker_events_ws_session(state.runtime, worker_ref, query, socket)
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
async fn worker_events_ws_session(
|
||||
runtime: Runtime,
|
||||
worker_ref: WorkerRef,
|
||||
query: RuntimeWorkerEventsWsQuery,
|
||||
mut socket: WebSocket,
|
||||
) {
|
||||
let mut cursor = match query.cursor.as_deref() {
|
||||
Some(raw) => match WorkerObservationCursor::decode(raw) {
|
||||
Some(cursor) => cursor,
|
||||
None => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.cursor_malformed",
|
||||
format!("malformed worker observation cursor: {raw}"),
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
None => match runtime.worker_observation_cursor_now(&worker_ref) {
|
||||
Ok(cursor) => cursor,
|
||||
Err(error) => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.worker_not_found",
|
||||
error.to_string(),
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let mut receiver = match runtime.subscribe_worker_observation() {
|
||||
Ok(receiver) => receiver,
|
||||
Err(error) => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.unavailable",
|
||||
format!("runtime observation bus unavailable: {error}"),
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let snapshot = match runtime.worker_observation_snapshot(&worker_ref) {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.worker_not_found",
|
||||
error.to_string(),
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let snapshot_cursor = cursor.encode();
|
||||
let snapshot_frame = RuntimeWorkerEventWsFrame::event(
|
||||
snapshot_cursor.clone(),
|
||||
format!("snapshot:{snapshot_cursor}"),
|
||||
worker_ref.worker_id.clone(),
|
||||
snapshot,
|
||||
);
|
||||
if !send_ws_frame(&mut socket, &snapshot_frame).await {
|
||||
return;
|
||||
}
|
||||
|
||||
match runtime.read_worker_observation_events(&worker_ref, cursor) {
|
||||
Ok(backlog) => {
|
||||
for event in backlog {
|
||||
cursor = WorkerObservationCursor::new(event.sequence);
|
||||
let frame = RuntimeWorkerEventWsFrame::event(
|
||||
event.cursor,
|
||||
event.event_id,
|
||||
event.worker_ref.worker_id,
|
||||
event.payload,
|
||||
);
|
||||
if !send_ws_frame(&mut socket, &frame).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.cursor_unknown_or_expired",
|
||||
error.to_string(),
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
inbound = socket.next() => {
|
||||
match inbound {
|
||||
Some(Ok(WsMessage::Close(_))) | None => return,
|
||||
Some(Ok(WsMessage::Ping(payload))) => {
|
||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Some(Ok(WsMessage::Pong(_))) => {}
|
||||
Some(Ok(_)) => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.observation_only",
|
||||
"runtime worker event WebSocket is observation-only",
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.websocket_error",
|
||||
format!("runtime WebSocket receive error: {error}"),
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
event = receiver.recv() => {
|
||||
match event {
|
||||
Ok(event) if event.worker_ref == worker_ref && event.sequence > cursor.sequence => {
|
||||
cursor = WorkerObservationCursor::new(event.sequence);
|
||||
let frame = RuntimeWorkerEventWsFrame::event(
|
||||
event.cursor,
|
||||
event.event_id,
|
||||
event.worker_ref.worker_id,
|
||||
event.payload,
|
||||
);
|
||||
if !send_ws_frame(&mut socket, &frame).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.cursor_expired",
|
||||
"runtime observation backlog was overrun",
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
let frame = RuntimeWorkerEventWsFrame::diagnostic(
|
||||
"runtime.upstream_closed",
|
||||
"runtime observation bus closed",
|
||||
);
|
||||
let _ = send_ws_frame(&mut socket, &frame).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_worker_input(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
Path(worker_id): Path<String>,
|
||||
|
|
@ -1409,7 +1243,7 @@ mod ws_tests {
|
|||
runtime,
|
||||
worker.worker_ref.clone(),
|
||||
format!(
|
||||
"ws://{addr}/v1/workers/{}/events/ws",
|
||||
"ws://{addr}/v1/workers/{}/protocol/ws",
|
||||
worker.worker_ref.worker_id
|
||||
),
|
||||
)
|
||||
|
|
@ -1419,7 +1253,7 @@ mod ws_tests {
|
|||
stream: &mut tokio_tungstenite::WebSocketStream<
|
||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||
>,
|
||||
) -> RuntimeWorkerEventWsFrame {
|
||||
) -> protocol::Event {
|
||||
let message = stream.next().await.unwrap().unwrap();
|
||||
let Message::Text(text) = message else {
|
||||
panic!("expected text frame");
|
||||
|
|
@ -1428,21 +1262,16 @@ mod ws_tests {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_ws_connect_sends_snapshot_and_live_worker_events() {
|
||||
async fn protocol_ws_connect_sends_snapshot_and_live_worker_events() {
|
||||
let (runtime, worker_ref, url) = spawn_runtime_server().await;
|
||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||
|
||||
match next_frame(&mut stream).await {
|
||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
||||
assert_eq!(envelope.worker_id, worker_ref.worker_id);
|
||||
assert!(matches!(envelope.payload, protocol::Event::Snapshot { .. }));
|
||||
}
|
||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
||||
panic!("unexpected diagnostic: {diagnostic:?}");
|
||||
}
|
||||
}
|
||||
assert!(matches!(
|
||||
next_frame(&mut stream).await,
|
||||
protocol::Event::Snapshot { .. }
|
||||
));
|
||||
|
||||
let stored = runtime
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
&worker_ref,
|
||||
protocol::Event::TextDelta {
|
||||
|
|
@ -1450,23 +1279,14 @@ mod ws_tests {
|
|||
},
|
||||
)
|
||||
.unwrap();
|
||||
match next_frame(&mut stream).await {
|
||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
||||
assert_eq!(envelope.worker_id, worker_ref.worker_id);
|
||||
assert_eq!(envelope.cursor, stored.cursor);
|
||||
assert!(matches!(
|
||||
envelope.payload,
|
||||
next_frame(&mut stream).await,
|
||||
protocol::Event::TextDelta { .. }
|
||||
));
|
||||
}
|
||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
||||
panic!("unexpected diagnostic: {diagnostic:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_ws_cursor_resume_is_duplicate_safe_and_filters_workers() {
|
||||
async fn protocol_ws_cursor_resume_is_duplicate_safe_and_filters_workers() {
|
||||
let (runtime, worker_ref, url) = spawn_runtime_server().await;
|
||||
let other = runtime.create_worker(ws_create_request()).unwrap();
|
||||
let first = runtime
|
||||
|
|
@ -1481,7 +1301,7 @@ mod ws_tests {
|
|||
.observe_worker_event(
|
||||
&other.worker_ref,
|
||||
protocol::Event::TextDelta {
|
||||
text: "started".into(),
|
||||
text: "other".into(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
|
@ -1491,10 +1311,10 @@ mod ws_tests {
|
|||
.unwrap();
|
||||
assert!(matches!(
|
||||
next_frame(&mut stream).await,
|
||||
RuntimeWorkerEventWsFrame::Event { envelope } if matches!(envelope.payload, protocol::Event::Snapshot { .. })
|
||||
protocol::Event::Snapshot { .. }
|
||||
));
|
||||
|
||||
let second = runtime
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
&worker_ref,
|
||||
protocol::Event::TextDone {
|
||||
|
|
@ -1502,41 +1322,27 @@ mod ws_tests {
|
|||
},
|
||||
)
|
||||
.unwrap();
|
||||
match next_frame(&mut stream).await {
|
||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
||||
assert_eq!(envelope.cursor, second.cursor);
|
||||
assert_ne!(envelope.cursor, first.cursor);
|
||||
assert!(matches!(envelope.payload, protocol::Event::TextDone { .. }));
|
||||
}
|
||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
||||
panic!("unexpected diagnostic: {diagnostic:?}");
|
||||
}
|
||||
}
|
||||
assert!(matches!(
|
||||
next_frame(&mut stream).await,
|
||||
protocol::Event::TextDone { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_ws_reports_malformed_cursor_and_observation_only_input() {
|
||||
async fn protocol_ws_reports_malformed_cursor_and_method_frame() {
|
||||
let (_runtime, _worker_ref, url) = spawn_runtime_server().await;
|
||||
let (mut malformed, _) = connect_async(format!("{url}?cursor=bad")).await.unwrap();
|
||||
match next_frame(&mut malformed).await {
|
||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
||||
assert_eq!(diagnostic.code, "runtime.cursor_malformed");
|
||||
}
|
||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
||||
panic!("unexpected event: {envelope:?}");
|
||||
}
|
||||
}
|
||||
assert!(matches!(
|
||||
next_frame(&mut malformed).await,
|
||||
protocol::Event::Error { .. }
|
||||
));
|
||||
|
||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||
let _ = next_frame(&mut stream).await;
|
||||
stream.send(Message::Text("{}".into())).await.unwrap();
|
||||
match next_frame(&mut stream).await {
|
||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
||||
assert_eq!(diagnostic.code, "runtime.observation_only");
|
||||
}
|
||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
||||
panic!("unexpected event: {envelope:?}");
|
||||
}
|
||||
}
|
||||
assert!(matches!(
|
||||
next_frame(&mut stream).await,
|
||||
protocol::Event::Error { .. }
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2128,7 +2128,7 @@ impl RemoteWorkerRuntime {
|
|||
} else if let Some(rest) = base.strip_prefix("http://") {
|
||||
base = format!("ws://{rest}");
|
||||
}
|
||||
format!("{base}/v1/workers/{worker_id}/events/ws")
|
||||
format!("{base}/v1/workers/{worker_id}/protocol/ws")
|
||||
}
|
||||
|
||||
fn get_json<T>(&self, path: &str) -> Result<T, RuntimeDiagnostic>
|
||||
|
|
@ -4285,7 +4285,7 @@ mod tests {
|
|||
panic!("remote runtime should expose a remote WS observation source");
|
||||
};
|
||||
assert!(observation.endpoint.starts_with("ws://127.0.0.1:"));
|
||||
assert!(observation.endpoint.ends_with("/v1/workers/1/events/ws"));
|
||||
assert!(observation.endpoint.ends_with("/v1/workers/1/protocol/ws"));
|
||||
assert_eq!(observation.bearer_token.as_deref(), Some(secret.as_str()));
|
||||
|
||||
let workers = registry.list_workers(10);
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ use serde::{Deserialize, Serialize};
|
|||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tokio_tungstenite::tungstenite::{Error as TungsteniteError, Message as TungsteniteMessage};
|
||||
use worker_runtime::http_server::{RuntimeWorkerEventWsEnvelope, RuntimeWorkerEventWsFrame};
|
||||
|
||||
/// Backend-private source for a runtime worker observation stream.
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
|
|
@ -104,42 +103,12 @@ pub struct RuntimeObservationUpstreamEvent {
|
|||
pub payload: protocol::Event,
|
||||
}
|
||||
|
||||
/// Backend-local frame exposed to browser/future-TUI clients.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum ClientWorkerEventWsFrame {
|
||||
Event {
|
||||
envelope: ClientWorkerEventWsEnvelope,
|
||||
},
|
||||
Diagnostic {
|
||||
diagnostic: ClientWorkerEventWsDiagnostic,
|
||||
},
|
||||
}
|
||||
|
||||
/// Backend-owned event envelope. It intentionally omits Runtime endpoints,
|
||||
/// credentials, sockets and session paths.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ClientWorkerEventWsEnvelope {
|
||||
pub event_id: String,
|
||||
pub runtime_id: String,
|
||||
pub worker_id: String,
|
||||
pub payload: protocol::Event,
|
||||
}
|
||||
|
||||
/// Client-facing typed observation diagnostic.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ClientWorkerEventWsDiagnostic {
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ObservationProxyError {
|
||||
RuntimeUnavailable(String),
|
||||
WorkerNotFound(String),
|
||||
UpstreamDisconnect(String),
|
||||
MalformedFrame(String),
|
||||
ObservationOnly,
|
||||
}
|
||||
|
||||
impl ObservationProxyError {
|
||||
|
|
@ -149,7 +118,6 @@ impl ObservationProxyError {
|
|||
ObservationProxyError::WorkerNotFound(_) => "backend.worker_not_found",
|
||||
ObservationProxyError::UpstreamDisconnect(_) => "backend.upstream_disconnect",
|
||||
ObservationProxyError::MalformedFrame(_) => "backend.malformed_frame",
|
||||
ObservationProxyError::ObservationOnly => "backend.observation_only",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -159,24 +127,6 @@ impl ObservationProxyError {
|
|||
| ObservationProxyError::WorkerNotFound(message)
|
||||
| ObservationProxyError::UpstreamDisconnect(message)
|
||||
| ObservationProxyError::MalformedFrame(message) => message,
|
||||
ObservationProxyError::ObservationOnly => {
|
||||
"backend worker event WebSocket is observation-only"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientWorkerEventWsFrame {
|
||||
pub fn event(envelope: ClientWorkerEventWsEnvelope) -> Self {
|
||||
Self::Event { envelope }
|
||||
}
|
||||
|
||||
pub fn diagnostic(error: ObservationProxyError) -> Self {
|
||||
Self::Diagnostic {
|
||||
diagnostic: ClientWorkerEventWsDiagnostic {
|
||||
code: error.code().to_string(),
|
||||
message: error.message().to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -238,15 +188,6 @@ impl BackendObservationProxy {
|
|||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn map_event(&self, event: RuntimeObservationUpstreamEvent) -> ClientWorkerEventWsEnvelope {
|
||||
ClientWorkerEventWsEnvelope {
|
||||
event_id: event.runtime_event_id,
|
||||
runtime_id: event.runtime_id,
|
||||
worker_id: event.worker_id,
|
||||
payload: event.payload,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_runtime_connect_error(error: TungsteniteError) -> ObservationProxyError {
|
||||
|
|
@ -266,24 +207,6 @@ fn map_runtime_connect_error(error: TungsteniteError) -> ObservationProxyError {
|
|||
}
|
||||
}
|
||||
|
||||
fn map_runtime_diagnostic(code: String, message: String) -> ObservationProxyError {
|
||||
match code.as_str() {
|
||||
"runtime.worker_not_found" => ObservationProxyError::WorkerNotFound(message),
|
||||
"runtime.cursor_malformed"
|
||||
| "runtime.cursor_unknown_or_expired"
|
||||
| "runtime.cursor_expired" => ObservationProxyError::RuntimeUnavailable(message),
|
||||
"runtime.unavailable" => ObservationProxyError::RuntimeUnavailable(message),
|
||||
"runtime.upstream_closed" | "runtime.websocket_error" => {
|
||||
ObservationProxyError::UpstreamDisconnect(message)
|
||||
}
|
||||
"runtime.serialize_failed" => ObservationProxyError::MalformedFrame(message),
|
||||
"runtime.observation_only" => ObservationProxyError::ObservationOnly,
|
||||
_ => ObservationProxyError::RuntimeUnavailable(format!(
|
||||
"runtime diagnostic {code}: {message}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RuntimeWsObservationClient {
|
||||
runtime_id: String,
|
||||
worker_id: String,
|
||||
|
|
@ -361,32 +284,17 @@ impl RuntimeWsObservationClient {
|
|||
));
|
||||
}
|
||||
};
|
||||
let frame: RuntimeWorkerEventWsFrame =
|
||||
serde_json::from_str(&text).map_err(|error| {
|
||||
let payload: protocol::Event = serde_json::from_str(&text).map_err(|error| {
|
||||
ObservationProxyError::MalformedFrame(format!(
|
||||
"failed to decode runtime observation frame: {error}"
|
||||
"failed to decode runtime protocol event frame: {error}"
|
||||
))
|
||||
})?;
|
||||
match frame {
|
||||
RuntimeWorkerEventWsFrame::Event { envelope } => {
|
||||
return Ok(self.map_envelope(envelope));
|
||||
}
|
||||
RuntimeWorkerEventWsFrame::Diagnostic { diagnostic } => {
|
||||
return Err(map_runtime_diagnostic(diagnostic.code, diagnostic.message));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_envelope(
|
||||
&self,
|
||||
envelope: RuntimeWorkerEventWsEnvelope,
|
||||
) -> RuntimeObservationUpstreamEvent {
|
||||
RuntimeObservationUpstreamEvent {
|
||||
return Ok(RuntimeObservationUpstreamEvent {
|
||||
runtime_id: self.runtime_id.clone(),
|
||||
worker_id: self.worker_id.clone(),
|
||||
runtime_event_id: envelope.event_id,
|
||||
payload: envelope.payload,
|
||||
runtime_event_id: "protocol".to_string(),
|
||||
payload,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -548,7 +456,8 @@ mod tests {
|
|||
RuntimeObservationSourceConfig {
|
||||
runtime_id: "remote-runtime".to_string(),
|
||||
worker_id: "worker-1".to_string(),
|
||||
endpoint: "wss://remote.example.invalid/private/workers/worker-1/events/ws".to_string(),
|
||||
endpoint: "wss://remote.example.invalid/private/workers/worker-1/protocol/ws"
|
||||
.to_string(),
|
||||
bearer_token: Some("top-secret-bearer-token".to_string()),
|
||||
}
|
||||
}
|
||||
|
|
@ -563,7 +472,7 @@ mod tests {
|
|||
assert!(debug.contains("<redacted>"));
|
||||
for forbidden in [
|
||||
"remote.example.invalid",
|
||||
"/private/workers/worker-1/events/ws",
|
||||
"/private/workers/worker-1/protocol/ws",
|
||||
"top-secret-bearer-token",
|
||||
] {
|
||||
assert!(
|
||||
|
|
@ -582,7 +491,7 @@ mod tests {
|
|||
assert!(debug.contains("source_count"));
|
||||
for forbidden in [
|
||||
"remote.example.invalid",
|
||||
"/private/workers/worker-1/events/ws",
|
||||
"/private/workers/worker-1/protocol/ws",
|
||||
"top-secret-bearer-token",
|
||||
] {
|
||||
assert!(
|
||||
|
|
|
|||
|
|
@ -41,8 +41,8 @@ use crate::hosts::{
|
|||
};
|
||||
use crate::identity::WorkspaceIdentity;
|
||||
use crate::observation::{
|
||||
BackendObservationProxy, ClientWorkerEventWsFrame, ObservationProxyError,
|
||||
RuntimeObservationClient, RuntimeObservationSourceConfig,
|
||||
BackendObservationProxy, ObservationProxyError, RuntimeObservationClient,
|
||||
RuntimeObservationSourceConfig,
|
||||
};
|
||||
use crate::profile_settings::{
|
||||
CreateWorkspaceProfileSourceRequest, DeleteWorkspaceProfileSourceRequest,
|
||||
|
|
@ -547,14 +547,6 @@ pub fn build_router(api: WorkspaceApi) -> Router {
|
|||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/cancel",
|
||||
post(scoped_cancel_runtime_worker),
|
||||
)
|
||||
.route(
|
||||
"/api/runtimes/{runtime_id}/workers/{worker_id}/events/ws",
|
||||
get(worker_observation_ws),
|
||||
)
|
||||
.route(
|
||||
"/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/events/ws",
|
||||
get(scoped_worker_observation_ws),
|
||||
)
|
||||
.route(
|
||||
"/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws",
|
||||
get(worker_protocol_ws),
|
||||
|
|
@ -2452,19 +2444,6 @@ async fn scoped_cancel_runtime_worker(
|
|||
.await
|
||||
}
|
||||
|
||||
async fn scoped_worker_observation_ws(
|
||||
ws: WebSocketUpgrade,
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
|
||||
) -> Response {
|
||||
if let Err(err) = validate_workspace_scope(&api, &path.workspace_id) {
|
||||
return err.into_response();
|
||||
}
|
||||
worker_observation_ws(State(api), AxumPath((path.runtime_id, path.worker_id)), ws)
|
||||
.await
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn scoped_worker_protocol_ws(
|
||||
ws: WebSocketUpgrade,
|
||||
State(api): State<WorkspaceApi>,
|
||||
|
|
@ -3414,23 +3393,51 @@ async fn worker_protocol_ws(
|
|||
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> impl IntoResponse {
|
||||
match api.runtime.worker(&runtime_id, &worker_id) {
|
||||
Ok(_) => ws.on_upgrade(move |socket| {
|
||||
worker_protocol_ws_session(api.runtime, runtime_id, worker_id, socket)
|
||||
}),
|
||||
Err(error) => ApiError::from(error.into_error()).into_response(),
|
||||
let source = match api.observation_proxy.source(&runtime_id, &worker_id) {
|
||||
Ok(source) => source,
|
||||
Err(ObservationProxyError::WorkerNotFound(_)) => {
|
||||
match api.runtime.observation_source(&runtime_id, &worker_id) {
|
||||
Ok(source) => source,
|
||||
Err(error) => return ApiError::from(error.into_error()).into_response(),
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
return (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(serde_json::json!({
|
||||
"error": error.code(),
|
||||
"message": error.message(),
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
ws.on_upgrade(move |socket| {
|
||||
worker_protocol_ws_session(api.runtime, source, runtime_id, worker_id, socket)
|
||||
})
|
||||
}
|
||||
|
||||
async fn worker_protocol_ws_session(
|
||||
runtime: Arc<RuntimeRegistry>,
|
||||
source: crate::observation::RuntimeObservationSource,
|
||||
runtime_id: String,
|
||||
worker_id: String,
|
||||
mut socket: WebSocket,
|
||||
) {
|
||||
while let Some(frame) = socket.next().await {
|
||||
match frame {
|
||||
Ok(WsMessage::Text(text)) => match serde_json::from_str::<protocol::Method>(&text) {
|
||||
let mut upstream = match RuntimeObservationClient::connect(&source).await {
|
||||
Ok(client) => client,
|
||||
Err(error) => {
|
||||
let event = protocol_error_event(error.message());
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
inbound = socket.next() => {
|
||||
match inbound {
|
||||
Some(Ok(WsMessage::Text(text))) => match serde_json::from_str::<protocol::Method>(&text) {
|
||||
Ok(method) => match runtime.send_protocol_method(&runtime_id, &worker_id, method) {
|
||||
Ok(events) => {
|
||||
for event in events {
|
||||
|
|
@ -3454,20 +3461,36 @@ async fn worker_protocol_ws_session(
|
|||
}
|
||||
}
|
||||
},
|
||||
Ok(WsMessage::Close(_)) => return,
|
||||
Ok(WsMessage::Ping(payload)) => {
|
||||
Some(Ok(WsMessage::Close(_))) | None => return,
|
||||
Some(Ok(WsMessage::Ping(payload))) => {
|
||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Ok(WsMessage::Pong(_)) | Ok(WsMessage::Binary(_)) => {}
|
||||
Err(error) => {
|
||||
Some(Ok(WsMessage::Pong(_))) | Some(Ok(WsMessage::Binary(_))) => {}
|
||||
Some(Err(error)) => {
|
||||
let event = protocol_error_event(format!("protocol WebSocket error: {error}"));
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
upstream_event = upstream.next_event() => {
|
||||
match upstream_event {
|
||||
Ok(event) => {
|
||||
if !send_protocol_event(&mut socket, &event.payload).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let event = protocol_error_event(error.message());
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_protocol_event(socket: &mut WebSocket, event: &protocol::Event) -> bool {
|
||||
|
|
@ -3492,116 +3515,6 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
|||
}
|
||||
}
|
||||
|
||||
async fn worker_observation_ws(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> impl IntoResponse {
|
||||
match api.observation_proxy.source(&runtime_id, &worker_id) {
|
||||
Ok(source) => ws.on_upgrade(move |socket| {
|
||||
worker_observation_ws_session(api.observation_proxy, source, socket)
|
||||
}),
|
||||
Err(ObservationProxyError::WorkerNotFound(_)) => {
|
||||
match api.runtime.observation_source(&runtime_id, &worker_id) {
|
||||
Ok(source) => ws.on_upgrade(move |socket| {
|
||||
worker_observation_ws_session(api.observation_proxy, source, socket)
|
||||
}),
|
||||
Err(error) => ApiError::from(error.into_error()).into_response(),
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let status = StatusCode::BAD_REQUEST;
|
||||
(
|
||||
status,
|
||||
Json(serde_json::json!({
|
||||
"error": error.code(),
|
||||
"message": error.message(),
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn worker_observation_ws_session(
|
||||
proxy: BackendObservationProxy,
|
||||
source: crate::observation::RuntimeObservationSource,
|
||||
mut socket: WebSocket,
|
||||
) {
|
||||
let mut upstream = match RuntimeObservationClient::connect(&source).await {
|
||||
Ok(client) => client,
|
||||
Err(error) => {
|
||||
let _ = send_client_ws_frame(&mut socket, ClientWorkerEventWsFrame::diagnostic(error))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
inbound = socket.next() => {
|
||||
match inbound {
|
||||
Some(Ok(WsMessage::Close(_))) | None => return,
|
||||
Some(Ok(WsMessage::Ping(payload))) => {
|
||||
if socket.send(WsMessage::Pong(payload)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Some(Ok(WsMessage::Pong(_))) => {}
|
||||
Some(Ok(_)) => {
|
||||
let _ = send_client_ws_frame(
|
||||
&mut socket,
|
||||
ClientWorkerEventWsFrame::diagnostic(ObservationProxyError::ObservationOnly),
|
||||
).await;
|
||||
return;
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
let _ = send_client_ws_frame(
|
||||
&mut socket,
|
||||
ClientWorkerEventWsFrame::diagnostic(
|
||||
ObservationProxyError::MalformedFrame(format!(
|
||||
"client WebSocket receive error: {error}"
|
||||
)),
|
||||
),
|
||||
).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
upstream_event = upstream.next_event() => {
|
||||
match upstream_event {
|
||||
Ok(event) => {
|
||||
let envelope = proxy.map_event(event);
|
||||
if !send_client_ws_frame(&mut socket, ClientWorkerEventWsFrame::event(envelope)).await {
|
||||
return;
|
||||
}
|
||||
},
|
||||
Err(error) => {
|
||||
let _ = send_client_ws_frame(&mut socket, ClientWorkerEventWsFrame::diagnostic(error)).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_client_ws_frame(socket: &mut WebSocket, frame: ClientWorkerEventWsFrame) -> bool {
|
||||
match serde_json::to_string(&frame) {
|
||||
Ok(text) => socket.send(WsMessage::Text(text.into())).await.is_ok(),
|
||||
Err(error) => {
|
||||
let fallback =
|
||||
ClientWorkerEventWsFrame::diagnostic(ObservationProxyError::MalformedFrame(
|
||||
format!("failed to serialize backend observation frame: {error}"),
|
||||
));
|
||||
let Ok(text) = serde_json::to_string(&fallback) else {
|
||||
return false;
|
||||
};
|
||||
socket.send(WsMessage::Text(text.into())).await.is_ok()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_host_workers(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(host_id): AxumPath<String>,
|
||||
|
|
@ -5436,7 +5349,7 @@ mod tests {
|
|||
use super::*;
|
||||
use axum::body::{Body, to_bytes};
|
||||
use axum::http::Request;
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use futures::StreamExt;
|
||||
use serde_json::{Value, json};
|
||||
use std::{fs, sync::Arc};
|
||||
use tokio_tungstenite::connect_async;
|
||||
|
|
@ -5449,7 +5362,6 @@ mod tests {
|
|||
TicketWorkerRole, WorkerInputKind, WorkerOperationState, WorkerSpawnAcceptanceRequirement,
|
||||
WorkerSpawnIntent,
|
||||
};
|
||||
use crate::observation::ClientWorkerEventWsDiagnostic;
|
||||
use crate::store::SqliteWorkspaceStore;
|
||||
|
||||
const TEST_WORKSPACE_ID: &str = "0192f0e8-4d84-7d6e-a000-000000000001";
|
||||
|
|
@ -7861,53 +7773,20 @@ mod tests {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxies_worker_observation_ws_with_backend_cursors_and_diagnostics() {
|
||||
let (runtime, worker_ref) = runtime_with_worker();
|
||||
let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let runtime_addr = runtime_listener.local_addr().unwrap();
|
||||
tokio::spawn({
|
||||
let runtime = runtime.clone();
|
||||
async move {
|
||||
worker_runtime::http_server::serve_runtime_http(runtime, runtime_listener, None)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
});
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||
let mut config = test_server_config(dir.path());
|
||||
config
|
||||
.runtime_event_sources
|
||||
.push(RuntimeObservationSourceConfig {
|
||||
async fn proxies_worker_protocol_ws_as_raw_events() {
|
||||
let (runtime, worker_ref, endpoint) = spawn_runtime_worker().await;
|
||||
let source = RuntimeObservationSourceConfig {
|
||||
runtime_id: "runtime-a".into(),
|
||||
worker_id: "worker-a".into(),
|
||||
endpoint: format!(
|
||||
"ws://{runtime_addr}/v1/workers/{}/events/ws",
|
||||
worker_ref.worker_id
|
||||
),
|
||||
endpoint,
|
||||
bearer_token: None,
|
||||
});
|
||||
let api = WorkspaceApi::new_with_execution_backend(
|
||||
config,
|
||||
Arc::new(store),
|
||||
Arc::new(DeterministicExecutionBackend::default()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let app_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let app_addr = app_listener.local_addr().unwrap();
|
||||
tokio::spawn(async move { axum::serve(app_listener, build_router(api)).await.unwrap() });
|
||||
|
||||
let url = format!("ws://{app_addr}/api/runtimes/runtime-a/workers/worker-a/events/ws");
|
||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||
let snapshot = next_client_frame(&mut stream).await;
|
||||
let ClientWorkerEventWsFrame::Event { envelope: snapshot } = snapshot else {
|
||||
panic!("expected snapshot event");
|
||||
};
|
||||
assert_eq!(snapshot.runtime_id, "runtime-a");
|
||||
assert_eq!(snapshot.worker_id, "worker-a");
|
||||
assert!(matches!(snapshot.payload, protocol::Event::Snapshot { .. }));
|
||||
let (url, _dir) = spawn_workspace_proxy(source).await;
|
||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||
assert!(matches!(
|
||||
next_client_frame(&mut stream).await,
|
||||
protocol::Event::Snapshot { .. }
|
||||
));
|
||||
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
|
|
@ -7917,59 +7796,16 @@ mod tests {
|
|||
},
|
||||
)
|
||||
.unwrap();
|
||||
let live = next_client_frame(&mut stream).await;
|
||||
let ClientWorkerEventWsFrame::Event { envelope: live } = live else {
|
||||
panic!("expected live event");
|
||||
};
|
||||
assert_eq!(live.runtime_id, "runtime-a");
|
||||
assert_eq!(live.worker_id, "worker-a");
|
||||
assert!(matches!(live.payload, protocol::Event::TextDelta { .. }));
|
||||
|
||||
let (mut fresh, _) = connect_async(&url).await.unwrap();
|
||||
let fresh_snapshot = next_client_frame(&mut fresh).await;
|
||||
assert!(matches!(
|
||||
fresh_snapshot,
|
||||
ClientWorkerEventWsFrame::Event { envelope } if matches!(envelope.payload, protocol::Event::Snapshot { .. })
|
||||
next_client_frame(&mut stream).await,
|
||||
protocol::Event::TextDelta { .. }
|
||||
));
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
&worker_ref,
|
||||
protocol::Event::TextDone {
|
||||
text: "fresh".into(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let fresh_event = next_client_frame(&mut fresh).await;
|
||||
assert!(matches!(
|
||||
fresh_event,
|
||||
ClientWorkerEventWsFrame::Event { envelope } if matches!(envelope.payload, protocol::Event::TextDone { .. })
|
||||
));
|
||||
|
||||
let (mut query_stream, _) = connect_async(format!("{url}?cursor=bad")).await.unwrap();
|
||||
let query_snapshot = next_client_frame(&mut query_stream).await;
|
||||
assert!(matches!(
|
||||
query_snapshot,
|
||||
ClientWorkerEventWsFrame::Event { envelope } if matches!(envelope.payload, protocol::Event::Snapshot { .. })
|
||||
));
|
||||
|
||||
stream.send(Message::Text("{}".into())).await.unwrap();
|
||||
let mut saw_observation_only = false;
|
||||
for _ in 0..3 {
|
||||
if let ClientWorkerEventWsFrame::Diagnostic { diagnostic } =
|
||||
next_client_frame(&mut stream).await
|
||||
{
|
||||
assert_eq!(diagnostic.code, "backend.observation_only");
|
||||
saw_observation_only = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(saw_observation_only, "expected observation-only diagnostic");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_maps_runtime_worker_not_found_http_404_to_typed_backend_diagnostic() {
|
||||
async fn proxy_maps_runtime_worker_not_found_http_404_to_protocol_error_event() {
|
||||
let (_runtime, _worker_ref, endpoint) = spawn_runtime_worker().await;
|
||||
let endpoint = endpoint.replace("/events/ws", "/missing-worker/events/ws");
|
||||
let endpoint = endpoint.replace("/protocol/ws", "/missing-worker/protocol/ws");
|
||||
let source = RuntimeObservationSourceConfig {
|
||||
runtime_id: "runtime-a".into(),
|
||||
worker_id: "worker-a".into(),
|
||||
|
|
@ -7978,30 +7814,17 @@ mod tests {
|
|||
};
|
||||
let (url, _dir) = spawn_workspace_proxy(source).await;
|
||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||
let diagnostic = next_client_diagnostic(&mut stream).await;
|
||||
assert_eq!(diagnostic.code, "backend.worker_not_found");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_reports_actual_upstream_disconnect_separately() {
|
||||
let endpoint = spawn_closing_runtime_ws().await;
|
||||
let source = RuntimeObservationSourceConfig {
|
||||
runtime_id: "runtime-a".into(),
|
||||
worker_id: "worker-a".into(),
|
||||
endpoint,
|
||||
bearer_token: None,
|
||||
};
|
||||
let (url, _dir) = spawn_workspace_proxy(source).await;
|
||||
let (mut stream, _) = connect_async(&url).await.unwrap();
|
||||
let diagnostic = next_client_diagnostic(&mut stream).await;
|
||||
assert_eq!(diagnostic.code, "backend.upstream_disconnect");
|
||||
assert!(matches!(
|
||||
next_client_frame(&mut stream).await,
|
||||
protocol::Event::Error { .. }
|
||||
));
|
||||
}
|
||||
|
||||
async fn next_client_frame(
|
||||
stream: &mut tokio_tungstenite::WebSocketStream<
|
||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||
>,
|
||||
) -> ClientWorkerEventWsFrame {
|
||||
) -> protocol::Event {
|
||||
let message = stream.next().await.unwrap().unwrap();
|
||||
let Message::Text(text) = message else {
|
||||
panic!("expected text frame");
|
||||
|
|
@ -8009,19 +7832,6 @@ mod tests {
|
|||
serde_json::from_str(&text).unwrap()
|
||||
}
|
||||
|
||||
async fn next_client_diagnostic(
|
||||
stream: &mut tokio_tungstenite::WebSocketStream<
|
||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||
>,
|
||||
) -> ClientWorkerEventWsDiagnostic {
|
||||
match next_client_frame(stream).await {
|
||||
ClientWorkerEventWsFrame::Diagnostic { diagnostic } => diagnostic,
|
||||
ClientWorkerEventWsFrame::Event { envelope } => {
|
||||
panic!("expected diagnostic, got event: {envelope:?}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_runtime_worker() -> (
|
||||
worker_runtime::Runtime,
|
||||
worker_runtime::identity::WorkerRef,
|
||||
|
|
@ -8039,7 +7849,7 @@ mod tests {
|
|||
}
|
||||
});
|
||||
let endpoint = format!(
|
||||
"ws://{runtime_addr}/v1/workers/{}/events/ws",
|
||||
"ws://{runtime_addr}/v1/workers/{}/protocol/ws",
|
||||
worker_ref.worker_id
|
||||
);
|
||||
(runtime, worker_ref, endpoint)
|
||||
|
|
@ -8065,22 +7875,10 @@ mod tests {
|
|||
let app_addr = app_listener.local_addr().unwrap();
|
||||
tokio::spawn(async move { axum::serve(app_listener, build_router(api)).await.unwrap() });
|
||||
(
|
||||
format!("ws://{app_addr}/api/runtimes/{runtime_id}/workers/{worker_id}/events/ws"),
|
||||
format!("ws://{app_addr}/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws"),
|
||||
dir,
|
||||
)
|
||||
}
|
||||
|
||||
async fn spawn_closing_runtime_ws() -> String {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = tokio_tungstenite::accept_async(stream).await.unwrap();
|
||||
let _ = websocket.close(None).await;
|
||||
});
|
||||
format!("ws://{addr}/events/ws")
|
||||
}
|
||||
|
||||
async fn get_json(app: Router, uri: &str) -> Value {
|
||||
let response = app
|
||||
.oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
|
||||
|
|
|
|||
|
|
@ -65,17 +65,17 @@ Deno.test("Worker Console uses protocol observation events without transcript fe
|
|||
);
|
||||
|
||||
assert(
|
||||
consolePage.includes("seenObservationEventIds") &&
|
||||
consolePage.includes(
|
||||
"rememberObservationEvent(frame.envelope.event_id)",
|
||||
) &&
|
||||
consolePage.includes("connectProtocolTransport") &&
|
||||
consolePage.includes("handleIncomingProtocolEvent") &&
|
||||
consolePage.includes("/protocol/ws") &&
|
||||
!consolePage.includes("seenObservationEventIds") &&
|
||||
consolePage.includes("createConsoleProjector") &&
|
||||
consolePage.includes("consoleProjector.append(eventBatch)") &&
|
||||
consolePage.includes("{#each lines as item (item.id)}") &&
|
||||
!consolePage.includes("projectConsole(observedEvents.map") &&
|
||||
!consolePage.includes("/transcript") &&
|
||||
!consolePage.includes("WorkerTranscriptProjection"),
|
||||
"Console should render protocol observation replay/live events directly and dedupe repeated frames by event id",
|
||||
"Console should render raw protocol replay/live events directly from the unified protocol WS",
|
||||
);
|
||||
});
|
||||
|
||||
|
|
@ -285,8 +285,11 @@ Deno.test("Worker Console page is routed by runtime_id and worker_id through bac
|
|||
);
|
||||
assert(
|
||||
!consolePage.includes("/transcript") &&
|
||||
consolePage.includes("/events/ws") && consolePage.includes("/input"),
|
||||
"Console should use protocol observation WS and input APIs without a transcript API",
|
||||
consolePage.includes("/protocol/ws") &&
|
||||
!consolePage.includes("/events" + "/ws") &&
|
||||
!consolePage.includes("/input") &&
|
||||
!consolePage.includes("/completions"),
|
||||
"Console should use only the protocol WS for observation and operations",
|
||||
);
|
||||
assert(
|
||||
!consolePage.includes("/api/companion"),
|
||||
|
|
|
|||
|
|
@ -245,22 +245,6 @@ export type WorkerInputResult = {
|
|||
diagnostics: Diagnostic[];
|
||||
};
|
||||
|
||||
export type ClientWorkerEventWsEnvelope = {
|
||||
event_id: string;
|
||||
runtime_id: string;
|
||||
worker_id: string;
|
||||
payload: PodProtocolEvent;
|
||||
};
|
||||
|
||||
export type ClientWorkerEventWsDiagnostic = {
|
||||
code: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type ClientWorkerEventWsFrame =
|
||||
| { kind: "event"; envelope: ClientWorkerEventWsEnvelope }
|
||||
| { kind: "diagnostic"; diagnostic: ClientWorkerEventWsDiagnostic };
|
||||
|
||||
export type ListResponse<T> = {
|
||||
workspace_id: string;
|
||||
limit: number;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@
|
|||
import type { Event as ProtocolEvent, Method as ProtocolMethod, RewindTarget, Segment } from "$lib/generated/protocol";
|
||||
import { workspaceApiPath } from "$lib/workspace/api/http";
|
||||
import type {
|
||||
ClientWorkerEventWsFrame,
|
||||
Diagnostic,
|
||||
Worker,
|
||||
PodProtocolEvent,
|
||||
|
|
@ -98,13 +97,10 @@
|
|||
let rewindHeadEntries = $state(0);
|
||||
let controlNotice = $state<string | null>(null);
|
||||
let composerNotice = $state<string | null>(null);
|
||||
let streamState = $state<"connecting" | "open" | "closed" | "error">(
|
||||
let protocolState = $state<"connecting" | "open" | "closed" | "error">(
|
||||
"connecting",
|
||||
);
|
||||
let commandState = $state<"connecting" | "open" | "closed" | "error">(
|
||||
"connecting",
|
||||
);
|
||||
let commandSocket: WebSocket | null = null;
|
||||
let protocolSocket: WebSocket | null = null;
|
||||
let pendingCompletionRequest: {
|
||||
resolve: (entries: ComposerCompletionEntry[]) => void;
|
||||
reject: (error: Error) => void;
|
||||
|
|
@ -129,8 +125,8 @@
|
|||
let consoleProjection = $state.raw<ConsoleProjection>(
|
||||
consoleProjector.snapshot(),
|
||||
);
|
||||
let seenObservationEventIds = new Set<string>();
|
||||
let pendingObservationEvents: ConsoleEventInput[] = [];
|
||||
let protocolEventSequence = 0;
|
||||
let pendingObservedStates: Array<string | null> = [];
|
||||
let pendingStreamDiagnostics: Diagnostic[] = [];
|
||||
let observationFlushHandle: number | null = null;
|
||||
|
|
@ -208,7 +204,6 @@
|
|||
consoleProjection = consoleProjector.reset();
|
||||
eventObservedAtById.clear();
|
||||
advanceEventObservedAtVersion();
|
||||
seenObservationEventIds = new Set();
|
||||
}
|
||||
|
||||
function cancelObservationFlush() {
|
||||
|
|
@ -253,23 +248,27 @@
|
|||
}
|
||||
}
|
||||
|
||||
function queueObservationEvent(
|
||||
frame: ClientWorkerEventWsFrame & { kind: "event" },
|
||||
) {
|
||||
if (!rememberObservationEvent(frame.envelope.event_id)) {
|
||||
function handleIncomingProtocolEvent(payload: ProtocolEvent) {
|
||||
handleProtocolCommandEvent(payload);
|
||||
if (payload.event === "completions" || payload.event === "error") {
|
||||
if (payload.event === "error") {
|
||||
queueObservationDiagnostic({
|
||||
code: payload.data.code,
|
||||
severity: "error",
|
||||
message: payload.data.message,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const eventId = `protocol-${++protocolEventSequence}`;
|
||||
const observedAtMs = Date.now();
|
||||
eventObservedAtById.set(frame.envelope.event_id, observedAtMs);
|
||||
const payload = frame.envelope.payload;
|
||||
eventObservedAtById.set(eventId, observedAtMs);
|
||||
pendingObservationEvents.push({
|
||||
eventId: frame.envelope.event_id,
|
||||
eventId,
|
||||
event: payload,
|
||||
observedAtMs,
|
||||
});
|
||||
if (payload.event === "rewind_targets") {
|
||||
handleProtocolCommandEvent(payload as ProtocolEvent);
|
||||
}
|
||||
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
|
||||
scheduleObservationFlush();
|
||||
}
|
||||
|
|
@ -279,14 +278,6 @@
|
|||
scheduleObservationFlush();
|
||||
}
|
||||
|
||||
function rememberObservationEvent(eventId: string): boolean {
|
||||
if (seenObservationEventIds.has(eventId)) {
|
||||
return false;
|
||||
}
|
||||
seenObservationEventIds.add(eventId);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function applyComposerCompletion(event: KeyboardEvent) {
|
||||
const target = event.currentTarget;
|
||||
if (!(target instanceof HTMLTextAreaElement)) {
|
||||
|
|
@ -502,89 +493,16 @@
|
|||
}
|
||||
}
|
||||
|
||||
function connectObservation(
|
||||
function connectProtocolTransport(
|
||||
targetWorker: Worker | null,
|
||||
token: number,
|
||||
target: ConsoleTarget,
|
||||
) {
|
||||
if (!targetWorker) {
|
||||
streamState = "closed";
|
||||
protocolState = "closed";
|
||||
return;
|
||||
}
|
||||
streamState = "connecting";
|
||||
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
|
||||
const wsPath = workerApiPath(
|
||||
`/runtimes/${encodeURIComponent(target.runtimeId)}/workers/${encodeURIComponent(
|
||||
target.workerId,
|
||||
)}/events/ws`,
|
||||
);
|
||||
const ws = new WebSocket(
|
||||
`${protocol}//${window.location.host}${wsPath}`,
|
||||
);
|
||||
|
||||
ws.onopen = () => {
|
||||
if (token === reloadToken) {
|
||||
streamState = "open";
|
||||
}
|
||||
};
|
||||
ws.onmessage = (message) => {
|
||||
if (token !== reloadToken) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const frame = JSON.parse(
|
||||
String(message.data),
|
||||
) as ClientWorkerEventWsFrame;
|
||||
if (frame.kind === "event") {
|
||||
queueObservationEvent(frame);
|
||||
} else {
|
||||
queueObservationDiagnostic({
|
||||
code: frame.diagnostic.code,
|
||||
severity: "warning",
|
||||
message: frame.diagnostic.message,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
queueObservationDiagnostic({
|
||||
code: "worker_observation_frame_invalid",
|
||||
severity: "warning",
|
||||
message:
|
||||
error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
};
|
||||
ws.onerror = () => {
|
||||
if (token === reloadToken) {
|
||||
streamState = "error";
|
||||
streamDiagnostics = [
|
||||
...streamDiagnostics,
|
||||
{
|
||||
code: "worker_observation_ws_error",
|
||||
severity: "error",
|
||||
message: "Worker observation WebSocket failed.",
|
||||
},
|
||||
];
|
||||
}
|
||||
};
|
||||
ws.onclose = () => {
|
||||
if (token === reloadToken && streamState !== "error") {
|
||||
streamState = "closed";
|
||||
}
|
||||
};
|
||||
|
||||
return () => ws.close();
|
||||
}
|
||||
|
||||
function connectProtocolCommands(
|
||||
targetWorker: Worker | null,
|
||||
token: number,
|
||||
target: ConsoleTarget,
|
||||
) {
|
||||
if (!targetWorker) {
|
||||
commandState = "closed";
|
||||
return;
|
||||
}
|
||||
commandState = "connecting";
|
||||
protocolState = "connecting";
|
||||
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
|
||||
const wsPath = workerApiPath(
|
||||
`/runtimes/${encodeURIComponent(target.runtimeId)}/workers/${encodeURIComponent(
|
||||
|
|
@ -594,11 +512,11 @@
|
|||
const ws = new WebSocket(
|
||||
`${protocol}//${window.location.host}${wsPath}`,
|
||||
);
|
||||
commandSocket = ws;
|
||||
protocolSocket = ws;
|
||||
|
||||
ws.onopen = () => {
|
||||
if (token === reloadToken) {
|
||||
commandState = "open";
|
||||
protocolState = "open";
|
||||
}
|
||||
};
|
||||
ws.onmessage = (message) => {
|
||||
|
|
@ -606,14 +524,14 @@
|
|||
return;
|
||||
}
|
||||
try {
|
||||
handleProtocolCommandEvent(
|
||||
handleIncomingProtocolEvent(
|
||||
JSON.parse(String(message.data)) as ProtocolEvent,
|
||||
);
|
||||
} catch (error) {
|
||||
streamDiagnostics = [
|
||||
...streamDiagnostics,
|
||||
{
|
||||
code: "worker_protocol_command_frame_invalid",
|
||||
code: "worker_protocol_frame_invalid",
|
||||
severity: "warning",
|
||||
message:
|
||||
error instanceof Error ? error.message : String(error),
|
||||
|
|
@ -623,42 +541,42 @@
|
|||
};
|
||||
ws.onerror = () => {
|
||||
if (token === reloadToken) {
|
||||
commandState = "error";
|
||||
protocolState = "error";
|
||||
streamDiagnostics = [
|
||||
...streamDiagnostics,
|
||||
{
|
||||
code: "worker_protocol_command_ws_error",
|
||||
code: "worker_protocol_ws_error",
|
||||
severity: "error",
|
||||
message: "Worker protocol command WebSocket failed.",
|
||||
message: "Worker protocol WebSocket failed.",
|
||||
},
|
||||
];
|
||||
}
|
||||
};
|
||||
ws.onclose = () => {
|
||||
if (commandSocket === ws) {
|
||||
commandSocket = null;
|
||||
if (protocolSocket === ws) {
|
||||
protocolSocket = null;
|
||||
}
|
||||
if (token === reloadToken && commandState !== "error") {
|
||||
commandState = "closed";
|
||||
if (token === reloadToken && protocolState !== "error") {
|
||||
protocolState = "closed";
|
||||
}
|
||||
rejectPendingCompletion(
|
||||
new Error("Worker protocol command WebSocket closed."),
|
||||
new Error("Worker protocol WebSocket closed."),
|
||||
);
|
||||
};
|
||||
|
||||
return () => {
|
||||
if (commandSocket === ws) {
|
||||
commandSocket = null;
|
||||
if (protocolSocket === ws) {
|
||||
protocolSocket = null;
|
||||
}
|
||||
ws.close();
|
||||
};
|
||||
}
|
||||
|
||||
function sendProtocolMethod(method: ProtocolMethod) {
|
||||
if (!commandSocket || commandSocket.readyState !== WebSocket.OPEN) {
|
||||
throw new Error("Worker protocol command WebSocket is not open.");
|
||||
if (!protocolSocket || protocolSocket.readyState !== WebSocket.OPEN) {
|
||||
throw new Error("Worker protocol WebSocket is not open.");
|
||||
}
|
||||
commandSocket.send(JSON.stringify(method));
|
||||
protocolSocket.send(JSON.stringify(method));
|
||||
}
|
||||
|
||||
function handleProtocolCommandEvent(event: ProtocolEvent) {
|
||||
|
|
@ -1162,8 +1080,7 @@
|
|||
void loadConsoleData(target);
|
||||
});
|
||||
|
||||
$effect(() => connectObservation(worker, reloadToken, consoleTarget));
|
||||
$effect(() => connectProtocolCommands(worker, reloadToken, consoleTarget));
|
||||
$effect(() => connectProtocolTransport(worker, reloadToken, consoleTarget));
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
|
|
@ -1182,14 +1099,14 @@
|
|||
<div class="console-header-actions">
|
||||
<div
|
||||
class="console-status-pill"
|
||||
class:warn={streamState !== "open"}
|
||||
class:warn={protocolState !== "open"}
|
||||
>
|
||||
{workerState} · stream {streamState} · command {commandState}
|
||||
{workerState} · protocol {protocolState}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
class="secondary-button"
|
||||
disabled={commandState !== "open"}
|
||||
disabled={protocolState !== "open"}
|
||||
onclick={() => sendControl({ method: "cancel" }, "Cancel")}
|
||||
>
|
||||
Cancel
|
||||
|
|
@ -1197,7 +1114,7 @@
|
|||
<button
|
||||
type="button"
|
||||
class="secondary-button"
|
||||
disabled={commandState !== "open"}
|
||||
disabled={protocolState !== "open"}
|
||||
onclick={() => sendControl({ method: "pause" }, "Pause")}
|
||||
>
|
||||
Pause
|
||||
|
|
@ -1205,7 +1122,7 @@
|
|||
<button
|
||||
type="button"
|
||||
class="secondary-button"
|
||||
disabled={commandState !== "open"}
|
||||
disabled={protocolState !== "open"}
|
||||
onclick={() => sendControl({ method: "resume" }, "Resume")}
|
||||
>
|
||||
Resume
|
||||
|
|
@ -1213,7 +1130,7 @@
|
|||
<button
|
||||
type="button"
|
||||
class="secondary-button"
|
||||
disabled={commandState !== "open"}
|
||||
disabled={protocolState !== "open"}
|
||||
onclick={() => sendControl({ method: "compact" }, "Compact")}
|
||||
>
|
||||
Compact
|
||||
|
|
@ -1221,7 +1138,7 @@
|
|||
<button
|
||||
type="button"
|
||||
class="secondary-button"
|
||||
disabled={commandState !== "open"}
|
||||
disabled={protocolState !== "open"}
|
||||
onclick={requestRewindTargets}
|
||||
>
|
||||
Rewind
|
||||
|
|
@ -1249,7 +1166,7 @@
|
|||
<button
|
||||
type="button"
|
||||
class="secondary-button"
|
||||
disabled={commandState !== "open" || !target.eligible}
|
||||
disabled={protocolState !== "open" || !target.eligible}
|
||||
title={target.disabled_reason ?? target.warning ?? undefined}
|
||||
onclick={() => rewindTo(target)}
|
||||
>
|
||||
|
|
@ -1377,7 +1294,7 @@
|
|||
{#if diagnostics.length > 0}
|
||||
<details
|
||||
class="metadata-details"
|
||||
open={streamState === "error"}
|
||||
open={protocolState === "error"}
|
||||
>
|
||||
<summary>Diagnostics ({diagnostics.length})</summary>
|
||||
<ul>
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user