Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: live reload runtime connections

Browse Source
This commit is contained in:
Hare
2026-07-04 04:47:23 +09:00
parent 296da3a440
commit b3dac8ea68
19 changed files with 1040 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/Cargo.toml
+1 -1
View File
@@ -23,7 +23,7 @@ thiserror.workspace = true
ticket.workspace = true
tokio = { workspace = true, features = ["fs", "macros", "net", "rt-multi-thread", "sync"] }
tokio-tungstenite.workspace = true
worker = { workspace = true, features = ["runtime-adapter"] }
worker.workspace = true
worker-runtime = { workspace = true, features = ["ws-server", "fs-store"] }
toml.workspace = true
tracing.workspace = true
crates/workspace-server/src/config.rs
+3 -1
View File
@@ -299,7 +299,9 @@ impl ResolvedWorkspaceBackendConfig {
}
}
fn resolve_remote_runtime(config: &RemoteRuntimeConfigFile) -> Result<RemoteRuntimeConfig> {
pub(crate) fn resolve_remote_runtime(
config: &RemoteRuntimeConfigFile,
) -> Result<RemoteRuntimeConfig> {
if let Some(token_ref) = config.token_ref.as_deref() {
return Err(Error::Config(format!(
"remote runtime `{}` uses token_ref `{token_ref}`, but secret ref resolution is not implemented for workspace backend config yet",
crates/workspace-server/src/hosts.rs
+173 -62
View File
@@ -6,7 +6,11 @@ use reqwest::header::{AUTHORIZATION, CONTENT_TYPE};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{path::PathBuf, sync::Arc, time::Duration};
use std::{
path::PathBuf,
sync::{Arc, RwLock},
time::Duration,
};
use worker_runtime::catalog::{
ConfigBundleRef, CreateWorkerRequest, ProfileSelector, WorkerDetail as EmbeddedWorkerDetail,
WorkerStatus as EmbeddedWorkerStatus,
@@ -45,6 +49,22 @@ const MAX_HOST_SCAN: usize = 256;
const MAX_IDENTIFIER_LEN: usize = 120;
const ID_DIGEST_HEX_LEN: usize = 16;
fn run_blocking_http<T, F>(operation: F) -> T
where
T: Send + 'static,
F: FnOnce() -> T + Send + 'static,
{
match tokio::runtime::Handle::try_current() {
Ok(handle) if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => {
tokio::task::block_in_place(operation)
}
Ok(_) => std::thread::spawn(operation)
.join()
.expect("blocking HTTP thread panicked"),
Err(_) => operation(),
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RuntimeDiagnostic {
pub code: String,
@@ -643,31 +663,95 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
}
}
#[derive(Debug, Clone)]
pub enum RuntimeRegistryUnregisterResult {
Removed,
NotFound,
BlockedByWorkers {
worker_count: usize,
diagnostics: Vec<RuntimeDiagnostic>,
},
}
#[derive(Clone)]
pub struct RuntimeRegistry {
runtimes: Vec<Arc<dyn WorkspaceWorkerRuntime>>,
runtimes: Arc<RwLock<Vec<Arc<dyn WorkspaceWorkerRuntime>>>>,
}
impl RuntimeRegistry {
pub fn new(runtimes: Vec<Arc<dyn WorkspaceWorkerRuntime>>) -> Self {
Self { runtimes }
Self {
runtimes: Arc::new(RwLock::new(runtimes)),
}
}
pub fn for_workspace(embedded_runtime: EmbeddedWorkerRuntime) -> Self {
Self::new(vec![Arc::new(embedded_runtime)])
}
pub fn register<R>(&mut self, runtime: R)
pub fn register<R>(&self, runtime: R)
where
R: WorkspaceWorkerRuntime + 'static,
{
self.runtimes.push(Arc::new(runtime));
self.runtimes
.write()
.expect("runtime registry lock poisoned")
.push(Arc::new(runtime));
}
pub fn register_or_replace<R>(&self, runtime: R)
where
R: WorkspaceWorkerRuntime + 'static,
{
let runtime = Arc::new(runtime);
let runtime_id = runtime.runtime_id().to_string();
let mut runtimes = self
.runtimes
.write()
.expect("runtime registry lock poisoned");
runtimes.retain(|existing| existing.runtime_id() != runtime_id);
runtimes.push(runtime);
}
pub fn unregister_if_idle(
&self,
runtime_id: &str,
worker_scan_limit: usize,
) -> Result<RuntimeRegistryUnregisterResult, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
let runtime = self
.runtimes_snapshot()
.into_iter()
.find(|runtime| runtime.runtime_id() == runtime_id);
if let Some(runtime) = runtime {
let worker_list = runtime.list_workers(worker_scan_limit);
if !worker_list.items.is_empty() {
return Ok(RuntimeRegistryUnregisterResult::BlockedByWorkers {
worker_count: worker_list.items.len(),
diagnostics: worker_list.diagnostics,
});
}
} else {
return Ok(RuntimeRegistryUnregisterResult::NotFound);
}
let mut runtimes = self
.runtimes
.write()
.expect("runtime registry lock poisoned");
let before = runtimes.len();
runtimes.retain(|runtime| runtime.runtime_id() != runtime_id);
if runtimes.len() == before {
Ok(RuntimeRegistryUnregisterResult::NotFound)
} else {
Ok(RuntimeRegistryUnregisterResult::Removed)
}
}
pub fn list_runtimes(&self, limit: usize) -> RuntimeList<RuntimeSummary> {
let mut diagnostics = Vec::new();
let mut items = Vec::new();
for runtime in self.runtimes.iter().take(limit) {
for runtime in self.runtimes_snapshot().iter().take(limit) {
let summary = runtime.runtime_summary(limit);
diagnostics.extend(summary.diagnostics.iter().cloned());
items.push(summary);
@@ -679,7 +763,7 @@ impl RuntimeRegistry {
pub fn list_hosts(&self, limit: usize) -> RuntimeList<HostSummary> {
let mut items = Vec::new();
let mut diagnostics = Vec::new();
for runtime in &self.runtimes {
for runtime in self.runtimes_snapshot() {
if items.len() >= limit {
break;
}
@@ -694,7 +778,7 @@ impl RuntimeRegistry {
pub fn list_workers(&self, limit: usize) -> RuntimeList<WorkerSummary> {
let mut items = Vec::new();
let mut diagnostics = Vec::new();
for runtime in &self.runtimes {
for runtime in self.runtimes_snapshot() {
if items.len() >= limit {
break;
}
@@ -716,7 +800,7 @@ impl RuntimeRegistry {
let mut host_found = false;
let mut diagnostics = Vec::new();
let mut items = Vec::new();
for runtime in &self.runtimes {
for runtime in self.runtimes_snapshot() {
let host_list = runtime.list_hosts(MAX_HOST_SCAN);
diagnostics.extend(host_list.diagnostics);
if !host_list.items.iter().any(|host| host.host_id == host_id) {
@@ -894,13 +978,23 @@ impl RuntimeRegistry {
})
}
fn runtimes_snapshot(&self) -> Vec<Arc<dyn WorkspaceWorkerRuntime>> {
self.runtimes
.read()
.expect("runtime registry lock poisoned")
.clone()
}
fn runtime(
&self,
runtime_id: &str,
) -> Result<&Arc<dyn WorkspaceWorkerRuntime>, RuntimeRegistryError> {
) -> Result<Arc<dyn WorkspaceWorkerRuntime>, RuntimeRegistryError> {
self.runtimes
.read()
.expect("runtime registry lock poisoned")
.iter()
.find(|runtime| runtime.runtime_id() == runtime_id)
.cloned()
.ok_or_else(|| RuntimeRegistryError::UnknownRuntime(runtime_id.to_string()))
}
}
@@ -1548,7 +1642,7 @@ impl RemoteRuntimeConfig {
display_name: display_name.into(),
base_url: base_url.into(),
bearer_token,
cached_capabilities: remote_runtime_capabilities(200, false),
cached_capabilities: remote_runtime_capabilities(200, false, false),
cached_status: "configured".to_string(),
timeout: Duration::from_secs(10),
}
@@ -1586,14 +1680,14 @@ impl RemoteWorkerRuntime {
pub fn new(config: RemoteRuntimeConfig) -> Result<Self, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", &config.runtime_id)?;
let base_url = config.base_url.trim_end_matches('/').to_string();
let http = BlockingHttpClient::builder()
.timeout(config.timeout)
.build()
.map_err(|err| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: config.runtime_id.clone(),
code: "remote_runtime_client_build_failed".to_string(),
message: err.to_string(),
})?;
let timeout = config.timeout;
let http =
run_blocking_http(move || BlockingHttpClient::builder().timeout(timeout).build())
.map_err(|err| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: config.runtime_id.clone(),
code: "remote_runtime_client_build_failed".to_string(),
message: err.to_string(),
})?;
Ok(Self {
host_id: host_id_for_remote_runtime(&config.runtime_id),
runtime_id: config.runtime_id,
@@ -1628,18 +1722,9 @@ impl RemoteWorkerRuntime {
format!("{base}/v1/workers/{worker_id}/events/ws")
}
fn authorize(&self, request: RequestBuilder) -> RequestBuilder {
let request = request.header(CONTENT_TYPE, "application/json");
if let Some(token) = self.bearer_token.as_deref() {
request.header(AUTHORIZATION, format!("Bearer {token}"))
} else {
request
}
}
fn get_json<T>(&self, path: &str) -> Result<T, RuntimeDiagnostic>
where
T: DeserializeOwned,
T: DeserializeOwned + Send + 'static,
{
self.send_json(self.http.get(self.endpoint(path)))
}
@@ -1647,38 +1732,43 @@ impl RemoteWorkerRuntime {
fn post_json<B, T>(&self, path: &str, body: &B) -> Result<T, RuntimeDiagnostic>
where
B: Serialize + ?Sized,
T: DeserializeOwned,
T: DeserializeOwned + Send + 'static,
{
self.send_json(self.http.post(self.endpoint(path)).json(body))
}
fn send_json<T>(&self, request: RequestBuilder) -> Result<T, RuntimeDiagnostic>
where
T: DeserializeOwned,
T: DeserializeOwned + Send + 'static,
{
let response = self
.authorize(request)
.send()
.map_err(|err| remote_reqwest_diagnostic(&self.runtime_id, err))?;
let status = response.status();
if status.is_success() {
response.json::<T>().map_err(|err| {
diagnostic(
"remote_runtime_malformed_response",
DiagnosticSeverity::Error,
format!(
"Remote Runtime returned malformed JSON for '{}': {err}",
self.runtime_id
),
)
})
} else {
Err(remote_http_status_diagnostic(
&self.runtime_id,
status,
response,
))
}
let runtime_id = self.runtime_id.clone();
let bearer_token = self.bearer_token.clone();
run_blocking_http(move || {
let request = request.header(CONTENT_TYPE, "application/json");
let request = if let Some(token) = bearer_token.as_deref() {
request.header(AUTHORIZATION, format!("Bearer {token}"))
} else {
request
};
let response = request
.send()
.map_err(|err| remote_reqwest_diagnostic(&runtime_id, err))?;
let status = response.status();
if status.is_success() {
response.json::<T>().map_err(|err| {
diagnostic(
"remote_runtime_malformed_response",
DiagnosticSeverity::Error,
format!(
"Remote Runtime returned malformed JSON for '{}': {err}",
runtime_id
),
)
})
} else {
Err(remote_http_status_diagnostic(&runtime_id, status, response))
}
})
}
fn map_worker_summary(&self, summary: worker_runtime::catalog::WorkerSummary) -> WorkerSummary {
@@ -1790,7 +1880,11 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
} else {
vec![self.host_id.clone()]
},
capabilities: remote_runtime_capabilities(limit, true),
capabilities: remote_runtime_capabilities(
limit,
true,
response.runtime.worker_creation_available,
),
diagnostics: vec![diagnostic(
"remote_runtime_backend_proxy",
DiagnosticSeverity::Info,
@@ -1827,7 +1921,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
status: "configured".to_string(),
observed_at: Utc::now().to_rfc3339(),
last_seen_at: None,
capabilities: remote_runtime_capabilities(limit, true),
capabilities: remote_runtime_capabilities(limit, true, false),
diagnostics: vec![diagnostic(
"remote_runtime_backend_proxy",
DiagnosticSeverity::Info,
@@ -2489,14 +2583,18 @@ fn percent_encode(input: &str, keep: impl Fn(u8) -> bool) -> String {
encoded
}
fn remote_runtime_capabilities(limit: usize, available: bool) -> RuntimeCapabilitySummary {
fn remote_runtime_capabilities(
limit: usize,
available: bool,
worker_creation_available: bool,
) -> RuntimeCapabilitySummary {
RuntimeCapabilitySummary {
can_list_hosts: true,
can_list_workers: available,
can_get_worker: available,
can_spawn_worker: available,
can_spawn_worker: available && worker_creation_available,
can_stop_worker: available,
can_accept_input: available,
can_accept_input: available && worker_creation_available,
has_workspace_fs: false,
has_shell: false,
has_git: false,
@@ -3327,6 +3425,19 @@ mod tests {
);
}
#[tokio::test]
async fn remote_runtime_client_can_initialize_inside_tokio_context() {
let runtime = RemoteWorkerRuntime::new(RemoteRuntimeConfig::new(
"remote:async-init",
"Remote Async Init",
"http://127.0.0.1:9",
None,
))
.unwrap();
assert_eq!(runtime.runtime_id(), "remote:async-init");
}
#[test]
fn remote_runtime_registry_routes_commands_without_browser_secret_leaks() {
let worker_json = worker_json("remote:primary", "worker-remote-1");
@@ -3362,7 +3473,7 @@ mod tests {
),
]);
let secret = "secret-token-do-not-leak".to_string();
let mut registry = RuntimeRegistry::new(Vec::new());
let registry = RuntimeRegistry::new(Vec::new());
registry.register(
RemoteWorkerRuntime::new(RemoteRuntimeConfig::new(
"remote:primary",
@@ -3559,7 +3670,7 @@ mod tests {
.to_string(),
),
]);
let mut registry = RuntimeRegistry::new(Vec::new());
let registry = RuntimeRegistry::new(Vec::new());
registry.register(
RemoteWorkerRuntime::new(RemoteRuntimeConfig::new(
"remote:primary",
@@ -3605,7 +3716,7 @@ mod tests {
401,
json!({ "error": { "code": "unauthorized", "message": "bad token" } }).to_string(),
)]);
let mut registry = RuntimeRegistry::new(Vec::new());
let registry = RuntimeRegistry::new(Vec::new());
registry.register(
RemoteWorkerRuntime::new(RemoteRuntimeConfig::new(
"remote:primary",
crates/workspace-server/src/server.rs
+299 -48
View File
@@ -12,20 +12,20 @@ use chrono::{SecondsFormat, Utc};
use futures::StreamExt;
use serde::{Deserialize, Serialize};
use tokio::net::TcpListener;
use worker::runtime_adapter::WorkerRuntimeExecutionBackend;
use worker_runtime::worker_backend::WorkerRuntimeExecutionBackend;
use crate::companion::{
CompanionCancelRequest, CompanionConsole, CompanionMessageRequest, CompanionMessageResponse,
CompanionStatusResponse, CompanionTranscriptProjection,
};
use crate::config::{RemoteRuntimeConfigFile, WorkspaceBackendConfigFile};
use crate::config::{RemoteRuntimeConfigFile, WorkspaceBackendConfigFile, resolve_remote_runtime};
use crate::hosts::{
ConfigBundleCheckResult, ConfigBundleSyncResult, DiagnosticSeverity, EmbeddedWorkerRuntime,
HostSummary, RemoteRuntimeConfig, RemoteWorkerRuntime, RuntimeDiagnostic, RuntimeRegistry,
RuntimeSummary, WorkerInputRequest, WorkerInputResult, WorkerLifecycleRequest,
WorkerLifecycleResult, WorkerOperationState, WorkerSpawnAcceptanceRequirement,
WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult, WorkerSummary,
WorkerTranscriptProjection,
RuntimeRegistryUnregisterResult, RuntimeSummary, WorkerInputRequest, WorkerInputResult,
WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState,
WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult,
WorkerSummary, WorkerTranscriptProjection,
};
use crate::identity::WorkspaceIdentity;
use crate::observation::{
@@ -171,7 +171,7 @@ impl WorkspaceApi {
updated_at: config.workspace_created_at.clone(),
})
.await?;
let mut runtime = RuntimeRegistry::for_workspace(
let runtime = RuntimeRegistry::for_workspace(
EmbeddedWorkerRuntime::new_fs_store_with_execution_backend(
config.workspace_id.clone(),
config.embedded_runtime_store_root.clone(),
@@ -782,7 +782,7 @@ async fn add_remote_runtime_connection(
"a remote Runtime connection with that id is already configured",
));
}
local_config.runtimes.remote.push(RemoteRuntimeConfigFile {
let remote_config = RemoteRuntimeConfigFile {
id,
endpoint: request.endpoint.trim().to_string(),
display_name: request
@@ -792,9 +792,30 @@ async fn add_remote_runtime_connection(
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
token_ref: None,
});
};
let active_config = remote_runtime_config_from_file(&remote_config).map_err(|diagnostic| {
ApiError::with_diagnostics(
Error::RuntimeOperationFailed {
runtime_id: remote_config.id.clone(),
code: diagnostic.code.clone(),
message: diagnostic.message.clone(),
},
vec![diagnostic],
)
})?;
let active_runtime = RemoteWorkerRuntime::new(active_config).map_err(|err| err.into_error())?;
local_config.runtimes.remote.push(remote_config);
write_workspace_backend_config_for_settings(&api, &local_config)?;
let mut response = runtime_connection_mutation_response(&api, &local_config);
api.runtime.register_or_replace(active_runtime);
let mut response = runtime_connection_mutation_response(
&api,
&local_config,
vec![settings_diagnostic(
"runtime_registry_applied",
DiagnosticSeverity::Info,
"Remote Runtime config was persisted and applied to the active Runtime registry without restarting the Workspace backend.",
)],
);
response.diagnostics.push(settings_diagnostic(
"workspace_backend_config_rewritten",
DiagnosticSeverity::Info,
@@ -822,8 +843,44 @@ async fn delete_remote_runtime_connection(
if before == local_config.runtimes.remote.len() {
return Err(Error::UnknownRuntime(runtime_id).into());
}
match api
.runtime
.unregister_if_idle(&runtime_id, api.config.max_records.min(200))
.map_err(|err| err.into_error())?
{
RuntimeRegistryUnregisterResult::Removed | RuntimeRegistryUnregisterResult::NotFound => {}
RuntimeRegistryUnregisterResult::BlockedByWorkers {
worker_count,
diagnostics,
} => {
let mut diagnostics = diagnostics;
diagnostics.push(settings_diagnostic(
"remote_runtime_delete_blocked",
DiagnosticSeverity::Error,
format!(
"Remote Runtime '{runtime_id}' has {worker_count} active worker(s); stop or move them before deleting the connection."
),
));
return Err(ApiError::with_diagnostics(
Error::RuntimeOperationFailed {
runtime_id,
code: "remote_runtime_delete_blocked".to_string(),
message: "Remote Runtime connection has active workers".to_string(),
},
diagnostics,
));
}
}
write_workspace_backend_config_for_settings(&api, &local_config)?;
let mut response = runtime_connection_mutation_response(&api, &local_config);
let mut response = runtime_connection_mutation_response(
&api,
&local_config,
vec![settings_diagnostic(
"runtime_registry_applied",
DiagnosticSeverity::Info,
"Remote Runtime config was removed from persisted config and the active Runtime registry without restarting the Workspace backend.",
)],
);
response.diagnostics.push(settings_diagnostic(
"workspace_backend_config_rewritten",
DiagnosticSeverity::Info,
@@ -1277,16 +1334,13 @@ fn runtime_connection_settings_response(
fn runtime_connection_mutation_response(
api: &WorkspaceApi,
local_config: &WorkspaceBackendConfigFile,
diagnostics: Vec<RuntimeDiagnostic>,
) -> RuntimeConnectionMutationResponse {
RuntimeConnectionMutationResponse {
workspace_id: api.config.workspace_id.clone(),
restart_required: true,
remotes: remote_runtime_connection_summaries(api, local_config, true),
diagnostics: vec![settings_diagnostic(
"runtime_registry_restart_required",
DiagnosticSeverity::Warning,
"Runtime connection config changed; restart the Workspace backend for the live Runtime registry to use the new config.",
)],
restart_required: false,
remotes: remote_runtime_connection_summaries(api, local_config, false),
diagnostics,
}
}
@@ -1443,6 +1497,18 @@ fn validate_public_runtime_id(runtime_id: &str) -> ApiResult<()> {
Ok(())
}
fn remote_runtime_config_from_file(
remote: &RemoteRuntimeConfigFile,
) -> std::result::Result<RemoteRuntimeConfig, RuntimeDiagnostic> {
resolve_remote_runtime(remote).map_err(|err| {
settings_diagnostic(
"remote_runtime_apply_failed",
DiagnosticSeverity::Error,
err.to_string(),
)
})
}
async fn test_remote_runtime_config(
api: &WorkspaceApi,
remote: &RemoteRuntimeConfigFile,
@@ -1529,7 +1595,10 @@ async fn test_remote_runtime_config(
);
}
};
observation.available("runtime.summary", "Runtime summary was readable.");
observation.available(
"runtime.summary",
"Connected: /v1/runtime responded with a recognized worker-runtime summary.",
);
let workers_url = match remote_probe_url(remote, "/v1/workers") {
Ok(url) => url,
@@ -1544,7 +1613,10 @@ async fn test_remote_runtime_config(
match probe_remote_json(&client, workers_url, "workers.list", "Worker list").await {
Ok(payload) => match serde_json::from_value::<RuntimeHttpWorkersResponse>(payload) {
Ok(workers) => {
observation.available("workers.list", "Worker list was readable.");
observation.available(
"workers.list",
"Verified: /v1/workers responded with a recognized worker list.",
);
Some(workers)
}
Err(_) => {
@@ -1574,7 +1646,10 @@ async fn test_remote_runtime_config(
match remote_probe_url(remote, &path) {
Ok(url) => match probe_remote_json(&client, url, "workers.detail", "Worker detail").await {
Ok(payload) => match serde_json::from_value::<RuntimeHttpWorkerResponse>(payload) {
Ok(_) => observation.available("workers.detail", "Worker detail was readable."),
Ok(_) => observation.available(
"workers.detail",
"Verified: worker detail responded for an existing worker reported by the remote Runtime.",
),
Err(_) => observation.incompatible(
"workers.detail",
settings_diagnostic(
@@ -1591,13 +1666,13 @@ async fn test_remote_runtime_config(
} else {
observation.unknown(
"workers.detail",
"Worker detail compatibility could not be proven because the Runtime reported no workers during the lightweight probe.",
"No connection problem found. Worker detail was not checked because the remote Runtime reported no workers during the lightweight probe.",
);
}
observation.available(
"workers.events_ws.construct",
"Worker event websocket URL can be constructed from the configured HTTP(S) Runtime endpoint, but no websocket connection was opened during this lightweight test.",
"Verified: worker event websocket URL can be constructed from the configured HTTP(S) Runtime endpoint. The lightweight test does not open a websocket stream.",
);
let bundles_url = match remote_probe_url(remote, "/v1/config-bundles") {
@@ -1621,8 +1696,10 @@ async fn test_remote_runtime_config(
Ok(payload) => {
match serde_json::from_value::<RuntimeHttpConfigBundlesResponse>(payload) {
Ok(bundles) => {
observation
.available("config_bundles.list", "Config-bundle list was readable.");
observation.available(
"config_bundles.list",
"Verified: /v1/config-bundles responded with a recognized config-bundle list.",
);
Some(bundles)
}
Err(_) => {
@@ -1665,7 +1742,7 @@ async fn test_remote_runtime_config(
{
Ok(_) => observation.available(
"config_bundles.availability",
"Config-bundle availability was readable for an advertised bundle.",
"Verified: config-bundle availability was confirmed for an advertised bundle.",
),
Err(_) => observation.incompatible(
"config_bundles.availability",
@@ -1686,21 +1763,32 @@ async fn test_remote_runtime_config(
} else {
observation.unknown(
"config_bundles.availability",
"Config-bundle availability compatibility could not be proven because the Runtime advertised no bundles during the lightweight probe.",
"No connection problem found. Config-bundle availability was not checked because the remote Runtime advertised no bundles during the lightweight probe.",
);
}
observation.unknown(
"workers.spawn",
"Worker spawn compatibility was not proven because the lightweight test does not create remote workers as a side effect.",
);
if summary.runtime.worker_creation_available {
observation.available(
"workers.spawn",
"Verified: /v1/runtime reports worker creation is enabled by a Runtime execution backend. The lightweight test does not create a worker.",
);
} else {
observation.incompatible(
"workers.spawn",
settings_diagnostic(
"remote_runtime_worker_creation_unavailable",
DiagnosticSeverity::Error,
"Connected to the Runtime, but worker creation is unavailable because this Runtime process has no execution backend attached.",
),
);
}
observation.unknown(
"workers.input_dispatch",
"Worker input dispatch compatibility was not proven because the lightweight test does not send model-visible input as a side effect.",
"No connection problem found. Worker input dispatch was not checked because this lightweight test does not send model-visible input as a side effect.",
);
observation.unknown(
"config_bundles.sync",
"Config-bundle sync compatibility was not proven because the lightweight test does not upload bundles as a side effect.",
"No connection problem found. Config-bundle sync was not checked because this lightweight test does not upload bundles as a side effect.",
);
RemoteRuntimeTestResponse {
@@ -1709,11 +1797,14 @@ async fn test_remote_runtime_config(
checked_at,
state: observation.state().to_string(),
protocol_version,
compatibility_basis: "Observed worker-runtime HTTP endpoints for summary, worker listing/detail when available, event websocket URL construction, and config-bundle listing/availability when available; side-effecting spawn/input/sync operations are reported unknown unless the Runtime API proves them.".to_string(),
compatibility_basis: "Connected to /v1/runtime and verified non-side-effecting worker-runtime HTTP endpoints. No incompatible operation was found; warning items below are unproven optional or side-effecting checks, not connection failures.".to_string(),
capabilities: observation.capabilities,
health_result: format!(
"runtime_status={:?}; incompatible={}; unknown={}",
summary.runtime.status, observation.incompatible_count, observation.unknown_count
"connected=true; runtime_status={:?}; available={}; incompatible={}; warnings={}",
summary.runtime.status,
observation.available_count,
observation.incompatible_count,
observation.unknown_count
),
diagnostics: observation.diagnostics,
}
@@ -1747,13 +1838,20 @@ fn remote_runtime_test_failed(
struct RuntimeCompatibilityObservation {
capabilities: Vec<String>,
diagnostics: Vec<RuntimeDiagnostic>,
available_count: usize,
incompatible_count: usize,
unknown_count: usize,
}
impl RuntimeCompatibilityObservation {
fn available(&mut self, operation: &str, _message: &str) {
fn available(&mut self, operation: &str, message: impl Into<String>) {
self.available_count += 1;
self.capabilities.push(format!("{operation}:available"));
self.diagnostics.push(settings_diagnostic(
format!("{operation}.available"),
DiagnosticSeverity::Info,
message,
));
}
fn unknown(&mut self, operation: &str, message: impl Into<String>) {
@@ -1775,8 +1873,6 @@ impl RuntimeCompatibilityObservation {
fn state(&self) -> &'static str {
if self.incompatible_count > 0 {
"incompatible"
} else if self.unknown_count > 0 {
"unknown"
} else {
"compatible"
}
@@ -2144,6 +2240,9 @@ impl IntoResponse for ApiError {
Error::RuntimeOperationFailed { code, .. } if code == "remote_runtime_unsupported" => {
StatusCode::NOT_IMPLEMENTED
}
Error::RuntimeOperationFailed { code, .. } if code.ends_with("_blocked") => {
StatusCode::CONFLICT
}
Error::RuntimeOperationFailed { code, .. }
if code.starts_with("workspace_settings_")
|| code.starts_with("invalid_")
@@ -2365,7 +2464,7 @@ mod tests {
}
#[tokio::test]
async fn runtime_connection_settings_add_delete_persist_restart_required() {
async fn runtime_connection_settings_add_delete_apply_live_registry() {
let dir = tempfile::tempdir().unwrap();
let app = test_app(dir.path()).await;
@@ -2383,9 +2482,16 @@ mod tests {
}),
)
.await;
assert_eq!(added["restart_required"], true);
assert_eq!(added["restart_required"], false);
assert_eq!(added["remotes"][0]["runtime_id"], "team-runtime");
assert_eq!(added["remotes"][0]["endpoint_configured"], true);
assert!(
added["diagnostics"]
.as_array()
.unwrap()
.iter()
.any(|diagnostic| diagnostic["code"] == "runtime_registry_applied")
);
let projected = serde_json::to_string(&added).unwrap();
assert!(!projected.contains("runtime.example.invalid"));
@@ -2397,22 +2503,113 @@ mod tests {
"https://runtime.example.invalid"
);
let launch_options = get_json(app.clone(), "/api/workers/launch-options").await;
assert!(
launch_options["runtimes"]
.as_array()
.unwrap()
.iter()
.any(|runtime| runtime["runtime_id"] == "team-runtime")
);
let deleted = request_json(
app,
app.clone(),
"DELETE",
"/api/settings/runtime-connections/remotes/team-runtime",
None,
StatusCode::OK,
)
.await;
assert_eq!(deleted["restart_required"], true);
assert_eq!(deleted["restart_required"], false);
assert_eq!(deleted["remotes"].as_array().unwrap().len(), 0);
let launch_options = get_json(app.clone(), "/api/workers/launch-options").await;
assert!(
!launch_options["runtimes"]
.as_array()
.unwrap()
.iter()
.any(|runtime| runtime["runtime_id"] == "team-runtime")
);
let persisted = WorkspaceBackendConfigFile::load_for_workspace(dir.path()).unwrap();
assert!(persisted.runtimes.remote.is_empty());
}
#[tokio::test]
async fn runtime_connection_test_reports_observed_and_unknown_capabilities_without_endpoint_leak()
#[tokio::test(flavor = "multi_thread")]
async fn runtime_connection_delete_rejects_active_remote_workers() {
let (runtime, _worker_ref) = runtime_with_worker();
let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let runtime_addr = runtime_listener.local_addr().unwrap();
tokio::spawn({
let runtime = runtime.clone();
async move {
worker_runtime::http_server::serve_runtime_http(runtime, runtime_listener, None)
.await
.unwrap()
}
});
let dir = tempfile::tempdir().unwrap();
let app = test_app(dir.path()).await;
let added = post_json(
app.clone(),
"/api/settings/runtime-connections/remotes",
serde_json::json!({
"runtime_id": "busy-runtime",
"display_name": "Busy Runtime",
"endpoint": format!("http://{runtime_addr}")
}),
)
.await;
assert_eq!(added["restart_required"], false);
let created = post_json(
app.clone(),
"/api/workers",
serde_json::json!({
"runtime_id": "busy-runtime",
"display_name": "Remote Test Worker",
"profile": "runtime_default",
"initial_text": ""
}),
)
.await;
assert_eq!(created["runtime_id"], "busy-runtime");
let workers = get_json(app.clone(), "/api/workers").await;
assert!(
workers["items"]
.as_array()
.unwrap()
.iter()
.any(|worker| worker["runtime_id"] == "busy-runtime"),
"expected remote runtime to report at least one worker, got {workers}"
);
let response = request_json(
app,
"DELETE",
"/api/settings/runtime-connections/remotes/busy-runtime",
None,
StatusCode::CONFLICT,
)
.await;
assert!(
response["message"]
.as_str()
.unwrap()
.contains("remote_runtime_delete_blocked")
);
assert!(
response["diagnostics"]
.as_array()
.unwrap()
.iter()
.any(|diagnostic| { diagnostic["code"] == "remote_runtime_delete_blocked" })
);
let persisted = WorkspaceBackendConfigFile::load_for_workspace(dir.path()).unwrap();
assert_eq!(persisted.runtimes.remote.len(), 1);
}
#[tokio::test(flavor = "multi_thread")]
async fn runtime_connection_test_reports_compatible_with_unknown_warnings_without_endpoint_leak()
{
let (runtime, _worker_ref) = runtime_with_worker();
let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -2449,7 +2646,7 @@ mod tests {
serde_json::json!({}),
)
.await;
assert_eq!(response["state"], "unknown");
assert_eq!(response["state"], "compatible");
let capabilities = response["capabilities"].as_array().unwrap();
assert!(
capabilities
@@ -2464,14 +2661,14 @@ mod tests {
assert!(
capabilities
.iter()
.any(|value| value == "workers.spawn:unknown")
.any(|value| value == "workers.spawn:available")
);
assert!(
response["diagnostics"]
.as_array()
.unwrap()
.iter()
.any(|diagnostic| { diagnostic["code"] == "workers.spawn.unknown" })
.any(|diagnostic| { diagnostic["code"] == "workers.spawn.available" })
);
let projected = serde_json::to_string(&response).unwrap();
assert!(!projected.contains(&endpoint));
@@ -2479,6 +2676,60 @@ mod tests {
assert_eq!(response["protocol_version"], serde_json::Value::Null);
}
#[tokio::test(flavor = "multi_thread")]
async fn runtime_connection_test_marks_missing_execution_backend_incompatible() {
let runtime =
worker_runtime::Runtime::with_options(worker_runtime::RuntimeOptions::default());
let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let runtime_addr = runtime_listener.local_addr().unwrap();
tokio::spawn(async move {
worker_runtime::http_server::serve_runtime_http(runtime, runtime_listener, None)
.await
.unwrap()
});
let dir = tempfile::tempdir().unwrap();
let endpoint = format!("http://{runtime_addr}");
WorkspaceBackendConfigFile {
runtimes: crate::config::WorkspaceBackendRuntimesConfig {
remote: vec![RemoteRuntimeConfigFile {
id: "control-only-runtime".to_string(),
display_name: Some("Control-only Runtime".to_string()),
endpoint,
token_ref: None,
}],
},
..WorkspaceBackendConfigFile::default()
}
.write_for_workspace(dir.path())
.unwrap();
let app = test_app(dir.path()).await;
let response = post_json(
app,
"/api/settings/runtime-connections/remotes/control-only-runtime/test",
serde_json::json!({}),
)
.await;
assert_eq!(response["state"], "incompatible");
assert!(
response["capabilities"]
.as_array()
.unwrap()
.iter()
.any(|value| { value == "workers.spawn:incompatible" })
);
assert!(
response["diagnostics"]
.as_array()
.unwrap()
.iter()
.any(|diagnostic| {
diagnostic["code"] == "remote_runtime_worker_creation_unavailable"
})
);
}
#[tokio::test]
async fn browser_worker_create_succeeds_and_preserves_unsupported_diagnostics() {
let dir = tempfile::tempdir().unwrap();