fix: interrupt revoked Plugin transports and enforce full control-plane exclusions
This commit is contained in:
@@ -44,6 +44,12 @@ pub trait PluginHostOwner: Send + Sync {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Finite generated-client response budget for the supported one-MiB binary
|
||||||
|
/// body, its worst-case JSON numeric-array expansion, headers and envelope.
|
||||||
|
/// Set `response_body_limit` on the injected client before constructing a Remote
|
||||||
|
/// owner; a smaller operator-selected limit may reject otherwise valid replies.
|
||||||
|
pub const PLUGIN_HOST_RESPONSE_BODY_LIMIT: usize = 4 * 1024 * 1024 + 512 * 1024;
|
||||||
|
|
||||||
/// Uses an injected, already configured generated client (including its existing
|
/// Uses an injected, already configured generated client (including its existing
|
||||||
/// Runtime request authorizer). This adapter never creates or owns credentials.
|
/// Runtime request authorizer). This adapter never creates or owns credentials.
|
||||||
pub struct RemotePluginHostOwner<A> {
|
pub struct RemotePluginHostOwner<A> {
|
||||||
|
|||||||
@@ -108,6 +108,7 @@ pub(crate) struct Peer {
|
|||||||
pub(super) root: reqwest::Certificate,
|
pub(super) root: reqwest::Certificate,
|
||||||
pub(super) observations: mpsc::Receiver<Vec<u8>>,
|
pub(super) observations: mpsc::Receiver<Vec<u8>>,
|
||||||
pub(super) connections: Arc<AtomicUsize>,
|
pub(super) connections: Arc<AtomicUsize>,
|
||||||
|
closed: mpsc::Receiver<()>,
|
||||||
task: JoinHandle<()>,
|
task: JoinHandle<()>,
|
||||||
_directory: tempfile::TempDir,
|
_directory: tempfile::TempDir,
|
||||||
}
|
}
|
||||||
@@ -164,6 +165,7 @@ impl Peer {
|
|||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let address = listener.local_addr().unwrap();
|
let address = listener.local_addr().unwrap();
|
||||||
let (observed, observations) = mpsc::channel(16);
|
let (observed, observations) = mpsc::channel(16);
|
||||||
|
let (closed_sender, closed) = mpsc::channel(16);
|
||||||
let connections = Arc::new(AtomicUsize::new(0));
|
let connections = Arc::new(AtomicUsize::new(0));
|
||||||
let count = connections.clone();
|
let count = connections.clone();
|
||||||
let mut replies = VecDeque::from(replies);
|
let mut replies = VecDeque::from(replies);
|
||||||
@@ -176,6 +178,7 @@ impl Peer {
|
|||||||
count.fetch_add(1, Ordering::SeqCst);
|
count.fetch_add(1, Ordering::SeqCst);
|
||||||
let acceptor = acceptor.clone();
|
let acceptor = acceptor.clone();
|
||||||
let observed = observed.clone();
|
let observed = observed.clone();
|
||||||
|
let closed_sender = closed_sender.clone();
|
||||||
let reply = replies.pop_front().unwrap_or(Reply::Close);
|
let reply = replies.pop_front().unwrap_or(Reply::Close);
|
||||||
handlers.spawn(async move {
|
handlers.spawn(async move {
|
||||||
let Ok(mut socket) = acceptor.accept(socket).await else { return; };
|
let Ok(mut socket) = acceptor.accept(socket).await else { return; };
|
||||||
@@ -185,7 +188,7 @@ impl Peer {
|
|||||||
let Ok(length) = socket.read(&mut buffer).await else { return; };
|
let Ok(length) = socket.read(&mut buffer).await else { return; };
|
||||||
if length == 0 { return; }
|
if length == 0 { return; }
|
||||||
bytes.extend_from_slice(&buffer[..length]);
|
bytes.extend_from_slice(&buffer[..length]);
|
||||||
assert!(bytes.len() <= 64 * 1024, "test request exceeded fixture bound");
|
assert!(bytes.len() <= 2 * 1024 * 1024, "test request exceeded fixture bound");
|
||||||
if let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") {
|
if let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") {
|
||||||
let text = String::from_utf8_lossy(&bytes[..end]).to_lowercase();
|
let text = String::from_utf8_lossy(&bytes[..end]).to_lowercase();
|
||||||
let length = text.lines().find_map(|line| line.strip_prefix("content-length: "))
|
let length = text.lines().find_map(|line| line.strip_prefix("content-length: "))
|
||||||
@@ -197,7 +200,13 @@ impl Peer {
|
|||||||
match reply {
|
match reply {
|
||||||
Reply::Bytes(bytes) => { let _ = socket.write_all(&bytes).await; let _ = socket.shutdown().await; }
|
Reply::Bytes(bytes) => { let _ = socket.write_all(&bytes).await; let _ = socket.shutdown().await; }
|
||||||
Reply::StallAfter(bytes) => { let _ = socket.write_all(&bytes).await; pending::<()>().await; }
|
Reply::StallAfter(bytes) => { let _ = socket.write_all(&bytes).await; pending::<()>().await; }
|
||||||
Reply::Stall => pending::<()>().await,
|
Reply::Stall => {
|
||||||
|
let mut byte = [0];
|
||||||
|
match socket.read(&mut byte).await {
|
||||||
|
Ok(0) | Err(_) => { let _ = closed_sender.send(()).await; }
|
||||||
|
Ok(_) => panic!("unexpected bytes after complete request"),
|
||||||
|
}
|
||||||
|
},
|
||||||
Reply::Close => {},
|
Reply::Close => {},
|
||||||
Reply::Gated(release) => {
|
Reply::Gated(release) => {
|
||||||
if let Ok(bytes) = release.await {
|
if let Ok(bytes) = release.await {
|
||||||
@@ -217,6 +226,7 @@ impl Peer {
|
|||||||
root: reqwest::Certificate::from_pem(&certificate).unwrap(),
|
root: reqwest::Certificate::from_pem(&certificate).unwrap(),
|
||||||
observations,
|
observations,
|
||||||
connections,
|
connections,
|
||||||
|
closed,
|
||||||
task,
|
task,
|
||||||
_directory: directory,
|
_directory: directory,
|
||||||
}
|
}
|
||||||
@@ -232,6 +242,19 @@ impl Peer {
|
|||||||
self.connections.load(Ordering::SeqCst)
|
self.connections.load(Ordering::SeqCst)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn closed(&mut self) {
|
||||||
|
tokio::time::timeout(Duration::from_secs(2), self.closed.recv())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn transport_with_policy(&self, policy: TransportPolicy) -> HttpsTransport {
|
||||||
|
let mut transport = self.transport_for_requests(4);
|
||||||
|
transport.policy = policy;
|
||||||
|
transport
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn transport(&self) -> HttpsTransport {
|
pub(crate) fn transport(&self) -> HttpsTransport {
|
||||||
self.transport_for_requests(1)
|
self.transport_for_requests(1)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -307,6 +307,9 @@ pub struct ServerConfig {
|
|||||||
pub runtime_event_sources: Vec<RuntimeObservationSourceConfig>,
|
pub runtime_event_sources: Vec<RuntimeObservationSourceConfig>,
|
||||||
pub remote_runtime_sources: Vec<RemoteRuntimeConfig>,
|
pub remote_runtime_sources: Vec<RemoteRuntimeConfig>,
|
||||||
pub backend_base_url: Option<String>,
|
pub backend_base_url: Option<String>,
|
||||||
|
// Operational deny inventory survives Workspace capability filtering. It is
|
||||||
|
// never deserialized from Workspace configuration or returned to a guest.
|
||||||
|
pub(crate) control_plane_endpoints: std::collections::BTreeSet<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ServerConfig {
|
impl ServerConfig {
|
||||||
@@ -344,6 +347,7 @@ impl ServerConfig {
|
|||||||
runtime_event_sources: Vec::new(),
|
runtime_event_sources: Vec::new(),
|
||||||
remote_runtime_sources: Vec::new(),
|
remote_runtime_sources: Vec::new(),
|
||||||
backend_base_url: None,
|
backend_base_url: None,
|
||||||
|
control_plane_endpoints: Default::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -414,6 +418,28 @@ impl ServerConfig {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn plugin_control_plane_endpoints(&self) -> std::collections::BTreeSet<String> {
|
||||||
|
let AuthConfig::Passkey {
|
||||||
|
public_base_url, ..
|
||||||
|
} = &self.auth;
|
||||||
|
self.control_plane_endpoints
|
||||||
|
.iter()
|
||||||
|
.chain(std::iter::once(public_base_url))
|
||||||
|
.chain(self.backend_base_url.iter())
|
||||||
|
.chain(
|
||||||
|
self.remote_runtime_sources
|
||||||
|
.iter()
|
||||||
|
.map(|source| &source.base_url),
|
||||||
|
)
|
||||||
|
.chain(
|
||||||
|
self.runtime_event_sources
|
||||||
|
.iter()
|
||||||
|
.map(|source| &source.endpoint),
|
||||||
|
)
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn for_catalog_workspace(
|
fn for_catalog_workspace(
|
||||||
&self,
|
&self,
|
||||||
workspace: &WorkspaceRecord,
|
workspace: &WorkspaceRecord,
|
||||||
@@ -456,6 +482,7 @@ impl ServerConfig {
|
|||||||
scoped.repositories = repositories;
|
scoped.repositories = repositories;
|
||||||
// Runtime trust is server-global. Only explicitly assigned sources enter
|
// Runtime trust is server-global. Only explicitly assigned sources enter
|
||||||
// this Workspace's registry and receive Workspace-scoped capabilities.
|
// this Workspace's registry and receive Workspace-scoped capabilities.
|
||||||
|
scoped.control_plane_endpoints = self.plugin_control_plane_endpoints();
|
||||||
scoped.remote_runtime_sources.retain(|runtime| {
|
scoped.remote_runtime_sources.retain(|runtime| {
|
||||||
runtime.workspace_id.as_deref() == Some(workspace.workspace_id.as_str())
|
runtime.workspace_id.as_deref() == Some(workspace.workspace_id.as_str())
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ impl Fixture {
|
|||||||
runtime_event_sources: Vec::new(),
|
runtime_event_sources: Vec::new(),
|
||||||
remote_runtime_sources: Vec::new(),
|
remote_runtime_sources: Vec::new(),
|
||||||
backend_base_url: Some("http://localhost:8787".into()),
|
backend_base_url: Some("http://localhost:8787".into()),
|
||||||
|
control_plane_endpoints: Default::default(),
|
||||||
};
|
};
|
||||||
let store = Arc::new(
|
let store = Arc::new(
|
||||||
SqliteWorkspaceStore::open_for_test(&config.database_path)
|
SqliteWorkspaceStore::open_for_test(&config.database_path)
|
||||||
|
|||||||
@@ -77,7 +77,14 @@ use tokio_util::sync::CancellationToken;
|
|||||||
const RECEIPT_RETENTION_MS: i64 = 30 * 24 * 60 * 60 * 1000;
|
const RECEIPT_RETENTION_MS: i64 = 30 * 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
pub(super) fn router(service: Arc<ServerApiContractService>) -> Router {
|
pub(super) fn router(service: Arc<ServerApiContractService>) -> Router {
|
||||||
server_api_axum::plugin_host(service).layer(DefaultBodyLimit::max(2 * 1024 * 1024))
|
// Vec<u8> uses JSON's numeric array representation (up to four wire bytes
|
||||||
|
// per decoded byte). Reserve bounded room for IDs, URL/query and escaped
|
||||||
|
// headers; the shared owner still enforces every decoded field/body limit.
|
||||||
|
let body_ceiling = crate::plugin_binding::PluginConnectionPolicyCeiling::default()
|
||||||
|
.limits
|
||||||
|
.max_request_bytes as usize;
|
||||||
|
server_api_axum::plugin_host(service)
|
||||||
|
.layer(DefaultBodyLimit::max(4 * body_ceiling + 512 * 1024))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn denied() -> PluginHostError {
|
fn denied() -> PluginHostError {
|
||||||
@@ -623,32 +630,41 @@ async fn execute_with_transport(
|
|||||||
None => Arc::new(transport(&api).await?),
|
None => Arc::new(transport(&api).await?),
|
||||||
};
|
};
|
||||||
let cancellation = CancellationToken::new();
|
let cancellation = CancellationToken::new();
|
||||||
let result = transport
|
let sending = transport.execute_with_guard(
|
||||||
.execute_with_guard(
|
binding,
|
||||||
binding,
|
prepared,
|
||||||
prepared,
|
secret.as_deref().map(|v| v.as_str()),
|
||||||
secret.as_deref().map(|v| v.as_str()),
|
&cancellation,
|
||||||
&cancellation,
|
|| async {
|
||||||
|| async {
|
invocation(
|
||||||
invocation(
|
api.clone(),
|
||||||
api.clone(),
|
&context,
|
||||||
&context,
|
&workspace,
|
||||||
&workspace,
|
&invocation_id,
|
||||||
&invocation_id,
|
operation.clone(),
|
||||||
operation.clone(),
|
)
|
||||||
)
|
.await
|
||||||
.await
|
.map(|_| ())
|
||||||
.map(|_| ())
|
.map_err(|_| https::TransportError::Denied)
|
||||||
.map_err(|_| https::TransportError::Denied)
|
},
|
||||||
},
|
);
|
||||||
)
|
// Monitor the existing durable owners rather than maintaining
|
||||||
.await
|
// another revocation registry. Dropping the transport future
|
||||||
.map(|response| PluginHttpResponse {
|
// shuts down its single-use client/socket. A stalled authority
|
||||||
|
// owner also closes the attempt; it is never permission to run
|
||||||
|
// until the HTTP deadline. Admission remains consumed even if
|
||||||
|
// the shutdown acknowledgement or completion write is lost.
|
||||||
|
let result = tokio::select! {
|
||||||
|
result = sending => result.map(|response| PluginHttpResponse {
|
||||||
status: response.status,
|
status: response.status,
|
||||||
headers: response.headers,
|
headers: response.headers,
|
||||||
body: response.body,
|
body: response.body,
|
||||||
})
|
}).map_err(sent_transport_error),
|
||||||
.map_err(sent_transport_error);
|
error = monitor_invocation(api.clone(), &context, &workspace, &invocation_id, operation.clone()) => {
|
||||||
|
cancellation.cancel();
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
};
|
||||||
let store = api.config_store.clone();
|
let store = api.config_store.clone();
|
||||||
let r = record;
|
let r = record;
|
||||||
let fp = fingerprint;
|
let fp = fingerprint;
|
||||||
@@ -699,25 +715,92 @@ async fn execute_with_transport(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn monitor_invocation(
|
||||||
|
api: Arc<WorkspaceApi>,
|
||||||
|
context: &ServerRequestContext,
|
||||||
|
workspace: &str,
|
||||||
|
invocation_id: &str,
|
||||||
|
operation: PluginHostOperation,
|
||||||
|
) -> PluginHostError {
|
||||||
|
loop {
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
|
let checked = tokio::time::timeout(
|
||||||
|
std::time::Duration::from_secs(1),
|
||||||
|
invocation(
|
||||||
|
api.clone(),
|
||||||
|
context,
|
||||||
|
workspace,
|
||||||
|
invocation_id,
|
||||||
|
operation.clone(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let error = match checked {
|
||||||
|
Ok(Ok(_)) => continue,
|
||||||
|
Ok(Err(error)) => error,
|
||||||
|
Err(_) => policy::map_error(policy::failure(PluginHostErrorCode::Unavailable)),
|
||||||
|
};
|
||||||
|
// After admission the request may have reached the peer. Neither a
|
||||||
|
// failed check nor a local connection shutdown proves remote rollback.
|
||||||
|
return PluginHostError {
|
||||||
|
effect: PluginEffectObservation::Indeterminate,
|
||||||
|
..error
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn transport(
|
async fn transport(
|
||||||
api: &WorkspaceApi,
|
api: &WorkspaceApi,
|
||||||
) -> std::result::Result<https::HttpsTransport, PluginHostError> {
|
) -> std::result::Result<https::HttpsTransport, PluginHostError> {
|
||||||
|
let policy = transport_policy(api, |host, port| async move {
|
||||||
|
tokio::net::lookup_host((host, port))
|
||||||
|
.await
|
||||||
|
.map(|addresses| addresses.take(65).collect::<Vec<_>>())
|
||||||
|
.map_err(|_| https::TransportError::Dns)
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
Ok(https::HttpsTransport::new(policy))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn transport_policy<F, Fut>(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
resolve: F,
|
||||||
|
) -> std::result::Result<https::TransportPolicy, PluginHostError>
|
||||||
|
where
|
||||||
|
F: Fn(String, u16) -> Fut,
|
||||||
|
Fut: std::future::Future<
|
||||||
|
Output = std::result::Result<Vec<std::net::SocketAddr>, https::TransportError>,
|
||||||
|
>,
|
||||||
|
{
|
||||||
let mut hosts = std::collections::BTreeSet::new();
|
let mut hosts = std::collections::BTreeSet::new();
|
||||||
let mut ips = std::collections::BTreeSet::new();
|
let mut ips = std::collections::BTreeSet::new();
|
||||||
let AuthConfig::Passkey {
|
// Current server-owned bindings include dynamically registered endpoints in
|
||||||
public_base_url, ..
|
// other Workspaces, even when they were absent from the startup config.
|
||||||
} = &api.config.auth;
|
let store = api.config_store.clone();
|
||||||
for address in std::iter::once(public_base_url)
|
let registered = store
|
||||||
.chain(api.config.backend_base_url.iter())
|
.db_executor()
|
||||||
.chain(
|
.run(move || {
|
||||||
api.config
|
store.with_conn(|conn| {
|
||||||
.remote_runtime_sources
|
let mut statement =
|
||||||
.iter()
|
conn.prepare("SELECT DISTINCT base_url FROM workspace_runtime_bindings")?;
|
||||||
.map(|source| &source.base_url),
|
let rows = statement.query_map([], |row| row.get::<_, String>(0))?;
|
||||||
)
|
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(policy::map_error)?;
|
||||||
|
for address in api
|
||||||
|
.config
|
||||||
|
.plugin_control_plane_endpoints()
|
||||||
|
.iter()
|
||||||
|
.chain(registered.iter())
|
||||||
{
|
{
|
||||||
if let Ok(url) = url::Url::parse(address) {
|
if let Ok(url) = url::Url::parse(address) {
|
||||||
|
if !matches!(url.scheme(), "http" | "https" | "ws" | "wss") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if let Some(host) = url.host_str() {
|
if let Some(host) = url.host_str() {
|
||||||
|
let host = host.trim_start_matches('[').trim_end_matches(']');
|
||||||
hosts.insert(host.to_ascii_lowercase());
|
hosts.insert(host.to_ascii_lowercase());
|
||||||
if let Ok(ip) = host.parse() {
|
if let Ok(ip) = host.parse() {
|
||||||
ips.insert(ip);
|
ips.insert(ip);
|
||||||
@@ -725,12 +808,15 @@ async fn transport(
|
|||||||
let port = url.port_or_known_default().ok_or_else(denied)?;
|
let port = url.port_or_known_default().ok_or_else(denied)?;
|
||||||
let resolved = tokio::time::timeout(
|
let resolved = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(5),
|
std::time::Duration::from_secs(5),
|
||||||
tokio::net::lookup_host((host, port)),
|
resolve(host.to_owned(), port),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|_| transport_error(https::TransportError::Dns))?
|
.map_err(|_| transport_error(https::TransportError::Dns))?
|
||||||
.map_err(|_| transport_error(https::TransportError::Dns))?;
|
.map_err(|_| transport_error(https::TransportError::Dns))?;
|
||||||
for (index, address) in resolved.enumerate() {
|
if resolved.is_empty() {
|
||||||
|
return Err(transport_error(https::TransportError::Dns));
|
||||||
|
}
|
||||||
|
for (index, address) in resolved.into_iter().enumerate() {
|
||||||
if index >= 64 {
|
if index >= 64 {
|
||||||
return Err(transport_error(https::TransportError::Dns));
|
return Err(transport_error(https::TransportError::Dns));
|
||||||
}
|
}
|
||||||
@@ -740,10 +826,10 @@ async fn transport(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(https::HttpsTransport::new(https::TransportPolicy {
|
Ok(https::TransportPolicy {
|
||||||
denied_addresses: ips,
|
denied_addresses: ips,
|
||||||
denied_hosts: hosts,
|
denied_hosts: hosts,
|
||||||
}))
|
})
|
||||||
}
|
}
|
||||||
fn sent_transport_error(error: https::TransportError) -> PluginHostError {
|
fn sent_transport_error(error: https::TransportError) -> PluginHostError {
|
||||||
match error {
|
match error {
|
||||||
|
|||||||
@@ -1258,6 +1258,7 @@ async fn remote_owner_for(
|
|||||||
}));
|
}));
|
||||||
let client = server_api::ServerApiClient::builder(format!("http://{address}"))
|
let client = server_api::ServerApiClient::builder(format!("http://{address}"))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
.response_body_limit(yoi_plugin_runtime::owner::PLUGIN_HOST_RESPONSE_BODY_LIMIT)
|
||||||
.client(
|
.client(
|
||||||
reqwest::Client::builder()
|
reqwest::Client::builder()
|
||||||
.no_proxy()
|
.no_proxy()
|
||||||
@@ -1557,3 +1558,323 @@ async fn reserved_job_without_canonical_job_binding_cannot_fall_back_to_ordinary
|
|||||||
assert_eq!(f.count("plugin_effect_receipts"), 0);
|
assert_eq!(f.count("plugin_effect_receipts"), 0);
|
||||||
f.shutdown().await;
|
f.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn active_remote_and_embedded_sends_close_on_cancel_revoke_permission_loss_and_owner_failure()
|
||||||
|
{
|
||||||
|
use crate::plugin_https::test_support::Reply;
|
||||||
|
use server_api::PluginEffectObservation;
|
||||||
|
for remote in [false, true] {
|
||||||
|
for action in ["cancel", "revoke", "permission", "unavailable"] {
|
||||||
|
let mut peer = Peer::new(HOST, vec![Reply::Stall]).await;
|
||||||
|
let mut f = Fixture::new(Some(peer_binding(&peer))).await;
|
||||||
|
f.api.plugin_https_transport = Some(Arc::new(peer.transport_for_requests(4)));
|
||||||
|
let (remote_owner, server) =
|
||||||
|
remote_owner_for(&f, &f.source.worker_id, TEST_WORKSPACE_ID).await;
|
||||||
|
let owner = if remote {
|
||||||
|
remote_owner
|
||||||
|
} else {
|
||||||
|
f.owner(Some(peer.transport_for_requests(4))).await
|
||||||
|
};
|
||||||
|
let host = f.host(owner.clone(), "active-interruption").await;
|
||||||
|
let id = host.context().invocation_id.clone();
|
||||||
|
let request = send(&id, PluginHttpMethod::Post, "once");
|
||||||
|
let mut recovered_api = f.api.clone();
|
||||||
|
recovered_api.config_store =
|
||||||
|
Arc::new(SqliteWorkspaceStore::open_for_test(&f.api.config.database_path).unwrap());
|
||||||
|
let attempt = owner.call(request.clone());
|
||||||
|
tokio::pin!(attempt);
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut attempt => panic!("request must reach the stalled peer: {result:?}"),
|
||||||
|
_ = peer.observed() => {},
|
||||||
|
}
|
||||||
|
match action {
|
||||||
|
"cancel" => host.cancel().await.unwrap(),
|
||||||
|
"revoke" => f.revoke(),
|
||||||
|
"permission" => f.deny(),
|
||||||
|
"unavailable" => f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch(
|
||||||
|
"ALTER TABLE plugin_execution_plans RENAME TO unavailable_plugin_plans",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
// No response is released. HTTP has a five-second timeout; authority
|
||||||
|
// loss must close the owner transport substantially before that.
|
||||||
|
let error = tokio::time::timeout(std::time::Duration::from_secs(2), &mut attempt)
|
||||||
|
.await
|
||||||
|
.expect("active transport did not stop")
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
error.effect,
|
||||||
|
PluginEffectObservation::Indeterminate,
|
||||||
|
"{remote}/{action}"
|
||||||
|
);
|
||||||
|
peer.closed().await;
|
||||||
|
if action == "unavailable" {
|
||||||
|
f.api
|
||||||
|
.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch(
|
||||||
|
"ALTER TABLE unavailable_plugin_plans RENAME TO plugin_execution_plans",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
if action == "permission" {
|
||||||
|
set_policy(
|
||||||
|
&f.api,
|
||||||
|
&f.subject,
|
||||||
|
&f.installed.installation.installation_id,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Another actual owner with an independent SQLite connection has no
|
||||||
|
// local attempt state. Reopen after ordinary cancellation/revocation;
|
||||||
|
// the unavailable-owner fault deliberately changes SQLite DDL quotes.
|
||||||
|
if action != "unavailable" {
|
||||||
|
recovered_api.config_store = Arc::new(
|
||||||
|
SqliteWorkspaceStore::open_for_test(&f.api.config.database_path).unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let recovered = plugin_host::EmbeddedPluginHostOwner::new(
|
||||||
|
Arc::new(recovered_api),
|
||||||
|
f.verified_context().await,
|
||||||
|
TEST_WORKSPACE_ID.into(),
|
||||||
|
)
|
||||||
|
.with_transport(peer.transport_for_requests(4));
|
||||||
|
assert!(recovered.call(request).await.is_err());
|
||||||
|
assert_eq!(f.count("plugin_effect_receipts"), 1);
|
||||||
|
assert_eq!(
|
||||||
|
peer.connection_count(),
|
||||||
|
1,
|
||||||
|
"recovery resent {remote}/{action}"
|
||||||
|
);
|
||||||
|
drop(server);
|
||||||
|
f.shutdown().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn binary_ceiling_matches_generated_remote_router_and_embedded_owner_limits() {
|
||||||
|
let ceiling = crate::plugin_binding::PluginConnectionPolicyCeiling::default().limits;
|
||||||
|
let body = vec![255; ceiling.max_request_bytes as usize];
|
||||||
|
let mut peer = Peer::new(
|
||||||
|
HOST,
|
||||||
|
vec![response(200, &[], &body), response(200, &[], &body)],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut binding = peer_binding(&peer);
|
||||||
|
binding.limits.max_request_bytes = ceiling.max_request_bytes;
|
||||||
|
binding.limits.max_response_bytes = ceiling.max_response_bytes;
|
||||||
|
let mut f = Fixture::new(Some(binding)).await;
|
||||||
|
f.api.plugin_https_transport = Some(Arc::new(peer.transport_for_requests(4)));
|
||||||
|
let embedded = f.owner(Some(peer.transport_for_requests(4))).await;
|
||||||
|
let (remote, server) = remote_owner_for(&f, &f.source.worker_id, TEST_WORKSPACE_ID).await;
|
||||||
|
let host = f.host(remote.clone(), "binary-boundary").await;
|
||||||
|
let id = &host.context().invocation_id;
|
||||||
|
for (index, owner) in [&embedded, &remote].into_iter().enumerate() {
|
||||||
|
let mut input = send(id, PluginHttpMethod::Post, &format!("binary-{index}"));
|
||||||
|
if let PluginHostRequest::Send { request, .. } = &mut input {
|
||||||
|
request.body = body.clone();
|
||||||
|
}
|
||||||
|
assert!(serde_json::to_vec(&input).unwrap().len() > 4 * 1024 * 1024);
|
||||||
|
let PluginHostResponse::Response { response } = owner.call(input.clone()).await.unwrap()
|
||||||
|
else {
|
||||||
|
panic!("expected response")
|
||||||
|
};
|
||||||
|
assert_eq!(response.status, 200);
|
||||||
|
assert_eq!(response.body.len(), body.len());
|
||||||
|
assert!(response.body.iter().all(|byte| *byte == 255));
|
||||||
|
let observed = peer.observed().await;
|
||||||
|
assert!(observed.ends_with(&body));
|
||||||
|
assert_eq!(peer.connection_count(), index + 1);
|
||||||
|
// Replays cross adapters without a new connection.
|
||||||
|
embedded.call(input).await.unwrap();
|
||||||
|
let mut oversized = send(id, PluginHttpMethod::Post, &format!("oversized-{index}"));
|
||||||
|
if let PluginHostRequest::Send { request, .. } = &mut oversized {
|
||||||
|
request.body = vec![255; body.len() + 1];
|
||||||
|
}
|
||||||
|
let error = owner.call(oversized).await.unwrap_err();
|
||||||
|
assert_eq!(error.code, PluginHostErrorCode::QuotaExceeded);
|
||||||
|
assert_eq!(peer.connection_count(), index + 1);
|
||||||
|
}
|
||||||
|
assert_eq!(f.count("plugin_effect_receipts"), 2);
|
||||||
|
let mut enormous = send(id, PluginHttpMethod::Post, "wire-overflow");
|
||||||
|
if let PluginHostRequest::Send { request, .. } = &mut enormous {
|
||||||
|
request.body = vec![255; 2 * body.len()];
|
||||||
|
}
|
||||||
|
let path = format!("/api/w/{TEST_WORKSPACE_ID}/plugin-host");
|
||||||
|
let response = build_router(f.api.clone())
|
||||||
|
.oneshot(runtime_source_request(
|
||||||
|
&f.identity,
|
||||||
|
Some(&f.source.worker_id),
|
||||||
|
"POST",
|
||||||
|
&path,
|
||||||
|
serde_json::to_vec(&enormous).unwrap(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||||
|
assert_eq!(peer.connection_count(), 2);
|
||||||
|
drop(server);
|
||||||
|
f.shutdown().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn other_workspace_control_plane_hosts_ips_and_dns_aliases_never_connect() {
|
||||||
|
for dynamic in [false, true] {
|
||||||
|
for host in [HOST, "8.8.8.8", "runtime-alias.example.com"] {
|
||||||
|
let peer = Peer::new(HOST, vec![]).await;
|
||||||
|
let mut binding = peer_binding(&peer);
|
||||||
|
binding.origin = format!("https://{host}:{}", peer.public_address().port());
|
||||||
|
let mut f = Fixture::new(Some(binding)).await;
|
||||||
|
let endpoint = format!("https://{HOST}:{}", peer.public_address().port());
|
||||||
|
if dynamic {
|
||||||
|
let mut workspace = test_workspace();
|
||||||
|
workspace.workspace_id = "workspace-b".into();
|
||||||
|
f.api.store.upsert_workspace(&workspace).await.unwrap();
|
||||||
|
f.api
|
||||||
|
.store
|
||||||
|
.upsert_workspace_runtime_binding_record(
|
||||||
|
WorkspaceRuntimeBinding {
|
||||||
|
workspace_id: "workspace-b".into(),
|
||||||
|
runtime_id: "only-b".into(),
|
||||||
|
display_name: "Only B".into(),
|
||||||
|
base_url: endpoint,
|
||||||
|
public_key: f.identity.public_key.clone(),
|
||||||
|
public_key_fingerprint: String::new(),
|
||||||
|
binding_id: String::new(),
|
||||||
|
authentication_mode:
|
||||||
|
StoredRuntimeAuthenticationMode::LegacyServerIssuer,
|
||||||
|
created_at: "1".into(),
|
||||||
|
updated_at: "1".into(),
|
||||||
|
revoked_at: None,
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
} else {
|
||||||
|
let mut operational = f.api.config.clone();
|
||||||
|
operational.remote_runtime_sources.push(
|
||||||
|
RemoteRuntimeConfig::new("only-b", "Only B", endpoint, None)
|
||||||
|
.with_workspace_id("workspace-b"),
|
||||||
|
);
|
||||||
|
f.api.config = operational
|
||||||
|
.for_catalog_workspace(&test_workspace(), vec![])
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!f.api
|
||||||
|
.config
|
||||||
|
.remote_runtime_sources
|
||||||
|
.iter()
|
||||||
|
.any(|source| source.runtime_id == "only-b")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Only external DNS/socket/trust are substituted. The operational
|
||||||
|
// inventory and concrete shared owner/adapters remain production code.
|
||||||
|
let policy = plugin_host::transport_policy(&f.api, |name, port| async move {
|
||||||
|
let ip = if name == HOST { "8.8.8.8" } else { "9.9.9.9" };
|
||||||
|
Ok(vec![format!("{ip}:{port}").parse().unwrap()])
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(policy.denied_hosts.contains(HOST));
|
||||||
|
assert!(
|
||||||
|
policy
|
||||||
|
.denied_addresses
|
||||||
|
.contains(&"8.8.8.8".parse().unwrap())
|
||||||
|
);
|
||||||
|
f.api.plugin_https_transport =
|
||||||
|
Some(Arc::new(peer.transport_with_policy(policy.clone())));
|
||||||
|
let embedded = f.owner(Some(peer.transport_with_policy(policy))).await;
|
||||||
|
let (remote, server) =
|
||||||
|
remote_owner_for(&f, &f.source.worker_id, TEST_WORKSPACE_ID).await;
|
||||||
|
let invocation = f.host(embedded.clone(), "forbidden-control-plane").await;
|
||||||
|
for owner in [&embedded, &remote] {
|
||||||
|
assert_denied(
|
||||||
|
owner
|
||||||
|
.call(send(
|
||||||
|
&invocation.context().invocation_id,
|
||||||
|
PluginHttpMethod::Get,
|
||||||
|
"blocked",
|
||||||
|
))
|
||||||
|
.await,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(peer.connection_count(), 0, "{dynamic}/{host}");
|
||||||
|
drop(server);
|
||||||
|
f.shutdown().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stalled_authority_monitor_closes_active_transport_and_consumes_effect() {
|
||||||
|
use crate::plugin_https::test_support::Reply;
|
||||||
|
for remote in [false, true] {
|
||||||
|
let mut peer = Peer::new(HOST, vec![Reply::Stall]).await;
|
||||||
|
let mut f = Fixture::new(Some(peer_binding(&peer))).await;
|
||||||
|
f.api.plugin_https_transport = Some(Arc::new(peer.transport_for_requests(4)));
|
||||||
|
let (remote_owner, server) =
|
||||||
|
remote_owner_for(&f, &f.source.worker_id, TEST_WORKSPACE_ID).await;
|
||||||
|
let owner = if remote {
|
||||||
|
remote_owner
|
||||||
|
} else {
|
||||||
|
f.owner(Some(peer.transport_for_requests(4))).await
|
||||||
|
};
|
||||||
|
let host = f.host(owner.clone(), "monitor-unavailable").await;
|
||||||
|
let request = send(
|
||||||
|
&host.context().invocation_id,
|
||||||
|
PluginHttpMethod::Post,
|
||||||
|
"once",
|
||||||
|
);
|
||||||
|
let attempt = owner.call(request.clone());
|
||||||
|
tokio::pin!(attempt);
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut attempt => panic!("request must reach peer: {result:?}"),
|
||||||
|
_ = peer.observed() => {},
|
||||||
|
}
|
||||||
|
// Hold the actual authority DB connection. The monitor cannot complete
|
||||||
|
// its read, and may not treat a past permission as continued authority.
|
||||||
|
let store = f.api.config_store.clone();
|
||||||
|
let (started, waiting) = tokio::sync::oneshot::channel();
|
||||||
|
let (release, gate) = tokio::sync::oneshot::channel();
|
||||||
|
let held = tokio::task::spawn_blocking(move || {
|
||||||
|
store.with_conn(|_| {
|
||||||
|
let _ = started.send(());
|
||||||
|
let _ = gate.blocking_recv();
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
});
|
||||||
|
waiting.await.unwrap();
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut attempt => panic!("DB is held but owner finished: {result:?}"),
|
||||||
|
_ = peer.closed() => {},
|
||||||
|
}
|
||||||
|
release.send(()).unwrap();
|
||||||
|
held.await.unwrap().unwrap();
|
||||||
|
let error = tokio::time::timeout(std::time::Duration::from_secs(3), &mut attempt)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
error.effect,
|
||||||
|
server_api::PluginEffectObservation::Indeterminate
|
||||||
|
);
|
||||||
|
assert_eq!(error.code, PluginHostErrorCode::Unavailable);
|
||||||
|
assert!(owner.call(request).await.is_err());
|
||||||
|
assert_eq!(f.count("plugin_effect_receipts"), 1);
|
||||||
|
assert_eq!(peer.connection_count(), 1);
|
||||||
|
drop(server);
|
||||||
|
f.shutdown().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -454,6 +454,7 @@ mod tests {
|
|||||||
runtime_event_sources: vec![],
|
runtime_event_sources: vec![],
|
||||||
remote_runtime_sources: vec![],
|
remote_runtime_sources: vec![],
|
||||||
backend_base_url: Some("server-auth".into()),
|
backend_base_url: Some("server-auth".into()),
|
||||||
|
control_plane_endpoints: Default::default(),
|
||||||
};
|
};
|
||||||
let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap();
|
let identity = RuntimeIdentityMaterial::generate("runtime-auth").unwrap();
|
||||||
store
|
store
|
||||||
|
|||||||
Reference in New Issue
Block a user