feat: enforce live Plugin Host authority and durable HTTPS effects

This commit is contained in:
2026-10-30 04:07:19 +09:00
parent c65f145376
commit 732fe5c4c1
35 changed files with 7660 additions and 25 deletions
+2 -1
View File
@@ -8,6 +8,7 @@ rust-version.workspace = true
[dependencies]
async-trait.workspace = true
manifest.workspace = true
server-api.workspace = true
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
thiserror.workspace = true
@@ -23,4 +24,4 @@ wit-component = { version = "0.244.0", features = ["dummy-module"] }
wit-parser = "0.244.0"
wasmprinter = "0.244.0"
wat = "1"
tokio = { workspace = true, features = ["rt-multi-thread", "test-util"] }
tokio = { workspace = true, features = ["rt-multi-thread", "test-util", "net"] }
+1
View File
@@ -12,5 +12,6 @@ pub mod bindings {
mod executor;
pub mod host;
pub mod owner;
mod sandbox;
pub use executor::*;
+926
View File
@@ -0,0 +1,926 @@
//! Shared owner transport and WIT Host adapter. In-process owners implement the
//! same trait directly; a failed remote call never selects a local fallback.
use crate::host::{Host, InvocationContext, Operation, errors, http, logging};
use async_trait::async_trait;
use server_api::{
PluginConsumer, PluginEffectObservation, PluginHostError, PluginHostErrorCode,
PluginHostOperation, PluginHostRequest, PluginHostResponse, PluginHttpMethod,
PluginHttpRequest, PluginLogEntry, PluginLogField, PluginLogLevel, PluginLogScalar,
};
use std::{
any::Any,
sync::Arc,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use tokio::time::Instant;
/// Bound to one authenticated caller and Workspace by the integration. The owner
/// resolves all identity/policy from that binding and live-checks every operation,
/// including Open/Send/Log. Implementations must not detach effects or auto-retry.
#[async_trait]
pub trait PluginHostOwner: Send + Sync {
async fn call(&self, request: PluginHostRequest)
-> Result<PluginHostResponse, PluginHostError>;
}
/// Uses an injected, already configured generated client (including its existing
/// Runtime request authorizer). This adapter never creates or owns credentials.
pub struct RemotePluginHostOwner<A> {
client: Arc<server_api::ServerApiClient<A>>,
workspace_id: String,
}
impl<A> RemotePluginHostOwner<A> {
pub fn new(client: Arc<server_api::ServerApiClient<A>>, workspace_id: String) -> Self {
Self {
client,
workspace_id,
}
}
}
#[async_trait]
impl<A: server_api::client_support::RequestAuthorizer> PluginHostOwner
for RemotePluginHostOwner<A>
{
async fn call(
&self,
request: PluginHostRequest,
) -> Result<PluginHostResponse, PluginHostError> {
let may_dispatch = matches!(request, PluginHostRequest::Send { .. });
self.client
.plugin_host(self.workspace_id.clone(), request)
.await
.map_err(|error| {
match error {
server_api::client_support::ClientError::Public { error, .. } => error,
// A missing response is not proof that a Send was not dispatched.
// Preserve uncertainty; never synthesize a success or retry it.
server_api::client_support::ClientError::Failure(_) => local_error(
if may_dispatch {
PluginHostErrorCode::Indeterminate
} else {
PluginHostErrorCode::Unavailable
},
if may_dispatch {
PluginEffectObservation::Indeterminate
} else {
PluginEffectObservation::NotDispatched
},
),
}
})
}
}
/// Exactly one owner-issued invocation, with a local snapshot upper bound.
/// Obtain the execution context through `context()` after Begin; the caller's
/// pre-Begin invocation ID is never sent to, or adopted as authority by, the owner.
pub struct OwnerHost {
owner: Arc<dyn PluginHostOwner>,
context: InvocationContext,
binding_identity: Arc<()>,
}
struct OwnerConnection {
invocation_id: String,
slot: String,
identity: Arc<()>,
}
impl OwnerHost {
pub async fn begin(
owner: Arc<dyn PluginHostOwner>,
mut context: InvocationContext,
consumer: PluginConsumer,
) -> Result<Self, PluginHostError> {
if let PluginConsumer::ToolCall { tool_name, .. } = &consumer {
if *tool_name != context.tool_name {
return Err(local_error(
PluginHostErrorCode::Denied,
PluginEffectObservation::NotDispatched,
));
}
}
check_lifetime(&context)?;
let response = call_owner(
owner.as_ref(),
&context,
PluginHostRequest::Begin {
plan_id: context.plan.clone(),
installation_id: context.installation.clone(),
consumer,
},
)
.await?;
let PluginHostResponse::Invocation {
invocation_id,
expires_at_ms,
} = response
else {
return Err(protocol_error(false));
};
if invocation_id.is_empty() || invocation_id.len() > 1024 {
return Err(protocol_error(false));
}
let now_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.and_then(|d| i64::try_from(d.as_millis()).ok())
.ok_or_else(|| protocol_error(false))?;
let remaining = expires_at_ms
.checked_sub(now_ms)
.filter(|ms| *ms > 0)
.ok_or_else(|| {
local_error(
PluginHostErrorCode::DeadlineExceeded,
PluginEffectObservation::NotDispatched,
)
})?;
let expires = Instant::now()
.checked_add(Duration::from_millis(remaining as u64))
.ok_or_else(|| protocol_error(false))?;
context.deadline = context.deadline.min(expires);
context.invocation_id = invocation_id;
check_lifetime(&context)?;
Ok(Self {
owner,
context,
binding_identity: Arc::new(()),
})
}
pub fn context(&self) -> &InvocationContext {
&self.context
}
/// Persist cancellation at the owner as well as stopping local guest waits.
/// This never claims to undo an already admitted external effect. The caller
/// must reconcile an unavailable response, not restore under a new identity.
pub async fn cancel(&self) -> Result<(), PluginHostError> {
self.context.cancellation.cancel();
let response = tokio::time::timeout(
Duration::from_secs(5),
self.owner.call(PluginHostRequest::Cancel {
invocation_id: self.context.invocation_id.clone(),
}),
)
.await
.map_err(|_| {
local_error(
PluginHostErrorCode::Unavailable,
PluginEffectObservation::Indeterminate,
)
})??;
if response != PluginHostResponse::Authorized {
return Err(protocol_error(false));
}
Ok(())
}
fn check_context(&self, context: &InvocationContext) -> Result<(), PluginHostError> {
let bound = &self.context;
if context.workspace != bound.workspace
|| context.installation != bound.installation
|| context.plan != bound.plan
|| context.subject != bound.subject
|| context.invocation_id != bound.invocation_id
|| context.artifact != bound.artifact
|| context.tool_name != bound.tool_name
|| context.grants.logging != bound.grants.logging
|| context.grants.connection_slots != bound.grants.connection_slots
// The executor may shorten the deadline and derive a child token.
|| context.deadline > bound.deadline
{
return Err(local_error(
PluginHostErrorCode::Denied,
PluginEffectObservation::NotDispatched,
));
}
check_lifetime(bound)?;
check_lifetime(context)
}
fn connection<'a>(
&self,
binding: &'a (dyn Any + Send + Sync),
) -> Result<&'a OwnerConnection, PluginHostError> {
binding
.downcast_ref::<OwnerConnection>()
.filter(|b| {
b.invocation_id == self.context.invocation_id
&& Arc::ptr_eq(&b.identity, &self.binding_identity)
})
.ok_or_else(|| {
local_error(
PluginHostErrorCode::Denied,
PluginEffectObservation::NotDispatched,
)
})
}
fn check_slot(&self, slot: &str) -> Result<(), PluginHostError> {
if !self.context.grants.connection_slots.contains(slot) {
return Err(local_error(
PluginHostErrorCode::Denied,
PluginEffectObservation::NotDispatched,
));
}
Ok(())
}
}
#[async_trait]
impl Host for OwnerHost {
async fn authorize(
&self,
context: &InvocationContext,
operation: Operation<'_>,
) -> Result<(), errors::HostError> {
self.check_context(context).map_err(wit_error)?;
let operation = match operation {
Operation::Admit => PluginHostOperation::Admit,
Operation::Publish => PluginHostOperation::Publish,
Operation::Open { slot } => {
self.check_slot(slot).map_err(wit_error)?;
PluginHostOperation::Open { slot: slot.into() }
}
Operation::Send { slot, binding, .. } => {
let connection = self.connection(binding).map_err(wit_error)?;
self.check_slot(slot).map_err(wit_error)?;
if connection.slot != slot {
return Err(wit_error(local_error(
PluginHostErrorCode::Denied,
PluginEffectObservation::NotDispatched,
)));
}
PluginHostOperation::Send { slot: slot.into() }
}
Operation::Log => {
if !context.grants.logging {
return Err(wit_error(local_error(
PluginHostErrorCode::Denied,
PluginEffectObservation::NotDispatched,
)));
}
PluginHostOperation::Log
}
};
match call_owner(
self.owner.as_ref(),
context,
PluginHostRequest::Authorize {
invocation_id: self.context.invocation_id.clone(),
operation,
},
)
.await
.map_err(wit_error)?
{
PluginHostResponse::Authorized => Ok(()),
_ => Err(wit_error(protocol_error(false))),
}
}
async fn open(
&self,
context: &InvocationContext,
slot: &str,
) -> Result<Arc<dyn Any + Send + Sync>, errors::HostError> {
self.check_context(context).map_err(wit_error)?;
self.check_slot(slot).map_err(wit_error)?;
match call_owner(
self.owner.as_ref(),
context,
PluginHostRequest::Open {
invocation_id: self.context.invocation_id.clone(),
slot: slot.into(),
},
)
.await
.map_err(wit_error)?
{
PluginHostResponse::Opened => Ok(Arc::new(OwnerConnection {
invocation_id: self.context.invocation_id.clone(),
slot: slot.into(),
identity: self.binding_identity.clone(),
})),
_ => Err(wit_error(protocol_error(false))),
}
}
async fn send(
&self,
context: &InvocationContext,
binding: &(dyn Any + Send + Sync),
request: http::Request,
) -> Result<http::Response, errors::HostError> {
self.check_context(context).map_err(wit_error)?;
let connection = self.connection(binding).map_err(wit_error)?;
self.check_slot(&connection.slot).map_err(wit_error)?;
let request = PluginHttpRequest {
method: match request.method {
http::Method::Get => PluginHttpMethod::Get,
http::Method::Head => PluginHttpMethod::Head,
http::Method::Post => PluginHttpMethod::Post,
http::Method::Put => PluginHttpMethod::Put,
http::Method::Patch => PluginHttpMethod::Patch,
http::Method::Delete => PluginHttpMethod::Delete,
http::Method::Options => PluginHttpMethod::Options,
},
path: request.path,
query: pairs(request.query),
headers: pairs(request.headers),
body: request.body,
timeout_ms: request.timeout_ms,
effect_key: request.effect_key,
};
match call_owner(
self.owner.as_ref(),
context,
PluginHostRequest::Send {
invocation_id: self.context.invocation_id.clone(),
slot: connection.slot.clone(),
request,
},
)
.await
.map_err(wit_error)?
{
PluginHostResponse::Response { response } => Ok(http::Response {
status: response.status,
headers: response
.headers
.into_iter()
.map(|(name, value)| http::Pair { name, value })
.collect(),
body: response.body,
}),
_ => Err(wit_error(protocol_error(true))),
}
}
async fn log(
&self,
context: &InvocationContext,
entry: logging::Entry,
) -> Result<(), errors::HostError> {
self.check_context(context).map_err(wit_error)?;
if !context.grants.logging {
return Err(wit_error(local_error(
PluginHostErrorCode::Denied,
PluginEffectObservation::NotDispatched,
)));
}
let entry = PluginLogEntry {
level: match entry.level {
logging::Level::Trace => PluginLogLevel::Trace,
logging::Level::Debug => PluginLogLevel::Debug,
logging::Level::Info => PluginLogLevel::Info,
logging::Level::Warn => PluginLogLevel::Warn,
logging::Level::Error => PluginLogLevel::Error,
},
event_code: entry.event_code,
message: entry.message,
fields: entry
.fields
.into_iter()
.map(|f| {
Ok(PluginLogField {
name: f.name,
value: match f.value {
logging::Scalar::Text(v) => PluginLogScalar::Text(v),
logging::Scalar::Boolean(v) => PluginLogScalar::Boolean(v),
logging::Scalar::Signed(v) => PluginLogScalar::Signed(v),
logging::Scalar::Unsigned(v) => PluginLogScalar::Unsigned(v),
logging::Scalar::Number(v) if v.is_finite() => {
PluginLogScalar::Number(v)
}
logging::Scalar::Number(_) => {
return Err(wit_error(local_error(
PluginHostErrorCode::InvalidInput,
PluginEffectObservation::NotDispatched,
)));
}
},
})
})
.collect::<Result<_, errors::HostError>>()?,
};
match call_owner(
self.owner.as_ref(),
context,
PluginHostRequest::Log {
invocation_id: self.context.invocation_id.clone(),
entry,
},
)
.await
.map_err(wit_error)?
{
PluginHostResponse::Logged => Ok(()),
_ => Err(wit_error(protocol_error(false))),
}
}
}
async fn call_owner(
owner: &dyn PluginHostOwner,
context: &InvocationContext,
request: PluginHostRequest,
) -> Result<PluginHostResponse, PluginHostError> {
check_lifetime(context)?;
let effect = if matches!(request, PluginHostRequest::Send { .. }) {
PluginEffectObservation::Indeterminate
} else {
PluginEffectObservation::NotDispatched
};
tokio::select! {
biased;
_ = context.cancellation.cancelled() => Err(local_error(PluginHostErrorCode::Cancelled, effect)),
_ = tokio::time::sleep_until(context.deadline) => Err(local_error(PluginHostErrorCode::DeadlineExceeded, effect)),
response = owner.call(request) => response,
}
}
fn pairs(pairs: Vec<http::Pair>) -> Vec<(String, String)> {
pairs.into_iter().map(|p| (p.name, p.value)).collect()
}
fn local_error(code: PluginHostErrorCode, effect: PluginEffectObservation) -> PluginHostError {
PluginHostError {
code,
effect,
diagnostic_id: String::new(),
}
}
fn protocol_error(may_dispatch: bool) -> PluginHostError {
local_error(
if may_dispatch {
PluginHostErrorCode::Indeterminate
} else {
PluginHostErrorCode::Unavailable
},
if may_dispatch {
PluginEffectObservation::Indeterminate
} else {
PluginEffectObservation::NotDispatched
},
)
}
fn check_lifetime(context: &InvocationContext) -> Result<(), PluginHostError> {
if context.cancellation.is_cancelled() {
return Err(local_error(
PluginHostErrorCode::Cancelled,
PluginEffectObservation::NotDispatched,
));
}
if Instant::now() >= context.deadline {
return Err(local_error(
PluginHostErrorCode::DeadlineExceeded,
PluginEffectObservation::NotDispatched,
));
}
Ok(())
}
fn wit_error(error: PluginHostError) -> errors::HostError {
let message = error.message().to_owned();
errors::HostError {
code: match error.code {
PluginHostErrorCode::InvalidInput => errors::Code::InvalidInput,
PluginHostErrorCode::Denied => errors::Code::Denied,
PluginHostErrorCode::NotFound => errors::Code::NotFound,
PluginHostErrorCode::Conflict => errors::Code::Conflict,
PluginHostErrorCode::Unsupported => errors::Code::Unsupported,
PluginHostErrorCode::QuotaExceeded => errors::Code::QuotaExceeded,
PluginHostErrorCode::DeadlineExceeded => errors::Code::DeadlineExceeded,
PluginHostErrorCode::Cancelled => errors::Code::Cancelled,
PluginHostErrorCode::Unavailable => errors::Code::Unavailable,
PluginHostErrorCode::Indeterminate => errors::Code::Indeterminate,
PluginHostErrorCode::ReceiptExpired => errors::Code::ReceiptExpired,
},
effect: match error.effect {
PluginEffectObservation::NotDispatched => errors::EffectObservation::NotDispatched,
PluginEffectObservation::ResponseObserved => {
errors::EffectObservation::ResponseObserved
}
PluginEffectObservation::Indeterminate => errors::EffectObservation::Indeterminate,
},
message,
diagnostic_id: error.diagnostic_id,
}
}
#[cfg(test)]
mod tests {
// Adapter contract tests: the real generated HTTP client and OwnerHost run
// against one deterministic owner, directly or through an isolated loopback
// endpoint. Server authentication, HTTPS policy and durable receipts are not
// substituted claims of this suite; their concrete owner tests live in Server.
use super::*;
use crate::{
ArtifactIdentity,
host::{CancellationToken, Grants},
};
use server_api::{PluginHttpResponse, client_support, server_support};
use std::sync::{
Mutex,
atomic::{AtomicBool, Ordering},
};
#[derive(Default)]
struct TestOwner {
requests: Mutex<Vec<PluginHostRequest>>,
revoked: AtomicBool,
}
#[async_trait]
impl PluginHostOwner for TestOwner {
async fn call(
&self,
request: PluginHostRequest,
) -> Result<PluginHostResponse, PluginHostError> {
self.requests.lock().unwrap().push(request.clone());
if self.revoked.load(Ordering::SeqCst) {
return Err(PluginHostError {
code: PluginHostErrorCode::Denied,
effect: PluginEffectObservation::NotDispatched,
diagnostic_id: "revoked-diagnostic".into(),
});
}
Ok(match request {
PluginHostRequest::Begin { .. } => PluginHostResponse::Invocation {
invocation_id: "owner-issued".into(),
expires_at_ms: i64::try_from(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis(),
)
.unwrap()
+ 60_000,
},
PluginHostRequest::Cancel { .. } | PluginHostRequest::Authorize { .. } => {
PluginHostResponse::Authorized
}
PluginHostRequest::Open { .. } => PluginHostResponse::Opened,
PluginHostRequest::Send { request, .. } => PluginHostResponse::Response {
response: PluginHttpResponse {
status: 207,
headers: request.headers,
body: request.body,
},
},
PluginHostRequest::Log { .. } => PluginHostResponse::Logged,
})
}
}
fn context() -> InvocationContext {
InvocationContext {
workspace: "ws".into(),
installation: "install".into(),
plan: "plan".into(),
subject: "subject".into(),
invocation_id: "not-authority".into(),
artifact: ArtifactIdentity {
package_digest: "package".into(),
artifact_digest: "artifact".into(),
artifact_size: 123,
},
tool_name: "tool".into(),
grants: Grants {
logging: true,
connection_slots: ["api".into(), "other".into()].into(),
},
deadline: Instant::now() + Duration::from_secs(60),
cancellation: CancellationToken::new(),
}
}
fn consumer() -> PluginConsumer {
PluginConsumer::ToolCall {
tool_call_id: "call".into(),
tool_name: "tool".into(),
}
}
fn request() -> http::Request {
http::Request {
method: http::Method::Post,
path: "/v1/items".into(),
query: vec![
http::Pair {
name: "q".into(),
value: "one".into(),
},
http::Pair {
name: "q".into(),
value: "two".into(),
},
],
headers: vec![http::Pair {
name: "x-test".into(),
value: "value".into(),
}],
body: vec![0, 255, 42],
timeout_ms: 1234,
effect_key: "effect".into(),
}
}
struct TestAuthorizer;
impl client_support::RequestAuthorizer for TestAuthorizer {
fn authorize(
&self,
_: client_support::AuthorizerRequest<'_>,
) -> Result<client_support::framework::header::HeaderMap, client_support::AuthorizationError>
{
let mut headers = client_support::framework::header::HeaderMap::new();
headers.insert("authorization", "Bearer isolated-test".parse().unwrap());
Ok(headers)
}
}
struct Endpoint {
url: String,
task: tokio::task::JoinHandle<()>,
}
impl Drop for Endpoint {
fn drop(&mut self) {
self.task.abort();
}
}
async fn endpoint(owner: Arc<TestOwner>) -> Endpoint {
use server_support::framework::{self as axum, HeaderMap, Json, Path, Response, State};
async fn call(
State(owner): State<Arc<TestOwner>>,
Path(workspace): Path<String>,
headers: HeaderMap,
Json(request): Json<PluginHostRequest>,
) -> Response {
assert_eq!(workspace, "ws");
assert_eq!(
headers.get("authorization").unwrap(),
"Bearer isolated-test"
);
match owner.call(request).await {
Ok(response) => server_support::json_response(axum::StatusCode::OK, response),
Err(error) => server_support::error_response(axum::StatusCode::FORBIDDEN, error),
}
}
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let router = axum::Router::new()
.route("/api/w/{workspace_id}/plugin-host", axum::post(call))
.with_state(owner);
let task = tokio::spawn(async move {
axum::serve(listener, router).await.unwrap();
});
Endpoint { url, task }
}
async fn remote(owner: Arc<TestOwner>) -> (Arc<dyn PluginHostOwner>, Endpoint) {
let endpoint = endpoint(owner).await;
let http = client_support::framework::Client::builder()
.no_proxy()
.build()
.unwrap();
let core = client_support::ClientBuilder::try_new(&endpoint.url)
.unwrap()
.client(http)
.authorizer(TestAuthorizer)
.request_timeout(Duration::from_secs(5))
.build()
.unwrap();
let api = Arc::new(server_api::ServerApiClient::from_core(core));
(
Arc::new(RemotePluginHostOwner::new(api, "ws".into())),
endpoint,
)
}
#[tokio::test]
async fn remote_and_direct_owner_preserve_requests_results_and_live_denial() {
let mut transcripts = Vec::new();
for use_remote in [false, true] {
let owner = Arc::new(TestOwner::default());
let (transport, _endpoint): (Arc<dyn PluginHostOwner>, _) = if use_remote {
let (transport, endpoint) = remote(owner.clone()).await;
(transport, Some(endpoint))
} else {
(owner.clone(), None)
};
let host = OwnerHost::begin(transport, context(), consumer())
.await
.unwrap();
let ctx = host.context();
assert_eq!(ctx.invocation_id, "owner-issued");
host.authorize(ctx, Operation::Admit).await.unwrap();
host.authorize(ctx, Operation::Open { slot: "api" })
.await
.unwrap();
let connection = host.open(ctx, "api").await.unwrap();
host.authorize(
ctx,
Operation::Send {
slot: "api",
binding: connection.as_ref(),
request: &request(),
},
)
.await
.unwrap();
let response = host
.send(ctx, connection.as_ref(), request())
.await
.unwrap();
assert_eq!(response.status, 207);
assert_eq!(response.body, vec![0, 255, 42]);
assert_eq!(response.headers[0].value, "value");
host.authorize(ctx, Operation::Log).await.unwrap();
host.log(
ctx,
logging::Entry {
level: logging::Level::Warn,
event_code: "test.event".into(),
message: "bounded".into(),
fields: vec![logging::Field {
name: "count".into(),
value: logging::Scalar::Signed(-2),
}],
},
)
.await
.unwrap();
host.authorize(ctx, Operation::Publish).await.unwrap();
owner.revoked.store(true, Ordering::SeqCst);
for operation in [
Operation::Admit,
Operation::Publish,
Operation::Open { slot: "api" },
Operation::Log,
] {
let error = host.authorize(ctx, operation).await.unwrap_err();
assert_eq!(error.code, errors::Code::Denied);
assert_eq!(error.diagnostic_id, "revoked-diagnostic");
assert_eq!(error.effect, errors::EffectObservation::NotDispatched);
}
assert_eq!(
host.send(ctx, connection.as_ref(), request())
.await
.unwrap_err()
.code,
errors::Code::Denied
);
transcripts.push(owner.requests.lock().unwrap().clone());
}
assert_eq!(transcripts[0], transcripts[1]);
}
#[tokio::test]
async fn changed_context_or_cross_invocation_connection_never_reaches_owner() {
let owner = Arc::new(TestOwner::default());
let host = OwnerHost::begin(owner.clone(), context(), consumer())
.await
.unwrap();
let other = OwnerHost::begin(owner.clone(), context(), consumer())
.await
.unwrap();
let connection = other.open(other.context(), "api").await.unwrap();
let count = owner.requests.lock().unwrap().len();
// Even an owner reusing an invocation ID cannot make private handles interchangeable.
assert_eq!(
host.send(host.context(), connection.as_ref(), request())
.await
.unwrap_err()
.code,
errors::Code::Denied
);
let mutations: Vec<Box<dyn Fn(&mut InvocationContext)>> = vec![
Box::new(|c| c.workspace.push('x')),
Box::new(|c| c.installation.push('x')),
Box::new(|c| c.plan.push('x')),
Box::new(|c| c.subject.push('x')),
Box::new(|c| c.invocation_id.push('x')),
Box::new(|c| c.artifact.artifact_digest.push('x')),
Box::new(|c| c.tool_name.push('x')),
Box::new(|c| c.grants.logging = false),
Box::new(|c| {
c.grants.connection_slots.insert("new".into());
}),
Box::new(|c| c.deadline += Duration::from_secs(1)),
];
for mutate in mutations {
let mut changed = host.context().clone();
mutate(&mut changed);
assert_eq!(
host.authorize(&changed, Operation::Admit)
.await
.unwrap_err()
.code,
errors::Code::Denied
);
}
let own = host.open(host.context(), "api").await.unwrap();
assert_eq!(
host.authorize(
host.context(),
Operation::Send {
slot: "other",
binding: own.as_ref(),
request: &request()
}
)
.await
.unwrap_err()
.code,
errors::Code::Denied
);
assert_eq!(owner.requests.lock().unwrap().len(), count + 1);
host.context().cancellation.cancel();
assert_eq!(
host.authorize(host.context(), Operation::Admit)
.await
.unwrap_err()
.code,
errors::Code::Cancelled
);
assert_eq!(owner.requests.lock().unwrap().len(), count + 1);
}
struct PendingOwner {
inner: TestOwner,
begin: bool,
dropped: Arc<AtomicBool>,
started: tokio::sync::Notify,
}
struct Dropped(Arc<AtomicBool>);
impl Drop for Dropped {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
#[async_trait]
impl PluginHostOwner for PendingOwner {
async fn call(
&self,
request: PluginHostRequest,
) -> Result<PluginHostResponse, PluginHostError> {
if self.begin || matches!(request, PluginHostRequest::Send { .. }) {
let _guard = Dropped(self.dropped.clone());
self.started.notify_one();
std::future::pending().await
} else {
self.inner.call(request).await
}
}
}
#[tokio::test(start_paused = true)]
async fn begin_deadline_drops_pending_owner_call() {
let dropped = Arc::new(AtomicBool::new(false));
let owner = Arc::new(PendingOwner {
inner: TestOwner::default(),
begin: true,
dropped: dropped.clone(),
started: tokio::sync::Notify::new(),
});
let error = OwnerHost::begin(owner, context(), consumer())
.await
.err()
.unwrap();
assert_eq!(error.code, PluginHostErrorCode::DeadlineExceeded);
assert_eq!(error.effect, PluginEffectObservation::NotDispatched);
assert!(dropped.load(Ordering::SeqCst));
}
#[tokio::test]
async fn cancelled_pending_send_drops_owner_future_and_preserves_uncertainty() {
let dropped = Arc::new(AtomicBool::new(false));
let owner = Arc::new(PendingOwner {
inner: TestOwner::default(),
begin: false,
dropped: dropped.clone(),
started: tokio::sync::Notify::new(),
});
let host = OwnerHost::begin(owner.clone(), context(), consumer())
.await
.unwrap();
let binding = host.open(host.context(), "api").await.unwrap();
let cancellation = host.context().cancellation.clone();
let (result, ()) = tokio::join!(
host.send(host.context(), binding.as_ref(), request()),
async {
owner.started.notified().await;
cancellation.cancel();
}
);
let error = result.unwrap_err();
assert_eq!(error.code, errors::Code::Cancelled);
assert_eq!(error.effect, errors::EffectObservation::Indeterminate);
assert!(dropped.load(Ordering::SeqCst));
}
#[tokio::test]
async fn remote_connection_loss_is_indeterminate_and_never_retried() {
let owner = Arc::new(TestOwner::default());
let (transport, mut endpoint) = remote(owner.clone()).await;
let host = OwnerHost::begin(transport, context(), consumer())
.await
.unwrap();
let binding = host.open(host.context(), "api").await.unwrap();
endpoint.task.abort();
// Await listener teardown, not a timing-based sleep.
let _ = (&mut endpoint.task).await;
let error = host
.send(host.context(), binding.as_ref(), request())
.await
.unwrap_err();
assert_eq!(error.code, errors::Code::Indeterminate);
assert_eq!(error.effect, errors::EffectObservation::Indeterminate);
assert!(!error.message.contains("127.0.0.1"));
assert_eq!(owner.requests.lock().unwrap().len(), 2);
}
}
+16 -2
View File
@@ -10,11 +10,13 @@ pub mod authorization;
mod drive;
mod notification;
mod plugin;
pub mod plugin_host;
mod plugin_types;
pub use notification::*;
pub use plugin::*;
pub use plugin_host::*;
#[cfg(feature = "typescript")]
pub use plugin_types::plugin_api_typescript;
pub use plugin_types::{plugin_api_typescript, plugin_host_typescript};
mod ticket_browse;
pub use ticket_browse::*;
pub mod repository_openapi_typescript;
@@ -210,6 +212,9 @@ macro_rules! impl_openapi_schema {
}
impl_openapi_schema!(
PluginHostRequest,
PluginHostResponse,
PluginHostError,
PluginInstallRequest,
PluginAddArtifactRequest,
PluginRevokeRequest,
@@ -577,6 +582,15 @@ pub struct WorkspaceWorkerSessionHistoryResponse {
/// Response loss does not imply rollback.
#[api(reqwest, axum, openapi)]
pub trait ServerApi {
/// Runtime Host owner boundary; human plugin-management authority is not accepted here.
#[post("/api/w/{workspace_id}/plugin-host", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,413,429,503], normalize_body_errors = true)]
async fn plugin_host(
&self,
#[extension] context: ServerRequestContext,
#[path] workspace_id: String,
#[body] request: PluginHostRequest,
) -> Result<PluginHostResponse, PluginHostError>;
#[get("/health", status = 200, error_status = 400)]
async fn health(&self) -> Result<HealthResponse, RepositoryApiError>;
@@ -14968,7 +14982,7 @@ mod openapi_artifact_tests {
// Session observation retains its existing explicit non-OpenAPI boundary independently of
// the Runtime/Worker-source signed operation inventory above.
const OTHER_OPENAPI_EXCLUDED: &[&str] = &["worker_session", "worker_session_history"];
const DOCUMENTED_SIGNED_INTERNAL: &[&str] = &["workspace_runtime_config"];
const DOCUMENTED_SIGNED_INTERNAL: &[&str] = &["workspace_runtime_config", "plugin_host"];
const BROWSER_ONLY: &[&str] = &["merge_request_selector_repair"];
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
+356
View File
@@ -0,0 +1,356 @@
//! Runtime-to-owner Host protocol. Authentication is transport-bound, never a DTO claim.
use crate::PluginHttpMethod;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum PluginHostRequest {
Begin {
plan_id: String,
installation_id: String,
consumer: PluginConsumer,
},
/// Irreversible cancellation of this authenticated logical invocation.
Cancel {
invocation_id: String,
},
Authorize {
invocation_id: String,
operation: PluginHostOperation,
},
Open {
invocation_id: String,
slot: String,
},
Send {
invocation_id: String,
slot: String,
request: PluginHttpRequest,
},
Log {
invocation_id: String,
entry: PluginLogEntry,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum PluginConsumer {
ToolCall {
tool_call_id: String,
tool_name: String,
},
/// Reserved consumer identity; this does not enable service/event delivery.
Consumer {
consumer_id: String,
event_id: String,
export_name: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(
tag = "kind",
content = "data",
rename_all = "snake_case",
deny_unknown_fields
)]
pub enum PluginHostOperation {
Admit,
Publish,
Open { slot: String },
Send { slot: String },
Log,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
pub struct PluginHttpRequest {
pub method: PluginHttpMethod,
pub path: String,
pub query: Vec<(String, String)>,
pub headers: Vec<(String, String)>,
pub body: Vec<u8>,
pub timeout_ms: u32,
pub effect_key: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
pub struct PluginHttpResponse {
pub status: u16,
pub headers: Vec<(String, String)>,
pub body: Vec<u8>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(
tag = "kind",
content = "data",
rename_all = "snake_case",
deny_unknown_fields
)]
pub enum PluginHostResponse {
Invocation {
invocation_id: String,
#[schemars(range(min = 0, max = 9_007_199_254_740_991_i64))]
#[cfg_attr(feature = "typescript", ts(type = "number"))]
expires_at_ms: i64,
},
Authorized,
Opened,
Response {
response: PluginHttpResponse,
},
Logged,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
pub struct PluginLogEntry {
pub level: PluginLogLevel,
pub event_code: String,
pub message: String,
pub fields: Vec<PluginLogField>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum PluginLogLevel {
Trace,
Debug,
Info,
Warn,
Error,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
pub struct PluginLogField {
pub name: String,
pub value: PluginLogScalar,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(
tag = "kind",
content = "value",
rename_all = "snake_case",
deny_unknown_fields
)]
pub enum PluginLogScalar {
Text(String),
Boolean(bool),
Signed(
#[serde(with = "decimal_integer")]
#[schemars(with = "String")]
#[cfg_attr(feature = "typescript", ts(type = "string"))]
i64,
),
Unsigned(
#[serde(with = "decimal_integer")]
#[schemars(with = "String")]
#[cfg_attr(feature = "typescript", ts(type = "string"))]
u64,
),
Number(f64),
}
// WIT scalars retain their full 64-bit range without JSON/TypeScript precision loss.
mod decimal_integer {
pub fn serialize<T: std::fmt::Display, S: serde::Serializer>(
value: &T,
serializer: S,
) -> Result<S::Ok, S::Error> {
serializer.collect_str(value)
}
pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
where
T: std::str::FromStr + std::fmt::Display,
D: serde::Deserializer<'de>,
{
let text = <String as serde::Deserialize>::deserialize(deserializer)?;
let value: T = text
.parse()
.map_err(|_| serde::de::Error::custom("invalid decimal integer"))?;
if value.to_string() != text {
return Err(serde::de::Error::custom("non-canonical decimal integer"));
}
Ok(value)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum PluginHostErrorCode {
InvalidInput,
Denied,
NotFound,
Conflict,
Unsupported,
QuotaExceeded,
DeadlineExceeded,
Cancelled,
Unavailable,
Indeterminate,
ReceiptExpired,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum PluginEffectObservation {
NotDispatched,
ResponseObserved,
Indeterminate,
}
/// Deliberately excludes raw diagnostics, transport bodies and arbitrary messages.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
pub struct PluginHostError {
pub code: PluginHostErrorCode,
pub effect: PluginEffectObservation,
pub diagnostic_id: String,
}
impl PluginHostError {
pub fn message(&self) -> &'static str {
match self.code {
PluginHostErrorCode::InvalidInput => "invalid plugin Host input",
PluginHostErrorCode::Denied => "plugin Host operation denied",
PluginHostErrorCode::NotFound => "plugin Host target not found",
PluginHostErrorCode::Conflict => "plugin Host operation conflict",
PluginHostErrorCode::Unsupported => "plugin Host operation unsupported",
PluginHostErrorCode::QuotaExceeded => "plugin Host quota exceeded",
PluginHostErrorCode::DeadlineExceeded => "plugin Host deadline exceeded",
PluginHostErrorCode::Cancelled => "plugin Host operation cancelled",
PluginHostErrorCode::Unavailable => "plugin Host unavailable",
PluginHostErrorCode::Indeterminate => "plugin Host outcome indeterminate",
PluginHostErrorCode::ReceiptExpired => "plugin Host receipt expired",
}
}
}
impl std::fmt::Display for PluginHostError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.message())
}
}
impl std::error::Error for PluginHostError {}
impl api_macros::HttpError for PluginHostError {
fn status_code(&self) -> u16 {
match self.code {
PluginHostErrorCode::InvalidInput | PluginHostErrorCode::Unsupported => 400,
PluginHostErrorCode::Denied => 403,
PluginHostErrorCode::NotFound => 404,
PluginHostErrorCode::Conflict
| PluginHostErrorCode::Indeterminate
| PluginHostErrorCode::ReceiptExpired => 409,
PluginHostErrorCode::QuotaExceeded => 429,
PluginHostErrorCode::DeadlineExceeded
| PluginHostErrorCode::Cancelled
| PluginHostErrorCode::Unavailable => 503,
}
}
}
impl api_macros::HttpRequestError for PluginHostError {
fn from_request_rejection(status: u16, _message: String) -> Self {
Self {
code: if status == 413 {
PluginHostErrorCode::QuotaExceeded
} else {
PluginHostErrorCode::InvalidInput
},
effect: PluginEffectObservation::NotDispatched,
diagnostic_id: String::new(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn host_protocol_rejects_identity_claims_and_unknown_fields() {
for value in [
serde_json::json!({"kind":"begin","plan_id":"p","installation_id":"i","consumer":{"kind":"tool_call","tool_call_id":"t","tool_name":"n"},"subject":"owner"}),
serde_json::json!({"kind":"begin","plan_id":"p","installation_id":"i","consumer":{"kind":"tool_call","tool_call_id":"t","tool_name":"n","worker_id":"w"}}),
serde_json::json!({"kind":"authorize","invocation_id":"i","operation":{"kind":"admit","subject":"owner"}}),
serde_json::json!({"kind":"send","invocation_id":"i","slot":"s","request":{"method":"GET","path":"/","query":[],"headers":[],"body":[],"timeout_ms":1,"effect_key":"e","url":"https://untrusted"}}),
] {
assert!(serde_json::from_value::<PluginHostRequest>(value).is_err());
}
assert!(
serde_json::from_value::<PluginHostResponse>(
serde_json::json!({"kind":"authorized","subject":"owner"})
)
.is_err()
);
assert!(serde_json::from_value::<PluginHostError>(serde_json::json!({"code":"denied","effect":"not_dispatched","diagnostic_id":"d","message":"secret"})).is_err());
}
#[test]
fn log_integer_wire_preserves_full_wit_range_without_json_precision_loss() {
for scalar in [
PluginLogScalar::Signed(i64::MIN),
PluginLogScalar::Signed(i64::MAX),
PluginLogScalar::Unsigned(u64::MAX),
] {
let wire = serde_json::to_value(&scalar).unwrap();
assert!(wire["value"].is_string());
assert_eq!(
serde_json::from_value::<PluginLogScalar>(wire).unwrap(),
scalar
);
}
for value in [
serde_json::json!(1),
serde_json::json!("01"),
serde_json::json!("+1"),
serde_json::json!("18446744073709551616"),
] {
assert!(
serde_json::from_value::<PluginLogScalar>(
serde_json::json!({"kind":"unsigned","value":value})
)
.is_err()
);
}
}
#[test]
fn host_route_uses_runtime_context_and_closed_error_schema() {
let document = crate::canonical_openapi_document().unwrap();
let value: serde_json::Value = serde_json::from_str(&document.to_json().unwrap()).unwrap();
let route = &value["paths"]["/api/w/{workspace_id}/plugin-host"]["post"];
for status in [200, 400, 401, 403, 404, 409, 413, 429, 503] {
assert!(
route["responses"][status.to_string()].is_object(),
"{status}"
);
}
assert!(
route["security"].is_null(),
"signed Runtime context, not human bearer/browser auth"
);
let params = route["parameters"].as_array().unwrap();
assert_eq!(params.len(), 1);
assert_eq!(params[0]["name"], "workspace_id");
let error = &value["components"]["schemas"]["PluginHostError"];
assert_eq!(error["additionalProperties"], false);
assert_eq!(error["properties"].as_object().unwrap().len(), 3);
}
}
+44 -1
View File
@@ -1,4 +1,28 @@
//! Generated management types share the exact DTOs used by the HTTP declaration.
//! Generated plugin types share the exact DTOs used by the HTTP declarations.
#[cfg(feature = "typescript")]
pub fn plugin_host_typescript() -> String {
use crate::*;
use ts_rs::TS;
let config = ts_rs::Config::default();
let declarations = [
PluginHttpMethod::decl(&config),
PluginConsumer::decl(&config),
PluginHostRequest::decl(&config),
PluginHostOperation::decl(&config),
PluginHttpRequest::decl(&config),
PluginHttpResponse::decl(&config),
PluginHostResponse::decl(&config),
PluginLogEntry::decl(&config),
PluginLogLevel::decl(&config),
PluginLogField::decl(&config),
PluginLogScalar::decl(&config),
PluginHostError::decl(&config),
PluginHostErrorCode::decl(&config),
PluginEffectObservation::decl(&config),
];
declarations.map(|d| format!("export {d}")).join("\n\n")
}
#[cfg(feature = "typescript")]
pub fn plugin_api_typescript() -> String {
use crate::*;
@@ -31,6 +55,25 @@ pub fn plugin_api_typescript() -> String {
#[cfg(test)]
mod tests {
#[cfg(feature = "typescript")]
#[test]
fn host_typescript_preserves_wire_numbers_and_lossless_scalar_strings() {
let text = super::plugin_host_typescript();
for name in [
"PluginHostRequest",
"PluginHostResponse",
"PluginHostError",
"PluginConsumer",
"PluginHttpRequest",
"PluginHttpResponse",
"PluginLogEntry",
] {
assert!(text.contains(&format!("export type {name} =")), "{name}");
}
assert!(text.contains("expires_at_ms: number"));
assert!(text.contains("\"signed\", \"value\": string"));
assert!(text.contains("\"unsigned\", \"value\": string"));
}
#[test]
fn plugin_routes_use_binary_upload_and_closed_management_contracts() {
let document = crate::canonical_openapi_document().unwrap();
+6
View File
@@ -30,6 +30,10 @@ job.workspace = true
protocol = { workspace = true }
runtime-api.workspace = true
server-api.workspace = true
secrets.workspace = true
yoi-plugin-runtime = { path = "../plugin-runtime" }
tokio-util = { version = "0.7", features = ["rt"] }
flate2 = "1"
project-record.workspace = true
reqwest = { version = "0.13", default-features = false, features = ["blocking", "json", "native-tls"] }
ring.workspace = true
@@ -61,6 +65,8 @@ webauthn-rs = { workspace = true, features = ["conditional-ui"] }
webauthn-rs-proto = "0.5.5"
[dev-dependencies]
native-tls = "0.2"
tokio-native-tls = "0.3"
wip-protocol.workspace = true
agen.workspace = true
tempfile.workspace = true
@@ -516,7 +516,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
108,
"Subject Ticket responsibility and current-body notification routing"
.into()
)
),
(109, "Durable Plugin Host invocations and HTTPS effect receipts".into())
]
);
for (table, rows) in retained.iter().zip(original.iter()) {
+5
View File
@@ -14,7 +14,10 @@ pub mod memory_staging;
pub mod observation;
pub mod plugin_binding;
pub mod plugin_catalog;
pub mod plugin_host;
pub mod plugin_https;
pub mod plugin_plan;
pub mod plugin_receipt;
pub mod plugin_settings;
pub mod profile_settings;
pub mod prompt_settings;
@@ -167,6 +170,8 @@ pub enum Error {
WorkspaceSigningIdentity { code: String, message: String },
#[error("Plugin catalog error: {0}")]
PluginCatalog(#[from] plugin_catalog::PluginCatalogError),
#[error("Plugin Host operation rejected: {0}")]
PluginHost(#[from] server_api::PluginHostError),
#[error("Workspace Drive error: {0}")]
Drive(#[from] workspace_drive::Error),
#[error("DB execution capacity exceeded")]
@@ -30,7 +30,7 @@ fn plugin_catalog_migration_matches_fresh_schema_from_oldest_retained_and_105()
// Plugin catalog remains migration 106 even when startup applies later steps.
assert_eq!(after.iter().filter(|(version, _)| *version == 106).cloned().collect::<Vec<_>>(),
vec![(106, crate::plugin_catalog::MIGRATION_NAME.into())]);
assert_eq!(after.last().unwrap(), &(108, "Subject Ticket responsibility and current-body notification routing".into()));
assert_eq!(after.last().unwrap(), &(109, "Durable Plugin Host invocations and HTTPS effect receipts".into()));
assert_eq!(current_schema_version(c)?, LATEST_SCHEMA_VERSION);
assert_eq!(c.query_row("SELECT display_name FROM workspaces WHERE workspace_id='plugin-workspace'",[],|r|r.get::<_,String>(0))?,"Plugins");
assert_eq!(c.query_row("SELECT count(*) FROM pragma_foreign_key_check",[],|r|r.get::<_,i64>(0))?,0);
@@ -72,6 +72,10 @@ fn plugin_catalog_migration_marker_failure_rolls_back_then_retries_without_chang
108,
"Subject Ticket responsibility and current-body notification routing".into(),
));
expected.push((
109,
"Durable Plugin Host invocations and HTTPS effect receipts".into(),
));
assert_eq!(workspace_schema_migration_history(c)?, expected);
crate::plugin_catalog::verify(c)?;
Ok(())
+238
View File
@@ -0,0 +1,238 @@
//! Server execution policy composes authenticated Host identity with retained plan
//! ceilings. Catalog management, artifact possession and ordinary configuration
//! selection never issue execution authority. Callers must run the existing live
//! operation/delegation authorizer before this same-database boundary.
use crate::{Error, Result, plugin_binding::PluginConnectionBinding, plugin_plan::*};
use rusqlite::{OptionalExtension, Transaction, params};
use server_api::*;
use sha2::{Digest, Sha256};
pub(crate) fn failure(code: PluginHostErrorCode) -> Error {
PluginHostError {
code,
effect: PluginEffectObservation::NotDispatched,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
}
.into()
}
/// This value stays in the Server. It is never a wire connection capability.
#[derive(Clone)]
pub struct AdmittedPluginOperation {
pub plan: StoredPluginPlan,
pub pin: WorkerPluginPin,
pub binding: Option<PluginConnectionBinding>,
}
/// `execution` is derived by the authenticated Host boundary from current Worker
/// registration, Subject policy and delegation lineage, never deserialized input.
/// Reads deliberately use the current catalog operation, not the launch-time one:
/// adding another artifact must not invalidate an existing immutable pin.
pub fn authorize(
tx: &Transaction<'_>,
workspace: &str,
execution: &HostExecutionIdentity,
plan_id: &str,
installation: &str,
consumer: &PluginConsumer,
operation: &PluginHostOperation,
) -> Result<AdmittedPluginOperation> {
let plan = plugin_plan_load(tx, workspace, plan_id)?;
if plan.plan.execution() != execution || plan.pin_state == PlanPinState::Released {
return Err(failure(PluginHostErrorCode::Denied));
}
let pin = plan
.plan
.plugins()
.iter()
.find(|pin| pin.installation_id() == installation)
.cloned()
.ok_or_else(|| failure(PluginHostErrorCode::Denied))?;
let tool_name = match consumer {
PluginConsumer::ToolCall {
tool_call_id,
tool_name,
} if !tool_call_id.is_empty()
&& tool_call_id.len() <= 512
&& !tool_call_id.chars().any(char::is_control) =>
{
tool_name
}
// The scope is reserved for future authenticated consumers. This initial
// Tool implementation cannot manufacture Service/event admission.
_ => return Err(failure(PluginHostErrorCode::Unsupported)),
};
if !pin.projection().surfaces.contains(&PluginSurface::Tool)
|| !pin.projection().tools.contains(tool_name)
{
return Err(failure(PluginHostErrorCode::Denied));
}
let expected_kind = match plan.pin_state {
PlanPinState::Pending => "pending",
PlanPinState::Active => "active",
PlanPinState::Restorable => "restorable",
PlanPinState::Released => unreachable!(),
};
let retained: bool = tx.query_row(
"SELECT EXISTS(SELECT 1 FROM plugin_references WHERE workspace_id=?1 AND kind=?2 AND owner=?3 AND installation_id=?4 AND artifact_digest=?5)",
params![workspace, expected_kind, plan_id, installation, pin.artifact_digest()], |r| r.get(0),
)?;
if !retained {
return Err(failure(PluginHostErrorCode::Denied));
}
let current: Option<String> = tx.query_row(
"SELECT last_operation FROM plugin_installations WHERE workspace_id=?1 AND installation_id=?2 AND uninstalled=0",
params![workspace, installation], |r| r.get(0),
).optional()?;
let current = current.ok_or_else(|| failure(PluginHostErrorCode::Denied))?;
let artifact = crate::plugin_catalog::plugin_artifact_admit(
tx,
workspace,
installation,
&current,
pin.package_digest(),
pin.artifact_digest(),
)?;
if !pin.matches_artifact(&artifact)? {
return Err(failure(PluginHostErrorCode::Denied));
}
let slot = match operation {
PluginHostOperation::Admit | PluginHostOperation::Publish => {
for reference in &pin.projection().bindings {
live_binding(tx, workspace, reference)?;
}
None
}
PluginHostOperation::Log => {
if !pin.projection().grants.contains(&PluginGrant::Logging) {
return Err(failure(PluginHostErrorCode::Denied));
}
None
}
PluginHostOperation::Open { slot } | PluginHostOperation::Send { slot } => Some(slot),
};
let binding = if let Some(slot) = slot {
if !pin
.projection()
.grants
.contains(&PluginGrant::Https { slot: slot.clone() })
{
return Err(failure(PluginHostErrorCode::Denied));
}
let reference = pin
.projection()
.bindings
.iter()
.find(|r| &r.slot == slot)
.ok_or_else(|| failure(PluginHostErrorCode::Denied))?;
Some(live_binding(tx, workspace, reference)?)
} else {
None
};
Ok(AdmittedPluginOperation { plan, pin, binding })
}
fn live_binding(
tx: &Transaction<'_>,
workspace: &str,
reference: &BindingPolicyReference,
) -> Result<PluginConnectionBinding> {
let state = crate::config_source::load_state(tx, workspace)?
.ok_or_else(|| failure(PluginHostErrorCode::Unavailable))?;
let evaluation = config_source::SnapshotEnvironment::new(state.snapshot)
.evaluate_contract(&state.contract)
.map_err(|_| failure(PluginHostErrorCode::Unavailable))?;
let settings = crate::plugin_settings::settings_from_evaluation(&evaluation)?;
let binding = settings
.bindings
.get(&reference.binding_id)
.ok_or_else(|| failure(PluginHostErrorCode::Denied))?
.clone();
binding
.validate(&crate::plugin_binding::PluginConnectionPolicyCeiling::default())
.map_err(|_| failure(PluginHostErrorCode::Denied))?;
if binding.digest() != reference.policy_digest {
return Err(failure(PluginHostErrorCode::Denied));
}
Ok(binding)
}
/// Only request identity is persisted, never request body, credential values or
/// transient deadlines. Query/header ordering remains part of protocol identity.
pub fn request_fingerprint(
admitted: &AdmittedPluginOperation,
slot: &str,
request: &PluginHttpRequest,
) -> Result<String> {
let reference = admitted
.pin
.projection()
.bindings
.iter()
.find(|r| r.slot == slot)
.ok_or_else(|| failure(PluginHostErrorCode::Denied))?;
let body_digest: String = Sha256::digest(&request.body)
.iter()
.map(|b| format!("{b:02x}"))
.collect();
let content = serde_json::to_vec(&(
"yoi-plugin-effect-v1",
&admitted.plan.plan_id,
admitted.pin.installation_id(),
&reference.binding_id,
slot,
&reference.policy_digest,
&request.method,
&request.path,
&request.query,
&request.headers,
body_digest,
))
.map_err(|_| failure(PluginHostErrorCode::InvalidInput))?;
let hex: String = Sha256::digest(content)
.iter()
.map(|b| format!("{b:02x}"))
.collect();
Ok(format!("sha256:{hex}"))
}
pub(crate) fn verify_receipt_schema(conn: &rusqlite::Connection) -> Result<()> {
let expected = rusqlite::Connection::open_in_memory()?;
expected.execute_batch(crate::plugin_receipt::SCHEMA_SQL)?;
let mut statement = expected.prepare("SELECT name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND tbl_name IN ('plugin_invocations','plugin_effect_receipts') ORDER BY name")?;
for item in statement.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})? {
let (name, sql) = item?;
let actual: Option<String> = conn
.query_row(
"SELECT sql FROM sqlite_schema WHERE name=?1",
[&name],
|row| row.get(0),
)
.optional()?;
if actual.as_deref() != Some(&sql) {
return Err(Error::Store("Plugin receipt schema has drifted".into()));
}
}
Ok(())
}
pub(crate) fn map_error(error: Error) -> PluginHostError {
match error {
Error::PluginHost(error) => error,
Error::PluginCatalog(_) | Error::WorkspacePermissionDenied(_) => {
if let Error::PluginHost(e) = failure(PluginHostErrorCode::Denied) {
e
} else {
unreachable!()
}
}
_ => PluginHostError {
code: PluginHostErrorCode::Unavailable,
effect: PluginEffectObservation::NotDispatched,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
},
}
}
@@ -0,0 +1,66 @@
// Retained migrations build the real predecessor; this is not a latest-schema
// fixture with its version number relabelled. Existing 107/108 history is frozen.
#[test]
fn plugin_receipt_tail_migrates_true_predecessor_reopens_and_matches_fresh_schema() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("server.db");
prepare_retained_schema(&path, 108);
let conn = Connection::open(&path).unwrap();
seed_migration_workspace(&conn);
let history = workspace_schema_migration_history(&conn).unwrap();
assert_eq!(history.last().unwrap().0, 108);
assert!(!table_exists(&conn, "plugin_invocations").unwrap());
let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap();
assert_eq!(plan.current_schema_version, 108);
assert_eq!(plan.migrations.iter().map(|step| step.version).collect::<Vec<_>>(), [109]);
assert_eq!(current_schema_version(&conn).unwrap(), 108);
drop(conn);
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
store.with_conn(|conn| {
crate::plugin_host::verify_receipt_schema(conn)?;
assert_eq!(current_schema_version(conn)?, 109);
let upgraded = workspace_schema_migration_history(conn)?;
assert_eq!(&upgraded[..history.len()], history.as_slice());
assert_eq!(upgraded.last().unwrap().1, "Durable Plugin Host invocations and HTTPS effect receipts");
assert_eq!(conn.query_row("SELECT count(*) FROM workspaces WHERE workspace_id='space'", [], |row| row.get::<_, i64>(0))?, 1);
assert_eq!(conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| row.get::<_, i64>(0))?, 0);
Ok(())
}).unwrap();
drop(store);
for _ in 0..2 {
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
reopened.with_conn(|conn| {
crate::plugin_host::verify_receipt_schema(conn)?;
assert_eq!(workspace_schema_migration_history(conn)?.iter().filter(|(v, _)| *v == 109).count(), 1);
Ok(())
}).unwrap();
}
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
fresh.with_conn(crate::plugin_host::verify_receipt_schema).unwrap();
}
#[test]
fn plugin_receipt_tail_rolls_back_partial_ddl_and_retries_without_history_rewrite() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("server.db");
prepare_retained_schema(&path, 108);
let conn = Connection::open(&path).unwrap();
configure_sqlite(&conn).unwrap();
// A late-name collision fails after the first CREATEs. The owner transaction
// must roll back all new objects and the migration receipt together.
conn.execute_batch("CREATE TABLE plugin_effect_receipts(unrelated INTEGER)").unwrap();
let history = workspace_schema_migration_history(&conn).unwrap();
let before: String = conn.query_row("SELECT group_concat(sql,char(10)) FROM (SELECT sql FROM sqlite_schema WHERE sql IS NOT NULL ORDER BY name)", [], |r| r.get(0)).unwrap();
assert!(migrate_plugin_receipts_v108_to_v109(&conn).is_err());
let after: String = conn.query_row("SELECT group_concat(sql,char(10)) FROM (SELECT sql FROM sqlite_schema WHERE sql IS NOT NULL ORDER BY name)", [], |r| r.get(0)).unwrap();
assert_eq!(before, after);
assert_eq!(workspace_schema_migration_history(&conn).unwrap(), history);
assert!(!table_exists(&conn, "plugin_invocations").unwrap());
conn.execute_batch("DROP TABLE plugin_effect_receipts").unwrap();
migrate_plugin_receipts_v108_to_v109(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 109);
assert!(migrate_plugin_receipts_v108_to_v109(&conn).is_err());
crate::plugin_host::verify_receipt_schema(&conn).unwrap();
conn.execute("UPDATE __yoi_schema_migrations SET name='wrong predecessor' WHERE version=108", []).unwrap();
assert!(verify_schema_history(&conn, 109).is_err());
}
+534
View File
@@ -0,0 +1,534 @@
//! Bounded, single-attempt HTTPS for an already admitted Plugin operation.
//!
//! The caller owns live authorization, secret lookup and durable effect admission.
//! In particular, an execution error is not evidence that an external effect did
//! not happen, and must never cause an automatic replay. No raw network error is
//! retained here. Debug output also omits request/response data and credentials.
use std::collections::BTreeSet;
use std::fmt;
use std::io::Read;
use std::net::{IpAddr, SocketAddr};
use std::time::Duration;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use serde::{Deserialize, Serialize};
use server_api::PluginHttpMethod;
use tokio_util::sync::CancellationToken;
use url::Url;
use crate::plugin_binding::{PluginConnectionBinding, PluginConnectionPolicyCeiling, path_within};
mod policy;
use policy::{PinnedResolver, check_addresses, safe_header};
const MAX_HEADERS: usize = 64;
const MAX_URL_BYTES: usize = 16 * 1024;
const MAX_QUERY_PAIRS: usize = 256;
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HttpsRequest {
pub method: PluginHttpMethod,
pub path: String,
pub query: Vec<(String, String)>,
pub headers: Vec<(String, String)>,
pub body: Vec<u8>,
pub timeout_ms: u32,
pub effect_key: String,
}
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HttpsResponse {
pub status: u16,
pub headers: Vec<(String, String)>,
pub body: Vec<u8>,
}
impl fmt::Debug for HttpsRequest {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("HttpsRequest").finish_non_exhaustive()
}
}
impl fmt::Debug for HttpsResponse {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("HttpsResponse")
.field("status", &self.status)
.finish_non_exhaustive()
}
}
/// Closed, safe categories: no reqwest, URL, DNS, TLS or header parsing source
/// error is exposed. These categories do not authorize retry after execution.
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
pub enum TransportError {
#[error("invalid HTTPS binding")]
InvalidBinding,
#[error("invalid HTTPS request")]
InvalidRequest,
#[error("HTTPS request denied")]
Denied,
#[error("HTTPS header limit exceeded")]
HeaderLimit,
#[error("HTTPS request limit exceeded")]
RequestLimit,
#[error("HTTPS response limit exceeded")]
ResponseLimit,
#[error("HTTPS name resolution failed")]
Dns,
#[error("HTTPS transport unavailable")]
Unavailable,
#[error("HTTPS deadline exceeded")]
DeadlineExceeded,
#[error("HTTPS request cancelled")]
Cancelled,
#[error("HTTPS credential unavailable or invalid")]
InvalidSecret,
#[error("unsupported HTTPS response encoding")]
UnsupportedEncoding,
}
/// Private fields prevent changing the checked request or swapping its binding.
/// The caller can hash/serialize `normalized()` for its durable fingerprint;
/// secret values are never part of this representation. Receipt fingerprinting
/// must exclude timeout and replace the body with its hash as required by API.md.
#[derive(Clone)]
pub struct PreparedRequest {
request: HttpsRequest,
binding_digest: String,
url: Url,
}
impl PreparedRequest {
pub fn normalized(&self) -> &HttpsRequest {
&self.request
}
}
impl fmt::Debug for PreparedRequest {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("PreparedRequest").finish_non_exhaustive()
}
}
/// Additional operator-owned exclusions, not Workspace-granted exceptions.
/// Operators must include all public control-plane IPs/names here. IP exclusions
/// apply to every port, including literal-IP origins and every DNS candidate.
#[derive(Debug, Clone)]
pub struct TransportPolicy {
pub denied_addresses: BTreeSet<IpAddr>,
pub denied_hosts: BTreeSet<String>,
}
pub struct HttpsTransport {
policy: TransportPolicy,
#[cfg(test)]
test_overrides: Option<test_support::TestOverrides>,
}
pub fn prepare(
binding: &PluginConnectionBinding,
mut request: HttpsRequest,
) -> Result<PreparedRequest, TransportError> {
binding
.validate(&PluginConnectionPolicyCeiling::default())
.map_err(|_| TransportError::InvalidBinding)?;
if !binding.methods.contains(&request.method)
|| !path_within("/", &request.path)
|| !binding.allows_path(&request.path)
{
return Err(TransportError::Denied);
}
if request.timeout_ms == 0
|| request.effect_key.is_empty()
|| request.effect_key.len() > 128
|| request.effect_key.chars().any(char::is_control)
{
return Err(TransportError::InvalidRequest);
}
if request.body.len() > binding.limits.max_request_bytes as usize
|| request.query.len() > MAX_QUERY_PAIRS
{
return Err(TransportError::RequestLimit);
}
request.timeout_ms = request.timeout_ms.min(binding.limits.timeout_ms);
if request.headers.len() > MAX_HEADERS {
return Err(TransportError::HeaderLimit);
}
let mut header_bytes = 0usize;
for (name, value) in &mut request.headers {
let parsed =
HeaderName::from_bytes(name.as_bytes()).map_err(|_| TransportError::InvalidRequest)?;
*name = parsed.as_str().to_owned();
if !safe_header(name) || !binding.request_headers.contains(name) {
return Err(TransportError::Denied);
}
// Exclude folding, control bytes and non-ASCII ambiguities even if the
// HTTP parser would accept them. Header values are never trimmed.
if !value.bytes().all(|b| (0x20..=0x7e).contains(&b)) {
return Err(TransportError::InvalidRequest);
}
HeaderValue::from_str(value).map_err(|_| TransportError::InvalidRequest)?;
header_bytes = header_bytes.saturating_add(name.len() + value.len() + 4);
}
if header_bytes > binding.limits.max_header_bytes as usize {
return Err(TransportError::HeaderLimit);
}
// Bound raw query input before percent encoding (whose expansion is at most
// threefold), then bound the actual request target after encoding as well.
let query_bytes = request
.query
.iter()
.try_fold(0usize, |sum, (k, v)| {
sum.checked_add(k.len())?
.checked_add(v.len())?
.checked_add(2)
})
.ok_or(TransportError::RequestLimit)?;
if query_bytes > MAX_URL_BYTES {
return Err(TransportError::RequestLimit);
}
let mut url = Url::parse(&format!("{}{}", binding.origin, request.path))
.map_err(|_| TransportError::InvalidRequest)?;
if url.path() != request.path {
return Err(TransportError::Denied);
}
if !request.query.is_empty() {
url.query_pairs_mut()
.extend_pairs(request.query.iter().map(|(k, v)| (k, v)));
}
if url.as_str().len() > MAX_URL_BYTES {
return Err(TransportError::RequestLimit);
}
Ok(PreparedRequest {
request,
binding_digest: binding.digest(),
url,
})
}
impl HttpsTransport {
pub fn new(policy: TransportPolicy) -> Self {
Self {
policy,
#[cfg(test)]
test_overrides: None,
}
}
/// `secret` is the complete Authorization field value (including its scheme),
/// not a secret reference. Its presence must agree with binding.secret_ref.
/// A fresh client and a pinned resolver are used for every single attempt;
/// neither a cached connection nor a second OS resolution can bypass checks.
/// The caller must cancel this token at the enclosing invocation deadline.
pub async fn execute(
&self,
binding: &PluginConnectionBinding,
prepared: PreparedRequest,
secret: Option<&str>,
cancellation: &CancellationToken,
) -> Result<HttpsResponse, TransportError> {
self.execute_with_guard(binding, prepared, secret, cancellation, || async { Ok(()) })
.await
}
/// Run the caller's fresh authority check only after DNS, address admission,
/// credential/header checks and complete client/request construction. Denial
/// is returned unchanged and no connection is opened. The guard shares the
/// transport deadline and cancellation; there is no await between a successful
/// guard and dispatch. It cannot make authorization atomic with remote I/O.
pub async fn execute_with_guard<G, F>(
&self,
binding: &PluginConnectionBinding,
prepared: PreparedRequest,
secret: Option<&str>,
cancellation: &CancellationToken,
guard: G,
) -> Result<HttpsResponse, TransportError>
where
G: FnOnce() -> F + Send,
F: std::future::Future<Output = Result<(), TransportError>> + Send,
{
let deadline =
tokio::time::Instant::now() + Duration::from_millis(prepared.request.timeout_ms.into());
tokio::select! {
biased;
_ = cancellation.cancelled() => Err(TransportError::Cancelled),
result = tokio::time::timeout_at(deadline, self.execute_inner(binding, prepared, secret, cancellation, deadline, guard)) => {
result.unwrap_or(Err(TransportError::DeadlineExceeded))
}
}
}
async fn execute_inner<G, F>(
&self,
binding: &PluginConnectionBinding,
prepared: PreparedRequest,
secret: Option<&str>,
cancellation: &CancellationToken,
deadline: tokio::time::Instant,
guard: G,
) -> Result<HttpsResponse, TransportError>
where
G: FnOnce() -> F + Send,
F: std::future::Future<Output = Result<(), TransportError>> + Send,
{
// Recheck against the supplied current binding, not only the preparation
// snapshot. A stale prepared policy cannot silently gain new authority.
if prepared.binding_digest != binding.digest() {
return Err(TransportError::Denied);
}
let prepared = prepare(binding, prepared.request)?;
let host = prepared
.url
.host_str()
.ok_or(TransportError::InvalidBinding)?;
let host = host.trim_start_matches('[').trim_end_matches(']');
if self
.policy
.denied_hosts
.iter()
.any(|name| name.eq_ignore_ascii_case(host))
{
return Err(TransportError::Denied);
}
let port = prepared
.url
.port_or_known_default()
.ok_or(TransportError::InvalidBinding)?;
let addresses = self.resolve(host, port).await?;
check_addresses(&addresses, &self.policy)?;
let mut headers = HeaderMap::new();
for (name, value) in &prepared.request.headers {
headers.append(
HeaderName::from_bytes(name.as_bytes())
.map_err(|_| TransportError::InvalidRequest)?,
HeaderValue::from_str(value).map_err(|_| TransportError::InvalidRequest)?,
);
}
match (binding.secret_ref.is_empty(), secret) {
(true, None) => {}
(false, Some(value)) if !value.is_empty() => {
if value.len() > binding.limits.max_header_bytes as usize {
return Err(TransportError::HeaderLimit);
}
if !value.bytes().all(|b| (0x20..=0x7e).contains(&b)) {
return Err(TransportError::InvalidSecret);
}
let mut value =
HeaderValue::from_str(value).map_err(|_| TransportError::InvalidSecret)?;
value.set_sensitive(true);
headers.insert(reqwest::header::AUTHORIZATION, value);
}
_ => return Err(TransportError::InvalidSecret),
}
headers.insert(
reqwest::header::ACCEPT_ENCODING,
HeaderValue::from_static("gzip"),
);
headers.insert(
reqwest::header::CONNECTION,
HeaderValue::from_static("close"),
);
headers
.entry(reqwest::header::ACCEPT)
.or_insert(HeaderValue::from_static("*/*"));
// Include generated fields and the credential in the wire header budget.
let authority = prepared.url[url::Position::BeforeHost..url::Position::AfterPort].len();
check_header_budget(&headers, authority + "host: \r\n".len() + 32, binding)?;
#[cfg(test)]
let addresses = self
.test_overrides
.as_ref()
.and_then(|o| o.connect_to)
.map(|address| vec![address])
.unwrap_or(addresses);
let resolver = PinnedResolver::new(host.to_owned(), addresses);
let builder = reqwest::Client::builder()
.https_only(true)
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.retry(reqwest::retry::never())
.http1_only()
.no_gzip()
.no_brotli()
.no_deflate()
.no_zstd()
.tls_backend_native()
.tls_version_min(reqwest::tls::Version::TLS_1_2)
.tls_sni(true)
.dns_resolver(resolver)
.pool_max_idle_per_host(0);
#[cfg(test)]
let builder = match self.test_overrides.as_ref().and_then(|o| o.root.clone()) {
Some(root) => builder.tls_certs_only([root]),
None => builder,
};
let client = builder.build().map_err(|_| TransportError::Unavailable)?;
let request = client
.request(method(&prepared.request.method), prepared.url)
.headers(headers)
.body(prepared.request.body)
.build()
.map_err(|_| TransportError::InvalidRequest)?;
guard().await?;
if cancellation.is_cancelled() {
return Err(TransportError::Cancelled);
}
if tokio::time::Instant::now() >= deadline {
return Err(TransportError::DeadlineExceeded);
}
let mut response = client.execute(request).await.map_err(network_error)?;
check_header_budget(response.headers(), 0, binding)?;
let status = response.status().as_u16();
let encoding = response_encoding(response.headers())?;
let limit = binding.limits.max_response_bytes as usize;
if response.content_length().is_some_and(|n| n > limit as u64) {
return Err(TransportError::ResponseLimit);
}
let connection_headers = connection_headers(response.headers())?;
let mut returned_headers = Vec::new();
for (name, value) in response.headers() {
if safe_header(name.as_str())
&& !connection_headers.contains(name.as_str())
&& binding.response_headers.iter().any(|h| h == name.as_str())
&& !(encoding == Encoding::Gzip && name == reqwest::header::CONTENT_ENCODING)
{
let value = value.to_str().map_err(|_| TransportError::Unavailable)?;
returned_headers.push((name.as_str().to_owned(), value.to_owned()));
}
}
let mut body = Vec::new();
while let Some(chunk) = response.chunk().await.map_err(network_error)? {
if chunk.len() > limit.saturating_sub(body.len()) {
return Err(TransportError::ResponseLimit);
}
body.extend_from_slice(&chunk);
}
if encoding == Encoding::Gzip {
body = decode_gzip(&body, limit).await?;
}
Ok(HttpsResponse {
status,
headers: returned_headers,
body,
})
}
async fn resolve(&self, host: &str, port: u16) -> Result<Vec<SocketAddr>, TransportError> {
#[cfg(test)]
if let Some(overrides) = &self.test_overrides {
return overrides.resolve().await;
}
if let Ok(ip) = host.parse::<IpAddr>() {
return Ok(vec![SocketAddr::new(ip, port)]);
}
tokio::net::lookup_host((host, port))
.await
.map(|addresses| addresses.collect())
.map_err(|_| TransportError::Dns)
}
}
fn check_header_budget(
headers: &HeaderMap,
generated_bytes: usize,
binding: &PluginConnectionBinding,
) -> Result<(), TransportError> {
let bytes = headers.iter().fold(generated_bytes, |sum, (name, value)| {
sum.saturating_add(name.as_str().len() + value.as_bytes().len() + 4)
});
// Generated Host and Content-Length consume two additional fields.
let generated_count = if generated_bytes == 0 { 0 } else { 2 };
if headers.len() + generated_count > MAX_HEADERS
|| bytes > binding.limits.max_header_bytes as usize
{
return Err(TransportError::HeaderLimit);
}
Ok(())
}
fn connection_headers(headers: &HeaderMap) -> Result<BTreeSet<String>, TransportError> {
let mut nominated = BTreeSet::new();
for value in headers.get_all(reqwest::header::CONNECTION) {
let value = value.to_str().map_err(|_| TransportError::Unavailable)?;
for name in value.split(',') {
let name = HeaderName::from_bytes(name.trim().as_bytes())
.map_err(|_| TransportError::Unavailable)?;
nominated.insert(name.as_str().to_owned());
}
}
Ok(nominated)
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Encoding {
Identity,
Gzip,
}
fn response_encoding(headers: &HeaderMap) -> Result<Encoding, TransportError> {
let mut values = headers.get_all(reqwest::header::CONTENT_ENCODING).iter();
let first = values.next();
if values.next().is_some() {
return Err(TransportError::UnsupportedEncoding);
}
match first {
None => Ok(Encoding::Identity),
Some(value) if value.as_bytes().eq_ignore_ascii_case(b"identity") => Ok(Encoding::Identity),
Some(value) if value.as_bytes().eq_ignore_ascii_case(b"gzip") => Ok(Encoding::Gzip),
_ => Err(TransportError::UnsupportedEncoding),
}
}
async fn decode_gzip(bytes: &[u8], limit: usize) -> Result<Vec<u8>, TransportError> {
let mut decoder = flate2::read::MultiGzDecoder::new(bytes);
let mut output = Vec::new();
let mut chunk = [0u8; 8192];
loop {
let length = decoder
.read(&mut chunk)
.map_err(|_| TransportError::Unavailable)?;
if length == 0 {
break;
}
if length > limit.saturating_sub(output.len()) {
return Err(TransportError::ResponseLimit);
}
output.extend_from_slice(&chunk[..length]);
// Keep cancellation and the enclosing wall deadline observable even for
// concatenated compressed members and expansion-heavy payloads.
tokio::task::yield_now().await;
}
Ok(output)
}
fn network_error(error: reqwest::Error) -> TransportError {
if error.is_timeout() {
TransportError::DeadlineExceeded
} else {
TransportError::Unavailable
}
}
fn method(method: &PluginHttpMethod) -> reqwest::Method {
match method {
PluginHttpMethod::Get => reqwest::Method::GET,
PluginHttpMethod::Head => reqwest::Method::HEAD,
PluginHttpMethod::Post => reqwest::Method::POST,
PluginHttpMethod::Put => reqwest::Method::PUT,
PluginHttpMethod::Patch => reqwest::Method::PATCH,
PluginHttpMethod::Delete => reqwest::Method::DELETE,
PluginHttpMethod::Options => reqwest::Method::OPTIONS,
}
}
#[cfg(test)]
pub(crate) mod test_support;
#[cfg(test)]
mod tests;
@@ -0,0 +1,110 @@
use std::io;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
use super::{TransportError, TransportPolicy};
pub(super) fn safe_header(name: &str) -> bool {
!name.is_empty()
&& name.len() <= 128
&& name
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
&& !name.starts_with("proxy-")
&& !name.starts_with("sec-")
&& !name.starts_with("x-forwarded-")
&& !matches!(
name,
"authorization"
| "www-authenticate"
| "authentication-info"
| "cookie"
| "cookie2"
| "set-cookie"
| "set-cookie2"
| "host"
| "connection"
| "keep-alive"
| "transfer-encoding"
| "te"
| "trailer"
| "upgrade"
| "content-length"
| "expect"
| "forwarded"
| "via"
| "x-api-key"
| "api-key"
| "accept-encoding"
)
}
pub(super) fn check_addresses(
addresses: &[SocketAddr],
policy: &TransportPolicy,
) -> Result<(), TransportError> {
if addresses.is_empty() || addresses.len() > 64 {
return Err(TransportError::Dns);
}
// Reject the complete answer if even one candidate is disallowed. Filtering
// the answer would hide mixed public/private DNS or rebinding attempts.
for address in addresses {
let public = match address.ip() {
IpAddr::V4(ip) => public_ipv4(ip),
IpAddr::V6(ip) => public_ipv6(ip),
};
if !public || policy.denied_addresses.contains(&address.ip()) {
return Err(TransportError::Denied);
}
}
Ok(())
}
fn public_ipv4(ip: Ipv4Addr) -> bool {
let [a, b, c, _] = ip.octets();
ip != Ipv4Addr::new(168, 63, 129, 16)
&& !matches!(a, 0 | 10 | 127 | 224..=255)
&& !(a == 100 && (64..=127).contains(&b))
&& !(a == 169 && b == 254)
&& !(a == 172 && (16..=31).contains(&b))
&& !(a == 192 && (b == 0 || b == 168 || (b == 88 && c == 99)))
&& !(a == 198 && (b == 18 || b == 19 || (b == 51 && c == 100)))
&& !(a == 203 && b == 0 && c == 113)
}
fn public_ipv6(ip: Ipv6Addr) -> bool {
let [a, b, ..] = ip.segments();
(0x2000..=0x3fff).contains(&a)
&& !(a == 0x2001 && (b <= 0x01ff || b == 0x0db8))
&& a != 0x2002
&& !(a == 0x3fff && b <= 0x0fff)
}
/// The HTTP connector cannot fall back to the OS resolver. TLS still receives
/// the original URL hostname for certificate verification and SNI.
pub(super) struct PinnedResolver {
host: String,
addresses: Vec<SocketAddr>,
}
impl PinnedResolver {
pub(super) fn new(host: String, addresses: Vec<SocketAddr>) -> Self {
Self { host, addresses }
}
}
impl Resolve for PinnedResolver {
fn resolve(&self, name: Name) -> Resolving {
let matches = name.as_str() == self.host;
let addresses = self.addresses.clone();
Box::pin(async move {
if !matches {
return Err(
io::Error::new(io::ErrorKind::PermissionDenied, "destination denied").into(),
);
}
Ok(Box::new(addresses.into_iter()) as Addrs)
})
}
}
@@ -0,0 +1,299 @@
//! Isolated TLS peer for crate-local integration tests. Production builds do
//! not contain its private DNS/socket/trust overrides. The certificate hostname
//! still passes through the real HTTPS client's name verification and SNI.
use std::collections::VecDeque;
use std::future::pending;
use std::io::Write;
use std::process::Command;
use std::sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::sync::mpsc;
use tokio::task::{JoinHandle, JoinSet};
use super::*;
use crate::plugin_binding::{PluginConnectionLimits, PluginNetworkPolicy};
pub(crate) const HOST: &str = "api.example.com";
pub(super) struct TestOverrides {
pub(super) connect_to: Option<SocketAddr>,
pub(super) root: Option<reqwest::Certificate>,
pub(super) answers: Mutex<VecDeque<DnsAnswer>>,
pub(super) calls: Arc<AtomicUsize>,
}
pub(super) enum DnsAnswer {
Addresses(Vec<SocketAddr>),
Pending(Option<tokio::sync::oneshot::Sender<()>>),
Barrier {
started: tokio::sync::oneshot::Sender<()>,
release: tokio::sync::oneshot::Receiver<Vec<SocketAddr>>,
},
}
impl TestOverrides {
pub(super) async fn resolve(&self) -> Result<Vec<SocketAddr>, TransportError> {
self.calls.fetch_add(1, Ordering::SeqCst);
let answer = self.answers.lock().unwrap().pop_front();
match answer {
Some(DnsAnswer::Addresses(addresses)) => Ok(addresses),
Some(DnsAnswer::Barrier { started, release }) => {
let _ = started.send(());
release.await.map_err(|_| TransportError::Dns)
}
Some(DnsAnswer::Pending(started)) => {
if let Some(started) = started {
let _ = started.send(());
}
pending().await
}
None => Err(TransportError::Dns),
}
}
}
pub(super) fn policy() -> TransportPolicy {
TransportPolicy {
denied_addresses: BTreeSet::new(),
denied_hosts: BTreeSet::new(),
}
}
pub(super) fn binding() -> PluginConnectionBinding {
PluginConnectionBinding {
origin: format!("https://{HOST}"),
network: PluginNetworkPolicy::PublicInternet,
methods: vec![PluginHttpMethod::Get, PluginHttpMethod::Post],
path_prefixes: vec!["/api".into()],
request_headers: vec!["accept".into(), "x-order".into()],
response_headers: vec![
"content-type".into(),
"x-order".into(),
"location".into(),
"content-encoding".into(),
],
secret_ref: String::new(),
limits: PluginConnectionLimits {
timeout_ms: 5_000,
max_request_bytes: 1024,
max_response_bytes: 1024,
max_header_bytes: 4096,
},
}
}
pub(super) fn request() -> HttpsRequest {
HttpsRequest {
method: PluginHttpMethod::Get,
path: "/api/items".into(),
query: Vec::new(),
headers: Vec::new(),
body: Vec::new(),
timeout_ms: 5_000,
effect_key: "effect-1".into(),
}
}
pub(super) fn addr(ip: &str, port: u16) -> SocketAddr {
SocketAddr::new(ip.parse().unwrap(), port)
}
pub(crate) struct Peer {
pub(super) address: SocketAddr,
pub(super) root: reqwest::Certificate,
pub(super) observations: mpsc::Receiver<Vec<u8>>,
pub(super) connections: Arc<AtomicUsize>,
task: JoinHandle<()>,
_directory: tempfile::TempDir,
}
impl Drop for Peer {
fn drop(&mut self) {
self.task.abort();
}
}
pub(crate) enum Reply {
Bytes(Vec<u8>),
StallAfter(Vec<u8>),
Stall,
Close,
Gated(tokio::sync::oneshot::Receiver<Vec<u8>>),
}
impl Peer {
pub(crate) async fn new(certificate_host: &str, replies: Vec<Reply>) -> Self {
let directory = tempfile::tempdir().unwrap();
let output = Command::new("openssl")
.current_dir(directory.path())
.args([
"req",
"-x509",
"-newkey",
"rsa:2048",
"-noenc",
"-days",
"1",
"-keyout",
"key.pem",
"-out",
"cert.pem",
"-subj",
"/CN=isolated-plugin-peer",
"-addext",
&format!("subjectAltName=DNS:{certificate_host}"),
"-addext",
"extendedKeyUsage=serverAuth",
])
.output()
.expect("TLS tests require openssl");
assert!(
output.status.success(),
"isolated TLS certificate generation failed"
);
let certificate = std::fs::read(directory.path().join("cert.pem")).unwrap();
let key = std::fs::read(directory.path().join("key.pem")).unwrap();
let identity = native_tls::Identity::from_pkcs8(&certificate, &key).unwrap();
let acceptor =
tokio_native_tls::TlsAcceptor::from(native_tls::TlsAcceptor::new(identity).unwrap());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let (observed, observations) = mpsc::channel(16);
let connections = Arc::new(AtomicUsize::new(0));
let count = connections.clone();
let mut replies = VecDeque::from(replies);
let task = tokio::spawn(async move {
let mut handlers = JoinSet::new();
loop {
tokio::select! {
accepted = listener.accept() => {
let (socket, _) = accepted.unwrap();
count.fetch_add(1, Ordering::SeqCst);
let acceptor = acceptor.clone();
let observed = observed.clone();
let reply = replies.pop_front().unwrap_or(Reply::Close);
handlers.spawn(async move {
let Ok(mut socket) = acceptor.accept(socket).await else { return; };
let mut bytes = Vec::new();
let mut buffer = [0; 4096];
loop {
let Ok(length) = socket.read(&mut buffer).await else { return; };
if length == 0 { return; }
bytes.extend_from_slice(&buffer[..length]);
assert!(bytes.len() <= 64 * 1024, "test request exceeded fixture bound");
if let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") {
let text = String::from_utf8_lossy(&bytes[..end]).to_lowercase();
let length = text.lines().find_map(|line| line.strip_prefix("content-length: "))
.map(|n| n.parse::<usize>().unwrap()).unwrap_or(0);
if bytes.len() >= end + 4 + length { break; }
}
}
observed.send(bytes).await.unwrap();
match reply {
Reply::Bytes(bytes) => { let _ = socket.write_all(&bytes).await; let _ = socket.shutdown().await; }
Reply::StallAfter(bytes) => { let _ = socket.write_all(&bytes).await; pending::<()>().await; }
Reply::Stall => pending::<()>().await,
Reply::Close => {},
Reply::Gated(release) => {
if let Ok(bytes) = release.await {
let _ = socket.write_all(&bytes).await;
let _ = socket.shutdown().await;
}
}
}
});
}
Some(result) = handlers.join_next(), if !handlers.is_empty() => { result.unwrap(); }
}
}
});
Self {
address,
root: reqwest::Certificate::from_pem(&certificate).unwrap(),
observations,
connections,
task,
_directory: directory,
}
}
pub(crate) fn binding(&self) -> PluginConnectionBinding {
let mut binding = binding();
binding.origin = format!("https://{HOST}:{}", self.address.port());
binding
}
pub(crate) fn connection_count(&self) -> usize {
self.connections.load(Ordering::SeqCst)
}
pub(crate) fn transport(&self) -> HttpsTransport {
self.transport_for_requests(1)
}
pub(crate) fn transport_for_requests(&self, requests: usize) -> HttpsTransport {
self.transport_with_answers(vec![vec![addr("8.8.8.8", self.address.port())]; requests])
}
pub(crate) fn transport_with_answers(&self, answers: Vec<Vec<SocketAddr>>) -> HttpsTransport {
let mut transport = HttpsTransport::new(policy());
transport.test_overrides = Some(TestOverrides {
connect_to: Some(self.address),
root: Some(self.root.clone()),
answers: Mutex::new(answers.into_iter().map(DnsAnswer::Addresses).collect()),
calls: Arc::new(AtomicUsize::new(0)),
});
transport
}
pub(crate) fn transport_with_dns_gate(
&self,
) -> (
HttpsTransport,
tokio::sync::oneshot::Receiver<()>,
tokio::sync::oneshot::Sender<Vec<SocketAddr>>,
) {
let (started, waiting) = tokio::sync::oneshot::channel();
let (release, answer) = tokio::sync::oneshot::channel();
let mut transport = self.transport();
transport.test_overrides.as_mut().unwrap().answers = Mutex::new(
[DnsAnswer::Barrier {
started,
release: answer,
}]
.into_iter()
.collect(),
);
(transport, waiting, release)
}
pub(crate) fn public_address(&self) -> SocketAddr {
addr("8.8.8.8", self.address.port())
}
pub(crate) async fn observed(&mut self) -> Vec<u8> {
tokio::time::timeout(Duration::from_secs(5), self.observations.recv())
.await
.unwrap()
.unwrap()
}
}
pub(crate) fn response(status: u16, headers: &[(&str, &str)], body: &[u8]) -> Reply {
let mut response = format!(
"HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n",
body.len()
)
.into_bytes();
for (name, value) in headers {
write!(response, "{name}: {value}\r\n").unwrap();
}
response.extend_from_slice(b"\r\n");
response.extend_from_slice(body);
Reply::Bytes(response)
}
@@ -0,0 +1,866 @@
//! Component/connection tests for the production HTTPS adapter. Only DNS answers,
//! the socket route to an isolated peer, and that peer's trust root are replaced;
//! request validation, public-address admission, pinning, TLS name verification,
//! HTTP serialization, response limits and cancellation remain production code.
//! Run with the crate's normal lib tests (`plugin_https` filter). TLS fixtures
//! require the `openssl` executable and never change process environment or trust.
use std::collections::VecDeque;
use std::io::Write;
use std::sync::{
Arc, Mutex,
atomic::{AtomicBool, AtomicUsize, Ordering},
};
use super::test_support::*;
use super::*;
#[test]
fn ambiguous_or_out_of_scope_paths_are_denied() {
let binding = binding();
for path in [
"https://other.example/api",
"//other.example/api",
"/api?x=y",
"/api#fragment",
"/api\\item",
"/api/../item",
"/api/./item",
"/api//item",
"/api/item/",
"/api/%2e%2e/item",
"/api/%2Fitem",
"/api/%5citem",
"/api/%252fitem",
"/apix",
"/api;param",
"/",
"/api/é",
"/api/\nitem",
] {
let mut request = request();
request.path = path.into();
assert!(
matches!(prepare(&binding, request), Err(TransportError::Denied)),
"{path:?}"
);
}
for path in ["/api", "/api/items", "/api/items-1.json", "/api/a:b@c"] {
let mut request = request();
request.path = path.into();
let prepared = prepare(&binding, request).unwrap();
assert_eq!(prepared.url.path(), path);
assert_eq!(prepared.normalized().path, path);
}
}
#[test]
fn query_encoding_and_normalized_headers_preserve_duplicates_and_order() {
let mut request = request();
request.query = vec![("q".into(), "a b&c".into()), ("q".into(), "/?é".into())];
request.headers = vec![
("X-Order".into(), "second".into()),
("x-order".into(), "first".into()),
];
let prepared = prepare(&binding(), request.clone()).unwrap();
assert_eq!(prepared.url.query(), Some("q=a+b%26c&q=%2F%3F%C3%A9"));
assert_eq!(prepared.normalized().query, request.query);
assert_eq!(
prepared.normalized().headers,
vec![
("x-order".into(), "second".into()),
("x-order".into(), "first".into())
]
);
let json = serde_json::to_value(prepared.normalized()).unwrap();
assert_eq!(json["method"], "GET");
assert_eq!(json["headers"][0][0], "x-order");
}
#[test]
fn credential_and_control_headers_never_gain_authority_from_allowlists() {
for name in [
"authorization",
"cookie",
"cookie2",
"set-cookie",
"host",
"connection",
"keep-alive",
"transfer-encoding",
"te",
"trailer",
"upgrade",
"content-length",
"expect",
"proxy-authorization",
"proxy-anything",
"sec-websocket-key",
"forwarded",
"via",
"x-forwarded-arbitrary",
"x-api-key",
"api-key",
"accept-encoding",
] {
let mut binding = binding();
binding.request_headers.push(name.into());
let mut request = request();
request
.headers
.push((name.to_uppercase(), "must-not-send".into()));
assert!(prepare(&binding, request).is_err(), "{name}");
}
for value in ["a\r\nb", "a\0b", "a\tb", "é"] {
let mut request = request();
request.headers.push(("accept".into(), value.into()));
assert!(matches!(
prepare(&binding(), request),
Err(TransportError::InvalidRequest)
));
}
}
#[test]
fn request_limits_timeout_and_effect_identity_are_validated_before_dispatch() {
let mut binding = binding();
binding.limits.max_request_bytes = 4;
let mut input = request();
input.body = vec![0; 4];
input.timeout_ms = 60_000;
assert_eq!(
prepare(&binding, input.clone())
.unwrap()
.normalized()
.timeout_ms,
5_000
);
input.body.push(0);
assert!(matches!(
prepare(&binding, input),
Err(TransportError::RequestLimit)
));
let mut input = request();
input.headers = vec![("accept".into(), "ok".into()); 65];
assert!(matches!(
prepare(&binding, input),
Err(TransportError::HeaderLimit)
));
let mut input = request();
input.headers = vec![("accept".into(), "x".repeat(4096))];
assert!(matches!(
prepare(&binding, input),
Err(TransportError::HeaderLimit)
));
let mut input = request();
input.query = vec![("q".into(), "é".repeat(4000))];
assert!(matches!(
prepare(&binding, input),
Err(TransportError::RequestLimit)
));
for key in [String::new(), "a".repeat(129), "a\nb".into()] {
let mut input = request();
input.effect_key = key;
assert!(matches!(
prepare(&binding, input),
Err(TransportError::InvalidRequest)
));
}
let mut input = request();
input.timeout_ms = 0;
assert!(matches!(
prepare(&binding, input),
Err(TransportError::InvalidRequest)
));
let mut input = request();
input.method = PluginHttpMethod::Delete;
assert!(matches!(
prepare(&binding, input),
Err(TransportError::Denied)
));
}
#[test]
fn all_dns_candidates_must_be_public_and_outside_operator_exclusions() {
let public = addr("8.8.8.8", 443);
for ip in [
"0.0.0.0",
"10.1.2.3",
"100.100.100.200",
"127.0.0.1",
"169.254.169.254",
"172.16.0.1",
"192.168.1.1",
"192.0.0.1",
"198.18.0.1",
"198.51.100.1",
"203.0.113.1",
"224.0.0.1",
"255.255.255.255",
"168.63.129.16",
"::",
"::1",
"::ffff:8.8.8.8",
"fc00::1",
"fe80::1",
"ff02::1",
"2001:db8::1",
"2001::1",
"2002:0808:0808::1",
"64:ff9b::808:808",
"3fff::1",
] {
for candidates in [vec![addr(ip, 443), public], vec![public, addr(ip, 443)]] {
assert_eq!(
check_addresses(&candidates, &policy()),
Err(TransportError::Denied),
"{ip}"
);
}
}
assert_eq!(
check_addresses(&[public, addr("2606:4700:4700::1111", 443)], &policy()),
Ok(())
);
let mut denied = policy();
denied.denied_addresses.insert(public.ip());
assert_eq!(
check_addresses(&[public], &denied),
Err(TransportError::Denied)
);
assert_eq!(check_addresses(&[], &policy()), Err(TransportError::Dns));
assert_eq!(
check_addresses(&vec![public; 65], &policy()),
Err(TransportError::Dns)
);
}
#[tokio::test]
async fn pinned_connector_rejects_any_other_hostname_without_dns_fallback() {
use reqwest::dns::Resolve;
let resolver = PinnedResolver::new(HOST.into(), vec![addr("8.8.8.8", 443)]);
let resolved: Vec<_> = resolver
.resolve(HOST.parse().unwrap())
.await
.unwrap()
.collect();
assert_eq!(resolved, vec![addr("8.8.8.8", 443)]);
assert!(
resolver
.resolve("other.example.com".parse().unwrap())
.await
.is_err()
);
}
async fn execute(
transport: &HttpsTransport,
binding: &PluginConnectionBinding,
input: HttpsRequest,
secret: Option<&str>,
) -> Result<HttpsResponse, TransportError> {
transport
.execute(
binding,
prepare(binding, input)?,
secret,
&CancellationToken::new(),
)
.await
}
#[tokio::test]
async fn tls_peer_receives_encoded_query_duplicate_headers_and_scoped_secret() {
let mut peer = Peer::new(
HOST,
vec![response(
503,
&[
("Content-Type", "application/octet-stream"),
("X-Order", "first"),
("X-Order", "second"),
("Set-Cookie", "must-not-return"),
("Authorization", "must-not-return"),
],
&[0, 255, 128],
)],
)
.await;
let mut binding = peer.binding();
binding.secret_ref = "credential-ref".into();
let mut input = request();
input.method = PluginHttpMethod::Post;
input.query = vec![("q".into(), "a & b".into()), ("q".into(), "/?".into())];
input.headers = vec![
("x-order".into(), "second".into()),
("X-Order".into(), "first".into()),
];
input.body = vec![1, 0, 255];
let output = execute(
&peer.transport(),
&binding,
input,
Some("Bearer isolated-test-secret"),
)
.await
.unwrap();
assert_eq!(output.status, 503);
assert_eq!(output.body, vec![0, 255, 128]);
assert_eq!(
output
.headers
.iter()
.filter(|(k, _)| k == "x-order")
.map(|(_, v)| v.as_str())
.collect::<Vec<_>>(),
["first", "second"]
);
assert!(
!output
.headers
.iter()
.any(|(k, _)| k == "set-cookie" || k == "authorization")
);
let wire = peer.observed().await;
let text = String::from_utf8_lossy(&wire).to_lowercase();
assert!(text.starts_with("post /api/items?q=a+%26+b&q=%2f%3f http/1.1\r\n"));
assert!(text.contains(&format!("host: {HOST}:{}\r\n", peer.address.port())));
assert!(text.contains("authorization: bearer isolated-test-secret\r\n"));
assert!(text.contains("x-order: second\r\nx-order: first\r\n"));
assert!(wire.ends_with(&[1, 0, 255]));
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn response_connection_nominated_headers_are_not_exposed() {
let peer = Peer::new(
HOST,
vec![response(
200,
&[
("Connection", "x-order"),
("X-Order", "hop-value"),
("Content-Type", "text/plain"),
],
b"ok",
)],
)
.await;
let output = execute(&peer.transport(), &peer.binding(), request(), None)
.await
.unwrap();
assert!(!output.headers.iter().any(|(name, _)| name == "x-order"));
assert!(
output
.headers
.iter()
.any(|(name, _)| name == "content-type")
);
}
#[tokio::test]
async fn redirects_are_returned_without_following_or_forwarding_credentials() {
let peer = Peer::new(
HOST,
vec![response(
302,
&[("Location", "https://127.0.0.1/private")],
b"redirect",
)],
)
.await;
let mut binding = peer.binding();
binding.secret_ref = "credential-ref".into();
let output = execute(
&peer.transport(),
&binding,
request(),
Some("Bearer secret"),
)
.await
.unwrap();
assert_eq!(output.status, 302);
assert_eq!(output.body, b"redirect");
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn tls_name_mismatch_and_untrusted_certificate_are_rejected() {
for (host, trust) in [("wrong.example.com", true), (HOST, false)] {
let peer = Peer::new(host, vec![]).await;
let mut transport = peer.transport();
if !trust {
transport.test_overrides.as_mut().unwrap().root = None;
}
assert_eq!(
execute(&transport, &peer.binding(), request(), None).await,
Err(TransportError::Unavailable)
);
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
assert!(peer.observations.is_empty());
}
}
#[tokio::test]
async fn response_loss_never_retries_even_a_get() {
let mut peer = Peer::new(
HOST,
vec![Reply::Close, response(200, &[], b"must-not-retry")],
)
.await;
assert_eq!(
execute(&peer.transport(), &peer.binding(), request(), None).await,
Err(TransportError::Unavailable)
);
peer.observed().await;
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn mixed_dns_is_denied_before_connect_and_rebinding_is_rechecked_per_execution() {
let peer = Peer::new(HOST, vec![response(200, &[], b"ok")]).await;
let port = peer.address.port();
let transport =
peer.transport_with_answers(vec![vec![addr("8.8.8.8", port), addr("127.0.0.1", port)]]);
assert_eq!(
execute(&transport, &peer.binding(), request(), None).await,
Err(TransportError::Denied)
);
assert_eq!(peer.connections.load(Ordering::SeqCst), 0);
let transport = peer.transport_with_answers(vec![
vec![addr("8.8.8.8", port)],
vec![addr("127.0.0.1", port)],
]);
assert_eq!(
execute(&transport, &peer.binding(), request(), None)
.await
.unwrap()
.body,
b"ok"
);
assert_eq!(
transport
.test_overrides
.as_ref()
.unwrap()
.calls
.load(Ordering::SeqCst),
1
);
assert_eq!(
execute(&transport, &peer.binding(), request(), None).await,
Err(TransportError::Denied)
);
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
assert_eq!(
transport
.test_overrides
.as_ref()
.unwrap()
.calls
.load(Ordering::SeqCst),
2
);
}
#[tokio::test]
async fn operator_control_plane_exclusions_apply_before_connect() {
let peer = Peer::new(HOST, vec![]).await;
for denied_host in [false, true] {
let mut transport = peer.transport();
if denied_host {
transport.policy.denied_hosts.insert(HOST.to_uppercase());
} else {
transport
.policy
.denied_addresses
.insert("8.8.8.8".parse().unwrap());
}
assert_eq!(
execute(&transport, &peer.binding(), request(), None).await,
Err(TransportError::Denied)
);
}
assert_eq!(peer.connections.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn binding_swap_invalid_credentials_and_generated_header_overflow_never_dispatch() {
let peer = Peer::new(HOST, vec![]).await;
let transport = peer.transport();
let mut binding = peer.binding();
let prepared = prepare(&binding, request()).unwrap();
binding.path_prefixes = vec!["/api/items".into()];
assert_eq!(
transport
.execute(&binding, prepared, None, &CancellationToken::new())
.await,
Err(TransportError::Denied)
);
for (reference, secret) in [
("", Some("Bearer unexpected")),
("credential", None),
("credential", Some("Bearer a\r\nb")),
] {
let mut binding = peer.binding();
binding.secret_ref = reference.into();
// Each rejected operation gets an explicit answer; none opens a socket.
let transport = peer.transport();
assert_eq!(
execute(&transport, &binding, request(), secret).await,
Err(TransportError::InvalidSecret)
);
}
let mut binding = peer.binding();
binding.secret_ref = "credential".into();
binding.limits.max_header_bytes = 64;
assert_eq!(
execute(
&peer.transport(),
&binding,
request(),
Some(&"s".repeat(100))
)
.await,
Err(TransportError::HeaderLimit)
);
assert_eq!(peer.connections.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn response_wire_size_count_and_header_bytes_are_bounded_before_return() {
let cases = [
response(200, &[], &vec![0; 1025]),
Reply::Bytes(
b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n401\r\n"
.iter()
.copied()
.chain(vec![0; 1025])
.chain(b"\r\n0\r\n\r\n".iter().copied())
.collect(),
),
response(200, &[("X-Order", &"x".repeat(4096))], b"ok"),
response(200, &vec![("X-Order", "x"); 65], b"ok"),
];
for (index, reply) in cases.into_iter().enumerate() {
let peer = Peer::new(HOST, vec![reply]).await;
let result = execute(&peer.transport(), &peer.binding(), request(), None).await;
let expected = if index < 2 {
TransportError::ResponseLimit
} else {
TransportError::HeaderLimit
};
assert_eq!(result, Err(expected), "case {index}");
}
}
fn gzip(bytes: &[u8]) -> Vec<u8> {
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
encoder.write_all(bytes).unwrap();
encoder.finish().unwrap()
}
#[tokio::test]
async fn gzip_is_bounded_after_expansion_including_concatenated_members() {
for (payload, accepted) in [
(gzip(&vec![0; 1024]), true),
(gzip(&vec![0; 1025]), false),
([gzip(&vec![0; 600]), gzip(&vec![0; 600])].concat(), false),
] {
let peer = Peer::new(
HOST,
vec![response(200, &[("Content-Encoding", "gzip")], &payload)],
)
.await;
let result = execute(&peer.transport(), &peer.binding(), request(), None).await;
if accepted {
let result = result.unwrap();
assert_eq!(result.body.len(), 1024);
assert!(!result.headers.iter().any(|(k, _)| k == "content-encoding"));
} else {
assert_eq!(result, Err(TransportError::ResponseLimit));
}
}
}
#[tokio::test]
async fn unsupported_or_corrupt_compression_is_not_returned_as_a_success() {
for (headers, expected) in [
(
vec![("Content-Encoding", "br")],
TransportError::UnsupportedEncoding,
),
(
vec![
("Content-Encoding", "gzip"),
("Content-Encoding", "identity"),
],
TransportError::UnsupportedEncoding,
),
(
vec![("Content-Encoding", "gzip")],
TransportError::Unavailable,
),
] {
let peer = Peer::new(HOST, vec![response(200, &headers, b"not-compressed")]).await;
assert_eq!(
execute(&peer.transport(), &peer.binding(), request(), None).await,
Err(expected)
);
}
}
#[tokio::test]
async fn cancellation_covers_dns_and_pre_cancelled_requests_without_network() {
let mut transport = HttpsTransport::new(policy());
let calls = Arc::new(AtomicUsize::new(0));
transport.test_overrides = Some(TestOverrides {
connect_to: None,
root: None,
answers: Mutex::new(VecDeque::from([DnsAnswer::Pending(None)])),
calls: calls.clone(),
});
let cancellation = CancellationToken::new();
cancellation.cancel();
let binding = binding();
assert_eq!(
transport
.execute(
&binding,
prepare(&binding, request()).unwrap(),
None,
&cancellation
)
.await,
Err(TransportError::Cancelled)
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let mut input = request();
input.timeout_ms = 20;
assert_eq!(
execute(&transport, &binding, input, None).await,
Err(TransportError::DeadlineExceeded)
);
assert_eq!(calls.load(Ordering::SeqCst), 1);
let (started, observed) = tokio::sync::oneshot::channel();
transport
.test_overrides
.as_mut()
.unwrap()
.answers
.lock()
.unwrap()
.push_back(DnsAnswer::Pending(Some(started)));
let cancellation = CancellationToken::new();
let operation = transport.execute(
&binding,
prepare(&binding, request()).unwrap(),
None,
&cancellation,
);
tokio::pin!(operation);
tokio::select! {
value = &mut operation => panic!("pending DNS completed unexpectedly: {value:?}"),
value = observed => value.unwrap(),
}
cancellation.cancel();
assert_eq!(operation.await, Err(TransportError::Cancelled));
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn cancellation_and_deadline_abort_inflight_response_waits() {
for reply in [
Reply::Stall,
Reply::StallAfter(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\na".to_vec()),
] {
let mut peer = Peer::new(HOST, vec![reply]).await;
let transport = peer.transport();
let binding = peer.binding();
let cancellation = CancellationToken::new();
let operation = transport.execute(
&binding,
prepare(&binding, request()).unwrap(),
None,
&cancellation,
);
tokio::pin!(operation);
tokio::select! {
value = &mut operation => panic!("response completed before cancellation: {value:?}"),
_ = peer.observed() => {},
}
cancellation.cancel();
assert_eq!(operation.await, Err(TransportError::Cancelled));
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
}
let mut peer = Peer::new(HOST, vec![Reply::Stall]).await;
let mut input = request();
input.timeout_ms = 250;
let result = execute(&peer.transport(), &peer.binding(), input, None).await;
assert_eq!(result, Err(TransportError::DeadlineExceeded));
peer.observed().await;
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn revocation_during_dns_is_checked_by_final_guard_without_opening_peer_connection() {
let peer = Peer::new(HOST, vec![]).await;
let mut transport = peer.transport();
let (dns_started, dns_observed) = tokio::sync::oneshot::channel();
let (dns_release, dns_answer) = tokio::sync::oneshot::channel();
transport.test_overrides.as_mut().unwrap().answers =
Mutex::new(VecDeque::from([DnsAnswer::Barrier {
started: dns_started,
release: dns_answer,
}]));
let authority = AtomicBool::new(true);
let guard_called = AtomicBool::new(false);
let binding = peer.binding();
let cancellation = CancellationToken::new();
let operation = transport.execute_with_guard(
&binding,
prepare(&binding, request()).unwrap(),
None,
&cancellation,
|| async {
guard_called.store(true, Ordering::SeqCst);
tokio::task::yield_now().await;
if authority.load(Ordering::SeqCst) {
Ok(())
} else {
Err(TransportError::Denied)
}
},
);
tokio::pin!(operation);
tokio::select! {
value = &mut operation => panic!("operation completed during blocked DNS: {value:?}"),
value = dns_observed => value.unwrap(),
}
assert!(!guard_called.load(Ordering::SeqCst));
assert_eq!(peer.connection_count(), 0);
authority.store(false, Ordering::SeqCst);
dns_release
.send(vec![addr("8.8.8.8", peer.address.port())])
.unwrap();
assert_eq!(operation.await, Err(TransportError::Denied));
assert!(guard_called.load(Ordering::SeqCst));
assert_eq!(peer.connection_count(), 0);
assert!(peer.observations.is_empty());
}
#[tokio::test]
async fn successful_async_dispatch_guard_runs_once_and_allows_the_prepared_request() {
let peer = Peer::new(HOST, vec![response(200, &[], b"guarded")]).await;
let transport = peer.transport();
let binding = peer.binding();
let calls = AtomicUsize::new(0);
let output = transport
.execute_with_guard(
&binding,
prepare(&binding, request()).unwrap(),
None,
&CancellationToken::new(),
|| async {
tokio::task::yield_now().await;
calls.fetch_add(1, Ordering::SeqCst);
Ok(())
},
)
.await
.unwrap();
assert_eq!(output.body, b"guarded");
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(peer.connection_count(), 1);
}
#[tokio::test]
async fn dispatch_guard_is_cancelled_with_the_operation_and_never_opens_a_connection() {
let peer = Peer::new(HOST, vec![]).await;
let transport = peer.transport();
let binding = peer.binding();
let cancellation = CancellationToken::new();
let (started, observed) = tokio::sync::oneshot::channel();
let operation = transport.execute_with_guard(
&binding,
prepare(&binding, request()).unwrap(),
None,
&cancellation,
|| async {
started.send(()).unwrap();
std::future::pending::<Result<(), TransportError>>().await
},
);
tokio::pin!(operation);
tokio::select! {
value = &mut operation => panic!("pending authority guard completed: {value:?}"),
value = observed => value.unwrap(),
}
cancellation.cancel();
assert_eq!(operation.await, Err(TransportError::Cancelled));
assert_eq!(peer.connection_count(), 0);
}
#[tokio::test(start_paused = true)]
async fn dispatch_guard_deadline_expiry_prevents_connection_even_if_guard_returns_success() {
let peer = Peer::new(HOST, vec![]).await;
let transport = peer.transport();
let binding = peer.binding();
let result = transport
.execute_with_guard(
&binding,
prepare(&binding, request()).unwrap(),
None,
&CancellationToken::new(),
|| async {
tokio::time::advance(Duration::from_secs(6)).await;
Ok(())
},
)
.await;
assert_eq!(result, Err(TransportError::DeadlineExceeded));
assert_eq!(peer.connection_count(), 0);
}
#[tokio::test]
async fn dispatch_guard_cancellation_prevents_connection_even_if_guard_returns_success() {
let peer = Peer::new(HOST, vec![]).await;
let transport = peer.transport();
let binding = peer.binding();
let cancellation = CancellationToken::new();
let result = transport
.execute_with_guard(
&binding,
prepare(&binding, request()).unwrap(),
None,
&cancellation,
|| async {
cancellation.cancel();
Ok(())
},
)
.await;
assert_eq!(result, Err(TransportError::Cancelled));
assert_eq!(peer.connection_count(), 0);
}
#[test]
fn diagnostic_formats_omit_request_response_urls_paths_and_secrets() {
let mut input = request();
input.path = "/api/private-path".into();
input.body = b"secret-body".to_vec();
let prepared = prepare(&binding(), input.clone()).unwrap();
let output = HttpsResponse {
status: 200,
headers: vec![("x-order".into(), "secret-header".into())],
body: b"secret-body".to_vec(),
};
let debug = format!("{input:?} {prepared:?} {output:?}");
for forbidden in [HOST, "private-path", "secret-body", "secret-header"] {
assert!(!debug.contains(forbidden));
}
let error = TransportError::Unavailable;
assert_eq!(error.to_string(), "HTTPS transport unavailable");
assert!(std::error::Error::source(&error).is_none());
}
@@ -30,6 +30,10 @@ fn plugin_plan_migration_marker_failure_preserves_catalog_then_retries_and_match
108,
"Subject Ticket responsibility and current-body notification routing".into(),
));
expected.push((
109,
"Durable Plugin Host invocations and HTTPS effect receipts".into(),
));
assert_eq!(workspace_schema_migration_history(conn)?, expected);
subject_assignment_migration::verify(conn)?;
crate::plugin_plan::verify_plan_schema(conn)?;
@@ -0,0 +1,477 @@
//! Durable Host-issued logical invocations and all-method effect receipts.
//!
//! These owner functions are storage, NOT authentication or admission authority.
//! The caller verifies the authenticated source, current plan/installation,
//! delegation, grants, binding and result-read authority on *every* call, including
//! replay. Consumer identities come from durable Host Tool-call/event records,
//! never the guest. A saved invocation or receipt confers no execution authority.
//!
//! Use an IMMEDIATE owner transaction. Commit `begin` and `effect_admit` before
//! dispatching any network request; a returned `Dispatch` from an uncommitted or
//! failed commit is NOT permission to send. No transaction is held across I/O.
//! Every inserted receipt starts with a durable indeterminate error. Losing the
//! dispatch opportunity (even before sending) sacrifices availability, never
//! safety: no existing receipt is dispatchable, for ANY HTTP method.
//!
//! The parent owns migration registration and Workspace deletion. This module
//! neither registers a schema at runtime nor creates pins/grants/catalog state.
use crate::{
Error, Result,
plugin_plan::{HostExecutionIdentity, StoredPluginPlan},
};
use rusqlite::{Connection, OptionalExtension, Transaction, params};
use server_api::{
PluginConsumer, PluginEffectObservation, PluginHostError, PluginHostErrorCode,
PluginHttpResponse,
};
pub const SCHEMA_SQL: &str = include_str!("plugin_receipt/schema.sql");
/// Buffered body ceiling; encoded allowed headers have their separate finite bound.
/// Both are protected Workspace data rather than diagnostic text.
pub const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
pub const MAX_RESPONSE_HEADER_BYTES: usize = 64 * 1024;
pub const MAX_RESPONSE_HEADERS: usize = 128;
const MAX_ERROR_BYTES: usize = 4096;
type HttpResult = std::result::Result<PluginHttpResponse, PluginHostError>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct InvocationRecord {
pub invocation_id: String,
pub workspace_id: String,
pub plan_id: String,
pub installation_id: String,
pub execution: HostExecutionIdentity,
pub consumer: PluginConsumer,
pub expires_at_ms: i64,
pub cancelled: bool,
pub expired: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum EffectAdmission {
/// Only a first INSERT, to be committed durably before external dispatch.
Dispatch,
/// Includes a saved indeterminate error for pending/unknown outcomes.
Replay(HttpResult),
}
fn host_error(code: PluginHostErrorCode, effect: PluginEffectObservation) -> PluginHostError {
PluginHostError {
code,
effect,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
}
}
fn fail<T>(code: PluginHostErrorCode, effect: PluginEffectObservation) -> Result<T> {
Err(host_error(code, effect).into())
}
fn invalid<T>() -> Result<T> {
fail(
PluginHostErrorCode::InvalidInput,
PluginEffectObservation::NotDispatched,
)
}
fn conflict<T>() -> Result<T> {
// A conflicting key or logical identity might already have external effects.
fail(
PluginHostErrorCode::Conflict,
PluginEffectObservation::Indeterminate,
)
}
fn expired<T>() -> Result<T> {
fail(
PluginHostErrorCode::ReceiptExpired,
PluginEffectObservation::Indeterminate,
)
}
fn valid_identity(value: &str, max: usize) -> bool {
!value.is_empty() && value.len() <= max && !value.chars().any(char::is_control)
}
fn valid_fingerprint(value: &str) -> bool {
value.strip_prefix("sha256:").is_some_and(|hex| {
hex.len() == 64
&& hex
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
})
}
fn encode(value: &impl serde::Serialize) -> Result<String> {
serde_json::to_string(value).map_err(|_| Error::Store("invalid Plugin receipt data".into()))
}
fn decode<T: serde::de::DeserializeOwned>(value: &str) -> Result<T> {
serde_json::from_str(value)
.map_err(|_| Error::Store("invalid persisted Plugin receipt data".into()))
}
// The exported name is immutable input, NOT part of logical identity. Renaming
// an export/tool on the same durable call/event conflicts, rather than opening a
// fresh effect-key namespace. JSON tuples avoid separator/escaping collisions.
fn consumer_key(consumer: &PluginConsumer) -> Result<String> {
match consumer {
PluginConsumer::ToolCall {
tool_call_id,
tool_name,
} => {
if !valid_identity(tool_call_id, 512) || !valid_identity(tool_name, 512) {
return invalid();
}
encode(&["tool_call", tool_call_id])
}
PluginConsumer::Consumer {
consumer_id,
event_id,
export_name,
} => {
if !valid_identity(consumer_id, 512)
|| !valid_identity(event_id, 512)
|| !valid_identity(export_name, 512)
{
return invalid();
}
encode(&["consumer", consumer_id, event_id])
}
}
}
/// Establish/recover a Host UUID for a durable consumer under the stored plan.
/// The first receipt-retention deadline is fixed; retries do not extend it.
/// Replay returns the identity even after cancellation/expiry, not new execution
/// authority. `effect_admit` refuses expired invocations, including unused keys.
/// A new Tool-call/event identity must represent an explicit new request.
pub fn begin(
tx: &Transaction<'_>,
stored: &StoredPluginPlan,
installation_id: &str,
consumer: &PluginConsumer,
now_ms: i64,
retention_ms: i64,
) -> Result<InvocationRecord> {
let Some(expires_at_ms) = now_ms.checked_add(retention_ms) else {
return invalid();
};
if now_ms < 0 || retention_ms <= 0 || !valid_identity(installation_id, 512) {
return invalid();
}
// Snapshot integrity only. Live security authority belongs to the caller.
stored.plan.validate_canonical()?;
if !valid_identity(&stored.plan_id, 512)
|| stored.fingerprint != stored.plan.fingerprint()?
|| !stored
.plan
.plugins()
.iter()
.any(|pin| pin.installation_id() == installation_id)
{
return invalid();
}
let workspace = stored.plan.workspace_id();
let execution = stored.plan.execution();
let key = consumer_key(consumer)?;
let execution_json = encode(execution)?;
let consumer_json = encode(consumer)?;
let saved = tx.query_row(
"SELECT invocation_id,plan_fingerprint FROM plugin_invocations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND installation_id=?4 AND consumer_key=?5",
params![workspace, execution.runtime_id, execution.worker_id, installation_id, key],
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
).optional()?;
if let Some((id, fingerprint)) = saved {
let record = lookup(tx, workspace, &id)?;
if record.plan_id != stored.plan_id
|| fingerprint != stored.fingerprint
|| record.execution != *execution
|| record.consumer != *consumer
{
return conflict();
}
return Ok(record);
}
let invocation_id = uuid::Uuid::now_v7().to_string();
tx.execute(
"INSERT INTO plugin_invocations(workspace_id,invocation_id,plan_id,plan_fingerprint,installation_id,runtime_id,worker_id,execution_json,consumer_key,consumer_json,created_at_ms,expires_at_ms) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",
params![workspace, invocation_id, stored.plan_id, stored.fingerprint, installation_id, execution.runtime_id, execution.worker_id, execution_json, key, consumer_json, now_ms, expires_at_ms],
)?;
lookup(tx, workspace, &invocation_id)
}
/// Persisted identity lookup only; the caller checks current read authority.
pub fn lookup(conn: &Connection, workspace: &str, invocation_id: &str) -> Result<InvocationRecord> {
if !valid_identity(workspace, 512) || !valid_identity(invocation_id, 512) {
return invalid();
}
let row = conn.query_row(
"SELECT plan_id,installation_id,execution_json,consumer_json,expires_at_ms,cancelled,expired FROM plugin_invocations WHERE workspace_id=?1 AND invocation_id=?2",
params![workspace, invocation_id],
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?, row.get::<_, String>(3)?, row.get::<_, i64>(4)?, row.get::<_, bool>(5)?, row.get::<_, bool>(6)?)),
).optional()?;
let Some((plan_id, installation_id, execution, consumer, expires_at_ms, cancelled, expired)) =
row
else {
return fail(
PluginHostErrorCode::NotFound,
PluginEffectObservation::NotDispatched,
);
};
Ok(InvocationRecord {
invocation_id: invocation_id.into(),
workspace_id: workspace.into(),
plan_id,
installation_id,
execution: decode(&execution)?,
consumer: decode(&consumer)?,
expires_at_ms,
cancelled,
expired,
})
}
// Never trust a passed record's mutable flags/deadline or accept a fabricated
// plan/execution/consumer. Records may be stale only in monotonic terminal flags.
fn verify_record(conn: &Connection, supplied: &InvocationRecord) -> Result<InvocationRecord> {
let saved = lookup(conn, &supplied.workspace_id, &supplied.invocation_id)?;
if supplied.plan_id != saved.plan_id
|| supplied.installation_id != saved.installation_id
|| supplied.execution != saved.execution
|| supplied.consumer != saved.consumer
|| supplied.expires_at_ms != saved.expires_at_ms
{
return conflict();
}
Ok(saved)
}
fn is_expired(conn: &Connection, record: &InvocationRecord) -> Result<bool> {
Ok(conn.query_row(
"SELECT expired FROM plugin_invocations WHERE workspace_id=?1 AND invocation_id=?2",
params![record.workspace_id, record.invocation_id],
|row| row.get(0),
)?)
}
fn validate_key(effect_key: &str, fingerprint: &str) -> Result<()> {
if !valid_identity(effect_key, 128) || !valid_fingerprint(fingerprint) {
return invalid();
}
Ok(())
}
struct SavedEffect {
fingerprint: String,
state: String,
status: Option<u16>,
headers: Option<String>,
body: Option<Vec<u8>>,
error: Option<String>,
}
fn saved_effect(
conn: &Connection,
record: &InvocationRecord,
key: &str,
) -> Result<Option<SavedEffect>> {
Ok(conn.query_row(
"SELECT fingerprint,state,response_status,response_headers,response_body,error_json FROM plugin_effect_receipts WHERE workspace_id=?1 AND invocation_id=?2 AND effect_key=?3",
params![record.workspace_id, record.invocation_id, key],
|row| Ok(SavedEffect { fingerprint: row.get(0)?, state: row.get(1)?, status: row.get(2)?, headers: row.get(3)?, body: row.get(4)?, error: row.get(5)? }),
).optional()?)
}
impl SavedEffect {
fn result(self) -> Result<HttpResult> {
if self.state == "expired" {
return expired();
}
if let Some(json) = self.error {
return Ok(Err(decode(&json)?));
}
match (self.status, self.headers, self.body) {
(Some(status), Some(headers), Some(body)) if self.state == "completed" => {
Ok(Ok(PluginHttpResponse {
status,
headers: decode(&headers)?,
body,
}))
}
_ => Err(Error::Store(
"invalid persisted Plugin receipt result".into(),
)),
}
}
}
/// The parent supplies a canonical request fingerprint covering all HTTP methods,
/// binding/slot/policy, plan, path, ordered query/headers and body digest, excluding
/// secret values and remaining timeout. Never accepts/stores raw request data.
/// Commit the insertion before I/O. Repeated pending/unknown results NEVER dispatch.
pub fn effect_admit(
tx: &Transaction<'_>,
record: &InvocationRecord,
effect_key: &str,
fingerprint: &str,
now_ms: i64,
) -> Result<EffectAdmission> {
validate_key(effect_key, fingerprint)?;
if now_ms < 0 {
return invalid();
}
let record = verify_record(tx, record)?;
// Retention expiry takes precedence: it must not look like an unused key.
if now_ms >= record.expires_at_ms || is_expired(tx, &record)? {
return expired();
}
if record.cancelled {
return fail(
PluginHostErrorCode::Cancelled,
PluginEffectObservation::Indeterminate,
);
}
if let Some(saved) = saved_effect(tx, &record, effect_key)? {
if saved.fingerprint != fingerprint {
return conflict();
}
return Ok(EffectAdmission::Replay(saved.result()?));
}
let unknown = host_error(
PluginHostErrorCode::Indeterminate,
PluginEffectObservation::Indeterminate,
);
tx.execute(
"INSERT INTO plugin_effect_receipts(workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms,state,error_json) VALUES(?1,?2,?3,?4,?5,'pending',?6)",
params![record.workspace_id, record.invocation_id, effect_key, fingerprint, now_ms, encode(&unknown)?],
)?;
Ok(EffectAdmission::Dispatch)
}
/// Save one bounded observed outcome. This records an already-admitted effect;
/// it does not authorize dispatch or publication. An in-flight outcome may be
/// recorded after cancellation. Once pruned, payloads can never be restored.
/// Exact completion replay is harmless; different/completed results conflict.
/// On invalid/oversized completion the original indeterminate receipt survives.
pub fn effect_complete(
tx: &Transaction<'_>,
record: &InvocationRecord,
effect_key: &str,
fingerprint: &str,
result: &std::result::Result<PluginHttpResponse, PluginHostError>,
) -> Result<()> {
validate_key(effect_key, fingerprint)?;
let record = verify_record(tx, record)?;
if is_expired(tx, &record)? {
return expired();
}
let Some(saved) = saved_effect(tx, &record, effect_key)? else {
return fail(
PluginHostErrorCode::NotFound,
PluginEffectObservation::NotDispatched,
);
};
if saved.fingerprint != fingerprint {
return conflict();
}
if saved.state == "expired" {
return expired();
}
if saved.state == "completed" {
return if saved.result()? == *result {
Ok(())
} else {
conflict()
};
}
let (status, headers, body, error) = match result {
Ok(response) => {
if !(100..=599).contains(&response.status) {
return fail(
PluginHostErrorCode::InvalidInput,
PluginEffectObservation::ResponseObserved,
);
}
if response.headers.len() > MAX_RESPONSE_HEADERS {
return fail(
PluginHostErrorCode::QuotaExceeded,
PluginEffectObservation::ResponseObserved,
);
}
let headers = encode(&response.headers)?;
if headers.len() > MAX_RESPONSE_HEADER_BYTES || response.body.len() > MAX_RESPONSE_BYTES
{
return fail(
PluginHostErrorCode::QuotaExceeded,
PluginEffectObservation::ResponseObserved,
);
}
(
Some(response.status),
Some(headers),
Some(response.body.as_slice()),
None,
)
}
Err(error) => {
if !valid_identity(&error.diagnostic_id, 128) {
return fail(
PluginHostErrorCode::InvalidInput,
PluginEffectObservation::Indeterminate,
);
}
let json = encode(error)?;
if json.len() > MAX_ERROR_BYTES {
return fail(
PluginHostErrorCode::InvalidInput,
PluginEffectObservation::Indeterminate,
);
}
(None, None, None, Some(json))
}
};
let changed = tx.execute(
"UPDATE plugin_effect_receipts SET state='completed',response_status=?4,response_headers=?5,response_body=?6,error_json=?7 WHERE workspace_id=?1 AND invocation_id=?2 AND effect_key=?3 AND state='pending' AND fingerprint=?8",
params![record.workspace_id, record.invocation_id, effect_key, status, headers, body, error, fingerprint],
)?;
if changed != 1 {
return conflict();
}
Ok(())
}
/// Irreversible, idempotent cancellation. Does not claim to undo external effects.
pub fn cancel(tx: &Transaction<'_>, workspace: &str, invocation_id: &str) -> Result<()> {
lookup(tx, workspace, invocation_id)?;
tx.execute(
"UPDATE plugin_invocations SET cancelled=1 WHERE workspace_id=?1 AND invocation_id=?2 AND cancelled=0",
params![workspace, invocation_id],
)?;
Ok(())
}
/// Expire matching invocations and erase response/error payloads, never identity,
/// fingerprints or keys. Returns the number of receipts newly tombstoned. Even a
/// subsequent backwards clock cannot resurrect an explicitly expired invocation.
/// Tombstones survive until the parent's Workspace deletion cascade.
pub fn prune(tx: &Transaction<'_>, workspace: &str, now_ms: i64) -> Result<usize> {
if !valid_identity(workspace, 512) || now_ms < 0 {
return invalid();
}
// A local savepoint keeps expiry atomic if the owner handles an error and
// commits unrelated work in its surrounding transaction.
tx.execute_batch("SAVEPOINT plugin_receipt_prune")?;
let work = (|| {
tx.execute(
"UPDATE plugin_invocations SET expired=1 WHERE workspace_id=?1 AND expires_at_ms<=?2 AND expired=0",
params![workspace, now_ms],
)?;
Ok(tx.execute(
"UPDATE plugin_effect_receipts SET state='expired',response_status=NULL,response_headers=NULL,response_body=NULL,error_json=NULL WHERE workspace_id=?1 AND state<>'expired' AND invocation_id IN (SELECT invocation_id FROM plugin_invocations WHERE workspace_id=?1 AND expired=1)",
[workspace],
)?)
})();
match work {
Ok(count) => {
tx.execute_batch("RELEASE plugin_receipt_prune")?;
Ok(count)
}
Err(error) => {
tx.execute_batch("ROLLBACK TO plugin_receipt_prune; RELEASE plugin_receipt_prune")?;
Err(error)
}
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,79 @@
-- Durable Host identity and all-method HTTPS receipts. No request body, raw
-- request headers, secret value, grant authority or catalog pin lives here.
CREATE TABLE plugin_invocations (
workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
invocation_id TEXT NOT NULL CHECK(length(CAST(invocation_id AS BLOB)) BETWEEN 1 AND 512),
plan_id TEXT NOT NULL,
plan_fingerprint TEXT NOT NULL CHECK(length(plan_fingerprint)=71 AND substr(plan_fingerprint,1,7)='sha256:' AND substr(plan_fingerprint,8) NOT GLOB '*[^0-9a-f]*'),
installation_id TEXT NOT NULL,
runtime_id TEXT NOT NULL CHECK(length(CAST(runtime_id AS BLOB)) BETWEEN 1 AND 512),
worker_id TEXT NOT NULL CHECK(length(CAST(worker_id AS BLOB)) BETWEEN 1 AND 512),
execution_json TEXT NOT NULL CHECK(json_valid(execution_json) AND json_type(execution_json)='object' AND length(CAST(execution_json AS BLOB)) <= 16384),
consumer_key TEXT NOT NULL CHECK(json_valid(consumer_key) AND json_type(consumer_key)='array' AND length(CAST(consumer_key AS BLOB)) <= 8192),
consumer_json TEXT NOT NULL CHECK(json_valid(consumer_json) AND json_type(consumer_json)='object' AND length(CAST(consumer_json AS BLOB)) <= 16384),
created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0),
expires_at_ms INTEGER NOT NULL CHECK(expires_at_ms > created_at_ms),
cancelled INTEGER NOT NULL DEFAULT 0 CHECK(cancelled IN (0,1)),
expired INTEGER NOT NULL DEFAULT 0 CHECK(expired IN (0,1)),
PRIMARY KEY(workspace_id, invocation_id),
UNIQUE(workspace_id, runtime_id, worker_id, installation_id, consumer_key),
FOREIGN KEY(workspace_id, plan_id) REFERENCES plugin_execution_plans(workspace_id, plan_id),
FOREIGN KEY(workspace_id, installation_id) REFERENCES plugin_installations(workspace_id, installation_id)
);
CREATE INDEX plugin_invocation_expiry ON plugin_invocations(workspace_id, expires_at_ms) WHERE expired=0;
CREATE TRIGGER plugin_invocation_immutable
BEFORE UPDATE OF workspace_id,invocation_id,plan_id,plan_fingerprint,installation_id,runtime_id,worker_id,execution_json,consumer_key,consumer_json,created_at_ms,expires_at_ms ON plugin_invocations
BEGIN SELECT RAISE(ABORT,'Plugin invocation identity is immutable'); END;
CREATE TRIGGER plugin_invocation_terminal BEFORE UPDATE OF cancelled,expired ON plugin_invocations
WHEN NEW.cancelled < OLD.cancelled OR NEW.expired < OLD.expired
BEGIN SELECT RAISE(ABORT,'Plugin invocation cancellation and expiry are irreversible'); END;
CREATE TRIGGER plugin_invocation_retained BEFORE DELETE ON plugin_invocations
WHEN EXISTS(SELECT 1 FROM workspaces WHERE workspace_id=OLD.workspace_id)
BEGIN SELECT RAISE(ABORT,'Plugin invocation tombstone must be retained'); END;
-- REPLACE must not bypass tombstones even with recursive_triggers disabled.
CREATE TRIGGER plugin_invocation_no_replace BEFORE INSERT ON plugin_invocations
WHEN EXISTS(SELECT 1 FROM plugin_invocations WHERE workspace_id=NEW.workspace_id AND
(invocation_id=NEW.invocation_id OR (runtime_id=NEW.runtime_id AND worker_id=NEW.worker_id AND installation_id=NEW.installation_id AND consumer_key=NEW.consumer_key)))
BEGIN SELECT RAISE(ABORT,'Plugin invocation identity cannot be reused'); END;
CREATE TABLE plugin_effect_receipts (
workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
invocation_id TEXT NOT NULL,
effect_key TEXT NOT NULL CHECK(length(CAST(effect_key AS BLOB)) BETWEEN 1 AND 128),
fingerprint TEXT NOT NULL CHECK(length(fingerprint)=71 AND substr(fingerprint,1,7)='sha256:' AND substr(fingerprint,8) NOT GLOB '*[^0-9a-f]*'),
admitted_at_ms INTEGER NOT NULL CHECK(admitted_at_ms >= 0),
state TEXT NOT NULL CHECK(state IN ('pending','completed','expired')),
response_status INTEGER CHECK(response_status BETWEEN 100 AND 599),
response_headers TEXT CHECK(json_valid(response_headers) AND json_type(response_headers)='array' AND json_array_length(response_headers) <= 128 AND length(CAST(response_headers AS BLOB)) <= 65536),
response_body BLOB CHECK(response_body IS NULL OR (typeof(response_body)='blob' AND length(response_body) <= 1048576)),
error_json TEXT CHECK(json_valid(error_json) AND json_type(error_json)='object' AND length(CAST(error_json AS BLOB)) <= 4096),
CHECK(COALESCE(length(response_body),0)+COALESCE(length(CAST(response_headers AS BLOB)),0) <= 1114112),
CHECK(
(state='pending' AND response_status IS NULL AND response_headers IS NULL AND response_body IS NULL AND error_json IS NOT NULL AND COALESCE(json_extract(error_json,'$.code'),'')='indeterminate' AND COALESCE(json_extract(error_json,'$.effect'),'')='indeterminate') OR
(state='completed' AND ((response_status IS NOT NULL AND response_headers IS NOT NULL AND response_body IS NOT NULL AND error_json IS NULL) OR
(response_status IS NULL AND response_headers IS NULL AND response_body IS NULL AND error_json IS NOT NULL))) OR
(state='expired' AND response_status IS NULL AND response_headers IS NULL AND response_body IS NULL AND error_json IS NULL)
),
PRIMARY KEY(workspace_id, invocation_id, effect_key),
FOREIGN KEY(workspace_id, invocation_id) REFERENCES plugin_invocations(workspace_id, invocation_id) ON DELETE CASCADE
);
CREATE TRIGGER plugin_effect_immutable BEFORE UPDATE OF workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms ON plugin_effect_receipts
BEGIN SELECT RAISE(ABORT,'Plugin effect identity is immutable'); END;
CREATE TRIGGER plugin_effect_transition BEFORE UPDATE ON plugin_effect_receipts
WHEN NOT (
(OLD.state='pending' AND NEW.state='completed') OR
(OLD.state IN ('pending','completed') AND NEW.state='expired' AND EXISTS(
SELECT 1 FROM plugin_invocations WHERE workspace_id=OLD.workspace_id AND invocation_id=OLD.invocation_id AND expired=1))
)
BEGIN SELECT RAISE(ABORT,'Plugin effect result is immutable'); END;
CREATE TRIGGER plugin_effect_admission BEFORE INSERT ON plugin_effect_receipts
WHEN NEW.state <> 'pending' OR NOT EXISTS(
SELECT 1 FROM plugin_invocations WHERE workspace_id=NEW.workspace_id AND invocation_id=NEW.invocation_id
AND cancelled=0 AND expired=0 AND expires_at_ms > NEW.admitted_at_ms)
BEGIN SELECT RAISE(ABORT,'Plugin effect invocation is not admissible'); END;
CREATE TRIGGER plugin_effect_no_replace BEFORE INSERT ON plugin_effect_receipts
WHEN EXISTS(SELECT 1 FROM plugin_effect_receipts WHERE workspace_id=NEW.workspace_id AND invocation_id=NEW.invocation_id AND effect_key=NEW.effect_key)
BEGIN SELECT RAISE(ABORT,'Plugin effect key cannot be reused'); END;
CREATE TRIGGER plugin_effect_retained BEFORE DELETE ON plugin_effect_receipts
WHEN EXISTS(SELECT 1 FROM workspaces WHERE workspace_id=OLD.workspace_id)
BEGIN SELECT RAISE(ABORT,'Plugin effect tombstone must be retained'); END;
@@ -0,0 +1,987 @@
//! File-backed SQLite tests of the owner boundary, using production receipt,
//! plan and catalog DDL. The minimal Workspace parent is a fixture, not a second
//! migration path. Parent migration transition/rollback/retry tests live upstream.
use super::*;
use crate::plugin_plan::{
ConfigProvenance, PlanPinState, PluginPinProjection, PluginSurface, ProfileProvenance,
WorkerPluginExecutionPlan, WorkerPluginPin, WorkerPluginPlanInput,
};
use rusqlite::TransactionBehavior;
use server_api::{PluginArtifact, PluginSchemaContribution, PluginToolMetadata};
use sha2::{Digest, Sha256};
use std::{
path::Path,
sync::{
Arc, Barrier,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use tempfile::{TempDir, tempdir};
const NOW: i64 = 1000;
const RETENTION: i64 = 500;
const INSTALLATION: &str = "installation";
fn fingerprint(seed: &str) -> String {
let hex: String = Sha256::digest(seed.as_bytes())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect();
format!("sha256:{hex}")
}
fn consumer(id: &str) -> PluginConsumer {
PluginConsumer::ToolCall {
tool_call_id: id.into(),
tool_name: "example_echo".into(),
}
}
fn response() -> HttpResult {
Ok(PluginHttpResponse {
status: 503, // HTTP failure is still an observed response, not transport error.
headers: vec![
("x-result".into(), "one".into()),
("x-result".into(), "two".into()),
],
body: vec![0, 255, 128, 13, 10],
})
}
fn open(path: &Path) -> Connection {
let conn = Connection::open(path).unwrap();
conn.execute_batch("PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;")
.unwrap();
conn.busy_timeout(Duration::from_secs(10)).unwrap();
conn
}
struct Fixture {
dir: TempDir,
plan: StoredPluginPlan,
}
impl Fixture {
fn new() -> Self {
let dir = tempdir().unwrap();
let conn = open(&dir.path().join("receipts.db"));
conn.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY); INSERT INTO workspaces VALUES('a'),('b');").unwrap();
conn.execute_batch(include_str!("../plugin_catalog_schema.sql"))
.unwrap();
conn.execute_batch(crate::plugin_plan::SCHEMA_SQL).unwrap();
conn.execute_batch(SCHEMA_SQL).unwrap();
for ws in ["a", "b"] {
conn.execute(
"INSERT INTO plugin_installations VALUES(?1,?2,'alias','package','operation',0)",
params![ws, INSTALLATION],
)
.unwrap();
}
let plan = make_plan("plan", "config", "a", "runtime", "worker", "principal");
save_plan(&conn, &plan);
Self { dir, plan }
}
fn path(&self) -> std::path::PathBuf {
self.dir.path().join("receipts.db")
}
fn conn(&self) -> Connection {
open(&self.path())
}
fn begin(&self, id: &str) -> InvocationRecord {
let mut conn = self.conn();
let tx = conn
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let record = begin(&tx, &self.plan, INSTALLATION, &consumer(id), NOW, RETENTION).unwrap();
tx.commit().unwrap();
record
}
fn write<T>(&self, work: impl FnOnce(&Transaction<'_>) -> Result<T>) -> Result<T> {
let mut conn = self.conn();
let tx = conn
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let value = work(&tx)?;
tx.commit().unwrap();
Ok(value)
}
fn admit(
&self,
record: &InvocationRecord,
key: &str,
seed: &str,
now: i64,
) -> Result<EffectAdmission> {
self.write(|tx| effect_admit(tx, record, key, &fingerprint(seed), now))
}
}
fn make_plan(
id: &str,
config: &str,
workspace: &str,
runtime: &str,
worker: &str,
principal: &str,
) -> StoredPluginPlan {
let artifact = PluginArtifact {
package_id: "package".into(),
name: "package".into(),
version: "1.0.0".into(),
description: None,
package_digest: fingerprint("package"),
artifact_digest: fingerprint("artifact"),
artifact_size: 1,
schema_version: manifest::plugin::SUPPORTED_PLUGIN_API_VERSION,
world: manifest::plugin::PLUGIN_COMPONENT_TOOL_WORLD.into(),
required_imports: vec![],
tools: vec![PluginToolMetadata {
name: "example_echo".into(),
description: "echo".into(),
input_schema: serde_json::json!({}),
output_schema: serde_json::json!({}),
error_codes: vec![],
external_write: false,
}],
connections: vec![],
requested_grants: vec![],
config_schema: PluginSchemaContribution {
identity: "schema".into(),
version: "1".into(),
digest: fingerprint("schema"),
namespace: "package".into(),
dcdl: "".into(),
},
revoked: false,
revocation_reason: None,
};
let pin = WorkerPluginPin::from_artifact(
INSTALLATION.into(),
"operation".into(),
&artifact,
PluginPinProjection {
config: serde_json::json!({}),
surfaces: vec![PluginSurface::Tool],
tools: vec!["example_echo".into()],
grants: vec![],
bindings: vec![],
},
)
.unwrap();
let plan = WorkerPluginExecutionPlan::canonicalize(WorkerPluginPlanInput {
workspace_id: workspace.into(),
config: ConfigProvenance {
content_digest: fingerprint(config),
toolchain_fingerprint: "toolchain".into(),
},
profile: ProfileProvenance {
identity: "profile".into(),
projection_digest: fingerprint("profile"),
},
execution: HostExecutionIdentity {
runtime_id: runtime.into(),
worker_id: worker.into(),
principal_id: principal.into(),
delegation_id: Some("delegation".into()),
},
plugins: vec![pin],
})
.unwrap();
StoredPluginPlan {
plan_id: id.into(),
fingerprint: plan.fingerprint().unwrap(),
pin_state: PlanPinState::Active,
plan,
}
}
fn save_plan(conn: &Connection, plan: &StoredPluginPlan) {
conn.execute("INSERT INTO plugin_execution_plans(workspace_id,plan_id,actor,operation_key,fingerprint,plan_json,pin_state) VALUES(?1,?2,'fixture',?2,?3,?4,'active')",
params![plan.plan.workspace_id(), plan.plan_id, plan.fingerprint, plan.plan.canonical_json().unwrap()]).unwrap();
}
fn error<T: std::fmt::Debug>(result: Result<T>, code: PluginHostErrorCode) -> PluginHostError {
match result {
Err(Error::PluginHost(error)) if error.code == code => error,
other => panic!("expected {code:?}, got {other:?}"),
}
}
fn replay_unknown(result: Result<EffectAdmission>) -> PluginHostError {
match result.unwrap() {
EffectAdmission::Replay(Err(error)) => {
assert_eq!(error.code, PluginHostErrorCode::Indeterminate);
assert_eq!(error.effect, PluginEffectObservation::Indeterminate);
error
}
other => panic!("must never redispatch pending/unknown: {other:?}"),
}
}
#[test]
fn begin_replay_reopen_preserves_uuid_and_original_deadline_and_conflicts_on_input() {
let f = Fixture::new();
let first = f.begin("call");
uuid::Uuid::parse_str(&first.invocation_id).unwrap();
let again = f
.write(|tx| {
begin(
tx,
&f.plan,
INSTALLATION,
&consumer("call"),
NOW + 100,
RETENTION * 10,
)
})
.unwrap();
assert_eq!(first, again);
assert_eq!(first.expires_at_ms, NOW + RETENTION);
assert_eq!(lookup(&f.conn(), "a", &first.invocation_id).unwrap(), first);
error(
lookup(&f.conn(), "b", &first.invocation_id),
PluginHostErrorCode::NotFound,
);
let other_plan = make_plan(
"different-plan",
"config",
"a",
"runtime",
"worker",
"principal",
);
save_plan(&f.conn(), &other_plan);
error(
f.write(|tx| {
begin(
tx,
&other_plan,
INSTALLATION,
&consumer("call"),
NOW,
RETENTION,
)
}),
PluginHostErrorCode::Conflict,
);
let other_input = make_plan(
"changed-input",
"new-config",
"a",
"runtime",
"worker",
"principal",
);
save_plan(&f.conn(), &other_input);
error(
f.write(|tx| {
begin(
tx,
&other_input,
INSTALLATION,
&consumer("call"),
NOW,
RETENTION,
)
}),
PluginHostErrorCode::Conflict,
);
let renamed = PluginConsumer::ToolCall {
tool_call_id: "call".into(),
tool_name: "different".into(),
};
error(
f.write(|tx| begin(tx, &f.plan, INSTALLATION, &renamed, NOW, RETENTION)),
PluginHostErrorCode::Conflict,
);
let changed_execution = make_plan(
"principal-change",
"config",
"a",
"runtime",
"worker",
"other-principal",
);
save_plan(&f.conn(), &changed_execution);
error(
f.write(|tx| {
begin(
tx,
&changed_execution,
INSTALLATION,
&consumer("call"),
NOW,
RETENTION,
)
}),
PluginHostErrorCode::Conflict,
);
assert_ne!(f.begin("new-call").invocation_id, first.invocation_id);
}
#[test]
fn identities_isolate_workspaces_runtimes_workers_installations_and_consumers() {
let f = Fixture::new();
let original = f.begin("same-id");
let mut ids = std::collections::BTreeSet::from([original.invocation_id]);
for (id, ws, runtime, worker) in [
("other-workspace", "b", "runtime", "worker"),
("other-runtime", "a", "runtime-2", "worker"),
("other-worker", "a", "runtime", "worker-2"),
] {
let plan = make_plan(id, "config", ws, runtime, worker, "principal");
save_plan(&f.conn(), &plan);
let record = f
.write(|tx| {
begin(
tx,
&plan,
INSTALLATION,
&consumer("same-id"),
NOW,
RETENTION,
)
})
.unwrap();
assert!(ids.insert(record.invocation_id));
}
// Another immutable installation selection gets an independent namespace.
let mut plan_json = serde_json::to_value(&f.plan.plan).unwrap();
plan_json["input"]["plugins"][0]["installation_id"] = serde_json::json!("other-installation");
let other_snapshot: WorkerPluginExecutionPlan = serde_json::from_value(plan_json).unwrap();
let other = StoredPluginPlan {
plan_id: "other-installation-plan".into(),
fingerprint: other_snapshot.fingerprint().unwrap(),
pin_state: PlanPinState::Active,
plan: other_snapshot,
};
f.conn().execute("INSERT INTO plugin_installations VALUES('a','other-installation','other-alias','package','operation',0)", []).unwrap();
save_plan(&f.conn(), &other);
let record = f
.write(|tx| {
begin(
tx,
&other,
"other-installation",
&consumer("same-id"),
NOW,
RETENTION,
)
})
.unwrap();
assert!(ids.insert(record.invocation_id));
for consumer_id in ["consumer-1", "consumer-2"] {
let c = PluginConsumer::Consumer {
consumer_id: consumer_id.into(),
event_id: "same-id".into(),
export_name: "export".into(),
};
let record = f
.write(|tx| begin(tx, &f.plan, INSTALLATION, &c, NOW, RETENTION))
.unwrap();
assert!(ids.insert(record.invocation_id.clone()));
assert_eq!(
f.write(|tx| begin(tx, &f.plan, INSTALLATION, &c, NOW + 5, RETENTION))
.unwrap(),
record
);
let renamed = PluginConsumer::Consumer {
consumer_id: consumer_id.into(),
event_id: "same-id".into(),
export_name: "other".into(),
};
error(
f.write(|tx| begin(tx, &f.plan, INSTALLATION, &renamed, NOW, RETENTION)),
PluginHostErrorCode::Conflict,
);
}
// Missing installation cannot open a namespace absent from the immutable plan.
error(
f.write(|tx| {
begin(
tx,
&f.plan,
"not-in-plan",
&consumer("same-id"),
NOW,
RETENTION,
)
}),
PluginHostErrorCode::InvalidInput,
);
}
#[test]
fn two_connections_racing_admission_commit_exactly_one_dispatch() {
let f = Fixture::new();
let record = f.begin("race");
let barrier = Arc::new(Barrier::new(2));
let sends = Arc::new(AtomicUsize::new(0));
let handles: Vec<_> = (0..2)
.map(|_| {
let (path, record, barrier, sends) =
(f.path(), record.clone(), barrier.clone(), sends.clone());
std::thread::spawn(move || {
let mut conn = open(&path);
barrier.wait();
let tx = conn
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let admission =
effect_admit(&tx, &record, "GET-key", &fingerprint("GET /"), NOW).unwrap();
tx.commit().unwrap();
if admission == EffectAdmission::Dispatch {
sends.fetch_add(1, Ordering::SeqCst);
}
admission
})
})
.collect();
let outcomes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
assert_eq!(sends.load(Ordering::SeqCst), 1);
assert_eq!(
outcomes
.iter()
.filter(|a| **a == EffectAdmission::Dispatch)
.count(),
1
);
assert!(outcomes.iter().any(|a| matches!(a, EffectAdmission::Replay(Err(error)) if error.code == PluginHostErrorCode::Indeterminate)));
error(
f.admit(&record, "GET-key", "POST /", NOW),
PluginHostErrorCode::Conflict,
);
}
#[test]
fn crash_after_durable_admission_reopens_pending_with_zero_redispatch() {
let f = Fixture::new();
let record = f.begin("crash");
let sends = AtomicUsize::new(0);
if f.admit(&record, "key", "request", NOW).unwrap() == EffectAdmission::Dispatch {
sends.fetch_add(1, Ordering::SeqCst);
}
// All helpers close their connections. Simulate crash before any completion.
let saved_json: String = f
.conn()
.query_row("SELECT error_json FROM plugin_effect_receipts", [], |r| {
r.get(0)
})
.unwrap();
let saved: PluginHostError = serde_json::from_str(&saved_json).unwrap();
for _ in 0..3 {
let recovered = lookup(&f.conn(), "a", &record.invocation_id).unwrap();
assert_eq!(
replay_unknown(f.admit(&recovered, "key", "request", NOW + 1)),
saved
);
}
assert_eq!(sends.load(Ordering::SeqCst), 1);
}
#[test]
fn completion_replays_exact_binary_response_headers_and_http_status_without_dispatch() {
let f = Fixture::new();
let record = f.begin("response");
assert_eq!(
f.admit(&record, "key", "request", NOW).unwrap(),
EffectAdmission::Dispatch
);
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &response()))
.unwrap();
assert_eq!(
f.admit(&record, "key", "request", NOW).unwrap(),
EffectAdmission::Replay(response())
);
// Successful completion replay is a read, not an overwrite.
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &response()))
.unwrap();
error(
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("different"), &response())),
PluginHostErrorCode::Conflict,
);
let changed = Ok(PluginHttpResponse {
status: 200,
headers: vec![],
body: vec![],
});
error(
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &changed)),
PluginHostErrorCode::Conflict,
);
error(
f.write(|tx| {
effect_complete(
tx,
&record,
"never-admitted",
&fingerprint("request"),
&changed,
)
}),
PluginHostErrorCode::NotFound,
);
}
#[test]
fn completion_saves_transport_failure_for_each_observation_and_never_redispatches() {
let f = Fixture::new();
let record = f.begin("errors");
for (key, observation) in [
("before", PluginEffectObservation::NotDispatched),
("after", PluginEffectObservation::Indeterminate),
("observed", PluginEffectObservation::ResponseObserved),
] {
assert_eq!(
f.admit(&record, key, key, NOW).unwrap(),
EffectAdmission::Dispatch
);
let failure = Err(PluginHostError {
code: PluginHostErrorCode::DeadlineExceeded,
effect: observation,
diagnostic_id: format!("diagnostic-{key}"),
});
f.write(|tx| effect_complete(tx, &record, key, &fingerprint(key), &failure))
.unwrap();
assert_eq!(
f.admit(&record, key, key, NOW + 1).unwrap(),
EffectAdmission::Replay(failure)
);
}
}
#[test]
fn cancellation_is_irreversible_and_stale_records_cannot_bypass_it() {
let f = Fixture::new();
let record = f.begin("cancel");
assert_eq!(
f.admit(&record, "in-flight", "request", NOW).unwrap(),
EffectAdmission::Dispatch
);
f.write(|tx| cancel(tx, "a", &record.invocation_id))
.unwrap();
f.write(|tx| cancel(tx, "a", &record.invocation_id))
.unwrap();
assert!(
lookup(&f.conn(), "a", &record.invocation_id)
.unwrap()
.cancelled
);
for key in ["in-flight", "new-key"] {
let e = error(
f.admit(&record, key, "request", NOW),
PluginHostErrorCode::Cancelled,
);
assert_eq!(e.effect, PluginEffectObservation::Indeterminate);
}
// Recording an observed in-flight response does not publish it after cancel.
f.write(|tx| {
effect_complete(
tx,
&record,
"in-flight",
&fingerprint("request"),
&response(),
)
})
.unwrap();
error(
f.admit(&record, "in-flight", "request", NOW),
PluginHostErrorCode::Cancelled,
);
assert!(
f.conn()
.execute("UPDATE plugin_invocations SET cancelled=0", [])
.is_err()
);
assert!(f.begin("cancel").cancelled);
}
#[test]
fn deadline_expiry_refuses_old_and_new_keys_and_prune_preserves_tombstones() {
let f = Fixture::new();
let record = f.begin("expire");
let untouched = f.begin("no-effects");
for key in ["complete", "pending"] {
assert_eq!(
f.admit(&record, key, "request", NOW).unwrap(),
EffectAdmission::Dispatch
);
}
f.write(|tx| {
effect_complete(
tx,
&record,
"complete",
&fingerprint("request"),
&response(),
)
})
.unwrap();
for key in ["complete", "pending", "unused"] {
error(
f.admit(&record, key, "request", NOW + RETENTION),
PluginHostErrorCode::ReceiptExpired,
);
}
error(
f.admit(&untouched, "unused", "request", NOW + RETENTION),
PluginHostErrorCode::ReceiptExpired,
);
let recovered = f
.write(|tx| {
begin(
tx,
&f.plan,
INSTALLATION,
&consumer("expire"),
NOW + RETENTION + 10,
RETENTION * 100,
)
})
.unwrap();
assert_eq!(record, recovered);
assert_eq!(f.write(|tx| prune(tx, "a", NOW + RETENTION)).unwrap(), 2);
assert_eq!(f.write(|tx| prune(tx, "a", NOW + RETENTION)).unwrap(), 0);
let conn = f.conn();
let counts: (i64, i64, i64) = conn.query_row("SELECT count(*),sum(state='expired'),sum(response_body IS NULL AND response_headers IS NULL AND error_json IS NULL) FROM plugin_effect_receipts", [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))).unwrap();
assert_eq!(counts, (2, 2, 2));
// Explicit pruning is monotonic even if the caller's clock moves backwards.
error(
f.admit(&record, "unused", "request", NOW),
PluginHostErrorCode::ReceiptExpired,
);
error(
f.admit(&untouched, "unused", "request", NOW),
PluginHostErrorCode::ReceiptExpired,
);
error(
f.write(|tx| {
effect_complete(
tx,
&record,
"complete",
&fingerprint("request"),
&response(),
)
}),
PluginHostErrorCode::ReceiptExpired,
);
assert!(
conn.execute("UPDATE plugin_invocations SET expired=0", [])
.is_err()
);
assert_eq!(
lookup(&conn, "a", &record.invocation_id).unwrap(),
InvocationRecord {
expired: true,
..record
}
);
assert!(
conn.execute("DELETE FROM plugin_effect_receipts", [])
.is_err()
);
assert!(conn.execute("DELETE FROM plugin_invocations", []).is_err());
assert!(
conn.execute("UPDATE plugin_effect_receipts SET state='pending'", [])
.is_err()
);
// Workspace deletion is the only identity/key purge. All FK checks remain on.
conn.execute("DELETE FROM workspaces WHERE workspace_id='a'", [])
.unwrap();
for table in ["plugin_invocations", "plugin_effect_receipts"] {
let count: i64 = conn
.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0);
}
let fk_errors: i64 = conn
.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| {
r.get(0)
})
.unwrap();
assert_eq!(fk_errors, 0);
}
#[test]
fn response_and_key_bounds_reject_without_erasing_indeterminate_receipt() {
let f = Fixture::new();
let record = f.begin("bounds");
assert_eq!(
f.admit(&record, "key", "request", NOW).unwrap(),
EffectAdmission::Dispatch
);
for body_len in [MAX_RESPONSE_BYTES + 1] {
let big = Ok(PluginHttpResponse {
status: 200,
headers: vec![],
body: vec![255; body_len],
});
error(
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &big)),
PluginHostErrorCode::QuotaExceeded,
);
replay_unknown(f.admit(&record, "key", "request", NOW));
}
let too_many_headers = Ok(PluginHttpResponse {
status: 200,
headers: vec![("x".into(), "y".into()); MAX_RESPONSE_HEADERS + 1],
body: vec![],
});
error(
f.write(|tx| {
effect_complete(
tx,
&record,
"key",
&fingerprint("request"),
&too_many_headers,
)
}),
PluginHostErrorCode::QuotaExceeded,
);
let big_headers = Ok(PluginHttpResponse {
status: 200,
headers: vec![("x".into(), "y".repeat(MAX_RESPONSE_HEADER_BYTES))],
body: vec![],
});
error(
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &big_headers)),
PluginHostErrorCode::QuotaExceeded,
);
for key in [
"".to_owned(),
"x".repeat(129),
"é".repeat(65),
"bad\0key".into(),
] {
error(
f.admit(&record, &key, "request", NOW),
PluginHostErrorCode::InvalidInput,
);
}
error(
f.write(|tx| effect_admit(tx, &record, "key", "raw request must not be stored", NOW)),
PluginHostErrorCode::InvalidInput,
);
let max = Ok(PluginHttpResponse {
status: 200,
headers: vec![],
body: vec![255; MAX_RESPONSE_BYTES],
});
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &max))
.unwrap();
assert_eq!(
f.admit(&record, "key", "request", NOW).unwrap(),
EffectAdmission::Replay(max)
);
}
#[test]
fn schema_guards_identity_result_overwrites_replace_and_unadmitted_completion() {
let f = Fixture::new();
let record = f.begin("guards");
f.admit(&record, "key", "request", NOW).unwrap();
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &response()))
.unwrap();
let conn = f.conn();
// REPLACE protection must not depend on PRAGMA recursive_triggers.
conn.execute_batch("PRAGMA recursive_triggers=OFF;")
.unwrap();
for sql in [
"DELETE FROM plugin_invocations",
"DELETE FROM plugin_effect_receipts",
"UPDATE plugin_invocations SET invocation_id='replacement'",
"UPDATE plugin_invocations SET expires_at_ms=99999",
"UPDATE plugin_invocations SET consumer_json='{}'",
"UPDATE plugin_effect_receipts SET fingerprint='sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'",
"UPDATE plugin_effect_receipts SET response_body=X'01'",
"UPDATE plugin_effect_receipts SET state='pending',response_status=NULL,response_headers=NULL,response_body=NULL,error_json='{}'",
"UPDATE plugin_effect_receipts SET state='expired',response_status=NULL,response_headers=NULL,response_body=NULL,error_json=NULL",
"INSERT OR REPLACE INTO plugin_invocations SELECT * FROM plugin_invocations",
"INSERT OR REPLACE INTO plugin_effect_receipts SELECT * FROM plugin_effect_receipts",
"INSERT INTO plugin_effect_receipts SELECT workspace_id,invocation_id,'new-key',fingerprint,admitted_at_ms,state,response_status,response_headers,response_body,error_json FROM plugin_effect_receipts",
] {
assert!(conn.execute(sql, []).is_err(), "must reject {sql}");
}
let mut forged = record.clone();
forged.expires_at_ms += 1000;
error(
f.admit(&forged, "new-key", "request", NOW),
PluginHostErrorCode::Conflict,
);
forged = record.clone();
forged.execution.worker_id = "another-worker".into();
error(
f.admit(&forged, "new-key", "request", NOW),
PluginHostErrorCode::Conflict,
);
assert_eq!(
f.admit(&record, "key", "request", NOW).unwrap(),
EffectAdmission::Replay(response())
);
}
#[test]
fn uncommitted_admission_rolls_back_and_failed_prune_is_atomic() {
let f = Fixture::new();
let record = f.begin("rollback");
{
let mut conn = f.conn();
let tx = conn
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
assert_eq!(
effect_admit(&tx, &record, "key", &fingerprint("request"), NOW).unwrap(),
EffectAdmission::Dispatch
);
// No send: commit was not successful. Transaction rollback is safe here.
}
assert_eq!(
f.admit(&record, "key", "request", NOW).unwrap(),
EffectAdmission::Dispatch
);
let mut conn = f.conn();
conn.execute_batch("CREATE TRIGGER refuse_prune BEFORE UPDATE ON plugin_effect_receipts WHEN NEW.state='expired' BEGIN SELECT RAISE(ABORT,'injected'); END;").unwrap();
let tx = conn
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
assert!(prune(&tx, "a", NOW + RETENTION).is_err());
tx.commit().unwrap();
let expired: bool = conn
.query_row("SELECT expired FROM plugin_invocations", [], |r| r.get(0))
.unwrap();
assert!(
!expired,
"caller catching an error must not commit partial expiry"
);
replay_unknown(f.admit(&record, "key", "request", NOW));
}
#[test]
fn invalid_begin_does_not_create_identity_or_overflow_deadline() {
let f = Fixture::new();
for (now, retention) in [(-1, 1), (NOW, 0), (NOW, -1), (i64::MAX, 1)] {
error(
f.write(|tx| {
begin(
tx,
&f.plan,
INSTALLATION,
&consumer("invalid"),
now,
retention,
)
}),
PluginHostErrorCode::InvalidInput,
);
}
let invalid = PluginConsumer::ToolCall {
tool_call_id: "".into(),
tool_name: "tool".into(),
};
error(
f.write(|tx| begin(tx, &f.plan, INSTALLATION, &invalid, NOW, RETENTION)),
PluginHostErrorCode::InvalidInput,
);
let mut plan = f.plan.clone();
plan.fingerprint = fingerprint("different");
error(
f.write(|tx| {
begin(
tx,
&plan,
INSTALLATION,
&consumer("invalid"),
NOW,
RETENTION,
)
}),
PluginHostErrorCode::InvalidInput,
);
let count: i64 = f
.conn()
.query_row("SELECT count(*) FROM plugin_invocations", [], |r| r.get(0))
.unwrap();
assert_eq!(count, 0);
}
#[test]
fn all_http_methods_including_get_and_head_have_the_same_one_shot_receipt_semantics() {
let f = Fixture::new();
let record = f.begin("all-methods");
let sends = AtomicUsize::new(0);
for method in ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] {
if f.admit(&record, method, method, NOW).unwrap() == EffectAdmission::Dispatch {
sends.fetch_add(1, Ordering::SeqCst);
}
replay_unknown(f.admit(&record, method, method, NOW));
error(
f.admit(&record, method, "changed-method-or-body", NOW),
PluginHostErrorCode::Conflict,
);
f.write(|tx| effect_complete(tx, &record, method, &fingerprint(method), &response()))
.unwrap();
assert_eq!(
f.admit(&record, method, method, NOW).unwrap(),
EffectAdmission::Replay(response())
);
}
assert_eq!(sends.load(Ordering::SeqCst), 7);
}
#[test]
fn schema_checks_initial_indeterminate_result_and_bounded_payloads_independently_of_rust() {
let f = Fixture::new();
let record = f.begin("sql-checks");
let conn = f.conn();
for json in [
"{}",
r#"{"code":"unavailable","effect":"not_dispatched","diagnostic_id":"d"}"#,
] {
assert!(conn.execute("INSERT INTO plugin_effect_receipts(workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms,state,error_json) VALUES('a',?1,'invalid-pending',?2,1000,'pending',?3)", params![record.invocation_id, fingerprint("request"), json]).is_err());
}
assert!(conn.execute("INSERT INTO plugin_effect_receipts(workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms,state,error_json) VALUES('a',?1,'expired-admission',?2,1500,'pending',?3)", params![record.invocation_id, fingerprint("request"), encode(&host_error(PluginHostErrorCode::Indeterminate, PluginEffectObservation::Indeterminate)).unwrap()]).is_err());
f.admit(&record, "key", "request", NOW).unwrap();
for sql in [
"UPDATE plugin_effect_receipts SET state='completed',response_status=200,response_headers='[]',response_body=zeroblob(1048577),error_json=NULL",
"UPDATE plugin_effect_receipts SET state='completed',response_status=200,response_headers='[]',response_body='not-a-blob',error_json=NULL",
"UPDATE plugin_effect_receipts SET state='completed',response_status=99,response_headers='[]',response_body=X'',error_json=NULL",
"UPDATE plugin_effect_receipts SET state='completed',response_status=200,response_headers='{}',response_body=X'',error_json=NULL",
] {
assert!(conn.execute(sql, []).is_err(), "must reject {sql}");
}
replay_unknown(f.admit(&record, "key", "request", NOW));
}
#[test]
fn pruning_one_workspace_does_not_remove_another_workspaces_saved_result() {
let f = Fixture::new();
let a = f.begin("prune-scope");
let plan_b = make_plan("plan-b", "config", "b", "runtime", "worker", "principal");
save_plan(&f.conn(), &plan_b);
let b = f
.write(|tx| {
begin(
tx,
&plan_b,
INSTALLATION,
&consumer("prune-scope"),
NOW,
RETENTION,
)
})
.unwrap();
for record in [&a, &b] {
f.admit(record, "key", "request", NOW).unwrap();
f.write(|tx| effect_complete(tx, record, "key", &fingerprint("request"), &response()))
.unwrap();
}
assert_eq!(f.write(|tx| prune(tx, "a", NOW + RETENTION)).unwrap(), 1);
assert_eq!(
f.admit(&b, "key", "request", NOW).unwrap(),
EffectAdmission::Replay(response())
);
error(
f.admit(&a, "key", "request", NOW),
PluginHostErrorCode::ReceiptExpired,
);
// Unexpired and pending invocations are not accidentally pruned early.
assert_eq!(
f.write(|tx| prune(tx, "b", NOW + RETENTION - 1)).unwrap(),
0
);
}
@@ -112,6 +112,10 @@ mod schema_convergence {
version: 108,
name: "Subject Ticket responsibility and current-body notification routing".into(),
},
WorkspaceSchemaMigrationStep {
version: 109,
name: "Durable Plugin Host invocations and HTTPS effect receipts".into(),
},
]
);
assert_eq!(
@@ -131,6 +135,7 @@ mod schema_convergence {
expected.push((106, crate::plugin_catalog::MIGRATION_NAME.into()));
expected.push((107, "Immutable Worker Plugin plans and retained pins".into()));
expected.push((108, "Subject Ticket responsibility and current-body notification routing".into()));
expected.push((109, "Durable Plugin Host invocations and HTTPS effect receipts".into()));
assert_eq!(workspace_schema_migration_history(conn)?, expected);
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
+26
View File
@@ -48,6 +48,8 @@ mod drive;
mod drive_grants;
#[path = "server_plugin.rs"]
mod plugin;
#[path = "server_plugin_host.rs"]
mod plugin_host;
#[path = "server_workspace_config.rs"]
mod workspace_config;
use std::collections::{BTreeMap, HashMap, HashSet};
@@ -1999,6 +2001,8 @@ pub struct WorkspaceApi {
workdir_remove_attempt_owner: WorkdirRemovalAttemptOwner,
worker_control_locks: Arc<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>>,
attachment_upload_grants: Arc<Mutex<HashMap<String, AttachmentUploadGrant>>>,
#[cfg(test)]
plugin_https_transport: Option<Arc<crate::plugin_https::HttpsTransport>>,
}
#[derive(Clone)]
@@ -4510,6 +4514,8 @@ impl WorkspaceApi {
workdir_remove_attempt_owner: current_workdir_removal_attempt_owner()?,
worker_control_locks: Arc::new(Mutex::new(HashMap::new())),
attachment_upload_grants: Arc::new(Mutex::new(HashMap::new())),
#[cfg(test)]
plugin_https_transport: None,
};
// Restore the domain fence before this API can issue any Drive handle,
// including blocked/failed deletions not scheduled for automatic recovery.
@@ -7646,6 +7652,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou
service.clone(),
))
.merge(plugin::router(service.clone()))
.merge(plugin_host::router(service.clone()))
.merge(server_api::server_api_axum::current_worker_workspace_config_get(service.clone()))
.merge(server_api::server_api_axum::current_worker_workspace_config_attach(service.clone()))
.merge(
@@ -13488,6 +13495,22 @@ impl server_api::ServerApi for ServerApiContractService {
.map_err(ApiError::into_repository_api_error)
}
async fn plugin_host(
&self,
context: server_api::ServerRequestContext,
workspace_id: String,
request: server_api::PluginHostRequest,
) -> std::result::Result<server_api::PluginHostResponse, server_api::PluginHostError> {
let api = self
.workspace_api()
.map_err(|_| server_api::PluginHostError {
code: server_api::PluginHostErrorCode::Unavailable,
effect: server_api::PluginEffectObservation::NotDispatched,
diagnostic_id: String::new(),
})?;
plugin_host::execute(Arc::new(api.clone()), context, workspace_id, request).await
}
async fn plugin_upload(
&self,
actor: RequestActor,
@@ -38643,6 +38666,9 @@ mod tests {
mod plugin_tests {
include!("server_plugin_tests.rs");
}
mod plugin_host_tests {
include!("server_plugin_host_tests.rs");
}
mod ticket_tags_tests {
include!("server_ticket_tags_tests.rs");
@@ -0,0 +1,585 @@
//! Both authenticated Remote requests and Embedded Host calls enter this owner.
//! It does not trust human catalog credentials or accept execution identities in
//! JSON. Network dispatch is single-attempt and follows durable effect admission.
use super::*;
use crate::{plugin_host as policy, plugin_https as https, plugin_receipt as receipt};
use server_api::*;
use tokio_util::sync::CancellationToken;
const RECEIPT_RETENTION_MS: i64 = 30 * 24 * 60 * 60 * 1000;
pub(super) fn router(service: Arc<ServerApiContractService>) -> Router {
server_api_axum::plugin_host(service).layer(DefaultBodyLimit::max(2 * 1024 * 1024))
}
fn denied() -> PluginHostError {
policy::map_error(policy::failure(PluginHostErrorCode::Denied))
}
fn now_ms() -> i64 {
chrono::Utc::now().timestamp_millis()
}
fn operation_name(op: &PluginHostOperation) -> &'static str {
match op {
PluginHostOperation::Admit => "plugin.admit",
PluginHostOperation::Publish => "plugin.publish",
PluginHostOperation::Open { .. } => "plugin.connection.open",
PluginHostOperation::Send { .. } => "plugin.http.send",
PluginHostOperation::Log => "plugin.log",
}
}
async fn check(
api: Arc<WorkspaceApi>,
context: ServerRequestContext,
workspace: String,
plan_id: String,
installation: String,
consumer: PluginConsumer,
operation: PluginHostOperation,
) -> std::result::Result<policy::AdmittedPluginOperation, PluginHostError> {
validate_workspace_scope(&api, &workspace).map_err(|e| policy::map_error(e.error))?;
let worker = subject_authorization::source_from_context(&context)
.map_err(|e| policy::map_error(e.error))?;
let execution = api
.plugin_execution_identity(&worker)
.await
.map_err(policy::map_error)?;
let executor = api.config_store.db_executor();
executor
.run(move || {
subject_authorization::authorize_scope(
&api,
&worker,
&server_api::authorization::OperationAuthorizationRequest {
operation: operation_name(&operation).into(),
resource: server_api::authorization::PermissionResource::PluginInstallation {
workspace_id: workspace.clone(),
installation_id: installation.clone(),
},
},
)
.map_err(|e| e.error)?;
api.config_store.with_immediate_transaction(|tx| {
policy::authorize(
tx,
&workspace,
&execution,
&plan_id,
&installation,
&consumer,
&operation,
)
})
})
.await
.map_err(policy::map_error)
}
async fn invocation(
api: Arc<WorkspaceApi>,
context: &ServerRequestContext,
workspace: &str,
invocation_id: &str,
operation: PluginHostOperation,
) -> std::result::Result<
(receipt::InvocationRecord, policy::AdmittedPluginOperation),
PluginHostError,
> {
let store = api.config_store.clone();
let w = workspace.to_owned();
let id = invocation_id.to_owned();
let record = store
.db_executor()
.run(move || store.with_conn(|conn| receipt::lookup(conn, &w, &id)))
.await
.map_err(policy::map_error)?;
let admitted = check(
api,
context.clone(),
workspace.into(),
record.plan_id.clone(),
record.installation_id.clone(),
record.consumer.clone(),
operation,
)
.await?;
if record.execution != *admitted.plan.plan.execution() {
return Err(denied());
}
if record.cancelled {
return Err(PluginHostError {
code: PluginHostErrorCode::Cancelled,
effect: PluginEffectObservation::Indeterminate,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
});
}
if record.expired || record.expires_at_ms <= now_ms() {
return Err(PluginHostError {
code: PluginHostErrorCode::ReceiptExpired,
effect: PluginEffectObservation::Indeterminate,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
});
}
Ok((record, admitted))
}
/// An Embedded transport retains this context only after the ordinary Runtime
/// source proof verifier has established it. Merely holding WorkspaceApi or human
/// management access does not produce a Host adapter.
#[derive(Clone)]
pub struct EmbeddedPluginHostOwner {
api: Arc<WorkspaceApi>,
context: ServerRequestContext,
workspace: String,
transport_override: Option<Arc<https::HttpsTransport>>,
}
impl EmbeddedPluginHostOwner {
pub(crate) fn new(
api: Arc<WorkspaceApi>,
context: ServerRequestContext,
workspace: String,
) -> Self {
Self {
api,
context,
workspace,
transport_override: None,
}
}
#[cfg(test)]
pub(crate) fn with_transport(mut self, transport: https::HttpsTransport) -> Self {
self.transport_override = Some(Arc::new(transport));
self
}
}
#[async_trait]
impl yoi_plugin_runtime::owner::PluginHostOwner for EmbeddedPluginHostOwner {
async fn call(
&self,
request: PluginHostRequest,
) -> std::result::Result<PluginHostResponse, PluginHostError> {
execute_with_transport(
self.api.clone(),
self.context.clone(),
self.workspace.clone(),
request,
self.transport_override.clone(),
)
.await
}
}
pub(super) async fn execute(
api: Arc<WorkspaceApi>,
context: ServerRequestContext,
workspace: String,
request: PluginHostRequest,
) -> std::result::Result<PluginHostResponse, PluginHostError> {
#[cfg(test)]
let transport_override = api.plugin_https_transport.clone();
#[cfg(not(test))]
let transport_override = None;
execute_with_transport(api, context, workspace, request, transport_override).await
}
async fn execute_with_transport(
api: Arc<WorkspaceApi>,
context: ServerRequestContext,
workspace: String,
request: PluginHostRequest,
transport_override: Option<Arc<https::HttpsTransport>>,
) -> std::result::Result<PluginHostResponse, PluginHostError> {
// Human management and a Runtime-only (no Worker) source both fail closed.
subject_authorization::source_from_context(&context).map_err(|_| denied())?;
match request {
PluginHostRequest::Begin {
plan_id,
installation_id,
consumer,
} => {
let admitted = check(
api.clone(),
context,
workspace,
plan_id,
installation_id.clone(),
consumer.clone(),
PluginHostOperation::Admit,
)
.await?;
let store = api.config_store.clone();
let record = store
.db_executor()
.run(move || {
store.with_immediate_transaction(|tx| {
receipt::begin(
tx,
&admitted.plan,
&installation_id,
&consumer,
now_ms(),
RECEIPT_RETENTION_MS,
)
})
})
.await
.map_err(policy::map_error)?;
if record.cancelled || record.expired || record.expires_at_ms <= now_ms() {
return Err(PluginHostError {
code: if record.cancelled {
PluginHostErrorCode::Cancelled
} else {
PluginHostErrorCode::ReceiptExpired
},
effect: PluginEffectObservation::Indeterminate,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
});
}
Ok(PluginHostResponse::Invocation {
invocation_id: record.invocation_id,
expires_at_ms: record.expires_at_ms,
})
}
PluginHostRequest::Cancel { invocation_id } => {
// Do not use invocation(): exact cancellation replay must work after
// the irreversible bit has already been committed.
let store = api.config_store.clone();
let w = workspace.clone();
let id = invocation_id.clone();
let record = store
.db_executor()
.run(move || store.with_conn(|conn| receipt::lookup(conn, &w, &id)))
.await
.map_err(policy::map_error)?;
check(
api.clone(),
context,
workspace.clone(),
record.plan_id,
record.installation_id,
record.consumer,
PluginHostOperation::Admit,
)
.await?;
let store = api.config_store.clone();
store
.db_executor()
.run(move || {
store.with_immediate_transaction(|tx| {
receipt::cancel(tx, &workspace, &invocation_id)
})
})
.await
.map_err(policy::map_error)?;
Ok(PluginHostResponse::Authorized)
}
PluginHostRequest::Authorize {
invocation_id,
operation,
} => {
invocation(api, &context, &workspace, &invocation_id, operation).await?;
Ok(PluginHostResponse::Authorized)
}
PluginHostRequest::Open {
invocation_id,
slot,
} => {
invocation(
api,
&context,
&workspace,
&invocation_id,
PluginHostOperation::Open { slot },
)
.await?;
Ok(PluginHostResponse::Opened)
}
PluginHostRequest::Log {
invocation_id,
entry,
} => {
invocation(
api,
&context,
&workspace,
&invocation_id,
PluginHostOperation::Log,
)
.await?;
// Guest text/fields are untrusted and may contain echoed credentials.
// Standard diagnostics retain only this safe event, not their payload.
if entry.message.len() > 4096
|| entry.event_code.is_empty()
|| entry.event_code.len() > 128
|| !entry
.event_code
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.'))
|| entry.fields.len() > 32 || entry.fields.iter().any(|field| {
field.name.len() > 128
|| field.name.is_empty()
|| field.name.chars().any(char::is_control)
|| matches!(&field.value, PluginLogScalar::Text(value) if value.len() > 1024)
|| matches!(&field.value, PluginLogScalar::Number(value) if !value.is_finite())
}) {
return Err(policy::map_error(policy::failure(
PluginHostErrorCode::QuotaExceeded,
)));
}
tracing::debug!(invocation = %invocation_id, "plugin log event admitted");
Ok(PluginHostResponse::Logged)
}
PluginHostRequest::Send {
invocation_id,
slot,
request,
} => {
let operation = PluginHostOperation::Send { slot: slot.clone() };
let (record, admitted) = invocation(
api.clone(),
&context,
&workspace,
&invocation_id,
operation.clone(),
)
.await?;
let binding = admitted.binding.as_ref().ok_or_else(denied)?;
let prepared = https::prepare(
binding,
https::HttpsRequest {
method: request.method.clone(),
path: request.path.clone(),
query: request.query.clone(),
headers: request.headers.clone(),
body: request.body.clone(),
timeout_ms: request.timeout_ms,
effect_key: request.effect_key.clone(),
},
)
.map_err(transport_error)?;
let normalized = prepared.normalized();
let normalized_dto = PluginHttpRequest {
method: normalized.method.clone(),
path: normalized.path.clone(),
query: normalized.query.clone(),
headers: normalized.headers.clone(),
body: normalized.body.clone(),
timeout_ms: normalized.timeout_ms,
effect_key: normalized.effect_key.clone(),
};
let fingerprint = policy::request_fingerprint(&admitted, &slot, &normalized_dto)
.map_err(policy::map_error)?;
let store = api.config_store.clone();
let r = record.clone();
let fp = fingerprint.clone();
let key = request.effect_key.clone();
let admission = store
.db_executor()
.run(move || {
store.with_immediate_transaction(|tx| {
receipt::effect_admit(tx, &r, &key, &fp, now_ms())
})
})
.await
.map_err(policy::map_error)?;
let result = match admission {
receipt::EffectAdmission::Replay(result) => result,
receipt::EffectAdmission::Dispatch => {
// A second current check fences cancellation/revocation that
// happened while waiting for durable admission. Any lost
// acknowledgement from this point remains indeterminate.
invocation(
api.clone(),
&context,
&workspace,
&invocation_id,
operation.clone(),
)
.await?;
let secret = if binding.secret_ref.is_empty() {
None
} else {
let root = ServerConfig::workspace_backend_data_root_for_data_dir(
&api.config.data_root,
&workspace,
);
Some(zeroize::Zeroizing::new(
secrets::SecretStore::new(root)
.get(&format!("plugin/{}", binding.secret_ref))
.map_err(|_| denied())?
.into_string(),
))
};
let transport = match transport_override {
Some(transport) => transport,
None => Arc::new(transport(&api).await?),
};
let cancellation = CancellationToken::new();
let result = transport
.execute_with_guard(
binding,
prepared,
secret.as_deref().map(|v| v.as_str()),
&cancellation,
|| async {
invocation(
api.clone(),
&context,
&workspace,
&invocation_id,
operation.clone(),
)
.await
.map(|_| ())
.map_err(|_| https::TransportError::Denied)
},
)
.await
.map(|response| PluginHttpResponse {
status: response.status,
headers: response.headers,
body: response.body,
})
.map_err(sent_transport_error);
let store = api.config_store.clone();
let r = record;
let fp = fingerprint;
let key = request.effect_key;
let saved = result.clone();
store
.db_executor()
.run(move || {
store.with_immediate_transaction(|tx| {
receipt::effect_complete(tx, &r, &key, &fp, &saved)
})
})
.await
.map_err(|_| PluginHostError {
code: PluginHostErrorCode::Indeterminate,
effect: PluginEffectObservation::Indeterminate,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
})?;
result
}
};
// Both saved response replay and new network results require current
// read/publication authority. No denied payload reaches the Runtime.
let observation = result
.as_ref()
.map(|_| PluginEffectObservation::ResponseObserved)
.unwrap_or_else(|error| error.effect);
invocation(api.clone(), &context, &workspace, &invocation_id, operation)
.await
.map_err(|mut error| {
error.effect = observation;
error
})?;
invocation(
api,
&context,
&workspace,
&invocation_id,
PluginHostOperation::Publish,
)
.await
.map_err(|mut error| {
error.effect = observation;
error
})?;
result.map(|response| PluginHostResponse::Response { response })
}
}
}
async fn transport(
api: &WorkspaceApi,
) -> std::result::Result<https::HttpsTransport, PluginHostError> {
let mut hosts = std::collections::BTreeSet::new();
let mut ips = std::collections::BTreeSet::new();
let AuthConfig::Passkey {
public_base_url, ..
} = &api.config.auth;
for address in std::iter::once(public_base_url)
.chain(api.config.backend_base_url.iter())
.chain(
api.config
.remote_runtime_sources
.iter()
.map(|source| &source.base_url),
)
{
if let Ok(url) = url::Url::parse(address) {
if let Some(host) = url.host_str() {
hosts.insert(host.to_ascii_lowercase());
if let Ok(ip) = host.parse() {
ips.insert(ip);
} else {
let port = url.port_or_known_default().ok_or_else(denied)?;
let resolved = tokio::time::timeout(
std::time::Duration::from_secs(5),
tokio::net::lookup_host((host, port)),
)
.await
.map_err(|_| transport_error(https::TransportError::Dns))?
.map_err(|_| transport_error(https::TransportError::Dns))?;
for (index, address) in resolved.enumerate() {
if index >= 64 {
return Err(transport_error(https::TransportError::Dns));
}
ips.insert(address.ip());
}
}
}
}
}
Ok(https::HttpsTransport::new(https::TransportPolicy {
denied_addresses: ips,
denied_hosts: hosts,
}))
}
fn sent_transport_error(error: https::TransportError) -> PluginHostError {
match error {
https::TransportError::ResponseLimit => PluginHostError {
code: PluginHostErrorCode::QuotaExceeded,
effect: PluginEffectObservation::ResponseObserved,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
},
https::TransportError::UnsupportedEncoding => PluginHostError {
code: PluginHostErrorCode::Unsupported,
effect: PluginEffectObservation::ResponseObserved,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
},
https::TransportError::HeaderLimit => PluginHostError {
code: PluginHostErrorCode::QuotaExceeded,
effect: PluginEffectObservation::Indeterminate,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
},
https::TransportError::Unavailable
| https::TransportError::DeadlineExceeded
| https::TransportError::Cancelled => PluginHostError {
code: PluginHostErrorCode::Indeterminate,
effect: PluginEffectObservation::Indeterminate,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
},
other => transport_error(other),
}
}
fn transport_error(error: https::TransportError) -> PluginHostError {
let code = match error {
https::TransportError::Denied | https::TransportError::InvalidBinding => {
PluginHostErrorCode::Denied
}
https::TransportError::InvalidRequest => PluginHostErrorCode::InvalidInput,
https::TransportError::HeaderLimit
| https::TransportError::RequestLimit
| https::TransportError::ResponseLimit => PluginHostErrorCode::QuotaExceeded,
https::TransportError::DeadlineExceeded => PluginHostErrorCode::DeadlineExceeded,
https::TransportError::Cancelled => PluginHostErrorCode::Cancelled,
_ => PluginHostErrorCode::Unavailable,
};
PluginHostError {
code,
effect: PluginEffectObservation::NotDispatched,
diagnostic_id: uuid::Uuid::now_v7().to_string(),
}
}
File diff suppressed because it is too large Load Diff
+34 -2
View File
@@ -129,7 +129,7 @@ pub use ticket::{
};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 108;
const LATEST_SCHEMA_VERSION: i64 = 109;
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
@@ -477,6 +477,11 @@ const MIGRATIONS: &[Migration] = &[
name: "Subject Ticket responsibility and current-body notification routing",
apply: migrate_subject_assignments_v107_to_v108,
},
Migration {
version: 109,
name: "Durable Plugin Host invocations and HTTPS effect receipts",
apply: migrate_plugin_receipts_v108_to_v109,
},
];
// Both schema-103 identities shipped. Keep the alternate implementation and accept
@@ -15851,6 +15856,27 @@ fn migrate_subject_assignments_v107_to_v108(conn: &Connection) -> Result<()> {
Ok(())
}
fn migrate_plugin_receipts_v108_to_v109(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 108 {
return Err(Error::Store(
"Plugin receipt migration requires schema 108".into(),
));
}
verify_schema_history(conn, 108)?;
crate::plugin_catalog::verify(conn)?;
crate::plugin_plan::verify_plan_schema(conn)?;
subject_assignment_migration::verify(conn)?;
let tx = conn.unchecked_transaction()?;
tx.execute_batch(crate::plugin_receipt::SCHEMA_SQL)?;
crate::plugin_host::verify_receipt_schema(&tx)?;
tx.execute(
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(109,?1)",
["Durable Plugin Host invocations and HTTPS effect receipts"],
)?;
tx.commit()?;
Ok(())
}
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
create_workspace_schema_structure(conn)?;
assignment_delivery::install(conn)?;
@@ -15889,6 +15915,7 @@ fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
conn.execute_batch(include_str!("activation_schema.sql"))?;
conn.execute_batch(include_str!("passkey_results_schema.sql"))?;
conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?;
conn.execute_batch(crate::plugin_receipt::SCHEMA_SQL)?;
Ok(())
}
@@ -16931,6 +16958,7 @@ fn apply_migrations_with_secret_source(
verify_schema_history(conn, LATEST_SCHEMA_VERSION)?;
crate::plugin_catalog::verify(conn)?;
crate::plugin_plan::verify_plan_schema(conn)?;
crate::plugin_host::verify_receipt_schema(conn)?;
subject_assignment_migration::verify(conn)?;
verify_workspace_runtime_binding_schema(conn)?;
verify_workspace_runtime_verification_schema(conn)?;
@@ -19214,6 +19242,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
include!("ticket_tag_upgrade_tests.rs");
include!("ticket_post_upgrade_tests.rs");
include!("migration_entrypoint_tests.rs");
include!("plugin_host_migration_tests.rs");
include!("legacy_evidence_column_tests.rs");
#[test]
@@ -19629,6 +19658,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
WorkspaceSchemaMigrationStep {version:106, name:crate::plugin_catalog::MIGRATION_NAME.into()},
WorkspaceSchemaMigrationStep {version:107, name:"Immutable Worker Plugin plans and retained pins".into()},
WorkspaceSchemaMigrationStep {version:108, name:"Subject Ticket responsibility and current-body notification routing".into()},
WorkspaceSchemaMigrationStep {version:109, name:"Durable Plugin Host invocations and HTTPS effect receipts".into()},
]
);
@@ -19747,6 +19777,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(106, crate::plugin_catalog::MIGRATION_NAME.to_string()),
(107, "Immutable Worker Plugin plans and retained pins".to_string()),
(108, "Subject Ticket responsibility and current-body notification routing".to_string()),
(109, "Durable Plugin Host invocations and HTTPS effect receipts".to_string()),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -23036,7 +23067,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(105, "Durable Subject deletion snapshots and Job admission fences".into()),
(106, crate::plugin_catalog::MIGRATION_NAME.into()),
(107, "Immutable Worker Plugin plans and retained pins".into()),
(108, "Subject Ticket responsibility and current-body notification routing".into())]);
(108, "Subject Ticket responsibility and current-body notification routing".into()),
(109, "Durable Plugin Host invocations and HTTPS effect receipts".into())]);
for (table, rows) in unchanged.iter().zip(before.iter()) {
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
}
@@ -1094,7 +1094,7 @@ mod subject_assignment_migration_tests {
}
fn expected_tail(version: i64) -> Vec<(i64, String)> {
match version {
let mut tail = match version {
96 => vec![
(
97,
@@ -1127,7 +1127,9 @@ mod subject_assignment_migration_tests {
106 => vec![(107, PLAN_NAME.into()), (108, SUBJECT_NAME.into())],
107 => vec![(108, SUBJECT_NAME.into())],
_ => panic!("unsupported fixture origin"),
}
};
tail.push((109, "Durable Plugin Host invocations and HTTPS effect receipts".into()));
tail
}
fn plan_steps(version: i64) -> Vec<WorkspaceSchemaMigrationStep> {
@@ -1176,12 +1178,12 @@ mod subject_assignment_migration_tests {
fn plugin_schema(conn: &Connection) -> Rows {
fixture_query_rows(
conn,
"SELECT type,name,tbl_name,rootpage,sql FROM sqlite_schema WHERE tbl_name LIKE 'plugin_%' ORDER BY type,name",
"SELECT type,name,tbl_name,rootpage,sql FROM sqlite_schema WHERE tbl_name LIKE 'plugin_%' AND tbl_name NOT IN ('plugin_invocations','plugin_effect_receipts') ORDER BY type,name",
)
}
#[tokio::test]
async fn applied107_only108_preserves_canonical_plans_pins_and_released_replay() {
async fn applied107_preserves_canonical_plans_pins_and_released_replay_through_later_tail() {
let (_dir, path, conn, receipts) = populated107().await;
let before = snapshot(&conn);
let rows = data(&conn, true);
@@ -1193,10 +1195,7 @@ mod subject_assignment_migration_tests {
assert_eq!(plan.current_schema_version, 107);
assert_eq!(
plan.migrations,
vec![WorkspaceSchemaMigrationStep {
version: 108,
name: SUBJECT_NAME.into()
}]
plan_steps(107)
);
assert_eq!(snapshot(&conn), before);
drop(conn);
@@ -16,6 +16,8 @@ use crate::{Error, Result};
/// Explicit domain-owned purge inventory. The deletion operation tombstone is intentionally
/// excluded so retries and audit remain available after the Workspace row is gone.
const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"plugin_effect_receipts",
"plugin_invocations",
"plugin_execution_plans",
"plugin_references",
"plugin_operations",
@@ -509,6 +511,8 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
// commit; update guards remain installed throughout.
let mut ticket_delete_guards = Vec::new();
for (table, trigger) in [
("plugin_effect_receipts", "plugin_effect_retained"),
("plugin_invocations", "plugin_invocation_retained"),
("plugin_execution_plans", "plugin_plan_replay_retained"),
("notification_source_preparation_failures", "notification_source_preparation_failures_no_delete"),
("notification_job_targets", "notification_job_targets_no_delete"),
@@ -1303,6 +1307,12 @@ mod tests {
for kind in ["config","pending","active","restorable"] {
conn.execute("INSERT INTO plugin_references VALUES(?1,?2,'owner','installation',?3)", params![workspace,kind,digest])?;
}
conn.execute("INSERT INTO plugin_execution_plans VALUES(?1,'plan','actor','create',?2,'{}','active')", params![workspace,digest])?;
conn.execute("INSERT INTO plugin_invocations(workspace_id,invocation_id,plan_id,plan_fingerprint,installation_id,runtime_id,worker_id,execution_json,consumer_key,consumer_json,created_at_ms,expires_at_ms) VALUES(?1,'invocation','plan',?2,'installation','runtime','worker','{}','[\"tool_call\",\"call\"]','{}',1,1000)", params![workspace,digest])?;
conn.execute("INSERT INTO plugin_effect_receipts(workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms,state,error_json) VALUES(?1,'invocation','key',?2,2,'pending',?3)", params![workspace,digest,r#"{"code":"indeterminate","effect":"indeterminate","diagnostic_id":"safe"}"#])?;
for table in ["plugin_effect_receipts", "plugin_invocations", "plugin_execution_plans"] {
assert!(conn.execute(&format!("DELETE FROM {table} WHERE workspace_id=?1"), [workspace]).is_err());
}
Ok(())
}).unwrap();
}
@@ -1310,6 +1320,9 @@ mod tests {
store
.with_conn(|conn| {
[
"plugin_effect_receipts",
"plugin_invocations",
"plugin_execution_plans",
"plugin_references",
"plugin_operations",
"plugin_uploads",
@@ -1329,7 +1342,7 @@ mod tests {
.unwrap()
};
let other = counts("workspace-b");
assert_eq!(counts("workspace-a"), vec![4, 1, 1, 1, 1]);
assert_eq!(counts("workspace-a"), vec![1, 1, 1, 4, 1, 1, 1, 1]);
let preflight = store
.workspace_deletion_preflight(&owner, &workspace_id)
.unwrap();
@@ -1345,6 +1358,24 @@ mod tests {
)
.unwrap()
.operation;
store.with_conn(|conn| {
conn.execute_batch("CREATE TRIGGER plugin_purge_failure BEFORE DELETE ON workspaces WHEN OLD.workspace_id='workspace-a' BEGIN SELECT RAISE(ABORT,'injected purge failure'); END;")?;
Ok(())
}).unwrap();
assert!(
store
.finalize_workspace_deletion(&operation.operation_id)
.is_err()
);
assert_eq!(counts("workspace-a"), vec![1, 1, 1, 4, 1, 1, 1, 1]);
assert_eq!(counts("workspace-b"), other);
store
.with_conn(|conn| {
crate::plugin_host::verify_receipt_schema(conn)?;
conn.execute_batch("DROP TRIGGER plugin_purge_failure")?;
Ok(())
})
.unwrap();
assert_eq!(
store
.finalize_workspace_deletion(&operation.operation_id)
@@ -1352,10 +1383,22 @@ mod tests {
.state,
WorkspaceDeletionState::Succeeded
);
assert_eq!(counts("workspace-a"), vec![0; 5]);
assert_eq!(counts("workspace-a"), vec![0; 8]);
assert_eq!(counts("workspace-b"), other);
store
.with_conn(|conn| {
crate::plugin_host::verify_receipt_schema(conn)?;
assert_eq!(
conn.query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0))?,
"ok"
);
assert!(
conn.execute(
"DELETE FROM plugin_effect_receipts WHERE workspace_id='workspace-b'",
[]
)
.is_err()
);
assert_eq!(
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r
.get::<_, i64>(