feat: enforce live Plugin Host authority and durable HTTPS effects
This commit is contained in:
@@ -8,6 +8,7 @@ rust-version.workspace = true
|
||||
[dependencies]
|
||||
async-trait.workspace = true
|
||||
manifest.workspace = true
|
||||
server-api.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
@@ -23,4 +24,4 @@ wit-component = { version = "0.244.0", features = ["dummy-module"] }
|
||||
wit-parser = "0.244.0"
|
||||
wasmprinter = "0.244.0"
|
||||
wat = "1"
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "test-util"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "test-util", "net"] }
|
||||
|
||||
@@ -12,5 +12,6 @@ pub mod bindings {
|
||||
|
||||
mod executor;
|
||||
pub mod host;
|
||||
pub mod owner;
|
||||
mod sandbox;
|
||||
pub use executor::*;
|
||||
|
||||
@@ -0,0 +1,926 @@
|
||||
//! Shared owner transport and WIT Host adapter. In-process owners implement the
|
||||
//! same trait directly; a failed remote call never selects a local fallback.
|
||||
use crate::host::{Host, InvocationContext, Operation, errors, http, logging};
|
||||
use async_trait::async_trait;
|
||||
use server_api::{
|
||||
PluginConsumer, PluginEffectObservation, PluginHostError, PluginHostErrorCode,
|
||||
PluginHostOperation, PluginHostRequest, PluginHostResponse, PluginHttpMethod,
|
||||
PluginHttpRequest, PluginLogEntry, PluginLogField, PluginLogLevel, PluginLogScalar,
|
||||
};
|
||||
use std::{
|
||||
any::Any,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
use tokio::time::Instant;
|
||||
|
||||
/// Bound to one authenticated caller and Workspace by the integration. The owner
|
||||
/// resolves all identity/policy from that binding and live-checks every operation,
|
||||
/// including Open/Send/Log. Implementations must not detach effects or auto-retry.
|
||||
#[async_trait]
|
||||
pub trait PluginHostOwner: Send + Sync {
|
||||
async fn call(&self, request: PluginHostRequest)
|
||||
-> Result<PluginHostResponse, PluginHostError>;
|
||||
}
|
||||
|
||||
/// Uses an injected, already configured generated client (including its existing
|
||||
/// Runtime request authorizer). This adapter never creates or owns credentials.
|
||||
pub struct RemotePluginHostOwner<A> {
|
||||
client: Arc<server_api::ServerApiClient<A>>,
|
||||
workspace_id: String,
|
||||
}
|
||||
impl<A> RemotePluginHostOwner<A> {
|
||||
pub fn new(client: Arc<server_api::ServerApiClient<A>>, workspace_id: String) -> Self {
|
||||
Self {
|
||||
client,
|
||||
workspace_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl<A: server_api::client_support::RequestAuthorizer> PluginHostOwner
|
||||
for RemotePluginHostOwner<A>
|
||||
{
|
||||
async fn call(
|
||||
&self,
|
||||
request: PluginHostRequest,
|
||||
) -> Result<PluginHostResponse, PluginHostError> {
|
||||
let may_dispatch = matches!(request, PluginHostRequest::Send { .. });
|
||||
self.client
|
||||
.plugin_host(self.workspace_id.clone(), request)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
match error {
|
||||
server_api::client_support::ClientError::Public { error, .. } => error,
|
||||
// A missing response is not proof that a Send was not dispatched.
|
||||
// Preserve uncertainty; never synthesize a success or retry it.
|
||||
server_api::client_support::ClientError::Failure(_) => local_error(
|
||||
if may_dispatch {
|
||||
PluginHostErrorCode::Indeterminate
|
||||
} else {
|
||||
PluginHostErrorCode::Unavailable
|
||||
},
|
||||
if may_dispatch {
|
||||
PluginEffectObservation::Indeterminate
|
||||
} else {
|
||||
PluginEffectObservation::NotDispatched
|
||||
},
|
||||
),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Exactly one owner-issued invocation, with a local snapshot upper bound.
|
||||
/// Obtain the execution context through `context()` after Begin; the caller's
|
||||
/// pre-Begin invocation ID is never sent to, or adopted as authority by, the owner.
|
||||
pub struct OwnerHost {
|
||||
owner: Arc<dyn PluginHostOwner>,
|
||||
context: InvocationContext,
|
||||
binding_identity: Arc<()>,
|
||||
}
|
||||
struct OwnerConnection {
|
||||
invocation_id: String,
|
||||
slot: String,
|
||||
identity: Arc<()>,
|
||||
}
|
||||
impl OwnerHost {
|
||||
pub async fn begin(
|
||||
owner: Arc<dyn PluginHostOwner>,
|
||||
mut context: InvocationContext,
|
||||
consumer: PluginConsumer,
|
||||
) -> Result<Self, PluginHostError> {
|
||||
if let PluginConsumer::ToolCall { tool_name, .. } = &consumer {
|
||||
if *tool_name != context.tool_name {
|
||||
return Err(local_error(
|
||||
PluginHostErrorCode::Denied,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
));
|
||||
}
|
||||
}
|
||||
check_lifetime(&context)?;
|
||||
let response = call_owner(
|
||||
owner.as_ref(),
|
||||
&context,
|
||||
PluginHostRequest::Begin {
|
||||
plan_id: context.plan.clone(),
|
||||
installation_id: context.installation.clone(),
|
||||
consumer,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let PluginHostResponse::Invocation {
|
||||
invocation_id,
|
||||
expires_at_ms,
|
||||
} = response
|
||||
else {
|
||||
return Err(protocol_error(false));
|
||||
};
|
||||
if invocation_id.is_empty() || invocation_id.len() > 1024 {
|
||||
return Err(protocol_error(false));
|
||||
}
|
||||
let now_ms = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_millis()).ok())
|
||||
.ok_or_else(|| protocol_error(false))?;
|
||||
let remaining = expires_at_ms
|
||||
.checked_sub(now_ms)
|
||||
.filter(|ms| *ms > 0)
|
||||
.ok_or_else(|| {
|
||||
local_error(
|
||||
PluginHostErrorCode::DeadlineExceeded,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
)
|
||||
})?;
|
||||
let expires = Instant::now()
|
||||
.checked_add(Duration::from_millis(remaining as u64))
|
||||
.ok_or_else(|| protocol_error(false))?;
|
||||
context.deadline = context.deadline.min(expires);
|
||||
context.invocation_id = invocation_id;
|
||||
check_lifetime(&context)?;
|
||||
Ok(Self {
|
||||
owner,
|
||||
context,
|
||||
binding_identity: Arc::new(()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn context(&self) -> &InvocationContext {
|
||||
&self.context
|
||||
}
|
||||
|
||||
/// Persist cancellation at the owner as well as stopping local guest waits.
|
||||
/// This never claims to undo an already admitted external effect. The caller
|
||||
/// must reconcile an unavailable response, not restore under a new identity.
|
||||
pub async fn cancel(&self) -> Result<(), PluginHostError> {
|
||||
self.context.cancellation.cancel();
|
||||
let response = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
self.owner.call(PluginHostRequest::Cancel {
|
||||
invocation_id: self.context.invocation_id.clone(),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
local_error(
|
||||
PluginHostErrorCode::Unavailable,
|
||||
PluginEffectObservation::Indeterminate,
|
||||
)
|
||||
})??;
|
||||
if response != PluginHostResponse::Authorized {
|
||||
return Err(protocol_error(false));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_context(&self, context: &InvocationContext) -> Result<(), PluginHostError> {
|
||||
let bound = &self.context;
|
||||
if context.workspace != bound.workspace
|
||||
|| context.installation != bound.installation
|
||||
|| context.plan != bound.plan
|
||||
|| context.subject != bound.subject
|
||||
|| context.invocation_id != bound.invocation_id
|
||||
|| context.artifact != bound.artifact
|
||||
|| context.tool_name != bound.tool_name
|
||||
|| context.grants.logging != bound.grants.logging
|
||||
|| context.grants.connection_slots != bound.grants.connection_slots
|
||||
// The executor may shorten the deadline and derive a child token.
|
||||
|| context.deadline > bound.deadline
|
||||
{
|
||||
return Err(local_error(
|
||||
PluginHostErrorCode::Denied,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
));
|
||||
}
|
||||
check_lifetime(bound)?;
|
||||
check_lifetime(context)
|
||||
}
|
||||
fn connection<'a>(
|
||||
&self,
|
||||
binding: &'a (dyn Any + Send + Sync),
|
||||
) -> Result<&'a OwnerConnection, PluginHostError> {
|
||||
binding
|
||||
.downcast_ref::<OwnerConnection>()
|
||||
.filter(|b| {
|
||||
b.invocation_id == self.context.invocation_id
|
||||
&& Arc::ptr_eq(&b.identity, &self.binding_identity)
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
local_error(
|
||||
PluginHostErrorCode::Denied,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
)
|
||||
})
|
||||
}
|
||||
fn check_slot(&self, slot: &str) -> Result<(), PluginHostError> {
|
||||
if !self.context.grants.connection_slots.contains(slot) {
|
||||
return Err(local_error(
|
||||
PluginHostErrorCode::Denied,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Host for OwnerHost {
|
||||
async fn authorize(
|
||||
&self,
|
||||
context: &InvocationContext,
|
||||
operation: Operation<'_>,
|
||||
) -> Result<(), errors::HostError> {
|
||||
self.check_context(context).map_err(wit_error)?;
|
||||
let operation = match operation {
|
||||
Operation::Admit => PluginHostOperation::Admit,
|
||||
Operation::Publish => PluginHostOperation::Publish,
|
||||
Operation::Open { slot } => {
|
||||
self.check_slot(slot).map_err(wit_error)?;
|
||||
PluginHostOperation::Open { slot: slot.into() }
|
||||
}
|
||||
Operation::Send { slot, binding, .. } => {
|
||||
let connection = self.connection(binding).map_err(wit_error)?;
|
||||
self.check_slot(slot).map_err(wit_error)?;
|
||||
if connection.slot != slot {
|
||||
return Err(wit_error(local_error(
|
||||
PluginHostErrorCode::Denied,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
)));
|
||||
}
|
||||
PluginHostOperation::Send { slot: slot.into() }
|
||||
}
|
||||
Operation::Log => {
|
||||
if !context.grants.logging {
|
||||
return Err(wit_error(local_error(
|
||||
PluginHostErrorCode::Denied,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
)));
|
||||
}
|
||||
PluginHostOperation::Log
|
||||
}
|
||||
};
|
||||
match call_owner(
|
||||
self.owner.as_ref(),
|
||||
context,
|
||||
PluginHostRequest::Authorize {
|
||||
invocation_id: self.context.invocation_id.clone(),
|
||||
operation,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(wit_error)?
|
||||
{
|
||||
PluginHostResponse::Authorized => Ok(()),
|
||||
_ => Err(wit_error(protocol_error(false))),
|
||||
}
|
||||
}
|
||||
async fn open(
|
||||
&self,
|
||||
context: &InvocationContext,
|
||||
slot: &str,
|
||||
) -> Result<Arc<dyn Any + Send + Sync>, errors::HostError> {
|
||||
self.check_context(context).map_err(wit_error)?;
|
||||
self.check_slot(slot).map_err(wit_error)?;
|
||||
match call_owner(
|
||||
self.owner.as_ref(),
|
||||
context,
|
||||
PluginHostRequest::Open {
|
||||
invocation_id: self.context.invocation_id.clone(),
|
||||
slot: slot.into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(wit_error)?
|
||||
{
|
||||
PluginHostResponse::Opened => Ok(Arc::new(OwnerConnection {
|
||||
invocation_id: self.context.invocation_id.clone(),
|
||||
slot: slot.into(),
|
||||
identity: self.binding_identity.clone(),
|
||||
})),
|
||||
_ => Err(wit_error(protocol_error(false))),
|
||||
}
|
||||
}
|
||||
async fn send(
|
||||
&self,
|
||||
context: &InvocationContext,
|
||||
binding: &(dyn Any + Send + Sync),
|
||||
request: http::Request,
|
||||
) -> Result<http::Response, errors::HostError> {
|
||||
self.check_context(context).map_err(wit_error)?;
|
||||
let connection = self.connection(binding).map_err(wit_error)?;
|
||||
self.check_slot(&connection.slot).map_err(wit_error)?;
|
||||
let request = PluginHttpRequest {
|
||||
method: match request.method {
|
||||
http::Method::Get => PluginHttpMethod::Get,
|
||||
http::Method::Head => PluginHttpMethod::Head,
|
||||
http::Method::Post => PluginHttpMethod::Post,
|
||||
http::Method::Put => PluginHttpMethod::Put,
|
||||
http::Method::Patch => PluginHttpMethod::Patch,
|
||||
http::Method::Delete => PluginHttpMethod::Delete,
|
||||
http::Method::Options => PluginHttpMethod::Options,
|
||||
},
|
||||
path: request.path,
|
||||
query: pairs(request.query),
|
||||
headers: pairs(request.headers),
|
||||
body: request.body,
|
||||
timeout_ms: request.timeout_ms,
|
||||
effect_key: request.effect_key,
|
||||
};
|
||||
match call_owner(
|
||||
self.owner.as_ref(),
|
||||
context,
|
||||
PluginHostRequest::Send {
|
||||
invocation_id: self.context.invocation_id.clone(),
|
||||
slot: connection.slot.clone(),
|
||||
request,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(wit_error)?
|
||||
{
|
||||
PluginHostResponse::Response { response } => Ok(http::Response {
|
||||
status: response.status,
|
||||
headers: response
|
||||
.headers
|
||||
.into_iter()
|
||||
.map(|(name, value)| http::Pair { name, value })
|
||||
.collect(),
|
||||
body: response.body,
|
||||
}),
|
||||
_ => Err(wit_error(protocol_error(true))),
|
||||
}
|
||||
}
|
||||
async fn log(
|
||||
&self,
|
||||
context: &InvocationContext,
|
||||
entry: logging::Entry,
|
||||
) -> Result<(), errors::HostError> {
|
||||
self.check_context(context).map_err(wit_error)?;
|
||||
if !context.grants.logging {
|
||||
return Err(wit_error(local_error(
|
||||
PluginHostErrorCode::Denied,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
)));
|
||||
}
|
||||
let entry = PluginLogEntry {
|
||||
level: match entry.level {
|
||||
logging::Level::Trace => PluginLogLevel::Trace,
|
||||
logging::Level::Debug => PluginLogLevel::Debug,
|
||||
logging::Level::Info => PluginLogLevel::Info,
|
||||
logging::Level::Warn => PluginLogLevel::Warn,
|
||||
logging::Level::Error => PluginLogLevel::Error,
|
||||
},
|
||||
event_code: entry.event_code,
|
||||
message: entry.message,
|
||||
fields: entry
|
||||
.fields
|
||||
.into_iter()
|
||||
.map(|f| {
|
||||
Ok(PluginLogField {
|
||||
name: f.name,
|
||||
value: match f.value {
|
||||
logging::Scalar::Text(v) => PluginLogScalar::Text(v),
|
||||
logging::Scalar::Boolean(v) => PluginLogScalar::Boolean(v),
|
||||
logging::Scalar::Signed(v) => PluginLogScalar::Signed(v),
|
||||
logging::Scalar::Unsigned(v) => PluginLogScalar::Unsigned(v),
|
||||
logging::Scalar::Number(v) if v.is_finite() => {
|
||||
PluginLogScalar::Number(v)
|
||||
}
|
||||
logging::Scalar::Number(_) => {
|
||||
return Err(wit_error(local_error(
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
)));
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
.collect::<Result<_, errors::HostError>>()?,
|
||||
};
|
||||
match call_owner(
|
||||
self.owner.as_ref(),
|
||||
context,
|
||||
PluginHostRequest::Log {
|
||||
invocation_id: self.context.invocation_id.clone(),
|
||||
entry,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(wit_error)?
|
||||
{
|
||||
PluginHostResponse::Logged => Ok(()),
|
||||
_ => Err(wit_error(protocol_error(false))),
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn call_owner(
|
||||
owner: &dyn PluginHostOwner,
|
||||
context: &InvocationContext,
|
||||
request: PluginHostRequest,
|
||||
) -> Result<PluginHostResponse, PluginHostError> {
|
||||
check_lifetime(context)?;
|
||||
let effect = if matches!(request, PluginHostRequest::Send { .. }) {
|
||||
PluginEffectObservation::Indeterminate
|
||||
} else {
|
||||
PluginEffectObservation::NotDispatched
|
||||
};
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = context.cancellation.cancelled() => Err(local_error(PluginHostErrorCode::Cancelled, effect)),
|
||||
_ = tokio::time::sleep_until(context.deadline) => Err(local_error(PluginHostErrorCode::DeadlineExceeded, effect)),
|
||||
response = owner.call(request) => response,
|
||||
}
|
||||
}
|
||||
|
||||
fn pairs(pairs: Vec<http::Pair>) -> Vec<(String, String)> {
|
||||
pairs.into_iter().map(|p| (p.name, p.value)).collect()
|
||||
}
|
||||
fn local_error(code: PluginHostErrorCode, effect: PluginEffectObservation) -> PluginHostError {
|
||||
PluginHostError {
|
||||
code,
|
||||
effect,
|
||||
diagnostic_id: String::new(),
|
||||
}
|
||||
}
|
||||
fn protocol_error(may_dispatch: bool) -> PluginHostError {
|
||||
local_error(
|
||||
if may_dispatch {
|
||||
PluginHostErrorCode::Indeterminate
|
||||
} else {
|
||||
PluginHostErrorCode::Unavailable
|
||||
},
|
||||
if may_dispatch {
|
||||
PluginEffectObservation::Indeterminate
|
||||
} else {
|
||||
PluginEffectObservation::NotDispatched
|
||||
},
|
||||
)
|
||||
}
|
||||
fn check_lifetime(context: &InvocationContext) -> Result<(), PluginHostError> {
|
||||
if context.cancellation.is_cancelled() {
|
||||
return Err(local_error(
|
||||
PluginHostErrorCode::Cancelled,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
));
|
||||
}
|
||||
if Instant::now() >= context.deadline {
|
||||
return Err(local_error(
|
||||
PluginHostErrorCode::DeadlineExceeded,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn wit_error(error: PluginHostError) -> errors::HostError {
|
||||
let message = error.message().to_owned();
|
||||
errors::HostError {
|
||||
code: match error.code {
|
||||
PluginHostErrorCode::InvalidInput => errors::Code::InvalidInput,
|
||||
PluginHostErrorCode::Denied => errors::Code::Denied,
|
||||
PluginHostErrorCode::NotFound => errors::Code::NotFound,
|
||||
PluginHostErrorCode::Conflict => errors::Code::Conflict,
|
||||
PluginHostErrorCode::Unsupported => errors::Code::Unsupported,
|
||||
PluginHostErrorCode::QuotaExceeded => errors::Code::QuotaExceeded,
|
||||
PluginHostErrorCode::DeadlineExceeded => errors::Code::DeadlineExceeded,
|
||||
PluginHostErrorCode::Cancelled => errors::Code::Cancelled,
|
||||
PluginHostErrorCode::Unavailable => errors::Code::Unavailable,
|
||||
PluginHostErrorCode::Indeterminate => errors::Code::Indeterminate,
|
||||
PluginHostErrorCode::ReceiptExpired => errors::Code::ReceiptExpired,
|
||||
},
|
||||
effect: match error.effect {
|
||||
PluginEffectObservation::NotDispatched => errors::EffectObservation::NotDispatched,
|
||||
PluginEffectObservation::ResponseObserved => {
|
||||
errors::EffectObservation::ResponseObserved
|
||||
}
|
||||
PluginEffectObservation::Indeterminate => errors::EffectObservation::Indeterminate,
|
||||
},
|
||||
message,
|
||||
diagnostic_id: error.diagnostic_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
// Adapter contract tests: the real generated HTTP client and OwnerHost run
|
||||
// against one deterministic owner, directly or through an isolated loopback
|
||||
// endpoint. Server authentication, HTTPS policy and durable receipts are not
|
||||
// substituted claims of this suite; their concrete owner tests live in Server.
|
||||
use super::*;
|
||||
use crate::{
|
||||
ArtifactIdentity,
|
||||
host::{CancellationToken, Grants},
|
||||
};
|
||||
use server_api::{PluginHttpResponse, client_support, server_support};
|
||||
use std::sync::{
|
||||
Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
struct TestOwner {
|
||||
requests: Mutex<Vec<PluginHostRequest>>,
|
||||
revoked: AtomicBool,
|
||||
}
|
||||
#[async_trait]
|
||||
impl PluginHostOwner for TestOwner {
|
||||
async fn call(
|
||||
&self,
|
||||
request: PluginHostRequest,
|
||||
) -> Result<PluginHostResponse, PluginHostError> {
|
||||
self.requests.lock().unwrap().push(request.clone());
|
||||
if self.revoked.load(Ordering::SeqCst) {
|
||||
return Err(PluginHostError {
|
||||
code: PluginHostErrorCode::Denied,
|
||||
effect: PluginEffectObservation::NotDispatched,
|
||||
diagnostic_id: "revoked-diagnostic".into(),
|
||||
});
|
||||
}
|
||||
Ok(match request {
|
||||
PluginHostRequest::Begin { .. } => PluginHostResponse::Invocation {
|
||||
invocation_id: "owner-issued".into(),
|
||||
expires_at_ms: i64::try_from(
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis(),
|
||||
)
|
||||
.unwrap()
|
||||
+ 60_000,
|
||||
},
|
||||
PluginHostRequest::Cancel { .. } | PluginHostRequest::Authorize { .. } => {
|
||||
PluginHostResponse::Authorized
|
||||
}
|
||||
PluginHostRequest::Open { .. } => PluginHostResponse::Opened,
|
||||
PluginHostRequest::Send { request, .. } => PluginHostResponse::Response {
|
||||
response: PluginHttpResponse {
|
||||
status: 207,
|
||||
headers: request.headers,
|
||||
body: request.body,
|
||||
},
|
||||
},
|
||||
PluginHostRequest::Log { .. } => PluginHostResponse::Logged,
|
||||
})
|
||||
}
|
||||
}
|
||||
fn context() -> InvocationContext {
|
||||
InvocationContext {
|
||||
workspace: "ws".into(),
|
||||
installation: "install".into(),
|
||||
plan: "plan".into(),
|
||||
subject: "subject".into(),
|
||||
invocation_id: "not-authority".into(),
|
||||
artifact: ArtifactIdentity {
|
||||
package_digest: "package".into(),
|
||||
artifact_digest: "artifact".into(),
|
||||
artifact_size: 123,
|
||||
},
|
||||
tool_name: "tool".into(),
|
||||
grants: Grants {
|
||||
logging: true,
|
||||
connection_slots: ["api".into(), "other".into()].into(),
|
||||
},
|
||||
deadline: Instant::now() + Duration::from_secs(60),
|
||||
cancellation: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
fn consumer() -> PluginConsumer {
|
||||
PluginConsumer::ToolCall {
|
||||
tool_call_id: "call".into(),
|
||||
tool_name: "tool".into(),
|
||||
}
|
||||
}
|
||||
fn request() -> http::Request {
|
||||
http::Request {
|
||||
method: http::Method::Post,
|
||||
path: "/v1/items".into(),
|
||||
query: vec![
|
||||
http::Pair {
|
||||
name: "q".into(),
|
||||
value: "one".into(),
|
||||
},
|
||||
http::Pair {
|
||||
name: "q".into(),
|
||||
value: "two".into(),
|
||||
},
|
||||
],
|
||||
headers: vec![http::Pair {
|
||||
name: "x-test".into(),
|
||||
value: "value".into(),
|
||||
}],
|
||||
body: vec![0, 255, 42],
|
||||
timeout_ms: 1234,
|
||||
effect_key: "effect".into(),
|
||||
}
|
||||
}
|
||||
|
||||
struct TestAuthorizer;
|
||||
impl client_support::RequestAuthorizer for TestAuthorizer {
|
||||
fn authorize(
|
||||
&self,
|
||||
_: client_support::AuthorizerRequest<'_>,
|
||||
) -> Result<client_support::framework::header::HeaderMap, client_support::AuthorizationError>
|
||||
{
|
||||
let mut headers = client_support::framework::header::HeaderMap::new();
|
||||
headers.insert("authorization", "Bearer isolated-test".parse().unwrap());
|
||||
Ok(headers)
|
||||
}
|
||||
}
|
||||
struct Endpoint {
|
||||
url: String,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
impl Drop for Endpoint {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
async fn endpoint(owner: Arc<TestOwner>) -> Endpoint {
|
||||
use server_support::framework::{self as axum, HeaderMap, Json, Path, Response, State};
|
||||
async fn call(
|
||||
State(owner): State<Arc<TestOwner>>,
|
||||
Path(workspace): Path<String>,
|
||||
headers: HeaderMap,
|
||||
Json(request): Json<PluginHostRequest>,
|
||||
) -> Response {
|
||||
assert_eq!(workspace, "ws");
|
||||
assert_eq!(
|
||||
headers.get("authorization").unwrap(),
|
||||
"Bearer isolated-test"
|
||||
);
|
||||
match owner.call(request).await {
|
||||
Ok(response) => server_support::json_response(axum::StatusCode::OK, response),
|
||||
Err(error) => server_support::error_response(axum::StatusCode::FORBIDDEN, error),
|
||||
}
|
||||
}
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let router = axum::Router::new()
|
||||
.route("/api/w/{workspace_id}/plugin-host", axum::post(call))
|
||||
.with_state(owner);
|
||||
let task = tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
});
|
||||
Endpoint { url, task }
|
||||
}
|
||||
async fn remote(owner: Arc<TestOwner>) -> (Arc<dyn PluginHostOwner>, Endpoint) {
|
||||
let endpoint = endpoint(owner).await;
|
||||
let http = client_support::framework::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.unwrap();
|
||||
let core = client_support::ClientBuilder::try_new(&endpoint.url)
|
||||
.unwrap()
|
||||
.client(http)
|
||||
.authorizer(TestAuthorizer)
|
||||
.request_timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap();
|
||||
let api = Arc::new(server_api::ServerApiClient::from_core(core));
|
||||
(
|
||||
Arc::new(RemotePluginHostOwner::new(api, "ws".into())),
|
||||
endpoint,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_and_direct_owner_preserve_requests_results_and_live_denial() {
|
||||
let mut transcripts = Vec::new();
|
||||
for use_remote in [false, true] {
|
||||
let owner = Arc::new(TestOwner::default());
|
||||
let (transport, _endpoint): (Arc<dyn PluginHostOwner>, _) = if use_remote {
|
||||
let (transport, endpoint) = remote(owner.clone()).await;
|
||||
(transport, Some(endpoint))
|
||||
} else {
|
||||
(owner.clone(), None)
|
||||
};
|
||||
let host = OwnerHost::begin(transport, context(), consumer())
|
||||
.await
|
||||
.unwrap();
|
||||
let ctx = host.context();
|
||||
assert_eq!(ctx.invocation_id, "owner-issued");
|
||||
host.authorize(ctx, Operation::Admit).await.unwrap();
|
||||
host.authorize(ctx, Operation::Open { slot: "api" })
|
||||
.await
|
||||
.unwrap();
|
||||
let connection = host.open(ctx, "api").await.unwrap();
|
||||
host.authorize(
|
||||
ctx,
|
||||
Operation::Send {
|
||||
slot: "api",
|
||||
binding: connection.as_ref(),
|
||||
request: &request(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let response = host
|
||||
.send(ctx, connection.as_ref(), request())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status, 207);
|
||||
assert_eq!(response.body, vec![0, 255, 42]);
|
||||
assert_eq!(response.headers[0].value, "value");
|
||||
host.authorize(ctx, Operation::Log).await.unwrap();
|
||||
host.log(
|
||||
ctx,
|
||||
logging::Entry {
|
||||
level: logging::Level::Warn,
|
||||
event_code: "test.event".into(),
|
||||
message: "bounded".into(),
|
||||
fields: vec![logging::Field {
|
||||
name: "count".into(),
|
||||
value: logging::Scalar::Signed(-2),
|
||||
}],
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
host.authorize(ctx, Operation::Publish).await.unwrap();
|
||||
owner.revoked.store(true, Ordering::SeqCst);
|
||||
for operation in [
|
||||
Operation::Admit,
|
||||
Operation::Publish,
|
||||
Operation::Open { slot: "api" },
|
||||
Operation::Log,
|
||||
] {
|
||||
let error = host.authorize(ctx, operation).await.unwrap_err();
|
||||
assert_eq!(error.code, errors::Code::Denied);
|
||||
assert_eq!(error.diagnostic_id, "revoked-diagnostic");
|
||||
assert_eq!(error.effect, errors::EffectObservation::NotDispatched);
|
||||
}
|
||||
assert_eq!(
|
||||
host.send(ctx, connection.as_ref(), request())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
errors::Code::Denied
|
||||
);
|
||||
transcripts.push(owner.requests.lock().unwrap().clone());
|
||||
}
|
||||
assert_eq!(transcripts[0], transcripts[1]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn changed_context_or_cross_invocation_connection_never_reaches_owner() {
|
||||
let owner = Arc::new(TestOwner::default());
|
||||
let host = OwnerHost::begin(owner.clone(), context(), consumer())
|
||||
.await
|
||||
.unwrap();
|
||||
let other = OwnerHost::begin(owner.clone(), context(), consumer())
|
||||
.await
|
||||
.unwrap();
|
||||
let connection = other.open(other.context(), "api").await.unwrap();
|
||||
let count = owner.requests.lock().unwrap().len();
|
||||
// Even an owner reusing an invocation ID cannot make private handles interchangeable.
|
||||
assert_eq!(
|
||||
host.send(host.context(), connection.as_ref(), request())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
errors::Code::Denied
|
||||
);
|
||||
let mutations: Vec<Box<dyn Fn(&mut InvocationContext)>> = vec![
|
||||
Box::new(|c| c.workspace.push('x')),
|
||||
Box::new(|c| c.installation.push('x')),
|
||||
Box::new(|c| c.plan.push('x')),
|
||||
Box::new(|c| c.subject.push('x')),
|
||||
Box::new(|c| c.invocation_id.push('x')),
|
||||
Box::new(|c| c.artifact.artifact_digest.push('x')),
|
||||
Box::new(|c| c.tool_name.push('x')),
|
||||
Box::new(|c| c.grants.logging = false),
|
||||
Box::new(|c| {
|
||||
c.grants.connection_slots.insert("new".into());
|
||||
}),
|
||||
Box::new(|c| c.deadline += Duration::from_secs(1)),
|
||||
];
|
||||
for mutate in mutations {
|
||||
let mut changed = host.context().clone();
|
||||
mutate(&mut changed);
|
||||
assert_eq!(
|
||||
host.authorize(&changed, Operation::Admit)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
errors::Code::Denied
|
||||
);
|
||||
}
|
||||
let own = host.open(host.context(), "api").await.unwrap();
|
||||
assert_eq!(
|
||||
host.authorize(
|
||||
host.context(),
|
||||
Operation::Send {
|
||||
slot: "other",
|
||||
binding: own.as_ref(),
|
||||
request: &request()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
errors::Code::Denied
|
||||
);
|
||||
assert_eq!(owner.requests.lock().unwrap().len(), count + 1);
|
||||
host.context().cancellation.cancel();
|
||||
assert_eq!(
|
||||
host.authorize(host.context(), Operation::Admit)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.code,
|
||||
errors::Code::Cancelled
|
||||
);
|
||||
assert_eq!(owner.requests.lock().unwrap().len(), count + 1);
|
||||
}
|
||||
|
||||
struct PendingOwner {
|
||||
inner: TestOwner,
|
||||
begin: bool,
|
||||
dropped: Arc<AtomicBool>,
|
||||
started: tokio::sync::Notify,
|
||||
}
|
||||
struct Dropped(Arc<AtomicBool>);
|
||||
impl Drop for Dropped {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl PluginHostOwner for PendingOwner {
|
||||
async fn call(
|
||||
&self,
|
||||
request: PluginHostRequest,
|
||||
) -> Result<PluginHostResponse, PluginHostError> {
|
||||
if self.begin || matches!(request, PluginHostRequest::Send { .. }) {
|
||||
let _guard = Dropped(self.dropped.clone());
|
||||
self.started.notify_one();
|
||||
std::future::pending().await
|
||||
} else {
|
||||
self.inner.call(request).await
|
||||
}
|
||||
}
|
||||
}
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn begin_deadline_drops_pending_owner_call() {
|
||||
let dropped = Arc::new(AtomicBool::new(false));
|
||||
let owner = Arc::new(PendingOwner {
|
||||
inner: TestOwner::default(),
|
||||
begin: true,
|
||||
dropped: dropped.clone(),
|
||||
started: tokio::sync::Notify::new(),
|
||||
});
|
||||
let error = OwnerHost::begin(owner, context(), consumer())
|
||||
.await
|
||||
.err()
|
||||
.unwrap();
|
||||
assert_eq!(error.code, PluginHostErrorCode::DeadlineExceeded);
|
||||
assert_eq!(error.effect, PluginEffectObservation::NotDispatched);
|
||||
assert!(dropped.load(Ordering::SeqCst));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn cancelled_pending_send_drops_owner_future_and_preserves_uncertainty() {
|
||||
let dropped = Arc::new(AtomicBool::new(false));
|
||||
let owner = Arc::new(PendingOwner {
|
||||
inner: TestOwner::default(),
|
||||
begin: false,
|
||||
dropped: dropped.clone(),
|
||||
started: tokio::sync::Notify::new(),
|
||||
});
|
||||
let host = OwnerHost::begin(owner.clone(), context(), consumer())
|
||||
.await
|
||||
.unwrap();
|
||||
let binding = host.open(host.context(), "api").await.unwrap();
|
||||
let cancellation = host.context().cancellation.clone();
|
||||
let (result, ()) = tokio::join!(
|
||||
host.send(host.context(), binding.as_ref(), request()),
|
||||
async {
|
||||
owner.started.notified().await;
|
||||
cancellation.cancel();
|
||||
}
|
||||
);
|
||||
let error = result.unwrap_err();
|
||||
assert_eq!(error.code, errors::Code::Cancelled);
|
||||
assert_eq!(error.effect, errors::EffectObservation::Indeterminate);
|
||||
assert!(dropped.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_connection_loss_is_indeterminate_and_never_retried() {
|
||||
let owner = Arc::new(TestOwner::default());
|
||||
let (transport, mut endpoint) = remote(owner.clone()).await;
|
||||
let host = OwnerHost::begin(transport, context(), consumer())
|
||||
.await
|
||||
.unwrap();
|
||||
let binding = host.open(host.context(), "api").await.unwrap();
|
||||
endpoint.task.abort();
|
||||
// Await listener teardown, not a timing-based sleep.
|
||||
let _ = (&mut endpoint.task).await;
|
||||
let error = host
|
||||
.send(host.context(), binding.as_ref(), request())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code, errors::Code::Indeterminate);
|
||||
assert_eq!(error.effect, errors::EffectObservation::Indeterminate);
|
||||
assert!(!error.message.contains("127.0.0.1"));
|
||||
assert_eq!(owner.requests.lock().unwrap().len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -10,11 +10,13 @@ pub mod authorization;
|
||||
mod drive;
|
||||
mod notification;
|
||||
mod plugin;
|
||||
pub mod plugin_host;
|
||||
mod plugin_types;
|
||||
pub use notification::*;
|
||||
pub use plugin::*;
|
||||
pub use plugin_host::*;
|
||||
#[cfg(feature = "typescript")]
|
||||
pub use plugin_types::plugin_api_typescript;
|
||||
pub use plugin_types::{plugin_api_typescript, plugin_host_typescript};
|
||||
mod ticket_browse;
|
||||
pub use ticket_browse::*;
|
||||
pub mod repository_openapi_typescript;
|
||||
@@ -210,6 +212,9 @@ macro_rules! impl_openapi_schema {
|
||||
}
|
||||
|
||||
impl_openapi_schema!(
|
||||
PluginHostRequest,
|
||||
PluginHostResponse,
|
||||
PluginHostError,
|
||||
PluginInstallRequest,
|
||||
PluginAddArtifactRequest,
|
||||
PluginRevokeRequest,
|
||||
@@ -577,6 +582,15 @@ pub struct WorkspaceWorkerSessionHistoryResponse {
|
||||
/// Response loss does not imply rollback.
|
||||
#[api(reqwest, axum, openapi)]
|
||||
pub trait ServerApi {
|
||||
/// Runtime Host owner boundary; human plugin-management authority is not accepted here.
|
||||
#[post("/api/w/{workspace_id}/plugin-host", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,413,429,503], normalize_body_errors = true)]
|
||||
async fn plugin_host(
|
||||
&self,
|
||||
#[extension] context: ServerRequestContext,
|
||||
#[path] workspace_id: String,
|
||||
#[body] request: PluginHostRequest,
|
||||
) -> Result<PluginHostResponse, PluginHostError>;
|
||||
|
||||
#[get("/health", status = 200, error_status = 400)]
|
||||
async fn health(&self) -> Result<HealthResponse, RepositoryApiError>;
|
||||
|
||||
@@ -14968,7 +14982,7 @@ mod openapi_artifact_tests {
|
||||
// Session observation retains its existing explicit non-OpenAPI boundary independently of
|
||||
// the Runtime/Worker-source signed operation inventory above.
|
||||
const OTHER_OPENAPI_EXCLUDED: &[&str] = &["worker_session", "worker_session_history"];
|
||||
const DOCUMENTED_SIGNED_INTERNAL: &[&str] = &["workspace_runtime_config"];
|
||||
const DOCUMENTED_SIGNED_INTERNAL: &[&str] = &["workspace_runtime_config", "plugin_host"];
|
||||
const BROWSER_ONLY: &[&str] = &["merge_request_selector_repair"];
|
||||
|
||||
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
//! Runtime-to-owner Host protocol. Authentication is transport-bound, never a DTO claim.
|
||||
use crate::PluginHttpMethod;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum PluginHostRequest {
|
||||
Begin {
|
||||
plan_id: String,
|
||||
installation_id: String,
|
||||
consumer: PluginConsumer,
|
||||
},
|
||||
/// Irreversible cancellation of this authenticated logical invocation.
|
||||
Cancel {
|
||||
invocation_id: String,
|
||||
},
|
||||
Authorize {
|
||||
invocation_id: String,
|
||||
operation: PluginHostOperation,
|
||||
},
|
||||
Open {
|
||||
invocation_id: String,
|
||||
slot: String,
|
||||
},
|
||||
Send {
|
||||
invocation_id: String,
|
||||
slot: String,
|
||||
request: PluginHttpRequest,
|
||||
},
|
||||
Log {
|
||||
invocation_id: String,
|
||||
entry: PluginLogEntry,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum PluginConsumer {
|
||||
ToolCall {
|
||||
tool_call_id: String,
|
||||
tool_name: String,
|
||||
},
|
||||
/// Reserved consumer identity; this does not enable service/event delivery.
|
||||
Consumer {
|
||||
consumer_id: String,
|
||||
event_id: String,
|
||||
export_name: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(
|
||||
tag = "kind",
|
||||
content = "data",
|
||||
rename_all = "snake_case",
|
||||
deny_unknown_fields
|
||||
)]
|
||||
pub enum PluginHostOperation {
|
||||
Admit,
|
||||
Publish,
|
||||
Open { slot: String },
|
||||
Send { slot: String },
|
||||
Log,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PluginHttpRequest {
|
||||
pub method: PluginHttpMethod,
|
||||
pub path: String,
|
||||
pub query: Vec<(String, String)>,
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub body: Vec<u8>,
|
||||
pub timeout_ms: u32,
|
||||
pub effect_key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PluginHttpResponse {
|
||||
pub status: u16,
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(
|
||||
tag = "kind",
|
||||
content = "data",
|
||||
rename_all = "snake_case",
|
||||
deny_unknown_fields
|
||||
)]
|
||||
pub enum PluginHostResponse {
|
||||
Invocation {
|
||||
invocation_id: String,
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_i64))]
|
||||
#[cfg_attr(feature = "typescript", ts(type = "number"))]
|
||||
expires_at_ms: i64,
|
||||
},
|
||||
Authorized,
|
||||
Opened,
|
||||
Response {
|
||||
response: PluginHttpResponse,
|
||||
},
|
||||
Logged,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PluginLogEntry {
|
||||
pub level: PluginLogLevel,
|
||||
pub event_code: String,
|
||||
pub message: String,
|
||||
pub fields: Vec<PluginLogField>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PluginLogLevel {
|
||||
Trace,
|
||||
Debug,
|
||||
Info,
|
||||
Warn,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PluginLogField {
|
||||
pub name: String,
|
||||
pub value: PluginLogScalar,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(
|
||||
tag = "kind",
|
||||
content = "value",
|
||||
rename_all = "snake_case",
|
||||
deny_unknown_fields
|
||||
)]
|
||||
pub enum PluginLogScalar {
|
||||
Text(String),
|
||||
Boolean(bool),
|
||||
Signed(
|
||||
#[serde(with = "decimal_integer")]
|
||||
#[schemars(with = "String")]
|
||||
#[cfg_attr(feature = "typescript", ts(type = "string"))]
|
||||
i64,
|
||||
),
|
||||
Unsigned(
|
||||
#[serde(with = "decimal_integer")]
|
||||
#[schemars(with = "String")]
|
||||
#[cfg_attr(feature = "typescript", ts(type = "string"))]
|
||||
u64,
|
||||
),
|
||||
Number(f64),
|
||||
}
|
||||
|
||||
// WIT scalars retain their full 64-bit range without JSON/TypeScript precision loss.
|
||||
mod decimal_integer {
|
||||
pub fn serialize<T: std::fmt::Display, S: serde::Serializer>(
|
||||
value: &T,
|
||||
serializer: S,
|
||||
) -> Result<S::Ok, S::Error> {
|
||||
serializer.collect_str(value)
|
||||
}
|
||||
pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
|
||||
where
|
||||
T: std::str::FromStr + std::fmt::Display,
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let text = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
let value: T = text
|
||||
.parse()
|
||||
.map_err(|_| serde::de::Error::custom("invalid decimal integer"))?;
|
||||
if value.to_string() != text {
|
||||
return Err(serde::de::Error::custom("non-canonical decimal integer"));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PluginHostErrorCode {
|
||||
InvalidInput,
|
||||
Denied,
|
||||
NotFound,
|
||||
Conflict,
|
||||
Unsupported,
|
||||
QuotaExceeded,
|
||||
DeadlineExceeded,
|
||||
Cancelled,
|
||||
Unavailable,
|
||||
Indeterminate,
|
||||
ReceiptExpired,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PluginEffectObservation {
|
||||
NotDispatched,
|
||||
ResponseObserved,
|
||||
Indeterminate,
|
||||
}
|
||||
|
||||
/// Deliberately excludes raw diagnostics, transport bodies and arbitrary messages.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PluginHostError {
|
||||
pub code: PluginHostErrorCode,
|
||||
pub effect: PluginEffectObservation,
|
||||
pub diagnostic_id: String,
|
||||
}
|
||||
|
||||
impl PluginHostError {
|
||||
pub fn message(&self) -> &'static str {
|
||||
match self.code {
|
||||
PluginHostErrorCode::InvalidInput => "invalid plugin Host input",
|
||||
PluginHostErrorCode::Denied => "plugin Host operation denied",
|
||||
PluginHostErrorCode::NotFound => "plugin Host target not found",
|
||||
PluginHostErrorCode::Conflict => "plugin Host operation conflict",
|
||||
PluginHostErrorCode::Unsupported => "plugin Host operation unsupported",
|
||||
PluginHostErrorCode::QuotaExceeded => "plugin Host quota exceeded",
|
||||
PluginHostErrorCode::DeadlineExceeded => "plugin Host deadline exceeded",
|
||||
PluginHostErrorCode::Cancelled => "plugin Host operation cancelled",
|
||||
PluginHostErrorCode::Unavailable => "plugin Host unavailable",
|
||||
PluginHostErrorCode::Indeterminate => "plugin Host outcome indeterminate",
|
||||
PluginHostErrorCode::ReceiptExpired => "plugin Host receipt expired",
|
||||
}
|
||||
}
|
||||
}
|
||||
impl std::fmt::Display for PluginHostError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.message())
|
||||
}
|
||||
}
|
||||
impl std::error::Error for PluginHostError {}
|
||||
impl api_macros::HttpError for PluginHostError {
|
||||
fn status_code(&self) -> u16 {
|
||||
match self.code {
|
||||
PluginHostErrorCode::InvalidInput | PluginHostErrorCode::Unsupported => 400,
|
||||
PluginHostErrorCode::Denied => 403,
|
||||
PluginHostErrorCode::NotFound => 404,
|
||||
PluginHostErrorCode::Conflict
|
||||
| PluginHostErrorCode::Indeterminate
|
||||
| PluginHostErrorCode::ReceiptExpired => 409,
|
||||
PluginHostErrorCode::QuotaExceeded => 429,
|
||||
PluginHostErrorCode::DeadlineExceeded
|
||||
| PluginHostErrorCode::Cancelled
|
||||
| PluginHostErrorCode::Unavailable => 503,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl api_macros::HttpRequestError for PluginHostError {
|
||||
fn from_request_rejection(status: u16, _message: String) -> Self {
|
||||
Self {
|
||||
code: if status == 413 {
|
||||
PluginHostErrorCode::QuotaExceeded
|
||||
} else {
|
||||
PluginHostErrorCode::InvalidInput
|
||||
},
|
||||
effect: PluginEffectObservation::NotDispatched,
|
||||
diagnostic_id: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn host_protocol_rejects_identity_claims_and_unknown_fields() {
|
||||
for value in [
|
||||
serde_json::json!({"kind":"begin","plan_id":"p","installation_id":"i","consumer":{"kind":"tool_call","tool_call_id":"t","tool_name":"n"},"subject":"owner"}),
|
||||
serde_json::json!({"kind":"begin","plan_id":"p","installation_id":"i","consumer":{"kind":"tool_call","tool_call_id":"t","tool_name":"n","worker_id":"w"}}),
|
||||
serde_json::json!({"kind":"authorize","invocation_id":"i","operation":{"kind":"admit","subject":"owner"}}),
|
||||
serde_json::json!({"kind":"send","invocation_id":"i","slot":"s","request":{"method":"GET","path":"/","query":[],"headers":[],"body":[],"timeout_ms":1,"effect_key":"e","url":"https://untrusted"}}),
|
||||
] {
|
||||
assert!(serde_json::from_value::<PluginHostRequest>(value).is_err());
|
||||
}
|
||||
assert!(
|
||||
serde_json::from_value::<PluginHostResponse>(
|
||||
serde_json::json!({"kind":"authorized","subject":"owner"})
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(serde_json::from_value::<PluginHostError>(serde_json::json!({"code":"denied","effect":"not_dispatched","diagnostic_id":"d","message":"secret"})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn log_integer_wire_preserves_full_wit_range_without_json_precision_loss() {
|
||||
for scalar in [
|
||||
PluginLogScalar::Signed(i64::MIN),
|
||||
PluginLogScalar::Signed(i64::MAX),
|
||||
PluginLogScalar::Unsigned(u64::MAX),
|
||||
] {
|
||||
let wire = serde_json::to_value(&scalar).unwrap();
|
||||
assert!(wire["value"].is_string());
|
||||
assert_eq!(
|
||||
serde_json::from_value::<PluginLogScalar>(wire).unwrap(),
|
||||
scalar
|
||||
);
|
||||
}
|
||||
for value in [
|
||||
serde_json::json!(1),
|
||||
serde_json::json!("01"),
|
||||
serde_json::json!("+1"),
|
||||
serde_json::json!("18446744073709551616"),
|
||||
] {
|
||||
assert!(
|
||||
serde_json::from_value::<PluginLogScalar>(
|
||||
serde_json::json!({"kind":"unsigned","value":value})
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_route_uses_runtime_context_and_closed_error_schema() {
|
||||
let document = crate::canonical_openapi_document().unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(&document.to_json().unwrap()).unwrap();
|
||||
let route = &value["paths"]["/api/w/{workspace_id}/plugin-host"]["post"];
|
||||
for status in [200, 400, 401, 403, 404, 409, 413, 429, 503] {
|
||||
assert!(
|
||||
route["responses"][status.to_string()].is_object(),
|
||||
"{status}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
route["security"].is_null(),
|
||||
"signed Runtime context, not human bearer/browser auth"
|
||||
);
|
||||
let params = route["parameters"].as_array().unwrap();
|
||||
assert_eq!(params.len(), 1);
|
||||
assert_eq!(params[0]["name"], "workspace_id");
|
||||
let error = &value["components"]["schemas"]["PluginHostError"];
|
||||
assert_eq!(error["additionalProperties"], false);
|
||||
assert_eq!(error["properties"].as_object().unwrap().len(), 3);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,28 @@
|
||||
//! Generated management types share the exact DTOs used by the HTTP declaration.
|
||||
//! Generated plugin types share the exact DTOs used by the HTTP declarations.
|
||||
#[cfg(feature = "typescript")]
|
||||
pub fn plugin_host_typescript() -> String {
|
||||
use crate::*;
|
||||
use ts_rs::TS;
|
||||
let config = ts_rs::Config::default();
|
||||
let declarations = [
|
||||
PluginHttpMethod::decl(&config),
|
||||
PluginConsumer::decl(&config),
|
||||
PluginHostRequest::decl(&config),
|
||||
PluginHostOperation::decl(&config),
|
||||
PluginHttpRequest::decl(&config),
|
||||
PluginHttpResponse::decl(&config),
|
||||
PluginHostResponse::decl(&config),
|
||||
PluginLogEntry::decl(&config),
|
||||
PluginLogLevel::decl(&config),
|
||||
PluginLogField::decl(&config),
|
||||
PluginLogScalar::decl(&config),
|
||||
PluginHostError::decl(&config),
|
||||
PluginHostErrorCode::decl(&config),
|
||||
PluginEffectObservation::decl(&config),
|
||||
];
|
||||
declarations.map(|d| format!("export {d}")).join("\n\n")
|
||||
}
|
||||
|
||||
#[cfg(feature = "typescript")]
|
||||
pub fn plugin_api_typescript() -> String {
|
||||
use crate::*;
|
||||
@@ -31,6 +55,25 @@ pub fn plugin_api_typescript() -> String {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[cfg(feature = "typescript")]
|
||||
#[test]
|
||||
fn host_typescript_preserves_wire_numbers_and_lossless_scalar_strings() {
|
||||
let text = super::plugin_host_typescript();
|
||||
for name in [
|
||||
"PluginHostRequest",
|
||||
"PluginHostResponse",
|
||||
"PluginHostError",
|
||||
"PluginConsumer",
|
||||
"PluginHttpRequest",
|
||||
"PluginHttpResponse",
|
||||
"PluginLogEntry",
|
||||
] {
|
||||
assert!(text.contains(&format!("export type {name} =")), "{name}");
|
||||
}
|
||||
assert!(text.contains("expires_at_ms: number"));
|
||||
assert!(text.contains("\"signed\", \"value\": string"));
|
||||
assert!(text.contains("\"unsigned\", \"value\": string"));
|
||||
}
|
||||
#[test]
|
||||
fn plugin_routes_use_binary_upload_and_closed_management_contracts() {
|
||||
let document = crate::canonical_openapi_document().unwrap();
|
||||
|
||||
@@ -30,6 +30,10 @@ job.workspace = true
|
||||
protocol = { workspace = true }
|
||||
runtime-api.workspace = true
|
||||
server-api.workspace = true
|
||||
secrets.workspace = true
|
||||
yoi-plugin-runtime = { path = "../plugin-runtime" }
|
||||
tokio-util = { version = "0.7", features = ["rt"] }
|
||||
flate2 = "1"
|
||||
project-record.workspace = true
|
||||
reqwest = { version = "0.13", default-features = false, features = ["blocking", "json", "native-tls"] }
|
||||
ring.workspace = true
|
||||
@@ -61,6 +65,8 @@ webauthn-rs = { workspace = true, features = ["conditional-ui"] }
|
||||
webauthn-rs-proto = "0.5.5"
|
||||
|
||||
[dev-dependencies]
|
||||
native-tls = "0.2"
|
||||
tokio-native-tls = "0.3"
|
||||
wip-protocol.workspace = true
|
||||
agen.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -516,7 +516,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
108,
|
||||
"Subject Ticket responsibility and current-body notification routing"
|
||||
.into()
|
||||
)
|
||||
),
|
||||
(109, "Durable Plugin Host invocations and HTTPS effect receipts".into())
|
||||
]
|
||||
);
|
||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||
|
||||
@@ -14,7 +14,10 @@ pub mod memory_staging;
|
||||
pub mod observation;
|
||||
pub mod plugin_binding;
|
||||
pub mod plugin_catalog;
|
||||
pub mod plugin_host;
|
||||
pub mod plugin_https;
|
||||
pub mod plugin_plan;
|
||||
pub mod plugin_receipt;
|
||||
pub mod plugin_settings;
|
||||
pub mod profile_settings;
|
||||
pub mod prompt_settings;
|
||||
@@ -167,6 +170,8 @@ pub enum Error {
|
||||
WorkspaceSigningIdentity { code: String, message: String },
|
||||
#[error("Plugin catalog error: {0}")]
|
||||
PluginCatalog(#[from] plugin_catalog::PluginCatalogError),
|
||||
#[error("Plugin Host operation rejected: {0}")]
|
||||
PluginHost(#[from] server_api::PluginHostError),
|
||||
#[error("Workspace Drive error: {0}")]
|
||||
Drive(#[from] workspace_drive::Error),
|
||||
#[error("DB execution capacity exceeded")]
|
||||
|
||||
@@ -30,7 +30,7 @@ fn plugin_catalog_migration_matches_fresh_schema_from_oldest_retained_and_105()
|
||||
// Plugin catalog remains migration 106 even when startup applies later steps.
|
||||
assert_eq!(after.iter().filter(|(version, _)| *version == 106).cloned().collect::<Vec<_>>(),
|
||||
vec![(106, crate::plugin_catalog::MIGRATION_NAME.into())]);
|
||||
assert_eq!(after.last().unwrap(), &(108, "Subject Ticket responsibility and current-body notification routing".into()));
|
||||
assert_eq!(after.last().unwrap(), &(109, "Durable Plugin Host invocations and HTTPS effect receipts".into()));
|
||||
assert_eq!(current_schema_version(c)?, LATEST_SCHEMA_VERSION);
|
||||
assert_eq!(c.query_row("SELECT display_name FROM workspaces WHERE workspace_id='plugin-workspace'",[],|r|r.get::<_,String>(0))?,"Plugins");
|
||||
assert_eq!(c.query_row("SELECT count(*) FROM pragma_foreign_key_check",[],|r|r.get::<_,i64>(0))?,0);
|
||||
@@ -72,6 +72,10 @@ fn plugin_catalog_migration_marker_failure_rolls_back_then_retries_without_chang
|
||||
108,
|
||||
"Subject Ticket responsibility and current-body notification routing".into(),
|
||||
));
|
||||
expected.push((
|
||||
109,
|
||||
"Durable Plugin Host invocations and HTTPS effect receipts".into(),
|
||||
));
|
||||
assert_eq!(workspace_schema_migration_history(c)?, expected);
|
||||
crate::plugin_catalog::verify(c)?;
|
||||
Ok(())
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
//! Server execution policy composes authenticated Host identity with retained plan
|
||||
//! ceilings. Catalog management, artifact possession and ordinary configuration
|
||||
//! selection never issue execution authority. Callers must run the existing live
|
||||
//! operation/delegation authorizer before this same-database boundary.
|
||||
|
||||
use crate::{Error, Result, plugin_binding::PluginConnectionBinding, plugin_plan::*};
|
||||
use rusqlite::{OptionalExtension, Transaction, params};
|
||||
use server_api::*;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub(crate) fn failure(code: PluginHostErrorCode) -> Error {
|
||||
PluginHostError {
|
||||
code,
|
||||
effect: PluginEffectObservation::NotDispatched,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
/// This value stays in the Server. It is never a wire connection capability.
|
||||
#[derive(Clone)]
|
||||
pub struct AdmittedPluginOperation {
|
||||
pub plan: StoredPluginPlan,
|
||||
pub pin: WorkerPluginPin,
|
||||
pub binding: Option<PluginConnectionBinding>,
|
||||
}
|
||||
|
||||
/// `execution` is derived by the authenticated Host boundary from current Worker
|
||||
/// registration, Subject policy and delegation lineage, never deserialized input.
|
||||
/// Reads deliberately use the current catalog operation, not the launch-time one:
|
||||
/// adding another artifact must not invalidate an existing immutable pin.
|
||||
pub fn authorize(
|
||||
tx: &Transaction<'_>,
|
||||
workspace: &str,
|
||||
execution: &HostExecutionIdentity,
|
||||
plan_id: &str,
|
||||
installation: &str,
|
||||
consumer: &PluginConsumer,
|
||||
operation: &PluginHostOperation,
|
||||
) -> Result<AdmittedPluginOperation> {
|
||||
let plan = plugin_plan_load(tx, workspace, plan_id)?;
|
||||
if plan.plan.execution() != execution || plan.pin_state == PlanPinState::Released {
|
||||
return Err(failure(PluginHostErrorCode::Denied));
|
||||
}
|
||||
let pin = plan
|
||||
.plan
|
||||
.plugins()
|
||||
.iter()
|
||||
.find(|pin| pin.installation_id() == installation)
|
||||
.cloned()
|
||||
.ok_or_else(|| failure(PluginHostErrorCode::Denied))?;
|
||||
let tool_name = match consumer {
|
||||
PluginConsumer::ToolCall {
|
||||
tool_call_id,
|
||||
tool_name,
|
||||
} if !tool_call_id.is_empty()
|
||||
&& tool_call_id.len() <= 512
|
||||
&& !tool_call_id.chars().any(char::is_control) =>
|
||||
{
|
||||
tool_name
|
||||
}
|
||||
// The scope is reserved for future authenticated consumers. This initial
|
||||
// Tool implementation cannot manufacture Service/event admission.
|
||||
_ => return Err(failure(PluginHostErrorCode::Unsupported)),
|
||||
};
|
||||
if !pin.projection().surfaces.contains(&PluginSurface::Tool)
|
||||
|| !pin.projection().tools.contains(tool_name)
|
||||
{
|
||||
return Err(failure(PluginHostErrorCode::Denied));
|
||||
}
|
||||
let expected_kind = match plan.pin_state {
|
||||
PlanPinState::Pending => "pending",
|
||||
PlanPinState::Active => "active",
|
||||
PlanPinState::Restorable => "restorable",
|
||||
PlanPinState::Released => unreachable!(),
|
||||
};
|
||||
let retained: bool = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM plugin_references WHERE workspace_id=?1 AND kind=?2 AND owner=?3 AND installation_id=?4 AND artifact_digest=?5)",
|
||||
params![workspace, expected_kind, plan_id, installation, pin.artifact_digest()], |r| r.get(0),
|
||||
)?;
|
||||
if !retained {
|
||||
return Err(failure(PluginHostErrorCode::Denied));
|
||||
}
|
||||
let current: Option<String> = tx.query_row(
|
||||
"SELECT last_operation FROM plugin_installations WHERE workspace_id=?1 AND installation_id=?2 AND uninstalled=0",
|
||||
params![workspace, installation], |r| r.get(0),
|
||||
).optional()?;
|
||||
let current = current.ok_or_else(|| failure(PluginHostErrorCode::Denied))?;
|
||||
let artifact = crate::plugin_catalog::plugin_artifact_admit(
|
||||
tx,
|
||||
workspace,
|
||||
installation,
|
||||
¤t,
|
||||
pin.package_digest(),
|
||||
pin.artifact_digest(),
|
||||
)?;
|
||||
if !pin.matches_artifact(&artifact)? {
|
||||
return Err(failure(PluginHostErrorCode::Denied));
|
||||
}
|
||||
let slot = match operation {
|
||||
PluginHostOperation::Admit | PluginHostOperation::Publish => {
|
||||
for reference in &pin.projection().bindings {
|
||||
live_binding(tx, workspace, reference)?;
|
||||
}
|
||||
None
|
||||
}
|
||||
PluginHostOperation::Log => {
|
||||
if !pin.projection().grants.contains(&PluginGrant::Logging) {
|
||||
return Err(failure(PluginHostErrorCode::Denied));
|
||||
}
|
||||
None
|
||||
}
|
||||
PluginHostOperation::Open { slot } | PluginHostOperation::Send { slot } => Some(slot),
|
||||
};
|
||||
let binding = if let Some(slot) = slot {
|
||||
if !pin
|
||||
.projection()
|
||||
.grants
|
||||
.contains(&PluginGrant::Https { slot: slot.clone() })
|
||||
{
|
||||
return Err(failure(PluginHostErrorCode::Denied));
|
||||
}
|
||||
let reference = pin
|
||||
.projection()
|
||||
.bindings
|
||||
.iter()
|
||||
.find(|r| &r.slot == slot)
|
||||
.ok_or_else(|| failure(PluginHostErrorCode::Denied))?;
|
||||
Some(live_binding(tx, workspace, reference)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(AdmittedPluginOperation { plan, pin, binding })
|
||||
}
|
||||
|
||||
fn live_binding(
|
||||
tx: &Transaction<'_>,
|
||||
workspace: &str,
|
||||
reference: &BindingPolicyReference,
|
||||
) -> Result<PluginConnectionBinding> {
|
||||
let state = crate::config_source::load_state(tx, workspace)?
|
||||
.ok_or_else(|| failure(PluginHostErrorCode::Unavailable))?;
|
||||
let evaluation = config_source::SnapshotEnvironment::new(state.snapshot)
|
||||
.evaluate_contract(&state.contract)
|
||||
.map_err(|_| failure(PluginHostErrorCode::Unavailable))?;
|
||||
let settings = crate::plugin_settings::settings_from_evaluation(&evaluation)?;
|
||||
let binding = settings
|
||||
.bindings
|
||||
.get(&reference.binding_id)
|
||||
.ok_or_else(|| failure(PluginHostErrorCode::Denied))?
|
||||
.clone();
|
||||
binding
|
||||
.validate(&crate::plugin_binding::PluginConnectionPolicyCeiling::default())
|
||||
.map_err(|_| failure(PluginHostErrorCode::Denied))?;
|
||||
if binding.digest() != reference.policy_digest {
|
||||
return Err(failure(PluginHostErrorCode::Denied));
|
||||
}
|
||||
Ok(binding)
|
||||
}
|
||||
|
||||
/// Only request identity is persisted, never request body, credential values or
|
||||
/// transient deadlines. Query/header ordering remains part of protocol identity.
|
||||
pub fn request_fingerprint(
|
||||
admitted: &AdmittedPluginOperation,
|
||||
slot: &str,
|
||||
request: &PluginHttpRequest,
|
||||
) -> Result<String> {
|
||||
let reference = admitted
|
||||
.pin
|
||||
.projection()
|
||||
.bindings
|
||||
.iter()
|
||||
.find(|r| r.slot == slot)
|
||||
.ok_or_else(|| failure(PluginHostErrorCode::Denied))?;
|
||||
let body_digest: String = Sha256::digest(&request.body)
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect();
|
||||
let content = serde_json::to_vec(&(
|
||||
"yoi-plugin-effect-v1",
|
||||
&admitted.plan.plan_id,
|
||||
admitted.pin.installation_id(),
|
||||
&reference.binding_id,
|
||||
slot,
|
||||
&reference.policy_digest,
|
||||
&request.method,
|
||||
&request.path,
|
||||
&request.query,
|
||||
&request.headers,
|
||||
body_digest,
|
||||
))
|
||||
.map_err(|_| failure(PluginHostErrorCode::InvalidInput))?;
|
||||
let hex: String = Sha256::digest(content)
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect();
|
||||
Ok(format!("sha256:{hex}"))
|
||||
}
|
||||
|
||||
pub(crate) fn verify_receipt_schema(conn: &rusqlite::Connection) -> Result<()> {
|
||||
let expected = rusqlite::Connection::open_in_memory()?;
|
||||
expected.execute_batch(crate::plugin_receipt::SCHEMA_SQL)?;
|
||||
let mut statement = expected.prepare("SELECT name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND tbl_name IN ('plugin_invocations','plugin_effect_receipts') ORDER BY name")?;
|
||||
for item in statement.query_map([], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||
})? {
|
||||
let (name, sql) = item?;
|
||||
let actual: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT sql FROM sqlite_schema WHERE name=?1",
|
||||
[&name],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
if actual.as_deref() != Some(&sql) {
|
||||
return Err(Error::Store("Plugin receipt schema has drifted".into()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn map_error(error: Error) -> PluginHostError {
|
||||
match error {
|
||||
Error::PluginHost(error) => error,
|
||||
Error::PluginCatalog(_) | Error::WorkspacePermissionDenied(_) => {
|
||||
if let Error::PluginHost(e) = failure(PluginHostErrorCode::Denied) {
|
||||
e
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
_ => PluginHostError {
|
||||
code: PluginHostErrorCode::Unavailable,
|
||||
effect: PluginEffectObservation::NotDispatched,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Retained migrations build the real predecessor; this is not a latest-schema
|
||||
// fixture with its version number relabelled. Existing 107/108 history is frozen.
|
||||
#[test]
|
||||
fn plugin_receipt_tail_migrates_true_predecessor_reopens_and_matches_fresh_schema() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("server.db");
|
||||
prepare_retained_schema(&path, 108);
|
||||
let conn = Connection::open(&path).unwrap();
|
||||
seed_migration_workspace(&conn);
|
||||
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||
assert_eq!(history.last().unwrap().0, 108);
|
||||
assert!(!table_exists(&conn, "plugin_invocations").unwrap());
|
||||
let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap();
|
||||
assert_eq!(plan.current_schema_version, 108);
|
||||
assert_eq!(plan.migrations.iter().map(|step| step.version).collect::<Vec<_>>(), [109]);
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 108);
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
store.with_conn(|conn| {
|
||||
crate::plugin_host::verify_receipt_schema(conn)?;
|
||||
assert_eq!(current_schema_version(conn)?, 109);
|
||||
let upgraded = workspace_schema_migration_history(conn)?;
|
||||
assert_eq!(&upgraded[..history.len()], history.as_slice());
|
||||
assert_eq!(upgraded.last().unwrap().1, "Durable Plugin Host invocations and HTTPS effect receipts");
|
||||
assert_eq!(conn.query_row("SELECT count(*) FROM workspaces WHERE workspace_id='space'", [], |row| row.get::<_, i64>(0))?, 1);
|
||||
assert_eq!(conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| row.get::<_, i64>(0))?, 0);
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
drop(store);
|
||||
for _ in 0..2 {
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
reopened.with_conn(|conn| {
|
||||
crate::plugin_host::verify_receipt_schema(conn)?;
|
||||
assert_eq!(workspace_schema_migration_history(conn)?.iter().filter(|(v, _)| *v == 109).count(), 1);
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||
fresh.with_conn(crate::plugin_host::verify_receipt_schema).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plugin_receipt_tail_rolls_back_partial_ddl_and_retries_without_history_rewrite() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("server.db");
|
||||
prepare_retained_schema(&path, 108);
|
||||
let conn = Connection::open(&path).unwrap();
|
||||
configure_sqlite(&conn).unwrap();
|
||||
// A late-name collision fails after the first CREATEs. The owner transaction
|
||||
// must roll back all new objects and the migration receipt together.
|
||||
conn.execute_batch("CREATE TABLE plugin_effect_receipts(unrelated INTEGER)").unwrap();
|
||||
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||
let before: String = conn.query_row("SELECT group_concat(sql,char(10)) FROM (SELECT sql FROM sqlite_schema WHERE sql IS NOT NULL ORDER BY name)", [], |r| r.get(0)).unwrap();
|
||||
assert!(migrate_plugin_receipts_v108_to_v109(&conn).is_err());
|
||||
let after: String = conn.query_row("SELECT group_concat(sql,char(10)) FROM (SELECT sql FROM sqlite_schema WHERE sql IS NOT NULL ORDER BY name)", [], |r| r.get(0)).unwrap();
|
||||
assert_eq!(before, after);
|
||||
assert_eq!(workspace_schema_migration_history(&conn).unwrap(), history);
|
||||
assert!(!table_exists(&conn, "plugin_invocations").unwrap());
|
||||
conn.execute_batch("DROP TABLE plugin_effect_receipts").unwrap();
|
||||
migrate_plugin_receipts_v108_to_v109(&conn).unwrap();
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 109);
|
||||
assert!(migrate_plugin_receipts_v108_to_v109(&conn).is_err());
|
||||
crate::plugin_host::verify_receipt_schema(&conn).unwrap();
|
||||
conn.execute("UPDATE __yoi_schema_migrations SET name='wrong predecessor' WHERE version=108", []).unwrap();
|
||||
assert!(verify_schema_history(&conn, 109).is_err());
|
||||
}
|
||||
@@ -0,0 +1,534 @@
|
||||
//! Bounded, single-attempt HTTPS for an already admitted Plugin operation.
|
||||
//!
|
||||
//! The caller owns live authorization, secret lookup and durable effect admission.
|
||||
//! In particular, an execution error is not evidence that an external effect did
|
||||
//! not happen, and must never cause an automatic replay. No raw network error is
|
||||
//! retained here. Debug output also omits request/response data and credentials.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::io::Read;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use server_api::PluginHttpMethod;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use url::Url;
|
||||
|
||||
use crate::plugin_binding::{PluginConnectionBinding, PluginConnectionPolicyCeiling, path_within};
|
||||
|
||||
mod policy;
|
||||
use policy::{PinnedResolver, check_addresses, safe_header};
|
||||
|
||||
const MAX_HEADERS: usize = 64;
|
||||
const MAX_URL_BYTES: usize = 16 * 1024;
|
||||
const MAX_QUERY_PAIRS: usize = 256;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HttpsRequest {
|
||||
pub method: PluginHttpMethod,
|
||||
pub path: String,
|
||||
pub query: Vec<(String, String)>,
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub body: Vec<u8>,
|
||||
pub timeout_ms: u32,
|
||||
pub effect_key: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HttpsResponse {
|
||||
pub status: u16,
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for HttpsRequest {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("HttpsRequest").finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for HttpsResponse {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("HttpsResponse")
|
||||
.field("status", &self.status)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Closed, safe categories: no reqwest, URL, DNS, TLS or header parsing source
|
||||
/// error is exposed. These categories do not authorize retry after execution.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum TransportError {
|
||||
#[error("invalid HTTPS binding")]
|
||||
InvalidBinding,
|
||||
#[error("invalid HTTPS request")]
|
||||
InvalidRequest,
|
||||
#[error("HTTPS request denied")]
|
||||
Denied,
|
||||
#[error("HTTPS header limit exceeded")]
|
||||
HeaderLimit,
|
||||
#[error("HTTPS request limit exceeded")]
|
||||
RequestLimit,
|
||||
#[error("HTTPS response limit exceeded")]
|
||||
ResponseLimit,
|
||||
#[error("HTTPS name resolution failed")]
|
||||
Dns,
|
||||
#[error("HTTPS transport unavailable")]
|
||||
Unavailable,
|
||||
#[error("HTTPS deadline exceeded")]
|
||||
DeadlineExceeded,
|
||||
#[error("HTTPS request cancelled")]
|
||||
Cancelled,
|
||||
#[error("HTTPS credential unavailable or invalid")]
|
||||
InvalidSecret,
|
||||
#[error("unsupported HTTPS response encoding")]
|
||||
UnsupportedEncoding,
|
||||
}
|
||||
|
||||
/// Private fields prevent changing the checked request or swapping its binding.
|
||||
/// The caller can hash/serialize `normalized()` for its durable fingerprint;
|
||||
/// secret values are never part of this representation. Receipt fingerprinting
|
||||
/// must exclude timeout and replace the body with its hash as required by API.md.
|
||||
#[derive(Clone)]
|
||||
pub struct PreparedRequest {
|
||||
request: HttpsRequest,
|
||||
binding_digest: String,
|
||||
url: Url,
|
||||
}
|
||||
|
||||
impl PreparedRequest {
|
||||
pub fn normalized(&self) -> &HttpsRequest {
|
||||
&self.request
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for PreparedRequest {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("PreparedRequest").finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Additional operator-owned exclusions, not Workspace-granted exceptions.
|
||||
/// Operators must include all public control-plane IPs/names here. IP exclusions
|
||||
/// apply to every port, including literal-IP origins and every DNS candidate.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TransportPolicy {
|
||||
pub denied_addresses: BTreeSet<IpAddr>,
|
||||
pub denied_hosts: BTreeSet<String>,
|
||||
}
|
||||
|
||||
pub struct HttpsTransport {
|
||||
policy: TransportPolicy,
|
||||
#[cfg(test)]
|
||||
test_overrides: Option<test_support::TestOverrides>,
|
||||
}
|
||||
|
||||
pub fn prepare(
|
||||
binding: &PluginConnectionBinding,
|
||||
mut request: HttpsRequest,
|
||||
) -> Result<PreparedRequest, TransportError> {
|
||||
binding
|
||||
.validate(&PluginConnectionPolicyCeiling::default())
|
||||
.map_err(|_| TransportError::InvalidBinding)?;
|
||||
if !binding.methods.contains(&request.method)
|
||||
|| !path_within("/", &request.path)
|
||||
|| !binding.allows_path(&request.path)
|
||||
{
|
||||
return Err(TransportError::Denied);
|
||||
}
|
||||
if request.timeout_ms == 0
|
||||
|| request.effect_key.is_empty()
|
||||
|| request.effect_key.len() > 128
|
||||
|| request.effect_key.chars().any(char::is_control)
|
||||
{
|
||||
return Err(TransportError::InvalidRequest);
|
||||
}
|
||||
if request.body.len() > binding.limits.max_request_bytes as usize
|
||||
|| request.query.len() > MAX_QUERY_PAIRS
|
||||
{
|
||||
return Err(TransportError::RequestLimit);
|
||||
}
|
||||
request.timeout_ms = request.timeout_ms.min(binding.limits.timeout_ms);
|
||||
if request.headers.len() > MAX_HEADERS {
|
||||
return Err(TransportError::HeaderLimit);
|
||||
}
|
||||
let mut header_bytes = 0usize;
|
||||
for (name, value) in &mut request.headers {
|
||||
let parsed =
|
||||
HeaderName::from_bytes(name.as_bytes()).map_err(|_| TransportError::InvalidRequest)?;
|
||||
*name = parsed.as_str().to_owned();
|
||||
if !safe_header(name) || !binding.request_headers.contains(name) {
|
||||
return Err(TransportError::Denied);
|
||||
}
|
||||
// Exclude folding, control bytes and non-ASCII ambiguities even if the
|
||||
// HTTP parser would accept them. Header values are never trimmed.
|
||||
if !value.bytes().all(|b| (0x20..=0x7e).contains(&b)) {
|
||||
return Err(TransportError::InvalidRequest);
|
||||
}
|
||||
HeaderValue::from_str(value).map_err(|_| TransportError::InvalidRequest)?;
|
||||
header_bytes = header_bytes.saturating_add(name.len() + value.len() + 4);
|
||||
}
|
||||
if header_bytes > binding.limits.max_header_bytes as usize {
|
||||
return Err(TransportError::HeaderLimit);
|
||||
}
|
||||
// Bound raw query input before percent encoding (whose expansion is at most
|
||||
// threefold), then bound the actual request target after encoding as well.
|
||||
let query_bytes = request
|
||||
.query
|
||||
.iter()
|
||||
.try_fold(0usize, |sum, (k, v)| {
|
||||
sum.checked_add(k.len())?
|
||||
.checked_add(v.len())?
|
||||
.checked_add(2)
|
||||
})
|
||||
.ok_or(TransportError::RequestLimit)?;
|
||||
if query_bytes > MAX_URL_BYTES {
|
||||
return Err(TransportError::RequestLimit);
|
||||
}
|
||||
let mut url = Url::parse(&format!("{}{}", binding.origin, request.path))
|
||||
.map_err(|_| TransportError::InvalidRequest)?;
|
||||
if url.path() != request.path {
|
||||
return Err(TransportError::Denied);
|
||||
}
|
||||
if !request.query.is_empty() {
|
||||
url.query_pairs_mut()
|
||||
.extend_pairs(request.query.iter().map(|(k, v)| (k, v)));
|
||||
}
|
||||
if url.as_str().len() > MAX_URL_BYTES {
|
||||
return Err(TransportError::RequestLimit);
|
||||
}
|
||||
Ok(PreparedRequest {
|
||||
request,
|
||||
binding_digest: binding.digest(),
|
||||
url,
|
||||
})
|
||||
}
|
||||
|
||||
impl HttpsTransport {
|
||||
pub fn new(policy: TransportPolicy) -> Self {
|
||||
Self {
|
||||
policy,
|
||||
#[cfg(test)]
|
||||
test_overrides: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `secret` is the complete Authorization field value (including its scheme),
|
||||
/// not a secret reference. Its presence must agree with binding.secret_ref.
|
||||
/// A fresh client and a pinned resolver are used for every single attempt;
|
||||
/// neither a cached connection nor a second OS resolution can bypass checks.
|
||||
/// The caller must cancel this token at the enclosing invocation deadline.
|
||||
pub async fn execute(
|
||||
&self,
|
||||
binding: &PluginConnectionBinding,
|
||||
prepared: PreparedRequest,
|
||||
secret: Option<&str>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<HttpsResponse, TransportError> {
|
||||
self.execute_with_guard(binding, prepared, secret, cancellation, || async { Ok(()) })
|
||||
.await
|
||||
}
|
||||
|
||||
/// Run the caller's fresh authority check only after DNS, address admission,
|
||||
/// credential/header checks and complete client/request construction. Denial
|
||||
/// is returned unchanged and no connection is opened. The guard shares the
|
||||
/// transport deadline and cancellation; there is no await between a successful
|
||||
/// guard and dispatch. It cannot make authorization atomic with remote I/O.
|
||||
pub async fn execute_with_guard<G, F>(
|
||||
&self,
|
||||
binding: &PluginConnectionBinding,
|
||||
prepared: PreparedRequest,
|
||||
secret: Option<&str>,
|
||||
cancellation: &CancellationToken,
|
||||
guard: G,
|
||||
) -> Result<HttpsResponse, TransportError>
|
||||
where
|
||||
G: FnOnce() -> F + Send,
|
||||
F: std::future::Future<Output = Result<(), TransportError>> + Send,
|
||||
{
|
||||
let deadline =
|
||||
tokio::time::Instant::now() + Duration::from_millis(prepared.request.timeout_ms.into());
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => Err(TransportError::Cancelled),
|
||||
result = tokio::time::timeout_at(deadline, self.execute_inner(binding, prepared, secret, cancellation, deadline, guard)) => {
|
||||
result.unwrap_or(Err(TransportError::DeadlineExceeded))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_inner<G, F>(
|
||||
&self,
|
||||
binding: &PluginConnectionBinding,
|
||||
prepared: PreparedRequest,
|
||||
secret: Option<&str>,
|
||||
cancellation: &CancellationToken,
|
||||
deadline: tokio::time::Instant,
|
||||
guard: G,
|
||||
) -> Result<HttpsResponse, TransportError>
|
||||
where
|
||||
G: FnOnce() -> F + Send,
|
||||
F: std::future::Future<Output = Result<(), TransportError>> + Send,
|
||||
{
|
||||
// Recheck against the supplied current binding, not only the preparation
|
||||
// snapshot. A stale prepared policy cannot silently gain new authority.
|
||||
if prepared.binding_digest != binding.digest() {
|
||||
return Err(TransportError::Denied);
|
||||
}
|
||||
let prepared = prepare(binding, prepared.request)?;
|
||||
let host = prepared
|
||||
.url
|
||||
.host_str()
|
||||
.ok_or(TransportError::InvalidBinding)?;
|
||||
let host = host.trim_start_matches('[').trim_end_matches(']');
|
||||
if self
|
||||
.policy
|
||||
.denied_hosts
|
||||
.iter()
|
||||
.any(|name| name.eq_ignore_ascii_case(host))
|
||||
{
|
||||
return Err(TransportError::Denied);
|
||||
}
|
||||
let port = prepared
|
||||
.url
|
||||
.port_or_known_default()
|
||||
.ok_or(TransportError::InvalidBinding)?;
|
||||
let addresses = self.resolve(host, port).await?;
|
||||
check_addresses(&addresses, &self.policy)?;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in &prepared.request.headers {
|
||||
headers.append(
|
||||
HeaderName::from_bytes(name.as_bytes())
|
||||
.map_err(|_| TransportError::InvalidRequest)?,
|
||||
HeaderValue::from_str(value).map_err(|_| TransportError::InvalidRequest)?,
|
||||
);
|
||||
}
|
||||
match (binding.secret_ref.is_empty(), secret) {
|
||||
(true, None) => {}
|
||||
(false, Some(value)) if !value.is_empty() => {
|
||||
if value.len() > binding.limits.max_header_bytes as usize {
|
||||
return Err(TransportError::HeaderLimit);
|
||||
}
|
||||
if !value.bytes().all(|b| (0x20..=0x7e).contains(&b)) {
|
||||
return Err(TransportError::InvalidSecret);
|
||||
}
|
||||
let mut value =
|
||||
HeaderValue::from_str(value).map_err(|_| TransportError::InvalidSecret)?;
|
||||
value.set_sensitive(true);
|
||||
headers.insert(reqwest::header::AUTHORIZATION, value);
|
||||
}
|
||||
_ => return Err(TransportError::InvalidSecret),
|
||||
}
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT_ENCODING,
|
||||
HeaderValue::from_static("gzip"),
|
||||
);
|
||||
headers.insert(
|
||||
reqwest::header::CONNECTION,
|
||||
HeaderValue::from_static("close"),
|
||||
);
|
||||
headers
|
||||
.entry(reqwest::header::ACCEPT)
|
||||
.or_insert(HeaderValue::from_static("*/*"));
|
||||
// Include generated fields and the credential in the wire header budget.
|
||||
let authority = prepared.url[url::Position::BeforeHost..url::Position::AfterPort].len();
|
||||
check_header_budget(&headers, authority + "host: \r\n".len() + 32, binding)?;
|
||||
|
||||
#[cfg(test)]
|
||||
let addresses = self
|
||||
.test_overrides
|
||||
.as_ref()
|
||||
.and_then(|o| o.connect_to)
|
||||
.map(|address| vec![address])
|
||||
.unwrap_or(addresses);
|
||||
let resolver = PinnedResolver::new(host.to_owned(), addresses);
|
||||
let builder = reqwest::Client::builder()
|
||||
.https_only(true)
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.retry(reqwest::retry::never())
|
||||
.http1_only()
|
||||
.no_gzip()
|
||||
.no_brotli()
|
||||
.no_deflate()
|
||||
.no_zstd()
|
||||
.tls_backend_native()
|
||||
.tls_version_min(reqwest::tls::Version::TLS_1_2)
|
||||
.tls_sni(true)
|
||||
.dns_resolver(resolver)
|
||||
.pool_max_idle_per_host(0);
|
||||
#[cfg(test)]
|
||||
let builder = match self.test_overrides.as_ref().and_then(|o| o.root.clone()) {
|
||||
Some(root) => builder.tls_certs_only([root]),
|
||||
None => builder,
|
||||
};
|
||||
let client = builder.build().map_err(|_| TransportError::Unavailable)?;
|
||||
let request = client
|
||||
.request(method(&prepared.request.method), prepared.url)
|
||||
.headers(headers)
|
||||
.body(prepared.request.body)
|
||||
.build()
|
||||
.map_err(|_| TransportError::InvalidRequest)?;
|
||||
guard().await?;
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(TransportError::Cancelled);
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(TransportError::DeadlineExceeded);
|
||||
}
|
||||
let mut response = client.execute(request).await.map_err(network_error)?;
|
||||
check_header_budget(response.headers(), 0, binding)?;
|
||||
let status = response.status().as_u16();
|
||||
let encoding = response_encoding(response.headers())?;
|
||||
let limit = binding.limits.max_response_bytes as usize;
|
||||
if response.content_length().is_some_and(|n| n > limit as u64) {
|
||||
return Err(TransportError::ResponseLimit);
|
||||
}
|
||||
let connection_headers = connection_headers(response.headers())?;
|
||||
let mut returned_headers = Vec::new();
|
||||
for (name, value) in response.headers() {
|
||||
if safe_header(name.as_str())
|
||||
&& !connection_headers.contains(name.as_str())
|
||||
&& binding.response_headers.iter().any(|h| h == name.as_str())
|
||||
&& !(encoding == Encoding::Gzip && name == reqwest::header::CONTENT_ENCODING)
|
||||
{
|
||||
let value = value.to_str().map_err(|_| TransportError::Unavailable)?;
|
||||
returned_headers.push((name.as_str().to_owned(), value.to_owned()));
|
||||
}
|
||||
}
|
||||
let mut body = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await.map_err(network_error)? {
|
||||
if chunk.len() > limit.saturating_sub(body.len()) {
|
||||
return Err(TransportError::ResponseLimit);
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
if encoding == Encoding::Gzip {
|
||||
body = decode_gzip(&body, limit).await?;
|
||||
}
|
||||
Ok(HttpsResponse {
|
||||
status,
|
||||
headers: returned_headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
async fn resolve(&self, host: &str, port: u16) -> Result<Vec<SocketAddr>, TransportError> {
|
||||
#[cfg(test)]
|
||||
if let Some(overrides) = &self.test_overrides {
|
||||
return overrides.resolve().await;
|
||||
}
|
||||
if let Ok(ip) = host.parse::<IpAddr>() {
|
||||
return Ok(vec![SocketAddr::new(ip, port)]);
|
||||
}
|
||||
tokio::net::lookup_host((host, port))
|
||||
.await
|
||||
.map(|addresses| addresses.collect())
|
||||
.map_err(|_| TransportError::Dns)
|
||||
}
|
||||
}
|
||||
|
||||
fn check_header_budget(
|
||||
headers: &HeaderMap,
|
||||
generated_bytes: usize,
|
||||
binding: &PluginConnectionBinding,
|
||||
) -> Result<(), TransportError> {
|
||||
let bytes = headers.iter().fold(generated_bytes, |sum, (name, value)| {
|
||||
sum.saturating_add(name.as_str().len() + value.as_bytes().len() + 4)
|
||||
});
|
||||
// Generated Host and Content-Length consume two additional fields.
|
||||
let generated_count = if generated_bytes == 0 { 0 } else { 2 };
|
||||
if headers.len() + generated_count > MAX_HEADERS
|
||||
|| bytes > binding.limits.max_header_bytes as usize
|
||||
{
|
||||
return Err(TransportError::HeaderLimit);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn connection_headers(headers: &HeaderMap) -> Result<BTreeSet<String>, TransportError> {
|
||||
let mut nominated = BTreeSet::new();
|
||||
for value in headers.get_all(reqwest::header::CONNECTION) {
|
||||
let value = value.to_str().map_err(|_| TransportError::Unavailable)?;
|
||||
for name in value.split(',') {
|
||||
let name = HeaderName::from_bytes(name.trim().as_bytes())
|
||||
.map_err(|_| TransportError::Unavailable)?;
|
||||
nominated.insert(name.as_str().to_owned());
|
||||
}
|
||||
}
|
||||
Ok(nominated)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Encoding {
|
||||
Identity,
|
||||
Gzip,
|
||||
}
|
||||
|
||||
fn response_encoding(headers: &HeaderMap) -> Result<Encoding, TransportError> {
|
||||
let mut values = headers.get_all(reqwest::header::CONTENT_ENCODING).iter();
|
||||
let first = values.next();
|
||||
if values.next().is_some() {
|
||||
return Err(TransportError::UnsupportedEncoding);
|
||||
}
|
||||
match first {
|
||||
None => Ok(Encoding::Identity),
|
||||
Some(value) if value.as_bytes().eq_ignore_ascii_case(b"identity") => Ok(Encoding::Identity),
|
||||
Some(value) if value.as_bytes().eq_ignore_ascii_case(b"gzip") => Ok(Encoding::Gzip),
|
||||
_ => Err(TransportError::UnsupportedEncoding),
|
||||
}
|
||||
}
|
||||
|
||||
async fn decode_gzip(bytes: &[u8], limit: usize) -> Result<Vec<u8>, TransportError> {
|
||||
let mut decoder = flate2::read::MultiGzDecoder::new(bytes);
|
||||
let mut output = Vec::new();
|
||||
let mut chunk = [0u8; 8192];
|
||||
loop {
|
||||
let length = decoder
|
||||
.read(&mut chunk)
|
||||
.map_err(|_| TransportError::Unavailable)?;
|
||||
if length == 0 {
|
||||
break;
|
||||
}
|
||||
if length > limit.saturating_sub(output.len()) {
|
||||
return Err(TransportError::ResponseLimit);
|
||||
}
|
||||
output.extend_from_slice(&chunk[..length]);
|
||||
// Keep cancellation and the enclosing wall deadline observable even for
|
||||
// concatenated compressed members and expansion-heavy payloads.
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn network_error(error: reqwest::Error) -> TransportError {
|
||||
if error.is_timeout() {
|
||||
TransportError::DeadlineExceeded
|
||||
} else {
|
||||
TransportError::Unavailable
|
||||
}
|
||||
}
|
||||
|
||||
fn method(method: &PluginHttpMethod) -> reqwest::Method {
|
||||
match method {
|
||||
PluginHttpMethod::Get => reqwest::Method::GET,
|
||||
PluginHttpMethod::Head => reqwest::Method::HEAD,
|
||||
PluginHttpMethod::Post => reqwest::Method::POST,
|
||||
PluginHttpMethod::Put => reqwest::Method::PUT,
|
||||
PluginHttpMethod::Patch => reqwest::Method::PATCH,
|
||||
PluginHttpMethod::Delete => reqwest::Method::DELETE,
|
||||
PluginHttpMethod::Options => reqwest::Method::OPTIONS,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,110 @@
|
||||
use std::io;
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
|
||||
|
||||
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
|
||||
|
||||
use super::{TransportError, TransportPolicy};
|
||||
|
||||
pub(super) fn safe_header(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.len() <= 128
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
|
||||
&& !name.starts_with("proxy-")
|
||||
&& !name.starts_with("sec-")
|
||||
&& !name.starts_with("x-forwarded-")
|
||||
&& !matches!(
|
||||
name,
|
||||
"authorization"
|
||||
| "www-authenticate"
|
||||
| "authentication-info"
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "set-cookie"
|
||||
| "set-cookie2"
|
||||
| "host"
|
||||
| "connection"
|
||||
| "keep-alive"
|
||||
| "transfer-encoding"
|
||||
| "te"
|
||||
| "trailer"
|
||||
| "upgrade"
|
||||
| "content-length"
|
||||
| "expect"
|
||||
| "forwarded"
|
||||
| "via"
|
||||
| "x-api-key"
|
||||
| "api-key"
|
||||
| "accept-encoding"
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn check_addresses(
|
||||
addresses: &[SocketAddr],
|
||||
policy: &TransportPolicy,
|
||||
) -> Result<(), TransportError> {
|
||||
if addresses.is_empty() || addresses.len() > 64 {
|
||||
return Err(TransportError::Dns);
|
||||
}
|
||||
// Reject the complete answer if even one candidate is disallowed. Filtering
|
||||
// the answer would hide mixed public/private DNS or rebinding attempts.
|
||||
for address in addresses {
|
||||
let public = match address.ip() {
|
||||
IpAddr::V4(ip) => public_ipv4(ip),
|
||||
IpAddr::V6(ip) => public_ipv6(ip),
|
||||
};
|
||||
if !public || policy.denied_addresses.contains(&address.ip()) {
|
||||
return Err(TransportError::Denied);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn public_ipv4(ip: Ipv4Addr) -> bool {
|
||||
let [a, b, c, _] = ip.octets();
|
||||
ip != Ipv4Addr::new(168, 63, 129, 16)
|
||||
&& !matches!(a, 0 | 10 | 127 | 224..=255)
|
||||
&& !(a == 100 && (64..=127).contains(&b))
|
||||
&& !(a == 169 && b == 254)
|
||||
&& !(a == 172 && (16..=31).contains(&b))
|
||||
&& !(a == 192 && (b == 0 || b == 168 || (b == 88 && c == 99)))
|
||||
&& !(a == 198 && (b == 18 || b == 19 || (b == 51 && c == 100)))
|
||||
&& !(a == 203 && b == 0 && c == 113)
|
||||
}
|
||||
|
||||
fn public_ipv6(ip: Ipv6Addr) -> bool {
|
||||
let [a, b, ..] = ip.segments();
|
||||
(0x2000..=0x3fff).contains(&a)
|
||||
&& !(a == 0x2001 && (b <= 0x01ff || b == 0x0db8))
|
||||
&& a != 0x2002
|
||||
&& !(a == 0x3fff && b <= 0x0fff)
|
||||
}
|
||||
|
||||
/// The HTTP connector cannot fall back to the OS resolver. TLS still receives
|
||||
/// the original URL hostname for certificate verification and SNI.
|
||||
pub(super) struct PinnedResolver {
|
||||
host: String,
|
||||
addresses: Vec<SocketAddr>,
|
||||
}
|
||||
|
||||
impl PinnedResolver {
|
||||
pub(super) fn new(host: String, addresses: Vec<SocketAddr>) -> Self {
|
||||
Self { host, addresses }
|
||||
}
|
||||
}
|
||||
|
||||
impl Resolve for PinnedResolver {
|
||||
fn resolve(&self, name: Name) -> Resolving {
|
||||
let matches = name.as_str() == self.host;
|
||||
let addresses = self.addresses.clone();
|
||||
Box::pin(async move {
|
||||
if !matches {
|
||||
return Err(
|
||||
io::Error::new(io::ErrorKind::PermissionDenied, "destination denied").into(),
|
||||
);
|
||||
}
|
||||
Ok(Box::new(addresses.into_iter()) as Addrs)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
//! Isolated TLS peer for crate-local integration tests. Production builds do
|
||||
//! not contain its private DNS/socket/trust overrides. The certificate hostname
|
||||
//! still passes through the real HTTPS client's name verification and SNI.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::future::pending;
|
||||
use std::io::Write;
|
||||
use std::process::Command;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::{JoinHandle, JoinSet};
|
||||
|
||||
use super::*;
|
||||
use crate::plugin_binding::{PluginConnectionLimits, PluginNetworkPolicy};
|
||||
|
||||
pub(crate) const HOST: &str = "api.example.com";
|
||||
|
||||
pub(super) struct TestOverrides {
|
||||
pub(super) connect_to: Option<SocketAddr>,
|
||||
pub(super) root: Option<reqwest::Certificate>,
|
||||
pub(super) answers: Mutex<VecDeque<DnsAnswer>>,
|
||||
pub(super) calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
pub(super) enum DnsAnswer {
|
||||
Addresses(Vec<SocketAddr>),
|
||||
Pending(Option<tokio::sync::oneshot::Sender<()>>),
|
||||
Barrier {
|
||||
started: tokio::sync::oneshot::Sender<()>,
|
||||
release: tokio::sync::oneshot::Receiver<Vec<SocketAddr>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl TestOverrides {
|
||||
pub(super) async fn resolve(&self) -> Result<Vec<SocketAddr>, TransportError> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let answer = self.answers.lock().unwrap().pop_front();
|
||||
match answer {
|
||||
Some(DnsAnswer::Addresses(addresses)) => Ok(addresses),
|
||||
Some(DnsAnswer::Barrier { started, release }) => {
|
||||
let _ = started.send(());
|
||||
release.await.map_err(|_| TransportError::Dns)
|
||||
}
|
||||
Some(DnsAnswer::Pending(started)) => {
|
||||
if let Some(started) = started {
|
||||
let _ = started.send(());
|
||||
}
|
||||
pending().await
|
||||
}
|
||||
None => Err(TransportError::Dns),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn policy() -> TransportPolicy {
|
||||
TransportPolicy {
|
||||
denied_addresses: BTreeSet::new(),
|
||||
denied_hosts: BTreeSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn binding() -> PluginConnectionBinding {
|
||||
PluginConnectionBinding {
|
||||
origin: format!("https://{HOST}"),
|
||||
network: PluginNetworkPolicy::PublicInternet,
|
||||
methods: vec![PluginHttpMethod::Get, PluginHttpMethod::Post],
|
||||
path_prefixes: vec!["/api".into()],
|
||||
request_headers: vec!["accept".into(), "x-order".into()],
|
||||
response_headers: vec![
|
||||
"content-type".into(),
|
||||
"x-order".into(),
|
||||
"location".into(),
|
||||
"content-encoding".into(),
|
||||
],
|
||||
secret_ref: String::new(),
|
||||
limits: PluginConnectionLimits {
|
||||
timeout_ms: 5_000,
|
||||
max_request_bytes: 1024,
|
||||
max_response_bytes: 1024,
|
||||
max_header_bytes: 4096,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn request() -> HttpsRequest {
|
||||
HttpsRequest {
|
||||
method: PluginHttpMethod::Get,
|
||||
path: "/api/items".into(),
|
||||
query: Vec::new(),
|
||||
headers: Vec::new(),
|
||||
body: Vec::new(),
|
||||
timeout_ms: 5_000,
|
||||
effect_key: "effect-1".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn addr(ip: &str, port: u16) -> SocketAddr {
|
||||
SocketAddr::new(ip.parse().unwrap(), port)
|
||||
}
|
||||
|
||||
pub(crate) struct Peer {
|
||||
pub(super) address: SocketAddr,
|
||||
pub(super) root: reqwest::Certificate,
|
||||
pub(super) observations: mpsc::Receiver<Vec<u8>>,
|
||||
pub(super) connections: Arc<AtomicUsize>,
|
||||
task: JoinHandle<()>,
|
||||
_directory: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl Drop for Peer {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) enum Reply {
|
||||
Bytes(Vec<u8>),
|
||||
StallAfter(Vec<u8>),
|
||||
Stall,
|
||||
Close,
|
||||
Gated(tokio::sync::oneshot::Receiver<Vec<u8>>),
|
||||
}
|
||||
|
||||
impl Peer {
|
||||
pub(crate) async fn new(certificate_host: &str, replies: Vec<Reply>) -> Self {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let output = Command::new("openssl")
|
||||
.current_dir(directory.path())
|
||||
.args([
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-noenc",
|
||||
"-days",
|
||||
"1",
|
||||
"-keyout",
|
||||
"key.pem",
|
||||
"-out",
|
||||
"cert.pem",
|
||||
"-subj",
|
||||
"/CN=isolated-plugin-peer",
|
||||
"-addext",
|
||||
&format!("subjectAltName=DNS:{certificate_host}"),
|
||||
"-addext",
|
||||
"extendedKeyUsage=serverAuth",
|
||||
])
|
||||
.output()
|
||||
.expect("TLS tests require openssl");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"isolated TLS certificate generation failed"
|
||||
);
|
||||
let certificate = std::fs::read(directory.path().join("cert.pem")).unwrap();
|
||||
let key = std::fs::read(directory.path().join("key.pem")).unwrap();
|
||||
let identity = native_tls::Identity::from_pkcs8(&certificate, &key).unwrap();
|
||||
let acceptor =
|
||||
tokio_native_tls::TlsAcceptor::from(native_tls::TlsAcceptor::new(identity).unwrap());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let (observed, observations) = mpsc::channel(16);
|
||||
let connections = Arc::new(AtomicUsize::new(0));
|
||||
let count = connections.clone();
|
||||
let mut replies = VecDeque::from(replies);
|
||||
let task = tokio::spawn(async move {
|
||||
let mut handlers = JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
accepted = listener.accept() => {
|
||||
let (socket, _) = accepted.unwrap();
|
||||
count.fetch_add(1, Ordering::SeqCst);
|
||||
let acceptor = acceptor.clone();
|
||||
let observed = observed.clone();
|
||||
let reply = replies.pop_front().unwrap_or(Reply::Close);
|
||||
handlers.spawn(async move {
|
||||
let Ok(mut socket) = acceptor.accept(socket).await else { return; };
|
||||
let mut bytes = Vec::new();
|
||||
let mut buffer = [0; 4096];
|
||||
loop {
|
||||
let Ok(length) = socket.read(&mut buffer).await else { return; };
|
||||
if length == 0 { return; }
|
||||
bytes.extend_from_slice(&buffer[..length]);
|
||||
assert!(bytes.len() <= 64 * 1024, "test request exceeded fixture bound");
|
||||
if let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") {
|
||||
let text = String::from_utf8_lossy(&bytes[..end]).to_lowercase();
|
||||
let length = text.lines().find_map(|line| line.strip_prefix("content-length: "))
|
||||
.map(|n| n.parse::<usize>().unwrap()).unwrap_or(0);
|
||||
if bytes.len() >= end + 4 + length { break; }
|
||||
}
|
||||
}
|
||||
observed.send(bytes).await.unwrap();
|
||||
match reply {
|
||||
Reply::Bytes(bytes) => { let _ = socket.write_all(&bytes).await; let _ = socket.shutdown().await; }
|
||||
Reply::StallAfter(bytes) => { let _ = socket.write_all(&bytes).await; pending::<()>().await; }
|
||||
Reply::Stall => pending::<()>().await,
|
||||
Reply::Close => {},
|
||||
Reply::Gated(release) => {
|
||||
if let Ok(bytes) = release.await {
|
||||
let _ = socket.write_all(&bytes).await;
|
||||
let _ = socket.shutdown().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
Some(result) = handlers.join_next(), if !handlers.is_empty() => { result.unwrap(); }
|
||||
}
|
||||
}
|
||||
});
|
||||
Self {
|
||||
address,
|
||||
root: reqwest::Certificate::from_pem(&certificate).unwrap(),
|
||||
observations,
|
||||
connections,
|
||||
task,
|
||||
_directory: directory,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn binding(&self) -> PluginConnectionBinding {
|
||||
let mut binding = binding();
|
||||
binding.origin = format!("https://{HOST}:{}", self.address.port());
|
||||
binding
|
||||
}
|
||||
|
||||
pub(crate) fn connection_count(&self) -> usize {
|
||||
self.connections.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn transport(&self) -> HttpsTransport {
|
||||
self.transport_for_requests(1)
|
||||
}
|
||||
|
||||
pub(crate) fn transport_for_requests(&self, requests: usize) -> HttpsTransport {
|
||||
self.transport_with_answers(vec![vec![addr("8.8.8.8", self.address.port())]; requests])
|
||||
}
|
||||
|
||||
pub(crate) fn transport_with_answers(&self, answers: Vec<Vec<SocketAddr>>) -> HttpsTransport {
|
||||
let mut transport = HttpsTransport::new(policy());
|
||||
transport.test_overrides = Some(TestOverrides {
|
||||
connect_to: Some(self.address),
|
||||
root: Some(self.root.clone()),
|
||||
answers: Mutex::new(answers.into_iter().map(DnsAnswer::Addresses).collect()),
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
});
|
||||
transport
|
||||
}
|
||||
|
||||
pub(crate) fn transport_with_dns_gate(
|
||||
&self,
|
||||
) -> (
|
||||
HttpsTransport,
|
||||
tokio::sync::oneshot::Receiver<()>,
|
||||
tokio::sync::oneshot::Sender<Vec<SocketAddr>>,
|
||||
) {
|
||||
let (started, waiting) = tokio::sync::oneshot::channel();
|
||||
let (release, answer) = tokio::sync::oneshot::channel();
|
||||
let mut transport = self.transport();
|
||||
transport.test_overrides.as_mut().unwrap().answers = Mutex::new(
|
||||
[DnsAnswer::Barrier {
|
||||
started,
|
||||
release: answer,
|
||||
}]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
);
|
||||
(transport, waiting, release)
|
||||
}
|
||||
|
||||
pub(crate) fn public_address(&self) -> SocketAddr {
|
||||
addr("8.8.8.8", self.address.port())
|
||||
}
|
||||
|
||||
pub(crate) async fn observed(&mut self) -> Vec<u8> {
|
||||
tokio::time::timeout(Duration::from_secs(5), self.observations.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn response(status: u16, headers: &[(&str, &str)], body: &[u8]) -> Reply {
|
||||
let mut response = format!(
|
||||
"HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n",
|
||||
body.len()
|
||||
)
|
||||
.into_bytes();
|
||||
for (name, value) in headers {
|
||||
write!(response, "{name}: {value}\r\n").unwrap();
|
||||
}
|
||||
response.extend_from_slice(b"\r\n");
|
||||
response.extend_from_slice(body);
|
||||
Reply::Bytes(response)
|
||||
}
|
||||
@@ -0,0 +1,866 @@
|
||||
//! Component/connection tests for the production HTTPS adapter. Only DNS answers,
|
||||
//! the socket route to an isolated peer, and that peer's trust root are replaced;
|
||||
//! request validation, public-address admission, pinning, TLS name verification,
|
||||
//! HTTP serialization, response limits and cancellation remain production code.
|
||||
//! Run with the crate's normal lib tests (`plugin_https` filter). TLS fixtures
|
||||
//! require the `openssl` executable and never change process environment or trust.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::io::Write;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
use super::test_support::*;
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ambiguous_or_out_of_scope_paths_are_denied() {
|
||||
let binding = binding();
|
||||
for path in [
|
||||
"https://other.example/api",
|
||||
"//other.example/api",
|
||||
"/api?x=y",
|
||||
"/api#fragment",
|
||||
"/api\\item",
|
||||
"/api/../item",
|
||||
"/api/./item",
|
||||
"/api//item",
|
||||
"/api/item/",
|
||||
"/api/%2e%2e/item",
|
||||
"/api/%2Fitem",
|
||||
"/api/%5citem",
|
||||
"/api/%252fitem",
|
||||
"/apix",
|
||||
"/api;param",
|
||||
"/",
|
||||
"/api/é",
|
||||
"/api/\nitem",
|
||||
] {
|
||||
let mut request = request();
|
||||
request.path = path.into();
|
||||
assert!(
|
||||
matches!(prepare(&binding, request), Err(TransportError::Denied)),
|
||||
"{path:?}"
|
||||
);
|
||||
}
|
||||
for path in ["/api", "/api/items", "/api/items-1.json", "/api/a:b@c"] {
|
||||
let mut request = request();
|
||||
request.path = path.into();
|
||||
let prepared = prepare(&binding, request).unwrap();
|
||||
assert_eq!(prepared.url.path(), path);
|
||||
assert_eq!(prepared.normalized().path, path);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_encoding_and_normalized_headers_preserve_duplicates_and_order() {
|
||||
let mut request = request();
|
||||
request.query = vec![("q".into(), "a b&c".into()), ("q".into(), "/?é".into())];
|
||||
request.headers = vec![
|
||||
("X-Order".into(), "second".into()),
|
||||
("x-order".into(), "first".into()),
|
||||
];
|
||||
let prepared = prepare(&binding(), request.clone()).unwrap();
|
||||
assert_eq!(prepared.url.query(), Some("q=a+b%26c&q=%2F%3F%C3%A9"));
|
||||
assert_eq!(prepared.normalized().query, request.query);
|
||||
assert_eq!(
|
||||
prepared.normalized().headers,
|
||||
vec![
|
||||
("x-order".into(), "second".into()),
|
||||
("x-order".into(), "first".into())
|
||||
]
|
||||
);
|
||||
let json = serde_json::to_value(prepared.normalized()).unwrap();
|
||||
assert_eq!(json["method"], "GET");
|
||||
assert_eq!(json["headers"][0][0], "x-order");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_and_control_headers_never_gain_authority_from_allowlists() {
|
||||
for name in [
|
||||
"authorization",
|
||||
"cookie",
|
||||
"cookie2",
|
||||
"set-cookie",
|
||||
"host",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"transfer-encoding",
|
||||
"te",
|
||||
"trailer",
|
||||
"upgrade",
|
||||
"content-length",
|
||||
"expect",
|
||||
"proxy-authorization",
|
||||
"proxy-anything",
|
||||
"sec-websocket-key",
|
||||
"forwarded",
|
||||
"via",
|
||||
"x-forwarded-arbitrary",
|
||||
"x-api-key",
|
||||
"api-key",
|
||||
"accept-encoding",
|
||||
] {
|
||||
let mut binding = binding();
|
||||
binding.request_headers.push(name.into());
|
||||
let mut request = request();
|
||||
request
|
||||
.headers
|
||||
.push((name.to_uppercase(), "must-not-send".into()));
|
||||
assert!(prepare(&binding, request).is_err(), "{name}");
|
||||
}
|
||||
for value in ["a\r\nb", "a\0b", "a\tb", "é"] {
|
||||
let mut request = request();
|
||||
request.headers.push(("accept".into(), value.into()));
|
||||
assert!(matches!(
|
||||
prepare(&binding(), request),
|
||||
Err(TransportError::InvalidRequest)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_limits_timeout_and_effect_identity_are_validated_before_dispatch() {
|
||||
let mut binding = binding();
|
||||
binding.limits.max_request_bytes = 4;
|
||||
let mut input = request();
|
||||
input.body = vec![0; 4];
|
||||
input.timeout_ms = 60_000;
|
||||
assert_eq!(
|
||||
prepare(&binding, input.clone())
|
||||
.unwrap()
|
||||
.normalized()
|
||||
.timeout_ms,
|
||||
5_000
|
||||
);
|
||||
input.body.push(0);
|
||||
assert!(matches!(
|
||||
prepare(&binding, input),
|
||||
Err(TransportError::RequestLimit)
|
||||
));
|
||||
let mut input = request();
|
||||
input.headers = vec![("accept".into(), "ok".into()); 65];
|
||||
assert!(matches!(
|
||||
prepare(&binding, input),
|
||||
Err(TransportError::HeaderLimit)
|
||||
));
|
||||
let mut input = request();
|
||||
input.headers = vec![("accept".into(), "x".repeat(4096))];
|
||||
assert!(matches!(
|
||||
prepare(&binding, input),
|
||||
Err(TransportError::HeaderLimit)
|
||||
));
|
||||
let mut input = request();
|
||||
input.query = vec![("q".into(), "é".repeat(4000))];
|
||||
assert!(matches!(
|
||||
prepare(&binding, input),
|
||||
Err(TransportError::RequestLimit)
|
||||
));
|
||||
for key in [String::new(), "a".repeat(129), "a\nb".into()] {
|
||||
let mut input = request();
|
||||
input.effect_key = key;
|
||||
assert!(matches!(
|
||||
prepare(&binding, input),
|
||||
Err(TransportError::InvalidRequest)
|
||||
));
|
||||
}
|
||||
let mut input = request();
|
||||
input.timeout_ms = 0;
|
||||
assert!(matches!(
|
||||
prepare(&binding, input),
|
||||
Err(TransportError::InvalidRequest)
|
||||
));
|
||||
let mut input = request();
|
||||
input.method = PluginHttpMethod::Delete;
|
||||
assert!(matches!(
|
||||
prepare(&binding, input),
|
||||
Err(TransportError::Denied)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_dns_candidates_must_be_public_and_outside_operator_exclusions() {
|
||||
let public = addr("8.8.8.8", 443);
|
||||
for ip in [
|
||||
"0.0.0.0",
|
||||
"10.1.2.3",
|
||||
"100.100.100.200",
|
||||
"127.0.0.1",
|
||||
"169.254.169.254",
|
||||
"172.16.0.1",
|
||||
"192.168.1.1",
|
||||
"192.0.0.1",
|
||||
"198.18.0.1",
|
||||
"198.51.100.1",
|
||||
"203.0.113.1",
|
||||
"224.0.0.1",
|
||||
"255.255.255.255",
|
||||
"168.63.129.16",
|
||||
"::",
|
||||
"::1",
|
||||
"::ffff:8.8.8.8",
|
||||
"fc00::1",
|
||||
"fe80::1",
|
||||
"ff02::1",
|
||||
"2001:db8::1",
|
||||
"2001::1",
|
||||
"2002:0808:0808::1",
|
||||
"64:ff9b::808:808",
|
||||
"3fff::1",
|
||||
] {
|
||||
for candidates in [vec![addr(ip, 443), public], vec![public, addr(ip, 443)]] {
|
||||
assert_eq!(
|
||||
check_addresses(&candidates, &policy()),
|
||||
Err(TransportError::Denied),
|
||||
"{ip}"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
check_addresses(&[public, addr("2606:4700:4700::1111", 443)], &policy()),
|
||||
Ok(())
|
||||
);
|
||||
let mut denied = policy();
|
||||
denied.denied_addresses.insert(public.ip());
|
||||
assert_eq!(
|
||||
check_addresses(&[public], &denied),
|
||||
Err(TransportError::Denied)
|
||||
);
|
||||
assert_eq!(check_addresses(&[], &policy()), Err(TransportError::Dns));
|
||||
assert_eq!(
|
||||
check_addresses(&vec![public; 65], &policy()),
|
||||
Err(TransportError::Dns)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pinned_connector_rejects_any_other_hostname_without_dns_fallback() {
|
||||
use reqwest::dns::Resolve;
|
||||
let resolver = PinnedResolver::new(HOST.into(), vec![addr("8.8.8.8", 443)]);
|
||||
let resolved: Vec<_> = resolver
|
||||
.resolve(HOST.parse().unwrap())
|
||||
.await
|
||||
.unwrap()
|
||||
.collect();
|
||||
assert_eq!(resolved, vec![addr("8.8.8.8", 443)]);
|
||||
assert!(
|
||||
resolver
|
||||
.resolve("other.example.com".parse().unwrap())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
transport: &HttpsTransport,
|
||||
binding: &PluginConnectionBinding,
|
||||
input: HttpsRequest,
|
||||
secret: Option<&str>,
|
||||
) -> Result<HttpsResponse, TransportError> {
|
||||
transport
|
||||
.execute(
|
||||
binding,
|
||||
prepare(binding, input)?,
|
||||
secret,
|
||||
&CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tls_peer_receives_encoded_query_duplicate_headers_and_scoped_secret() {
|
||||
let mut peer = Peer::new(
|
||||
HOST,
|
||||
vec![response(
|
||||
503,
|
||||
&[
|
||||
("Content-Type", "application/octet-stream"),
|
||||
("X-Order", "first"),
|
||||
("X-Order", "second"),
|
||||
("Set-Cookie", "must-not-return"),
|
||||
("Authorization", "must-not-return"),
|
||||
],
|
||||
&[0, 255, 128],
|
||||
)],
|
||||
)
|
||||
.await;
|
||||
let mut binding = peer.binding();
|
||||
binding.secret_ref = "credential-ref".into();
|
||||
let mut input = request();
|
||||
input.method = PluginHttpMethod::Post;
|
||||
input.query = vec![("q".into(), "a & b".into()), ("q".into(), "/?".into())];
|
||||
input.headers = vec![
|
||||
("x-order".into(), "second".into()),
|
||||
("X-Order".into(), "first".into()),
|
||||
];
|
||||
input.body = vec![1, 0, 255];
|
||||
let output = execute(
|
||||
&peer.transport(),
|
||||
&binding,
|
||||
input,
|
||||
Some("Bearer isolated-test-secret"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(output.status, 503);
|
||||
assert_eq!(output.body, vec![0, 255, 128]);
|
||||
assert_eq!(
|
||||
output
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|(k, _)| k == "x-order")
|
||||
.map(|(_, v)| v.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["first", "second"]
|
||||
);
|
||||
assert!(
|
||||
!output
|
||||
.headers
|
||||
.iter()
|
||||
.any(|(k, _)| k == "set-cookie" || k == "authorization")
|
||||
);
|
||||
let wire = peer.observed().await;
|
||||
let text = String::from_utf8_lossy(&wire).to_lowercase();
|
||||
assert!(text.starts_with("post /api/items?q=a+%26+b&q=%2f%3f http/1.1\r\n"));
|
||||
assert!(text.contains(&format!("host: {HOST}:{}\r\n", peer.address.port())));
|
||||
assert!(text.contains("authorization: bearer isolated-test-secret\r\n"));
|
||||
assert!(text.contains("x-order: second\r\nx-order: first\r\n"));
|
||||
assert!(wire.ends_with(&[1, 0, 255]));
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_connection_nominated_headers_are_not_exposed() {
|
||||
let peer = Peer::new(
|
||||
HOST,
|
||||
vec![response(
|
||||
200,
|
||||
&[
|
||||
("Connection", "x-order"),
|
||||
("X-Order", "hop-value"),
|
||||
("Content-Type", "text/plain"),
|
||||
],
|
||||
b"ok",
|
||||
)],
|
||||
)
|
||||
.await;
|
||||
let output = execute(&peer.transport(), &peer.binding(), request(), None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!output.headers.iter().any(|(name, _)| name == "x-order"));
|
||||
assert!(
|
||||
output
|
||||
.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name == "content-type")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn redirects_are_returned_without_following_or_forwarding_credentials() {
|
||||
let peer = Peer::new(
|
||||
HOST,
|
||||
vec![response(
|
||||
302,
|
||||
&[("Location", "https://127.0.0.1/private")],
|
||||
b"redirect",
|
||||
)],
|
||||
)
|
||||
.await;
|
||||
let mut binding = peer.binding();
|
||||
binding.secret_ref = "credential-ref".into();
|
||||
let output = execute(
|
||||
&peer.transport(),
|
||||
&binding,
|
||||
request(),
|
||||
Some("Bearer secret"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(output.status, 302);
|
||||
assert_eq!(output.body, b"redirect");
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tls_name_mismatch_and_untrusted_certificate_are_rejected() {
|
||||
for (host, trust) in [("wrong.example.com", true), (HOST, false)] {
|
||||
let peer = Peer::new(host, vec![]).await;
|
||||
let mut transport = peer.transport();
|
||||
if !trust {
|
||||
transport.test_overrides.as_mut().unwrap().root = None;
|
||||
}
|
||||
assert_eq!(
|
||||
execute(&transport, &peer.binding(), request(), None).await,
|
||||
Err(TransportError::Unavailable)
|
||||
);
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
|
||||
assert!(peer.observations.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_loss_never_retries_even_a_get() {
|
||||
let mut peer = Peer::new(
|
||||
HOST,
|
||||
vec![Reply::Close, response(200, &[], b"must-not-retry")],
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
execute(&peer.transport(), &peer.binding(), request(), None).await,
|
||||
Err(TransportError::Unavailable)
|
||||
);
|
||||
peer.observed().await;
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mixed_dns_is_denied_before_connect_and_rebinding_is_rechecked_per_execution() {
|
||||
let peer = Peer::new(HOST, vec![response(200, &[], b"ok")]).await;
|
||||
let port = peer.address.port();
|
||||
let transport =
|
||||
peer.transport_with_answers(vec![vec![addr("8.8.8.8", port), addr("127.0.0.1", port)]]);
|
||||
assert_eq!(
|
||||
execute(&transport, &peer.binding(), request(), None).await,
|
||||
Err(TransportError::Denied)
|
||||
);
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 0);
|
||||
let transport = peer.transport_with_answers(vec![
|
||||
vec![addr("8.8.8.8", port)],
|
||||
vec![addr("127.0.0.1", port)],
|
||||
]);
|
||||
assert_eq!(
|
||||
execute(&transport, &peer.binding(), request(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.body,
|
||||
b"ok"
|
||||
);
|
||||
assert_eq!(
|
||||
transport
|
||||
.test_overrides
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.calls
|
||||
.load(Ordering::SeqCst),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
execute(&transport, &peer.binding(), request(), None).await,
|
||||
Err(TransportError::Denied)
|
||||
);
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
transport
|
||||
.test_overrides
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.calls
|
||||
.load(Ordering::SeqCst),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn operator_control_plane_exclusions_apply_before_connect() {
|
||||
let peer = Peer::new(HOST, vec![]).await;
|
||||
for denied_host in [false, true] {
|
||||
let mut transport = peer.transport();
|
||||
if denied_host {
|
||||
transport.policy.denied_hosts.insert(HOST.to_uppercase());
|
||||
} else {
|
||||
transport
|
||||
.policy
|
||||
.denied_addresses
|
||||
.insert("8.8.8.8".parse().unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
execute(&transport, &peer.binding(), request(), None).await,
|
||||
Err(TransportError::Denied)
|
||||
);
|
||||
}
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn binding_swap_invalid_credentials_and_generated_header_overflow_never_dispatch() {
|
||||
let peer = Peer::new(HOST, vec![]).await;
|
||||
let transport = peer.transport();
|
||||
let mut binding = peer.binding();
|
||||
let prepared = prepare(&binding, request()).unwrap();
|
||||
binding.path_prefixes = vec!["/api/items".into()];
|
||||
assert_eq!(
|
||||
transport
|
||||
.execute(&binding, prepared, None, &CancellationToken::new())
|
||||
.await,
|
||||
Err(TransportError::Denied)
|
||||
);
|
||||
for (reference, secret) in [
|
||||
("", Some("Bearer unexpected")),
|
||||
("credential", None),
|
||||
("credential", Some("Bearer a\r\nb")),
|
||||
] {
|
||||
let mut binding = peer.binding();
|
||||
binding.secret_ref = reference.into();
|
||||
// Each rejected operation gets an explicit answer; none opens a socket.
|
||||
let transport = peer.transport();
|
||||
assert_eq!(
|
||||
execute(&transport, &binding, request(), secret).await,
|
||||
Err(TransportError::InvalidSecret)
|
||||
);
|
||||
}
|
||||
let mut binding = peer.binding();
|
||||
binding.secret_ref = "credential".into();
|
||||
binding.limits.max_header_bytes = 64;
|
||||
assert_eq!(
|
||||
execute(
|
||||
&peer.transport(),
|
||||
&binding,
|
||||
request(),
|
||||
Some(&"s".repeat(100))
|
||||
)
|
||||
.await,
|
||||
Err(TransportError::HeaderLimit)
|
||||
);
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_wire_size_count_and_header_bytes_are_bounded_before_return() {
|
||||
let cases = [
|
||||
response(200, &[], &vec![0; 1025]),
|
||||
Reply::Bytes(
|
||||
b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n401\r\n"
|
||||
.iter()
|
||||
.copied()
|
||||
.chain(vec![0; 1025])
|
||||
.chain(b"\r\n0\r\n\r\n".iter().copied())
|
||||
.collect(),
|
||||
),
|
||||
response(200, &[("X-Order", &"x".repeat(4096))], b"ok"),
|
||||
response(200, &vec![("X-Order", "x"); 65], b"ok"),
|
||||
];
|
||||
for (index, reply) in cases.into_iter().enumerate() {
|
||||
let peer = Peer::new(HOST, vec![reply]).await;
|
||||
let result = execute(&peer.transport(), &peer.binding(), request(), None).await;
|
||||
let expected = if index < 2 {
|
||||
TransportError::ResponseLimit
|
||||
} else {
|
||||
TransportError::HeaderLimit
|
||||
};
|
||||
assert_eq!(result, Err(expected), "case {index}");
|
||||
}
|
||||
}
|
||||
|
||||
fn gzip(bytes: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
encoder.write_all(bytes).unwrap();
|
||||
encoder.finish().unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gzip_is_bounded_after_expansion_including_concatenated_members() {
|
||||
for (payload, accepted) in [
|
||||
(gzip(&vec![0; 1024]), true),
|
||||
(gzip(&vec![0; 1025]), false),
|
||||
([gzip(&vec![0; 600]), gzip(&vec![0; 600])].concat(), false),
|
||||
] {
|
||||
let peer = Peer::new(
|
||||
HOST,
|
||||
vec![response(200, &[("Content-Encoding", "gzip")], &payload)],
|
||||
)
|
||||
.await;
|
||||
let result = execute(&peer.transport(), &peer.binding(), request(), None).await;
|
||||
if accepted {
|
||||
let result = result.unwrap();
|
||||
assert_eq!(result.body.len(), 1024);
|
||||
assert!(!result.headers.iter().any(|(k, _)| k == "content-encoding"));
|
||||
} else {
|
||||
assert_eq!(result, Err(TransportError::ResponseLimit));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsupported_or_corrupt_compression_is_not_returned_as_a_success() {
|
||||
for (headers, expected) in [
|
||||
(
|
||||
vec![("Content-Encoding", "br")],
|
||||
TransportError::UnsupportedEncoding,
|
||||
),
|
||||
(
|
||||
vec![
|
||||
("Content-Encoding", "gzip"),
|
||||
("Content-Encoding", "identity"),
|
||||
],
|
||||
TransportError::UnsupportedEncoding,
|
||||
),
|
||||
(
|
||||
vec![("Content-Encoding", "gzip")],
|
||||
TransportError::Unavailable,
|
||||
),
|
||||
] {
|
||||
let peer = Peer::new(HOST, vec![response(200, &headers, b"not-compressed")]).await;
|
||||
assert_eq!(
|
||||
execute(&peer.transport(), &peer.binding(), request(), None).await,
|
||||
Err(expected)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_covers_dns_and_pre_cancelled_requests_without_network() {
|
||||
let mut transport = HttpsTransport::new(policy());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
transport.test_overrides = Some(TestOverrides {
|
||||
connect_to: None,
|
||||
root: None,
|
||||
answers: Mutex::new(VecDeque::from([DnsAnswer::Pending(None)])),
|
||||
calls: calls.clone(),
|
||||
});
|
||||
let cancellation = CancellationToken::new();
|
||||
cancellation.cancel();
|
||||
let binding = binding();
|
||||
assert_eq!(
|
||||
transport
|
||||
.execute(
|
||||
&binding,
|
||||
prepare(&binding, request()).unwrap(),
|
||||
None,
|
||||
&cancellation
|
||||
)
|
||||
.await,
|
||||
Err(TransportError::Cancelled)
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
let mut input = request();
|
||||
input.timeout_ms = 20;
|
||||
assert_eq!(
|
||||
execute(&transport, &binding, input, None).await,
|
||||
Err(TransportError::DeadlineExceeded)
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
let (started, observed) = tokio::sync::oneshot::channel();
|
||||
transport
|
||||
.test_overrides
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.answers
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push_back(DnsAnswer::Pending(Some(started)));
|
||||
let cancellation = CancellationToken::new();
|
||||
let operation = transport.execute(
|
||||
&binding,
|
||||
prepare(&binding, request()).unwrap(),
|
||||
None,
|
||||
&cancellation,
|
||||
);
|
||||
tokio::pin!(operation);
|
||||
tokio::select! {
|
||||
value = &mut operation => panic!("pending DNS completed unexpectedly: {value:?}"),
|
||||
value = observed => value.unwrap(),
|
||||
}
|
||||
cancellation.cancel();
|
||||
assert_eq!(operation.await, Err(TransportError::Cancelled));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_and_deadline_abort_inflight_response_waits() {
|
||||
for reply in [
|
||||
Reply::Stall,
|
||||
Reply::StallAfter(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\na".to_vec()),
|
||||
] {
|
||||
let mut peer = Peer::new(HOST, vec![reply]).await;
|
||||
let transport = peer.transport();
|
||||
let binding = peer.binding();
|
||||
let cancellation = CancellationToken::new();
|
||||
let operation = transport.execute(
|
||||
&binding,
|
||||
prepare(&binding, request()).unwrap(),
|
||||
None,
|
||||
&cancellation,
|
||||
);
|
||||
tokio::pin!(operation);
|
||||
tokio::select! {
|
||||
value = &mut operation => panic!("response completed before cancellation: {value:?}"),
|
||||
_ = peer.observed() => {},
|
||||
}
|
||||
cancellation.cancel();
|
||||
assert_eq!(operation.await, Err(TransportError::Cancelled));
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
let mut peer = Peer::new(HOST, vec![Reply::Stall]).await;
|
||||
let mut input = request();
|
||||
input.timeout_ms = 250;
|
||||
let result = execute(&peer.transport(), &peer.binding(), input, None).await;
|
||||
assert_eq!(result, Err(TransportError::DeadlineExceeded));
|
||||
peer.observed().await;
|
||||
assert_eq!(peer.connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn revocation_during_dns_is_checked_by_final_guard_without_opening_peer_connection() {
|
||||
let peer = Peer::new(HOST, vec![]).await;
|
||||
let mut transport = peer.transport();
|
||||
let (dns_started, dns_observed) = tokio::sync::oneshot::channel();
|
||||
let (dns_release, dns_answer) = tokio::sync::oneshot::channel();
|
||||
transport.test_overrides.as_mut().unwrap().answers =
|
||||
Mutex::new(VecDeque::from([DnsAnswer::Barrier {
|
||||
started: dns_started,
|
||||
release: dns_answer,
|
||||
}]));
|
||||
let authority = AtomicBool::new(true);
|
||||
let guard_called = AtomicBool::new(false);
|
||||
let binding = peer.binding();
|
||||
let cancellation = CancellationToken::new();
|
||||
let operation = transport.execute_with_guard(
|
||||
&binding,
|
||||
prepare(&binding, request()).unwrap(),
|
||||
None,
|
||||
&cancellation,
|
||||
|| async {
|
||||
guard_called.store(true, Ordering::SeqCst);
|
||||
tokio::task::yield_now().await;
|
||||
if authority.load(Ordering::SeqCst) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(TransportError::Denied)
|
||||
}
|
||||
},
|
||||
);
|
||||
tokio::pin!(operation);
|
||||
tokio::select! {
|
||||
value = &mut operation => panic!("operation completed during blocked DNS: {value:?}"),
|
||||
value = dns_observed => value.unwrap(),
|
||||
}
|
||||
assert!(!guard_called.load(Ordering::SeqCst));
|
||||
assert_eq!(peer.connection_count(), 0);
|
||||
authority.store(false, Ordering::SeqCst);
|
||||
dns_release
|
||||
.send(vec![addr("8.8.8.8", peer.address.port())])
|
||||
.unwrap();
|
||||
assert_eq!(operation.await, Err(TransportError::Denied));
|
||||
assert!(guard_called.load(Ordering::SeqCst));
|
||||
assert_eq!(peer.connection_count(), 0);
|
||||
assert!(peer.observations.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_async_dispatch_guard_runs_once_and_allows_the_prepared_request() {
|
||||
let peer = Peer::new(HOST, vec![response(200, &[], b"guarded")]).await;
|
||||
let transport = peer.transport();
|
||||
let binding = peer.binding();
|
||||
let calls = AtomicUsize::new(0);
|
||||
let output = transport
|
||||
.execute_with_guard(
|
||||
&binding,
|
||||
prepare(&binding, request()).unwrap(),
|
||||
None,
|
||||
&CancellationToken::new(),
|
||||
|| async {
|
||||
tokio::task::yield_now().await;
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(output.body, b"guarded");
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(peer.connection_count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dispatch_guard_is_cancelled_with_the_operation_and_never_opens_a_connection() {
|
||||
let peer = Peer::new(HOST, vec![]).await;
|
||||
let transport = peer.transport();
|
||||
let binding = peer.binding();
|
||||
let cancellation = CancellationToken::new();
|
||||
let (started, observed) = tokio::sync::oneshot::channel();
|
||||
let operation = transport.execute_with_guard(
|
||||
&binding,
|
||||
prepare(&binding, request()).unwrap(),
|
||||
None,
|
||||
&cancellation,
|
||||
|| async {
|
||||
started.send(()).unwrap();
|
||||
std::future::pending::<Result<(), TransportError>>().await
|
||||
},
|
||||
);
|
||||
tokio::pin!(operation);
|
||||
tokio::select! {
|
||||
value = &mut operation => panic!("pending authority guard completed: {value:?}"),
|
||||
value = observed => value.unwrap(),
|
||||
}
|
||||
cancellation.cancel();
|
||||
assert_eq!(operation.await, Err(TransportError::Cancelled));
|
||||
assert_eq!(peer.connection_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn dispatch_guard_deadline_expiry_prevents_connection_even_if_guard_returns_success() {
|
||||
let peer = Peer::new(HOST, vec![]).await;
|
||||
let transport = peer.transport();
|
||||
let binding = peer.binding();
|
||||
let result = transport
|
||||
.execute_with_guard(
|
||||
&binding,
|
||||
prepare(&binding, request()).unwrap(),
|
||||
None,
|
||||
&CancellationToken::new(),
|
||||
|| async {
|
||||
tokio::time::advance(Duration::from_secs(6)).await;
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result, Err(TransportError::DeadlineExceeded));
|
||||
assert_eq!(peer.connection_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dispatch_guard_cancellation_prevents_connection_even_if_guard_returns_success() {
|
||||
let peer = Peer::new(HOST, vec![]).await;
|
||||
let transport = peer.transport();
|
||||
let binding = peer.binding();
|
||||
let cancellation = CancellationToken::new();
|
||||
let result = transport
|
||||
.execute_with_guard(
|
||||
&binding,
|
||||
prepare(&binding, request()).unwrap(),
|
||||
None,
|
||||
&cancellation,
|
||||
|| async {
|
||||
cancellation.cancel();
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result, Err(TransportError::Cancelled));
|
||||
assert_eq!(peer.connection_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_formats_omit_request_response_urls_paths_and_secrets() {
|
||||
let mut input = request();
|
||||
input.path = "/api/private-path".into();
|
||||
input.body = b"secret-body".to_vec();
|
||||
let prepared = prepare(&binding(), input.clone()).unwrap();
|
||||
let output = HttpsResponse {
|
||||
status: 200,
|
||||
headers: vec![("x-order".into(), "secret-header".into())],
|
||||
body: b"secret-body".to_vec(),
|
||||
};
|
||||
let debug = format!("{input:?} {prepared:?} {output:?}");
|
||||
for forbidden in [HOST, "private-path", "secret-body", "secret-header"] {
|
||||
assert!(!debug.contains(forbidden));
|
||||
}
|
||||
let error = TransportError::Unavailable;
|
||||
assert_eq!(error.to_string(), "HTTPS transport unavailable");
|
||||
assert!(std::error::Error::source(&error).is_none());
|
||||
}
|
||||
@@ -30,6 +30,10 @@ fn plugin_plan_migration_marker_failure_preserves_catalog_then_retries_and_match
|
||||
108,
|
||||
"Subject Ticket responsibility and current-body notification routing".into(),
|
||||
));
|
||||
expected.push((
|
||||
109,
|
||||
"Durable Plugin Host invocations and HTTPS effect receipts".into(),
|
||||
));
|
||||
assert_eq!(workspace_schema_migration_history(conn)?, expected);
|
||||
subject_assignment_migration::verify(conn)?;
|
||||
crate::plugin_plan::verify_plan_schema(conn)?;
|
||||
|
||||
@@ -0,0 +1,477 @@
|
||||
//! Durable Host-issued logical invocations and all-method effect receipts.
|
||||
//!
|
||||
//! These owner functions are storage, NOT authentication or admission authority.
|
||||
//! The caller verifies the authenticated source, current plan/installation,
|
||||
//! delegation, grants, binding and result-read authority on *every* call, including
|
||||
//! replay. Consumer identities come from durable Host Tool-call/event records,
|
||||
//! never the guest. A saved invocation or receipt confers no execution authority.
|
||||
//!
|
||||
//! Use an IMMEDIATE owner transaction. Commit `begin` and `effect_admit` before
|
||||
//! dispatching any network request; a returned `Dispatch` from an uncommitted or
|
||||
//! failed commit is NOT permission to send. No transaction is held across I/O.
|
||||
//! Every inserted receipt starts with a durable indeterminate error. Losing the
|
||||
//! dispatch opportunity (even before sending) sacrifices availability, never
|
||||
//! safety: no existing receipt is dispatchable, for ANY HTTP method.
|
||||
//!
|
||||
//! The parent owns migration registration and Workspace deletion. This module
|
||||
//! neither registers a schema at runtime nor creates pins/grants/catalog state.
|
||||
use crate::{
|
||||
Error, Result,
|
||||
plugin_plan::{HostExecutionIdentity, StoredPluginPlan},
|
||||
};
|
||||
use rusqlite::{Connection, OptionalExtension, Transaction, params};
|
||||
use server_api::{
|
||||
PluginConsumer, PluginEffectObservation, PluginHostError, PluginHostErrorCode,
|
||||
PluginHttpResponse,
|
||||
};
|
||||
|
||||
pub const SCHEMA_SQL: &str = include_str!("plugin_receipt/schema.sql");
|
||||
/// Buffered body ceiling; encoded allowed headers have their separate finite bound.
|
||||
/// Both are protected Workspace data rather than diagnostic text.
|
||||
pub const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
|
||||
pub const MAX_RESPONSE_HEADER_BYTES: usize = 64 * 1024;
|
||||
pub const MAX_RESPONSE_HEADERS: usize = 128;
|
||||
const MAX_ERROR_BYTES: usize = 4096;
|
||||
|
||||
type HttpResult = std::result::Result<PluginHttpResponse, PluginHostError>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct InvocationRecord {
|
||||
pub invocation_id: String,
|
||||
pub workspace_id: String,
|
||||
pub plan_id: String,
|
||||
pub installation_id: String,
|
||||
pub execution: HostExecutionIdentity,
|
||||
pub consumer: PluginConsumer,
|
||||
pub expires_at_ms: i64,
|
||||
pub cancelled: bool,
|
||||
pub expired: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum EffectAdmission {
|
||||
/// Only a first INSERT, to be committed durably before external dispatch.
|
||||
Dispatch,
|
||||
/// Includes a saved indeterminate error for pending/unknown outcomes.
|
||||
Replay(HttpResult),
|
||||
}
|
||||
|
||||
fn host_error(code: PluginHostErrorCode, effect: PluginEffectObservation) -> PluginHostError {
|
||||
PluginHostError {
|
||||
code,
|
||||
effect,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
}
|
||||
}
|
||||
fn fail<T>(code: PluginHostErrorCode, effect: PluginEffectObservation) -> Result<T> {
|
||||
Err(host_error(code, effect).into())
|
||||
}
|
||||
fn invalid<T>() -> Result<T> {
|
||||
fail(
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
)
|
||||
}
|
||||
fn conflict<T>() -> Result<T> {
|
||||
// A conflicting key or logical identity might already have external effects.
|
||||
fail(
|
||||
PluginHostErrorCode::Conflict,
|
||||
PluginEffectObservation::Indeterminate,
|
||||
)
|
||||
}
|
||||
fn expired<T>() -> Result<T> {
|
||||
fail(
|
||||
PluginHostErrorCode::ReceiptExpired,
|
||||
PluginEffectObservation::Indeterminate,
|
||||
)
|
||||
}
|
||||
fn valid_identity(value: &str, max: usize) -> bool {
|
||||
!value.is_empty() && value.len() <= max && !value.chars().any(char::is_control)
|
||||
}
|
||||
fn valid_fingerprint(value: &str) -> bool {
|
||||
value.strip_prefix("sha256:").is_some_and(|hex| {
|
||||
hex.len() == 64
|
||||
&& hex
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
})
|
||||
}
|
||||
fn encode(value: &impl serde::Serialize) -> Result<String> {
|
||||
serde_json::to_string(value).map_err(|_| Error::Store("invalid Plugin receipt data".into()))
|
||||
}
|
||||
fn decode<T: serde::de::DeserializeOwned>(value: &str) -> Result<T> {
|
||||
serde_json::from_str(value)
|
||||
.map_err(|_| Error::Store("invalid persisted Plugin receipt data".into()))
|
||||
}
|
||||
|
||||
// The exported name is immutable input, NOT part of logical identity. Renaming
|
||||
// an export/tool on the same durable call/event conflicts, rather than opening a
|
||||
// fresh effect-key namespace. JSON tuples avoid separator/escaping collisions.
|
||||
fn consumer_key(consumer: &PluginConsumer) -> Result<String> {
|
||||
match consumer {
|
||||
PluginConsumer::ToolCall {
|
||||
tool_call_id,
|
||||
tool_name,
|
||||
} => {
|
||||
if !valid_identity(tool_call_id, 512) || !valid_identity(tool_name, 512) {
|
||||
return invalid();
|
||||
}
|
||||
encode(&["tool_call", tool_call_id])
|
||||
}
|
||||
PluginConsumer::Consumer {
|
||||
consumer_id,
|
||||
event_id,
|
||||
export_name,
|
||||
} => {
|
||||
if !valid_identity(consumer_id, 512)
|
||||
|| !valid_identity(event_id, 512)
|
||||
|| !valid_identity(export_name, 512)
|
||||
{
|
||||
return invalid();
|
||||
}
|
||||
encode(&["consumer", consumer_id, event_id])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Establish/recover a Host UUID for a durable consumer under the stored plan.
|
||||
/// The first receipt-retention deadline is fixed; retries do not extend it.
|
||||
/// Replay returns the identity even after cancellation/expiry, not new execution
|
||||
/// authority. `effect_admit` refuses expired invocations, including unused keys.
|
||||
/// A new Tool-call/event identity must represent an explicit new request.
|
||||
pub fn begin(
|
||||
tx: &Transaction<'_>,
|
||||
stored: &StoredPluginPlan,
|
||||
installation_id: &str,
|
||||
consumer: &PluginConsumer,
|
||||
now_ms: i64,
|
||||
retention_ms: i64,
|
||||
) -> Result<InvocationRecord> {
|
||||
let Some(expires_at_ms) = now_ms.checked_add(retention_ms) else {
|
||||
return invalid();
|
||||
};
|
||||
if now_ms < 0 || retention_ms <= 0 || !valid_identity(installation_id, 512) {
|
||||
return invalid();
|
||||
}
|
||||
// Snapshot integrity only. Live security authority belongs to the caller.
|
||||
stored.plan.validate_canonical()?;
|
||||
if !valid_identity(&stored.plan_id, 512)
|
||||
|| stored.fingerprint != stored.plan.fingerprint()?
|
||||
|| !stored
|
||||
.plan
|
||||
.plugins()
|
||||
.iter()
|
||||
.any(|pin| pin.installation_id() == installation_id)
|
||||
{
|
||||
return invalid();
|
||||
}
|
||||
let workspace = stored.plan.workspace_id();
|
||||
let execution = stored.plan.execution();
|
||||
let key = consumer_key(consumer)?;
|
||||
let execution_json = encode(execution)?;
|
||||
let consumer_json = encode(consumer)?;
|
||||
let saved = tx.query_row(
|
||||
"SELECT invocation_id,plan_fingerprint FROM plugin_invocations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND installation_id=?4 AND consumer_key=?5",
|
||||
params![workspace, execution.runtime_id, execution.worker_id, installation_id, key],
|
||||
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
|
||||
).optional()?;
|
||||
if let Some((id, fingerprint)) = saved {
|
||||
let record = lookup(tx, workspace, &id)?;
|
||||
if record.plan_id != stored.plan_id
|
||||
|| fingerprint != stored.fingerprint
|
||||
|| record.execution != *execution
|
||||
|| record.consumer != *consumer
|
||||
{
|
||||
return conflict();
|
||||
}
|
||||
return Ok(record);
|
||||
}
|
||||
let invocation_id = uuid::Uuid::now_v7().to_string();
|
||||
tx.execute(
|
||||
"INSERT INTO plugin_invocations(workspace_id,invocation_id,plan_id,plan_fingerprint,installation_id,runtime_id,worker_id,execution_json,consumer_key,consumer_json,created_at_ms,expires_at_ms) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",
|
||||
params![workspace, invocation_id, stored.plan_id, stored.fingerprint, installation_id, execution.runtime_id, execution.worker_id, execution_json, key, consumer_json, now_ms, expires_at_ms],
|
||||
)?;
|
||||
lookup(tx, workspace, &invocation_id)
|
||||
}
|
||||
|
||||
/// Persisted identity lookup only; the caller checks current read authority.
|
||||
pub fn lookup(conn: &Connection, workspace: &str, invocation_id: &str) -> Result<InvocationRecord> {
|
||||
if !valid_identity(workspace, 512) || !valid_identity(invocation_id, 512) {
|
||||
return invalid();
|
||||
}
|
||||
let row = conn.query_row(
|
||||
"SELECT plan_id,installation_id,execution_json,consumer_json,expires_at_ms,cancelled,expired FROM plugin_invocations WHERE workspace_id=?1 AND invocation_id=?2",
|
||||
params![workspace, invocation_id],
|
||||
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?, row.get::<_, String>(3)?, row.get::<_, i64>(4)?, row.get::<_, bool>(5)?, row.get::<_, bool>(6)?)),
|
||||
).optional()?;
|
||||
let Some((plan_id, installation_id, execution, consumer, expires_at_ms, cancelled, expired)) =
|
||||
row
|
||||
else {
|
||||
return fail(
|
||||
PluginHostErrorCode::NotFound,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
);
|
||||
};
|
||||
Ok(InvocationRecord {
|
||||
invocation_id: invocation_id.into(),
|
||||
workspace_id: workspace.into(),
|
||||
plan_id,
|
||||
installation_id,
|
||||
execution: decode(&execution)?,
|
||||
consumer: decode(&consumer)?,
|
||||
expires_at_ms,
|
||||
cancelled,
|
||||
expired,
|
||||
})
|
||||
}
|
||||
|
||||
// Never trust a passed record's mutable flags/deadline or accept a fabricated
|
||||
// plan/execution/consumer. Records may be stale only in monotonic terminal flags.
|
||||
fn verify_record(conn: &Connection, supplied: &InvocationRecord) -> Result<InvocationRecord> {
|
||||
let saved = lookup(conn, &supplied.workspace_id, &supplied.invocation_id)?;
|
||||
if supplied.plan_id != saved.plan_id
|
||||
|| supplied.installation_id != saved.installation_id
|
||||
|| supplied.execution != saved.execution
|
||||
|| supplied.consumer != saved.consumer
|
||||
|| supplied.expires_at_ms != saved.expires_at_ms
|
||||
{
|
||||
return conflict();
|
||||
}
|
||||
Ok(saved)
|
||||
}
|
||||
fn is_expired(conn: &Connection, record: &InvocationRecord) -> Result<bool> {
|
||||
Ok(conn.query_row(
|
||||
"SELECT expired FROM plugin_invocations WHERE workspace_id=?1 AND invocation_id=?2",
|
||||
params![record.workspace_id, record.invocation_id],
|
||||
|row| row.get(0),
|
||||
)?)
|
||||
}
|
||||
fn validate_key(effect_key: &str, fingerprint: &str) -> Result<()> {
|
||||
if !valid_identity(effect_key, 128) || !valid_fingerprint(fingerprint) {
|
||||
return invalid();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SavedEffect {
|
||||
fingerprint: String,
|
||||
state: String,
|
||||
status: Option<u16>,
|
||||
headers: Option<String>,
|
||||
body: Option<Vec<u8>>,
|
||||
error: Option<String>,
|
||||
}
|
||||
fn saved_effect(
|
||||
conn: &Connection,
|
||||
record: &InvocationRecord,
|
||||
key: &str,
|
||||
) -> Result<Option<SavedEffect>> {
|
||||
Ok(conn.query_row(
|
||||
"SELECT fingerprint,state,response_status,response_headers,response_body,error_json FROM plugin_effect_receipts WHERE workspace_id=?1 AND invocation_id=?2 AND effect_key=?3",
|
||||
params![record.workspace_id, record.invocation_id, key],
|
||||
|row| Ok(SavedEffect { fingerprint: row.get(0)?, state: row.get(1)?, status: row.get(2)?, headers: row.get(3)?, body: row.get(4)?, error: row.get(5)? }),
|
||||
).optional()?)
|
||||
}
|
||||
impl SavedEffect {
|
||||
fn result(self) -> Result<HttpResult> {
|
||||
if self.state == "expired" {
|
||||
return expired();
|
||||
}
|
||||
if let Some(json) = self.error {
|
||||
return Ok(Err(decode(&json)?));
|
||||
}
|
||||
match (self.status, self.headers, self.body) {
|
||||
(Some(status), Some(headers), Some(body)) if self.state == "completed" => {
|
||||
Ok(Ok(PluginHttpResponse {
|
||||
status,
|
||||
headers: decode(&headers)?,
|
||||
body,
|
||||
}))
|
||||
}
|
||||
_ => Err(Error::Store(
|
||||
"invalid persisted Plugin receipt result".into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The parent supplies a canonical request fingerprint covering all HTTP methods,
|
||||
/// binding/slot/policy, plan, path, ordered query/headers and body digest, excluding
|
||||
/// secret values and remaining timeout. Never accepts/stores raw request data.
|
||||
/// Commit the insertion before I/O. Repeated pending/unknown results NEVER dispatch.
|
||||
pub fn effect_admit(
|
||||
tx: &Transaction<'_>,
|
||||
record: &InvocationRecord,
|
||||
effect_key: &str,
|
||||
fingerprint: &str,
|
||||
now_ms: i64,
|
||||
) -> Result<EffectAdmission> {
|
||||
validate_key(effect_key, fingerprint)?;
|
||||
if now_ms < 0 {
|
||||
return invalid();
|
||||
}
|
||||
let record = verify_record(tx, record)?;
|
||||
// Retention expiry takes precedence: it must not look like an unused key.
|
||||
if now_ms >= record.expires_at_ms || is_expired(tx, &record)? {
|
||||
return expired();
|
||||
}
|
||||
if record.cancelled {
|
||||
return fail(
|
||||
PluginHostErrorCode::Cancelled,
|
||||
PluginEffectObservation::Indeterminate,
|
||||
);
|
||||
}
|
||||
if let Some(saved) = saved_effect(tx, &record, effect_key)? {
|
||||
if saved.fingerprint != fingerprint {
|
||||
return conflict();
|
||||
}
|
||||
return Ok(EffectAdmission::Replay(saved.result()?));
|
||||
}
|
||||
let unknown = host_error(
|
||||
PluginHostErrorCode::Indeterminate,
|
||||
PluginEffectObservation::Indeterminate,
|
||||
);
|
||||
tx.execute(
|
||||
"INSERT INTO plugin_effect_receipts(workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms,state,error_json) VALUES(?1,?2,?3,?4,?5,'pending',?6)",
|
||||
params![record.workspace_id, record.invocation_id, effect_key, fingerprint, now_ms, encode(&unknown)?],
|
||||
)?;
|
||||
Ok(EffectAdmission::Dispatch)
|
||||
}
|
||||
|
||||
/// Save one bounded observed outcome. This records an already-admitted effect;
|
||||
/// it does not authorize dispatch or publication. An in-flight outcome may be
|
||||
/// recorded after cancellation. Once pruned, payloads can never be restored.
|
||||
/// Exact completion replay is harmless; different/completed results conflict.
|
||||
/// On invalid/oversized completion the original indeterminate receipt survives.
|
||||
pub fn effect_complete(
|
||||
tx: &Transaction<'_>,
|
||||
record: &InvocationRecord,
|
||||
effect_key: &str,
|
||||
fingerprint: &str,
|
||||
result: &std::result::Result<PluginHttpResponse, PluginHostError>,
|
||||
) -> Result<()> {
|
||||
validate_key(effect_key, fingerprint)?;
|
||||
let record = verify_record(tx, record)?;
|
||||
if is_expired(tx, &record)? {
|
||||
return expired();
|
||||
}
|
||||
let Some(saved) = saved_effect(tx, &record, effect_key)? else {
|
||||
return fail(
|
||||
PluginHostErrorCode::NotFound,
|
||||
PluginEffectObservation::NotDispatched,
|
||||
);
|
||||
};
|
||||
if saved.fingerprint != fingerprint {
|
||||
return conflict();
|
||||
}
|
||||
if saved.state == "expired" {
|
||||
return expired();
|
||||
}
|
||||
if saved.state == "completed" {
|
||||
return if saved.result()? == *result {
|
||||
Ok(())
|
||||
} else {
|
||||
conflict()
|
||||
};
|
||||
}
|
||||
let (status, headers, body, error) = match result {
|
||||
Ok(response) => {
|
||||
if !(100..=599).contains(&response.status) {
|
||||
return fail(
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
PluginEffectObservation::ResponseObserved,
|
||||
);
|
||||
}
|
||||
if response.headers.len() > MAX_RESPONSE_HEADERS {
|
||||
return fail(
|
||||
PluginHostErrorCode::QuotaExceeded,
|
||||
PluginEffectObservation::ResponseObserved,
|
||||
);
|
||||
}
|
||||
let headers = encode(&response.headers)?;
|
||||
if headers.len() > MAX_RESPONSE_HEADER_BYTES || response.body.len() > MAX_RESPONSE_BYTES
|
||||
{
|
||||
return fail(
|
||||
PluginHostErrorCode::QuotaExceeded,
|
||||
PluginEffectObservation::ResponseObserved,
|
||||
);
|
||||
}
|
||||
(
|
||||
Some(response.status),
|
||||
Some(headers),
|
||||
Some(response.body.as_slice()),
|
||||
None,
|
||||
)
|
||||
}
|
||||
Err(error) => {
|
||||
if !valid_identity(&error.diagnostic_id, 128) {
|
||||
return fail(
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
PluginEffectObservation::Indeterminate,
|
||||
);
|
||||
}
|
||||
let json = encode(error)?;
|
||||
if json.len() > MAX_ERROR_BYTES {
|
||||
return fail(
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
PluginEffectObservation::Indeterminate,
|
||||
);
|
||||
}
|
||||
(None, None, None, Some(json))
|
||||
}
|
||||
};
|
||||
let changed = tx.execute(
|
||||
"UPDATE plugin_effect_receipts SET state='completed',response_status=?4,response_headers=?5,response_body=?6,error_json=?7 WHERE workspace_id=?1 AND invocation_id=?2 AND effect_key=?3 AND state='pending' AND fingerprint=?8",
|
||||
params![record.workspace_id, record.invocation_id, effect_key, status, headers, body, error, fingerprint],
|
||||
)?;
|
||||
if changed != 1 {
|
||||
return conflict();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Irreversible, idempotent cancellation. Does not claim to undo external effects.
|
||||
pub fn cancel(tx: &Transaction<'_>, workspace: &str, invocation_id: &str) -> Result<()> {
|
||||
lookup(tx, workspace, invocation_id)?;
|
||||
tx.execute(
|
||||
"UPDATE plugin_invocations SET cancelled=1 WHERE workspace_id=?1 AND invocation_id=?2 AND cancelled=0",
|
||||
params![workspace, invocation_id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Expire matching invocations and erase response/error payloads, never identity,
|
||||
/// fingerprints or keys. Returns the number of receipts newly tombstoned. Even a
|
||||
/// subsequent backwards clock cannot resurrect an explicitly expired invocation.
|
||||
/// Tombstones survive until the parent's Workspace deletion cascade.
|
||||
pub fn prune(tx: &Transaction<'_>, workspace: &str, now_ms: i64) -> Result<usize> {
|
||||
if !valid_identity(workspace, 512) || now_ms < 0 {
|
||||
return invalid();
|
||||
}
|
||||
// A local savepoint keeps expiry atomic if the owner handles an error and
|
||||
// commits unrelated work in its surrounding transaction.
|
||||
tx.execute_batch("SAVEPOINT plugin_receipt_prune")?;
|
||||
let work = (|| {
|
||||
tx.execute(
|
||||
"UPDATE plugin_invocations SET expired=1 WHERE workspace_id=?1 AND expires_at_ms<=?2 AND expired=0",
|
||||
params![workspace, now_ms],
|
||||
)?;
|
||||
Ok(tx.execute(
|
||||
"UPDATE plugin_effect_receipts SET state='expired',response_status=NULL,response_headers=NULL,response_body=NULL,error_json=NULL WHERE workspace_id=?1 AND state<>'expired' AND invocation_id IN (SELECT invocation_id FROM plugin_invocations WHERE workspace_id=?1 AND expired=1)",
|
||||
[workspace],
|
||||
)?)
|
||||
})();
|
||||
match work {
|
||||
Ok(count) => {
|
||||
tx.execute_batch("RELEASE plugin_receipt_prune")?;
|
||||
Ok(count)
|
||||
}
|
||||
Err(error) => {
|
||||
tx.execute_batch("ROLLBACK TO plugin_receipt_prune; RELEASE plugin_receipt_prune")?;
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,79 @@
|
||||
-- Durable Host identity and all-method HTTPS receipts. No request body, raw
|
||||
-- request headers, secret value, grant authority or catalog pin lives here.
|
||||
CREATE TABLE plugin_invocations (
|
||||
workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
|
||||
invocation_id TEXT NOT NULL CHECK(length(CAST(invocation_id AS BLOB)) BETWEEN 1 AND 512),
|
||||
plan_id TEXT NOT NULL,
|
||||
plan_fingerprint TEXT NOT NULL CHECK(length(plan_fingerprint)=71 AND substr(plan_fingerprint,1,7)='sha256:' AND substr(plan_fingerprint,8) NOT GLOB '*[^0-9a-f]*'),
|
||||
installation_id TEXT NOT NULL,
|
||||
runtime_id TEXT NOT NULL CHECK(length(CAST(runtime_id AS BLOB)) BETWEEN 1 AND 512),
|
||||
worker_id TEXT NOT NULL CHECK(length(CAST(worker_id AS BLOB)) BETWEEN 1 AND 512),
|
||||
execution_json TEXT NOT NULL CHECK(json_valid(execution_json) AND json_type(execution_json)='object' AND length(CAST(execution_json AS BLOB)) <= 16384),
|
||||
consumer_key TEXT NOT NULL CHECK(json_valid(consumer_key) AND json_type(consumer_key)='array' AND length(CAST(consumer_key AS BLOB)) <= 8192),
|
||||
consumer_json TEXT NOT NULL CHECK(json_valid(consumer_json) AND json_type(consumer_json)='object' AND length(CAST(consumer_json AS BLOB)) <= 16384),
|
||||
created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0),
|
||||
expires_at_ms INTEGER NOT NULL CHECK(expires_at_ms > created_at_ms),
|
||||
cancelled INTEGER NOT NULL DEFAULT 0 CHECK(cancelled IN (0,1)),
|
||||
expired INTEGER NOT NULL DEFAULT 0 CHECK(expired IN (0,1)),
|
||||
PRIMARY KEY(workspace_id, invocation_id),
|
||||
UNIQUE(workspace_id, runtime_id, worker_id, installation_id, consumer_key),
|
||||
FOREIGN KEY(workspace_id, plan_id) REFERENCES plugin_execution_plans(workspace_id, plan_id),
|
||||
FOREIGN KEY(workspace_id, installation_id) REFERENCES plugin_installations(workspace_id, installation_id)
|
||||
);
|
||||
CREATE INDEX plugin_invocation_expiry ON plugin_invocations(workspace_id, expires_at_ms) WHERE expired=0;
|
||||
CREATE TRIGGER plugin_invocation_immutable
|
||||
BEFORE UPDATE OF workspace_id,invocation_id,plan_id,plan_fingerprint,installation_id,runtime_id,worker_id,execution_json,consumer_key,consumer_json,created_at_ms,expires_at_ms ON plugin_invocations
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin invocation identity is immutable'); END;
|
||||
CREATE TRIGGER plugin_invocation_terminal BEFORE UPDATE OF cancelled,expired ON plugin_invocations
|
||||
WHEN NEW.cancelled < OLD.cancelled OR NEW.expired < OLD.expired
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin invocation cancellation and expiry are irreversible'); END;
|
||||
CREATE TRIGGER plugin_invocation_retained BEFORE DELETE ON plugin_invocations
|
||||
WHEN EXISTS(SELECT 1 FROM workspaces WHERE workspace_id=OLD.workspace_id)
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin invocation tombstone must be retained'); END;
|
||||
-- REPLACE must not bypass tombstones even with recursive_triggers disabled.
|
||||
CREATE TRIGGER plugin_invocation_no_replace BEFORE INSERT ON plugin_invocations
|
||||
WHEN EXISTS(SELECT 1 FROM plugin_invocations WHERE workspace_id=NEW.workspace_id AND
|
||||
(invocation_id=NEW.invocation_id OR (runtime_id=NEW.runtime_id AND worker_id=NEW.worker_id AND installation_id=NEW.installation_id AND consumer_key=NEW.consumer_key)))
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin invocation identity cannot be reused'); END;
|
||||
|
||||
CREATE TABLE plugin_effect_receipts (
|
||||
workspace_id TEXT NOT NULL REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
|
||||
invocation_id TEXT NOT NULL,
|
||||
effect_key TEXT NOT NULL CHECK(length(CAST(effect_key AS BLOB)) BETWEEN 1 AND 128),
|
||||
fingerprint TEXT NOT NULL CHECK(length(fingerprint)=71 AND substr(fingerprint,1,7)='sha256:' AND substr(fingerprint,8) NOT GLOB '*[^0-9a-f]*'),
|
||||
admitted_at_ms INTEGER NOT NULL CHECK(admitted_at_ms >= 0),
|
||||
state TEXT NOT NULL CHECK(state IN ('pending','completed','expired')),
|
||||
response_status INTEGER CHECK(response_status BETWEEN 100 AND 599),
|
||||
response_headers TEXT CHECK(json_valid(response_headers) AND json_type(response_headers)='array' AND json_array_length(response_headers) <= 128 AND length(CAST(response_headers AS BLOB)) <= 65536),
|
||||
response_body BLOB CHECK(response_body IS NULL OR (typeof(response_body)='blob' AND length(response_body) <= 1048576)),
|
||||
error_json TEXT CHECK(json_valid(error_json) AND json_type(error_json)='object' AND length(CAST(error_json AS BLOB)) <= 4096),
|
||||
CHECK(COALESCE(length(response_body),0)+COALESCE(length(CAST(response_headers AS BLOB)),0) <= 1114112),
|
||||
CHECK(
|
||||
(state='pending' AND response_status IS NULL AND response_headers IS NULL AND response_body IS NULL AND error_json IS NOT NULL AND COALESCE(json_extract(error_json,'$.code'),'')='indeterminate' AND COALESCE(json_extract(error_json,'$.effect'),'')='indeterminate') OR
|
||||
(state='completed' AND ((response_status IS NOT NULL AND response_headers IS NOT NULL AND response_body IS NOT NULL AND error_json IS NULL) OR
|
||||
(response_status IS NULL AND response_headers IS NULL AND response_body IS NULL AND error_json IS NOT NULL))) OR
|
||||
(state='expired' AND response_status IS NULL AND response_headers IS NULL AND response_body IS NULL AND error_json IS NULL)
|
||||
),
|
||||
PRIMARY KEY(workspace_id, invocation_id, effect_key),
|
||||
FOREIGN KEY(workspace_id, invocation_id) REFERENCES plugin_invocations(workspace_id, invocation_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TRIGGER plugin_effect_immutable BEFORE UPDATE OF workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms ON plugin_effect_receipts
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin effect identity is immutable'); END;
|
||||
CREATE TRIGGER plugin_effect_transition BEFORE UPDATE ON plugin_effect_receipts
|
||||
WHEN NOT (
|
||||
(OLD.state='pending' AND NEW.state='completed') OR
|
||||
(OLD.state IN ('pending','completed') AND NEW.state='expired' AND EXISTS(
|
||||
SELECT 1 FROM plugin_invocations WHERE workspace_id=OLD.workspace_id AND invocation_id=OLD.invocation_id AND expired=1))
|
||||
)
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin effect result is immutable'); END;
|
||||
CREATE TRIGGER plugin_effect_admission BEFORE INSERT ON plugin_effect_receipts
|
||||
WHEN NEW.state <> 'pending' OR NOT EXISTS(
|
||||
SELECT 1 FROM plugin_invocations WHERE workspace_id=NEW.workspace_id AND invocation_id=NEW.invocation_id
|
||||
AND cancelled=0 AND expired=0 AND expires_at_ms > NEW.admitted_at_ms)
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin effect invocation is not admissible'); END;
|
||||
CREATE TRIGGER plugin_effect_no_replace BEFORE INSERT ON plugin_effect_receipts
|
||||
WHEN EXISTS(SELECT 1 FROM plugin_effect_receipts WHERE workspace_id=NEW.workspace_id AND invocation_id=NEW.invocation_id AND effect_key=NEW.effect_key)
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin effect key cannot be reused'); END;
|
||||
CREATE TRIGGER plugin_effect_retained BEFORE DELETE ON plugin_effect_receipts
|
||||
WHEN EXISTS(SELECT 1 FROM workspaces WHERE workspace_id=OLD.workspace_id)
|
||||
BEGIN SELECT RAISE(ABORT,'Plugin effect tombstone must be retained'); END;
|
||||
@@ -0,0 +1,987 @@
|
||||
//! File-backed SQLite tests of the owner boundary, using production receipt,
|
||||
//! plan and catalog DDL. The minimal Workspace parent is a fixture, not a second
|
||||
//! migration path. Parent migration transition/rollback/retry tests live upstream.
|
||||
use super::*;
|
||||
use crate::plugin_plan::{
|
||||
ConfigProvenance, PlanPinState, PluginPinProjection, PluginSurface, ProfileProvenance,
|
||||
WorkerPluginExecutionPlan, WorkerPluginPin, WorkerPluginPlanInput,
|
||||
};
|
||||
use rusqlite::TransactionBehavior;
|
||||
use server_api::{PluginArtifact, PluginSchemaContribution, PluginToolMetadata};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
path::Path,
|
||||
sync::{
|
||||
Arc, Barrier,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tempfile::{TempDir, tempdir};
|
||||
|
||||
const NOW: i64 = 1000;
|
||||
const RETENTION: i64 = 500;
|
||||
const INSTALLATION: &str = "installation";
|
||||
|
||||
fn fingerprint(seed: &str) -> String {
|
||||
let hex: String = Sha256::digest(seed.as_bytes())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect();
|
||||
format!("sha256:{hex}")
|
||||
}
|
||||
fn consumer(id: &str) -> PluginConsumer {
|
||||
PluginConsumer::ToolCall {
|
||||
tool_call_id: id.into(),
|
||||
tool_name: "example_echo".into(),
|
||||
}
|
||||
}
|
||||
fn response() -> HttpResult {
|
||||
Ok(PluginHttpResponse {
|
||||
status: 503, // HTTP failure is still an observed response, not transport error.
|
||||
headers: vec![
|
||||
("x-result".into(), "one".into()),
|
||||
("x-result".into(), "two".into()),
|
||||
],
|
||||
body: vec![0, 255, 128, 13, 10],
|
||||
})
|
||||
}
|
||||
fn open(path: &Path) -> Connection {
|
||||
let conn = Connection::open(path).unwrap();
|
||||
conn.execute_batch("PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;")
|
||||
.unwrap();
|
||||
conn.busy_timeout(Duration::from_secs(10)).unwrap();
|
||||
conn
|
||||
}
|
||||
struct Fixture {
|
||||
dir: TempDir,
|
||||
plan: StoredPluginPlan,
|
||||
}
|
||||
impl Fixture {
|
||||
fn new() -> Self {
|
||||
let dir = tempdir().unwrap();
|
||||
let conn = open(&dir.path().join("receipts.db"));
|
||||
conn.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY); INSERT INTO workspaces VALUES('a'),('b');").unwrap();
|
||||
conn.execute_batch(include_str!("../plugin_catalog_schema.sql"))
|
||||
.unwrap();
|
||||
conn.execute_batch(crate::plugin_plan::SCHEMA_SQL).unwrap();
|
||||
conn.execute_batch(SCHEMA_SQL).unwrap();
|
||||
for ws in ["a", "b"] {
|
||||
conn.execute(
|
||||
"INSERT INTO plugin_installations VALUES(?1,?2,'alias','package','operation',0)",
|
||||
params![ws, INSTALLATION],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let plan = make_plan("plan", "config", "a", "runtime", "worker", "principal");
|
||||
save_plan(&conn, &plan);
|
||||
Self { dir, plan }
|
||||
}
|
||||
fn path(&self) -> std::path::PathBuf {
|
||||
self.dir.path().join("receipts.db")
|
||||
}
|
||||
fn conn(&self) -> Connection {
|
||||
open(&self.path())
|
||||
}
|
||||
fn begin(&self, id: &str) -> InvocationRecord {
|
||||
let mut conn = self.conn();
|
||||
let tx = conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let record = begin(&tx, &self.plan, INSTALLATION, &consumer(id), NOW, RETENTION).unwrap();
|
||||
tx.commit().unwrap();
|
||||
record
|
||||
}
|
||||
fn write<T>(&self, work: impl FnOnce(&Transaction<'_>) -> Result<T>) -> Result<T> {
|
||||
let mut conn = self.conn();
|
||||
let tx = conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let value = work(&tx)?;
|
||||
tx.commit().unwrap();
|
||||
Ok(value)
|
||||
}
|
||||
fn admit(
|
||||
&self,
|
||||
record: &InvocationRecord,
|
||||
key: &str,
|
||||
seed: &str,
|
||||
now: i64,
|
||||
) -> Result<EffectAdmission> {
|
||||
self.write(|tx| effect_admit(tx, record, key, &fingerprint(seed), now))
|
||||
}
|
||||
}
|
||||
fn make_plan(
|
||||
id: &str,
|
||||
config: &str,
|
||||
workspace: &str,
|
||||
runtime: &str,
|
||||
worker: &str,
|
||||
principal: &str,
|
||||
) -> StoredPluginPlan {
|
||||
let artifact = PluginArtifact {
|
||||
package_id: "package".into(),
|
||||
name: "package".into(),
|
||||
version: "1.0.0".into(),
|
||||
description: None,
|
||||
package_digest: fingerprint("package"),
|
||||
artifact_digest: fingerprint("artifact"),
|
||||
artifact_size: 1,
|
||||
schema_version: manifest::plugin::SUPPORTED_PLUGIN_API_VERSION,
|
||||
world: manifest::plugin::PLUGIN_COMPONENT_TOOL_WORLD.into(),
|
||||
required_imports: vec![],
|
||||
tools: vec![PluginToolMetadata {
|
||||
name: "example_echo".into(),
|
||||
description: "echo".into(),
|
||||
input_schema: serde_json::json!({}),
|
||||
output_schema: serde_json::json!({}),
|
||||
error_codes: vec![],
|
||||
external_write: false,
|
||||
}],
|
||||
connections: vec![],
|
||||
requested_grants: vec![],
|
||||
config_schema: PluginSchemaContribution {
|
||||
identity: "schema".into(),
|
||||
version: "1".into(),
|
||||
digest: fingerprint("schema"),
|
||||
namespace: "package".into(),
|
||||
dcdl: "".into(),
|
||||
},
|
||||
revoked: false,
|
||||
revocation_reason: None,
|
||||
};
|
||||
let pin = WorkerPluginPin::from_artifact(
|
||||
INSTALLATION.into(),
|
||||
"operation".into(),
|
||||
&artifact,
|
||||
PluginPinProjection {
|
||||
config: serde_json::json!({}),
|
||||
surfaces: vec![PluginSurface::Tool],
|
||||
tools: vec!["example_echo".into()],
|
||||
grants: vec![],
|
||||
bindings: vec![],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let plan = WorkerPluginExecutionPlan::canonicalize(WorkerPluginPlanInput {
|
||||
workspace_id: workspace.into(),
|
||||
config: ConfigProvenance {
|
||||
content_digest: fingerprint(config),
|
||||
toolchain_fingerprint: "toolchain".into(),
|
||||
},
|
||||
profile: ProfileProvenance {
|
||||
identity: "profile".into(),
|
||||
projection_digest: fingerprint("profile"),
|
||||
},
|
||||
execution: HostExecutionIdentity {
|
||||
runtime_id: runtime.into(),
|
||||
worker_id: worker.into(),
|
||||
principal_id: principal.into(),
|
||||
delegation_id: Some("delegation".into()),
|
||||
},
|
||||
plugins: vec![pin],
|
||||
})
|
||||
.unwrap();
|
||||
StoredPluginPlan {
|
||||
plan_id: id.into(),
|
||||
fingerprint: plan.fingerprint().unwrap(),
|
||||
pin_state: PlanPinState::Active,
|
||||
plan,
|
||||
}
|
||||
}
|
||||
fn save_plan(conn: &Connection, plan: &StoredPluginPlan) {
|
||||
conn.execute("INSERT INTO plugin_execution_plans(workspace_id,plan_id,actor,operation_key,fingerprint,plan_json,pin_state) VALUES(?1,?2,'fixture',?2,?3,?4,'active')",
|
||||
params![plan.plan.workspace_id(), plan.plan_id, plan.fingerprint, plan.plan.canonical_json().unwrap()]).unwrap();
|
||||
}
|
||||
fn error<T: std::fmt::Debug>(result: Result<T>, code: PluginHostErrorCode) -> PluginHostError {
|
||||
match result {
|
||||
Err(Error::PluginHost(error)) if error.code == code => error,
|
||||
other => panic!("expected {code:?}, got {other:?}"),
|
||||
}
|
||||
}
|
||||
fn replay_unknown(result: Result<EffectAdmission>) -> PluginHostError {
|
||||
match result.unwrap() {
|
||||
EffectAdmission::Replay(Err(error)) => {
|
||||
assert_eq!(error.code, PluginHostErrorCode::Indeterminate);
|
||||
assert_eq!(error.effect, PluginEffectObservation::Indeterminate);
|
||||
error
|
||||
}
|
||||
other => panic!("must never redispatch pending/unknown: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn begin_replay_reopen_preserves_uuid_and_original_deadline_and_conflicts_on_input() {
|
||||
let f = Fixture::new();
|
||||
let first = f.begin("call");
|
||||
uuid::Uuid::parse_str(&first.invocation_id).unwrap();
|
||||
let again = f
|
||||
.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&f.plan,
|
||||
INSTALLATION,
|
||||
&consumer("call"),
|
||||
NOW + 100,
|
||||
RETENTION * 10,
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(first, again);
|
||||
assert_eq!(first.expires_at_ms, NOW + RETENTION);
|
||||
assert_eq!(lookup(&f.conn(), "a", &first.invocation_id).unwrap(), first);
|
||||
error(
|
||||
lookup(&f.conn(), "b", &first.invocation_id),
|
||||
PluginHostErrorCode::NotFound,
|
||||
);
|
||||
let other_plan = make_plan(
|
||||
"different-plan",
|
||||
"config",
|
||||
"a",
|
||||
"runtime",
|
||||
"worker",
|
||||
"principal",
|
||||
);
|
||||
save_plan(&f.conn(), &other_plan);
|
||||
error(
|
||||
f.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&other_plan,
|
||||
INSTALLATION,
|
||||
&consumer("call"),
|
||||
NOW,
|
||||
RETENTION,
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
let other_input = make_plan(
|
||||
"changed-input",
|
||||
"new-config",
|
||||
"a",
|
||||
"runtime",
|
||||
"worker",
|
||||
"principal",
|
||||
);
|
||||
save_plan(&f.conn(), &other_input);
|
||||
error(
|
||||
f.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&other_input,
|
||||
INSTALLATION,
|
||||
&consumer("call"),
|
||||
NOW,
|
||||
RETENTION,
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
let renamed = PluginConsumer::ToolCall {
|
||||
tool_call_id: "call".into(),
|
||||
tool_name: "different".into(),
|
||||
};
|
||||
error(
|
||||
f.write(|tx| begin(tx, &f.plan, INSTALLATION, &renamed, NOW, RETENTION)),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
let changed_execution = make_plan(
|
||||
"principal-change",
|
||||
"config",
|
||||
"a",
|
||||
"runtime",
|
||||
"worker",
|
||||
"other-principal",
|
||||
);
|
||||
save_plan(&f.conn(), &changed_execution);
|
||||
error(
|
||||
f.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&changed_execution,
|
||||
INSTALLATION,
|
||||
&consumer("call"),
|
||||
NOW,
|
||||
RETENTION,
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
assert_ne!(f.begin("new-call").invocation_id, first.invocation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identities_isolate_workspaces_runtimes_workers_installations_and_consumers() {
|
||||
let f = Fixture::new();
|
||||
let original = f.begin("same-id");
|
||||
let mut ids = std::collections::BTreeSet::from([original.invocation_id]);
|
||||
for (id, ws, runtime, worker) in [
|
||||
("other-workspace", "b", "runtime", "worker"),
|
||||
("other-runtime", "a", "runtime-2", "worker"),
|
||||
("other-worker", "a", "runtime", "worker-2"),
|
||||
] {
|
||||
let plan = make_plan(id, "config", ws, runtime, worker, "principal");
|
||||
save_plan(&f.conn(), &plan);
|
||||
let record = f
|
||||
.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&plan,
|
||||
INSTALLATION,
|
||||
&consumer("same-id"),
|
||||
NOW,
|
||||
RETENTION,
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
assert!(ids.insert(record.invocation_id));
|
||||
}
|
||||
// Another immutable installation selection gets an independent namespace.
|
||||
let mut plan_json = serde_json::to_value(&f.plan.plan).unwrap();
|
||||
plan_json["input"]["plugins"][0]["installation_id"] = serde_json::json!("other-installation");
|
||||
let other_snapshot: WorkerPluginExecutionPlan = serde_json::from_value(plan_json).unwrap();
|
||||
let other = StoredPluginPlan {
|
||||
plan_id: "other-installation-plan".into(),
|
||||
fingerprint: other_snapshot.fingerprint().unwrap(),
|
||||
pin_state: PlanPinState::Active,
|
||||
plan: other_snapshot,
|
||||
};
|
||||
f.conn().execute("INSERT INTO plugin_installations VALUES('a','other-installation','other-alias','package','operation',0)", []).unwrap();
|
||||
save_plan(&f.conn(), &other);
|
||||
let record = f
|
||||
.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&other,
|
||||
"other-installation",
|
||||
&consumer("same-id"),
|
||||
NOW,
|
||||
RETENTION,
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
assert!(ids.insert(record.invocation_id));
|
||||
for consumer_id in ["consumer-1", "consumer-2"] {
|
||||
let c = PluginConsumer::Consumer {
|
||||
consumer_id: consumer_id.into(),
|
||||
event_id: "same-id".into(),
|
||||
export_name: "export".into(),
|
||||
};
|
||||
let record = f
|
||||
.write(|tx| begin(tx, &f.plan, INSTALLATION, &c, NOW, RETENTION))
|
||||
.unwrap();
|
||||
assert!(ids.insert(record.invocation_id.clone()));
|
||||
assert_eq!(
|
||||
f.write(|tx| begin(tx, &f.plan, INSTALLATION, &c, NOW + 5, RETENTION))
|
||||
.unwrap(),
|
||||
record
|
||||
);
|
||||
let renamed = PluginConsumer::Consumer {
|
||||
consumer_id: consumer_id.into(),
|
||||
event_id: "same-id".into(),
|
||||
export_name: "other".into(),
|
||||
};
|
||||
error(
|
||||
f.write(|tx| begin(tx, &f.plan, INSTALLATION, &renamed, NOW, RETENTION)),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
}
|
||||
// Missing installation cannot open a namespace absent from the immutable plan.
|
||||
error(
|
||||
f.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&f.plan,
|
||||
"not-in-plan",
|
||||
&consumer("same-id"),
|
||||
NOW,
|
||||
RETENTION,
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_connections_racing_admission_commit_exactly_one_dispatch() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("race");
|
||||
let barrier = Arc::new(Barrier::new(2));
|
||||
let sends = Arc::new(AtomicUsize::new(0));
|
||||
let handles: Vec<_> = (0..2)
|
||||
.map(|_| {
|
||||
let (path, record, barrier, sends) =
|
||||
(f.path(), record.clone(), barrier.clone(), sends.clone());
|
||||
std::thread::spawn(move || {
|
||||
let mut conn = open(&path);
|
||||
barrier.wait();
|
||||
let tx = conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
let admission =
|
||||
effect_admit(&tx, &record, "GET-key", &fingerprint("GET /"), NOW).unwrap();
|
||||
tx.commit().unwrap();
|
||||
if admission == EffectAdmission::Dispatch {
|
||||
sends.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
admission
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let outcomes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
|
||||
assert_eq!(sends.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
outcomes
|
||||
.iter()
|
||||
.filter(|a| **a == EffectAdmission::Dispatch)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert!(outcomes.iter().any(|a| matches!(a, EffectAdmission::Replay(Err(error)) if error.code == PluginHostErrorCode::Indeterminate)));
|
||||
error(
|
||||
f.admit(&record, "GET-key", "POST /", NOW),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crash_after_durable_admission_reopens_pending_with_zero_redispatch() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("crash");
|
||||
let sends = AtomicUsize::new(0);
|
||||
if f.admit(&record, "key", "request", NOW).unwrap() == EffectAdmission::Dispatch {
|
||||
sends.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
// All helpers close their connections. Simulate crash before any completion.
|
||||
let saved_json: String = f
|
||||
.conn()
|
||||
.query_row("SELECT error_json FROM plugin_effect_receipts", [], |r| {
|
||||
r.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
let saved: PluginHostError = serde_json::from_str(&saved_json).unwrap();
|
||||
for _ in 0..3 {
|
||||
let recovered = lookup(&f.conn(), "a", &record.invocation_id).unwrap();
|
||||
assert_eq!(
|
||||
replay_unknown(f.admit(&recovered, "key", "request", NOW + 1)),
|
||||
saved
|
||||
);
|
||||
}
|
||||
assert_eq!(sends.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_replays_exact_binary_response_headers_and_http_status_without_dispatch() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("response");
|
||||
assert_eq!(
|
||||
f.admit(&record, "key", "request", NOW).unwrap(),
|
||||
EffectAdmission::Dispatch
|
||||
);
|
||||
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &response()))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
f.admit(&record, "key", "request", NOW).unwrap(),
|
||||
EffectAdmission::Replay(response())
|
||||
);
|
||||
// Successful completion replay is a read, not an overwrite.
|
||||
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &response()))
|
||||
.unwrap();
|
||||
error(
|
||||
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("different"), &response())),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
let changed = Ok(PluginHttpResponse {
|
||||
status: 200,
|
||||
headers: vec![],
|
||||
body: vec![],
|
||||
});
|
||||
error(
|
||||
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &changed)),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
error(
|
||||
f.write(|tx| {
|
||||
effect_complete(
|
||||
tx,
|
||||
&record,
|
||||
"never-admitted",
|
||||
&fingerprint("request"),
|
||||
&changed,
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::NotFound,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_saves_transport_failure_for_each_observation_and_never_redispatches() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("errors");
|
||||
for (key, observation) in [
|
||||
("before", PluginEffectObservation::NotDispatched),
|
||||
("after", PluginEffectObservation::Indeterminate),
|
||||
("observed", PluginEffectObservation::ResponseObserved),
|
||||
] {
|
||||
assert_eq!(
|
||||
f.admit(&record, key, key, NOW).unwrap(),
|
||||
EffectAdmission::Dispatch
|
||||
);
|
||||
let failure = Err(PluginHostError {
|
||||
code: PluginHostErrorCode::DeadlineExceeded,
|
||||
effect: observation,
|
||||
diagnostic_id: format!("diagnostic-{key}"),
|
||||
});
|
||||
f.write(|tx| effect_complete(tx, &record, key, &fingerprint(key), &failure))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
f.admit(&record, key, key, NOW + 1).unwrap(),
|
||||
EffectAdmission::Replay(failure)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_is_irreversible_and_stale_records_cannot_bypass_it() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("cancel");
|
||||
assert_eq!(
|
||||
f.admit(&record, "in-flight", "request", NOW).unwrap(),
|
||||
EffectAdmission::Dispatch
|
||||
);
|
||||
f.write(|tx| cancel(tx, "a", &record.invocation_id))
|
||||
.unwrap();
|
||||
f.write(|tx| cancel(tx, "a", &record.invocation_id))
|
||||
.unwrap();
|
||||
assert!(
|
||||
lookup(&f.conn(), "a", &record.invocation_id)
|
||||
.unwrap()
|
||||
.cancelled
|
||||
);
|
||||
for key in ["in-flight", "new-key"] {
|
||||
let e = error(
|
||||
f.admit(&record, key, "request", NOW),
|
||||
PluginHostErrorCode::Cancelled,
|
||||
);
|
||||
assert_eq!(e.effect, PluginEffectObservation::Indeterminate);
|
||||
}
|
||||
// Recording an observed in-flight response does not publish it after cancel.
|
||||
f.write(|tx| {
|
||||
effect_complete(
|
||||
tx,
|
||||
&record,
|
||||
"in-flight",
|
||||
&fingerprint("request"),
|
||||
&response(),
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
error(
|
||||
f.admit(&record, "in-flight", "request", NOW),
|
||||
PluginHostErrorCode::Cancelled,
|
||||
);
|
||||
assert!(
|
||||
f.conn()
|
||||
.execute("UPDATE plugin_invocations SET cancelled=0", [])
|
||||
.is_err()
|
||||
);
|
||||
assert!(f.begin("cancel").cancelled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deadline_expiry_refuses_old_and_new_keys_and_prune_preserves_tombstones() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("expire");
|
||||
let untouched = f.begin("no-effects");
|
||||
for key in ["complete", "pending"] {
|
||||
assert_eq!(
|
||||
f.admit(&record, key, "request", NOW).unwrap(),
|
||||
EffectAdmission::Dispatch
|
||||
);
|
||||
}
|
||||
f.write(|tx| {
|
||||
effect_complete(
|
||||
tx,
|
||||
&record,
|
||||
"complete",
|
||||
&fingerprint("request"),
|
||||
&response(),
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
for key in ["complete", "pending", "unused"] {
|
||||
error(
|
||||
f.admit(&record, key, "request", NOW + RETENTION),
|
||||
PluginHostErrorCode::ReceiptExpired,
|
||||
);
|
||||
}
|
||||
error(
|
||||
f.admit(&untouched, "unused", "request", NOW + RETENTION),
|
||||
PluginHostErrorCode::ReceiptExpired,
|
||||
);
|
||||
let recovered = f
|
||||
.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&f.plan,
|
||||
INSTALLATION,
|
||||
&consumer("expire"),
|
||||
NOW + RETENTION + 10,
|
||||
RETENTION * 100,
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(record, recovered);
|
||||
assert_eq!(f.write(|tx| prune(tx, "a", NOW + RETENTION)).unwrap(), 2);
|
||||
assert_eq!(f.write(|tx| prune(tx, "a", NOW + RETENTION)).unwrap(), 0);
|
||||
let conn = f.conn();
|
||||
let counts: (i64, i64, i64) = conn.query_row("SELECT count(*),sum(state='expired'),sum(response_body IS NULL AND response_headers IS NULL AND error_json IS NULL) FROM plugin_effect_receipts", [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))).unwrap();
|
||||
assert_eq!(counts, (2, 2, 2));
|
||||
// Explicit pruning is monotonic even if the caller's clock moves backwards.
|
||||
error(
|
||||
f.admit(&record, "unused", "request", NOW),
|
||||
PluginHostErrorCode::ReceiptExpired,
|
||||
);
|
||||
error(
|
||||
f.admit(&untouched, "unused", "request", NOW),
|
||||
PluginHostErrorCode::ReceiptExpired,
|
||||
);
|
||||
error(
|
||||
f.write(|tx| {
|
||||
effect_complete(
|
||||
tx,
|
||||
&record,
|
||||
"complete",
|
||||
&fingerprint("request"),
|
||||
&response(),
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::ReceiptExpired,
|
||||
);
|
||||
assert!(
|
||||
conn.execute("UPDATE plugin_invocations SET expired=0", [])
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
lookup(&conn, "a", &record.invocation_id).unwrap(),
|
||||
InvocationRecord {
|
||||
expired: true,
|
||||
..record
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
conn.execute("DELETE FROM plugin_effect_receipts", [])
|
||||
.is_err()
|
||||
);
|
||||
assert!(conn.execute("DELETE FROM plugin_invocations", []).is_err());
|
||||
assert!(
|
||||
conn.execute("UPDATE plugin_effect_receipts SET state='pending'", [])
|
||||
.is_err()
|
||||
);
|
||||
// Workspace deletion is the only identity/key purge. All FK checks remain on.
|
||||
conn.execute("DELETE FROM workspaces WHERE workspace_id='a'", [])
|
||||
.unwrap();
|
||||
for table in ["plugin_invocations", "plugin_effect_receipts"] {
|
||||
let count: i64 = conn
|
||||
.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
let fk_errors: i64 = conn
|
||||
.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| {
|
||||
r.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(fk_errors, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_and_key_bounds_reject_without_erasing_indeterminate_receipt() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("bounds");
|
||||
assert_eq!(
|
||||
f.admit(&record, "key", "request", NOW).unwrap(),
|
||||
EffectAdmission::Dispatch
|
||||
);
|
||||
for body_len in [MAX_RESPONSE_BYTES + 1] {
|
||||
let big = Ok(PluginHttpResponse {
|
||||
status: 200,
|
||||
headers: vec![],
|
||||
body: vec![255; body_len],
|
||||
});
|
||||
error(
|
||||
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &big)),
|
||||
PluginHostErrorCode::QuotaExceeded,
|
||||
);
|
||||
replay_unknown(f.admit(&record, "key", "request", NOW));
|
||||
}
|
||||
let too_many_headers = Ok(PluginHttpResponse {
|
||||
status: 200,
|
||||
headers: vec![("x".into(), "y".into()); MAX_RESPONSE_HEADERS + 1],
|
||||
body: vec![],
|
||||
});
|
||||
error(
|
||||
f.write(|tx| {
|
||||
effect_complete(
|
||||
tx,
|
||||
&record,
|
||||
"key",
|
||||
&fingerprint("request"),
|
||||
&too_many_headers,
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::QuotaExceeded,
|
||||
);
|
||||
let big_headers = Ok(PluginHttpResponse {
|
||||
status: 200,
|
||||
headers: vec![("x".into(), "y".repeat(MAX_RESPONSE_HEADER_BYTES))],
|
||||
body: vec![],
|
||||
});
|
||||
error(
|
||||
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &big_headers)),
|
||||
PluginHostErrorCode::QuotaExceeded,
|
||||
);
|
||||
for key in [
|
||||
"".to_owned(),
|
||||
"x".repeat(129),
|
||||
"é".repeat(65),
|
||||
"bad\0key".into(),
|
||||
] {
|
||||
error(
|
||||
f.admit(&record, &key, "request", NOW),
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
);
|
||||
}
|
||||
error(
|
||||
f.write(|tx| effect_admit(tx, &record, "key", "raw request must not be stored", NOW)),
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
);
|
||||
let max = Ok(PluginHttpResponse {
|
||||
status: 200,
|
||||
headers: vec![],
|
||||
body: vec![255; MAX_RESPONSE_BYTES],
|
||||
});
|
||||
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &max))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
f.admit(&record, "key", "request", NOW).unwrap(),
|
||||
EffectAdmission::Replay(max)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_guards_identity_result_overwrites_replace_and_unadmitted_completion() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("guards");
|
||||
f.admit(&record, "key", "request", NOW).unwrap();
|
||||
f.write(|tx| effect_complete(tx, &record, "key", &fingerprint("request"), &response()))
|
||||
.unwrap();
|
||||
let conn = f.conn();
|
||||
// REPLACE protection must not depend on PRAGMA recursive_triggers.
|
||||
conn.execute_batch("PRAGMA recursive_triggers=OFF;")
|
||||
.unwrap();
|
||||
for sql in [
|
||||
"DELETE FROM plugin_invocations",
|
||||
"DELETE FROM plugin_effect_receipts",
|
||||
"UPDATE plugin_invocations SET invocation_id='replacement'",
|
||||
"UPDATE plugin_invocations SET expires_at_ms=99999",
|
||||
"UPDATE plugin_invocations SET consumer_json='{}'",
|
||||
"UPDATE plugin_effect_receipts SET fingerprint='sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'",
|
||||
"UPDATE plugin_effect_receipts SET response_body=X'01'",
|
||||
"UPDATE plugin_effect_receipts SET state='pending',response_status=NULL,response_headers=NULL,response_body=NULL,error_json='{}'",
|
||||
"UPDATE plugin_effect_receipts SET state='expired',response_status=NULL,response_headers=NULL,response_body=NULL,error_json=NULL",
|
||||
"INSERT OR REPLACE INTO plugin_invocations SELECT * FROM plugin_invocations",
|
||||
"INSERT OR REPLACE INTO plugin_effect_receipts SELECT * FROM plugin_effect_receipts",
|
||||
"INSERT INTO plugin_effect_receipts SELECT workspace_id,invocation_id,'new-key',fingerprint,admitted_at_ms,state,response_status,response_headers,response_body,error_json FROM plugin_effect_receipts",
|
||||
] {
|
||||
assert!(conn.execute(sql, []).is_err(), "must reject {sql}");
|
||||
}
|
||||
let mut forged = record.clone();
|
||||
forged.expires_at_ms += 1000;
|
||||
error(
|
||||
f.admit(&forged, "new-key", "request", NOW),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
forged = record.clone();
|
||||
forged.execution.worker_id = "another-worker".into();
|
||||
error(
|
||||
f.admit(&forged, "new-key", "request", NOW),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
assert_eq!(
|
||||
f.admit(&record, "key", "request", NOW).unwrap(),
|
||||
EffectAdmission::Replay(response())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uncommitted_admission_rolls_back_and_failed_prune_is_atomic() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("rollback");
|
||||
{
|
||||
let mut conn = f.conn();
|
||||
let tx = conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
effect_admit(&tx, &record, "key", &fingerprint("request"), NOW).unwrap(),
|
||||
EffectAdmission::Dispatch
|
||||
);
|
||||
// No send: commit was not successful. Transaction rollback is safe here.
|
||||
}
|
||||
assert_eq!(
|
||||
f.admit(&record, "key", "request", NOW).unwrap(),
|
||||
EffectAdmission::Dispatch
|
||||
);
|
||||
let mut conn = f.conn();
|
||||
conn.execute_batch("CREATE TRIGGER refuse_prune BEFORE UPDATE ON plugin_effect_receipts WHEN NEW.state='expired' BEGIN SELECT RAISE(ABORT,'injected'); END;").unwrap();
|
||||
let tx = conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)
|
||||
.unwrap();
|
||||
assert!(prune(&tx, "a", NOW + RETENTION).is_err());
|
||||
tx.commit().unwrap();
|
||||
let expired: bool = conn
|
||||
.query_row("SELECT expired FROM plugin_invocations", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert!(
|
||||
!expired,
|
||||
"caller catching an error must not commit partial expiry"
|
||||
);
|
||||
replay_unknown(f.admit(&record, "key", "request", NOW));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_begin_does_not_create_identity_or_overflow_deadline() {
|
||||
let f = Fixture::new();
|
||||
for (now, retention) in [(-1, 1), (NOW, 0), (NOW, -1), (i64::MAX, 1)] {
|
||||
error(
|
||||
f.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&f.plan,
|
||||
INSTALLATION,
|
||||
&consumer("invalid"),
|
||||
now,
|
||||
retention,
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
);
|
||||
}
|
||||
let invalid = PluginConsumer::ToolCall {
|
||||
tool_call_id: "".into(),
|
||||
tool_name: "tool".into(),
|
||||
};
|
||||
error(
|
||||
f.write(|tx| begin(tx, &f.plan, INSTALLATION, &invalid, NOW, RETENTION)),
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
);
|
||||
let mut plan = f.plan.clone();
|
||||
plan.fingerprint = fingerprint("different");
|
||||
error(
|
||||
f.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&plan,
|
||||
INSTALLATION,
|
||||
&consumer("invalid"),
|
||||
NOW,
|
||||
RETENTION,
|
||||
)
|
||||
}),
|
||||
PluginHostErrorCode::InvalidInput,
|
||||
);
|
||||
let count: i64 = f
|
||||
.conn()
|
||||
.query_row("SELECT count(*) FROM plugin_invocations", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_http_methods_including_get_and_head_have_the_same_one_shot_receipt_semantics() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("all-methods");
|
||||
let sends = AtomicUsize::new(0);
|
||||
for method in ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] {
|
||||
if f.admit(&record, method, method, NOW).unwrap() == EffectAdmission::Dispatch {
|
||||
sends.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
replay_unknown(f.admit(&record, method, method, NOW));
|
||||
error(
|
||||
f.admit(&record, method, "changed-method-or-body", NOW),
|
||||
PluginHostErrorCode::Conflict,
|
||||
);
|
||||
f.write(|tx| effect_complete(tx, &record, method, &fingerprint(method), &response()))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
f.admit(&record, method, method, NOW).unwrap(),
|
||||
EffectAdmission::Replay(response())
|
||||
);
|
||||
}
|
||||
assert_eq!(sends.load(Ordering::SeqCst), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_checks_initial_indeterminate_result_and_bounded_payloads_independently_of_rust() {
|
||||
let f = Fixture::new();
|
||||
let record = f.begin("sql-checks");
|
||||
let conn = f.conn();
|
||||
for json in [
|
||||
"{}",
|
||||
r#"{"code":"unavailable","effect":"not_dispatched","diagnostic_id":"d"}"#,
|
||||
] {
|
||||
assert!(conn.execute("INSERT INTO plugin_effect_receipts(workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms,state,error_json) VALUES('a',?1,'invalid-pending',?2,1000,'pending',?3)", params![record.invocation_id, fingerprint("request"), json]).is_err());
|
||||
}
|
||||
assert!(conn.execute("INSERT INTO plugin_effect_receipts(workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms,state,error_json) VALUES('a',?1,'expired-admission',?2,1500,'pending',?3)", params![record.invocation_id, fingerprint("request"), encode(&host_error(PluginHostErrorCode::Indeterminate, PluginEffectObservation::Indeterminate)).unwrap()]).is_err());
|
||||
f.admit(&record, "key", "request", NOW).unwrap();
|
||||
for sql in [
|
||||
"UPDATE plugin_effect_receipts SET state='completed',response_status=200,response_headers='[]',response_body=zeroblob(1048577),error_json=NULL",
|
||||
"UPDATE plugin_effect_receipts SET state='completed',response_status=200,response_headers='[]',response_body='not-a-blob',error_json=NULL",
|
||||
"UPDATE plugin_effect_receipts SET state='completed',response_status=99,response_headers='[]',response_body=X'',error_json=NULL",
|
||||
"UPDATE plugin_effect_receipts SET state='completed',response_status=200,response_headers='{}',response_body=X'',error_json=NULL",
|
||||
] {
|
||||
assert!(conn.execute(sql, []).is_err(), "must reject {sql}");
|
||||
}
|
||||
replay_unknown(f.admit(&record, "key", "request", NOW));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pruning_one_workspace_does_not_remove_another_workspaces_saved_result() {
|
||||
let f = Fixture::new();
|
||||
let a = f.begin("prune-scope");
|
||||
let plan_b = make_plan("plan-b", "config", "b", "runtime", "worker", "principal");
|
||||
save_plan(&f.conn(), &plan_b);
|
||||
let b = f
|
||||
.write(|tx| {
|
||||
begin(
|
||||
tx,
|
||||
&plan_b,
|
||||
INSTALLATION,
|
||||
&consumer("prune-scope"),
|
||||
NOW,
|
||||
RETENTION,
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
for record in [&a, &b] {
|
||||
f.admit(record, "key", "request", NOW).unwrap();
|
||||
f.write(|tx| effect_complete(tx, record, "key", &fingerprint("request"), &response()))
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(f.write(|tx| prune(tx, "a", NOW + RETENTION)).unwrap(), 1);
|
||||
assert_eq!(
|
||||
f.admit(&b, "key", "request", NOW).unwrap(),
|
||||
EffectAdmission::Replay(response())
|
||||
);
|
||||
error(
|
||||
f.admit(&a, "key", "request", NOW),
|
||||
PluginHostErrorCode::ReceiptExpired,
|
||||
);
|
||||
// Unexpired and pending invocations are not accidentally pruned early.
|
||||
assert_eq!(
|
||||
f.write(|tx| prune(tx, "b", NOW + RETENTION - 1)).unwrap(),
|
||||
0
|
||||
);
|
||||
}
|
||||
@@ -112,6 +112,10 @@ mod schema_convergence {
|
||||
version: 108,
|
||||
name: "Subject Ticket responsibility and current-body notification routing".into(),
|
||||
},
|
||||
WorkspaceSchemaMigrationStep {
|
||||
version: 109,
|
||||
name: "Durable Plugin Host invocations and HTTPS effect receipts".into(),
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -131,6 +135,7 @@ mod schema_convergence {
|
||||
expected.push((106, crate::plugin_catalog::MIGRATION_NAME.into()));
|
||||
expected.push((107, "Immutable Worker Plugin plans and retained pins".into()));
|
||||
expected.push((108, "Subject Ticket responsibility and current-body notification routing".into()));
|
||||
expected.push((109, "Durable Plugin Host invocations and HTTPS effect receipts".into()));
|
||||
assert_eq!(workspace_schema_migration_history(conn)?, expected);
|
||||
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
|
||||
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
|
||||
|
||||
@@ -48,6 +48,8 @@ mod drive;
|
||||
mod drive_grants;
|
||||
#[path = "server_plugin.rs"]
|
||||
mod plugin;
|
||||
#[path = "server_plugin_host.rs"]
|
||||
mod plugin_host;
|
||||
#[path = "server_workspace_config.rs"]
|
||||
mod workspace_config;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
@@ -1999,6 +2001,8 @@ pub struct WorkspaceApi {
|
||||
workdir_remove_attempt_owner: WorkdirRemovalAttemptOwner,
|
||||
worker_control_locks: Arc<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>>,
|
||||
attachment_upload_grants: Arc<Mutex<HashMap<String, AttachmentUploadGrant>>>,
|
||||
#[cfg(test)]
|
||||
plugin_https_transport: Option<Arc<crate::plugin_https::HttpsTransport>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -4510,6 +4514,8 @@ impl WorkspaceApi {
|
||||
workdir_remove_attempt_owner: current_workdir_removal_attempt_owner()?,
|
||||
worker_control_locks: Arc::new(Mutex::new(HashMap::new())),
|
||||
attachment_upload_grants: Arc::new(Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
plugin_https_transport: None,
|
||||
};
|
||||
// Restore the domain fence before this API can issue any Drive handle,
|
||||
// including blocked/failed deletions not scheduled for automatic recovery.
|
||||
@@ -7646,6 +7652,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou
|
||||
service.clone(),
|
||||
))
|
||||
.merge(plugin::router(service.clone()))
|
||||
.merge(plugin_host::router(service.clone()))
|
||||
.merge(server_api::server_api_axum::current_worker_workspace_config_get(service.clone()))
|
||||
.merge(server_api::server_api_axum::current_worker_workspace_config_attach(service.clone()))
|
||||
.merge(
|
||||
@@ -13488,6 +13495,22 @@ impl server_api::ServerApi for ServerApiContractService {
|
||||
.map_err(ApiError::into_repository_api_error)
|
||||
}
|
||||
|
||||
async fn plugin_host(
|
||||
&self,
|
||||
context: server_api::ServerRequestContext,
|
||||
workspace_id: String,
|
||||
request: server_api::PluginHostRequest,
|
||||
) -> std::result::Result<server_api::PluginHostResponse, server_api::PluginHostError> {
|
||||
let api = self
|
||||
.workspace_api()
|
||||
.map_err(|_| server_api::PluginHostError {
|
||||
code: server_api::PluginHostErrorCode::Unavailable,
|
||||
effect: server_api::PluginEffectObservation::NotDispatched,
|
||||
diagnostic_id: String::new(),
|
||||
})?;
|
||||
plugin_host::execute(Arc::new(api.clone()), context, workspace_id, request).await
|
||||
}
|
||||
|
||||
async fn plugin_upload(
|
||||
&self,
|
||||
actor: RequestActor,
|
||||
@@ -38643,6 +38666,9 @@ mod tests {
|
||||
mod plugin_tests {
|
||||
include!("server_plugin_tests.rs");
|
||||
}
|
||||
mod plugin_host_tests {
|
||||
include!("server_plugin_host_tests.rs");
|
||||
}
|
||||
|
||||
mod ticket_tags_tests {
|
||||
include!("server_ticket_tags_tests.rs");
|
||||
|
||||
@@ -0,0 +1,585 @@
|
||||
//! Both authenticated Remote requests and Embedded Host calls enter this owner.
|
||||
//! It does not trust human catalog credentials or accept execution identities in
|
||||
//! JSON. Network dispatch is single-attempt and follows durable effect admission.
|
||||
use super::*;
|
||||
use crate::{plugin_host as policy, plugin_https as https, plugin_receipt as receipt};
|
||||
use server_api::*;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const RECEIPT_RETENTION_MS: i64 = 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
pub(super) fn router(service: Arc<ServerApiContractService>) -> Router {
|
||||
server_api_axum::plugin_host(service).layer(DefaultBodyLimit::max(2 * 1024 * 1024))
|
||||
}
|
||||
|
||||
fn denied() -> PluginHostError {
|
||||
policy::map_error(policy::failure(PluginHostErrorCode::Denied))
|
||||
}
|
||||
fn now_ms() -> i64 {
|
||||
chrono::Utc::now().timestamp_millis()
|
||||
}
|
||||
fn operation_name(op: &PluginHostOperation) -> &'static str {
|
||||
match op {
|
||||
PluginHostOperation::Admit => "plugin.admit",
|
||||
PluginHostOperation::Publish => "plugin.publish",
|
||||
PluginHostOperation::Open { .. } => "plugin.connection.open",
|
||||
PluginHostOperation::Send { .. } => "plugin.http.send",
|
||||
PluginHostOperation::Log => "plugin.log",
|
||||
}
|
||||
}
|
||||
|
||||
async fn check(
|
||||
api: Arc<WorkspaceApi>,
|
||||
context: ServerRequestContext,
|
||||
workspace: String,
|
||||
plan_id: String,
|
||||
installation: String,
|
||||
consumer: PluginConsumer,
|
||||
operation: PluginHostOperation,
|
||||
) -> std::result::Result<policy::AdmittedPluginOperation, PluginHostError> {
|
||||
validate_workspace_scope(&api, &workspace).map_err(|e| policy::map_error(e.error))?;
|
||||
let worker = subject_authorization::source_from_context(&context)
|
||||
.map_err(|e| policy::map_error(e.error))?;
|
||||
let execution = api
|
||||
.plugin_execution_identity(&worker)
|
||||
.await
|
||||
.map_err(policy::map_error)?;
|
||||
let executor = api.config_store.db_executor();
|
||||
executor
|
||||
.run(move || {
|
||||
subject_authorization::authorize_scope(
|
||||
&api,
|
||||
&worker,
|
||||
&server_api::authorization::OperationAuthorizationRequest {
|
||||
operation: operation_name(&operation).into(),
|
||||
resource: server_api::authorization::PermissionResource::PluginInstallation {
|
||||
workspace_id: workspace.clone(),
|
||||
installation_id: installation.clone(),
|
||||
},
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.error)?;
|
||||
api.config_store.with_immediate_transaction(|tx| {
|
||||
policy::authorize(
|
||||
tx,
|
||||
&workspace,
|
||||
&execution,
|
||||
&plan_id,
|
||||
&installation,
|
||||
&consumer,
|
||||
&operation,
|
||||
)
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(policy::map_error)
|
||||
}
|
||||
|
||||
async fn invocation(
|
||||
api: Arc<WorkspaceApi>,
|
||||
context: &ServerRequestContext,
|
||||
workspace: &str,
|
||||
invocation_id: &str,
|
||||
operation: PluginHostOperation,
|
||||
) -> std::result::Result<
|
||||
(receipt::InvocationRecord, policy::AdmittedPluginOperation),
|
||||
PluginHostError,
|
||||
> {
|
||||
let store = api.config_store.clone();
|
||||
let w = workspace.to_owned();
|
||||
let id = invocation_id.to_owned();
|
||||
let record = store
|
||||
.db_executor()
|
||||
.run(move || store.with_conn(|conn| receipt::lookup(conn, &w, &id)))
|
||||
.await
|
||||
.map_err(policy::map_error)?;
|
||||
let admitted = check(
|
||||
api,
|
||||
context.clone(),
|
||||
workspace.into(),
|
||||
record.plan_id.clone(),
|
||||
record.installation_id.clone(),
|
||||
record.consumer.clone(),
|
||||
operation,
|
||||
)
|
||||
.await?;
|
||||
if record.execution != *admitted.plan.plan.execution() {
|
||||
return Err(denied());
|
||||
}
|
||||
if record.cancelled {
|
||||
return Err(PluginHostError {
|
||||
code: PluginHostErrorCode::Cancelled,
|
||||
effect: PluginEffectObservation::Indeterminate,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
});
|
||||
}
|
||||
if record.expired || record.expires_at_ms <= now_ms() {
|
||||
return Err(PluginHostError {
|
||||
code: PluginHostErrorCode::ReceiptExpired,
|
||||
effect: PluginEffectObservation::Indeterminate,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
});
|
||||
}
|
||||
Ok((record, admitted))
|
||||
}
|
||||
|
||||
/// An Embedded transport retains this context only after the ordinary Runtime
|
||||
/// source proof verifier has established it. Merely holding WorkspaceApi or human
|
||||
/// management access does not produce a Host adapter.
|
||||
#[derive(Clone)]
|
||||
pub struct EmbeddedPluginHostOwner {
|
||||
api: Arc<WorkspaceApi>,
|
||||
context: ServerRequestContext,
|
||||
workspace: String,
|
||||
transport_override: Option<Arc<https::HttpsTransport>>,
|
||||
}
|
||||
impl EmbeddedPluginHostOwner {
|
||||
pub(crate) fn new(
|
||||
api: Arc<WorkspaceApi>,
|
||||
context: ServerRequestContext,
|
||||
workspace: String,
|
||||
) -> Self {
|
||||
Self {
|
||||
api,
|
||||
context,
|
||||
workspace,
|
||||
transport_override: None,
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_transport(mut self, transport: https::HttpsTransport) -> Self {
|
||||
self.transport_override = Some(Arc::new(transport));
|
||||
self
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl yoi_plugin_runtime::owner::PluginHostOwner for EmbeddedPluginHostOwner {
|
||||
async fn call(
|
||||
&self,
|
||||
request: PluginHostRequest,
|
||||
) -> std::result::Result<PluginHostResponse, PluginHostError> {
|
||||
execute_with_transport(
|
||||
self.api.clone(),
|
||||
self.context.clone(),
|
||||
self.workspace.clone(),
|
||||
request,
|
||||
self.transport_override.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn execute(
|
||||
api: Arc<WorkspaceApi>,
|
||||
context: ServerRequestContext,
|
||||
workspace: String,
|
||||
request: PluginHostRequest,
|
||||
) -> std::result::Result<PluginHostResponse, PluginHostError> {
|
||||
#[cfg(test)]
|
||||
let transport_override = api.plugin_https_transport.clone();
|
||||
#[cfg(not(test))]
|
||||
let transport_override = None;
|
||||
execute_with_transport(api, context, workspace, request, transport_override).await
|
||||
}
|
||||
|
||||
async fn execute_with_transport(
|
||||
api: Arc<WorkspaceApi>,
|
||||
context: ServerRequestContext,
|
||||
workspace: String,
|
||||
request: PluginHostRequest,
|
||||
transport_override: Option<Arc<https::HttpsTransport>>,
|
||||
) -> std::result::Result<PluginHostResponse, PluginHostError> {
|
||||
// Human management and a Runtime-only (no Worker) source both fail closed.
|
||||
subject_authorization::source_from_context(&context).map_err(|_| denied())?;
|
||||
match request {
|
||||
PluginHostRequest::Begin {
|
||||
plan_id,
|
||||
installation_id,
|
||||
consumer,
|
||||
} => {
|
||||
let admitted = check(
|
||||
api.clone(),
|
||||
context,
|
||||
workspace,
|
||||
plan_id,
|
||||
installation_id.clone(),
|
||||
consumer.clone(),
|
||||
PluginHostOperation::Admit,
|
||||
)
|
||||
.await?;
|
||||
let store = api.config_store.clone();
|
||||
let record = store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store.with_immediate_transaction(|tx| {
|
||||
receipt::begin(
|
||||
tx,
|
||||
&admitted.plan,
|
||||
&installation_id,
|
||||
&consumer,
|
||||
now_ms(),
|
||||
RECEIPT_RETENTION_MS,
|
||||
)
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(policy::map_error)?;
|
||||
if record.cancelled || record.expired || record.expires_at_ms <= now_ms() {
|
||||
return Err(PluginHostError {
|
||||
code: if record.cancelled {
|
||||
PluginHostErrorCode::Cancelled
|
||||
} else {
|
||||
PluginHostErrorCode::ReceiptExpired
|
||||
},
|
||||
effect: PluginEffectObservation::Indeterminate,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
});
|
||||
}
|
||||
Ok(PluginHostResponse::Invocation {
|
||||
invocation_id: record.invocation_id,
|
||||
expires_at_ms: record.expires_at_ms,
|
||||
})
|
||||
}
|
||||
PluginHostRequest::Cancel { invocation_id } => {
|
||||
// Do not use invocation(): exact cancellation replay must work after
|
||||
// the irreversible bit has already been committed.
|
||||
let store = api.config_store.clone();
|
||||
let w = workspace.clone();
|
||||
let id = invocation_id.clone();
|
||||
let record = store
|
||||
.db_executor()
|
||||
.run(move || store.with_conn(|conn| receipt::lookup(conn, &w, &id)))
|
||||
.await
|
||||
.map_err(policy::map_error)?;
|
||||
check(
|
||||
api.clone(),
|
||||
context,
|
||||
workspace.clone(),
|
||||
record.plan_id,
|
||||
record.installation_id,
|
||||
record.consumer,
|
||||
PluginHostOperation::Admit,
|
||||
)
|
||||
.await?;
|
||||
let store = api.config_store.clone();
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store.with_immediate_transaction(|tx| {
|
||||
receipt::cancel(tx, &workspace, &invocation_id)
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(policy::map_error)?;
|
||||
Ok(PluginHostResponse::Authorized)
|
||||
}
|
||||
PluginHostRequest::Authorize {
|
||||
invocation_id,
|
||||
operation,
|
||||
} => {
|
||||
invocation(api, &context, &workspace, &invocation_id, operation).await?;
|
||||
Ok(PluginHostResponse::Authorized)
|
||||
}
|
||||
PluginHostRequest::Open {
|
||||
invocation_id,
|
||||
slot,
|
||||
} => {
|
||||
invocation(
|
||||
api,
|
||||
&context,
|
||||
&workspace,
|
||||
&invocation_id,
|
||||
PluginHostOperation::Open { slot },
|
||||
)
|
||||
.await?;
|
||||
Ok(PluginHostResponse::Opened)
|
||||
}
|
||||
PluginHostRequest::Log {
|
||||
invocation_id,
|
||||
entry,
|
||||
} => {
|
||||
invocation(
|
||||
api,
|
||||
&context,
|
||||
&workspace,
|
||||
&invocation_id,
|
||||
PluginHostOperation::Log,
|
||||
)
|
||||
.await?;
|
||||
// Guest text/fields are untrusted and may contain echoed credentials.
|
||||
// Standard diagnostics retain only this safe event, not their payload.
|
||||
if entry.message.len() > 4096
|
||||
|| entry.event_code.is_empty()
|
||||
|| entry.event_code.len() > 128
|
||||
|| !entry
|
||||
.event_code
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.'))
|
||||
|| entry.fields.len() > 32 || entry.fields.iter().any(|field| {
|
||||
field.name.len() > 128
|
||||
|| field.name.is_empty()
|
||||
|| field.name.chars().any(char::is_control)
|
||||
|| matches!(&field.value, PluginLogScalar::Text(value) if value.len() > 1024)
|
||||
|| matches!(&field.value, PluginLogScalar::Number(value) if !value.is_finite())
|
||||
}) {
|
||||
return Err(policy::map_error(policy::failure(
|
||||
PluginHostErrorCode::QuotaExceeded,
|
||||
)));
|
||||
}
|
||||
tracing::debug!(invocation = %invocation_id, "plugin log event admitted");
|
||||
Ok(PluginHostResponse::Logged)
|
||||
}
|
||||
PluginHostRequest::Send {
|
||||
invocation_id,
|
||||
slot,
|
||||
request,
|
||||
} => {
|
||||
let operation = PluginHostOperation::Send { slot: slot.clone() };
|
||||
let (record, admitted) = invocation(
|
||||
api.clone(),
|
||||
&context,
|
||||
&workspace,
|
||||
&invocation_id,
|
||||
operation.clone(),
|
||||
)
|
||||
.await?;
|
||||
let binding = admitted.binding.as_ref().ok_or_else(denied)?;
|
||||
let prepared = https::prepare(
|
||||
binding,
|
||||
https::HttpsRequest {
|
||||
method: request.method.clone(),
|
||||
path: request.path.clone(),
|
||||
query: request.query.clone(),
|
||||
headers: request.headers.clone(),
|
||||
body: request.body.clone(),
|
||||
timeout_ms: request.timeout_ms,
|
||||
effect_key: request.effect_key.clone(),
|
||||
},
|
||||
)
|
||||
.map_err(transport_error)?;
|
||||
let normalized = prepared.normalized();
|
||||
let normalized_dto = PluginHttpRequest {
|
||||
method: normalized.method.clone(),
|
||||
path: normalized.path.clone(),
|
||||
query: normalized.query.clone(),
|
||||
headers: normalized.headers.clone(),
|
||||
body: normalized.body.clone(),
|
||||
timeout_ms: normalized.timeout_ms,
|
||||
effect_key: normalized.effect_key.clone(),
|
||||
};
|
||||
let fingerprint = policy::request_fingerprint(&admitted, &slot, &normalized_dto)
|
||||
.map_err(policy::map_error)?;
|
||||
let store = api.config_store.clone();
|
||||
let r = record.clone();
|
||||
let fp = fingerprint.clone();
|
||||
let key = request.effect_key.clone();
|
||||
let admission = store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store.with_immediate_transaction(|tx| {
|
||||
receipt::effect_admit(tx, &r, &key, &fp, now_ms())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(policy::map_error)?;
|
||||
let result = match admission {
|
||||
receipt::EffectAdmission::Replay(result) => result,
|
||||
receipt::EffectAdmission::Dispatch => {
|
||||
// A second current check fences cancellation/revocation that
|
||||
// happened while waiting for durable admission. Any lost
|
||||
// acknowledgement from this point remains indeterminate.
|
||||
invocation(
|
||||
api.clone(),
|
||||
&context,
|
||||
&workspace,
|
||||
&invocation_id,
|
||||
operation.clone(),
|
||||
)
|
||||
.await?;
|
||||
let secret = if binding.secret_ref.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let root = ServerConfig::workspace_backend_data_root_for_data_dir(
|
||||
&api.config.data_root,
|
||||
&workspace,
|
||||
);
|
||||
Some(zeroize::Zeroizing::new(
|
||||
secrets::SecretStore::new(root)
|
||||
.get(&format!("plugin/{}", binding.secret_ref))
|
||||
.map_err(|_| denied())?
|
||||
.into_string(),
|
||||
))
|
||||
};
|
||||
let transport = match transport_override {
|
||||
Some(transport) => transport,
|
||||
None => Arc::new(transport(&api).await?),
|
||||
};
|
||||
let cancellation = CancellationToken::new();
|
||||
let result = transport
|
||||
.execute_with_guard(
|
||||
binding,
|
||||
prepared,
|
||||
secret.as_deref().map(|v| v.as_str()),
|
||||
&cancellation,
|
||||
|| async {
|
||||
invocation(
|
||||
api.clone(),
|
||||
&context,
|
||||
&workspace,
|
||||
&invocation_id,
|
||||
operation.clone(),
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|_| https::TransportError::Denied)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|response| PluginHttpResponse {
|
||||
status: response.status,
|
||||
headers: response.headers,
|
||||
body: response.body,
|
||||
})
|
||||
.map_err(sent_transport_error);
|
||||
let store = api.config_store.clone();
|
||||
let r = record;
|
||||
let fp = fingerprint;
|
||||
let key = request.effect_key;
|
||||
let saved = result.clone();
|
||||
store
|
||||
.db_executor()
|
||||
.run(move || {
|
||||
store.with_immediate_transaction(|tx| {
|
||||
receipt::effect_complete(tx, &r, &key, &fp, &saved)
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|_| PluginHostError {
|
||||
code: PluginHostErrorCode::Indeterminate,
|
||||
effect: PluginEffectObservation::Indeterminate,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
})?;
|
||||
result
|
||||
}
|
||||
};
|
||||
// Both saved response replay and new network results require current
|
||||
// read/publication authority. No denied payload reaches the Runtime.
|
||||
let observation = result
|
||||
.as_ref()
|
||||
.map(|_| PluginEffectObservation::ResponseObserved)
|
||||
.unwrap_or_else(|error| error.effect);
|
||||
invocation(api.clone(), &context, &workspace, &invocation_id, operation)
|
||||
.await
|
||||
.map_err(|mut error| {
|
||||
error.effect = observation;
|
||||
error
|
||||
})?;
|
||||
invocation(
|
||||
api,
|
||||
&context,
|
||||
&workspace,
|
||||
&invocation_id,
|
||||
PluginHostOperation::Publish,
|
||||
)
|
||||
.await
|
||||
.map_err(|mut error| {
|
||||
error.effect = observation;
|
||||
error
|
||||
})?;
|
||||
result.map(|response| PluginHostResponse::Response { response })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn transport(
|
||||
api: &WorkspaceApi,
|
||||
) -> std::result::Result<https::HttpsTransport, PluginHostError> {
|
||||
let mut hosts = std::collections::BTreeSet::new();
|
||||
let mut ips = std::collections::BTreeSet::new();
|
||||
let AuthConfig::Passkey {
|
||||
public_base_url, ..
|
||||
} = &api.config.auth;
|
||||
for address in std::iter::once(public_base_url)
|
||||
.chain(api.config.backend_base_url.iter())
|
||||
.chain(
|
||||
api.config
|
||||
.remote_runtime_sources
|
||||
.iter()
|
||||
.map(|source| &source.base_url),
|
||||
)
|
||||
{
|
||||
if let Ok(url) = url::Url::parse(address) {
|
||||
if let Some(host) = url.host_str() {
|
||||
hosts.insert(host.to_ascii_lowercase());
|
||||
if let Ok(ip) = host.parse() {
|
||||
ips.insert(ip);
|
||||
} else {
|
||||
let port = url.port_or_known_default().ok_or_else(denied)?;
|
||||
let resolved = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
tokio::net::lookup_host((host, port)),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| transport_error(https::TransportError::Dns))?
|
||||
.map_err(|_| transport_error(https::TransportError::Dns))?;
|
||||
for (index, address) in resolved.enumerate() {
|
||||
if index >= 64 {
|
||||
return Err(transport_error(https::TransportError::Dns));
|
||||
}
|
||||
ips.insert(address.ip());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(https::HttpsTransport::new(https::TransportPolicy {
|
||||
denied_addresses: ips,
|
||||
denied_hosts: hosts,
|
||||
}))
|
||||
}
|
||||
fn sent_transport_error(error: https::TransportError) -> PluginHostError {
|
||||
match error {
|
||||
https::TransportError::ResponseLimit => PluginHostError {
|
||||
code: PluginHostErrorCode::QuotaExceeded,
|
||||
effect: PluginEffectObservation::ResponseObserved,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
},
|
||||
https::TransportError::UnsupportedEncoding => PluginHostError {
|
||||
code: PluginHostErrorCode::Unsupported,
|
||||
effect: PluginEffectObservation::ResponseObserved,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
},
|
||||
https::TransportError::HeaderLimit => PluginHostError {
|
||||
code: PluginHostErrorCode::QuotaExceeded,
|
||||
effect: PluginEffectObservation::Indeterminate,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
},
|
||||
https::TransportError::Unavailable
|
||||
| https::TransportError::DeadlineExceeded
|
||||
| https::TransportError::Cancelled => PluginHostError {
|
||||
code: PluginHostErrorCode::Indeterminate,
|
||||
effect: PluginEffectObservation::Indeterminate,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
},
|
||||
other => transport_error(other),
|
||||
}
|
||||
}
|
||||
fn transport_error(error: https::TransportError) -> PluginHostError {
|
||||
let code = match error {
|
||||
https::TransportError::Denied | https::TransportError::InvalidBinding => {
|
||||
PluginHostErrorCode::Denied
|
||||
}
|
||||
https::TransportError::InvalidRequest => PluginHostErrorCode::InvalidInput,
|
||||
https::TransportError::HeaderLimit
|
||||
| https::TransportError::RequestLimit
|
||||
| https::TransportError::ResponseLimit => PluginHostErrorCode::QuotaExceeded,
|
||||
https::TransportError::DeadlineExceeded => PluginHostErrorCode::DeadlineExceeded,
|
||||
https::TransportError::Cancelled => PluginHostErrorCode::Cancelled,
|
||||
_ => PluginHostErrorCode::Unavailable,
|
||||
};
|
||||
PluginHostError {
|
||||
code,
|
||||
effect: PluginEffectObservation::NotDispatched,
|
||||
diagnostic_id: uuid::Uuid::now_v7().to_string(),
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -129,7 +129,7 @@ pub use ticket::{
|
||||
};
|
||||
|
||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 108;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 109;
|
||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||
@@ -477,6 +477,11 @@ const MIGRATIONS: &[Migration] = &[
|
||||
name: "Subject Ticket responsibility and current-body notification routing",
|
||||
apply: migrate_subject_assignments_v107_to_v108,
|
||||
},
|
||||
Migration {
|
||||
version: 109,
|
||||
name: "Durable Plugin Host invocations and HTTPS effect receipts",
|
||||
apply: migrate_plugin_receipts_v108_to_v109,
|
||||
},
|
||||
];
|
||||
|
||||
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
||||
@@ -15851,6 +15856,27 @@ fn migrate_subject_assignments_v107_to_v108(conn: &Connection) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_plugin_receipts_v108_to_v109(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? != 108 {
|
||||
return Err(Error::Store(
|
||||
"Plugin receipt migration requires schema 108".into(),
|
||||
));
|
||||
}
|
||||
verify_schema_history(conn, 108)?;
|
||||
crate::plugin_catalog::verify(conn)?;
|
||||
crate::plugin_plan::verify_plan_schema(conn)?;
|
||||
subject_assignment_migration::verify(conn)?;
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
tx.execute_batch(crate::plugin_receipt::SCHEMA_SQL)?;
|
||||
crate::plugin_host::verify_receipt_schema(&tx)?;
|
||||
tx.execute(
|
||||
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(109,?1)",
|
||||
["Durable Plugin Host invocations and HTTPS effect receipts"],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
||||
create_workspace_schema_structure(conn)?;
|
||||
assignment_delivery::install(conn)?;
|
||||
@@ -15889,6 +15915,7 @@ fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
|
||||
conn.execute_batch(include_str!("activation_schema.sql"))?;
|
||||
conn.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
||||
conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?;
|
||||
conn.execute_batch(crate::plugin_receipt::SCHEMA_SQL)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -16931,6 +16958,7 @@ fn apply_migrations_with_secret_source(
|
||||
verify_schema_history(conn, LATEST_SCHEMA_VERSION)?;
|
||||
crate::plugin_catalog::verify(conn)?;
|
||||
crate::plugin_plan::verify_plan_schema(conn)?;
|
||||
crate::plugin_host::verify_receipt_schema(conn)?;
|
||||
subject_assignment_migration::verify(conn)?;
|
||||
verify_workspace_runtime_binding_schema(conn)?;
|
||||
verify_workspace_runtime_verification_schema(conn)?;
|
||||
@@ -19214,6 +19242,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
include!("ticket_tag_upgrade_tests.rs");
|
||||
include!("ticket_post_upgrade_tests.rs");
|
||||
include!("migration_entrypoint_tests.rs");
|
||||
include!("plugin_host_migration_tests.rs");
|
||||
include!("legacy_evidence_column_tests.rs");
|
||||
|
||||
#[test]
|
||||
@@ -19629,6 +19658,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
WorkspaceSchemaMigrationStep {version:106, name:crate::plugin_catalog::MIGRATION_NAME.into()},
|
||||
WorkspaceSchemaMigrationStep {version:107, name:"Immutable Worker Plugin plans and retained pins".into()},
|
||||
WorkspaceSchemaMigrationStep {version:108, name:"Subject Ticket responsibility and current-body notification routing".into()},
|
||||
WorkspaceSchemaMigrationStep {version:109, name:"Durable Plugin Host invocations and HTTPS effect receipts".into()},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -19747,6 +19777,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(106, crate::plugin_catalog::MIGRATION_NAME.to_string()),
|
||||
(107, "Immutable Worker Plugin plans and retained pins".to_string()),
|
||||
(108, "Subject Ticket responsibility and current-body notification routing".to_string()),
|
||||
(109, "Durable Plugin Host invocations and HTTPS effect receipts".to_string()),
|
||||
]
|
||||
);
|
||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||
@@ -23036,7 +23067,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(105, "Durable Subject deletion snapshots and Job admission fences".into()),
|
||||
(106, crate::plugin_catalog::MIGRATION_NAME.into()),
|
||||
(107, "Immutable Worker Plugin plans and retained pins".into()),
|
||||
(108, "Subject Ticket responsibility and current-body notification routing".into())]);
|
||||
(108, "Subject Ticket responsibility and current-body notification routing".into()),
|
||||
(109, "Durable Plugin Host invocations and HTTPS effect receipts".into())]);
|
||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||
}
|
||||
|
||||
@@ -1094,7 +1094,7 @@ mod subject_assignment_migration_tests {
|
||||
}
|
||||
|
||||
fn expected_tail(version: i64) -> Vec<(i64, String)> {
|
||||
match version {
|
||||
let mut tail = match version {
|
||||
96 => vec![
|
||||
(
|
||||
97,
|
||||
@@ -1127,7 +1127,9 @@ mod subject_assignment_migration_tests {
|
||||
106 => vec![(107, PLAN_NAME.into()), (108, SUBJECT_NAME.into())],
|
||||
107 => vec![(108, SUBJECT_NAME.into())],
|
||||
_ => panic!("unsupported fixture origin"),
|
||||
}
|
||||
};
|
||||
tail.push((109, "Durable Plugin Host invocations and HTTPS effect receipts".into()));
|
||||
tail
|
||||
}
|
||||
|
||||
fn plan_steps(version: i64) -> Vec<WorkspaceSchemaMigrationStep> {
|
||||
@@ -1176,12 +1178,12 @@ mod subject_assignment_migration_tests {
|
||||
fn plugin_schema(conn: &Connection) -> Rows {
|
||||
fixture_query_rows(
|
||||
conn,
|
||||
"SELECT type,name,tbl_name,rootpage,sql FROM sqlite_schema WHERE tbl_name LIKE 'plugin_%' ORDER BY type,name",
|
||||
"SELECT type,name,tbl_name,rootpage,sql FROM sqlite_schema WHERE tbl_name LIKE 'plugin_%' AND tbl_name NOT IN ('plugin_invocations','plugin_effect_receipts') ORDER BY type,name",
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn applied107_only108_preserves_canonical_plans_pins_and_released_replay() {
|
||||
async fn applied107_preserves_canonical_plans_pins_and_released_replay_through_later_tail() {
|
||||
let (_dir, path, conn, receipts) = populated107().await;
|
||||
let before = snapshot(&conn);
|
||||
let rows = data(&conn, true);
|
||||
@@ -1193,10 +1195,7 @@ mod subject_assignment_migration_tests {
|
||||
assert_eq!(plan.current_schema_version, 107);
|
||||
assert_eq!(
|
||||
plan.migrations,
|
||||
vec![WorkspaceSchemaMigrationStep {
|
||||
version: 108,
|
||||
name: SUBJECT_NAME.into()
|
||||
}]
|
||||
plan_steps(107)
|
||||
);
|
||||
assert_eq!(snapshot(&conn), before);
|
||||
drop(conn);
|
||||
|
||||
@@ -16,6 +16,8 @@ use crate::{Error, Result};
|
||||
/// Explicit domain-owned purge inventory. The deletion operation tombstone is intentionally
|
||||
/// excluded so retries and audit remain available after the Workspace row is gone.
|
||||
const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
||||
"plugin_effect_receipts",
|
||||
"plugin_invocations",
|
||||
"plugin_execution_plans",
|
||||
"plugin_references",
|
||||
"plugin_operations",
|
||||
@@ -509,6 +511,8 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
||||
// commit; update guards remain installed throughout.
|
||||
let mut ticket_delete_guards = Vec::new();
|
||||
for (table, trigger) in [
|
||||
("plugin_effect_receipts", "plugin_effect_retained"),
|
||||
("plugin_invocations", "plugin_invocation_retained"),
|
||||
("plugin_execution_plans", "plugin_plan_replay_retained"),
|
||||
("notification_source_preparation_failures", "notification_source_preparation_failures_no_delete"),
|
||||
("notification_job_targets", "notification_job_targets_no_delete"),
|
||||
@@ -1303,6 +1307,12 @@ mod tests {
|
||||
for kind in ["config","pending","active","restorable"] {
|
||||
conn.execute("INSERT INTO plugin_references VALUES(?1,?2,'owner','installation',?3)", params![workspace,kind,digest])?;
|
||||
}
|
||||
conn.execute("INSERT INTO plugin_execution_plans VALUES(?1,'plan','actor','create',?2,'{}','active')", params![workspace,digest])?;
|
||||
conn.execute("INSERT INTO plugin_invocations(workspace_id,invocation_id,plan_id,plan_fingerprint,installation_id,runtime_id,worker_id,execution_json,consumer_key,consumer_json,created_at_ms,expires_at_ms) VALUES(?1,'invocation','plan',?2,'installation','runtime','worker','{}','[\"tool_call\",\"call\"]','{}',1,1000)", params![workspace,digest])?;
|
||||
conn.execute("INSERT INTO plugin_effect_receipts(workspace_id,invocation_id,effect_key,fingerprint,admitted_at_ms,state,error_json) VALUES(?1,'invocation','key',?2,2,'pending',?3)", params![workspace,digest,r#"{"code":"indeterminate","effect":"indeterminate","diagnostic_id":"safe"}"#])?;
|
||||
for table in ["plugin_effect_receipts", "plugin_invocations", "plugin_execution_plans"] {
|
||||
assert!(conn.execute(&format!("DELETE FROM {table} WHERE workspace_id=?1"), [workspace]).is_err());
|
||||
}
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
@@ -1310,6 +1320,9 @@ mod tests {
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
[
|
||||
"plugin_effect_receipts",
|
||||
"plugin_invocations",
|
||||
"plugin_execution_plans",
|
||||
"plugin_references",
|
||||
"plugin_operations",
|
||||
"plugin_uploads",
|
||||
@@ -1329,7 +1342,7 @@ mod tests {
|
||||
.unwrap()
|
||||
};
|
||||
let other = counts("workspace-b");
|
||||
assert_eq!(counts("workspace-a"), vec![4, 1, 1, 1, 1]);
|
||||
assert_eq!(counts("workspace-a"), vec![1, 1, 1, 4, 1, 1, 1, 1]);
|
||||
let preflight = store
|
||||
.workspace_deletion_preflight(&owner, &workspace_id)
|
||||
.unwrap();
|
||||
@@ -1345,6 +1358,24 @@ mod tests {
|
||||
)
|
||||
.unwrap()
|
||||
.operation;
|
||||
store.with_conn(|conn| {
|
||||
conn.execute_batch("CREATE TRIGGER plugin_purge_failure BEFORE DELETE ON workspaces WHEN OLD.workspace_id='workspace-a' BEGIN SELECT RAISE(ABORT,'injected purge failure'); END;")?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert!(
|
||||
store
|
||||
.finalize_workspace_deletion(&operation.operation_id)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(counts("workspace-a"), vec![1, 1, 1, 4, 1, 1, 1, 1]);
|
||||
assert_eq!(counts("workspace-b"), other);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
crate::plugin_host::verify_receipt_schema(conn)?;
|
||||
conn.execute_batch("DROP TRIGGER plugin_purge_failure")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.finalize_workspace_deletion(&operation.operation_id)
|
||||
@@ -1352,10 +1383,22 @@ mod tests {
|
||||
.state,
|
||||
WorkspaceDeletionState::Succeeded
|
||||
);
|
||||
assert_eq!(counts("workspace-a"), vec![0; 5]);
|
||||
assert_eq!(counts("workspace-a"), vec![0; 8]);
|
||||
assert_eq!(counts("workspace-b"), other);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
crate::plugin_host::verify_receipt_schema(conn)?;
|
||||
assert_eq!(
|
||||
conn.query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0))?,
|
||||
"ok"
|
||||
);
|
||||
assert!(
|
||||
conn.execute(
|
||||
"DELETE FROM plugin_effect_receipts WHERE workspace_id='workspace-b'",
|
||||
[]
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r
|
||||
.get::<_, i64>(
|
||||
|
||||
Reference in New Issue
Block a user