fix(web): align startup consumer with corrected T812 Rust boundary

This commit is contained in:
2026-11-06 12:01:17 +09:00
65 changed files with 9254 additions and 670 deletions
Generated
+5
View File
@@ -199,6 +199,7 @@ dependencies = [
"api-macros-impl", "api-macros-impl",
"axum", "axum",
"bytes", "bytes",
"error-contract",
"futures", "futures",
"oas3", "oas3",
"reqwest", "reqwest",
@@ -4325,12 +4326,15 @@ name = "runtime-api"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"api-macros", "api-macros",
"error-contract",
"manifest", "manifest",
"plugin-plan", "plugin-plan",
"protocol", "protocol",
"schemars",
"serde", "serde",
"serde_json", "serde_json",
"session-store", "session-store",
"sha2 0.11.0",
"tokio", "tokio",
"workdir", "workdir",
] ]
@@ -4764,6 +4768,7 @@ name = "server-api"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"api-macros", "api-macros",
"error-contract",
"feature-permissions", "feature-permissions",
"memory", "memory",
"merge-request", "merge-request",
+262 -113
View File
@@ -53,6 +53,7 @@ enum Method {
#[derive(Clone, Copy, Default)] #[derive(Clone, Copy, Default)]
struct AdapterConfig { struct AdapterConfig {
reqwest: bool, reqwest: bool,
client_error_hook: bool,
axum: bool, axum: bool,
openapi: bool, openapi: bool,
} }
@@ -64,9 +65,10 @@ impl Parse for AdapterConfig {
for adapter in adapters { for adapter in adapters {
match adapter.to_string().as_str() { match adapter.to_string().as_str() {
"reqwest" if !config.reqwest => config.reqwest = true, "reqwest" if !config.reqwest => config.reqwest = true,
"client_error_hook" if !config.client_error_hook => config.client_error_hook = true,
"axum" if !config.axum => config.axum = true, "axum" if !config.axum => config.axum = true,
"openapi" if !config.openapi => config.openapi = true, "openapi" if !config.openapi => config.openapi = true,
"reqwest" | "axum" | "openapi" => { "reqwest" | "axum" | "openapi" | "client_error_hook" => {
return Err(syn::Error::new(adapter.span(), "duplicate API adapter")); return Err(syn::Error::new(adapter.span(), "duplicate API adapter"));
} }
_ => { _ => {
@@ -77,6 +79,12 @@ impl Parse for AdapterConfig {
} }
} }
} }
if config.client_error_hook && !config.reqwest {
return Err(syn::Error::new(
Span::call_site(),
"`client_error_hook` requires the `reqwest` adapter",
));
}
Ok(config) Ok(config)
} }
} }
@@ -121,6 +129,8 @@ struct RouteArgs {
responses: Vec<DeclaredResponse>, responses: Vec<DeclaredResponse>,
error_status: Option<LitInt>, error_status: Option<LitInt>,
additional_error_statuses: Vec<LitInt>, additional_error_statuses: Vec<LitInt>,
additional_error_content_types: Option<Vec<LitStr>>,
client_error_hook: Option<syn::LitBool>,
bearer_auth: Option<syn::LitBool>, bearer_auth: Option<syn::LitBool>,
browser_auth: Option<syn::LitBool>, browser_auth: Option<syn::LitBool>,
normalize_body_errors: Option<syn::LitBool>, normalize_body_errors: Option<syn::LitBool>,
@@ -310,6 +320,8 @@ impl Parse for RouteArgs {
responses: Vec::new(), responses: Vec::new(),
error_status: None, error_status: None,
additional_error_statuses: Vec::new(), additional_error_statuses: Vec::new(),
additional_error_content_types: None,
client_error_hook: None,
bearer_auth: None, bearer_auth: None,
browser_auth: None, browser_auth: None,
normalize_body_errors: None, normalize_body_errors: None,
@@ -379,6 +391,25 @@ impl Parse for RouteArgs {
} }
} }
} }
"additional_error_content_types" => {
let content;
syn::bracketed!(content in input);
let values = Punctuated::<LitStr, Token![,]>::parse_terminated(&content)?
.into_iter()
.collect();
set_once(
&mut result.additional_error_content_types,
values,
&key,
"additional_error_content_types",
)?;
}
"client_error_hook" => set_once(
&mut result.client_error_hook,
input.parse::<syn::LitBool>()?,
&key,
"client_error_hook",
)?,
"bearer_auth" => set_once( "bearer_auth" => set_once(
&mut result.bearer_auth, &mut result.bearer_auth,
input.parse::<syn::LitBool>()?, input.parse::<syn::LitBool>()?,
@@ -406,7 +437,7 @@ impl Parse for RouteArgs {
_ => { _ => {
return Err(syn::Error::new( return Err(syn::Error::new(
key.span(), key.span(),
"unsupported route option; expected operation_id, status, alternate_status, responses, error_status, additional_error_statuses, bearer_auth, browser_auth, normalize_body_errors, or openapi", "unsupported route option; expected operation_id, status, alternate_status, responses, error_status, additional_error_statuses, additional_error_content_types, client_error_hook, bearer_auth, browser_auth, normalize_body_errors, or openapi",
)); ));
} }
} }
@@ -502,6 +533,8 @@ struct Operation {
alternate_status: Option<u16>, alternate_status: Option<u16>,
error_status: Option<u16>, error_status: Option<u16>,
additional_error_statuses: Vec<u16>, additional_error_statuses: Vec<u16>,
additional_error_content_types: Vec<String>,
client_error_hook: bool,
bearer_auth: bool, bearer_auth: bool,
browser_auth: bool, browser_auth: bool,
normalize_body_errors: bool, normalize_body_errors: bool,
@@ -546,7 +579,7 @@ fn normalize_api(
for mut trait_item in std::mem::take(&mut item.items) { for mut trait_item in std::mem::take(&mut item.items) {
let operation = match &mut trait_item { let operation = match &mut trait_item {
TraitItem::Fn(method) => { TraitItem::Fn(method) => {
let operation = normalize_operation(method)?; let operation = normalize_operation(method, adapters)?;
retained_items.push(trait_item); retained_items.push(trait_item);
operation operation
} }
@@ -778,6 +811,8 @@ fn normalize_websocket_operation(declaration: &mut syn::TraitItemType) -> syn::R
alternate_status: None, alternate_status: None,
error_status: None, error_status: None,
additional_error_statuses: Vec::new(), additional_error_statuses: Vec::new(),
additional_error_content_types: Vec::new(),
client_error_hook: false,
bearer_auth: false, bearer_auth: false,
browser_auth: false, browser_auth: false,
normalize_body_errors: false, normalize_body_errors: false,
@@ -796,7 +831,10 @@ fn validate_frame_type(ty: &Type) -> syn::Result<()> {
} }
} }
fn normalize_operation(method: &mut TraitItemFn) -> syn::Result<Operation> { fn normalize_operation(
method: &mut TraitItemFn,
adapters: AdapterConfig,
) -> syn::Result<Operation> {
validate_signature(method)?; validate_signature(method)?;
let mut route = None; let mut route = None;
@@ -845,6 +883,53 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result<Operation> {
validate_named_body_type(ty, "error response")?; validate_named_body_type(ty, "error response")?;
} }
let client_error_hook = route
.client_error_hook
.as_ref()
.is_some_and(|value| value.value);
if client_error_hook {
let option = route.client_error_hook.as_ref().expect("enabled hook");
if !adapters.reqwest {
return Err(syn::Error::new_spanned(
option,
"client_error_hook requires the `reqwest` adapter",
));
}
if error_body.is_none() {
return Err(syn::Error::new_spanned(
option,
"client_error_hook requires a public error response type",
));
}
}
let mut additional_error_content_types = Vec::new();
if let Some(content_types) = &route.additional_error_content_types {
if error_body.is_none() {
return Err(syn::Error::new_spanned(
&route.path,
"additional_error_content_types require a public error response type",
));
}
if content_types.is_empty() {
return Err(syn::Error::new_spanned(
&route.path,
"additional_error_content_types must not be empty",
));
}
for content_type in content_types {
let value = validate_error_content_type(content_type)?;
if value == "application/json" || additional_error_content_types.contains(&value) {
return Err(syn::Error::new_spanned(
content_type,
"additional error content types must be distinct from each other and application/json",
));
}
additional_error_content_types.push(value);
}
additional_error_content_types.sort();
}
let declared_responses = !route.responses.is_empty(); let declared_responses = !route.responses.is_empty();
if declared_responses && (route.status.is_some() || route.alternate_status.is_some()) { if declared_responses && (route.status.is_some() || route.alternate_status.is_some()) {
return Err(syn::Error::new_spanned( return Err(syn::Error::new_spanned(
@@ -1196,6 +1281,8 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result<Operation> {
alternate_status, alternate_status,
error_status, error_status,
additional_error_statuses, additional_error_statuses,
additional_error_content_types,
client_error_hook,
bearer_auth: route.bearer_auth.as_ref().is_some_and(|value| value.value), bearer_auth: route.bearer_auth.as_ref().is_some_and(|value| value.value),
browser_auth: route.browser_auth.as_ref().is_some_and(|value| value.value), browser_auth: route.browser_auth.as_ref().is_some_and(|value| value.value),
normalize_body_errors: route normalize_body_errors: route
@@ -1206,6 +1293,43 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result<Operation> {
}) })
} }
// Route media declarations are concrete type/subtype tokens, not media ranges or parameters.
fn validate_error_content_type(literal: &LitStr) -> syn::Result<String> {
let value = literal.value().to_ascii_lowercase();
let valid_token = |token: &str| {
!token.is_empty()
&& token.bytes().all(|byte| {
byte.is_ascii_alphanumeric()
|| matches!(
byte,
b'!' | b'#'
| b'$'
| b'%'
| b'&'
| b'\''
| b'+'
| b'-'
| b'.'
| b'^'
| b'_'
| b'`'
| b'|'
| b'~'
)
})
};
if !value
.split_once('/')
.is_some_and(|(kind, subtype)| valid_token(kind) && valid_token(subtype))
{
return Err(syn::Error::new_spanned(
literal,
"error content type must be a concrete type/subtype without parameters",
));
}
Ok(value)
}
fn validate_signature(method: &TraitItemFn) -> syn::Result<()> { fn validate_signature(method: &TraitItemFn) -> syn::Result<()> {
let sig = &method.sig; let sig = &method.sig;
if sig.asyncness.is_none() { if sig.asyncness.is_none() {
@@ -1815,17 +1939,27 @@ fn reqwest_adapter_tokens(
} }
} }
}; };
let public_error_decode = if api.adapters.client_error_hook || operation.client_error_hook {
quote! {
<#error_type as #api_crate::reqwest::HttpClientError>::decode_error(&__response)
.map_err(#api_crate::reqwest::ClientError::failure)?
}
} else {
quote! {
__response
.decode_json::<#error_type>(#api_crate::reqwest::DecodeKind::PublicError)
.map_err(|_| #api_crate::reqwest::ClientError::failure(
#api_crate::reqwest::ClientFailure::ErrorResponseDecode {
status: __status.as_u16(),
},
))?
}
};
let public_error = match operation.error_body.as_ref() { let public_error = match operation.error_body.as_ref() {
Some(error) => quote! { Some(_) => quote! {
if (400..=599).contains(&__actual) { if (400..=599).contains(&__actual) {
let __status = __response.status(); let __status = __response.status();
let __error = __response let __error = #public_error_decode;
.decode_json::<#error>(#api_crate::reqwest::DecodeKind::PublicError)
.map_err(|_| #api_crate::reqwest::ClientError::failure(
#api_crate::reqwest::ClientFailure::ErrorResponseDecode {
status: __status.as_u16(),
},
))?;
return ::core::result::Result::Err( return ::core::result::Result::Err(
#api_crate::reqwest::ClientError::public(__status, __error), #api_crate::reqwest::ClientError::public(__status, __error),
); );
@@ -2098,7 +2232,7 @@ fn axum_adapter_tokens(
status, status,
rejection.body_text(), rejection.body_text(),
); );
return #api_crate::axum::error_response( return #api_crate::axum::http_error_response(
#api_crate::axum::status(status), #api_crate::axum::status(status),
error, error,
); );
@@ -2132,7 +2266,7 @@ fn axum_adapter_tokens(
status, status,
rejection.body_text(), rejection.body_text(),
); );
return #api_crate::axum::error_response( return #api_crate::axum::http_error_response(
#api_crate::axum::status(status), #api_crate::axum::status(status),
error, error,
); );
@@ -2218,7 +2352,7 @@ fn axum_adapter_tokens(
let error = match operation.error_status { let error = match operation.error_status {
Some(_) => quote!({ Some(_) => quote!({
let status = #api_crate::HttpError::status_code(&error); let status = #api_crate::HttpError::status_code(&error);
#api_crate::axum::error_response(#api_crate::axum::status(status), error) #api_crate::axum::http_error_response(#api_crate::axum::status(status), error)
}), }),
None => quote!({ let _ = error; #api_crate::axum::empty_response(#api_crate::axum::framework::StatusCode::INTERNAL_SERVER_ERROR) }), None => quote!({ let _ = error; #api_crate::axum::empty_response(#api_crate::axum::framework::StatusCode::INTERNAL_SERVER_ERROR) }),
}; };
@@ -2315,110 +2449,115 @@ fn openapi_adapter_tokens(
matches!(operation.transport, Transport::Http) && !operation.openapi_skip matches!(operation.transport, Transport::Http) && !operation.openapi_skip
}) })
.map(|operation| { .map(|operation| {
let method = method_name(operation.method); let method = method_name(operation.method);
let path = &operation.path; let path = &operation.path;
let operation_id = &operation.operation_id; let operation_id = &operation.operation_id;
let parameters = operation.parameters.iter().filter_map(|parameter| { let parameters = operation.parameters.iter().filter_map(|parameter| {
if matches!(parameter.location, Location::Body | Location::Binary | Location::Extension) { if matches!(
return None; parameter.location,
} Location::Body | Location::Binary | Location::Extension
let ty = &parameter.ty; ) {
let name = &parameter.wire_name; return None;
let location = match parameter.location {
Location::Path => "path",
Location::Query => "query",
Location::Header => "header",
Location::Body | Location::Binary | Location::Extension => unreachable!(),
};
Some(quote! {
operation.parameter::<#ty>(#name, #location)?;
})
});
let request = operation.request_body.as_ref().map(|body| match body.wire_kind {
RequestWireKind::Json => {
let ty = &body.ty;
quote! {
operation.request_body::<#ty>("application/json")?;
}
}
RequestWireKind::Binary => quote! {
operation.binary_request_body()?;
},
});
let success_responses = operation.success_responses.iter().enumerate().map(|(index, response)| {
let status = response.status;
let description = if index == 0 {
"Successful response"
} else {
"Alternate successful response"
};
let body = match response.body.as_ref() {
Some(ty) if is_binary_body_type(ty) => quote! {
operation.binary_response(#status, #description)?;
},
Some(ty) => quote! {
operation.response::<#ty>(#status, "application/json", #description)?;
},
None => quote! {
operation.empty_response(#status, #description)?;
},
};
let headers = response.headers.iter().map(|header| {
let ty = &header.ty;
let name = &header.wire_name;
quote! {
operation.response_header::<#ty>(#status, #name)?;
} }
let ty = &parameter.ty;
let name = &parameter.wire_name;
let location = match parameter.location {
Location::Path => "path",
Location::Query => "query",
Location::Header => "header",
Location::Body | Location::Binary | Location::Extension => unreachable!(),
};
Some(quote! {
operation.parameter::<#ty>(#name, #location)?;
})
}); });
quote! { let request = operation
#body .request_body
#(#headers)* .as_ref()
} .map(|body| match body.wire_kind {
}); RequestWireKind::Json => {
let error = operation let ty = &body.ty;
.error_status
.zip(operation.error_body.as_ref())
.map(|(status, ty)| {
quote! {
operation.response::<#ty>(#status, "application/json", "Error response")?;
}
});
let additional_errors = operation
.error_body
.as_ref()
.map(|ty| {
operation
.additional_error_statuses
.iter()
.map(|status| {
quote! { quote! {
operation.response::<#ty>(#status, "application/json", "Error response")?; operation.request_body::<#ty>("application/json")?;
} }
}) }
.collect::<Vec<_>>() RequestWireKind::Binary => quote! {
}) operation.binary_request_body()?;
.unwrap_or_default(); },
let bearer_auth = operation.bearer_auth.then(|| { });
quote! { operation.bearer_authentication(); } let success_responses = operation.success_responses.iter().enumerate().map(
}); |(index, response)| {
let browser_auth = operation.browser_auth.then(|| { let status = response.status;
quote! { operation.browser_authentication(); } let description = if index == 0 {
}); "Successful response"
} else {
"Alternate successful response"
};
let body = match response.body.as_ref() {
Some(ty) if is_binary_body_type(ty) => quote! {
operation.binary_response(#status, #description)?;
},
Some(ty) => quote! {
operation.response::<#ty>(#status, "application/json", #description)?;
},
None => quote! {
operation.empty_response(#status, #description)?;
},
};
let headers = response.headers.iter().map(|header| {
let ty = &header.ty;
let name = &header.wire_name;
quote! {
operation.response_header::<#ty>(#status, #name)?;
}
});
quote! {
#body
#(#headers)*
}
},
);
let errors = operation
.error_body
.as_ref()
.map(|ty| {
let additional_content_types = &operation.additional_error_content_types;
operation
.error_status
.into_iter()
.chain(operation.additional_error_statuses.iter().copied())
.map(|status| {
quote! {
operation.response_content_types::<#ty>(
#status,
&["application/json", #(#additional_content_types),*],
"Error response",
)?;
}
})
.collect::<Vec<_>>()
})
.unwrap_or_default();
let bearer_auth = operation.bearer_auth.then(|| {
quote! { operation.bearer_authentication(); }
});
let browser_auth = operation.browser_auth.then(|| {
quote! { operation.browser_authentication(); }
});
quote! { quote! {
{ {
let mut operation = builder.operation(#method, #path, #operation_id)?; let mut operation = builder.operation(#method, #path, #operation_id)?;
#(#parameters)* #(#parameters)*
#bearer_auth #bearer_auth
#browser_auth #browser_auth
#request #request
#(#success_responses)* #(#success_responses)*
#error #(#errors)*
#(#additional_errors)* operation.finish()?;
operation.finish()?; }
} }
} });
});
quote! { quote! {
#[doc = "Build this trait's deterministic, deployment-independent OpenAPI 3.1 document."] #[doc = "Build this trait's deterministic, deployment-independent OpenAPI 3.1 document."]
@@ -2660,6 +2799,13 @@ fn metadata_tokens(
} else { } else {
quote!(#api_crate::WireKind::Empty) quote!(#api_crate::WireKind::Empty)
}; };
let content_types = if response.body.as_ref().is_some_and(is_binary_body_type) {
quote!(&["application/octet-stream"])
} else if response.body.is_some() {
quote!(&["application/json"])
} else {
quote!(&[])
};
let headers = response.headers.iter().map(|header| { let headers = response.headers.iter().map(|header| {
let wire_name = &header.wire_name; let wire_name = &header.wire_name;
let ty = &header.ty; let ty = &header.ty;
@@ -2674,6 +2820,7 @@ fn metadata_tokens(
#api_crate::ResponseMetadata { #api_crate::ResponseMetadata {
status: #status, status: #status,
body: #api_crate::BodyMetadata { wire_kind: #wire_kind }, body: #api_crate::BodyMetadata { wire_kind: #wire_kind },
content_types: #content_types,
headers: &[#(#headers),*], headers: &[#(#headers),*],
} }
} }
@@ -2683,10 +2830,12 @@ fn metadata_tokens(
.into_iter() .into_iter()
.chain(operation.additional_error_statuses.iter().copied()); .chain(operation.additional_error_statuses.iter().copied());
let error_responses = error_statuses.map(|status| { let error_responses = error_statuses.map(|status| {
let additional_content_types = &operation.additional_error_content_types;
quote! { quote! {
#api_crate::ResponseMetadata { #api_crate::ResponseMetadata {
status: #status, status: #status,
body: #api_crate::BodyMetadata { wire_kind: #api_crate::WireKind::Json }, body: #api_crate::BodyMetadata { wire_kind: #api_crate::WireKind::Json },
content_types: &["application/json", #(#additional_content_types),*],
headers: &[], headers: &[],
} }
} }
+2
View File
@@ -7,6 +7,7 @@ publish = false
[features] [features]
default = [] default = []
error-contract = ["dep:error-contract"]
axum = ["dep:axum", "dep:serde"] axum = ["dep:axum", "dep:serde"]
openapi = ["dep:schemars", "dep:serde_json", "api-macros-impl/openapi"] openapi = ["dep:schemars", "dep:serde_json", "api-macros-impl/openapi"]
reqwest = [ reqwest = [
@@ -20,6 +21,7 @@ reqwest = [
[dependencies] [dependencies]
api-macros-impl = { path = "../api-macros-impl" } api-macros-impl = { path = "../api-macros-impl" }
error-contract = { path = "../error-contract", optional = true }
bytes.workspace = true bytes.workspace = true
axum = { workspace = true, optional = true } axum = { workspace = true, optional = true }
futures = { workspace = true, optional = true } futures = { workspace = true, optional = true }
+2
View File
@@ -6,4 +6,6 @@ Defines transport-independent API metadata from public traits. The same declarat
- [`api`] — API declaration and validation - [`api`] — API declaration and validation
- [`ApiContract`], [`Operation`], and [`WebSocketOperation`] — typed operation contracts - [`ApiContract`], [`Operation`], and [`WebSocketOperation`] — typed operation contracts
- [`HttpError`] — opt-in public error encoding, with unchanged JSON defaults
- `axum`, `reqwest`, and `openapi` — adapters and document generation enabled by their respective features - `axum`, `reqwest`, and `openapi` — adapters and document generation enabled by their respective features
- `http` — projected error-contract response conversion and bounded reception, enabled by `error-contract` without enabling a framework
+28
View File
@@ -96,6 +96,34 @@ pub fn error_response<T: Serialize + Send + Sync + 'static>(
response response
} }
/// Apply the optional public encoding hook, retaining the original server-local error.
///
/// Invalid hook metadata fails closed with an empty 500 response and no typed extension.
/// A hook failure must not expose a potentially private legacy serialization.
pub fn http_error_response<T: crate::HttpError + Serialize + Send + Sync + 'static>(
status: StatusCode,
error: T,
) -> Response {
let Some(encoded) = error.custom_response() else {
return error_response(status, error);
};
let Ok(status) = StatusCode::from_u16(encoded.status) else {
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
};
let Ok(content_type) = axum::http::HeaderValue::from_str(encoded.content_type) else {
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
};
if !(status.is_client_error() || status.is_server_error()) || content_type.is_empty() {
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
}
let mut response = (status, encoded.body.into_bytes()).into_response();
response
.headers_mut()
.insert(axum::http::header::CONTENT_TYPE, content_type);
response.extensions_mut().insert(std::sync::Arc::new(error));
response
}
/// Emit an exact binary response with the contract status and octet-stream content type. /// Emit an exact binary response with the contract status and octet-stream content type.
pub fn binary_response(status: StatusCode, value: crate::BinaryBody) -> Response { pub fn binary_response(status: StatusCode, value: crate::BinaryBody) -> Response {
( (
+56
View File
@@ -0,0 +1,56 @@
//! Shared HTTP error-contract adapter, independent of Axum and Reqwest.
//!
//! Only Browser-projected failures can be encoded here. Receiving a valid failure does not
//! grant permission to forward it: domains must reproject with current authorization.
pub use error_contract::problem::{ApiBoundary, ProblemResponse};
use error_contract::{
Browser, Budget, ContractError, DecodeFailure, ProjectedFailure, WireDetails,
};
/// Encode a projected failure or a static, non-domain 500 fallback under the same budget.
///
/// `safe_instance` must be selected by the caller's disclosure policy (for example a local
/// opaque occurrence URI), never copied from a request URI containing credentials or queries.
/// URI syntax validation is not disclosure authorization. If even the static fallback exceeds
/// the budget, return an error instead of truncating JSON or silently exceeding the limit.
pub fn problem_response<D: WireDetails>(
projected: &ProjectedFailure<Browser, D>,
boundary: &impl ApiBoundary<D>,
safe_instance: &str,
budget: Budget,
) -> Result<ProblemResponse, ContractError> {
error_contract::problem::to_problem_details(projected, boundary, safe_instance, budget)
.or_else(|_| error_contract::problem::problem_fallback(budget))
}
impl From<ProblemResponse> for crate::HttpErrorResponse {
fn from(response: ProblemResponse) -> Self {
Self {
status: response.status,
content_type: response.content_type,
body: response.body().to_vec().into(),
}
}
}
/// Normalize a bounded HTTP error using actual transport metadata, not a body-claimed status.
///
/// Unsupported schema/reason, corrupt input, and `Corrupt(TooLarge)` remain distinct. None of
/// these results, including a known cause, establish remote execution or retry permission.
/// Streaming callers must also bound collection before calling this byte-slice adapter.
pub fn decode_problem_response<D: WireDetails>(
input: &[u8],
actual_status: u16,
content_type: &str,
boundary: &impl ApiBoundary<D>,
budget: Budget,
) -> DecodeFailure<D> {
error_contract::problem::decode_problem_details(
input,
actual_status,
content_type,
boundary,
budget,
)
}
+32
View File
@@ -14,6 +14,16 @@ extern crate self as api_macros;
/// leave authentication, authorization, body limits, and timeouts to the enclosing application. /// leave authentication, authorization, body limits, and timeouts to the enclosing application.
/// Declarations without adapters do not depend on an HTTP framework. /// Declarations without adapters do not depend on an HTTP framework.
/// ///
/// `#[api(reqwest, client_error_hook)]` delegates error decoding to the error type's
/// `reqwest::HttpClientError` implementation. A mixed-error API can instead set
/// `client_error_hook = true` on individual HTTP operations. Either flag enables the hook;
/// a method-level `false` does not override a trait-wide opt-in. Without opt-in, client-only
/// errors need only serde deserialization; server traits and codecs are not imposed on them.
///
/// `additional_error_content_types = ["application/problem+json"]` on an HTTP operation
/// declares extra representations for every declared error status, alongside legacy JSON.
/// Each representation uses the error type's schema; the producer still owns its encoding.
///
/// This declaration checks that a Rust trait and typed operation markers can be defined together. /// This declaration checks that a Rust trait and typed operation markers can be defined together.
/// ///
/// ``` /// ```
@@ -71,6 +81,8 @@ pub use api_macros_impl::api;
#[cfg(feature = "axum")] #[cfg(feature = "axum")]
pub mod axum; pub mod axum;
#[cfg(feature = "error-contract")]
pub mod http;
#[cfg(feature = "openapi")] #[cfg(feature = "openapi")]
pub mod openapi; pub mod openapi;
#[cfg(feature = "reqwest")] #[cfg(feature = "reqwest")]
@@ -213,6 +225,8 @@ pub struct ResponseHeaderMetadata {
pub struct ResponseMetadata { pub struct ResponseMetadata {
pub status: u16, pub status: u16,
pub body: BodyMetadata, pub body: BodyMetadata,
/// Declared media types, empty for bodyless responses. JSON remains the default error media.
pub content_types: &'static [&'static str],
pub headers: &'static [ResponseHeaderMetadata], pub headers: &'static [ResponseHeaderMetadata],
} }
@@ -272,6 +286,24 @@ pub trait WebSocketOperation {
/// service's typed 401/403/404/409 responses without introducing duplicate route handlers. /// service's typed 401/403/404/409 responses without introducing duplicate route handlers.
pub trait HttpError { pub trait HttpError {
fn status_code(&self) -> u16; fn status_code(&self) -> u16;
/// Opt in to an already encoded public response instead of the legacy JSON representation.
/// The adapter retains the original error as a server-local `Arc<Self>` extension.
/// Returning `None` preserves the existing JSON/status behavior exactly.
fn custom_response(&self) -> Option<HttpErrorResponse> {
None
}
}
/// An explicitly encoded HTTP error, independent of any server or client framework.
///
/// Producers own disclosure and byte limits. Adapters reject non-error statuses and invalid
/// content types rather than substituting the legacy representation, which may not be public.
#[derive(Debug)]
pub struct HttpErrorResponse {
pub status: u16,
pub content_type: &'static str,
pub body: BinaryBody,
} }
/// Successful JSON response that selects one of an operation's declared success statuses. /// Successful JSON response that selects one of an operation's declared success statuses.
+46 -6
View File
@@ -39,6 +39,19 @@ pub trait OpenApiSchema: JsonSchema {
fn openapi_schema_name() -> Cow<'static, str> { fn openapi_schema_name() -> Cow<'static, str> {
Self::schema_name() Self::schema_name()
} }
/// Select the exact wire schema for an explicitly declared response media type.
/// Custom error adapters may have distinct JSON and Problem Details bodies;
/// this avoids claiming an ambiguous untagged body union for either medium.
fn response_schema(
builder: &mut OpenApiBuilder,
_content_type: &str,
) -> Result<Value, OpenApiError>
where
Self: Sized,
{
builder.schema_ref::<Self>()
}
} }
macro_rules! impl_openapi_schema { macro_rules! impl_openapi_schema {
@@ -256,7 +269,8 @@ impl OpenApiBuilder {
Ok(OpenApiDocument(document)) Ok(OpenApiDocument(document))
} }
fn schema_ref<T: OpenApiSchema>(&mut self) -> Result<Value, OpenApiError> { /// Register an explicitly asserted wire schema, retaining collision and bounds checks.
pub fn schema_ref<T: OpenApiSchema>(&mut self) -> Result<Value, OpenApiError> {
let name = T::openapi_schema_name().into_owned(); let name = T::openapi_schema_name().into_owned();
validate_component_name(&name)?; validate_component_name(&name)?;
@@ -411,13 +425,39 @@ impl OpenApiOperation<'_> {
content_type: &'static str, content_type: &'static str,
description: &'static str, description: &'static str,
) -> Result<(), OpenApiError> { ) -> Result<(), OpenApiError> {
let schema = self.parent.schema_ref::<T>()?; self.response_content_types::<T>(status, &[content_type], description)
}
/// Declare one response schema under several explicit representations, atomically.
/// This does not change conflict detection for repeated status declarations.
pub fn response_content_types<T: OpenApiSchema>(
&mut self,
status: u16,
content_types: &[&str],
description: &'static str,
) -> Result<(), OpenApiError> {
let mut content = Map::new();
for content_type in content_types {
let schema = T::response_schema(self.parent, content_type)?;
if content
.insert((*content_type).to_owned(), json!({ "schema": schema }))
.is_some()
{
return Err(OpenApiError::InvalidContract(format!(
"duplicate response content type for `{}`",
self.operation_id
)));
}
}
if content.is_empty() {
return Err(OpenApiError::InvalidContract(format!(
"response content types are empty for `{}`",
self.operation_id
)));
}
self.insert_response( self.insert_response(
status, status,
json!({ json!({ "description": description, "content": content }),
"description": description,
"content": { content_type: { "schema": schema } },
}),
) )
} }
+85 -7
View File
@@ -482,6 +482,17 @@ pub fn encode_binary(value: crate::BinaryBody) -> EncodedBody {
EncodedBody::binary(value) EncodedBody::binary(value)
} }
/// Explicit decoder for APIs declared with `#[api(reqwest, client_error_hook)]` or an
/// operation-level `client_error_hook = true` option.
///
/// There is deliberately no blanket `Deserialize` or `HttpError` bound: legacy client-only
/// errors keep serde decoding without implementing this trait. Opted-in decoders receive the
/// actual status/headers and bytes collected under the client's configured limit. Returning
/// a failure does not imply anything about remote execution or authorize a new request.
pub trait HttpClientError: Sized {
fn decode_error(response: &ReceivedResponse) -> Result<Self, ClientFailure>;
}
/// Response received under the configured byte limit. Its content is always redacted in Debug. /// Response received under the configured byte limit. Its content is always redacted in Debug.
pub struct ReceivedResponse { pub struct ReceivedResponse {
status: StatusCode, status: StatusCode,
@@ -494,6 +505,48 @@ impl ReceivedResponse {
self.status self.status
} }
/// Borrow bytes already collected under the configured response limit.
pub fn body(&self) -> &[u8] {
&self.bytes
}
/// Require exactly one valid Content-Type header; ambiguous metadata must not select a codec.
pub fn content_type(&self) -> Result<&str, ClientFailure> {
let name = "content-type";
let mut values = self.headers.get_all(name).iter();
let value = values
.next()
.ok_or(ClientFailure::MissingResponseHeader { name })?;
if values.next().is_some() {
return Err(ClientFailure::DuplicateResponseHeader { name });
}
value
.to_str()
.map_err(|_| ClientFailure::InvalidResponseHeader { name })
}
/// Decode without promoting received data to a public projection or execution result.
/// Transport collection and contract budgets are enforced independently.
#[cfg(feature = "error-contract")]
pub fn decode_problem<D: error_contract::WireDetails>(
&self,
boundary: &impl crate::http::ApiBoundary<D>,
budget: error_contract::Budget,
) -> error_contract::DecodeFailure<D> {
match self.content_type() {
Ok(content_type) => crate::http::decode_problem_response(
&self.bytes,
self.status.as_u16(),
content_type,
boundary,
budget,
),
Err(_) => {
error_contract::DecodeFailure::Corrupt(error_contract::ContractError::Invalid)
}
}
}
pub fn parse_header<T>(&self, name: &'static str) -> Result<T, ClientFailure> pub fn parse_header<T>(&self, name: &'static str) -> Result<T, ClientFailure>
where where
T: std::str::FromStr, T: std::str::FromStr,
@@ -555,13 +608,32 @@ pub enum ClientFailure {
Connect, Connect,
Transport, Transport,
Timeout, Timeout,
ResponseTooLarge { limit: usize }, ResponseTooLarge {
UnexpectedStatus { expected: u16, actual: u16 }, limit: usize,
ErrorResponseDecode { status: u16 }, },
Decode { kind: DecodeKind }, UnexpectedStatus {
MissingResponseHeader { name: &'static str }, expected: u16,
DuplicateResponseHeader { name: &'static str }, actual: u16,
InvalidResponseHeader { name: &'static str }, },
ErrorResponseDecode {
status: u16,
},
/// A bounded, structurally valid error used an unsupported schema or reason.
UnsupportedErrorResponse {
status: u16,
},
Decode {
kind: DecodeKind,
},
MissingResponseHeader {
name: &'static str,
},
DuplicateResponseHeader {
name: &'static str,
},
InvalidResponseHeader {
name: &'static str,
},
UnexpectedBody, UnexpectedBody,
} }
@@ -588,6 +660,12 @@ impl fmt::Display for ClientFailure {
"expected HTTP status {expected}, received {actual}" "expected HTTP status {expected}, received {actual}"
) )
} }
Self::UnsupportedErrorResponse { status } => {
write!(
formatter,
"HTTP {status} error response contract was unsupported"
)
}
Self::ErrorResponseDecode { status } => { Self::ErrorResponseDecode { status } => {
write!(formatter, "HTTP {status} error response JSON was malformed") write!(formatter, "HTTP {status} error response JSON was malformed")
} }
+463
View File
@@ -0,0 +1,463 @@
//! Connect generated Axum responses to generated Reqwest opt-in decoding over isolated loopback.
//! Domain disclosure is a fixture; HTTP encoding, bounded reads and metadata checks are production.
#![cfg(all(feature = "axum", feature = "reqwest", feature = "error-contract"))]
use std::{sync::Arc, time::Duration};
use api_macros::{
HttpErrorResponse, api,
reqwest::{ClientError, ClientFailure, HttpClientError, ReceivedResponse},
};
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode},
response::IntoResponse,
routing::get,
};
use error_contract::{
Browser, Budget, Category, ContractError, DecodeFailure, Disclosure, FailureCause,
FailureFacts, FailureIdentity, ProjectFor, ProjectedFailure, WireDetails, project_failure,
};
use serde::{Deserialize, Serialize};
use tower::ServiceExt;
#[derive(Serialize, Deserialize)]
pub struct Payload {
value: String,
}
#[derive(Clone, Serialize, Deserialize)]
pub struct Denied {
required_operation: String,
}
impl FailureCause for Denied {
fn identity(&self) -> FailureIdentity {
FailureIdentity {
domain: "subject",
reason: "operation_denied",
category: Category::Authorization,
details_version: 1,
}
}
fn validate(&self) -> Result<(), ContractError> {
if self.required_operation == "session.record" {
Ok(())
} else {
Err(ContractError::InvalidDetails)
}
}
}
impl WireDetails for Denied {
fn supports(domain: &str, reason: &str, version: u16) -> bool {
domain == "subject" && reason == "operation_denied" && version == 1
}
}
impl ProjectFor<Browser> for Denied {
type Authorization = ();
type Details = Self;
fn disclose(&self, _: &FailureFacts<Self>, _: &()) -> Result<Disclosure<Self>, ContractError> {
Ok(Disclosure::new(
self.clone(),
"Subject cannot record a session.",
))
}
}
struct Boundary;
impl api_macros::http::ApiBoundary<Denied> for Boundary {
fn classify(&self, _: &Denied) -> error_contract::problem::ProblemClass {
error_contract::problem::ProblemClass::authorization_denied()
}
}
fn budget() -> Budget {
Budget::new(4096, 4096)
}
fn projected() -> ProjectedFailure<Browser, Denied> {
project_failure(
&FailureFacts::new(Denied {
required_operation: "session.record".into(),
})
.unwrap(),
&(),
)
.unwrap()
}
fn encoded() -> HttpErrorResponse {
api_macros::http::problem_response(&projected(), &Boundary, "urn:yoi:failure:local-1", budget())
.unwrap()
.into()
}
// Not Clone: the original must be retained without duplicating a domain error.
#[derive(Serialize)]
pub struct ServerError {
legacy: &'static str,
#[serde(skip)]
mode: u8,
}
impl api_macros::HttpError for ServerError {
fn status_code(&self) -> u16 {
422
}
fn custom_response(&self) -> Option<HttpErrorResponse> {
match self.mode {
0 => Some(encoded()),
1 => None,
mode => Some(HttpErrorResponse {
status: if mode == 2 { 200 } else { 403 },
content_type: if mode == 3 {
"bad\ncontent-type"
} else {
"application/problem+json"
},
body: b"private-hook-body".to_vec().into(),
}),
}
}
}
impl api_macros::HttpRequestError for ServerError {
fn from_request_rejection(_: u16, _: String) -> Self {
Self {
legacy: "never serialize",
mode: 0,
}
}
}
#[api(axum)]
pub trait HookServer {
#[get("/problem", status = 200, error_status = 422)]
async fn problem(&self) -> Result<Payload, ServerError>;
#[post("/reject", error_status = 422, normalize_body_errors = true)]
async fn reject(&self, #[body] input: Payload) -> Result<Payload, ServerError>;
#[post("/reject-binary", error_status = 422, normalize_body_errors = true)]
async fn reject_binary(
&self,
#[binary] input: api_macros::BinaryBody,
) -> Result<Payload, ServerError>;
}
pub struct Service;
impl HookServer for Service {
async fn problem(&self) -> Result<Payload, ServerError> {
Err(ServerError {
legacy: "never serialize",
mode: 0,
})
}
async fn reject(&self, input: Payload) -> Result<Payload, ServerError> {
Ok(input)
}
async fn reject_binary(&self, _input: api_macros::BinaryBody) -> Result<Payload, ServerError> {
Ok(Payload { value: "ok".into() })
}
}
// Neither Deserialize nor HttpError: client hooks impose no server or serde error bounds.
pub struct DecodedError {
operation: String,
}
impl HttpClientError for DecodedError {
fn decode_error(response: &ReceivedResponse) -> Result<Self, ClientFailure> {
let status = response.status().as_u16();
match response.decode_problem::<Denied>(&Boundary, budget()) {
DecodeFailure::Known(failure) => Ok(Self {
operation: failure.facts().cause().required_operation.clone(),
}),
DecodeFailure::Unsupported(_) => {
Err(ClientFailure::UnsupportedErrorResponse { status })
}
DecodeFailure::Corrupt(ContractError::TooLarge) => {
Err(ClientFailure::ResponseTooLarge {
limit: budget().bytes(),
})
}
DecodeFailure::Corrupt(_) => Err(ClientFailure::ErrorResponseDecode { status }),
}
}
}
#[api(reqwest)]
pub trait HookClient {
#[get("/problem", status = 200, error_status = 422, client_error_hook = true)]
async fn problem(&self) -> Result<Payload, DecodedError>;
#[get(
"/wire/{mode}",
status = 200,
error_status = 422,
client_error_hook = true
)]
async fn wire(&self, mode: String) -> Result<Payload, DecodedError>;
#[get("/legacy", error_status = 422)]
async fn legacy(&self) -> Result<Payload, LegacyClientError>;
}
// A false method flag cannot disable a trait-wide opt-in; the selection is logical OR.
#[api(reqwest, client_error_hook)]
pub trait TraitHookClient {
#[get("/problem", client_error_hook = false)]
async fn problem(&self) -> Result<Payload, DecodedError>;
}
// Client-only legacy errors must still compile and decode without either hook trait.
#[derive(Deserialize)]
pub struct LegacyClientError {
legacy: String,
}
#[api(reqwest)]
pub trait LegacyClient {
#[get("/legacy", error_status = 422)]
async fn legacy(&self) -> Result<Payload, LegacyClientError>;
}
#[tokio::test]
async fn generated_server_hook_overrides_wire_encoding_and_retains_arc_for_errors_and_rejections() {
let app = HookServerAxum::router(Service).layer(axum::extract::DefaultBodyLimit::max(32));
for request in [
Request::builder()
.uri("/problem")
.body(Body::empty())
.unwrap(),
Request::builder()
.method("POST")
.uri("/reject")
.header("content-type", "application/json")
.body(Body::from("{"))
.unwrap(),
Request::builder()
.method("POST")
.uri("/reject-binary")
.body(Body::from(vec![0; 64]))
.unwrap(),
] {
let response = app.clone().oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::FORBIDDEN);
assert_eq!(
response.headers()["content-type"],
"application/problem+json"
);
assert_eq!(
response
.extensions()
.get::<Arc<ServerError>>()
.unwrap()
.mode,
0
);
let bytes = to_bytes(response.into_body(), 4096).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(value["status"], 403);
assert_eq!(
value["yoi_failure"]["details"]["required_operation"],
"session.record"
);
assert!(!String::from_utf8_lossy(&bytes).contains("never serialize"));
}
}
#[tokio::test]
async fn absent_hook_preserves_json_and_invalid_hook_never_falls_back_to_legacy() {
let legacy = api_macros::axum::http_error_response(
StatusCode::UNPROCESSABLE_ENTITY,
ServerError {
legacy: "legacy-value",
mode: 1,
},
);
assert_eq!(legacy.status(), StatusCode::UNPROCESSABLE_ENTITY);
assert_eq!(legacy.headers()["content-type"], "application/json");
assert!(legacy.extensions().get::<Arc<ServerError>>().is_some());
let body = to_bytes(legacy.into_body(), 4096).await.unwrap();
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).unwrap()["legacy"],
"legacy-value"
);
for mode in [2, 3] {
let response = api_macros::axum::http_error_response(
StatusCode::UNPROCESSABLE_ENTITY,
ServerError {
legacy: "private",
mode,
},
);
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert!(response.extensions().get::<Arc<ServerError>>().is_none());
assert!(
to_bytes(response.into_body(), 4096)
.await
.unwrap()
.is_empty()
);
}
}
#[test]
fn problem_conversion_falls_back_without_exceeding_byte_or_depth_budgets() {
for (instance, budget) in [
("invalid URI", budget()),
("", Budget::new(256, 256)),
("", budget().with_depth(1, 1)),
] {
let response =
api_macros::http::problem_response(&projected(), &Boundary, instance, budget).unwrap();
assert_eq!(response.status, 500);
assert_eq!(
response.body(),
error_contract::problem::STATIC_PROBLEM_FALLBACK
);
assert!(response.body().len() <= budget.bytes());
}
assert!(matches!(
api_macros::http::problem_response(&projected(), &Boundary, "", Budget::new(1, 1)),
Err(ContractError::TooLarge)
));
}
async fn wire(axum::extract::Path(mode): axum::extract::Path<String>) -> axum::response::Response {
let mut value: serde_json::Value = serde_json::from_slice(encoded().body.as_ref()).unwrap();
let mut status = StatusCode::FORBIDDEN;
let mut content_type = "application/problem+json; charset=utf-8";
match mode.as_str() {
"unknown" => value["yoi_failure"]["reason"] = "future_reason".into(),
"version" => value["yoi_failure"]["version"] = 99.into(),
"status" => status = StatusCode::CONFLICT,
"content-type" => content_type = "application/json",
"corrupt" => {
return (
status,
[("content-type", content_type)],
"{private-raw-body",
)
.into_response();
}
"large" => {
return (status, [("content-type", content_type)], "x".repeat(8192)).into_response();
}
_ => {}
}
let mut response = (
status,
[("content-type", content_type)],
serde_json::to_vec(&value).unwrap(),
)
.into_response();
if mode == "duplicate" {
response
.headers_mut()
.append("content-type", "application/json".parse().unwrap());
}
if mode == "missing" {
response.headers_mut().remove("content-type");
}
response
}
struct ServerTask(tokio::task::JoinHandle<()>);
impl Drop for ServerTask {
fn drop(&mut self) {
self.0.abort();
}
}
async fn serve() -> (String, ServerTask) {
let app = HookServerAxum::router(Service)
.route("/wire/{mode}", get(wire))
.route(
"/legacy",
get(|| async {
api_macros::axum::http_error_response(
StatusCode::UNPROCESSABLE_ENTITY,
ServerError {
legacy: "legacy-value",
mode: 1,
},
)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let task = ServerTask(tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
}));
(url, task)
}
#[tokio::test]
async fn generated_client_hook_uses_actual_metadata_and_preserves_unknown_corrupt_and_size_failures()
{
let (url, _server) = serve().await;
let client = HookClientClient::builder(&url)
.unwrap()
.request_timeout(Duration::from_secs(5))
.response_body_limit(16384)
.build()
.unwrap();
match client.problem().await {
Err(ClientError::Public { status, error }) => {
assert_eq!(status.as_u16(), 403);
assert_eq!(error.operation, "session.record");
}
_ => panic!("expected a typed problem"),
}
for mode in ["unknown", "version"] {
assert!(matches!(
client.wire(mode.into()).await,
Err(ClientError::Failure(
ClientFailure::UnsupportedErrorResponse { status: 403 }
))
));
}
for (mode, expected) in [
("status", 409),
("content-type", 403),
("corrupt", 403),
("duplicate", 403),
("missing", 403),
] {
let result = client.wire(mode.into()).await;
assert!(
matches!(result, Err(ClientError::Failure(ClientFailure::ErrorResponseDecode { status })) if status == expected),
"mode={mode}"
);
}
// Contract budget and streaming transport budget are independent, bounded failures.
assert!(matches!(
client.wire("large".into()).await,
Err(ClientError::Failure(ClientFailure::ResponseTooLarge {
limit: 4096
}))
));
let small_client = HookClientClient::builder(&url)
.unwrap()
.request_timeout(Duration::from_secs(5))
.response_body_limit(128)
.build()
.unwrap();
assert!(matches!(
small_client.problem().await,
Err(ClientError::Failure(ClientFailure::ResponseTooLarge {
limit: 128
}))
));
match client.legacy().await {
Err(ClientError::Public { status, error }) => {
assert_eq!(status.as_u16(), 422);
assert_eq!(error.legacy, "legacy-value");
}
_ => panic!("unrelated error on the same trait must keep serde behavior"),
}
let trait_client = TraitHookClientClient::builder(&url)
.unwrap()
.request_timeout(Duration::from_secs(5))
.build()
.unwrap();
assert!(
matches!(trait_client.problem().await, Err(ClientError::Public { error, .. }) if error.operation == "session.record")
);
let legacy = LegacyClientClient::builder(&url)
.unwrap()
.request_timeout(Duration::from_secs(5))
.build()
.unwrap();
match legacy.legacy().await {
Err(ClientError::Public { status, error }) => {
assert_eq!(status.as_u16(), 422);
assert_eq!(error.legacy, "legacy-value");
}
_ => panic!("legacy client-only error must keep serde behavior"),
}
}
+152 -1
View File
@@ -3,7 +3,7 @@
use api_macros::{ use api_macros::{
BinaryBody, api, BinaryBody, api,
openapi::{OpenApiInfo, OpenApiSchema}, openapi::{OpenApiBuilder, OpenApiInfo, OpenApiSchema},
}; };
use schemars::JsonSchema; use schemars::JsonSchema;
use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -465,3 +465,154 @@ fn file_export_writes_the_canonical_document() {
std::fs::remove_file(path).expect("remove export"); std::fs::remove_file(path).expect("remove export");
assert_eq!(written, document.to_json().expect("canonical JSON")); assert_eq!(written, document.to_json().expect("canonical JSON"));
} }
#[derive(Deserialize, Serialize)]
#[serde(untagged, deny_unknown_fields)]
pub enum ProblemOrLegacy {
Legacy { code: String },
Problem { status: u16, detail: String },
}
impl JsonSchema for ProblemOrLegacy {
fn schema_name() -> std::borrow::Cow<'static, str> {
"ProblemOrLegacy".into()
}
fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
// The owner explicitly attests disjoint closed object shapes instead of admitting anyOf.
schemars::json_schema!({ "oneOf": [
{ "type": "object", "required": ["code"], "additionalProperties": false,
"properties": { "code": { "type": "string" } } },
{ "type": "object", "required": ["status", "detail"], "additionalProperties": false,
"properties": { "status": { "type": "integer", "format": "uint16" }, "detail": { "type": "string" } } }
] })
}
}
impl OpenApiSchema for ProblemOrLegacy {}
#[api(openapi)]
pub trait ErrorMediaApi {
#[post(
"/create",
error_status = 403,
additional_error_statuses = [409, 504],
additional_error_content_types = ["Application/Problem+JSON"]
)]
async fn create(&self) -> Result<Widget, ProblemOrLegacy>;
#[get("/legacy", error_status = 422)]
async fn legacy(&self) -> Result<Widget, PublicError>;
}
#[test]
fn additional_error_media_share_the_declared_schema_for_every_error_status_only() {
use api_macros::{ApiContract, Operation};
let document = error_media_api_openapi(OpenApiInfo {
title: "Error media",
version: "1",
source_digest: "fixture:error-media",
})
.unwrap();
let value = document.as_value();
let responses = &value["paths"]["/create"]["post"]["responses"];
for status in ["403", "409", "504"] {
let content = responses[status]["content"].as_object().unwrap();
assert_eq!(content.len(), 2);
assert_eq!(
content["application/json"]["schema"],
json!({ "$ref": "#/components/schemas/ProblemOrLegacy" })
);
assert_eq!(
content["application/problem+json"],
content["application/json"]
);
}
assert_eq!(responses["200"]["content"].as_object().unwrap().len(), 1);
assert!(
responses["200"]["content"]
.get("application/problem+json")
.is_none()
);
assert_eq!(
value["paths"]["/legacy"]["get"]["responses"]["422"]["content"]
.as_object()
.unwrap()
.len(),
1
);
assert!(value["components"]["schemas"]["ProblemOrLegacy"]["oneOf"].is_array());
let parsed = oas3::from_json(&document.to_json().unwrap()).unwrap();
assert_eq!(parsed.validate_version().unwrap().major, 3);
let operation = error_media_api_operations::Create::METADATA;
assert_eq!(
operation
.error_responses
.iter()
.map(|response| response.status)
.collect::<Vec<_>>(),
[403, 409, 504]
);
for response in operation.error_responses {
assert_eq!(
response.content_types,
["application/json", "application/problem+json"]
);
}
assert_eq!(
operation.success_responses[0].content_types,
["application/json"]
);
let legacy = ErrorMediaApiMetadata::OPERATIONS
.iter()
.find(|operation| operation.path == "/legacy")
.unwrap();
assert_eq!(
legacy.error_responses[0].content_types,
["application/json"]
);
}
#[derive(JsonSchema)]
pub struct MediaSpecificError;
impl OpenApiSchema for MediaSpecificError {
fn response_schema(
builder: &mut OpenApiBuilder,
content_type: &str,
) -> Result<Value, api_macros::openapi::OpenApiError> {
match content_type {
"application/json" => builder.schema_ref::<PublicError>(),
"application/problem+json" => builder.schema_ref::<ProblemOrLegacy>(),
_ => unreachable!("only declared media are selected"),
}
}
}
#[api(openapi)]
pub trait MediaSpecificApi {
#[post("/media", error_status = 409, additional_error_content_types = ["application/problem+json"])]
async fn create(&self) -> Result<Widget, MediaSpecificError>;
}
#[test]
fn explicitly_different_error_representations_register_their_actual_schema_closures() {
let document = media_specific_api_openapi(OpenApiInfo {
title: "Media-specific",
version: "1",
source_digest: "fixture:media-specific",
})
.unwrap();
let value = document.as_value();
let content = &value["paths"]["/media"]["post"]["responses"]["409"]["content"];
assert_eq!(
content["application/json"]["schema"],
json!({"$ref":"#/components/schemas/PublicError"})
);
assert_eq!(
content["application/problem+json"]["schema"],
json!({"$ref":"#/components/schemas/ProblemOrLegacy"})
);
assert!(value["components"]["schemas"]["MediaSpecificError"].is_null());
oas3::from_json(&document.to_json().unwrap())
.unwrap()
.validate_version()
.unwrap();
}
@@ -0,0 +1,6 @@
use api_macros::api;
#[api(client_error_hook)]
pub trait Invalid {}
fn main() {}
@@ -0,0 +1,7 @@
error: `client_error_hook` requires the `reqwest` adapter
--> tests/ui/client_hook_requires_reqwest.rs:3:1
|
3 | #[api(client_error_hook)]
| ^^^^^^^^^^^^^^^^^^^^^^^^^
|
= note: this error originates in the attribute macro `api` (in Nightly builds, run with -Z macro-backtrace for more info)
@@ -0,0 +1,47 @@
use api_macros::api;
pub struct Error;
#[api]
pub trait Wildcard {
#[get("/wildcard", additional_error_content_types = ["application/*"])]
async fn wildcard(&self) -> Result<(), Error>;
}
#[api]
pub trait DuplicateMedia {
#[get("/duplicate", additional_error_content_types = ["application/problem+json", "APPLICATION/PROBLEM+JSON"])]
async fn duplicate(&self) -> Result<(), Error>;
}
#[api]
pub trait DefaultMedia {
#[get("/default", additional_error_content_types = ["application/json"])]
async fn default_media(&self) -> Result<(), Error>;
}
#[api]
pub trait NoError {
#[get("/infallible", additional_error_content_types = ["application/problem+json"])]
async fn infallible(&self);
}
#[api]
pub trait EmptyMedia {
#[get("/empty", additional_error_content_types = [])]
async fn empty(&self) -> Result<(), Error>;
}
#[api]
pub trait DuplicateOption {
#[get("/duplicate-option", additional_error_content_types = [], additional_error_content_types = [])]
async fn duplicate_option(&self) -> Result<(), Error>;
}
#[api]
pub trait ParameterizedMedia {
#[get("/parameterized", additional_error_content_types = ["application/problem+json; charset=utf-8"])]
async fn parameterized(&self) -> Result<(), Error>;
}
fn main() {}
@@ -0,0 +1,41 @@
error: error content type must be a concrete type/subtype without parameters
--> tests/ui/invalid_error_media.rs:7:58
|
7 | #[get("/wildcard", additional_error_content_types = ["application/*"])]
| ^^^^^^^^^^^^^^^
error: additional error content types must be distinct from each other and application/json
--> tests/ui/invalid_error_media.rs:13:87
|
13 | #[get("/duplicate", additional_error_content_types = ["application/problem+json", "APPLICATION/PROBLEM+JSON"])]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^
error: additional error content types must be distinct from each other and application/json
--> tests/ui/invalid_error_media.rs:19:57
|
19 | #[get("/default", additional_error_content_types = ["application/json"])]
| ^^^^^^^^^^^^^^^^^^
error: additional_error_content_types require a public error response type
--> tests/ui/invalid_error_media.rs:25:11
|
25 | #[get("/infallible", additional_error_content_types = ["application/problem+json"])]
| ^^^^^^^^^^^^^
error: additional_error_content_types must not be empty
--> tests/ui/invalid_error_media.rs:31:11
|
31 | #[get("/empty", additional_error_content_types = [])]
| ^^^^^^^^
error: duplicate `additional_error_content_types` route option
--> tests/ui/invalid_error_media.rs:37:69
|
37 | #[get("/duplicate-option", additional_error_content_types = [], additional_error_content_types = [])]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error: error content type must be a concrete type/subtype without parameters
--> tests/ui/invalid_error_media.rs:43:63
|
43 | #[get("/parameterized", additional_error_content_types = ["application/problem+json; charset=utf-8"])]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -0,0 +1,23 @@
use api_macros::api;
pub struct Error;
#[api]
pub trait NoReqwest {
#[get("/error", client_error_hook = true)]
async fn error(&self) -> Result<(), Error>;
}
#[api(reqwest)]
pub trait NoError {
#[post("/infallible", client_error_hook = true)]
async fn infallible(&self);
}
#[api(reqwest)]
pub trait DuplicateHook {
#[get("/duplicate", client_error_hook = true, client_error_hook = false)]
async fn duplicate(&self) -> Result<(), Error>;
}
fn main() {}
@@ -0,0 +1,17 @@
error: client_error_hook requires the `reqwest` adapter
--> tests/ui/operation_hook_invalid_combinations.rs:7:41
|
7 | #[get("/error", client_error_hook = true)]
| ^^^^
error: client_error_hook requires a public error response type
--> tests/ui/operation_hook_invalid_combinations.rs:13:47
|
13 | #[post("/infallible", client_error_hook = true)]
| ^^^^
error: duplicate `client_error_hook` route option
--> tests/ui/operation_hook_invalid_combinations.rs:19:51
|
19 | #[get("/duplicate", client_error_hook = true, client_error_hook = false)]
| ^^^^^^^^^^^^^^^^^
+12
View File
@@ -253,6 +253,7 @@ pub enum BackendRuntimeClientError {
Api(BackendApiClientError), Api(BackendApiClientError),
SessionApi(server_api::ServerApiClientError), SessionApi(server_api::ServerApiClientError),
ContractApi(server_api::client_support::ClientError<server_api::RepositoryApiError>), ContractApi(server_api::client_support::ClientError<server_api::RepositoryApiError>),
WorkerCreateApi(server_api::client_support::ClientError<server_api::WorkerCreateApiError>),
Http(reqwest::Error), Http(reqwest::Error),
Protocol(String), Protocol(String),
} }
@@ -264,6 +265,7 @@ impl fmt::Display for BackendRuntimeClientError {
Self::Api(error) => write!(f, "{error}"), Self::Api(error) => write!(f, "{error}"),
Self::SessionApi(error) => write!(f, "{error}"), Self::SessionApi(error) => write!(f, "{error}"),
Self::ContractApi(error) => write!(f, "{error}"), Self::ContractApi(error) => write!(f, "{error}"),
Self::WorkerCreateApi(error) => write!(f, "{error}"),
Self::Http(error) => write!(f, "{error}"), Self::Http(error) => write!(f, "{error}"),
Self::Protocol(message) => f.write_str(message), Self::Protocol(message) => f.write_str(message),
} }
@@ -1016,3 +1018,13 @@ mod tests {
assert!(serde_json::from_value::<BackendWorkerSummary>(stale).is_err()); assert!(serde_json::from_value::<BackendWorkerSummary>(stale).is_err());
} }
} }
impl From<server_api::client_support::ClientError<server_api::WorkerCreateApiError>>
for BackendRuntimeClientError
{
fn from(
error: server_api::client_support::ClientError<server_api::WorkerCreateApiError>,
) -> Self {
Self::WorkerCreateApi(error)
}
}
+30
View File
@@ -39,6 +39,7 @@ pub enum BackendWorkspaceClientError {
Api(BackendApiClientError), Api(BackendApiClientError),
Http(reqwest::Error), Http(reqwest::Error),
ServerApi(server_api::client_support::ClientError<server_api::RepositoryApiError>), ServerApi(server_api::client_support::ClientError<server_api::RepositoryApiError>),
WorkerCreateApi(server_api::client_support::ClientError<server_api::WorkerCreateApiError>),
RuntimeManagementApi( RuntimeManagementApi(
server_api::client_support::ClientError<server_api::RuntimeManagementApiError>, server_api::client_support::ClientError<server_api::RuntimeManagementApiError>,
), ),
@@ -51,6 +52,7 @@ impl fmt::Display for BackendWorkspaceClientError {
Self::Api(error) => write!(f, "{error}"), Self::Api(error) => write!(f, "{error}"),
Self::Http(error) => write!(f, "{error}"), Self::Http(error) => write!(f, "{error}"),
Self::ServerApi(error) => write!(f, "{error}"), Self::ServerApi(error) => write!(f, "{error}"),
Self::WorkerCreateApi(error) => write!(f, "{error}"),
Self::RuntimeManagementApi(error) => write!(f, "{error}"), Self::RuntimeManagementApi(error) => write!(f, "{error}"),
} }
} }
@@ -454,3 +456,31 @@ mod tests {
assert_eq!(retry, request); assert_eq!(retry, request);
} }
} }
pub(crate) fn worker_create_client_error(
backend: &BackendApiClient,
error: server_api::client_support::ClientError<server_api::WorkerCreateApiError>,
) -> BackendWorkspaceClientError {
use server_api::client_support::ClientError;
// Retain typed startup facts even for an operator denial. A Subject execution
// condition is not a refusal of this client's credential or a login prompt.
match &error {
ClientError::Public {
status,
error: response,
} if response.failure().is_none() && *status == reqwest::StatusCode::UNAUTHORIZED => {
BackendWorkspaceClientError::Api(BackendApiClientError::Unauthorized {
origin: backend.origin().clone(),
})
}
ClientError::Public {
status,
error: response,
} if response.failure().is_none() && *status == reqwest::StatusCode::FORBIDDEN => {
BackendWorkspaceClientError::Api(BackendApiClientError::Forbidden {
origin: backend.origin().clone(),
})
}
_ => BackendWorkspaceClientError::WorkerCreateApi(error),
}
}
+20 -3
View File
@@ -81,6 +81,23 @@ impl BackendWorkspaceProductClient {
server_api::client_support::ClientError<server_api::RepositoryApiError>, server_api::client_support::ClientError<server_api::RepositoryApiError>,
>, >,
>, >,
{
self.generated_normalized(operation, crate::backend_workspace::server_client_error)
}
fn generated_normalized<T, E, F, Fut>(
&self,
operation: F,
normalize: impl FnOnce(
&BackendApiClient,
server_api::client_support::ClientError<E>,
) -> BackendWorkspaceClientError,
) -> Result<T, BackendWorkspaceClientError>
where
F: FnOnce(
server_api::ServerApiClient<crate::backend_workspace::ServerBearerAuthorizer>,
) -> Fut,
Fut: std::future::Future<Output = Result<T, server_api::client_support::ClientError<E>>>,
{ {
let client = crate::backend_workspace::server_api_client(&self.api)?; let client = crate::backend_workspace::server_api_client(&self.api)?;
let runtime = tokio::runtime::Builder::new_current_thread() let runtime = tokio::runtime::Builder::new_current_thread()
@@ -89,7 +106,7 @@ impl BackendWorkspaceProductClient {
.map_err(|error| BackendWorkspaceClientError::InvalidTarget(error.to_string()))?; .map_err(|error| BackendWorkspaceClientError::InvalidTarget(error.to_string()))?;
runtime runtime
.block_on(operation(client)) .block_on(operation(client))
.map_err(|error| crate::backend_workspace::server_client_error(&self.api, error)) .map_err(|error| normalize(&self.api, error))
} }
pub fn read_ticket_link_summary( pub fn read_ticket_link_summary(
@@ -584,9 +601,9 @@ impl BackendWorkspaceProductClient {
execution_grant: None, execution_grant: None,
}; };
let workspace_id = self.workspace_id.clone(); let workspace_id = self.workspace_id.clone();
let response: BrowserCreateWorkerResponse = self.generated(move |client| async move { let response: BrowserCreateWorkerResponse = self.generated_normalized(move |client| async move {
client.workspace_worker_create(workspace_id, request).await client.workspace_worker_create(workspace_id, request).await
})?; }, crate::backend_workspace::worker_create_client_error)?;
Ok(format!( Ok(format!(
"Started Intake Worker {}/{} for Ticket {ticket_id}", "Started Intake Worker {}/{} for Ticket {ticket_id}",
response.runtime_id, response.worker_id response.runtime_id, response.worker_id
+8
View File
@@ -162,6 +162,14 @@ fn validate_envelope_common(raw: &ReaderEnvelope) -> Result<(), ContractError> {
raw.context.validate() raw.context.validate()
} }
/// Validate byte and lexical depth limits before parsing an enclosing JSON DTO.
///
/// This is a resource guard only: callers must still use their typed JSON parser
/// to validate grammar and schema. It grants no disclosure or encoding authority.
pub fn validate_json_input(input: &[u8], budget: Budget) -> Result<(), ContractError> {
check_json_bounds(input, budget)
}
/// Linear lexical depth guard runs before serde allocates any JSON tree. Serde validates grammar. /// Linear lexical depth guard runs before serde allocates any JSON tree. Serde validates grammar.
pub(crate) fn check_json_bounds(input: &[u8], budget: Budget) -> Result<(), ContractError> { pub(crate) fn check_json_bounds(input: &[u8], budget: Budget) -> Result<(), ContractError> {
if input.len() > budget.bytes() { if input.len() > budget.bytes() {
+1
View File
@@ -107,6 +107,7 @@ pub struct CausalLink {
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", schemars(rename = "FailureDiagnostic"))]
pub struct Diagnostic { pub struct Diagnostic {
#[cfg_attr(feature = "json-schema", schemars(length(min = 1, max = 128)))] #[cfg_attr(feature = "json-schema", schemars(length(min = 1, max = 128)))]
pub code: String, pub code: String,
+4 -1
View File
@@ -10,7 +10,10 @@ pub mod schema;
#[cfg(feature = "test-support")] #[cfg(feature = "test-support")]
pub mod test_support; pub mod test_support;
pub use codec::{Budget, DecodeFailure, ReceivedFailure, decode_failure, encode_projected_failure}; pub use codec::{
Budget, DecodeFailure, ReceivedFailure, decode_failure, encode_projected_failure,
validate_json_input,
};
pub use facts::*; pub use facts::*;
pub use projection::*; pub use projection::*;
+6 -2
View File
@@ -6,10 +6,14 @@ edition.workspace = true
license.workspace = true license.workspace = true
[dependencies] [dependencies]
api-macros = { workspace = true, features = ["reqwest", "axum"] } api-macros = { workspace = true, features = ["reqwest", "axum", "openapi"] }
manifest.workspace = true manifest.workspace = true
plugin-plan = { path = "../plugin-plan" } plugin-plan = { path = "../plugin-plan" }
protocol.workspace = true protocol = { workspace = true, features = ["json-schema"] }
error-contract = { workspace = true, features = ["json-schema"] }
schemars.workspace = true
sha2.workspace = true
serde_json.workspace = true
session-store.workspace = true session-store.workspace = true
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
workdir.workspace = true workdir.workspace = true
+1
View File
@@ -8,3 +8,4 @@ HTTP contract for Worker Runtime management. Owns REST DTOs and generated client
- [`WorkerRestoreRequest`] / [`WorkerRestoreCoordinationRequest`] — Restore intent and Backend preparation contracts - [`WorkerRestoreRequest`] / [`WorkerRestoreCoordinationRequest`] — Restore intent and Backend preparation contracts
- [`LogicalWorkdirAttachment`] — Logical Workdir attachments determined by Workspace - [`LogicalWorkdirAttachment`] — Logical Workdir attachments determined by Workspace
- [`BackendJobExecutionBinding`] — Job/attempt execution identity fixed by the Host - [`BackendJobExecutionBinding`] — Job/attempt execution identity fixed by the Host
- [`startup`] — projected startup Problem Details, independent operation observations and bounded receiver normalization
+72 -1
View File
@@ -1,5 +1,7 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
pub mod startup;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use api_macros::api; use api_macros::api;
@@ -103,6 +105,9 @@ pub struct RuntimeApiError {
pub error: RuntimeApiErrorDetail, pub error: RuntimeApiErrorDetail,
#[serde(skip)] #[serde(skip)]
status: u16, status: u16,
/// Local owner/normalized observations, never a second wire attachment.
#[serde(skip)]
create_failure: Option<startup::WorkerCreateFailure>,
} }
/// Transient, allowlisted startup observations copied from the execution owner. /// Transient, allowlisted startup observations copied from the execution owner.
@@ -135,18 +140,37 @@ impl RuntimeApiError {
startup: None, startup: None,
}, },
status, status,
create_failure: None,
} }
} }
pub fn status(&self) -> u16 { pub fn status(&self) -> u16 {
self.status self.status
} }
/// Retain owner-selected facts locally; the HTTP hook projects them afresh.
pub fn with_create_failure(mut self, failure: startup::WorkerCreateFailure) -> Self {
self.create_failure = Some(failure);
self
}
pub fn create_failure(&self) -> Option<&startup::WorkerCreateFailure> {
self.create_failure.as_ref()
}
pub fn into_create_failure(self) -> Option<startup::WorkerCreateFailure> {
self.create_failure
}
} }
impl api_macros::HttpError for RuntimeApiError { impl api_macros::HttpError for RuntimeApiError {
fn status_code(&self) -> u16 { fn status_code(&self) -> u16 {
self.status self.status
} }
fn custom_response(&self) -> Option<api_macros::HttpErrorResponse> {
startup::http::runtime_error_response(self)
}
} }
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
@@ -1305,7 +1329,7 @@ pub struct SubjectJobRemovalPurgeResponse {
pub input_fingerprint: String, pub input_fingerprint: String,
} }
#[api(reqwest, axum)] #[api(reqwest, axum, client_error_hook)]
pub trait RuntimeApi { pub trait RuntimeApi {
#[post("/v1/subject-job-removal-purge", status = 200, error_status = 400)] #[post("/v1/subject-job-removal-purge", status = 200, error_status = 400)]
async fn purge_subject_job_removal( async fn purge_subject_job_removal(
@@ -1924,6 +1948,25 @@ mod tests {
} }
async fn get_worker(&self, worker_id: String) -> Result<WorkerResponse, RuntimeApiError> { async fn get_worker(&self, worker_id: String) -> Result<WorkerResponse, RuntimeApiError> {
if worker_id == "startup" {
use startup::*;
let failure = WorkerCreateFailure::from_cause(
StartupCause::Session(
protocol::session_failure::SessionRegistrationCause::denied(
protocol::session_failure::SessionPrincipal::Subject,
),
),
StartupOperationFacts {
acceptance: CreationAcceptance::Unknown,
execution: ExecutionOutcome::Unknown,
finalize: None,
backend_compensation: None,
correlation: None,
},
)
.unwrap();
return Err(test_error(500).with_create_failure(failure));
}
assert_eq!(worker_id, "missing worker"); assert_eq!(worker_id, "missing worker");
Err(test_error(404)) Err(test_error(404))
} }
@@ -2233,6 +2276,34 @@ mod tests {
assert!( assert!(
matches!(error, client_support::ClientError::Public { status, .. } if status.as_u16() == 404) matches!(error, client_support::ClientError::Public { status, .. } if status.as_u16() == 404)
); );
let error = client.get_worker("startup".to_string()).await.unwrap_err();
let client_support::ClientError::Public { status, error } = error else {
panic!("expected a normalized public startup failure");
};
assert_eq!(status.as_u16(), 403);
assert_eq!(error.status(), 403);
let failure = error
.create_failure()
.expect("hook retains non-HTTP observations");
assert_eq!(
failure.operation().acceptance,
startup::CreationAcceptance::Unknown
);
let browser = failure
.problem(
startup::StartupBoundary::Browser,
&startup::SessionFailureVisibility {
can_view_cause: true,
},
"",
)
.unwrap();
assert_eq!(browser.status, 409);
assert!(
std::str::from_utf8(browser.body())
.unwrap()
.contains("session.record")
);
server.abort(); server.abort();
} }
File diff suppressed because it is too large Load Diff
+193
View File
@@ -0,0 +1,193 @@
//! Runtime API HTTP integration. No HTTP status is converted into execution facts.
use api_macros::reqwest::{ClientFailure, HttpClientError, ReceivedResponse};
use error_contract::{DecodeFailure, InternalTransfer};
use serde::Deserialize;
use super::*;
use crate::RuntimeApiError;
fn unknown_operation() -> StartupOperationFacts {
StartupOperationFacts {
acceptance: CreationAcceptance::Unknown,
execution: ExecutionOutcome::Unknown,
finalize: None,
backend_compensation: None,
correlation: None,
}
}
/// RuntimeManagementApi invokes this only after checking authenticated Runtime
/// scope. That scope permits the minimum public-safe cause schema, not arbitrary
/// internal-transfer details. Resolve the public projection afresh on each call;
/// no stored/received projection is relabeled as Browser authorization.
pub(crate) fn runtime_error_response(
error: &RuntimeApiError,
) -> Option<api_macros::HttpErrorResponse> {
let legacy;
let failure = match error.create_failure() {
Some(failure) => failure,
None => {
let startup = error.error.startup.as_ref()?;
legacy = WorkerCreateFailure::new(startup.clone(), unknown_operation());
match &legacy {
Ok(failure) => failure,
Err(_) => return Some(static_fallback()),
}
}
};
let visibility = SessionFailureVisibility {
can_view_cause: true,
};
Some(
match failure.problem(StartupBoundary::Runtime, &visibility, "") {
Ok(response) => api_macros::HttpErrorResponse {
status: response.status,
content_type: response.content_type,
body: response.into_body().into(),
},
// Never expose the legacy/raw attachment if projection cannot be made.
Err(_) => static_fallback(),
},
)
}
fn static_fallback() -> api_macros::HttpErrorResponse {
api_macros::HttpErrorResponse {
status: 500,
content_type: PROBLEM_CONTENT_TYPE,
body: error_contract::problem::STATIC_PROBLEM_FALLBACK
.to_vec()
.into(),
}
}
fn normalized_error(
status: u16,
failure: WorkerCreateFailure,
) -> Result<RuntimeApiError, ClientFailure> {
// Received text and legacy labels are not sources of local explanations.
// InternalTransfer is an internal observation policy, not Browser authority.
let projected = failure
.startup()
.primary
.project_for::<InternalTransfer, StartupCause>(&())
.map_err(|_| ClientFailure::ErrorResponseDecode { status })?;
let identity = projected.identity();
let mut error = RuntimeApiError::new(
status,
format!("{}.{}", identity.domain, identity.reason),
projected.explanation(),
);
error.error.startup = Some(failure.startup().clone());
Ok(error.with_create_failure(failure))
}
fn local_observation(
status: u16,
reason: StartupBoundaryReason,
) -> Result<RuntimeApiError, ClientFailure> {
let failure = WorkerCreateFailure::from_cause(
StartupCause::Boundary(StartupBoundaryCause { reason }),
unknown_operation(),
)
.map_err(|_| ClientFailure::ErrorResponseDecode { status })?;
normalized_error(status, failure)
}
// A deliberately narrow legacy schema. Bounded typed parsing avoids arbitrary JSON
// allocation; startup attachment strings remain opaque until domain codec validation.
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyError {
error: LegacyDetail,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyDetail {
code: String,
message: String,
startup: Option<WorkerStartupFailure>,
}
impl HttpClientError for RuntimeApiError {
fn decode_error(response: &ReceivedResponse) -> Result<Self, ClientFailure> {
let status = response.status().as_u16();
if !(400..=599).contains(&status) {
return Err(ClientFailure::ErrorResponseDecode { status });
}
// The generated client has already bounded collection; enforce this
// smaller contract limit independently before any schema allocation.
if response.body().len() > MAX_STARTUP_PROBLEM_BYTES {
return local_observation(status, StartupBoundaryReason::TooLarge);
}
let content_type = match response.content_type() {
Ok(content_type) => content_type,
Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse),
};
let media_type = content_type.split(';').next().unwrap_or("").trim();
if media_type.eq_ignore_ascii_case(PROBLEM_CONTENT_TYPE) {
return match decode_startup_problem(
response.body(),
status,
content_type,
StartupBoundary::Runtime,
) {
DecodeStartupProblem::Known(failure) => normalized_error(status, failure),
DecodeStartupProblem::UnsupportedVersion => {
local_observation(status, StartupBoundaryReason::UnsupportedVersion)
}
DecodeStartupProblem::UnsupportedReason => {
local_observation(status, StartupBoundaryReason::UnsupportedReason)
}
DecodeStartupProblem::Invalid => {
local_observation(status, StartupBoundaryReason::InvalidResponse)
}
DecodeStartupProblem::TooLarge => {
local_observation(status, StartupBoundaryReason::TooLarge)
}
};
}
if !media_type.eq_ignore_ascii_case("application/json") {
return local_observation(status, StartupBoundaryReason::InvalidResponse);
}
// The typed legacy parser has fixed nesting and rejects unknown fields.
// Its total allocation/opaque attachment size is capped by the byte limit.
let legacy = match serde_json::from_slice::<LegacyError>(response.body()) {
Ok(legacy) => legacy.error,
Err(_) => return local_observation(status, StartupBoundaryReason::Decode),
};
if error_contract::validate_identifier(&legacy.code).is_err()
|| error_contract::validate_explanation(&legacy.message).is_err()
{
return local_observation(status, StartupBoundaryReason::InvalidResponse);
}
let Some(mut startup) = legacy.startup else {
// Preserve ordinary legacy errors exactly, including their actual status.
return Ok(RuntimeApiError::new(status, legacy.code, legacy.message));
};
let projected = match startup.primary.decode::<StartupCause>() {
DecodeFailure::Known(received) => match error_contract::project_failure::<
InternalTransfer,
_,
>(received.facts(), &())
{
Ok(projected) => projected,
Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse),
},
DecodeFailure::Corrupt(error_contract::ContractError::TooLarge) => {
return local_observation(status, StartupBoundaryReason::TooLarge);
}
// The legacy attachment codec intentionally exposes no unsupported
// metadata. Do not guess a reason/version from that opaque evidence.
_ => return local_observation(status, StartupBoundaryReason::InvalidResponse),
};
startup.primary = match StartupFailureAttachment::from_projected(&projected) {
Ok(primary) => primary,
Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse),
};
match WorkerCreateFailure::new(startup, unknown_operation()) {
Ok(failure) => normalized_error(status, failure),
Err(_) => local_observation(status, StartupBoundaryReason::InvalidResponse),
}
}
}
+162
View File
@@ -0,0 +1,162 @@
//! HTTP-independent snapshot of a Backend coordinator failure. This is a codec
//! for the existing reservation result, not a separate ledger or retry authority.
use super::*;
use crate::StoredStartupFailureAttachment;
use error_contract::{Storage, validate_json_input};
const STORAGE_VERSION: u16 = 1;
// Private DTO: only the concrete failure encoder may construct a stored snapshot.
// In particular no arbitrary error/source, HTTP response or private identity fits here.
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct StoredWorkerCreateFailure {
version: u16,
primary: StoredStartupFailureAttachment,
facts: StartupProblemFacts,
}
// Inspect versions from original input before attempting to understand future facts.
// IgnoredAny still checks JSON grammar without allocating arbitrary extension trees.
#[derive(Deserialize)]
struct StorageMetadata {
version: u16,
#[serde(rename = "primary")]
_primary: serde::de::IgnoredAny,
facts: ExtensionMetadata,
}
impl ProjectFor<Storage> for StartupBoundaryCause {
type Authorization = ();
type Details = Self;
fn disclose(&self, _: &FailureFacts<Self>, _: &()) -> Result<Disclosure<Self>, ContractError> {
// The boundary domain stores only its local enum, never received context.
Ok(Disclosure::new(*self, boundary_explanation(self)))
}
}
impl ProjectFor<Storage> for StartupCause {
type Authorization = ();
type Details = Self;
fn disclose(
&self,
facts: &FailureFacts<Self>,
_: &(),
) -> Result<Disclosure<Self>, ContractError> {
match self {
Self::Session(cause) => {
let concrete = domain_facts(*cause, facts)?;
let disclosed = <SessionRegistrationCause as ProjectFor<Storage>>::disclose(
cause,
&concrete,
&(),
)?;
Ok(wrap(disclosed, Self::Session, session_explanation(cause)))
}
Self::Runtime(cause) => {
let concrete = domain_facts(cause.clone(), facts)?;
let disclosed =
<RuntimeStartupCause as ProjectFor<Storage>>::disclose(cause, &concrete, &())?;
Ok(wrap(
disclosed,
Self::Runtime,
runtime_explanation(cause).into(),
))
}
Self::Boundary(cause) => {
let concrete = domain_facts(*cause, facts)?;
let disclosed =
<StartupBoundaryCause as ProjectFor<Storage>>::disclose(cause, &concrete, &())?;
Ok(wrap(
disclosed,
Self::Boundary,
boundary_explanation(cause).into(),
))
}
}
}
}
impl WorkerCreateFailure {
/// Encode a bounded version-1 storage snapshot through each concrete domain's
/// Storage allowlist. Never encodes or parses an HTTP body. The caller selects
/// the Backend persistence marker (including prospective Persisted for an
/// atomic reservation write); encoding alone never proves persistence.
pub fn encode_storage(&self) -> Result<String, ContractError> {
let facts = self.public_facts();
facts.validate()?;
let projected = self
.startup
.primary
.project_for::<Storage, StartupCause>(&())?;
let dto = StoredWorkerCreateFailure {
version: STORAGE_VERSION,
primary: StoredStartupFailureAttachment::from_projected(&projected)?,
facts,
};
let encoded = serde_json::to_string(&dto).map_err(|_| ContractError::Encoding)?;
// Includes escaped attachment bytes and the entire enclosing snapshot.
validate_json_input(encoded.as_bytes(), budget())?;
Ok(encoded)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum DecodeStoredWorkerCreateFailure {
Known(WorkerCreateFailure),
Unsupported,
Corrupt(ContractError),
}
/// Read a bounded storage snapshot without an HTTP classification or boundary.
/// Unsupported/corrupt evidence never becomes a fabricated primary cause, and a
/// Known result still needs current visibility for any later public projection.
/// Missing legacy columns are represented by the store's Option, not this codec.
pub fn decode_stored_worker_create_failure(input: &str) -> DecodeStoredWorkerCreateFailure {
match decode_checked(input) {
Ok(value) => value,
Err(error) => DecodeStoredWorkerCreateFailure::Corrupt(error),
}
}
fn decode_checked(input: &str) -> Result<DecodeStoredWorkerCreateFailure, ContractError> {
use DecodeStoredWorkerCreateFailure::*;
// Run the shared lexical guard over the whole input before any JSON parsing.
validate_json_input(input.as_bytes(), budget())?;
let metadata: StorageMetadata =
serde_json::from_str(input).map_err(|_| ContractError::Invalid)?;
if metadata.version == 0 || metadata.facts.version == 0 {
return Err(ContractError::Invalid);
}
if metadata.version != STORAGE_VERSION || metadata.facts.version != STARTUP_FACTS_VERSION {
return Ok(Unsupported);
}
// Original-byte typed decoding rejects duplicates rather than collapsing a Value.
let dto: StoredWorkerCreateFailure =
serde_json::from_str(input).map_err(|_| ContractError::InvalidDetails)?;
dto.facts.validate()?;
let received = match dto.primary.decode::<StartupCause>() {
DecodeFailure::Known(received) => received,
DecodeFailure::Unsupported(_) => return Ok(Unsupported),
DecodeFailure::Corrupt(error) => return Err(error),
};
// Deserialization is not a capability. Project again instead of relabeling or
// retaining raw loaded attachment bytes (including injected explanations/context).
let projected = project_failure::<Storage, _>(received.facts(), &())?;
let primary = StartupFailureAttachment::from_projected(&projected)?;
let facts = dto.facts;
let failure = WorkerCreateFailure::new(
WorkerStartupFailure {
primary,
secondary: facts.secondary,
resources: facts.resources,
persistence: facts.persistence,
io_kind: facts.io_kind,
},
facts.operation,
)?
.with_backend_result_persistence(facts.backend_result_persistence);
Ok(Known(failure))
}
@@ -0,0 +1,58 @@
{
"detail": "Worker creation could not complete Session registration: Subject is not authorized for required operation session.record. Creation acceptance and execution outcome are reported separately.",
"instance": "/workers/create/fixture-812",
"status": 409,
"title": "Operation conflict",
"type": "urn:yoi:problem:conflict",
"yoi_failure": {
"category": "authorization",
"context": {
"causes": [],
"correlation": null,
"diagnostics": [],
"operations": [],
"supplementary_omitted": false
},
"details": {
"principal": "subject",
"reason": "permission_denied",
"required_operation": "session.record"
},
"details_version": 1,
"domain": "session_registration",
"explanation": "Worker creation could not complete Session registration: Subject is not authorized for required operation session.record. Creation acceptance and execution outcome are reported separately.",
"reason": "permission_denied",
"required_extensions": [],
"version": 1
},
"yoi_startup": {
"io_kind": "permission_denied",
"operation": {
"acceptance": "rejected",
"backend_compensation": null,
"correlation": "startup-fixture-812",
"execution": "rejected",
"finalize": null
},
"persistence": "outcome_unknown",
"resources": {
"cleanup": {
"failed": "workdir"
},
"durable_effects_possible": true,
"resources": "worker_startup"
},
"secondary": [
{
"failure": "workdir",
"kind": "cleanup"
},
{
"kind": "persistence",
"outcome": "outcome_unknown",
"phase": "metadata_save"
}
],
"version": 1
}
}
@@ -0,0 +1,918 @@
//! Producer/consumer checks against the production constructor and decoder.
use error_contract::problem::ApiBoundary as _;
use error_contract::{
Browser, ContractError, DecodeFailure, FailureCause, FailureFacts, InternalTransfer,
OperationContext, ProjectFor, WireDetails, project_failure,
};
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
use runtime_api::startup::*;
use runtime_api::{
RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure,
StartupFailureAttachment, StartupFailurePhase, StartupIoKind, StartupPersistenceOutcome,
StartupResourceFacts, StartupResources, StartupSecondaryFailure, StartupStage,
WorkerStartupFailure,
};
use serde_json::{Value, json};
fn visible() -> SessionFailureVisibility {
SessionFailureVisibility {
can_view_cause: true,
}
}
fn operation(acceptance: CreationAcceptance, execution: ExecutionOutcome) -> StartupOperationFacts {
StartupOperationFacts {
acceptance,
execution,
finalize: None,
backend_compensation: None,
correlation: Some(StartupCorrelation::new("startup-fixture-812").unwrap()),
}
}
fn session() -> StartupCause {
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject))
}
fn failure(cause: StartupCause) -> WorkerCreateFailure {
WorkerCreateFailure::from_cause(
cause,
operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown),
)
.unwrap()
}
fn decode(value: &Value, status: u16, boundary: StartupBoundary) -> DecodeStartupProblem {
decode_startup_problem(
&serde_json::to_vec(value).unwrap(),
status,
PROBLEM_CONTENT_TYPE,
boundary,
)
}
fn known(result: DecodeStartupProblem) -> WorkerCreateFailure {
match result {
DecodeStartupProblem::Known(failure) => failure,
other => panic!("{other:?}"),
}
}
fn fixture_failure() -> WorkerCreateFailure {
let base = failure(session());
let (mut startup, _) = base.into_parts();
startup.secondary = vec![
StartupSecondaryFailure::Cleanup {
failure: StartupCleanupFailure::Workdir,
},
StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::MetadataSave,
outcome: StartupPersistenceOutcome::OutcomeUnknown,
},
];
startup.resources = Some(StartupResourceFacts {
resources: StartupResources::WorkerStartup,
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir),
durable_effects_possible: true,
});
startup.persistence = Some(StartupPersistenceOutcome::OutcomeUnknown);
startup.io_kind = Some(StartupIoKind::PermissionDenied);
WorkerCreateFailure::new(
startup,
operation(CreationAcceptance::Rejected, ExecutionOutcome::Rejected),
)
.unwrap()
}
#[test]
fn production_public_fixture_is_stable_and_consumable() {
let failure = fixture_failure();
let response = failure
.problem(
StartupBoundary::Browser,
&visible(),
"/workers/create/fixture-812",
)
.unwrap();
assert_eq!(response.status, 409);
assert_eq!(response.content_type, PROBLEM_CONTENT_TYPE);
let value: Value = serde_json::from_slice(response.body()).unwrap();
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/startup-session-record-denied.problem.json");
let fixture: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap();
assert_eq!(
value, fixture,
"fixture must be generated by the public production constructor"
);
let decoded = known(decode(&fixture, 409, StartupBoundary::Browser));
assert_eq!(decoded.operation(), failure.operation());
assert_eq!(decoded.public_facts(), failure.public_facts());
assert_eq!(
decoded
.startup()
.primary
.decode::<StartupCause>()
.known_cause(),
session()
);
assert!(fixture["detail"].as_str().unwrap().contains("Subject"));
assert!(
fixture["detail"]
.as_str()
.unwrap()
.contains("session.record")
);
assert!(fixture["detail"].as_str().unwrap().contains("Worker"));
assert!(fixture["yoi_startup"].get("primary").is_none());
}
/// Captured from the real signed Session registration → Runtime → Browser tests
/// in workspace-server. These are observed operation facts, not a declaration
/// that an HTTP conflict rejected creation or that successful cleanup erased effects.
#[test]
fn connected_browser_fixtures_preserve_observed_operation_and_storage_facts() {
for (body, status, cause) in [
(
include_bytes!("fixtures/startup-browser-session-record-denied.problem.json")
.as_slice(),
409,
SessionRegistrationCause::denied(SessionPrincipal::Subject),
),
(
include_bytes!("fixtures/startup-browser-registration-response-lost.problem.json")
.as_slice(),
502,
SessionRegistrationCause::outcome_unknown(),
),
] {
let failure = known(decode_startup_problem(
body,
status,
PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser,
));
assert_eq!(
failure
.startup()
.primary
.decode::<StartupCause>()
.known_cause(),
StartupCause::Session(cause)
);
assert_eq!(failure.operation().acceptance, CreationAcceptance::Unknown);
assert_eq!(failure.operation().execution, ExecutionOutcome::Errored);
assert_eq!(
failure.operation().backend_compensation,
Some(CompensationOutcome::Unknown)
);
assert_eq!(
failure.startup().persistence,
Some(StartupPersistenceOutcome::Persisted)
);
assert_eq!(
failure.backend_result_persistence(),
Some(StartupPersistenceOutcome::Persisted)
);
let encoded = failure.encode_storage().unwrap();
let DecodeStoredWorkerCreateFailure::Known(saved) =
decode_stored_worker_create_failure(&encoded)
else {
panic!("connected result must survive HTTP-independent storage");
};
assert_eq!(saved.public_facts(), failure.public_facts());
let response = saved
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
assert_eq!(response.status, status);
let original: Value = serde_json::from_slice(body).unwrap();
let projected: Value = serde_json::from_slice(response.body()).unwrap();
assert_eq!(projected["detail"], original["detail"]);
assert_eq!(projected["yoi_startup"], original["yoi_startup"]);
}
}
// A tiny test-only adapter so assertions never format raw received data.
trait CauseFromDecode {
fn known_cause(self) -> StartupCause;
}
impl CauseFromDecode for DecodeFailure<StartupCause> {
fn known_cause(self) -> StartupCause {
match self {
DecodeFailure::Known(received) => received.facts().cause().clone(),
_ => panic!("expected known cause"),
}
}
}
#[test]
fn original_domain_wire_identity_and_details_have_no_union_wrapper() {
for cause in [
session(),
StartupCause::Runtime(RuntimeStartupCause {
stage: StartupStage::Binding,
reason: RuntimeStartupReason::AttributionFailed,
}),
StartupCause::Boundary(StartupBoundaryCause {
reason: StartupBoundaryReason::Transport,
}),
] {
let response = failure(cause.clone())
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
let value: Value = serde_json::from_slice(response.body()).unwrap();
let id = cause.identity();
assert_eq!(value["yoi_failure"]["domain"], id.domain);
assert_eq!(value["yoi_failure"]["reason"], id.reason);
assert_eq!(value["yoi_failure"]["details_version"], id.details_version);
assert_eq!(
value["yoi_failure"]["details"],
serde_json::to_value(&cause).unwrap()
);
assert!(value["yoi_failure"]["details"].get("Session").is_none());
assert!(StartupCause::supports(
id.domain,
id.reason,
id.details_version
));
let decoded = known(decode(&value, response.status, StartupBoundary::Browser));
assert_eq!(
decoded
.startup()
.primary
.decode::<StartupCause>()
.known_cause(),
cause
);
}
}
#[test]
fn browser_session_projection_requires_current_visibility_even_after_receipt() {
let failure = failure(session());
assert_eq!(
failure
.problem(
StartupBoundary::Browser,
&SessionFailureVisibility::default(),
""
)
.unwrap_err(),
ContractError::InvalidDetails
);
let wire = failure
.problem(StartupBoundary::Runtime, &visible(), "")
.unwrap();
let received = known(decode_startup_problem(
wire.body(),
wire.status,
wire.content_type,
StartupBoundary::Runtime,
));
assert!(
received
.problem(
StartupBoundary::Browser,
&SessionFailureVisibility::default(),
""
)
.is_err()
);
assert_eq!(
received
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap()
.status,
409
);
}
#[test]
fn boundary_classification_is_explicit_not_category_text_or_inner_status() {
let cases = [
(
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Operator)),
403,
403,
),
(session(), 409, 403),
(
StartupCause::Session(SessionRegistrationCause::attribution_missing()),
409,
409,
),
(
StartupCause::Session(SessionRegistrationCause::binding_mismatch()),
409,
409,
),
(
StartupCause::Session(SessionRegistrationCause::transport()),
502,
502,
),
(
StartupCause::Session(SessionRegistrationCause::outcome_unknown()),
502,
502,
),
(
StartupCause::Session(SessionRegistrationCause::invalid_response()),
502,
502,
),
];
for (cause, browser, runtime) in cases {
assert_eq!(
StartupBoundary::Browser.classify(&cause).status.code(),
browser
);
assert_eq!(
StartupBoundary::Runtime.classify(&cause).status.code(),
runtime
);
}
for (reason, status) in [
(StartupBoundaryReason::Timeout, 504),
(StartupBoundaryReason::Transport, 502),
(StartupBoundaryReason::Decode, 502),
(StartupBoundaryReason::InvalidResponse, 502),
(StartupBoundaryReason::TooLarge, 502),
(StartupBoundaryReason::UnsupportedVersion, 502),
(StartupBoundaryReason::UnsupportedReason, 502),
(StartupBoundaryReason::FinalizeFailed, 500),
] {
let cause = StartupCause::Boundary(StartupBoundaryCause { reason });
for boundary in [StartupBoundary::Browser, StartupBoundary::Runtime] {
assert_eq!(boundary.classify(&cause).status.code(), status);
}
}
for (reason, status) in [
(RuntimeStartupReason::SegmentEmpty, 400),
(RuntimeStartupReason::SystemPromptMissing, 400),
(RuntimeStartupReason::MetadataPending, 409),
(RuntimeStartupReason::MetadataMissing, 409),
(RuntimeStartupReason::MetadataInactive, 409),
(RuntimeStartupReason::ManifestMissing, 409),
(RuntimeStartupReason::AllocationFailed, 503),
(RuntimeStartupReason::SessionReadFailed, 500),
(RuntimeStartupReason::MetadataReadFailed, 500),
(RuntimeStartupReason::ProviderFailed, 500),
(RuntimeStartupReason::AttributionFailed, 500),
(RuntimeStartupReason::FeatureFailed, 500),
(RuntimeStartupReason::StartupFailed, 500),
] {
let cause = StartupCause::Runtime(RuntimeStartupCause {
stage: StartupStage::Controller,
reason,
});
for boundary in [StartupBoundary::Browser, StartupBoundary::Runtime] {
assert_eq!(boundary.classify(&cause).status.code(), status);
}
}
}
#[test]
fn all_operation_facts_remain_independent_including_unknown_and_finalize() {
for acceptance in [
CreationAcceptance::Rejected,
CreationAcceptance::Accepted,
CreationAcceptance::Unknown,
] {
for execution in [
ExecutionOutcome::Accepted,
ExecutionOutcome::Busy,
ExecutionOutcome::Rejected,
ExecutionOutcome::Errored,
ExecutionOutcome::Unsupported,
ExecutionOutcome::Unknown,
] {
for compensation in [
CompensationOutcome::Completed,
CompensationOutcome::Residual,
CompensationOutcome::Unknown,
] {
let (startup, _) = fixture_failure().into_parts();
let mut operation = operation(acceptance, execution);
operation.finalize = Some(FinalizeFacts {
stage: FinalizeStage::WorkerRegistry,
compensation,
});
let original = WorkerCreateFailure::new(startup, operation).unwrap();
let response = original
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
assert_eq!(response.status, 409); // primary cause only, not outcomes/cleanup
let received = known(decode_startup_problem(
response.body(),
response.status,
response.content_type,
StartupBoundary::Browser,
));
assert_eq!(received.public_facts(), original.public_facts());
}
}
}
}
#[test]
fn transport_unknown_never_becomes_rejected_or_accepted_from_http_or_hints() {
let response = failure(StartupCause::Boundary(StartupBoundaryCause {
reason: StartupBoundaryReason::Timeout,
}))
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
assert_eq!(response.status, 504);
let mut value: Value = serde_json::from_slice(response.body()).unwrap();
value["retry_after"] = json!(0);
value["accepted"] = json!(false); // unrecognized RFC extension is not outcome authority
let decoded = known(decode(&value, 504, StartupBoundary::Browser));
assert_eq!(decoded.operation().acceptance, CreationAcceptance::Unknown);
assert_eq!(decoded.operation().execution, ExecutionOutcome::Unknown);
value["yoi_startup"]["operation"]
.as_object_mut()
.unwrap()
.remove("acceptance");
assert_eq!(
decode(&value, 504, StartupBoundary::Browser),
DecodeStartupProblem::Invalid
);
}
#[test]
fn received_text_and_context_are_reprojected_not_forwarded() {
let original = failure(session())
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
let mut value: Value = serde_json::from_slice(original.body()).unwrap();
let secret = "RAW_UPSTREAM_CREDENTIAL_OR_PATH";
value["detail"] = json!(secret);
value["yoi_failure"]["explanation"] = json!(secret);
value["yoi_failure"]["context"]["diagnostics"] =
json!([{ "code":"raw", "explanation":secret }]);
value["yoi_failure"]["context"]["operations"] = json!([
{"operation":"session.record","stage":"registration","operation_ref":secret},
{"operation":secret,"stage":"raw","operation_ref":secret}]);
value["yoi_failure"]["context"]["correlation"] = json!(secret);
let received = known(decode(&value, 409, StartupBoundary::Browser));
let public = received
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
let text = std::str::from_utf8(public.body()).unwrap();
assert!(!text.contains(secret));
assert!(text.contains("Subject"));
assert!(text.contains("session.record"));
assert_eq!(public.status, original.status);
let (mut startup, operation) = received.into_parts();
startup.secondary.reverse();
assert_eq!(
WorkerCreateFailure::new(startup, operation)
.unwrap()
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap()
.status,
409
);
}
#[test]
fn concrete_domain_allowlists_filter_projection_context() {
let mut facts = FailureFacts::new(session()).unwrap();
let mut allowed = OperationContext::new("session.record", "authorization").unwrap();
allowed.operation_ref = Some("DO_NOT_PUBLISH".into());
facts.with_context(allowed).unwrap();
facts
.with_context(OperationContext::new("DO_NOT_PUBLISH", "raw").unwrap())
.unwrap();
let projected = project_failure::<Browser, _>(&facts, &visible()).unwrap();
assert_eq!(projected.context().operations.len(), 1);
assert_eq!(projected.context().operations[0].operation_ref, None);
assert!(
<StartupCause as ProjectFor<InternalTransfer>>::disclose(facts.cause(), &facts, &())
.is_ok()
);
}
#[test]
fn unknown_version_reason_and_bad_envelopes_are_distinct() {
let response = fixture_failure()
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
let original: Value = serde_json::from_slice(response.body()).unwrap();
for field in ["version", "details_version"] {
let mut value = original.clone();
value["yoi_failure"][field] = json!(77);
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::UnsupportedVersion
);
}
let mut value = original.clone();
value["yoi_startup"]["version"] = json!(77);
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::UnsupportedVersion
);
value = original.clone();
value["yoi_failure"]["required_extensions"] = json!(["future-contract"]);
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::UnsupportedVersion
);
value = original.clone();
value["yoi_failure"]["reason"] = json!("future_reason");
value["yoi_failure"]["category"] = json!("future_category");
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::UnsupportedReason
);
value = original.clone();
value["yoi_failure"]["domain"] = json!("future_domain");
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::UnsupportedReason
);
for mutate in ["category", "domain", "reason"] {
let mut value = original.clone();
// Another KNOWN identity with mismatching typed details must be invalid.
value["yoi_failure"][mutate] = match mutate {
"category" => json!("internal"),
"domain" => json!("startup_boundary"),
_ => json!("transport"),
};
let result = decode(&value, 409, StartupBoundary::Browser);
if mutate != "domain" {
assert_eq!(result, DecodeStartupProblem::Invalid);
} else {
assert_eq!(result, DecodeStartupProblem::UnsupportedReason);
}
}
for field in ["version", "details_version"] {
let mut value = original.clone();
value["yoi_failure"][field] = json!(0);
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::Invalid
);
}
}
#[test]
fn decoder_rejects_incoherence_bad_shapes_duplicate_fields_and_limits() {
let response = failure(session())
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
let original: Value = serde_json::from_slice(response.body()).unwrap();
assert_eq!(
decode_startup_problem(
response.body(),
403,
PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser
),
DecodeStartupProblem::Invalid
);
assert_eq!(
decode_startup_problem(
response.body(),
409,
"application/json",
StartupBoundary::Browser
),
DecodeStartupProblem::Invalid
);
assert!(matches!(
decode_startup_problem(
response.body(),
409,
"Application/Problem+Json; charset=utf-8",
StartupBoundary::Browser
),
DecodeStartupProblem::Known(_)
));
assert_eq!(
decode_startup_problem(
response.body(),
409,
PROBLEM_CONTENT_TYPE,
StartupBoundary::Runtime
),
DecodeStartupProblem::Invalid
);
for (field, replacement) in [
("status", json!(502)),
("type", json!("urn:wrong")),
("title", json!("raw upstream")),
("detail", json!("different")),
("instance", json!("%zz")),
] {
let mut value = original.clone();
value[field] = replacement;
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::Invalid
);
}
for field in ["yoi_failure", "yoi_startup"] {
let mut value = original.clone();
value.as_object_mut().unwrap().remove(field);
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::Invalid
);
}
let text = std::str::from_utf8(response.body()).unwrap();
for (needle, duplicated) in [
("\"status\":409", "\"status\":409,\"status\":409"),
(
"\"acceptance\":\"unknown\"",
"\"acceptance\":\"unknown\",\"acceptance\":\"rejected\"",
),
(
"\"principal\":\"subject\"",
"\"principal\":\"subject\",\"principal\":\"operator\"",
),
] {
let duplicate = text.replace(needle, duplicated);
assert_ne!(duplicate, text);
assert_eq!(
decode_startup_problem(
duplicate.as_bytes(),
409,
PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser
),
DecodeStartupProblem::Invalid
);
}
let oversized = vec![b' '; MAX_STARTUP_PROBLEM_BYTES + 1];
assert_eq!(
decode_startup_problem(
&oversized,
409,
PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser
),
DecodeStartupProblem::TooLarge
);
let deep = format!("{}0{}", "[".repeat(1000), "]".repeat(1000));
assert_eq!(
decode_startup_problem(
deep.as_bytes(),
409,
PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser
),
DecodeStartupProblem::Invalid
);
for bytes in [b"not json".as_slice(), b"\xff".as_slice(), b"{}".as_slice()] {
assert_eq!(
decode_startup_problem(bytes, 409, PROBLEM_CONTENT_TYPE, StartupBoundary::Browser),
DecodeStartupProblem::Invalid
);
}
}
#[test]
fn safe_correlation_and_extension_validation_cannot_be_bypassed_by_serde() {
for unsafe_value in [
"",
"path/to/file",
"https://host",
"Bearer secret",
"query?token=secret",
"control\n",
] {
assert!(StartupCorrelation::new(unsafe_value).is_err());
assert!(serde_json::from_value::<StartupCorrelation>(json!(unsafe_value)).is_err());
}
assert!(StartupCorrelation::new("a".repeat(129)).is_err());
let failure = fixture_failure();
let mut facts = failure.public_facts();
facts.secondary = vec![
StartupSecondaryFailure::Cleanup {
failure: StartupCleanupFailure::Workdir
};
9
];
assert!(facts.validate().is_err());
let (mut startup, operation) = failure.into_parts();
startup.secondary = facts.secondary;
assert!(WorkerCreateFailure::new(startup, operation).is_err());
let response = fixture_failure()
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
let mut value: Value = serde_json::from_slice(response.body()).unwrap();
value["yoi_startup"]["operation"]["correlation"] = json!("/private/path");
assert_eq!(
decode(&value, 409, StartupBoundary::Browser),
DecodeStartupProblem::Invalid
);
assert!(
fixture_failure()
.problem(StartupBoundary::Browser, &visible(), "invalid%uri")
.is_err()
);
}
#[test]
fn unknown_or_corrupt_internal_attachments_are_retained_but_not_publishable() {
for raw in ["{invalid SECRET", "{}"] {
let primary: StartupFailureAttachment = serde_json::from_value(json!(raw)).unwrap();
let failure = WorkerCreateFailure::new(
WorkerStartupFailure {
primary,
secondary: vec![],
resources: None,
persistence: None,
io_kind: None,
},
operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown),
)
.unwrap();
assert!(
failure
.problem(StartupBoundary::Browser, &visible(), "")
.is_err()
);
assert!(!format!("{failure:?}").contains("SECRET"));
}
}
#[test]
fn unsupported_envelopes_still_reject_duplicate_safe_metadata() {
let response = failure(session())
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
let text = std::str::from_utf8(response.body())
.unwrap()
.replace("\"version\":1", "\"version\":77");
assert_eq!(
decode_startup_problem(
text.as_bytes(),
409,
PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser
),
DecodeStartupProblem::UnsupportedVersion
);
for (needle, duplicate) in [
(
"\"reason\":\"permission_denied\"",
"\"reason\":\"permission_denied\",\"reason\":\"future\"",
),
(
"\"domain\":\"session_registration\"",
"\"domain\":\"session_registration\",\"domain\":\"future\"",
),
(
"\"category\":\"authorization\"",
"\"category\":\"authorization\",\"category\":\"future\"",
),
] {
let bytes = text.replace(needle, duplicate);
assert_ne!(bytes, text);
assert_eq!(
decode_startup_problem(
bytes.as_bytes(),
409,
PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser
),
DecodeStartupProblem::Invalid
);
}
}
#[test]
fn backend_compensation_is_independent_of_finalize_runtime_cleanup_and_outcomes() {
for backend in [
None,
Some(CompensationOutcome::Completed),
Some(CompensationOutcome::Residual),
Some(CompensationOutcome::Unknown),
] {
for finalize in [
None,
Some(FinalizeFacts {
stage: FinalizeStage::WorkdirRegistry,
compensation: CompensationOutcome::Residual,
}),
] {
let (startup, mut operation) = fixture_failure().into_parts();
operation.backend_compensation = backend;
operation.finalize = finalize;
let original = WorkerCreateFailure::new(startup, operation).unwrap();
let response = original
.problem(StartupBoundary::Browser, &visible(), "")
.unwrap();
let received = known(decode_startup_problem(
response.body(),
response.status,
response.content_type,
StartupBoundary::Browser,
));
assert_eq!(received.public_facts(), original.public_facts());
assert_eq!(received.operation().backend_compensation, backend);
assert_eq!(received.operation().finalize, finalize);
}
}
for stage in [
FinalizeStage::TicketAssignmentBind,
FinalizeStage::TicketAssignmentCurrent,
FinalizeStage::TicketStateAccept,
FinalizeStage::WorkdirRegistry,
] {
assert_eq!(
serde_json::from_value::<FinalizeStage>(serde_json::to_value(stage).unwrap()).unwrap(),
stage
);
}
}
#[test]
fn telemetry_uses_typed_context_and_operation_correlation_not_attachment_identity() {
use error_contract::{CorrelationRef, Telemetry, observation_fields};
let mut facts = FailureFacts::new(session()).unwrap();
let mut context = OperationContext::new("session.record", "registration").unwrap();
context.operation_ref = Some("PRIVATE_OPERATION_ID".into());
facts.with_context(context).unwrap();
facts
.with_context(OperationContext::new("SECRET_CONTEXT", "SECRET_STAGE").unwrap())
.unwrap();
facts
.set_correlation(CorrelationRef::new("UNTRUSTED_ATTACHMENT_ID").unwrap())
.unwrap();
let projected = project_failure::<Telemetry, _>(&facts, &()).unwrap();
let fields = observation_fields(&projected);
assert_eq!(fields.domain, "session_registration");
assert_eq!(fields.reason, "permission_denied");
assert_eq!(fields.operation, Some("worker_startup"));
assert_eq!(fields.stage, Some("binding"));
assert_eq!(fields.correlation, Some("UNTRUSTED_ATTACHMENT_ID"));
assert!(
projected
.context()
.operations
.iter()
.all(|op| op.operation_ref.is_none())
);
let transfer = project_failure::<InternalTransfer, _>(&facts, &()).unwrap();
let startup = WorkerStartupFailure {
primary: StartupFailureAttachment::from_projected(&transfer).unwrap(),
secondary: vec![],
resources: None,
persistence: None,
io_kind: None,
};
let mut operation = operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown);
operation.correlation = Some(StartupCorrelation::new("fresh-backend-correlation").unwrap());
let failure = WorkerCreateFailure::new(startup, operation).unwrap();
let telemetry = failure.telemetry().unwrap();
assert_eq!(
observation_fields(&telemetry).correlation,
Some("fresh-backend-correlation")
);
assert_eq!(telemetry.identity(), session().identity());
let encoded = error_contract::encode_projected_failure(
&telemetry,
error_contract::Budget::new(32768, 32768),
)
.unwrap();
let text = std::str::from_utf8(encoded.as_bytes()).unwrap();
for secret in [
"PRIVATE_OPERATION_ID",
"SECRET_CONTEXT",
"SECRET_STAGE",
"UNTRUSTED_ATTACHMENT_ID",
] {
assert!(!text.contains(secret));
}
let (startup, mut operation) = failure.into_parts();
operation.correlation = None;
assert_eq!(
observation_fields(
&WorkerCreateFailure::new(startup, operation)
.unwrap()
.telemetry()
.unwrap()
)
.correlation,
None
);
}
#[test]
fn public_dto_schema_is_accessible_for_generator() {
let schema = schemars::schema_for!(StartupProblemDetails);
let value = serde_json::to_value(schema).unwrap();
for field in [
"type",
"title",
"status",
"detail",
"instance",
"yoi_failure",
"yoi_startup",
] {
assert!(value["properties"].get(field).is_some(), "{field}");
}
let schema_text = serde_json::to_string(&value).unwrap();
assert!(!schema_text.contains("StartupFailureAttachment"));
assert!(schema_text.contains("CreationAcceptance"));
assert!(schema_text.contains("ExecutionOutcome"));
}
+313
View File
@@ -0,0 +1,313 @@
//! Actual generated RuntimeApi client error hook with bounded HTTP metadata/bytes.
use api_macros::HttpError as _;
use error_contract::{DecodeFailure, FailureCause};
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
use runtime_api::startup::*;
use runtime_api::{RuntimeApiClient, RuntimeApiError, client_support, server_support};
use serde_json::{Value, json};
fn failure() -> WorkerCreateFailure {
WorkerCreateFailure::from_cause(
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)),
StartupOperationFacts {
acceptance: CreationAcceptance::Accepted,
execution: ExecutionOutcome::Errored,
finalize: Some(FinalizeFacts {
stage: FinalizeStage::TicketAssignmentBind,
compensation: CompensationOutcome::Unknown,
}),
backend_compensation: Some(CompensationOutcome::Residual),
correlation: Some(StartupCorrelation::new("backend-local-812").unwrap()),
},
)
.unwrap()
}
fn cause(error: &RuntimeApiError) -> StartupCause {
match error
.create_failure()
.unwrap()
.startup()
.primary
.decode::<StartupCause>()
{
DecodeFailure::Known(received) => received.facts().cause().clone(),
_ => panic!("expected locally typed cause"),
}
}
fn assert_unknown(error: &RuntimeApiError, reason: StartupBoundaryReason) {
let operation = error.create_failure().unwrap().operation();
assert_eq!(operation.acceptance, CreationAcceptance::Unknown);
assert_eq!(operation.execution, ExecutionOutcome::Unknown);
assert_eq!(operation.finalize, None);
assert_eq!(operation.backend_compensation, None);
assert_eq!(
cause(error),
StartupCause::Boundary(StartupBoundaryCause { reason })
);
}
async fn receive(status: u16, types: Vec<&'static str>, body: Vec<u8>) -> RuntimeApiError {
use server_support::framework::{Body, Response, Router, get};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let router = Router::new().route(
"/v1/workers/{worker_id}",
get(move || {
let types = types.clone();
let body = body.clone();
async move {
let mut response = Response::builder().status(status);
for content_type in types {
response = response.header("content-type", content_type);
}
response.body(Body::from(body)).unwrap()
}
}),
);
let server = tokio::spawn(async move {
server_support::framework::serve(listener, router)
.await
.unwrap();
});
let client = RuntimeApiClient::builder(&format!("http://{address}"))
.unwrap()
.request_timeout(std::time::Duration::from_secs(3))
.build()
.unwrap();
let result = client.get_worker("test".into()).await;
server.abort();
match result {
Err(client_support::ClientError::Public {
status: actual,
error,
}) => {
assert_eq!(actual.as_u16(), status);
assert_eq!(error.status(), status);
error
}
_ => panic!("expected normalized public error"),
}
}
#[test]
fn runtime_error_projects_public_schema_fresh_and_keeps_local_facts_off_serde() {
let failure = failure();
let error = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE")
.with_create_failure(failure.clone());
let legacy = serde_json::to_value(&error).unwrap();
assert!(legacy.get("create_failure").is_none());
let reparsed: RuntimeApiError = serde_json::from_value(legacy).unwrap();
assert!(reparsed.create_failure().is_none());
let response = error.custom_response().unwrap();
assert_eq!(response.status, 403); // runtime boundary is not Browser's 409
assert_eq!(response.content_type, PROBLEM_CONTENT_TYPE);
let text = std::str::from_utf8(response.body.as_ref()).unwrap();
assert!(!text.contains("PRIVATE_CODE"));
assert!(!text.contains("PRIVATE_MESSAGE"));
let decoded = match decode_startup_problem(
response.body.as_ref(),
response.status,
response.content_type,
StartupBoundary::Runtime,
) {
DecodeStartupProblem::Known(failure) => failure,
_ => panic!("expected known public-safe Runtime failure"),
};
assert_eq!(decoded.public_facts(), failure.public_facts());
assert_eq!(error.into_create_failure(), Some(failure));
let ordinary = RuntimeApiError::new(404, "not_found", "not found");
assert!(ordinary.custom_response().is_none());
}
#[test]
fn invalid_projection_uses_static_fallback_not_legacy_raw_serialization() {
let (mut startup, operation) = failure().into_parts();
startup.primary = serde_json::from_value(json!("RAW_PRIVATE_CREDENTIAL")).unwrap();
let error = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE")
.with_create_failure(WorkerCreateFailure::new(startup, operation).unwrap());
let response = error.custom_response().unwrap();
assert_eq!(response.status, 500);
assert_eq!(
response.body.as_ref(),
error_contract::problem::STATIC_PROBLEM_FALLBACK
);
let mut legacy = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE");
legacy.error.startup = error
.create_failure()
.map(|failure| failure.startup().clone());
assert_eq!(
legacy.custom_response().unwrap().body.as_ref(),
error_contract::problem::STATIC_PROBLEM_FALLBACK
);
}
#[tokio::test]
async fn generated_client_normalizes_known_problem_and_actual_metadata() {
let original = failure();
let response = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE")
.with_create_failure(original.clone())
.custom_response()
.unwrap();
let mut value: Value = serde_json::from_slice(response.body.as_ref()).unwrap();
value["detail"] = json!("WIRE_SECRET_EXPLANATION");
value["yoi_failure"]["explanation"] = json!("WIRE_SECRET_EXPLANATION");
let error = receive(
response.status,
vec![PROBLEM_CONTENT_TYPE],
serde_json::to_vec(&value).unwrap(),
)
.await;
assert_eq!(
error.create_failure().unwrap().public_facts(),
original.public_facts()
);
assert_eq!(
cause(&error).identity(),
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject))
.identity()
);
assert!(!error.error.message.contains("WIRE_SECRET"));
let browser = error
.create_failure()
.unwrap()
.problem(
StartupBoundary::Browser,
&SessionFailureVisibility {
can_view_cause: true,
},
"",
)
.unwrap();
assert_eq!(browser.status, 409);
let wrong_status = receive(
409,
vec![PROBLEM_CONTENT_TYPE],
serde_json::to_vec(&value).unwrap(),
)
.await;
assert_unknown(&wrong_status, StartupBoundaryReason::InvalidResponse);
}
#[tokio::test]
async fn unknown_versions_reasons_and_invalid_responses_remain_local_unknown_observations() {
let response = failure()
.problem(
StartupBoundary::Runtime,
&SessionFailureVisibility {
can_view_cause: true,
},
"",
)
.unwrap();
let original: Value = serde_json::from_slice(response.body()).unwrap();
for (field, replacement, reason) in [
(
"version",
json!(99),
StartupBoundaryReason::UnsupportedVersion,
),
(
"details_version",
json!(99),
StartupBoundaryReason::UnsupportedVersion,
),
(
"reason",
json!("future_reason"),
StartupBoundaryReason::UnsupportedReason,
),
(
"category",
json!("invalid_category"),
StartupBoundaryReason::InvalidResponse,
),
] {
let mut value = original.clone();
value["yoi_failure"][field] = replacement;
let error = receive(
403,
vec![PROBLEM_CONTENT_TYPE],
serde_json::to_vec(&value).unwrap(),
)
.await;
assert_unknown(&error, reason);
}
for content_types in [
vec![],
vec!["text/plain"],
vec![PROBLEM_CONTENT_TYPE, "application/json"],
] {
let error = receive(403, content_types, response.body().to_vec()).await;
assert_unknown(&error, StartupBoundaryReason::InvalidResponse);
}
let error = receive(
502,
vec![PROBLEM_CONTENT_TYPE],
vec![b'x'; MAX_STARTUP_PROBLEM_BYTES + 1],
)
.await;
assert_unknown(&error, StartupBoundaryReason::TooLarge);
let error = receive(
502,
vec![PROBLEM_CONTENT_TYPE],
b"not JSON PRIVATE_SECRET".to_vec(),
)
.await;
assert_unknown(&error, StartupBoundaryReason::InvalidResponse);
assert!(!error.error.message.contains("PRIVATE_SECRET"));
}
#[tokio::test]
async fn legacy_errors_are_bounded_and_startup_attachments_reprojected_without_outcome_inference() {
let ordinary = RuntimeApiError::new(404, "old_not_found", "old not found");
let error = receive(
404,
vec!["application/json"],
serde_json::to_vec(&ordinary).unwrap(),
)
.await;
assert_eq!(error.error, ordinary.error);
assert!(error.create_failure().is_none());
let original = failure();
let mut legacy = RuntimeApiError::new(502, "OLD_PRIVATE_CODE", "OLD_PRIVATE_MESSAGE");
legacy.error.startup = Some(original.startup().clone());
let error = receive(
502,
vec!["application/json; charset=utf-8"],
serde_json::to_vec(&legacy).unwrap(),
)
.await;
assert_eq!(
cause(&error).identity(),
cause(&RuntimeApiError::new(500, "unused", "unused").with_create_failure(original.clone()))
.identity()
);
let failure = error.create_failure().unwrap();
assert_eq!(failure.operation().acceptance, CreationAcceptance::Unknown);
assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown);
assert_eq!(failure.startup().secondary, original.startup().secondary);
assert!(!error.error.message.contains("OLD_PRIVATE"));
let mut value = serde_json::to_value(&legacy).unwrap();
value["error"]["startup"]["primary"] = json!("PRIVATE_MALFORMED_ATTACHMENT");
let malformed = receive(
502,
vec!["application/json"],
serde_json::to_vec(&value).unwrap(),
)
.await;
assert_unknown(&malformed, StartupBoundaryReason::InvalidResponse);
let invalid = receive(
502,
vec!["application/json"],
b"{invalid PRIVATE_SECRET".to_vec(),
)
.await;
assert_unknown(&invalid, StartupBoundaryReason::Decode);
let oversized = receive(
502,
vec!["application/json"],
vec![b'x'; MAX_STARTUP_PROBLEM_BYTES + 1],
)
.await;
assert_unknown(&oversized, StartupBoundaryReason::TooLarge);
}
+341
View File
@@ -0,0 +1,341 @@
//! Storage codecs use the production attachment projector, never an HTTP body.
use error_contract::{Budget, ContractError, DecodeFailure, validate_json_input};
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
use runtime_api::startup::*;
use runtime_api::{
RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure,
StartupFailureAttachment, StartupIoKind, StartupPersistenceOutcome, StartupResourceFacts,
StartupResources, StartupSecondaryFailure, StartupStage, WorkerStartupFailure,
};
use serde_json::{Value, json};
fn operation() -> StartupOperationFacts {
StartupOperationFacts {
acceptance: CreationAcceptance::Accepted,
execution: ExecutionOutcome::Errored,
finalize: Some(FinalizeFacts {
stage: FinalizeStage::Publication,
compensation: CompensationOutcome::Residual,
}),
backend_compensation: Some(CompensationOutcome::Unknown),
correlation: Some(StartupCorrelation::new("storage-test-812").unwrap()),
}
}
fn failure(cause: StartupCause) -> WorkerCreateFailure {
let base = WorkerCreateFailure::from_cause(cause, operation()).unwrap();
let (mut startup, operation) = base.into_parts();
startup.secondary.push(StartupSecondaryFailure::Cleanup {
failure: StartupCleanupFailure::Workdir,
});
startup.resources = Some(StartupResourceFacts {
resources: StartupResources::WorkerStartup,
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir),
durable_effects_possible: true,
});
// Session metadata write and Backend result write deliberately disagree.
startup.persistence = Some(StartupPersistenceOutcome::NotSaved);
startup.io_kind = Some(StartupIoKind::PermissionDenied);
WorkerCreateFailure::new(startup, operation)
.unwrap()
.with_backend_result_persistence(Some(StartupPersistenceOutcome::Persisted))
}
fn session() -> StartupCause {
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject))
}
fn known(input: &str) -> WorkerCreateFailure {
match decode_stored_worker_create_failure(input) {
DecodeStoredWorkerCreateFailure::Known(failure) => failure,
other => panic!("expected known storage snapshot, got {other:?}"),
}
}
fn change_primary(value: &mut Value, change: impl FnOnce(&mut Value)) {
let mut primary: Value = serde_json::from_str(value["primary"].as_str().unwrap()).unwrap();
change(&mut primary);
value["primary"] = Value::String(serde_json::to_string(&primary).unwrap());
}
#[test]
fn storage_semantic_roundtrip_covers_domains_and_independent_facts() {
for cause in [
session(),
StartupCause::Runtime(RuntimeStartupCause {
stage: StartupStage::Feature,
reason: RuntimeStartupReason::FeatureFailed,
}),
StartupCause::Boundary(StartupBoundaryCause {
reason: StartupBoundaryReason::FinalizeFailed,
}),
] {
let original = failure(cause.clone());
let encoded = original.encode_storage().unwrap();
let value: Value = serde_json::from_str(&encoded).unwrap();
assert_eq!(value["version"], 1);
assert!(value.get("status").is_none());
assert!(value.get("yoi_failure").is_none());
let loaded = known(&encoded);
assert_eq!(loaded.public_facts(), original.public_facts());
assert_eq!(loaded.operation(), original.operation());
assert_eq!(
loaded.startup().persistence,
Some(StartupPersistenceOutcome::NotSaved)
);
assert_eq!(
loaded.backend_result_persistence(),
Some(StartupPersistenceOutcome::Persisted)
);
match loaded.startup().primary.decode::<StartupCause>() {
DecodeFailure::Known(received) => assert_eq!(received.facts().cause(), &cause),
_ => panic!("primary cause lost"),
}
assert_eq!(loaded.encode_storage().unwrap(), encoded);
}
}
#[test]
fn encoding_uses_current_marker_without_replacing_primary_or_inferring_persistence() {
let original = failure(session());
for marker in [
None,
Some(StartupPersistenceOutcome::Persisted),
Some(StartupPersistenceOutcome::NotSaved),
Some(StartupPersistenceOutcome::Conflict),
Some(StartupPersistenceOutcome::OutcomeUnknown),
] {
let updated = original.clone().with_backend_result_persistence(marker);
assert_eq!(updated.startup(), original.startup());
assert_eq!(updated.operation(), original.operation());
let loaded = known(&updated.encode_storage().unwrap());
assert_eq!(loaded.backend_result_persistence(), marker);
assert_eq!(loaded.startup().persistence, original.startup().persistence);
assert_eq!(loaded.public_facts(), updated.public_facts());
}
}
#[test]
fn legacy_absent_marker_is_none_in_storage_and_public_reader() {
let original = failure(session()).with_backend_result_persistence(None);
let mut value: Value = serde_json::from_str(&original.encode_storage().unwrap()).unwrap();
value["facts"]
.as_object_mut()
.unwrap()
.remove("backend_result_persistence");
assert_eq!(known(&value.to_string()).backend_result_persistence(), None);
let response = original
.problem(
StartupBoundary::Browser,
&SessionFailureVisibility {
can_view_cause: true,
},
"",
)
.unwrap();
let public: Value = serde_json::from_slice(response.body()).unwrap();
assert!(
public["yoi_startup"]
.get("backend_result_persistence")
.is_none()
);
match decode_startup_problem(
response.body(),
response.status,
response.content_type,
StartupBoundary::Browser,
) {
DecodeStartupProblem::Known(failure) => {
assert_eq!(failure.backend_result_persistence(), None)
}
other => panic!("legacy public response failed: {other:?}"),
}
}
#[test]
fn public_receiver_restores_backend_marker_and_current_visibility_is_required() {
let original = failure(session());
let loaded = known(&original.encode_storage().unwrap());
assert!(
loaded
.problem(
StartupBoundary::Browser,
&SessionFailureVisibility {
can_view_cause: false
},
""
)
.is_err()
);
let response = loaded
.problem(
StartupBoundary::Browser,
&SessionFailureVisibility {
can_view_cause: true,
},
"",
)
.unwrap();
let received = match decode_startup_problem(
response.body(),
response.status,
response.content_type,
StartupBoundary::Browser,
) {
DecodeStartupProblem::Known(failure) => failure,
other => panic!("public response failed: {other:?}"),
};
assert_eq!(received.public_facts(), original.public_facts());
let mut updated = operation();
updated.acceptance = CreationAcceptance::Unknown;
let received = received.with_operation(updated.clone());
assert_eq!(received.operation(), &updated);
assert_eq!(
received.backend_result_persistence(),
Some(StartupPersistenceOutcome::Persisted)
);
// The receiving Backend clears remote evidence before its own write.
assert_eq!(
received
.with_backend_result_persistence(None)
.backend_result_persistence(),
None
);
}
#[test]
fn unknown_versions_and_primary_identities_are_unsupported() {
let base: Value = serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap();
for mutate in [
|v: &mut Value| v["version"] = json!(2),
|v: &mut Value| v["facts"]["version"] = json!(2),
|v: &mut Value| change_primary(v, |p| p["version"] = json!(2)),
|v: &mut Value| change_primary(v, |p| p["reason"] = json!("future_reason")),
|v: &mut Value| {
change_primary(v, |p| {
p["required_extensions"] = json!(["future_extension"])
})
},
] {
let mut value = base.clone();
mutate(&mut value);
assert_eq!(
decode_stored_worker_create_failure(&value.to_string()),
DecodeStoredWorkerCreateFailure::Unsupported
);
}
}
#[test]
fn malformed_and_duplicate_known_records_are_corrupt() {
let base: Value = serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap();
for input in [
"".to_owned(),
"null".to_owned(),
"{}".to_owned(),
"{".to_owned(),
base.to_string()
.replacen("\"version\":1", "\"version\":1,\"version\":1", 1),
] {
assert!(matches!(
decode_stored_worker_create_failure(&input),
DecodeStoredWorkerCreateFailure::Corrupt(_)
));
}
for mutate in [
|v: &mut Value| v["version"] = json!(0),
|v: &mut Value| v["primary"] = json!("{broken"),
|v: &mut Value| v["facts"]["backend_result_persistence"] = json!("invented"),
|v: &mut Value| {
v["facts"]["secondary"] = json!(vec![json!({"kind":"cleanup","failure":"workdir"}); 9])
},
|v: &mut Value| change_primary(v, |p| p["category"] = json!("internal")),
] {
let mut value = base.clone();
mutate(&mut value);
assert!(matches!(
decode_stored_worker_create_failure(&value.to_string()),
DecodeStoredWorkerCreateFailure::Corrupt(_)
));
}
}
#[test]
fn complete_input_and_embedded_attachment_are_bounded_before_parsing() {
let oversized = " ".repeat(32 * 1024 + 1);
assert_eq!(
decode_stored_worker_create_failure(&oversized),
DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooLarge)
);
let deep = format!("{}0{}", "[".repeat(17), "]".repeat(17));
assert_eq!(
decode_stored_worker_create_failure(&deep),
DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep)
);
// A supported-looking prefix must not bypass the whole-input guard.
let deep = format!("{{\"version\":2,\"extra\":{deep}}}");
assert_eq!(
decode_stored_worker_create_failure(&deep),
DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep)
);
let mut value: Value =
serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap();
change_primary(&mut value, |p| {
p["extra"] =
serde_json::from_str(&format!("{}0{}", "[".repeat(17), "]".repeat(17))).unwrap()
});
assert_eq!(
decode_stored_worker_create_failure(&value.to_string()),
DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep)
);
// Shared wrapper is a bounds check, not a grammar validator or encoder.
assert_eq!(validate_json_input(b"[", Budget::new(1024, 1024)), Ok(()));
}
#[test]
fn storage_encoder_and_reader_strip_untrusted_context_and_explanations() {
let original = failure(session());
let mut value: Value = serde_json::from_str(&original.encode_storage().unwrap()).unwrap();
change_primary(&mut value, |p| {
p["explanation"] = json!("TEST_ONLY_SECRET_RAW_HTTP");
p["context"]["correlation"] = json!("private-worker-id");
p["context"]["operations"] = json!([{
"operation":"session.record", "stage":"binding", "operation_ref":"TEST_ONLY_SECRET_RAW_HTTP"
}, {
"operation":"/private/path", "stage":"http-body", "operation_ref":null
}]);
});
// Simulate a received attachment without granting a projection capability.
let primary: StartupFailureAttachment =
serde_json::from_value(value["primary"].clone()).unwrap();
let (startup, operation) = original.clone().into_parts();
let incoming =
WorkerCreateFailure::new(WorkerStartupFailure { primary, ..startup }, operation).unwrap();
for encoded in [
incoming.encode_storage().unwrap(),
known(&value.to_string()).encode_storage().unwrap(),
] {
for secret in [
"TEST_ONLY_SECRET_RAW_HTTP",
"private-worker-id",
"/private/path",
"http-body",
] {
assert!(
!encoded.contains(secret),
"unexpected secret in stored snapshot"
);
}
let loaded = known(&encoded);
let response = loaded
.problem(
StartupBoundary::Browser,
&SessionFailureVisibility {
can_view_cause: true,
},
"",
)
.unwrap();
assert!(!String::from_utf8_lossy(response.body()).contains("TEST_ONLY_SECRET"));
}
}
+2 -1
View File
@@ -10,8 +10,9 @@ default = []
typescript = ["dep:ts-rs", "protocol/typescript"] typescript = ["dep:ts-rs", "protocol/typescript"]
[dependencies] [dependencies]
error-contract = { workspace = true, features = ["json-schema"] }
feature-permissions.workspace = true feature-permissions.workspace = true
api-macros = { workspace = true, features = ["reqwest", "axum", "openapi"] } api-macros = { workspace = true, features = ["reqwest", "axum", "openapi", "error-contract"] }
memory.workspace = true memory.workspace = true
plugin-plan.workspace = true plugin-plan.workspace = true
merge-request.workspace = true merge-request.workspace = true
+14 -6
View File
@@ -1,4 +1,6 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
mod worker_create_error;
pub use worker_create_error::WorkerCreateApiError;
use std::collections::BTreeMap; use std::collections::BTreeMap;
@@ -1907,7 +1909,9 @@ pub trait ServerApi {
"/api/workers", "/api/workers",
status = 200, status = 200,
error_status = 400, error_status = 400,
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503], additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504],
additional_error_content_types = ["application/problem+json"],
client_error_hook = true,
bearer_auth = true, bearer_auth = true,
browser_auth = true browser_auth = true
)] )]
@@ -1915,7 +1919,7 @@ pub trait ServerApi {
&self, &self,
#[extension] context: ServerRequestContext, #[extension] context: ServerRequestContext,
#[body] request: CreateWorkspaceWorkerRequest, #[body] request: CreateWorkspaceWorkerRequest,
) -> Result<BrowserCreateWorkerResponse, RepositoryApiError>; ) -> Result<BrowserCreateWorkerResponse, WorkerCreateApiError>;
#[get( #[get(
"/api/w/{workspace_id}/orchestrator", "/api/w/{workspace_id}/orchestrator",
@@ -1973,7 +1977,8 @@ pub trait ServerApi {
"/api/w/{workspace_id}/worker-control/workers", "/api/w/{workspace_id}/worker-control/workers",
status = 200, status = 200,
error_status = 400, error_status = 400,
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503], additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504],
client_error_hook = true,
openapi = false, openapi = false,
)] )]
async fn worker_control_spawn( async fn worker_control_spawn(
@@ -1981,7 +1986,7 @@ pub trait ServerApi {
#[extension] context: ServerRequestContext, #[extension] context: ServerRequestContext,
#[path] workspace_id: String, #[path] workspace_id: String,
#[body] request: CreateWorkspaceWorkerRequest, #[body] request: CreateWorkspaceWorkerRequest,
) -> Result<BrowserCreateWorkerResponse, RepositoryApiError>; ) -> Result<BrowserCreateWorkerResponse, WorkerCreateApiError>;
#[post( #[post(
"/api/w/{workspace_id}/worker-control/workers/{runtime_id}/{worker_id}/input", "/api/w/{workspace_id}/worker-control/workers/{runtime_id}/{worker_id}/input",
status = 200, status = 200,
@@ -2098,7 +2103,9 @@ pub trait ServerApi {
"/api/w/{workspace_id}/workers", "/api/w/{workspace_id}/workers",
status = 200, status = 200,
error_status = 400, error_status = 400,
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503], additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504],
additional_error_content_types = ["application/problem+json"],
client_error_hook = true,
bearer_auth = true, bearer_auth = true,
browser_auth = true browser_auth = true
)] )]
@@ -2107,7 +2114,7 @@ pub trait ServerApi {
#[extension] context: ServerRequestContext, #[extension] context: ServerRequestContext,
#[path] workspace_id: String, #[path] workspace_id: String,
#[body] request: CreateWorkspaceWorkerRequest, #[body] request: CreateWorkspaceWorkerRequest,
) -> Result<BrowserCreateWorkerResponse, RepositoryApiError>; ) -> Result<BrowserCreateWorkerResponse, WorkerCreateApiError>;
#[get( #[get(
"/api/w/{workspace_id}/workers/{worker_ref}", "/api/w/{workspace_id}/workers/{worker_ref}",
@@ -11099,6 +11106,7 @@ pub struct BrowserWorkspaceOrchestratorResponse {
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub enum WorkerOperationState { pub enum WorkerOperationState {
Unknown,
Accepted, Accepted,
Unsupported, Unsupported,
Rejected, Rejected,
@@ -0,0 +1,200 @@
//! Worker creation's HTTP projection is separate from its HTTP-independent result.
//! A decoded DTO is never an encoder capability: only a current visibility decision
//! can populate the cached public response. Legacy preflight routes keep their body.
use api_macros::{HttpError, HttpErrorResponse};
use runtime_api::startup::{
DecodeStartupProblem, SessionFailureVisibility, StartupBoundary, StartupProblemDetails,
WorkerCreateFailure, decode_startup_problem,
};
use schemars::{JsonSchema, Schema, SchemaGenerator};
use serde::{Deserialize, Serialize};
use crate::RepositoryApiError;
#[derive(Debug)]
pub struct WorkerCreateApiError {
legacy: RepositoryApiError,
failure: Option<WorkerCreateFailure>,
projected: Option<runtime_api::startup::ProblemResponse>,
}
impl WorkerCreateApiError {
/// The service has authenticated and authorized the current creation caller.
/// Visibility is a local decision; neither wire labels nor saved results supply it.
pub fn startup(failure: WorkerCreateFailure, visibility: &SessionFailureVisibility) -> Self {
let instance = failure
.operation()
.correlation
.as_ref()
.map(|correlation| format!("urn:yoi:worker-startup:{}", correlation.as_str()))
.unwrap_or_default();
let projected = failure
.problem(StartupBoundary::Browser, visibility, &instance)
.ok();
let status = projected.as_ref().map_or(500, |response| response.status);
Self {
legacy: RepositoryApiError::new(
status,
"worker_startup",
"Worker creation did not complete; inspect the typed result.",
Vec::new(),
),
failure: Some(failure),
projected,
}
}
pub fn failure(&self) -> Option<&WorkerCreateFailure> {
self.failure.as_ref()
}
pub fn legacy(&self) -> &RepositoryApiError {
&self.legacy
}
}
impl From<RepositoryApiError> for WorkerCreateApiError {
fn from(legacy: RepositoryApiError) -> Self {
Self {
legacy,
failure: None,
projected: None,
}
}
}
impl HttpError for WorkerCreateApiError {
fn status_code(&self) -> u16 {
self.projected
.as_ref()
.map_or_else(|| self.legacy.status_code(), |response| response.status)
}
fn custom_response(&self) -> Option<HttpErrorResponse> {
if let Some(projected) = &self.projected {
return Some(HttpErrorResponse {
status: projected.status,
content_type: projected.content_type,
body: projected.body().to_vec().into(),
});
}
// Projection failure does not serialize a possibly private internal result.
self.failure.as_ref().map(|_| {
error_contract::problem::problem_fallback(error_contract::Budget::new(
error_contract::MAX_FAILURE_BYTES,
error_contract::MAX_FAILURE_BYTES,
))
.expect("static fallback fits the fixed boundary budget")
.into()
})
}
}
impl api_macros::HttpRequestError for WorkerCreateApiError {
fn from_request_rejection(status: u16, message: String) -> Self {
<RepositoryApiError as api_macros::HttpRequestError>::from_request_rejection(
status, message,
)
.into()
}
}
impl Serialize for WorkerCreateApiError {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
// Generated HTTP adapters use custom_response. The ordinary serializer remains
// legacy-only and cannot serialize the retained internal startup attachment.
if let Some(projected) = &self.projected {
let dto: StartupProblemDetails =
serde_json::from_slice(projected.body()).map_err(serde::ser::Error::custom)?;
dto.serialize(serializer)
} else {
self.legacy.serialize(serializer)
}
}
}
impl<'de> Deserialize<'de> for WorkerCreateApiError {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
RepositoryApiError::deserialize(deserializer).map(Into::into)
}
}
#[allow(dead_code)]
#[derive(JsonSchema)]
#[serde(untagged)]
enum WorkerCreateErrorWire {
Problem(StartupProblemDetails),
Legacy(RepositoryApiError),
}
impl JsonSchema for WorkerCreateApiError {
fn schema_name() -> std::borrow::Cow<'static, str> {
"WorkerCreateApiError".into()
}
fn json_schema(generator: &mut SchemaGenerator) -> Schema {
WorkerCreateErrorWire::json_schema(generator)
}
}
impl api_macros::openapi::OpenApiSchema for WorkerCreateApiError {
fn response_schema(
builder: &mut api_macros::openapi::OpenApiBuilder,
content_type: &str,
) -> Result<serde_json::Value, api_macros::openapi::OpenApiError> {
match content_type {
"application/json" => builder.schema_ref::<RepositoryApiError>(),
runtime_api::startup::PROBLEM_CONTENT_TYPE => {
builder.schema_ref::<StartupProblemDetails>()
}
_ => Err(api_macros::openapi::OpenApiError::InvalidContract(
"unsupported Worker creation error representation".into(),
)),
}
}
}
impl api_macros::reqwest::HttpClientError for WorkerCreateApiError {
fn decode_error(
response: &api_macros::reqwest::ReceivedResponse,
) -> Result<Self, api_macros::reqwest::ClientFailure> {
use api_macros::reqwest::ClientFailure;
let content_type = response.content_type()?;
if content_type
.split(';')
.next()
.unwrap_or("")
.trim()
.eq_ignore_ascii_case(runtime_api::startup::PROBLEM_CONTENT_TYPE)
{
return match decode_startup_problem(
response.body(),
response.status().as_u16(),
content_type,
StartupBoundary::Browser,
) {
DecodeStartupProblem::Known(failure) => Ok(Self {
legacy: RepositoryApiError::new(
response.status().as_u16(),
"worker_startup",
"Worker creation returned a typed failure.",
Vec::new(),
),
failure: Some(failure),
projected: None,
}),
DecodeStartupProblem::TooLarge => Err(ClientFailure::ResponseTooLarge {
limit: runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
}),
DecodeStartupProblem::UnsupportedVersion
| DecodeStartupProblem::UnsupportedReason => {
Err(ClientFailure::UnsupportedErrorResponse {
status: response.status().as_u16(),
})
}
DecodeStartupProblem::Invalid => Err(ClientFailure::ErrorResponseDecode {
status: response.status().as_u16(),
}),
};
}
let legacy = response
.decode_json::<RepositoryApiError>(api_macros::reqwest::DecodeKind::PublicError)?;
Ok(legacy.into())
}
}
@@ -0,0 +1,96 @@
//! Generated Backend client → isolated loopback HTTP response → typed result.
//! The server response producer is substituted; authorization and persistence are
//! exercised by workspace-server's connected tests, not by this codec connection.
use runtime_api::startup::*;
use server_api::{CreateWorkspaceWorkerRequest, ServerApiClient, client_support, server_support};
use server_support::framework::{Body, Response, Router, post};
struct ServerTask(tokio::task::JoinHandle<()>);
impl Drop for ServerTask {
fn drop(&mut self) {
self.0.abort();
}
}
fn request() -> CreateWorkspaceWorkerRequest {
serde_json::from_value(serde_json::json!({
"runtime_id": "runtime", "display_name": "Worker", "feature_connections": {}
}))
.unwrap()
}
#[tokio::test]
async fn every_creation_client_decodes_actual_problem_status_and_operation_facts() {
let timeout = WorkerCreateFailure::from_cause(
StartupCause::Boundary(StartupBoundaryCause {
reason: StartupBoundaryReason::Timeout,
}),
StartupOperationFacts {
acceptance: CreationAcceptance::Unknown,
execution: ExecutionOutcome::Unknown,
finalize: None,
backend_compensation: None,
correlation: None,
},
)
.unwrap()
.problem(
StartupBoundary::Browser,
&SessionFailureVisibility::default(),
"",
)
.unwrap();
for (status, body) in [
(409, include_bytes!("../../runtime-api/tests/fixtures/startup-browser-session-record-denied.problem.json").to_vec()),
(502, include_bytes!("../../runtime-api/tests/fixtures/startup-browser-registration-response-lost.problem.json").to_vec()),
(504, timeout.into_body()),
] {
let DecodeStartupProblem::Known(expected) = decode_startup_problem(&body, status, PROBLEM_CONTENT_TYPE, StartupBoundary::Browser) else {
panic!("producer fixture must have a known typed cause");
};
let router = ["/api/workers", "/api/w/workspace/workers", "/api/w/workspace/worker-control/workers"]
.into_iter().fold(Router::new(), |router, path| {
let body = body.clone();
router.route(path, post(move || {
let body = body.clone();
async move { Response::builder().status(status).header("content-type", PROBLEM_CONTENT_TYPE).body(Body::from(body)).unwrap() }
}))
});
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let _server = ServerTask(tokio::spawn(async move {
server_support::framework::serve(listener, router).await.unwrap();
}));
let client = ServerApiClient::builder(format!("http://{address}"))
.unwrap().request_timeout(std::time::Duration::from_secs(3)).build().unwrap();
for result in [
client.workspace_worker_create("workspace".into(), request()).await,
client.workspace_worker_create_alias(request()).await,
client.worker_control_spawn("workspace".into(), request()).await,
] {
let Err(client_support::ClientError::Public { status: actual_status, error }) = result else {
panic!("generated route must invoke the Problem decoder, not legacy Deserialize");
};
assert_eq!(actual_status.as_u16(), status);
let actual = error.failure().expect("retain typed non-HTTP failure");
assert_eq!(actual.startup().primary, expected.startup().primary);
assert_eq!(actual.public_facts(), expected.public_facts());
}
}
}
#[test]
fn creation_openapi_declares_gateway_timeout_with_both_error_representations() {
let document: serde_json::Value = serde_json::from_str(
&server_api::canonical_openapi_document()
.unwrap()
.to_json()
.unwrap(),
)
.unwrap();
for path in ["/api/workers", "/api/w/{workspace_id}/workers"] {
let content = &document["paths"][path]["post"]["responses"]["504"]["content"];
assert!(content.get("application/problem+json").is_some());
assert!(content.get("application/json").is_some());
}
}
+24
View File
@@ -63,6 +63,15 @@ pub enum RuntimeError {
result: WorkerExecutionResult, result: WorkerExecutionResult,
}, },
/// Creation failed after connection; the original error remains internal source
/// evidence. Display deliberately never renders it or the startup attachment.
#[error("worker creation failed")]
WorkerCreateFailed {
failure: Box<runtime_api::startup::WorkerCreateFailure>,
#[source]
source: Box<RuntimeError>,
},
#[error("limit {requested} exceeds maximum {max}")] #[error("limit {requested} exceeds maximum {max}")]
LimitTooLarge { requested: usize, max: usize }, LimitTooLarge { requested: usize, max: usize },
@@ -149,11 +158,26 @@ pub enum RuntimeError {
} }
impl RuntimeError { impl RuntimeError {
/// Only an owning create boundary or a Spawn result can identify a creation
/// failure. Independent Input errors retain their ordinary API contract.
pub fn create_failure(&self) -> Option<runtime_api::startup::WorkerCreateFailure> {
match self {
Self::WorkerCreateFailed { failure, .. } => Some((**failure).clone()),
Self::WorkerExecutionRejected { result, .. } => {
crate::execution::startup_create_failure(result)
.ok()
.flatten()
}
_ => None,
}
}
/// The original execution result, including primary facts and independent secondary /// The original execution result, including primary facts and independent secondary
/// failures. Display is presentation only and must not be parsed to recover causes. /// failures. Display is presentation only and must not be parsed to recover causes.
pub fn execution_result(&self) -> Option<&WorkerExecutionResult> { pub fn execution_result(&self) -> Option<&WorkerExecutionResult> {
match self { match self {
Self::WorkerExecutionRejected { result, .. } => Some(result), Self::WorkerExecutionRejected { result, .. } => Some(result),
Self::WorkerCreateFailed { source, .. } => source.execution_result(),
_ => None, _ => None,
} }
} }
+56
View File
@@ -1690,3 +1690,59 @@ impl fmt::Debug for WorkerExecutionBackendRef {
.finish_non_exhaustive() .finish_non_exhaustive()
} }
} }
/// Transfer a spawn result without deriving operation acceptance from its error text,
/// HTTP status, cleanup disposition, or persistence outcome. Other operation kinds
/// keep their existing result contracts and are not reclassified as creation.
pub fn startup_create_failure(
result: &WorkerExecutionResult,
) -> Result<Option<runtime_api::startup::WorkerCreateFailure>, error_contract::ContractError> {
use runtime_api::startup::{
CreationAcceptance, ExecutionOutcome, StartupOperationFacts, WorkerCreateFailure,
};
if result.operation != WorkerExecutionOperation::Spawn {
return Ok(None);
}
let execution = match result.outcome {
WorkerExecutionOutcome::Accepted => ExecutionOutcome::Accepted,
WorkerExecutionOutcome::Busy => ExecutionOutcome::Busy,
WorkerExecutionOutcome::Rejected => ExecutionOutcome::Rejected,
WorkerExecutionOutcome::Errored => ExecutionOutcome::Errored,
WorkerExecutionOutcome::Unsupported => ExecutionOutcome::Unsupported,
};
let acceptance = match result.outcome {
WorkerExecutionOutcome::Accepted => CreationAcceptance::Accepted,
WorkerExecutionOutcome::Rejected | WorkerExecutionOutcome::Unsupported => {
CreationAcceptance::Rejected
}
WorkerExecutionOutcome::Busy | WorkerExecutionOutcome::Errored => {
CreationAcceptance::Unknown
}
};
let primary = match result.startup_failure.as_ref() {
Some(primary) => primary.clone(),
None => worker::startup_failure::runtime_startup_attachment(
protocol::startup_failure::RuntimeStartupCause {
stage: protocol::startup_failure::StartupStage::Controller,
reason: protocol::startup_failure::RuntimeStartupReason::StartupFailed,
},
)?,
};
WorkerCreateFailure::new(
runtime_api::WorkerStartupFailure {
primary,
secondary: result.secondary_failures.clone(),
resources: result.startup_resources,
persistence: result.startup_failure_persistence,
io_kind: result.startup_io_kind,
},
StartupOperationFacts {
acceptance,
execution,
finalize: None,
correlation: None,
backend_compensation: None,
},
)
.map(Some)
}
+13
View File
@@ -2326,6 +2326,7 @@ impl IntoResponse for RuntimeHttpWorkdirError {
#[derive(Debug)] #[derive(Debug)]
struct RuntimeHttpRestError { struct RuntimeHttpRestError {
create_failure: Option<runtime_api::startup::WorkerCreateFailure>,
status: StatusCode, status: StatusCode,
code: String, code: String,
message: String, message: String,
@@ -2339,12 +2340,14 @@ impl RuntimeHttpRestError {
code: code.into(), code: code.into(),
message: message.into(), message: message.into(),
startup: None, startup: None,
create_failure: None,
} }
} }
fn runtime(error: RuntimeError) -> Self { fn runtime(error: RuntimeError) -> Self {
let status = status_for_runtime_error(&error); let status = status_for_runtime_error(&error);
let code = code_for_runtime_error(&error); let code = code_for_runtime_error(&error);
let create_failure = error.create_failure();
let startup = match &error { let startup = match &error {
RuntimeError::WorkerExecutionRejected { result, .. } => startup_transfer(result), RuntimeError::WorkerExecutionRejected { result, .. } => startup_transfer(result),
_ => None, _ => None,
@@ -2359,6 +2362,7 @@ impl RuntimeHttpRestError {
code, code,
message, message,
startup, startup,
create_failure,
} }
} }
@@ -2391,6 +2395,13 @@ impl RuntimeHttpRestError {
impl IntoResponse for RuntimeHttpRestError { impl IntoResponse for RuntimeHttpRestError {
fn into_response(self) -> Response { fn into_response(self) -> Response {
if let Some(failure) = self.create_failure {
return runtime_api::server_support::http_error_response(
self.status,
runtime_api::RuntimeApiError::new(self.status.as_u16(), self.code, self.message)
.with_create_failure(failure),
);
}
let body = RuntimeHttpErrorResponse { let body = RuntimeHttpErrorResponse {
error: RuntimeHttpErrorDetail { error: RuntimeHttpErrorDetail {
code: self.code, code: self.code,
@@ -2422,6 +2433,7 @@ fn startup_transfer(
fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
match error { match error {
RuntimeError::WorkerCreateFailed { .. } => StatusCode::INTERNAL_SERVER_ERROR,
RuntimeError::WorkerNotFound { .. } | RuntimeError::ConfigBundleMissing { .. } => { RuntimeError::WorkerNotFound { .. } | RuntimeError::ConfigBundleMissing { .. } => {
StatusCode::NOT_FOUND StatusCode::NOT_FOUND
} }
@@ -2531,6 +2543,7 @@ fn code_for_runtime_error(error: &RuntimeError) -> String {
RuntimeError::ExecutionBackendUnavailable { .. } => { RuntimeError::ExecutionBackendUnavailable { .. } => {
"execution_backend_unavailable".to_string() "execution_backend_unavailable".to_string()
} }
RuntimeError::WorkerCreateFailed { .. } => "worker_create_failed".to_string(),
RuntimeError::WorkerExecutionRejected { .. } => "worker_execution_rejected".to_string(), RuntimeError::WorkerExecutionRejected { .. } => "worker_execution_rejected".to_string(),
RuntimeError::WorkspaceOwnerMismatch { .. } => "workspace_owner_mismatch".to_string(), RuntimeError::WorkspaceOwnerMismatch { .. } => "workspace_owner_mismatch".to_string(),
RuntimeError::LimitTooLarge { .. } => "limit_too_large".to_string(), RuntimeError::LimitTooLarge { .. } => "limit_too_large".to_string(),
@@ -227,6 +227,9 @@ fn api_error(error: RuntimeHttpRestError) -> runtime_api::RuntimeApiError {
let mut response = let mut response =
runtime_api::RuntimeApiError::new(error.status.as_u16(), error.code, error.message); runtime_api::RuntimeApiError::new(error.status.as_u16(), error.code, error.message);
response.error.startup = error.startup; response.error.startup = error.startup;
if let Some(failure) = error.create_failure {
return response.with_create_failure(failure);
}
response response
} }
+1
View File
@@ -31,6 +31,7 @@ mod session_material;
mod session_public; mod session_public;
mod session_retention; mod session_retention;
pub mod ssh_host_key_probe; pub mod ssh_host_key_probe;
mod startup_observation;
pub mod worker_backend; pub mod worker_backend;
pub mod worker_source; pub mod worker_source;
pub mod working_directory; pub mod working_directory;
+396 -231
View File
@@ -56,6 +56,11 @@ use protocol::subscription::{
SubscriptionWorkerState, SubscriptionWorkerWorkdirAttachment, SubscriptionWorkerState, SubscriptionWorkerWorkdirAttachment,
}; };
use protocol::{Event, Method}; use protocol::{Event, Method};
use runtime_api::startup::{
CompensationOutcome, CreationAcceptance, ExecutionOutcome, FinalizeFacts, FinalizeStage,
StartupBoundaryCause, StartupBoundaryReason, StartupCause, StartupCorrelation,
StartupOperationFacts, WorkerCreateFailure,
};
use server_api::WorkerRestoreState; use server_api::WorkerRestoreState;
use std::collections::BTreeMap; use std::collections::BTreeMap;
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
@@ -1381,7 +1386,34 @@ impl Runtime {
let worker_id = request.worker_id; let worker_id = request.worker_id;
let workspace_id = scope.map(|scope| scope.workspace_id.as_str()); let workspace_id = scope.map(|scope| scope.workspace_id.as_str());
let started_at = std::time::Instant::now(); let started_at = std::time::Instant::now();
let result = self.create_worker_with_workspace_inner(request, scope); let result = self
.create_worker_with_workspace_inner(request, scope)
.map_err(|error| {
// Use only the host-bound local identity, never received correlation.
// A poisoned/unbound Runtime cannot invent correlation authority.
let correlation = self.lock().ok().and_then(|state| {
state.runtime_identity.as_deref().map(|runtime_id| {
StartupCorrelation::for_worker(runtime_id, &worker_id.to_string())
})
});
match (error.create_failure(), correlation) {
(Some(failure), Some(correlation)) => {
let mut operation = failure.operation().clone();
operation.correlation = Some(correlation);
let failure = Box::new(failure.with_operation(operation));
match error {
RuntimeError::WorkerCreateFailed { source, .. } => {
RuntimeError::WorkerCreateFailed { failure, source }
}
source => RuntimeError::WorkerCreateFailed {
failure,
source: Box::new(source),
},
}
}
_ => error,
}
});
if let Err(error) = &result { if let Err(error) = &result {
write_runtime_worker_create_failure( write_runtime_worker_create_failure(
worker_id, worker_id,
@@ -1528,13 +1560,13 @@ impl Runtime {
WorkerExecutionSpawnResult::Rejected(mut result) WorkerExecutionSpawnResult::Rejected(mut result)
| WorkerExecutionSpawnResult::RolledBack(mut result) | WorkerExecutionSpawnResult::RolledBack(mut result)
| WorkerExecutionSpawnResult::Errored(mut result) => { | WorkerExecutionSpawnResult::Errored(mut result) => {
if let Err((error, phase, outcome)) = if let Err((_error, phase, outcome)) =
self.rollback_failed_create_with_phase(&worker_ref) self.rollback_failed_create_with_phase(&worker_ref)
{ {
tracing::error!( tracing::error!(
worker_id = %worker_ref.worker_id, worker_id = %worker_ref.worker_id,
?phase, ?phase,
%error, ?outcome,
"failed to roll back Worker creation" "failed to roll back Worker creation"
); );
// A later failure must not make a confirmed record deletion // A later failure must not make a confirmed record deletion
@@ -1601,7 +1633,8 @@ impl Runtime {
}); });
tracing::error!( tracing::error!(
worker_id = %worker_ref.worker_id, worker_id = %worker_ref.worker_id,
%error, ?phase,
?outcome,
"failed to persist Worker create reconciliation evidence" "failed to persist Worker create reconciliation evidence"
); );
} else { } else {
@@ -1620,80 +1653,119 @@ impl Runtime {
} }
}; };
if let Some(mut initial_input) = { // From Connected onward this owner, not the independent Input API,
let state = self.lock()?; // supplies creation context. Later cleanup must not replace the source.
state let mut execution = ExecutionOutcome::Unknown;
.worker(&worker_ref)? let mut stage = FinalizeStage::InitialInput;
.request let mut reason = StartupBoundaryReason::FinalizeFailed;
.as_ref() let mut persistence_phase = None;
.and_then(|request| request.initial_input.clone()) let mut input_result = None;
} { let result = (|| {
let expected_submission_id = initial_input let initial_input = {
.submission_request_id let state = self.lock()?;
.clone() state
.filter(|request_id| !request_id.trim().is_empty()) .worker(&worker_ref)?
.unwrap_or_else(|| Uuid::now_v7().to_string()); .request
initial_input.submission_request_id = Some(expected_submission_id.clone()); .as_ref()
let dispatch_result = backend.dispatch_input(&worker_ref, initial_input.clone()); .and_then(|request| request.initial_input.clone())
if !dispatch_result.is_accepted() {
self.cleanup_connected_failed_create(&backend, &worker_ref)?;
return Err(RuntimeError::WorkerExecutionRejected {
worker_id: worker_ref.worker_id.clone(),
operation: dispatch_result.operation,
outcome: dispatch_result.outcome,
message: dispatch_result.message_or_default(),
result: dispatch_result,
});
}
let has_durable_acceptance = dispatch_result
.submission
.as_ref()
.is_some_and(|ack| ack.submission_request_id == expected_submission_id);
if !has_durable_acceptance {
self.cleanup_connected_failed_create(&backend, &worker_ref)?;
let result = WorkerExecutionResult::rejected(
WorkerExecutionOperation::Input,
"execution backend accepted initial input without a durable submission acknowledgement",
);
return Err(RuntimeError::WorkerExecutionRejected {
worker_id: worker_ref.worker_id.clone(),
operation: result.operation,
outcome: result.outcome,
message: result.message_or_default(),
result,
});
}
let detail = match self.commit_created_worker(
&worker_ref,
initial_worker_state.clone(),
workdir_attachments,
dispatch_result,
) {
Ok(detail) => detail,
Err(error) => {
self.cleanup_connected_failed_create(&backend, &worker_ref)?;
return Err(error);
}
}; };
if let Err(error) = self.record_input_observation(&worker_ref, initial_input) { if let Some(mut initial_input) = initial_input {
self.cleanup_connected_failed_create(&backend, &worker_ref)?; let expected_submission_id = initial_input
return Err(error); .submission_request_id
} .clone()
Ok(detail) .filter(|request_id| !request_id.trim().is_empty())
} else { .unwrap_or_else(|| Uuid::now_v7().to_string());
match self.commit_created_worker( initial_input.submission_request_id = Some(expected_submission_id.clone());
&worker_ref, let dispatch_result = backend.dispatch_input(&worker_ref, initial_input.clone());
initial_worker_state, let has_durable_acceptance = dispatch_result
workdir_attachments, .submission
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn), .as_ref()
) { .is_some_and(|ack| ack.submission_request_id == expected_submission_id);
Ok(detail) => Ok(detail), input_result = Some(dispatch_result.clone());
Err(error) => { if !dispatch_result.is_accepted() {
self.cleanup_connected_failed_create(&backend, &worker_ref)?; // Confirmed rejection is failed initial execution. A lost
Err(error) // acknowledgement does not establish non-execution.
execution = match dispatch_result.outcome {
crate::execution::WorkerExecutionOutcome::Errored
if !has_durable_acceptance =>
{
ExecutionOutcome::Unknown
}
_ => ExecutionOutcome::Errored,
};
return Err(RuntimeError::WorkerExecutionRejected {
worker_id: worker_ref.worker_id,
operation: dispatch_result.operation,
outcome: dispatch_result.outcome,
message: dispatch_result.message_or_default(),
result: dispatch_result,
});
} }
if !has_durable_acceptance {
reason = StartupBoundaryReason::InvalidResponse;
return Err(RuntimeError::WorkerExecutionRejected {
worker_id: worker_ref.worker_id,
operation: dispatch_result.operation,
outcome: dispatch_result.outcome,
message: "initial input has no matching durable acknowledgement".into(),
result: dispatch_result,
});
}
execution = ExecutionOutcome::Accepted;
stage = FinalizeStage::Publication;
let detail = self.commit_created_worker(
&worker_ref,
initial_worker_state,
workdir_attachments,
dispatch_result,
&mut persistence_phase,
)?;
self.record_input_observation(&worker_ref, initial_input)?;
Ok(detail)
} else {
execution = ExecutionOutcome::Accepted;
stage = FinalizeStage::Publication;
self.commit_created_worker(
&worker_ref,
initial_worker_state,
workdir_attachments,
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
&mut persistence_phase,
)
} }
} })();
result.map_err(|source| {
let mut startup = connected_create_startup(&source, input_result.as_ref(), reason);
if let Some(phase) = persistence_phase {
push_create_secondary(
&mut startup,
StartupSecondaryFailure::Persistence {
phase,
outcome: create_persistence_outcome(&source),
},
);
}
let compensation =
self.cleanup_connected_failed_create(&backend, &worker_ref, &mut startup);
let failure = WorkerCreateFailure::new(
startup,
StartupOperationFacts {
acceptance: CreationAcceptance::Accepted,
execution,
finalize: Some(FinalizeFacts {
stage,
compensation,
}),
backend_compensation: None,
correlation: None,
},
)
.expect("connected create facts have bounded secondary observations");
RuntimeError::WorkerCreateFailed {
failure: Box::new(failure),
source: Box::new(source),
}
})
} }
/// List Workers known to this Runtime. /// List Workers known to this Runtime.
@@ -2267,7 +2339,7 @@ impl Runtime {
Some(RuntimeWorkerRestoreResult::failed( Some(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_canonical_write_unknown", "worker_restore_canonical_write_unknown",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
)), )),
None, None,
)); ));
@@ -2322,7 +2394,7 @@ impl Runtime {
Some(RuntimeWorkerRestoreResult::failed( Some(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_authority_changed", "worker_restore_authority_changed",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
)), )),
None, None,
)); ));
@@ -2432,7 +2504,7 @@ impl Runtime {
Some(RuntimeWorkerRestoreResult::failed( Some(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_admission_commit_unknown", "worker_restore_admission_commit_unknown",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
)), )),
None, None,
)); ));
@@ -2665,7 +2737,7 @@ impl Runtime {
return Ok(RuntimeWorkerRestoreResult::failed( return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_saved_binding_not_ready", "worker_restore_saved_binding_not_ready",
sanitize_worker_create_failure_message(&message), sanitize_worker_restore_failure_message(&message),
)); ));
} }
let result = RuntimeWorkerRestoreResult::accepted(detail); let result = RuntimeWorkerRestoreResult::accepted(detail);
@@ -2813,7 +2885,7 @@ impl Runtime {
WorkerRestoreState::Rejected WorkerRestoreState::Rejected
}, },
"worker_restore_preparation_rejected", "worker_restore_preparation_rejected",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
)); ));
} }
}; };
@@ -2922,7 +2994,7 @@ impl Runtime {
return Ok(RuntimeWorkerRestoreResult::failed( return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_observation_activation_pending", "worker_restore_observation_activation_pending",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
)); ));
} }
} }
@@ -2933,7 +3005,7 @@ impl Runtime {
return Ok(RuntimeWorkerRestoreResult::failed( return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::Rejected, WorkerRestoreState::Rejected,
"worker_restore_preflight_rejected", "worker_restore_preflight_rejected",
sanitize_worker_create_failure_message(&result.message_or_default()), sanitize_worker_restore_failure_message(&result.message_or_default()),
)); ));
} }
@@ -3001,7 +3073,7 @@ impl Runtime {
return Ok(RuntimeWorkerRestoreResult::failed( return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_activation_pending", "worker_restore_activation_pending",
sanitize_worker_create_failure_message(&error), sanitize_worker_restore_failure_message(&error),
)); ));
} }
#[cfg(feature = "restore-checkpoints")] #[cfg(feature = "restore-checkpoints")]
@@ -3021,7 +3093,7 @@ impl Runtime {
return Ok(RuntimeWorkerRestoreResult::failed( return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_observation_activation_pending", "worker_restore_observation_activation_pending",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
)); ));
} }
let worker = self.lock()?.worker(worker_ref)?.detail(); let worker = self.lock()?.worker(worker_ref)?.detail();
@@ -3030,7 +3102,7 @@ impl Runtime {
return Ok(RuntimeWorkerRestoreResult::failed( return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_receipt_commit_pending", "worker_restore_receipt_commit_pending",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
)); ));
} }
Ok(result) Ok(result)
@@ -3057,7 +3129,7 @@ impl Runtime {
let receipt = RuntimeWorkerRestoreResult::failed( let receipt = RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::RolledBack, WorkerRestoreState::RolledBack,
"worker_restore_decision_not_committed", "worker_restore_decision_not_committed",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
); );
if self.settle_restore_rollback(worker_ref, &receipt).is_ok() { if self.settle_restore_rollback(worker_ref, &receipt).is_ok() {
return Ok(receipt); return Ok(receipt);
@@ -3084,7 +3156,7 @@ impl Runtime {
Ok(RuntimeWorkerRestoreResult::failed( Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_commit_reconciliation_required", "worker_restore_commit_reconciliation_required",
sanitize_worker_create_failure_message(&error.to_string()), sanitize_worker_restore_failure_message(&error.to_string()),
)) ))
} }
} }
@@ -3094,14 +3166,14 @@ impl Runtime {
return Ok(RuntimeWorkerRestoreResult::failed( return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_reconciliation_rejected", "worker_restore_reconciliation_rejected",
sanitize_worker_create_failure_message(&result.message_or_default()), sanitize_worker_restore_failure_message(&result.message_or_default()),
)); ));
} }
self.discard_restore_candidate_context(worker_ref, &candidate_context)?; self.discard_restore_candidate_context(worker_ref, &candidate_context)?;
let receipt = RuntimeWorkerRestoreResult::failed( let receipt = RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::Rejected, WorkerRestoreState::Rejected,
"worker_restore_rejected", "worker_restore_rejected",
sanitize_worker_create_failure_message(&result.message_or_default()), sanitize_worker_restore_failure_message(&result.message_or_default()),
); );
if let Err(error) = self.settle_restore_rollback(worker_ref, &receipt) { if let Err(error) = self.settle_restore_rollback(worker_ref, &receipt) {
tracing::error!( tracing::error!(
@@ -3121,7 +3193,7 @@ impl Runtime {
let receipt = RuntimeWorkerRestoreResult::failed( let receipt = RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::RolledBack, WorkerRestoreState::RolledBack,
"worker_restore_rolled_back", "worker_restore_rolled_back",
sanitize_worker_create_failure_message(&_result.message_or_default()), sanitize_worker_restore_failure_message(&_result.message_or_default()),
); );
self.discard_restore_candidate_context(worker_ref, &candidate_context)?; self.discard_restore_candidate_context(worker_ref, &candidate_context)?;
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
@@ -3174,7 +3246,7 @@ impl Runtime {
Ok(RuntimeWorkerRestoreResult::failed( Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_reconciliation_required", "worker_restore_reconciliation_required",
sanitize_worker_create_failure_message(&format!( sanitize_worker_restore_failure_message(&format!(
"Worker restore requires reconciliation ({:?}): {}", "Worker restore requires reconciliation ({:?}): {}",
result.outcome, result.outcome,
result.message_or_default(), result.message_or_default(),
@@ -3194,7 +3266,7 @@ impl Runtime {
Ok(RuntimeWorkerRestoreResult::failed( Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired, WorkerRestoreState::ReconciliationRequired,
"worker_restore_backend_outcome_unknown", "worker_restore_backend_outcome_unknown",
sanitize_worker_create_failure_message(&_result.message_or_default()), sanitize_worker_restore_failure_message(&_result.message_or_default()),
)) ))
} }
} }
@@ -4007,6 +4079,7 @@ impl Runtime {
initial_worker_state: protocol::WorkerStateSnapshot, initial_worker_state: protocol::WorkerStateSnapshot,
workdir_attachments: Vec<crate::catalog::WorkingDirectoryAttachmentStatus>, workdir_attachments: Vec<crate::catalog::WorkingDirectoryAttachmentStatus>,
result: WorkerExecutionResult, result: WorkerExecutionResult,
persistence_phase: &mut Option<StartupFailurePhase>,
) -> Result<WorkerDetail, RuntimeError> { ) -> Result<WorkerDetail, RuntimeError> {
let mut state = self.lock()?; let mut state = self.lock()?;
let detail = { let detail = {
@@ -4030,41 +4103,155 @@ impl Runtime {
worker.workdir_attachments = workdir_attachments; worker.workdir_attachments = workdir_attachments;
worker.detail() worker.detail()
}; };
*persistence_phase = Some(StartupFailurePhase::RuntimeSnapshotSave);
state.persist_runtime_snapshot()?; state.persist_runtime_snapshot()?;
*persistence_phase = Some(StartupFailurePhase::RuntimeReconciliationRecordSave);
state.persist_worker(&worker_ref.worker_id)?; state.persist_worker(&worker_ref.worker_id)?;
*persistence_phase = None;
state.publish_worker_upsert(worker_ref.worker_id)?; state.publish_worker_upsert(worker_ref.worker_id)?;
Ok(detail) Ok(detail)
} }
/// Observe compensation separately from the original create failure. This
/// performs the same single stop and cleanup as before; no receipt lookup,
/// retry, reexecution or new durable lifecycle operation is introduced here.
fn cleanup_connected_failed_create( fn cleanup_connected_failed_create(
&self, &self,
backend: &WorkerExecutionBackendRef, backend: &WorkerExecutionBackendRef,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
) -> Result<(), RuntimeError> { startup: &mut runtime_api::WorkerStartupFailure,
) -> CompensationOutcome {
use protocol::startup_failure::{
StartupCleanup, StartupCleanupFailure, StartupResourceFacts, StartupResources,
};
let operation_id = WorkerLifecycleOperationId::new(); let operation_id = WorkerLifecycleOperationId::new();
let stop_result = backend.stop_worker_operation(WorkerExecutionStopRequest { let stop_result = backend.stop_worker_operation(WorkerExecutionStopRequest {
operation_id, operation_id,
worker_ref: worker_ref.clone(), worker_ref: worker_ref.clone(),
}); });
if stop_result.is_accepted() for secondary in &stop_result.secondary_failures {
push_create_secondary(startup, secondary.clone());
}
if startup.io_kind.is_none() {
startup.io_kind = stop_result.startup_io_kind;
}
// Preserve the original owner's resource facts, if present. In their
// absence, connection establishes startup resources but not their release.
let had_resources = startup.resources.is_some() || stop_result.startup_resources.is_some();
startup
.resources
.get_or_insert(
stop_result
.startup_resources
.unwrap_or(StartupResourceFacts {
resources: StartupResources::WorkerStartup,
cleanup: StartupCleanup::Required,
durable_effects_possible: true,
}),
);
if let Some(outcome) = stop_result.startup_failure_persistence {
push_create_secondary(
startup,
StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::MetadataSave,
outcome,
},
);
}
let terminated = stop_result.is_accepted()
&& stop_result.termination.as_ref().is_some_and(|termination| { && stop_result.termination.as_ref().is_some_and(|termination| {
termination.confirms_scope_shutdown(worker_ref, operation_id) termination.confirms_scope_shutdown(worker_ref, operation_id)
}) });
{ if terminated {
return self.rollback_failed_create(worker_ref); if !had_resources {
// Confirms scope shutdown only, not removal of Workdir contents
// or undoing already accepted initial-input effects.
startup.resources.as_mut().unwrap().cleanup = StartupCleanup::Confirmed;
}
match self.rollback_failed_create_with_phase(worker_ref) {
Ok(()) => CompensationOutcome::Completed,
Err((error, phase, outcome)) => {
if matches!(
phase,
StartupFailurePhase::RuntimeSnapshotSave
| StartupFailurePhase::RuntimeRemovalPublish
) {
push_create_secondary(
startup,
StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::RuntimeRecordDelete,
outcome: StartupPersistenceOutcome::Persisted,
},
);
}
if phase == StartupFailurePhase::RuntimeRemovalPublish {
push_create_secondary(
startup,
StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::RuntimeSnapshotSave,
outcome: StartupPersistenceOutcome::Persisted,
},
);
}
record_create_persistence_failure(startup, &error, phase, outcome);
if outcome == StartupPersistenceOutcome::OutcomeUnknown {
CompensationOutcome::Unknown
} else {
CompensationOutcome::Residual
}
}
}
} else {
let failed = !stop_result.is_accepted()
&& stop_result.outcome != crate::execution::WorkerExecutionOutcome::Errored;
let feature_failed = stop_result.termination.as_ref().is_some_and(|receipt| {
receipt.worker_ref == *worker_ref
&& receipt.stop_operation_id == operation_id
&& !receipt.feature_failures.is_empty()
});
if failed || feature_failed {
let failure = if feature_failed {
StartupCleanupFailure::Feature
} else if stop_result.outcome
== crate::execution::WorkerExecutionOutcome::Unsupported
{
StartupCleanupFailure::Unsupported
} else {
StartupCleanupFailure::Ownership
};
push_create_secondary(startup, StartupSecondaryFailure::Cleanup { failure });
if !had_resources {
startup.resources.as_mut().unwrap().cleanup = StartupCleanup::Failed(failure);
}
}
let mut phase = StartupFailurePhase::RuntimeSnapshotSave;
let retained = (|| {
let mut state = self.lock()?;
let record = state.worker_mut(worker_ref)?;
record.worker_state = stop_result.worker_state.clone();
state.persist_runtime_snapshot()?;
phase = StartupFailurePhase::RuntimeReconciliationRecordSave;
state.persist_worker(&worker_ref.worker_id)
})();
match retained {
Ok(()) => push_create_secondary(
startup,
StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::RuntimeReconciliationRecordSave,
outcome: StartupPersistenceOutcome::Persisted,
},
),
Err(error) => {
let outcome = create_persistence_outcome(&error);
record_create_persistence_failure(startup, &error, phase, outcome);
}
}
if failed || feature_failed {
CompensationOutcome::Residual
} else {
CompensationOutcome::Unknown
}
} }
let mut state = self.lock()?;
let record = state.worker_mut(worker_ref)?;
record.worker_state = stop_result.worker_state.clone();
state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?;
Ok(())
}
fn rollback_failed_create(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
self.rollback_failed_create_with_phase(worker_ref)
.map_err(|(error, _, _)| error)
} }
// The legacy caller still receives its RuntimeError. Spawn failures instead // The legacy caller still receives its RuntimeError. Spawn failures instead
@@ -5110,7 +5297,7 @@ impl Runtime {
let mut state = self.lock()?; let mut state = self.lock()?;
let diagnostic_id = state.next_diagnostic_id; let diagnostic_id = state.next_diagnostic_id;
state.next_diagnostic_id = state.next_diagnostic_id.saturating_add(1); state.next_diagnostic_id = state.next_diagnostic_id.saturating_add(1);
let message = sanitize_worker_create_failure_message( let message = sanitize_worker_restore_failure_message(
result result
.message .message
.as_deref() .as_deref()
@@ -7148,7 +7335,7 @@ impl RuntimeState {
result: &WorkerExecutionResult, result: &WorkerExecutionResult,
receipt: &RuntimeWorkerRestoreResult, receipt: &RuntimeWorkerRestoreResult,
) -> Result<(), RuntimeError> { ) -> Result<(), RuntimeError> {
let message = sanitize_worker_create_failure_message(&result.message_or_default()); let message = sanitize_worker_restore_failure_message(&result.message_or_default());
let diagnostic_id = self.next_diagnostic_id; let diagnostic_id = self.next_diagnostic_id;
let workspace_id = self let workspace_id = self
.workers .workers
@@ -8381,25 +8568,90 @@ fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), R
Ok(()) Ok(())
} }
fn worker_execution_failure_code(result: &WorkerExecutionResult) -> &'static str { /// Keep the original result's typed facts even when publication is the failing
let message = result.message.as_deref().unwrap_or_default(); /// boundary. No message/path parsing is used to choose the fallback cause.
if message.contains("timed out waiting for durable Worker Submit acceptance") { fn connected_create_startup(
"input_commit_timeout" source: &RuntimeError,
} else if message.contains("worker adapter task did not complete within") { result: Option<&WorkerExecutionResult>,
"adapter_task_timeout" reason: StartupBoundaryReason,
} else if message.contains("failed to send Worker method") { ) -> runtime_api::WorkerStartupFailure {
"worker_method_send_failed" let fallback = WorkerCreateFailure::from_cause(
} else if message.contains("worker rejected Submit") { StartupCause::Boundary(StartupBoundaryCause { reason }),
"worker_submit_rejected" StartupOperationFacts {
} else if message.contains("before durable acceptance") { acceptance: CreationAcceptance::Accepted,
"worker_failed_before_input_commit" execution: ExecutionOutcome::Unknown,
} else if message.contains("event stream closed") { finalize: None,
"worker_event_stream_closed" backend_compensation: None,
} else { correlation: None,
"worker_execution_failed" },
)
.expect("closed startup boundary cause is valid");
let (mut startup, _) = fallback.into_parts();
if let Some(result) = result {
if let Some(primary) = &result.startup_failure {
startup.primary = primary.clone();
}
for secondary in &result.secondary_failures {
push_create_secondary(&mut startup, secondary.clone());
}
startup.resources = result.startup_resources;
startup.persistence = result.startup_failure_persistence;
startup.io_kind = result.startup_io_kind;
}
if startup.io_kind.is_none() {
startup.io_kind = create_io_kind(source);
}
startup
}
fn create_io_kind(error: &RuntimeError) -> Option<protocol::startup_failure::StartupIoKind> {
match error {
RuntimeError::StoreIo { source, .. }
| RuntimeError::StoreCommitOutcomeUnknown { source, .. } => Some(source.kind().into()),
_ => None,
} }
} }
fn create_persistence_outcome(error: &RuntimeError) -> StartupPersistenceOutcome {
if matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) {
StartupPersistenceOutcome::OutcomeUnknown
} else {
StartupPersistenceOutcome::NotSaved
}
}
fn push_create_secondary(
startup: &mut runtime_api::WorkerStartupFailure,
secondary: StartupSecondaryFailure,
) {
// The create DTO has the core diagnostic bound; this is an observation,
// not another durable ledger. The original error remains boxed as source.
if startup.secondary.len() < error_contract::MAX_DIAGNOSTICS {
startup.secondary.push(secondary);
}
}
fn record_create_persistence_failure(
startup: &mut runtime_api::WorkerStartupFailure,
error: &RuntimeError,
phase: StartupFailurePhase,
outcome: StartupPersistenceOutcome,
) {
push_create_secondary(
startup,
StartupSecondaryFailure::Persistence { phase, outcome },
);
if startup.io_kind.is_none() {
startup.io_kind = create_io_kind(error);
}
tracing::error!(
?phase,
?outcome,
"Worker create compensation persistence failed"
);
}
#[cfg(test)]
fn runtime_worker_create_failure_fields( fn runtime_worker_create_failure_fields(
error: &RuntimeError, error: &RuntimeError,
) -> (&'static str, Option<String>, Option<String>) { ) -> (&'static str, Option<String>, Option<String>) {
@@ -8425,6 +8677,7 @@ fn runtime_worker_create_failure_fields(
RuntimeError::ExecutionBackendUnavailable { .. } => { RuntimeError::ExecutionBackendUnavailable { .. } => {
("execution_backend_unavailable", None, None) ("execution_backend_unavailable", None, None)
} }
RuntimeError::WorkerCreateFailed { .. } => ("worker_create_failed", None, None),
RuntimeError::WorkerExecutionRejected { RuntimeError::WorkerExecutionRejected {
operation, outcome, .. operation, outcome, ..
} => ( } => (
@@ -8454,9 +8707,11 @@ fn runtime_worker_create_failure_fields(
} }
} }
const WORKER_CREATE_FAILURE_MESSAGE_LIMIT: usize = 512; // Legacy Restore diagnostic presentation only. This is not a create failure
// projector or telemetry allowlist; creation uses startup_observation instead.
const WORKER_RESTORE_FAILURE_MESSAGE_LIMIT: usize = 512;
fn sanitize_worker_create_failure_message(message: &str) -> String { fn sanitize_worker_restore_failure_message(message: &str) -> String {
let mut redact_next = false; let mut redact_next = false;
let detail = message let detail = message
.split_whitespace() .split_whitespace()
@@ -8496,10 +8751,10 @@ fn sanitize_worker_create_failure_message(message: &str) -> String {
}) })
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join(" "); .join(" ");
let truncated = detail.chars().count() > WORKER_CREATE_FAILURE_MESSAGE_LIMIT; let truncated = detail.chars().count() > WORKER_RESTORE_FAILURE_MESSAGE_LIMIT;
let mut detail = detail let mut detail = detail
.chars() .chars()
.take(WORKER_CREATE_FAILURE_MESSAGE_LIMIT) .take(WORKER_RESTORE_FAILURE_MESSAGE_LIMIT)
.collect::<String>(); .collect::<String>();
if truncated { if truncated {
detail.push('…'); detail.push('…');
@@ -8507,43 +8762,13 @@ fn sanitize_worker_create_failure_message(message: &str) -> String {
detail detail
} }
fn runtime_worker_create_failure_message(error: &RuntimeError) -> Option<String> {
match error {
RuntimeError::WorkerExecutionRejected { message, .. } => {
Some(sanitize_worker_create_failure_message(message))
}
_ => None,
}
}
fn write_runtime_worker_create_failure( fn write_runtime_worker_create_failure(
worker_id: WorkerId, worker_id: WorkerId,
workspace_id: Option<&str>, workspace_id: Option<&str>,
error: &RuntimeError, error: &RuntimeError,
elapsed: std::time::Duration, elapsed: std::time::Duration,
) { ) {
let (error_kind, operation, outcome) = runtime_worker_create_failure_fields(error); crate::startup_observation::write_create_failure(worker_id, workspace_id, error, elapsed);
let execution_failure_code = match error {
RuntimeError::WorkerExecutionRejected { result, .. } => {
worker_execution_failure_code(result)
}
_ => "",
};
let execution_failure_message = runtime_worker_create_failure_message(error);
tracing::error!(
target: "yoi::worker_create",
event = "worker_create_failed",
component = "runtime",
workspace_id = workspace_id.unwrap_or(""),
worker_id = %worker_id,
error_kind,
operation = operation.as_deref().unwrap_or(""),
outcome = outcome.as_deref().unwrap_or(""),
execution_failure_code,
execution_failure_message = execution_failure_message.as_deref().unwrap_or(""),
duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
"Worker creation failed"
);
} }
fn validate_create_workspace_scope( fn validate_create_workspace_scope(
@@ -8692,6 +8917,11 @@ mod tests {
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Barrier, Condvar, Mutex}; use std::sync::{Arc, Barrier, Condvar, Mutex};
mod create_failure_tests {
use super::*;
include!("runtime/create_failure_tests.rs");
}
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
mod create_rollback_tests { mod create_rollback_tests {
//! Connect the real create/rollback path to FsRuntimeStore and reopen its disk state. //! Connect the real create/rollback path to FsRuntimeStore and reopen its disk state.
@@ -8911,57 +9141,6 @@ mod tests {
} }
} }
#[test]
fn worker_create_failure_fields_only_include_sanitized_execution_detail() {
let invalid_request = RuntimeError::InvalidRequest("private-token".to_string());
let (error_kind, operation, outcome) =
runtime_worker_create_failure_fields(&invalid_request);
assert_eq!(error_kind, "invalid_request");
assert_eq!(operation, None);
assert_eq!(outcome, None);
assert_eq!(
runtime_worker_create_failure_message(&invalid_request),
None
);
let result = WorkerExecutionResult::errored(
WorkerExecutionOperation::Spawn,
"failed to spawn controller in /private/path: File exists; token super-secret",
);
let error = RuntimeError::WorkerExecutionRejected {
worker_id: WorkerId::now_v7(),
operation: result.operation,
outcome: result.outcome,
message: result.message_or_default(),
result,
};
assert_eq!(
runtime_worker_create_failure_message(&error).as_deref(),
Some("failed to spawn controller in [redacted] File exists; token [redacted]")
);
assert_eq!(
sanitize_worker_create_failure_message("backend private-token failed"),
"backend [redacted] failed"
);
let truncated = sanitize_worker_create_failure_message(&"x".repeat(513));
assert_eq!(truncated.chars().count(), 513);
assert!(truncated.ends_with('…'));
}
#[test]
fn worker_create_failure_code_classifies_durable_acceptance_timeout() {
let timeout = WorkerExecutionResult::errored(
WorkerExecutionOperation::Input,
"timed out waiting for durable Worker Submit acceptance; private-token",
);
assert_eq!(
worker_execution_failure_code(&timeout),
"input_commit_timeout"
);
}
fn test_command() -> protocol::WorkerCommandEnvelope { fn test_command() -> protocol::WorkerCommandEnvelope {
protocol::WorkerCommandEnvelope { command_id: 1 } protocol::WorkerCommandEnvelope { command_id: 1 }
} }
@@ -9950,7 +10129,6 @@ mod tests {
recovery_absent: AtomicBool, recovery_absent: AtomicBool,
recovery_checks: std::sync::atomic::AtomicUsize, recovery_checks: std::sync::atomic::AtomicUsize,
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>, spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
#[cfg(feature = "fs-store")]
before_spawn_result: Mutex<Option<Box<dyn FnOnce() + Send>>>, before_spawn_result: Mutex<Option<Box<dyn FnOnce() + Send>>>,
job_bindings: Mutex<Vec<Option<crate::catalog::BackendJobExecutionBinding>>>, job_bindings: Mutex<Vec<Option<crate::catalog::BackendJobExecutionBinding>>>,
dispatch_result: Mutex<Option<WorkerExecutionResult>>, dispatch_result: Mutex<Option<WorkerExecutionResult>>,
@@ -10197,7 +10375,6 @@ mod tests {
.lock() .lock()
.unwrap() .unwrap()
.push(request.request.backend_job.clone()); .push(request.request.backend_job.clone());
#[cfg(feature = "fs-store")]
if let Some(before_result) = self.before_spawn_result.lock().unwrap().take() { if let Some(before_result) = self.before_spawn_result.lock().unwrap().take() {
before_result(); before_result();
} }
@@ -15503,13 +15680,7 @@ mod tests {
let error = runtime.create_worker(request).unwrap_err(); let error = runtime.create_worker(request).unwrap_err();
assert!(matches!( create_failure_tests::assert_ack_failure(&error);
error,
RuntimeError::WorkerExecutionRejected {
outcome: crate::execution::WorkerExecutionOutcome::Rejected,
..
}
));
assert!(runtime.list_workers().unwrap().is_empty()); assert!(runtime.list_workers().unwrap().is_empty());
} }
@@ -15524,13 +15695,7 @@ mod tests {
let error = runtime.create_worker(request).unwrap_err(); let error = runtime.create_worker(request).unwrap_err();
assert!(matches!( create_failure_tests::assert_ack_failure(&error);
error,
RuntimeError::WorkerExecutionRejected {
outcome: crate::execution::WorkerExecutionOutcome::Rejected,
..
}
));
assert!(runtime.list_workers().unwrap().is_empty()); assert!(runtime.list_workers().unwrap().is_empty());
} }
@@ -17160,7 +17325,7 @@ mod tests {
); );
assert!(!diagnostic.message.contains("credential-value")); assert!(!diagnostic.message.contains("credential-value"));
assert!(!diagnostic.message.contains("/private/path")); assert!(!diagnostic.message.contains("/private/path"));
assert!(diagnostic.message.chars().count() <= WORKER_CREATE_FAILURE_MESSAGE_LIMIT + 1); assert!(diagnostic.message.chars().count() <= WORKER_RESTORE_FAILURE_MESSAGE_LIMIT + 1);
let before = std::fs::read( let before = std::fs::read(
dir.path() dir.path()
.join("runtime/workers") .join("runtime/workers")
@@ -17346,7 +17511,7 @@ mod tests {
assert!(!message.contains("/private"), "{message}"); assert!(!message.contains("/private"), "{message}");
assert!(!message.contains("restore-token"), "{message}"); assert!(!message.contains("restore-token"), "{message}");
assert!(!message.contains("hidden-value"), "{message}"); assert!(!message.contains("hidden-value"), "{message}");
assert!(message.chars().count() <= WORKER_CREATE_FAILURE_MESSAGE_LIMIT + 1); assert!(message.chars().count() <= WORKER_RESTORE_FAILURE_MESSAGE_LIMIT + 1);
assert!( assert!(
runtime runtime
.lock() .lock()
@@ -0,0 +1,626 @@
// Registered by runtime.rs's normal library test module. Execution responses are
// fixtures; creation, persistence, projection and compensation use the real owner.
use protocol::startup_failure::{
RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure,
StartupIoKind, StartupResourceFacts, StartupResources, StartupStage,
};
fn boundary_reason(failure: &WorkerCreateFailure) -> StartupBoundaryReason {
let projected = failure
.startup()
.primary
.project_for::<error_contract::InternalTransfer, StartupCause>(&())
.unwrap();
let StartupCause::Boundary(cause) = projected.details() else {
panic!("expected a closed boundary cause");
};
cause.reason
}
pub(super) fn assert_ack_failure(error: &RuntimeError) {
assert!(matches!(error, RuntimeError::WorkerCreateFailed { .. }));
let failure = error.create_failure().unwrap();
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown);
assert_eq!(
failure.operation().finalize,
Some(FinalizeFacts {
stage: FinalizeStage::InitialInput,
compensation: CompensationOutcome::Completed,
})
);
assert_eq!(
boundary_reason(&failure),
StartupBoundaryReason::InvalidResponse
);
// The missing/foreign receipt is not rewritten into a confirmed rejection.
assert!(error.execution_result().unwrap().is_accepted());
assert_eq!(error.to_string(), "worker creation failed");
}
fn rejected_input() -> WorkerExecutionResult {
let attachment = worker::startup_failure::runtime_startup_attachment(RuntimeStartupCause {
stage: StartupStage::Controller,
reason: RuntimeStartupReason::StartupFailed,
})
.unwrap();
let mut result = WorkerExecutionResult::rejected(
WorkerExecutionOperation::Input,
"TEST_ONLY_SECRET original rejection /private/owner/path",
)
.with_startup_failure(attachment);
result
.secondary_failures
.push(StartupSecondaryFailure::Cleanup {
failure: StartupCleanupFailure::Workdir,
});
result.startup_resources = Some(StartupResourceFacts {
resources: StartupResources::Unproven,
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir),
durable_effects_possible: true,
});
result.startup_failure_persistence = Some(StartupPersistenceOutcome::NotSaved);
result.startup_io_kind = Some(StartupIoKind::PermissionDenied);
result
}
#[derive(Clone, Copy, Debug)]
enum DispatchFault {
DroppedAck,
MismatchedAck,
ConfirmedRejection,
LostResponse,
}
#[derive(Clone, Copy, Debug)]
enum CleanupFault {
None,
MissingTermination,
Rejected,
LostResponse,
MismatchedTermination,
}
#[test]
fn connected_create_dispatch_and_cleanup_fault_matrix() {
for dispatch in [
DispatchFault::DroppedAck,
DispatchFault::MismatchedAck,
DispatchFault::ConfirmedRejection,
DispatchFault::LostResponse,
] {
for cleanup in [
CleanupFault::None,
CleanupFault::MissingTermination,
CleanupFault::Rejected,
CleanupFault::LostResponse,
CleanupFault::MismatchedTermination,
] {
let (runtime, backend) = runtime_and_backend();
let original = match dispatch {
DispatchFault::DroppedAck => {
WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
}
DispatchFault::MismatchedAck => {
backend.preserve_submission_acknowledgement_id();
WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
"foreign-request",
"foreign-submission",
protocol::SubmissionDisposition::Started,
)
}
DispatchFault::ConfirmedRejection => rejected_input(),
DispatchFault::LostResponse => WorkerExecutionResult::errored(
WorkerExecutionOperation::Input,
"response lost; TEST_ONLY_SECRET",
),
};
backend.set_dispatch_result(original.clone());
let compensation = match cleanup {
CleanupFault::None => CompensationOutcome::Completed,
CleanupFault::MissingTermination => {
backend.set_stop_result(WorkerExecutionResult::accepted(
WorkerExecutionOperation::Stop,
));
CompensationOutcome::Unknown
}
CleanupFault::Rejected => {
backend.set_stop_result(WorkerExecutionResult::rejected(
WorkerExecutionOperation::Stop,
"TEST_ONLY_SECRET stop rejected",
));
CompensationOutcome::Residual
}
CleanupFault::LostResponse => {
backend.set_stop_result(WorkerExecutionResult::errored(
WorkerExecutionOperation::Stop,
"TEST_ONLY_SECRET stop response lost",
));
CompensationOutcome::Unknown
}
CleanupFault::MismatchedTermination => {
let mut stop = WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop);
stop.termination = Some(crate::execution::WorkerExecutionTermination {
scope: crate::execution::WorkerExecutionTerminationScope::Controller,
worker_ref: WorkerRef::new(WorkerId::now_v7()),
stop_operation_id: WorkerLifecycleOperationId::new(),
execution_id: Uuid::now_v7(),
shutdown_command_id: None,
submissions: Vec::new(),
feature_failures: Vec::new(),
});
backend.set_stop_result(stop);
CompensationOutcome::Unknown
}
};
let mut request = task_request("connected create fault matrix");
request.initial_input = Some(WorkerInput::user("start once"));
let error = runtime.create_worker(request).unwrap_err();
let failure = error.create_failure().unwrap();
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(
failure.operation().execution,
if matches!(dispatch, DispatchFault::ConfirmedRejection) {
ExecutionOutcome::Errored
} else {
ExecutionOutcome::Unknown
},
"{dispatch:?}/{cleanup:?}"
);
assert_eq!(
failure.operation().finalize,
Some(FinalizeFacts {
stage: FinalizeStage::InitialInput,
compensation,
})
);
assert_eq!(failure.operation().backend_compensation, None);
assert_eq!(failure.operation().correlation, None);
assert_eq!(error.execution_result(), Some(&original));
assert_eq!(error.to_string(), "worker creation failed");
if let Some(primary) = original.startup_failure.as_ref() {
assert_eq!(&failure.startup().primary, primary);
assert!(
failure
.startup()
.secondary
.starts_with(&original.secondary_failures)
);
assert_eq!(failure.startup().resources, original.startup_resources);
assert_eq!(
failure.startup().persistence,
original.startup_failure_persistence
);
assert_eq!(failure.startup().io_kind, original.startup_io_kind);
assert!(
std::error::Error::source(&error)
.unwrap()
.to_string()
.contains("TEST_ONLY_SECRET")
);
} else {
assert_eq!(
boundary_reason(&failure),
if matches!(dispatch, DispatchFault::LostResponse) {
StartupBoundaryReason::FinalizeFailed
} else {
StartupBoundaryReason::InvalidResponse
}
);
let resources = failure.startup().resources.unwrap();
assert!(resources.durable_effects_possible);
assert_eq!(
resources.cleanup,
match cleanup {
CleanupFault::None => StartupCleanup::Confirmed,
CleanupFault::Rejected =>
StartupCleanup::Failed(StartupCleanupFailure::Ownership),
_ => StartupCleanup::Required,
}
);
}
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
assert!(
backend
.submission_receipt_queries
.lock()
.unwrap()
.is_empty()
);
assert_eq!(
runtime.list_workers().unwrap().is_empty(),
matches!(cleanup, CleanupFault::None)
);
}
}
}
#[test]
fn independent_input_error_is_not_a_create_failure() {
let (runtime, backend) = runtime_and_backend();
let worker = runtime
.create_worker(task_request("independent input"))
.unwrap();
let original = rejected_input();
backend.set_dispatch_result(original.clone());
let error = runtime
.send_input(&worker.worker_ref, WorkerInput::user("ordinary input"))
.unwrap_err();
assert!(matches!(
&error,
RuntimeError::WorkerExecutionRejected { .. }
));
assert_eq!(error.create_failure(), None);
assert_eq!(error.execution_result(), Some(&original));
assert_eq!(*backend.stop_count.lock().unwrap(), 0);
}
#[test]
fn connected_create_initial_input_read_failure_keeps_create_context() {
let (runtime, backend) = runtime_and_backend();
let mut request = task_request("initial input read failure");
request.initial_input = Some(WorkerInput::user("never dispatched"));
let worker_id = request.worker_id;
*backend.before_spawn_result.lock().unwrap() = Some(Box::new({
let runtime = runtime.clone();
move || {
runtime.lock().unwrap().workers.remove(&worker_id);
}
}));
let error = runtime.create_worker(request).unwrap_err();
let RuntimeError::WorkerCreateFailed { failure, source } = error else {
panic!("post-Connected read failure must remain a create failure");
};
assert!(matches!(*source, RuntimeError::WorkerNotFound { .. }));
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown);
assert_eq!(
failure.operation().finalize.unwrap().stage,
FinalizeStage::InitialInput
);
assert_eq!(
boundary_reason(&failure),
StartupBoundaryReason::FinalizeFailed
);
assert!(backend.dispatched_inputs.lock().unwrap().is_empty());
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
}
#[test]
fn connected_create_publication_failure_preserves_known_input_acceptance() {
let (runtime, backend) = runtime_and_backend();
let mut request = task_request("publication fails after known acceptance");
request.initial_input = Some(WorkerInput::user("accepted once"));
let worker_ref = WorkerRef::new(request.worker_id);
// Substitute only Connected data. The invalid backend Workdir identifier
// reaches the real subscription projection after the owner commit.
backend
.contexts
.lock()
.unwrap()
.insert(request.worker_id, runtime.execution_context(worker_ref));
*backend.spawn_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Connected {
worker_state: protocol::WorkerStatus::Idle.into(),
workdir_attachments: vec![WorkingDirectoryAttachmentStatus {
alias: workdir::WorkdirAttachmentAlias::new("checkout").unwrap(),
working_directory: serde_json::from_value(serde_json::json!({
"summary": {
"working_directory_id": "",
"repository_id": "repo",
"materializer_kind": "runtime_git_clone",
"status": "active"
}
}))
.unwrap(),
}],
});
let error = runtime.create_worker(request).unwrap_err();
let failure = error.create_failure().unwrap();
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted);
assert_eq!(
failure.operation().finalize,
Some(FinalizeFacts {
stage: FinalizeStage::Publication,
compensation: CompensationOutcome::Completed,
})
);
assert_eq!(
boundary_reason(&failure),
StartupBoundaryReason::FinalizeFailed
);
let RuntimeError::WorkerCreateFailed { source, .. } = error else {
unreachable!()
};
assert!(matches!(*source, RuntimeError::InvalidRequest(_)));
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
}
#[test]
fn create_correlation_uses_bound_runtime_identity_for_connected_and_spawn_failures() {
for connected in [false, true] {
let (runtime, backend) = runtime_and_backend();
runtime.bind_runtime_identity("runtime-identity").unwrap();
let mut request = task_request("safe correlation");
let worker_id = request.worker_id;
if connected {
request.initial_input = Some(WorkerInput::user("start once"));
backend.set_dispatch_result(rejected_input());
} else {
let mut result = rejected_input();
result.operation = WorkerExecutionOperation::Spawn;
*backend.spawn_result.lock().unwrap() =
Some(WorkerExecutionSpawnResult::Rejected(result));
}
let error = runtime.create_worker(request).unwrap_err();
assert_eq!(
error.create_failure().unwrap().operation().correlation,
Some(StartupCorrelation::for_worker(
"runtime-identity",
&worker_id.to_string()
))
);
assert_eq!(error.to_string(), "worker creation failed");
}
}
#[cfg(feature = "fs-store")]
#[test]
fn connected_create_commit_write_faults_preserve_acceptance_and_io_facts() {
use crate::fs_store::AtomicWriteFault;
for fault in [
AtomicWriteFault::BeforeRename,
AtomicWriteFault::AfterRename,
] {
for has_initial_input in [false, true] {
let root = tempfile::tempdir().unwrap();
let options = FsRuntimeStoreOptions::new(root.path().join("runtime"))
.with_runtime_id("create-commit-fault");
let backend = Arc::new(TestExecutionBackend::default());
let runtime =
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone())
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
*backend.before_spawn_result.lock().unwrap() = Some(Box::new({
let runtime = runtime.clone();
move || {
runtime
.lock()
.unwrap()
.fs_store()
.unwrap()
.fail_next_worker_write(fault)
}
}));
let mut request = task_request("create commit fault");
if has_initial_input {
request.initial_input = Some(WorkerInput::user("accepted once"));
}
let worker_id = request.worker_id;
let error = runtime.create_worker(request).unwrap_err();
let failure = error.create_failure().unwrap();
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted);
assert_eq!(
failure.operation().finalize,
Some(FinalizeFacts {
stage: FinalizeStage::Publication,
compensation: CompensationOutcome::Completed,
})
);
assert_eq!(
boundary_reason(&failure),
StartupBoundaryReason::FinalizeFailed
);
assert_eq!(failure.startup().io_kind, Some(StartupIoKind::Other));
assert_eq!(failure.startup().persistence, None);
assert!(
failure
.startup()
.secondary
.contains(&StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::RuntimeReconciliationRecordSave,
outcome: if fault == AtomicWriteFault::AfterRename {
StartupPersistenceOutcome::OutcomeUnknown
} else {
StartupPersistenceOutcome::NotSaved
},
})
);
let RuntimeError::WorkerCreateFailed { source, .. } = error else {
unreachable!()
};
assert!(matches!(
*source,
RuntimeError::StoreIo { .. } | RuntimeError::StoreCommitOutcomeUnknown { .. }
));
assert_eq!(
backend.dispatched_inputs.lock().unwrap().len(),
usize::from(has_initial_input)
);
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
assert!(
!options
.root
.join("workers")
.join(worker_id.to_string())
.exists()
);
drop(runtime);
drop(backend);
let reopened = Runtime::with_fs_store(options).unwrap();
assert!(reopened.list_workers().unwrap().is_empty());
}
}
}
#[cfg(feature = "fs-store")]
#[test]
fn connected_create_cleanup_persistence_faults_never_replace_primary() {
for delete_fault in [false, true] {
let root = tempfile::tempdir().unwrap();
let store_root = root.path().join("runtime");
let options =
FsRuntimeStoreOptions::new(&store_root).with_runtime_id("create-cleanup-fault");
let backend = Arc::new(TestExecutionBackend::default());
let runtime =
Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let mut request = task_request("create cleanup persistence fault");
request.initial_input = Some(WorkerInput::user("rejected once"));
let worker_dir = store_root
.join("workers")
.join(request.worker_id.to_string());
let runtime_path = store_root.join("runtime.json");
*backend.before_spawn_result.lock().unwrap() = Some(Box::new(move || {
if delete_fault {
std::fs::create_dir(worker_dir.join("metadata.json")).unwrap();
} else {
std::fs::rename(&runtime_path, runtime_path.with_extension("saved")).unwrap();
std::fs::create_dir(&runtime_path).unwrap();
}
}));
let original = rejected_input();
backend.set_dispatch_result(original.clone());
let error = runtime.create_worker(request).unwrap_err();
let failure = error.create_failure().unwrap();
assert_eq!(error.execution_result(), Some(&original));
assert_eq!(failure.startup().primary, original.startup_failure.unwrap());
assert_eq!(failure.startup().io_kind, original.startup_io_kind);
assert_eq!(failure.operation().execution, ExecutionOutcome::Errored);
assert_eq!(
failure.operation().finalize.unwrap().compensation,
CompensationOutcome::Unknown
);
assert!(
failure
.startup()
.secondary
.contains(&StartupSecondaryFailure::Persistence {
phase: if delete_fault {
StartupFailurePhase::RuntimeRecordDelete
} else {
StartupFailurePhase::RuntimeSnapshotSave
},
outcome: StartupPersistenceOutcome::OutcomeUnknown,
})
);
if !delete_fault {
assert!(
failure
.startup()
.secondary
.contains(&StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::RuntimeRecordDelete,
outcome: StartupPersistenceOutcome::Persisted,
})
);
}
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
}
}
#[test]
fn connected_create_retains_cleanup_owner_resource_and_metadata_observations() {
let (runtime, backend) = runtime_and_backend();
backend.set_dispatch_result(WorkerExecutionResult::accepted(
WorkerExecutionOperation::Input,
));
let mut stop =
WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, "TEST_ONLY_SECRET");
let resources = StartupResourceFacts {
resources: StartupResources::Unproven,
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Feature),
durable_effects_possible: true,
};
stop.startup_resources = Some(resources);
stop.startup_io_kind = Some(StartupIoKind::TimedOut);
stop.startup_failure_persistence = Some(StartupPersistenceOutcome::OutcomeUnknown);
backend.set_stop_result(stop);
let mut request = task_request("cleanup owner observations");
request.initial_input = Some(WorkerInput::user("start once"));
let error = runtime.create_worker(request).unwrap_err();
let failure = error.create_failure().unwrap();
assert_eq!(failure.startup().resources, Some(resources));
assert_eq!(failure.startup().io_kind, Some(StartupIoKind::TimedOut));
assert_eq!(failure.startup().persistence, None);
assert!(
failure
.startup()
.secondary
.contains(&StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::MetadataSave,
outcome: StartupPersistenceOutcome::OutcomeUnknown,
})
);
assert_eq!(
failure.operation().finalize.unwrap().compensation,
CompensationOutcome::Unknown
);
assert_eq!(
boundary_reason(&failure),
StartupBoundaryReason::InvalidResponse
);
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
}
#[cfg(feature = "fs-store")]
#[test]
fn connected_create_unknown_cleanup_retention_write_failure_preserves_primary() {
use crate::fs_store::AtomicWriteFault;
let root = tempfile::tempdir().unwrap();
let options = FsRuntimeStoreOptions::new(root.path().join("runtime"))
.with_runtime_id("create-retain-fault");
let backend = Arc::new(TestExecutionBackend::default());
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
*backend.before_spawn_result.lock().unwrap() = Some(Box::new({
let runtime = runtime.clone();
move || {
runtime
.lock()
.unwrap()
.fs_store()
.unwrap()
.fail_next_worker_write(AtomicWriteFault::BeforeRename)
}
}));
let original = rejected_input();
backend.set_dispatch_result(original.clone());
// Accepted stop without a termination receipt cannot authorize deletion.
backend.set_stop_result(WorkerExecutionResult::accepted(
WorkerExecutionOperation::Stop,
));
let mut request = task_request("unknown cleanup retention persistence failure");
request.initial_input = Some(WorkerInput::user("rejected once"));
let error = runtime.create_worker(request).unwrap_err();
let failure = error.create_failure().unwrap();
assert_eq!(error.execution_result(), Some(&original));
assert_eq!(
failure.operation().finalize.unwrap().compensation,
CompensationOutcome::Unknown
);
assert!(
failure
.startup()
.secondary
.contains(&StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::RuntimeReconciliationRecordSave,
outcome: StartupPersistenceOutcome::NotSaved,
})
);
assert_eq!(runtime.list_workers().unwrap().len(), 1);
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
assert!(
backend
.submission_receipt_queries
.lock()
.unwrap()
.is_empty()
);
}
@@ -479,7 +479,7 @@ fn run(
} }
Ok(Step::Waiting) | Ok(Step::Contended) => None, Ok(Step::Waiting) | Ok(Step::Contended) => None,
Ok(Step::Retry(reason)) => Some(reason), Ok(Step::Retry(reason)) => Some(reason),
Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())), Err(error) => Some(sanitize_worker_restore_failure_message(&error.to_string())),
}; };
if let Some(reason) = &reason { if let Some(reason) = &reason {
repeated = repeated.saturating_add(1); repeated = repeated.saturating_add(1);
@@ -0,0 +1,205 @@
//! The Runtime creation responsibility sink records only projected typed fields.
//! Source/provider strings are never safe merely because a blacklist did not match.
//! Wrappers/projectors do not write events; a failing sink cannot change the result.
use crate::{error::RuntimeError, identity::WorkerId};
pub(crate) fn write_create_failure(
worker_id: WorkerId,
workspace_id: Option<&str>,
error: &RuntimeError,
elapsed: std::time::Duration,
) {
let failure = error.create_failure();
let projected = failure
.as_ref()
.and_then(|failure| failure.telemetry().ok());
let fields = projected.as_ref().map(error_contract::observation_fields);
let operation = failure.as_ref().map(|failure| failure.operation());
tracing::error!(
target: "yoi::worker_create",
event = "worker_create_failed",
component = "runtime",
workspace_id = workspace_id.unwrap_or(""),
worker_id = %worker_id,
domain = fields.as_ref().map_or("startup_boundary", |fields| fields.domain),
reason = fields.as_ref().map_or("observation_unavailable", |fields| fields.reason),
operation = fields.as_ref().and_then(|fields| fields.operation),
stage = fields.as_ref().and_then(|fields| fields.stage),
correlation = operation.and_then(|operation| operation.correlation.as_ref()).map(|value| value.as_str()),
acceptance = ?operation.map(|operation| operation.acceptance),
execution = ?operation.map(|operation| operation.execution),
finalize = ?operation.and_then(|operation| operation.finalize),
persistence = ?failure.as_ref().and_then(|failure| failure.startup().persistence),
resources = ?failure.as_ref().and_then(|failure| failure.startup().resources),
duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
"Worker creation returned a typed failure"
);
}
#[cfg(test)]
mod tests {
use super::*;
use crate::catalog::{CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveSource};
use crate::execution::{
WorkerExecutionBackend, WorkerExecutionOperation, WorkerExecutionResult,
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
};
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveInput};
use crate::{Runtime, RuntimeOptions};
use runtime_api::startup::*;
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use tracing_subscriber::prelude::*;
const CANARIES: &[&str] = &[
"aQ7xV2zN9mL5",
"/private/startup/provider",
"Bearer q3fV8aJ6",
"token=K7sM2pQ4",
];
struct FailingBackend;
impl WorkerExecutionBackend for FailingBackend {
fn backend_id(&self) -> &str {
"log-owner-fixture"
}
fn dispatch_input(
&self,
_: &crate::identity::WorkerRef,
_: crate::interaction::WorkerInput,
) -> WorkerExecutionResult {
panic!("rejected creation never dispatches input")
}
fn spawn_worker(&self, _: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
let result =
WorkerExecutionResult::errored(WorkerExecutionOperation::Spawn, CANARIES.join(" "))
.with_startup_failure(
worker::startup_failure::runtime_startup_attachment(
runtime_api::RuntimeStartupCause {
stage: runtime_api::StartupStage::Controller,
reason: runtime_api::RuntimeStartupReason::StartupFailed,
},
)
.unwrap(),
);
WorkerExecutionSpawnResult::Errored(result)
}
}
#[derive(Clone, Default)]
struct Events(Arc<Mutex<Vec<BTreeMap<String, String>>>>);
struct Fields(BTreeMap<String, String>);
impl tracing::field::Visit for Fields {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.0.insert(field.name().into(), value.into());
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.0.insert(field.name().into(), format!("{value:?}"));
}
}
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Events {
fn on_event(
&self,
event: &tracing::Event<'_>,
_: tracing_subscriber::layer::Context<'_, S>,
) {
if event.metadata().target() == "yoi::worker_create" {
let mut fields = Fields(BTreeMap::new());
event.record(&mut fields);
self.0.lock().unwrap().push(fields.0);
}
}
}
fn request() -> CreateWorkerRequest {
let archive = ProfileSourceArchive::build(ProfileSourceArchiveInput {
id: "log-profile".into(),
entrypoints: BTreeMap::from([("builtin:coder".into(), "profiles/coder.dcdl".into())]),
imports: BTreeMap::new(),
sources: BTreeMap::from([("profiles/coder.dcdl".into(), "{}".into())]),
})
.unwrap();
CreateWorkerRequest {
worker_id: WorkerId::now_v7(),
create_fingerprint: "log-create".into(),
profile: ProfileSelector::Builtin("builtin:coder".into()),
display_name: None,
profile_source: ProfileSourceArchiveSource::Embedded { archive },
config_bundle: None,
initial_input: None,
workdir_attachment_requests: Vec::new(),
workdir_attachments: Vec::new(),
worker_observation_enabled: false,
worker_observation_grants: Vec::new(),
workspace_api: None,
memory_settings: None,
plugin_execution: None,
subjektiv_attached: false,
backend_job: None,
}
}
#[test]
fn real_runtime_creation_sink_omits_unlabelled_provider_canaries_and_matches_typed_result() {
let runtime =
Runtime::with_execution_backend(RuntimeOptions::default(), Arc::new(FailingBackend))
.unwrap();
runtime
.bind_runtime_identity("runtime-log-fixture")
.unwrap();
let request = request();
let worker_id = request.worker_id;
let events = Events::default();
let subscriber = tracing_subscriber::registry().with(events.clone());
let error =
tracing::subscriber::with_default(subscriber, || runtime.create_worker(request))
.unwrap_err();
let failure = error
.create_failure()
.expect("creation owner must retain a typed cause");
let wire = failure
.problem(
StartupBoundary::Runtime,
&SessionFailureVisibility {
can_view_cause: true,
},
"",
)
.unwrap();
let encoded = failure.encode_storage().unwrap();
let DecodeStoredWorkerCreateFailure::Known(saved) =
decode_stored_worker_create_failure(&encoded)
else {
panic!("known storage");
};
assert_eq!(saved.public_facts(), failure.public_facts());
let captured = events.0.lock().unwrap();
assert_eq!(captured.len(), 1);
let event = &captured[0];
let projection = failure.telemetry().unwrap();
let fields = error_contract::observation_fields(&projection);
assert_eq!(event["domain"], fields.domain);
assert_eq!(event["reason"], fields.reason);
assert_eq!(
event["correlation"],
StartupCorrelation::for_worker("runtime-log-fixture", &worker_id.to_string()).as_str()
);
for canary in CANARIES {
assert!(!format!("{captured:?}").contains(canary));
assert!(!String::from_utf8_lossy(wire.body()).contains(canary));
assert!(!encoded.contains(canary));
}
assert!(!event.contains_key("execution_failure_message"));
}
#[test]
fn unsupported_failure_logs_only_fixed_fallback_not_source_text() {
let error = RuntimeError::InvalidRequest(CANARIES.join(" "));
let events = Events::default();
let subscriber = tracing_subscriber::registry().with(events.clone());
tracing::subscriber::with_default(subscriber, || {
write_create_failure(WorkerId::now_v7(), None, &error, std::time::Duration::ZERO)
});
let captured = events.0.lock().unwrap();
assert_eq!(captured.len(), 1);
assert_eq!(captured[0]["reason"], "observation_unavailable");
for canary in CANARIES {
assert!(!format!("{captured:?}").contains(canary));
}
}
}
+15 -3
View File
@@ -529,8 +529,8 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
} }
#[cfg(feature = "http-server")] #[cfg(feature = "http-server")]
fn generated_workspace_response<T: serde::Serialize>( fn generated_workspace_response<T: serde::Serialize, E: serde::Serialize>(
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>, response: Result<T, server_api::client_support::ClientError<E>>,
) -> Result<WorkspaceResponse, WorkspaceClientError> { ) -> Result<WorkspaceResponse, WorkspaceClientError> {
match response { match response {
Ok(response) => Ok(WorkspaceResponse { Ok(response) => Ok(WorkspaceResponse {
@@ -721,7 +721,19 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
), ),
WorkspaceServerOperation::WorkerControlSpawn(request) => { WorkspaceServerOperation::WorkerControlSpawn(request) => {
generated_workspace_response( generated_workspace_response(
client.worker_control_spawn(workspace_id, request).await, client.worker_control_spawn(workspace_id, request).await.map_err(|error| match error {
server_api::client_support::ClientError::Public { status, error } => {
let error = match error.failure().cloned() {
// This already authenticated Worker control caller may
// receive the enum-only public startup cause. The domain
// allowlist reprojects it, never forwards upstream text.
Some(failure) => server_api::WorkerCreateApiError::startup(failure, &protocol::session_failure::SessionFailureVisibility { can_view_cause: true }),
None => error,
};
server_api::client_support::ClientError::Public { status, error }
}
other => other,
}),
) )
} }
WorkspaceServerOperation::WorkerControlInput { WorkspaceServerOperation::WorkerControlInput {
+1 -1
View File
@@ -15,6 +15,7 @@ path = "src/main.rs"
test-support = [] test-support = []
[dependencies] [dependencies]
error-contract.workspace = true
api-macros = { workspace = true, features = ["axum"] } api-macros = { workspace = true, features = ["axum"] }
async-trait.workspace = true async-trait.workspace = true
axum = { workspace = true, features = ["ws"] } axum = { workspace = true, features = ["ws"] }
@@ -70,7 +71,6 @@ webauthn-rs = { workspace = true, features = ["conditional-ui"] }
webauthn-rs-proto = "0.5.5" webauthn-rs-proto = "0.5.5"
[dev-dependencies] [dev-dependencies]
error-contract.workspace = true
rusqlite = { workspace = true, features = ["hooks"] } rusqlite = { workspace = true, features = ["hooks"] }
feature-storage = { workspace = true, features = ["test-support"] } feature-storage = { workspace = true, features = ["test-support"] }
native-tls = "0.2" native-tls = "0.2"
@@ -1476,6 +1476,21 @@ fn captured_history_is_accepted_at_each_real_retained_prefix_and_cannot_be_forge
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let conn = legacy_fixture_to_85(&dir.path().join("prefixes.sqlite"), 85); let conn = legacy_fixture_to_85(&dir.path().join("prefixes.sqlite"), 85);
let original = pre_content_values(&conn, false); let original = pre_content_values(&conn, false);
let mut expected_receipts = rows(
&conn,
"SELECT version,name FROM __yoi_schema_migrations ORDER BY version",
);
expected_receipts.extend(
store_owner::MIGRATIONS
.iter()
.filter(|migration| migration.version > 85)
.map(|migration| {
vec![
Value::Integer(migration.version),
Value::Text(migration.name.into()),
]
}),
);
for migration in store_owner::MIGRATIONS for migration in store_owner::MIGRATIONS
.iter() .iter()
.filter(|migration| migration.version >= 86 && migration.version <= 122) .filter(|migration| migration.version >= 86 && migration.version <= 122)
@@ -1496,10 +1511,13 @@ fn captured_history_is_accepted_at_each_real_retained_prefix_and_cannot_be_forge
&conn, &conn,
"SELECT version,name FROM __yoi_schema_migrations ORDER BY version", "SELECT version,name FROM __yoi_schema_migrations ORDER BY version",
); );
assert_eq!(receipts, expected_receipts);
assert_eq!( assert_eq!(
receipts.last().unwrap(), store_owner::current_schema_version(&conn).unwrap(),
&vec![Value::Integer(123), Value::Text(MIGRATION_NAME.into())] store_owner::LATEST_SCHEMA_VERSION
); );
super::super::storage_formats::verify_history(&conn, store_owner::LATEST_SCHEMA_VERSION)
.unwrap();
assert_eq!(original, pre_content_values(&conn, true)); assert_eq!(original, pre_content_values(&conn, true));
assert!(conn.execute_batch("INSERT INTO backend_job_pre_content_history SELECT * FROM backend_job_pre_content_history LIMIT 1").is_err()); assert!(conn.execute_batch("INSERT INTO backend_job_pre_content_history SELECT * FROM backend_job_pre_content_history LIMIT 1").is_err());
} }
+177 -31
View File
@@ -628,6 +628,10 @@ fn initial_worker_input(segments: &[Segment]) -> Option<EmbeddedWorkerInput> {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkerSpawnResult { pub struct WorkerSpawnResult {
/// Typed failure for internal consumers, independent of diagnostic prose.
/// HTTP publication requires a fresh audience projection, not serialization here.
#[serde(skip)]
pub failure: Option<runtime_api::startup::WorkerCreateFailure>,
pub state: InternalWorkerOperationState, pub state: InternalWorkerOperationState,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub worker: Option<InternalWorkerSummary>, pub worker: Option<InternalWorkerSummary>,
@@ -672,6 +676,8 @@ fn required_worker_workspace_api(
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub enum InternalWorkerOperationState { pub enum InternalWorkerOperationState {
/// The boundary did not observe a conclusive operation result.
Unknown,
Accepted, Accepted,
Unsupported, Unsupported,
Rejected, Rejected,
@@ -680,6 +686,7 @@ pub enum InternalWorkerOperationState {
impl From<InternalWorkerOperationState> for server_api::WorkerOperationState { impl From<InternalWorkerOperationState> for server_api::WorkerOperationState {
fn from(state: InternalWorkerOperationState) -> Self { fn from(state: InternalWorkerOperationState) -> Self {
match state { match state {
InternalWorkerOperationState::Unknown => Self::Unknown,
InternalWorkerOperationState::Accepted => Self::Accepted, InternalWorkerOperationState::Accepted => Self::Accepted,
InternalWorkerOperationState::Unsupported => Self::Unsupported, InternalWorkerOperationState::Unsupported => Self::Unsupported,
InternalWorkerOperationState::Rejected => Self::Rejected, InternalWorkerOperationState::Rejected => Self::Rejected,
@@ -1137,6 +1144,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
request: WorkerSpawnRequest, request: WorkerSpawnRequest,
) -> WorkerSpawnResult { ) -> WorkerSpawnResult {
WorkerSpawnResult { WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Unsupported, state: InternalWorkerOperationState::Unsupported,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -3210,6 +3218,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
{ {
diagnostics.push(embedded_workdir_unsupported_diagnostic()); diagnostics.push(embedded_workdir_unsupported_diagnostic());
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -3226,6 +3235,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
"Embedded backend Runtime is transportless; use run_accepted/create acceptance for backend-internal Workers".to_string(), "Embedded backend Runtime is transportless; use run_accepted/create acceptance for backend-internal Workers".to_string(),
)); ));
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -3251,6 +3261,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
error, error,
)); ));
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -3263,6 +3274,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
Err(diagnostic) => { Err(diagnostic) => {
diagnostics.push(diagnostic); diagnostics.push(diagnostic);
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -3285,6 +3297,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
.create_worker_scoped(&workspace_scope, create_request) .create_worker_scoped(&workspace_scope, create_request)
{ {
Ok(detail) => WorkerSpawnResult { Ok(detail) => WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Accepted, state: InternalWorkerOperationState::Accepted,
worker: Some(self.map_worker_detail(detail)), worker: Some(self.map_worker_detail(detail)),
acceptance_evidence: vec![ acceptance_evidence: vec![
@@ -3303,9 +3316,15 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
diagnostics, diagnostics,
}, },
Err(err) => { Err(err) => {
let failure = err.create_failure();
diagnostics.push(embedded_runtime_diagnostic(&err)); diagnostics.push(embedded_runtime_diagnostic(&err));
let state = failure.as_ref().map_or(
InternalWorkerOperationState::Rejected,
startup_failure_operation_state,
);
WorkerSpawnResult { WorkerSpawnResult {
state: InternalWorkerOperationState::Rejected, failure,
state,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
diagnostics, diagnostics,
@@ -5750,6 +5769,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
WorkerSpawnAcceptanceRequirement::SocketReady WorkerSpawnAcceptanceRequirement::SocketReady
) { ) {
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -5765,6 +5785,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
Ok(archive) => archive, Ok(archive) => archive,
Err(error) => { Err(error) => {
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -5783,6 +5804,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
Ok(workspace_api) => workspace_api, Ok(workspace_api) => workspace_api,
Err(diagnostic) => { Err(diagnostic) => {
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -5802,6 +5824,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
Ok(create) => create, Ok(create) => create,
Err(diagnostic) => { Err(diagnostic) => {
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -5809,30 +5832,70 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
}; };
} }
}; };
match self let client = match self.runtime_api_client(
.run_runtime_api( REMOTE_WORKER_CREATE_TIMEOUT,
REMOTE_WORKER_CREATE_TIMEOUT, MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES, ) {
move |client| async move { client.create_worker(create).await }, Ok(client) => client,
) Err(diagnostic) => {
.and_then(|value| { return WorkerSpawnResult {
runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>(value.worker) failure: None,
}) { state: InternalWorkerOperationState::Rejected,
Ok(response) => WorkerSpawnResult { worker: None,
state: InternalWorkerOperationState::Accepted, acceptance_evidence: Vec::new(),
worker: Some(self.map_worker_detail(response)), diagnostics: vec![diagnostic],
acceptance_evidence: vec![WorkerSpawnAcceptanceEvidence { };
kind: "remote_runtime_worker_created".to_string(), }
detail: "worker-runtime REST create endpoint accepted the Worker".to_string(), };
}], let received = run_blocking_http(move || {
diagnostics: Vec::new(), tokio::runtime::Builder::new_current_thread()
}, .enable_all()
Err(diagnostic) => WorkerSpawnResult { .build()
state: InternalWorkerOperationState::Rejected, .map_err(|_| {
worker: None, runtime_api::client_support::ClientError::failure(
acceptance_evidence: Vec::new(), runtime_api::client_support::ClientFailure::Transport,
diagnostics: vec![diagnostic], )
}, })?
.block_on(client.create_worker(create))
});
match received {
Ok(response) => {
match runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>(
response.worker,
) {
Ok(worker) => WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Accepted,
worker: Some(self.map_worker_detail(worker)),
acceptance_evidence: vec![WorkerSpawnAcceptanceEvidence {
kind: "remote_runtime_worker_created".into(),
detail: "Runtime create returned a Worker summary".into(),
}],
diagnostics: Vec::new(),
},
Err(diagnostic) => WorkerSpawnResult {
failure: Some(startup_observation_failure(
runtime_api::startup::StartupBoundaryReason::Decode,
)),
state: InternalWorkerOperationState::Unknown,
worker: None,
acceptance_evidence: Vec::new(),
diagnostics: vec![diagnostic],
},
}
}
Err(error) => {
let failure = runtime_create_client_failure(&error);
let state = startup_failure_operation_state(&failure);
let diagnostic = runtime_api_diagnostic(&self.runtime_id, error);
WorkerSpawnResult {
failure: Some(failure),
state,
worker: None,
acceptance_evidence: Vec::new(),
diagnostics: vec![diagnostic],
}
}
} }
} }
@@ -6884,6 +6947,11 @@ fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnosti
message, message,
), ),
), ),
EmbeddedRuntimeError::WorkerCreateFailed { failure, .. } => diagnostic(
"embedded_worker_create_failed",
HostDiagnosticSeverity::Warning,
worker::startup_failure::internal_startup_failure_summary(&failure.startup().primary),
),
EmbeddedRuntimeError::WorkerExecutionRejected { EmbeddedRuntimeError::WorkerExecutionRejected {
operation, operation,
outcome, outcome,
@@ -6966,6 +7034,9 @@ fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnosti
}) })
}); });
} }
if let EmbeddedRuntimeError::WorkerCreateFailed { failure, .. } = error {
projection.startup = internal_startup_transfer(failure.startup().clone());
}
projection projection
} }
@@ -7367,6 +7438,7 @@ pub fn placeholder_worker(host_id: impl Into<String>) -> InternalWorkerSummary {
pub fn placeholder_spawn_response(host_id: impl Into<String>) -> WorkerSpawnResult { pub fn placeholder_spawn_response(host_id: impl Into<String>) -> WorkerSpawnResult {
WorkerSpawnResult { WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Unsupported, state: InternalWorkerOperationState::Unsupported,
worker: Some(placeholder_worker(host_id)), worker: Some(placeholder_worker(host_id)),
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -8811,7 +8883,7 @@ mod tests {
} }
#[test] #[test]
fn embedded_runtime_spawn_execution_failure_is_rejected_and_not_input_capable() { fn embedded_runtime_spawn_execution_error_preserves_unknown_acceptance_and_no_input() {
let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend( let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend(
"local:test", "local:test",
Arc::new(FailingSpawnBackend), Arc::new(FailingSpawnBackend),
@@ -8820,14 +8892,22 @@ mod tests {
let mut request = embedded_spawn_request(); let mut request = embedded_spawn_request();
request.requested_worker_name = Some("friendly-name".into()); request.requested_worker_name = Some("friendly-name".into());
let spawned = runtime.spawn_worker(test_create_binding(), request); let spawned = runtime.spawn_worker(test_create_binding(), request);
assert_eq!(spawned.state, InternalWorkerOperationState::Rejected); assert_eq!(spawned.state, InternalWorkerOperationState::Unknown);
let operation = spawned.failure.as_ref().unwrap().operation();
assert_eq!(
operation.acceptance,
runtime_api::startup::CreationAcceptance::Unknown
);
assert_eq!(
operation.execution,
runtime_api::startup::ExecutionOutcome::Errored
);
assert!(spawned.acceptance_evidence.is_empty()); assert!(spawned.acceptance_evidence.is_empty());
assert_eq!(spawned.diagnostics.len(), 1); assert_eq!(spawned.diagnostics.len(), 1);
assert!(spawned.diagnostics.iter().any(|diagnostic| { assert!(spawned.diagnostics.iter().any(|diagnostic| {
diagnostic.code == "embedded_worker_execution_rejected" diagnostic.code == "embedded_worker_create_failed"
&& diagnostic && diagnostic.startup.is_some()
.message && !diagnostic.message.contains("provider setup failed")
.contains("provider setup failed at [redacted]")
&& !diagnostic.message.contains("/tmp/secret-provider-config") && !diagnostic.message.contains("/tmp/secret-provider-config")
&& !diagnostic.message.contains("secret-value") && !diagnostic.message.contains("secret-value")
&& !diagnostic.message.contains("session-42") && !diagnostic.message.contains("session-42")
@@ -9977,3 +10057,69 @@ mod tests {
}) })
} }
} }
/// A local observation failure is not a received execution failure. Its operation
/// facts remain unknown even if HTTP or compatibility diagnostics say "rejected".
fn startup_failure_operation_state(
failure: &runtime_api::startup::WorkerCreateFailure,
) -> InternalWorkerOperationState {
use runtime_api::startup::{CreationAcceptance, ExecutionOutcome};
if failure.operation().acceptance == CreationAcceptance::Unknown
|| failure.operation().execution == ExecutionOutcome::Unknown
{
InternalWorkerOperationState::Unknown
} else if failure.operation().execution == ExecutionOutcome::Unsupported {
InternalWorkerOperationState::Unsupported
} else {
InternalWorkerOperationState::Rejected
}
}
fn startup_observation_failure(
reason: runtime_api::startup::StartupBoundaryReason,
) -> runtime_api::startup::WorkerCreateFailure {
use runtime_api::startup::*;
WorkerCreateFailure::from_cause(
StartupCause::Boundary(StartupBoundaryCause { reason }),
StartupOperationFacts {
acceptance: CreationAcceptance::Unknown,
execution: ExecutionOutcome::Unknown,
finalize: None,
correlation: None,
backend_compensation: None,
},
)
.expect("fixed enum-only observation fits the startup boundary")
}
fn runtime_create_client_failure(
error: &runtime_api::client_support::ClientError<runtime_api::RuntimeApiError>,
) -> runtime_api::startup::WorkerCreateFailure {
use runtime_api::client_support::{ClientError, ClientFailure};
use runtime_api::startup::StartupBoundaryReason;
match error {
ClientError::Public { error, .. } => error
.create_failure()
.cloned()
.unwrap_or_else(|| startup_observation_failure(StartupBoundaryReason::InvalidResponse)),
ClientError::Failure(failure) => startup_observation_failure(match failure {
ClientFailure::Timeout => StartupBoundaryReason::Timeout,
ClientFailure::ResponseTooLarge { .. } => StartupBoundaryReason::TooLarge,
ClientFailure::Decode { .. } | ClientFailure::ErrorResponseDecode { .. } => {
StartupBoundaryReason::Decode
}
ClientFailure::UnsupportedErrorResponse { .. } => {
StartupBoundaryReason::UnsupportedVersion
}
ClientFailure::UnexpectedStatus { .. }
| ClientFailure::MissingResponseHeader { .. }
| ClientFailure::DuplicateResponseHeader { .. }
| ClientFailure::InvalidResponseHeader { .. }
| ClientFailure::UnexpectedBody => StartupBoundaryReason::InvalidResponse,
ClientFailure::Authorization
| ClientFailure::RequestEncoding
| ClientFailure::Connect
| ClientFailure::Transport => StartupBoundaryReason::Transport,
}),
}
}
@@ -695,6 +695,8 @@ CREATE TABLE worker_create_reservations (
updated_at TEXT NOT NULL, request_fingerprint TEXT, memory_language TEXT, updated_at TEXT NOT NULL, request_fingerprint TEXT, memory_language TEXT,
singleton_key TEXT, singleton_key TEXT,
singleton_generation INTEGER CHECK (singleton_generation IS NULL OR singleton_generation > 0), singleton_generation INTEGER CHECK (singleton_generation IS NULL OR singleton_generation > 0),
startup_failure_json TEXT CHECK (startup_failure_json IS NULL OR
(typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768)),
PRIMARY KEY (workspace_id, allocation_key), PRIMARY KEY (workspace_id, allocation_key),
UNIQUE (workspace_id, worker_id), UNIQUE (workspace_id, worker_id),
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
+10
View File
@@ -54,6 +54,7 @@ mod ticket_browse_performance_tests;
mod ticket_item_checker; mod ticket_item_checker;
pub mod workdir_create_operations; pub mod workdir_create_operations;
mod workdir_removal; mod workdir_removal;
mod worker_create_failure_store;
mod worker_projection; mod worker_projection;
pub mod worker_source; pub mod worker_source;
pub mod workspace_catalog; pub mod workspace_catalog;
@@ -124,6 +125,15 @@ pub enum Error {
InvalidRuntimeIdentifier { kind: String, value: String }, InvalidRuntimeIdentifier { kind: String, value: String },
#[error("worker name is reserved for a dedicated Workspace service: {0}")] #[error("worker name is reserved for a dedicated Workspace service: {0}")]
ReservedWorkerName(String), ReservedWorkerName(String),
/// Typed startup result; HTTP and other callers project this without parsing Display.
#[error(
"Worker creation failed; the typed result retains its cause and operation observations"
)]
WorkerCreateFailed {
failure: Box<runtime_api::startup::WorkerCreateFailure>,
#[source]
source: Option<Box<Error>>,
},
#[error("runtime `{runtime_id}` operation failed ({code}): {message}")] #[error("runtime `{runtime_id}` operation failed ({code}): {message}")]
RuntimeOperationFailed { RuntimeOperationFailed {
runtime_id: String, runtime_id: String,
+329 -162
View File
@@ -40,6 +40,7 @@ mod subjektiv_memory_journal;
mod workdir_authorization; mod workdir_authorization;
mod worker_operations; mod worker_operations;
mod worker_protocol; mod worker_protocol;
mod worker_startup;
pub(crate) use worker_operations::{ pub(crate) use worker_operations::{
WorkerOperationContext, WorkspaceWorker, WorkspaceWorkerMethodSender, WorkerOperationContext, WorkspaceWorker, WorkspaceWorkerMethodSender,
}; };
@@ -5996,6 +5997,23 @@ impl WorkspaceApi {
} }
} }
/// Read retained creation evidence for this Workspace without changing lifecycle
/// state. Trusted internal callers must enforce their current authority before
/// redisclosure; a saved record never supplies visibility or retry permission.
pub fn worker_create_failure(
&self,
worker: &RuntimeWorkerRef,
) -> Result<Option<runtime_api::startup::DecodeStoredWorkerCreateFailure>> {
let worker_id = WorkerId::parse(&worker.worker_id)
.ok_or_else(|| Error::InvalidInput("invalid Worker identity".into()))?;
self.config_store
.worker_create_failure(self.workspace_id(), &worker.runtime_id, worker_id)
.map(|encoded| {
encoded
.map(|value| runtime_api::startup::decode_stored_worker_create_failure(&value))
})
}
fn spawn_workspace_worker( fn spawn_workspace_worker(
&self, &self,
runtime_id: &str, runtime_id: &str,
@@ -6012,7 +6030,46 @@ impl WorkspaceApi {
) -> ApiResult<WorkerSpawnResult> { ) -> ApiResult<WorkerSpawnResult> {
let assignment = request.ticket_assignment.clone(); let assignment = request.ticket_assignment.clone();
let mut dispatched = false; let mut dispatched = false;
let mut result = self.spawn_workspace_worker_inner(runtime_id, request, &mut dispatched); let mut reserved_worker = None;
let mut result = self.spawn_workspace_worker_inner(
runtime_id,
request,
&mut dispatched,
&mut reserved_worker,
);
if let Some(worker) = reserved_worker.as_ref() {
result = result
.map(|mut result| {
if let Some(failure) = result.failure.take() {
result.failure = Some(worker_startup::persist(self, worker, failure));
}
result
})
.map_err(|mut error| {
error.error = match error.error {
Error::WorkerCreateFailed { failure, source } => {
Error::WorkerCreateFailed {
failure: Box::new(worker_startup::persist(self, worker, *failure)),
source,
}
}
source if dispatched => {
// Dispatch is not proof of acceptance or execution.
let failure = worker_startup::persist(
self,
worker,
worker_startup::unknown_result(),
);
Error::WorkerCreateFailed {
failure: Box::new(failure),
source: Some(Box::new(source)),
}
}
source => source,
};
error
});
}
// Only a newly inserted durable receipt proves there was no earlier // Only a newly inserted durable receipt proves there was no earlier
// dispatch. A local rejection on replay says nothing about a prior // dispatch. A local rejection on replay says nothing about a prior
// unknown outcome; keep its fence until absence is confirmed. // unknown outcome; keep its fence until absence is confirmed.
@@ -6040,6 +6097,7 @@ impl WorkspaceApi {
runtime_id: &str, runtime_id: &str,
mut request: WorkerSpawnRequest, mut request: WorkerSpawnRequest,
dispatched: &mut bool, dispatched: &mut bool,
reserved_identity: &mut Option<RuntimeWorkerRef>,
) -> ApiResult<WorkerSpawnResult> { ) -> ApiResult<WorkerSpawnResult> {
if let Some(singleton_key) = request.singleton_key.as_deref() { if let Some(singleton_key) = request.singleton_key.as_deref() {
crate::store::validate_worker_singleton_key(singleton_key)?; crate::store::validate_worker_singleton_key(singleton_key)?;
@@ -6192,6 +6250,7 @@ impl WorkspaceApi {
.worker(&reserved_worker) .worker(&reserved_worker)
.unwrap_or_else(|_| worker_summary_from_registry(&record)); .unwrap_or_else(|_| worker_summary_from_registry(&record));
return Ok(WorkerSpawnResult { return Ok(WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Accepted, state: InternalWorkerOperationState::Accepted,
worker: Some(summary), worker: Some(summary),
acceptance_evidence: vec![crate::hosts::WorkerSpawnAcceptanceEvidence { acceptance_evidence: vec![crate::hosts::WorkerSpawnAcceptanceEvidence {
@@ -6230,6 +6289,7 @@ impl WorkspaceApi {
}; };
request.resolved_memory_settings = Some(reservation.memory_settings); request.resolved_memory_settings = Some(reservation.memory_settings);
let reservation_fingerprint = create_binding.create_fingerprint.clone(); let reservation_fingerprint = create_binding.create_fingerprint.clone();
*reserved_identity = Some(reserved_worker.clone());
let lifecycle_assignment = request.ticket_assignment.clone(); let lifecycle_assignment = request.ticket_assignment.clone();
let compensation_context = WorkerSpawnCompensationContext { let compensation_context = WorkerSpawnCompensationContext {
assignment: lifecycle_assignment.as_ref(), assignment: lifecycle_assignment.as_ref(),
@@ -6372,19 +6432,35 @@ impl WorkspaceApi {
return Err(ApiError::with_diagnostics(error.into_error(), diagnostics)); return Err(ApiError::with_diagnostics(error.into_error(), diagnostics));
} }
}; };
if result.state == InternalWorkerOperationState::Accepted && result.worker.is_none() {
// A malformed success is not conclusive evidence about the reserved Worker.
result.state = InternalWorkerOperationState::Unknown;
}
if result.state != InternalWorkerOperationState::Accepted && result.failure.is_none() {
result.failure = Some(if result.state == InternalWorkerOperationState::Unknown {
worker_startup::unknown_result()
} else {
worker_startup::local_rejection()
});
}
if let Some(failure) = result.failure.take() {
result.failure = Some(worker_startup::correlate(failure, &reserved_worker));
}
let Some(worker) = result.worker.as_mut() else { let Some(worker) = result.worker.as_mut() else {
result let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed(
.diagnostics self,
.extend(compensate_failed_workspace_worker_create( runtime_id,
self, worker_id,
runtime_id, &reservation_fingerprint,
worker_id, "runtime_spawn_failure",
&reservation_fingerprint, None,
"runtime_spawn_rejected", &compensation_context,
None, &reserved_attachments,
&compensation_context, );
&reserved_attachments, if let Some(failure) = result.failure.take() {
)); result.failure = Some(worker_startup::compensated(failure, compensation));
}
result.diagnostics.extend(diagnostics);
return Ok(result); return Ok(result);
}; };
// Runtime metadata is display-only. The lease returned by the same DB transaction that // Runtime metadata is display-only. The lease returned by the same DB transaction that
@@ -6437,21 +6513,26 @@ impl WorkspaceApi {
{ {
Ok(replacement) => replacement, Ok(replacement) => replacement,
Err(error) => { Err(error) => {
let diagnostics = compensate_failed_workspace_worker_create( let (diagnostics, compensation) =
self, compensate_failed_workspace_worker_create_observed(
runtime_id, self,
worker_id, runtime_id,
&reservation_fingerprint, worker_id,
"workspace_api_transport", &reservation_fingerprint,
Some(worker), "workspace_api_transport",
&compensation_context, Some(worker),
&reserved_attachments, &compensation_context,
); &reserved_attachments,
return Err(ApiError::with_diagnostics(error.into_error(), diagnostics)); );
return Err(worker_startup::finalize_error(
ApiError::with_diagnostics(error.into_error(), diagnostics),
WorkerSpawnFinalizeStage::WorkspaceApiBinding,
compensation,
));
} }
}; };
if replacement.state != InternalWorkerOperationState::Accepted { if replacement.state != InternalWorkerOperationState::Accepted {
let diagnostics = compensate_failed_workspace_worker_create( let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed(
self, self,
runtime_id, runtime_id,
worker_id, worker_id,
@@ -6461,23 +6542,27 @@ impl WorkspaceApi {
&compensation_context, &compensation_context,
&reserved_attachments, &reserved_attachments,
); );
return Err(ApiError::with_diagnostics( return Err(worker_startup::finalize_error(
Error::RuntimeOperationFailed { ApiError::with_diagnostics(
runtime_id: runtime_id.to_string(), Error::RuntimeOperationFailed {
code: "worker_workspace_api_replace_failed".to_string(), runtime_id: runtime_id.to_string(),
message: replacement code: "worker_workspace_api_replace_failed".to_string(),
.diagnostics message: replacement
.first() .diagnostics
.map(|diagnostic| diagnostic.message.clone()) .first()
.unwrap_or_else(|| { .map(|diagnostic| diagnostic.message.clone())
"Runtime rejected Workspace API replacement after spawn".to_string() .unwrap_or_else(|| {
}), "Runtime rejected Workspace API replacement after spawn".to_string()
}, }),
diagnostics, },
diagnostics,
),
WorkerSpawnFinalizeStage::WorkspaceApiBinding,
compensation,
)); ));
} }
if let Err(error) = parse_runtime_worker_id_for_registry(&worker.worker.worker_id) { if let Err(error) = parse_runtime_worker_id_for_registry(&worker.worker.worker_id) {
let diagnostics = compensate_failed_workspace_worker_create( let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed(
self, self,
runtime_id, runtime_id,
worker_id, worker_id,
@@ -6487,7 +6572,11 @@ impl WorkspaceApi {
&compensation_context, &compensation_context,
&reserved_attachments, &reserved_attachments,
); );
return Err(api_error_with_additional_diagnostics(error, diagnostics)); return Err(worker_startup::finalize_error(
api_error_with_additional_diagnostics(error, diagnostics),
WorkerSpawnFinalizeStage::WorkerIdentity,
compensation,
));
} }
let registry_result = record_worker_summary( let registry_result = record_worker_summary(
self, self,
@@ -6556,7 +6645,7 @@ impl WorkspaceApi {
.config_store .config_store
.complete_worker_create_reservation(&self.config.workspace_id, worker_id) .complete_worker_create_reservation(&self.config.workspace_id, worker_id)
{ {
let diagnostics = compensate_failed_workspace_worker_create( let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed(
self, self,
runtime_id, runtime_id,
worker_id, worker_id,
@@ -6566,9 +6655,10 @@ impl WorkspaceApi {
&compensation_context, &compensation_context,
&reserved_attachments, &reserved_attachments,
); );
return Err(ApiError::with_diagnostics( return Err(worker_startup::finalize_error(
Error::Config(error.to_string()), ApiError::with_diagnostics(Error::Config(error.to_string()), diagnostics),
diagnostics, WorkerSpawnFinalizeStage::CreateReservation,
compensation,
)); ));
} }
if let Err(error) = subjektiv_journal::reconcile_created_worker(self, &worker_ref) { if let Err(error) = subjektiv_journal::reconcile_created_worker(self, &worker_ref) {
@@ -10762,8 +10852,10 @@ impl server_api::ServerApi for ServerApiContractService {
&self, &self,
context: server_api::ServerRequestContext, context: server_api::ServerRequestContext,
request: server_api::CreateWorkspaceWorkerRequest, request: server_api::CreateWorkspaceWorkerRequest,
) -> std::result::Result<server_api::BrowserCreateWorkerResponse, server_api::RepositoryApiError> ) -> std::result::Result<
{ server_api::BrowserCreateWorkerResponse,
server_api::WorkerCreateApiError,
> {
create_workspace_worker_authenticated( create_workspace_worker_authenticated(
self.workspace_api()?.clone(), self.workspace_api()?.clone(),
contract_request_headers(&context)?, contract_request_headers(&context)?,
@@ -10773,7 +10865,7 @@ impl server_api::ServerApi for ServerApiContractService {
) )
.await .await
.map(|Json(response)| response) .map(|Json(response)| response)
.map_err(ApiError::into_repository_api_error) .map_err(ApiError::into_worker_create_api_error)
} }
async fn workspace_orchestrator_status( async fn workspace_orchestrator_status(
@@ -10869,8 +10961,10 @@ impl server_api::ServerApi for ServerApiContractService {
context: server_api::ServerRequestContext, context: server_api::ServerRequestContext,
workspace_id: String, workspace_id: String,
request: server_api::CreateWorkspaceWorkerRequest, request: server_api::CreateWorkspaceWorkerRequest,
) -> std::result::Result<server_api::BrowserCreateWorkerResponse, server_api::RepositoryApiError> ) -> std::result::Result<
{ server_api::BrowserCreateWorkerResponse,
server_api::WorkerCreateApiError,
> {
spawn_known_worker_authenticated( spawn_known_worker_authenticated(
State(self.workspace_api()?.clone()), State(self.workspace_api()?.clone()),
AxumPath(ScopedWorkspacePath { workspace_id }), AxumPath(ScopedWorkspacePath { workspace_id }),
@@ -10880,7 +10974,7 @@ impl server_api::ServerApi for ServerApiContractService {
) )
.await .await
.map(|Json(response)| response) .map(|Json(response)| response)
.map_err(ApiError::into_repository_api_error) .map_err(ApiError::into_worker_create_api_error)
} }
async fn worker_control_input( async fn worker_control_input(
@@ -11048,8 +11142,10 @@ impl server_api::ServerApi for ServerApiContractService {
context: server_api::ServerRequestContext, context: server_api::ServerRequestContext,
workspace_id: String, workspace_id: String,
request: server_api::CreateWorkspaceWorkerRequest, request: server_api::CreateWorkspaceWorkerRequest,
) -> std::result::Result<server_api::BrowserCreateWorkerResponse, server_api::RepositoryApiError> ) -> std::result::Result<
{ server_api::BrowserCreateWorkerResponse,
server_api::WorkerCreateApiError,
> {
validate_workspace_scope(self.workspace_api()?, &workspace_id) validate_workspace_scope(self.workspace_api()?, &workspace_id)
.map_err(|error| ApiError::from(error).into_repository_api_error())?; .map_err(|error| ApiError::from(error).into_repository_api_error())?;
create_workspace_worker_authenticated( create_workspace_worker_authenticated(
@@ -11061,7 +11157,7 @@ impl server_api::ServerApi for ServerApiContractService {
) )
.await .await
.map(|Json(response)| response) .map(|Json(response)| response)
.map_err(ApiError::into_repository_api_error) .map_err(ApiError::into_worker_create_api_error)
} }
async fn workspace_worker_detail( async fn workspace_worker_detail(
@@ -15015,6 +15111,15 @@ async fn log_failed_api_response(request: Request, next: Next) -> Response {
.and_then(|source| source.token_id_hash.clone()); .and_then(|source| source.token_id_hash.clone());
let response = next.run(request).await; let response = next.run(request).await;
let status = response.status(); let status = response.status();
if let Some(error) = response
.extensions()
.get::<Arc<server_api::WorkerCreateApiError>>()
{
if let Some(failure) = error.failure() {
worker_startup::observe_http(failure, status);
return response;
}
}
let is_api = uri.path().starts_with("/api/") let is_api = uri.path().starts_with("/api/")
|| canonical_ticket_operation_workspace_id(uri.path()).is_some(); || canonical_ticket_operation_workspace_id(uri.path()).is_some();
@@ -15024,6 +15129,12 @@ async fn log_failed_api_response(request: Request, next: Next) -> Response {
.extensions() .extensions()
.get::<Arc<server_api::RepositoryApiError>>() .get::<Arc<server_api::RepositoryApiError>>()
.map(|error| ApiErrorLog::from_repository_error(error)) .map(|error| ApiErrorLog::from_repository_error(error))
.or_else(|| {
response
.extensions()
.get::<Arc<server_api::WorkerCreateApiError>>()
.map(|error| ApiErrorLog::from_repository_error(error.legacy()))
})
.or_else(|| { .or_else(|| {
response response
.extensions() .extensions()
@@ -21673,6 +21784,7 @@ fn ticket_notification_delivery_error_category(
InternalWorkerOperationState::Accepted => None, InternalWorkerOperationState::Accepted => None,
InternalWorkerOperationState::Rejected => Some("runtime_rejected"), InternalWorkerOperationState::Rejected => Some("runtime_rejected"),
InternalWorkerOperationState::Unsupported => Some("runtime_unsupported"), InternalWorkerOperationState::Unsupported => Some("runtime_unsupported"),
InternalWorkerOperationState::Unknown => Some("runtime_outcome_unknown"),
}, },
Err(RuntimeRegistryError::InvalidIdentifier { .. }) => Some("invalid_identifier"), Err(RuntimeRegistryError::InvalidIdentifier { .. }) => Some("invalid_identifier"),
Err(RuntimeRegistryError::UnknownRuntime(_)) => Some("unknown_runtime"), Err(RuntimeRegistryError::UnknownRuntime(_)) => Some("unknown_runtime"),
@@ -34894,6 +35006,7 @@ fn record_browser_worker_spawn(
if result.state != InternalWorkerOperationState::Accepted { if result.state != InternalWorkerOperationState::Accepted {
return Err(worker_create_not_accepted_error( return Err(worker_create_not_accepted_error(
requested_runtime_id.clone(), requested_runtime_id.clone(),
result.failure,
result.diagnostics, result.diagnostics,
)); ));
} }
@@ -34912,20 +35025,24 @@ fn browser_worker_response_from_summary(
diagnostics: Vec<RuntimeDiagnostic>, diagnostics: Vec<RuntimeDiagnostic>,
assignment: Option<&WorkerTicketAssignmentRequest>, assignment: Option<&WorkerTicketAssignmentRequest>,
) -> ApiResult<BrowserCreateWorkerResponse> { ) -> ApiResult<BrowserCreateWorkerResponse> {
let worker_record = match record_worker_summary(
api,
&worker,
display_name.as_str(),
worker.profile.clone(),
WorkerRegistryDisplayNamePolicy::UseProvided,
) {
Ok(record) => record,
Err(error) => return Err(error.into()),
};
let compensation = WorkerSpawnCompensationContext { let compensation = WorkerSpawnCompensationContext {
assignment, assignment,
spawned_workdir_ids: &[], spawned_workdir_ids: &[],
}; };
let worker_record = finalize_worker_spawn_stage(
api,
&worker,
&compensation,
WorkerSpawnFinalizeStage::WorkerRegistry,
record_worker_summary(
api,
&worker,
display_name.as_str(),
worker.profile.clone(),
WorkerRegistryDisplayNamePolicy::UseProvided,
)
.map_err(ApiError::from),
)?;
if let Some(assignment) = assignment { if let Some(assignment) = assignment {
if let Err(error) = api.store.bind_ticket_assignment_operation_worker( if let Err(error) = api.store.bind_ticket_assignment_operation_worker(
&api.config.workspace_id, &api.config.workspace_id,
@@ -34979,12 +35096,15 @@ fn browser_worker_response_for_committed_summary(
let workspace_id = api.workspace_id().to_string(); let workspace_id = api.workspace_id().to_string();
let resource_key = api let resource_key = api
.store .store
.worker_resource_key(&workspace_id, &worker.worker)? .worker_resource_key(&workspace_id, &worker.worker)
.ok_or_else(|| { .and_then(|key| {
Error::Store(format!( key.ok_or_else(|| {
"Workspace Worker `{worker_id}` has no resource key after registration" Error::Store(format!(
)) "Workspace Worker `{worker_id}` has no resource key after registration"
})?; ))
})
})
.map_err(|error| worker_startup::publication_error(api, &worker.worker, error.into()))?;
let worker = worker_launch_worker_summary(worker); let worker = worker_launch_worker_summary(worker);
let console_href = browser_worker_console_href(&workspace_id, &resource_key); let console_href = browser_worker_console_href(&workspace_id, &resource_key);
Ok(BrowserCreateWorkerResponse { Ok(BrowserCreateWorkerResponse {
@@ -35594,6 +35714,9 @@ async fn list_runtime_workers(
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum WorkerSpawnFinalizeStage { enum WorkerSpawnFinalizeStage {
WorkerIdentity,
WorkspaceApiBinding,
CreateReservation,
WorkerRegistry, WorkerRegistry,
TicketAssignmentBind, TicketAssignmentBind,
TicketAssignmentCurrent, TicketAssignmentCurrent,
@@ -35602,19 +35725,6 @@ enum WorkerSpawnFinalizeStage {
WorkdirAttachment, WorkdirAttachment,
} }
impl WorkerSpawnFinalizeStage {
fn as_str(self) -> &'static str {
match self {
Self::WorkerRegistry => "worker_registry",
Self::TicketAssignmentBind => "ticket_assignment_bind",
Self::TicketAssignmentCurrent => "ticket_assignment_current",
Self::TicketStateAccept => "ticket_state_accept",
Self::WorkdirRegistry => "workdir_registry",
Self::WorkdirAttachment => "workdir_attachment",
}
}
}
struct WorkerSpawnCompensationContext<'a> { struct WorkerSpawnCompensationContext<'a> {
assignment: Option<&'a crate::hosts::WorkerTicketAssignmentRequest>, assignment: Option<&'a crate::hosts::WorkerTicketAssignmentRequest>,
spawned_workdir_ids: &'a [String], spawned_workdir_ids: &'a [String],
@@ -35630,52 +35740,19 @@ fn finalize_worker_spawn_stage<T>(
let Err(source) = result else { let Err(source) = result else {
return result; return result;
}; };
let source_message = sanitize_backend_error(&source.error.to_string()); let (compensation_diagnostics, compensation) =
let operation_id = context compensate_failed_worker_spawn_observed(api, worker, context);
.assignment let failure = worker_startup::persist(
.map(|assignment| assignment.operation_id.as_str()) api,
.unwrap_or("none"); &worker.worker,
let mut diagnostics = vec![RuntimeDiagnostic { worker_startup::finalize_failure(&source, stage, compensation),
startup: None, );
code: format!("worker_spawn_finalize_{}_failed", stage.as_str()), let mut diagnostics = source.diagnostics;
severity: HostDiagnosticSeverity::Error, diagnostics.extend(compensation_diagnostics);
message: format!(
"Worker spawn finalize failed at stage `{}` for Runtime Worker {}:{} (operation `{operation_id}`): {source_message}",
stage.as_str(),
worker.worker.runtime_id,
worker.worker.worker_id
),
}];
diagnostics.extend(source.diagnostics);
let compensation_errors = compensate_failed_worker_spawn(api, worker, context);
let compensation_failed = !compensation_errors.is_empty();
diagnostics.extend(compensation_errors);
if !compensation_failed {
diagnostics.push(RuntimeDiagnostic {
startup: None,
code: "worker_spawn_compensated".to_string(),
severity: HostDiagnosticSeverity::Info,
message: format!(
"Removed Runtime Worker {}:{} and rolled back Backend spawn state",
worker.worker.runtime_id, worker.worker.worker_id
),
});
}
let compensation = if compensation_failed {
"compensation left residual state; inspect diagnostics"
} else {
"compensation completed"
};
Err(ApiError::with_diagnostics( Err(ApiError::with_diagnostics(
Error::RuntimeOperationFailed { Error::WorkerCreateFailed {
runtime_id: worker.worker.runtime_id.clone(), failure: Box::new(failure),
code: format!("worker_spawn_finalize_{}_failed", stage.as_str()), source: Some(Box::new(source.error)),
message: format!(
"Worker spawn finalize failed at stage `{}` for Runtime Worker {}:{} (operation `{operation_id}`): {source_message}; {compensation}",
stage.as_str(),
worker.worker.runtime_id,
worker.worker.worker_id
),
}, },
diagnostics, diagnostics,
)) ))
@@ -35687,16 +35764,14 @@ fn write_workspace_worker_create_failure(
phase: &str, phase: &str,
diagnostics: &[RuntimeDiagnostic], diagnostics: &[RuntimeDiagnostic],
) { ) {
tracing::error!( tracing::warn!(
target: "yoi::worker_create", target: "yoi::worker_create",
event = "worker_create_failed", event = "worker_create_compensation",
component = "workspace_server", component = "workspace_server",
runtime_id, correlation = %worker_startup::correlation(runtime_id, &worker_id.to_string()),
worker_id = %worker_id,
phase, phase,
cleanup_succeeded = diagnostics.is_empty(), cleanup_completed = diagnostics.is_empty(),
cleanup_diagnostics = ?diagnostics, "Worker creation compensation observed"
"Worker creation failed"
); );
} }
@@ -35771,6 +35846,32 @@ fn compensate_failed_workspace_worker_create(
context: &WorkerSpawnCompensationContext<'_>, context: &WorkerSpawnCompensationContext<'_>,
attachment_reservations: &[(String, String, String, workdir::WorkdirSessionCapabilities)], attachment_reservations: &[(String, String, String, workdir::WorkdirSessionCapabilities)],
) -> Vec<RuntimeDiagnostic> { ) -> Vec<RuntimeDiagnostic> {
compensate_failed_workspace_worker_create_observed(
api,
runtime_id,
reservation_worker_id,
create_fingerprint,
failure_phase,
worker,
context,
attachment_reservations,
)
.0
}
fn compensate_failed_workspace_worker_create_observed(
api: &WorkspaceApi,
runtime_id: &str,
reservation_worker_id: WorkerId,
create_fingerprint: &str,
failure_phase: &str,
worker: Option<&InternalWorkerSummary>,
context: &WorkerSpawnCompensationContext<'_>,
attachment_reservations: &[(String, String, String, workdir::WorkdirSessionCapabilities)],
) -> (
Vec<RuntimeDiagnostic>,
runtime_api::startup::CompensationOutcome,
) {
let mut diagnostics = Vec::new(); let mut diagnostics = Vec::new();
let reserved_worker_ref = let reserved_worker_ref =
RuntimeWorkerRef::new(runtime_id.to_string(), reservation_worker_id.to_string()); RuntimeWorkerRef::new(runtime_id.to_string(), reservation_worker_id.to_string());
@@ -35840,18 +35941,31 @@ fn compensate_failed_workspace_worker_create(
failure_phase, failure_phase,
&diagnostics, &diagnostics,
); );
diagnostics let outcome = if !reserved_worker_absent {
runtime_api::startup::CompensationOutcome::Unknown
} else if diagnostics.is_empty() {
runtime_api::startup::CompensationOutcome::Completed
} else {
runtime_api::startup::CompensationOutcome::Residual
};
(diagnostics, outcome)
} }
fn compensate_failed_worker_spawn( fn compensate_failed_worker_spawn_observed(
api: &WorkspaceApi, api: &WorkspaceApi,
worker: &InternalWorkerSummary, worker: &InternalWorkerSummary,
context: &WorkerSpawnCompensationContext<'_>, context: &WorkerSpawnCompensationContext<'_>,
) -> Vec<RuntimeDiagnostic> { ) -> (
Vec<RuntimeDiagnostic>,
runtime_api::startup::CompensationOutcome,
) {
let (runtime_deleted, mut diagnostics) = let (runtime_deleted, mut diagnostics) =
WorkerRemovalService::new(api).delete_spawn_compensation_runtime_worker(&worker.worker); WorkerRemovalService::new(api).delete_spawn_compensation_runtime_worker(&worker.worker);
if !runtime_deleted { if !runtime_deleted {
return diagnostics; return (
diagnostics,
runtime_api::startup::CompensationOutcome::Unknown,
);
} }
diagnostics.extend(finalize_spawn_compensation_after_worker_delete( diagnostics.extend(finalize_spawn_compensation_after_worker_delete(
api, api,
@@ -35872,7 +35986,12 @@ fn compensate_failed_worker_spawn(
), ),
)); ));
} }
diagnostics let outcome = if diagnostics.is_empty() {
runtime_api::startup::CompensationOutcome::Completed
} else {
runtime_api::startup::CompensationOutcome::Residual
};
(diagnostics, outcome)
} }
fn finalize_spawn_compensation_after_worker_delete( fn finalize_spawn_compensation_after_worker_delete(
@@ -36148,6 +36267,7 @@ async fn create_runtime_worker_authenticated_on_host(
{ {
accept_queued_ticket_after_worker_spawn(&api, assignment)?; accept_queued_ticket_after_worker_spawn(&api, assignment)?;
return Ok(Json(worker_spawn_result_to_api(WorkerSpawnResult { return Ok(Json(worker_spawn_result_to_api(WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Accepted, state: InternalWorkerOperationState::Accepted,
worker: Some(worker), worker: Some(worker),
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -39159,19 +39279,15 @@ fn encode_path_segment(value: &str) -> String {
} }
fn worker_create_not_accepted_error( fn worker_create_not_accepted_error(
runtime_id: String, _runtime_id: String,
mut diagnostics: Vec<RuntimeDiagnostic>, failure: Option<runtime_api::startup::WorkerCreateFailure>,
diagnostics: Vec<RuntimeDiagnostic>,
) -> ApiError { ) -> ApiError {
diagnostics.push(settings_diagnostic( let failure = failure.unwrap_or_else(worker_startup::local_rejection);
"workspace_worker_create_not_accepted",
HostDiagnosticSeverity::Error,
"Runtime did not accept worker creation; see diagnostics for sanitized Runtime compatibility details.",
));
ApiError::with_diagnostics( ApiError::with_diagnostics(
Error::RuntimeOperationFailed { Error::WorkerCreateFailed {
runtime_id, failure: Box::new(failure),
code: "workspace_worker_create_failed".to_string(), source: None,
message: "Runtime did not accept worker creation".to_string(),
}, },
diagnostics, diagnostics,
) )
@@ -39666,6 +39782,18 @@ fn subjektiv_diagnostic_code(message: &str) -> Option<&'static str> {
} }
impl ApiError { impl ApiError {
fn into_worker_create_api_error(self) -> server_api::WorkerCreateApiError {
match self.error {
Error::WorkerCreateFailed { failure, .. } => server_api::WorkerCreateApiError::startup(
*failure,
&protocol::session_failure::SessionFailureVisibility {
can_view_cause: true,
},
),
_ => self.into_repository_api_error().into(),
}
}
fn with_diagnostics(error: Error, diagnostics: Vec<RuntimeDiagnostic>) -> Self { fn with_diagnostics(error: Error, diagnostics: Vec<RuntimeDiagnostic>) -> Self {
Self { Self {
error, error,
@@ -39852,6 +39980,13 @@ fn api_error_response_message(error: &Error) -> String {
impl IntoResponse for ApiError { impl IntoResponse for ApiError {
fn into_response(self) -> Response { fn into_response(self) -> Response {
if matches!(&self.error, Error::WorkerCreateFailed { .. }) {
let error = self.into_worker_create_api_error();
return api_macros::axum::http_error_response(
api_macros::axum::status(api_macros::HttpError::status_code(&error)),
error,
);
}
let status = api_error_status(&self.error); let status = api_error_status(&self.error);
let response_message = api_error_response_message(&self.error); let response_message = api_error_response_message(&self.error);
let log = ApiErrorLog { let log = ApiErrorLog {
@@ -39953,6 +40088,7 @@ mod tests {
mod protocol_backpressure_tests; mod protocol_backpressure_tests;
mod restore_workdir_tests; mod restore_workdir_tests;
mod restored_parent_control_tests; mod restored_parent_control_tests;
mod startup_result_tests;
mod subject_authorization_tests; mod subject_authorization_tests;
mod subject_deletion_tests; mod subject_deletion_tests;
mod notification_source_authorization_tests { mod notification_source_authorization_tests {
@@ -43038,6 +43174,7 @@ mod tests {
.unwrap_or_else(std::sync::PoisonError::into_inner), .unwrap_or_else(std::sync::PoisonError::into_inner),
) { ) {
return WorkerSpawnResult { return WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -43069,6 +43206,7 @@ mod tests {
.unwrap_or_else(std::sync::PoisonError::into_inner) .unwrap_or_else(std::sync::PoisonError::into_inner)
.push(worker.clone()); .push(worker.clone());
WorkerSpawnResult { WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Accepted, state: InternalWorkerOperationState::Accepted,
worker: Some(worker), worker: Some(worker),
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -47164,7 +47302,18 @@ mod tests {
guarded_spawn_payload(&fixture, "guarded-runtime-rejection"), guarded_spawn_payload(&fixture, "guarded-runtime-rejection"),
) )
.await; .await;
assert_eq!(response.status(), StatusCode::BAD_GATEWAY); assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json");
let bytes = to_bytes(
response.into_body(),
runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
)
.await
.unwrap();
let body: Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(body["yoi_failure"]["domain"], "worker_startup");
assert_eq!(body["yoi_failure"]["reason"], "failed");
assert_eq!(body["yoi_startup"]["operation"]["acceptance"], "rejected");
assert_eq!(fixture.runtime.spawn_requests().len(), 1); assert_eq!(fixture.runtime.spawn_requests().len(), 1);
assert_guarded_spawn_rolled_back(&fixture); assert_guarded_spawn_rolled_back(&fixture);
} }
@@ -47191,7 +47340,17 @@ mod tests {
guarded_spawn_payload(&fixture, "guarded-post-create-failure"), guarded_spawn_payload(&fixture, "guarded-post-create-failure"),
) )
.await; .await;
assert_eq!(response.status(), StatusCode::BAD_GATEWAY); assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json");
let bytes = to_bytes(
response.into_body(),
runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
)
.await
.unwrap();
let body: Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(body["yoi_startup"]["operation"]["acceptance"], "accepted");
assert_eq!(body["yoi_failure"]["reason"], "finalize_failed");
assert_eq!(fixture.runtime.spawn_requests().len(), 1); assert_eq!(fixture.runtime.spawn_requests().len(), 1);
assert_guarded_spawn_rolled_back(&fixture); assert_guarded_spawn_rolled_back(&fixture);
} }
@@ -61792,7 +61951,7 @@ mod tests {
sync_worker_observation(&api, &summary).unwrap(); sync_worker_observation(&api, &summary).unwrap();
let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap();
let _ = subscriber.take_snapshot(); let _ = subscriber.take_snapshot();
let diagnostics = compensate_failed_worker_spawn( let (diagnostics, _) = compensate_failed_worker_spawn_observed(
&api, &api,
&summary, &summary,
&WorkerSpawnCompensationContext { &WorkerSpawnCompensationContext {
@@ -68042,24 +68201,32 @@ mod tests {
); );
let response = worker_create_not_accepted_error( let response = worker_create_not_accepted_error(
"unsupported-runtime".to_string(), "unsupported-runtime".into(),
None,
vec![settings_diagnostic( vec![settings_diagnostic(
"remote_runtime_unsupported", "remote_runtime_unsupported",
HostDiagnosticSeverity::Warning, HostDiagnosticSeverity::Warning,
"Remote Runtime provisioning is unsupported by this v0 worker launch path.", "TEST_ONLY_DIAGNOSTIC_TEXT",
)], )],
) )
.into_response(); .into_response();
assert_eq!(response.status(), StatusCode::BAD_GATEWAY); assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json");
let bytes = to_bytes(
response.into_body(),
runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
)
.await
.unwrap();
let response: Value = serde_json::from_slice(&bytes).unwrap(); let response: Value = serde_json::from_slice(&bytes).unwrap();
let diagnostics = response["diagnostics"].as_array().unwrap(); assert_eq!(response["yoi_failure"]["domain"], "worker_startup");
assert!(diagnostics.len() >= 2); assert!(response.get("diagnostics").is_none());
assert!( assert!(
diagnostics !response
.iter() .to_string()
.any(|diagnostic| { diagnostic["code"] == "workspace_worker_create_not_accepted" }) .contains("workspace_worker_create_not_accepted")
); );
assert!(!response.to_string().contains("TEST_ONLY_DIAGNOSTIC_TEXT"));
let projected = serde_json::to_string(&response).unwrap(); let projected = serde_json::to_string(&response).unwrap();
assert!(!projected.contains("http://")); assert!(!projected.contains("http://"));
} }
@@ -1,14 +1,18 @@
//! Focused T840 connection: real Subject policy -> signed Session POST -> real //! Connected startup acceptance: real Subject policy -> signed Session POST ->
//! ProfileRuntimeWorkerFactory startup -> RuntimeHTTP/Host diagnostic -> saved //! ProfileRuntimeWorkerFactory -> Runtime HTTP -> Backend, including the real
//! Worker metadata. Only the unused external model is replaced. No restore, //! authenticated Browser worker-create HTTP route and saved Worker metadata.
//! operation replay, singleton-release, or Browser rendering contract is asserted. //! Only the unused external model is replaced. Read-only reconciliation is not
//! restore, operation replay, or a Browser rendering contract.
use super::*; use super::*;
use api_macros::HttpError as _; use api_macros::HttpError as _;
use protocol::session_failure::{ use protocol::session_failure::{
SessionPrincipal, SessionRegistrationCause, session_registration_attachment, SessionPrincipal, SessionRegistrationCause, session_registration_attachment,
}; };
use session_store::WorkerMetadataStore as _; use session_store::WorkerMetadataStore as _;
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use tracing::instrument::WithSubscriber as _;
use tracing_subscriber::{Layer as _, layer::SubscriberExt as _};
use worker_runtime::worker_backend::RuntimeWorkerFactory as _; use worker_runtime::worker_backend::RuntimeWorkerFactory as _;
#[derive(Clone, Default)] #[derive(Clone, Default)]
@@ -30,6 +34,57 @@ impl agen::llm_client::LlmClient for UnusedModel {
} }
} }
#[derive(Clone, Default)]
struct StartupEvents(Arc<Mutex<Vec<BTreeMap<String, String>>>>);
#[derive(Default)]
struct StartupEventFields(BTreeMap<String, String>);
impl tracing::field::Visit for StartupEventFields {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.0.insert(field.name().into(), value.into());
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.0.insert(field.name().into(), format!("{value:?}"));
}
}
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for StartupEvents {
fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) {
if event.metadata().target() == "yoi::worker_startup" {
let mut fields = StartupEventFields::default();
event.record(&mut fields);
self.0.lock().unwrap().push(fields.0);
}
}
}
/// A real formatter writer failure, not a replacement result/observer function.
/// The request-scoped subscriber avoids process-global test interference.
#[derive(Clone, Default)]
struct FailedStartupSink(Arc<AtomicUsize>);
impl std::io::Write for FailedStartupSink {
fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
self.0.fetch_add(1, Ordering::SeqCst);
Err(std::io::Error::other(
"test: startup observation sink unavailable",
))
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for FailedStartupSink {
type Writer = Self;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
type ReleaseFence = ( type ReleaseFence = (
crate::store::WorkerSingletonLease, crate::store::WorkerSingletonLease,
subjektiv::journal::JournalWriterBinding, subjektiv::journal::JournalWriterBinding,
@@ -41,6 +96,8 @@ struct RegistrationFixture {
callbacks: Arc<Mutex<Vec<(StatusCode, Value)>>>, callbacks: Arc<Mutex<Vec<(StatusCode, Value)>>>,
model: UnusedModel, model: UnusedModel,
release_fence: Arc<Mutex<Option<ReleaseFence>>>, release_fence: Arc<Mutex<Option<ReleaseFence>>>,
startup_events: StartupEvents,
failed_sink: FailedStartupSink,
} }
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
@@ -49,6 +106,7 @@ enum RegistrationFault {
Deny, Deny,
DenyWithMetadataReadFailure, DenyWithMetadataReadFailure,
ReleasePending, ReleasePending,
CommittedResponseLost,
} }
impl RegistrationFixture { impl RegistrationFixture {
@@ -63,13 +121,31 @@ impl RegistrationFixture {
let model = UnusedModel::default(); let model = UnusedModel::default();
let release_fence = Arc::new(Mutex::new(None)); let release_fence = Arc::new(Mutex::new(None));
let reserved = release_fence.clone(); let reserved = release_fence.clone();
let startup_events = StartupEvents::default();
let failed_sink = FailedStartupSink::default();
let events = startup_events.clone();
let sink = failed_sink.clone();
let connected = ConnectedWorkerHarness::start_with_layer(model.clone(), move |router| { let connected = ConnectedWorkerHarness::start_with_layer(model.clone(), move |router| {
router.layer(axum::middleware::from_fn( router.layer(axum::middleware::from_fn(
move |request: axum::extract::Request, next: axum::middleware::Next| { move |request: axum::extract::Request, next: axum::middleware::Next| {
let hook = hook.clone(); let hook = hook.clone();
let observed = observed.clone(); let observed = observed.clone();
let reserved = reserved.clone(); let reserved = reserved.clone();
let events = events.clone();
let sink = sink.clone();
async move { async move {
if request.method() == axum::http::Method::POST
&& request.uri().path() == format!("/api/w/{TEST_WORKSPACE_ID}/workers")
{
let subscriber = tracing_subscriber::registry()
.with(events)
.with(tracing_subscriber::fmt::layer()
.with_writer(sink)
.with_filter(tracing_subscriber::filter::filter_fn(|metadata| {
metadata.target() == "yoi::worker_startup"
})));
return next.run(request).with_subscriber(subscriber).await;
}
if request.method() != axum::http::Method::POST if request.method() != axum::http::Method::POST
|| request.uri().path() || request.uri().path()
!= format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions") != format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions")
@@ -108,7 +184,7 @@ impl RegistrationFixture {
assert!(store.journal_writer_active(&writer).unwrap()); assert!(store.journal_writer_active(&writer).unwrap());
assert_eq!(api.store.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key).unwrap(), Some(lease.clone())); assert_eq!(api.store.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key).unwrap(), Some(lease.clone()));
*reserved.lock().unwrap() = Some((lease, writer)); *reserved.lock().unwrap() = Some((lease, writer));
} else { } else if !matches!(fault, RegistrationFault::CommittedResponseLost) {
store store
.set_subject_permissions( .set_subject_permissions(
&subject, &subject,
@@ -163,6 +239,20 @@ impl RegistrationFixture {
std::fs::create_dir(&metadata).unwrap(); std::fs::create_dir(&metadata).unwrap();
} }
observed.lock().unwrap().push((response.status(), input)); observed.lock().unwrap().push((response.status(), input));
if matches!(fault, RegistrationFault::CommittedResponseLost) {
assert_eq!(response.status(), StatusCode::OK);
// The genuine signed handler committed before its response is
// lost. Do not replace its authorization, attribution, or write.
// A broken body fails the actual Runtime HTTP client rather
// than fabricating a typed Runtime/domain failure in this layer.
let (parts, _) = response.into_parts();
let body = Body::from_stream(futures::stream::once(async {
Err::<Vec<u8>, _>(std::io::Error::other(
"test: committed registration response lost",
))
}));
return Response::from_parts(parts, body);
}
response response
} }
}, },
@@ -200,6 +290,8 @@ impl RegistrationFixture {
callbacks, callbacks,
model, model,
release_fence, release_fence,
startup_events,
failed_sink,
} }
} }
@@ -222,6 +314,178 @@ impl RegistrationFixture {
} }
} }
async fn browser_failure(
&self,
expected_status: StatusCode,
) -> (
runtime_api::startup::StartupProblemDetails,
runtime_api::startup::WorkerCreateFailure,
) {
use runtime_api::startup::{
DecodeStartupProblem, PROBLEM_CONTENT_TYPE, StartupBoundary, decode_startup_problem,
};
let (status, headers, bytes) = self
.connected
.human_raw(
reqwest::Method::POST,
&format!("/api/w/{TEST_WORKSPACE_ID}/workers"),
serde_json::to_value(self.request()).unwrap(),
)
.await;
assert_eq!(status, expected_status);
let content_type = headers[axum::http::header::CONTENT_TYPE].to_str().unwrap();
assert_eq!(content_type, PROBLEM_CONTENT_TYPE);
let DecodeStartupProblem::Known(failure) = decode_startup_problem(
&bytes,
status.as_u16(),
content_type,
StartupBoundary::Browser,
) else {
panic!("real Browser HTTP response must decode as a bounded startup Problem");
};
let problem: runtime_api::startup::StartupProblemDetails =
serde_json::from_slice(&bytes).unwrap();
assert_eq!(problem.status, status.as_u16());
// Neither raw callback diagnostics nor authority-bearing identifiers may
// be copied into a public explanation/extension by the adapters.
let body = std::str::from_utf8(&bytes).unwrap();
for secret in [
&self.connected.owner,
&self.subject,
&self.connected.backend_url,
] {
assert!(
!body.contains(secret),
"Browser Problem disclosed private fixture data"
);
}
assert!(!body.contains("test: committed registration response lost"));
assert!(!body.contains("worker-runtime-"));
// Public, bounded handler bytes, after the non-disclosure checks above.
// These provide the actual shared Browser fixture source under --nocapture;
// no metadata, callback body, credential, or private upstream text is logged.
eprintln!(
"T812_BROWSER_STARTUP_PROBLEM status={status} content_type={content_type} body={body}"
);
(problem, failure)
}
fn assert_backend_failure(
&self,
worker: &str,
session: &str,
problem: &runtime_api::startup::StartupProblemDetails,
browser: &runtime_api::startup::WorkerCreateFailure,
expected: SessionRegistrationCause,
) {
use protocol::startup_failure::StartupPersistenceOutcome::Persisted;
use runtime_api::startup::DecodeStoredWorkerCreateFailure;
let reference = RuntimeWorkerRef::new(CONNECTED_RUNTIME, worker);
let Some(DecodeStoredWorkerCreateFailure::Known(saved)) = self
.connected
.api
.worker_create_failure(&reference)
.unwrap()
else {
panic!("real Browser failure must be readable from the canonical Backend reservation");
};
assert_startup_semantics(saved.startup(), browser.startup(), expected);
assert_eq!(
saved.operation(),
browser.operation(),
"HTTP cannot rewrite acceptance/execution/finalization/compensation/correlation"
);
assert_eq!(saved.backend_result_persistence(), Some(Persisted));
assert_eq!(browser.backend_result_persistence(), Some(Persisted));
assert_eq!(
problem.yoi_startup.backend_result_persistence,
Some(Persisted)
);
assert_eq!(saved.public_facts(), problem.yoi_startup);
let correlation = saved
.operation()
.correlation
.as_ref()
.expect("Backend-selected lookup token");
assert_eq!(
correlation.as_str(),
worker_startup::correlation(CONNECTED_RUNTIME, worker)
);
assert_eq!(
problem.instance,
format!("urn:yoi:worker-startup:{}", correlation.as_str())
);
let events = self.startup_events.0.lock().unwrap();
let result_events = events
.iter()
.filter(|event| {
event
.get("event")
.is_some_and(|value| value == "worker_create_result")
})
.collect::<Vec<_>>();
assert_eq!(
result_events.len(),
1,
"one HTTP responsibility boundary records the primary, not every adapter"
);
let event = result_events[0];
assert_eq!(event["domain"], problem.yoi_failure.domain);
assert_eq!(event["reason"], problem.yoi_failure.reason);
assert_eq!(event["correlation"], correlation.as_str());
assert_eq!(event["status"], problem.status.to_string());
assert_eq!(
event["acceptance"],
format!("{:?}", saved.operation().acceptance)
);
assert_eq!(
event["execution"],
format!("{:?}", saved.operation().execution)
);
let logged = serde_json::to_string(&*events).unwrap();
for secret in [
self.connected.owner.as_str(),
self.subject.as_str(),
self.connected.backend_url.as_str(),
session,
worker,
"test: committed registration response lost",
] {
assert!(
!logged.contains(secret),
"startup observation leaked private fixture data"
);
}
assert!(!logged.contains(self.connected.runtime_root.to_str().unwrap()));
drop(events);
assert!(
self.failed_sink.0.load(Ordering::SeqCst) > 0,
"actual tracing formatter sink failed while HTTP/Backend/metadata results were preserved"
);
assert_eq!(
self.connected
.api
.worker_create_failure(&reference)
.unwrap(),
Some(DecodeStoredWorkerCreateFailure::Known(saved)),
"lookup is read-only, never a create retry"
);
}
fn attempted_worker(&self) -> String {
let workers = std::fs::read_dir(self.connected.runtime_root.join("workers"))
.unwrap()
.map(|entry| entry.unwrap().file_name().into_string().unwrap())
.collect::<Vec<_>>();
assert_eq!(
workers.len(),
1,
"one real factory attempt, never a new create/replay"
);
workers[0].clone()
}
fn assert_callback(&self, expected: StatusCode) -> String { fn assert_callback(&self, expected: StatusCode) -> String {
let callbacks = self.callbacks.lock().unwrap(); let callbacks = self.callbacks.lock().unwrap();
assert!(!callbacks.is_empty(), "real signed startup callback"); assert!(!callbacks.is_empty(), "real signed startup callback");
@@ -250,6 +514,42 @@ fn contains_saved_value(saved: &Value, expected: &Value) -> bool {
} }
} }
// Runtime/Browser normalization reprojects for the receiving boundary, including
// its local explanation. Compare domain enums, not byte identity of envelopes;
// saved metadata below still checks the original Storage attachment independently.
fn session_primary(
primary: &protocol::startup_failure::StartupFailureAttachment,
) -> Option<SessionRegistrationCause> {
match primary.decode::<SessionRegistrationCause>() {
error_contract::DecodeFailure::Known(received) => Some(*received.facts().cause()),
_ => None,
}
}
fn assert_session_primary(
primary: &protocol::startup_failure::StartupFailureAttachment,
expected: SessionRegistrationCause,
) {
assert_eq!(
session_primary(primary),
Some(expected),
"Session failure must remain a known typed domain cause"
);
}
fn assert_startup_semantics(
host: &runtime_api::WorkerStartupFailure,
wire: &runtime_api::WorkerStartupFailure,
expected: SessionRegistrationCause,
) {
assert_session_primary(&host.primary, expected);
assert_session_primary(&wire.primary, expected);
assert_eq!(host.secondary, wire.secondary);
assert_eq!(host.persistence, wire.persistence);
assert_eq!(host.resources, wire.resources);
assert_eq!(host.io_kind, wire.io_kind);
}
fn read_metadata(root: &std::path::Path, worker: &str) -> Value { fn read_metadata(root: &std::path::Path, worker: &str) -> Value {
// Reopen the canonical metadata store, not the separate Runtime catalog. // Reopen the canonical metadata store, not the separate Runtime catalog.
let name = format!("worker-runtime-{worker}"); let name = format!("worker-runtime-{worker}");
@@ -346,6 +646,168 @@ fn assert_registration_cause(body: Value, cause: SessionRegistrationCause) {
); );
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
async fn browser_http_session_record_denial_retains_subject_cause_and_saved_resources() {
use protocol::startup_failure::{
StartupFailurePhase, StartupPersistenceOutcome, StartupSecondaryFailure,
};
use runtime_api::startup::{ExecutionOutcome, StartupCause};
let fixture = RegistrationFixture::new(RegistrationFault::Deny).await;
// This POST must pass the real Browser authentication/preflight and reach
// the signed callback. Only then does the fixture remove session.record.
let (problem, browser) = fixture.browser_failure(StatusCode::CONFLICT).await;
let session = fixture.assert_callback(StatusCode::FORBIDDEN);
assert_eq!(fixture.callbacks.lock().unwrap().len(), 1);
let expected = SessionRegistrationCause::denied(SessionPrincipal::Subject);
assert_eq!(problem.type_uri, "urn:yoi:problem:conflict");
assert_eq!(problem.yoi_failure.domain, "session_registration");
assert_eq!(problem.yoi_failure.reason, "permission_denied");
assert_eq!(problem.yoi_failure.details, StartupCause::Session(expected));
assert!(problem.detail.contains("Subject"), "{}", problem.detail);
assert!(
problem.detail.contains("session.record"),
"{}",
problem.detail
);
assert_eq!(browser.operation().execution, ExecutionOutcome::Errored);
assert_session_primary(&browser.startup().primary, expected);
let runtime = fixture.connected.runtime_startup_failures.lock().unwrap();
assert_eq!(runtime.len(), 1, "one actual Runtime HTTP failure");
assert_session_primary(&runtime[0].primary, expected);
assert_eq!(
browser.startup().persistence,
Some(StartupPersistenceOutcome::Persisted)
);
assert_eq!(browser.startup().persistence, runtime[0].persistence);
assert!(
runtime[0].resources.is_some(),
"resource disposition is independent of the denial"
);
assert_eq!(browser.startup().resources, runtime[0].resources);
assert!(
browser.startup().secondary.iter().any(|secondary| matches!(
secondary,
StartupSecondaryFailure::Persistence {
phase: StartupFailurePhase::RuntimeRecordDelete,
..
}
)),
"real retention pin prevents Runtime record deletion without replacing the denial"
);
drop(runtime);
let store = open_subjektiv_store(&fixture.connected.api).unwrap();
assert!(store.session_attribution(&session).unwrap().is_none());
assert!(
store
.subject_session_attributions(&fixture.subject)
.unwrap()
.is_empty()
);
let worker = fixture.attempted_worker();
fixture.assert_backend_failure(&worker, &session, &problem, &browser, expected);
let saved = read_metadata(&fixture.connected.runtime_root, &worker);
assert!(contains_saved_value(
&saved,
&serde_json::to_value(session_registration_attachment(expected).unwrap()).unwrap()
));
assert_eq!(
fixture.connected.runtime.list_workers().unwrap().len(),
1,
"retained Runtime resource is observed separately from persisted cause and HTTP409"
);
assert_eq!(
read_metadata(&fixture.connected.runtime_root, &worker),
saved,
"ordinary read-only reopen, not create/restore/retry"
);
assert_eq!(fixture.callbacks.lock().unwrap().len(), 1);
assert_eq!(fixture.model.0.load(Ordering::SeqCst), 0);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
async fn browser_http_committed_registration_response_loss_stays_unknown_without_reexecution() {
use protocol::startup_failure::StartupPersistenceOutcome;
use runtime_api::startup::StartupCause;
let fixture = RegistrationFixture::new(RegistrationFault::CommittedResponseLost).await;
let (problem, browser) = fixture.browser_failure(StatusCode::BAD_GATEWAY).await;
// This is the status of the original signed handler, before transport loss.
let session = fixture.assert_callback(StatusCode::OK);
assert_eq!(
fixture.callbacks.lock().unwrap().len(),
1,
"response loss must not automatically reexecute Session registration"
);
let expected = SessionRegistrationCause::outcome_unknown();
assert_eq!(problem.yoi_failure.domain, "session_registration");
assert_eq!(problem.yoi_failure.reason, "outcome_unknown");
assert_eq!(problem.yoi_failure.details, StartupCause::Session(expected));
assert!(
problem.detail.contains("session.record"),
"{}",
problem.detail
);
assert_session_primary(&browser.startup().primary, expected);
assert_eq!(
browser.startup().persistence,
Some(StartupPersistenceOutcome::Persisted)
);
let runtime = fixture.connected.runtime_startup_failures.lock().unwrap();
assert_eq!(runtime.len(), 1);
assert_session_primary(&runtime[0].primary, expected);
assert_eq!(browser.startup().resources, runtime[0].resources);
assert_eq!(browser.startup().persistence, runtime[0].persistence);
drop(runtime);
let store = open_subjektiv_store(&fixture.connected.api).unwrap();
let committed = store.session_attribution(&session).unwrap().unwrap();
assert_eq!(committed.subject_id, fixture.subject);
assert_eq!(committed.runtime_id, CONNECTED_RUNTIME);
let worker = fixture.attempted_worker();
fixture.assert_backend_failure(&worker, &session, &problem, &browser, expected);
assert_eq!(committed.worker_id, worker);
assert_eq!(
store
.subject_session_attributions(&fixture.subject)
.unwrap()
.len(),
1,
"HTTP failure is not proof of no Backend write"
);
let saved = read_metadata(&fixture.connected.runtime_root, &worker);
assert_eq!(
saved["subjektiv_session_attribution"]["state"], "outcome_unknown",
"saved Runtime observation must not claim confirmed registration or rejection"
);
assert_eq!(
saved["subjektiv_session_attribution"]["session_id"],
session
);
assert!(contains_saved_value(
&saved,
&serde_json::to_value(session_registration_attachment(expected).unwrap()).unwrap()
));
assert_eq!(
read_metadata(&fixture.connected.runtime_root, &worker),
saved
);
assert_eq!(
store.session_attribution(&session).unwrap(),
Some(committed)
);
assert_eq!(fixture.connected.runtime.list_workers().unwrap().len(), 1);
assert_eq!(fixture.attempted_worker(), worker);
assert_eq!(
fixture.callbacks.lock().unwrap().len(),
1,
"Problem decoding, attribution lookup and metadata reopen are read-only, never replay authority"
);
assert_eq!(fixture.model.0.load(Ordering::SeqCst), 0);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 6)] #[tokio::test(flavor = "multi_thread", worker_threads = 6)]
async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen() { async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen() {
let fixture = RegistrationFixture::new(RegistrationFault::Deny).await; let fixture = RegistrationFixture::new(RegistrationFault::Deny).await;
@@ -390,7 +852,10 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen()
SessionPrincipal::Subject, SessionPrincipal::Subject,
)) ))
.unwrap(); .unwrap();
assert_eq!(wire.primary, expected_primary); assert_session_primary(
&wire.primary,
SessionRegistrationCause::denied(SessionPrincipal::Subject),
);
assert_eq!( assert_eq!(
wire.persistence, wire.persistence,
Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted) Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted)
@@ -399,9 +864,16 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen()
.diagnostics .diagnostics
.iter() .iter()
.filter_map(|d| d.startup.as_ref()) .filter_map(|d| d.startup.as_ref())
.find(|startup| startup.primary == wire.primary) .find(|startup| {
session_primary(&startup.primary)
== Some(SessionRegistrationCause::denied(SessionPrincipal::Subject))
})
.expect("RemoteWorkerRuntime must preserve the typed startup primary in Host diagnostics"); .expect("RemoteWorkerRuntime must preserve the typed startup primary in Host diagnostics");
assert_eq!(host.primary, wire.primary); assert_startup_semantics(
host,
wire,
SessionRegistrationCause::denied(SessionPrincipal::Subject),
);
assert!( assert!(
host.secondary.iter().any(|failure| matches!( host.secondary.iter().any(|failure| matches!(
failure, failure,
@@ -412,7 +884,7 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen()
)), )),
"a real Session retention pin rejects outer Runtime deletion without hiding the primary" "a real Session retention pin rejects outer Runtime deletion without hiding the primary"
); );
let expected = serde_json::to_value(&wire.primary).unwrap(); let expected = serde_json::to_value(&expected_primary).unwrap();
assert!( assert!(
expected.is_string(), expected.is_string(),
"the shared attachment contains a codec envelope" "the shared attachment contains a codec envelope"
@@ -467,12 +939,9 @@ async fn real_session_record_denial_and_metadata_save_failure_remain_separate()
"one real startup attempt, not an operation replay" "one real startup attempt, not an operation replay"
); );
let wire = &failures[0]; let wire = &failures[0];
assert_eq!( assert_session_primary(
wire.primary, &wire.primary,
session_registration_attachment(SessionRegistrationCause::denied( SessionRegistrationCause::denied(SessionPrincipal::Subject),
SessionPrincipal::Subject
),)
.unwrap()
); );
assert_eq!(wire.persistence, Some(StartupPersistenceOutcome::NotSaved)); assert_eq!(wire.persistence, Some(StartupPersistenceOutcome::NotSaved));
assert!(wire.secondary.iter().any(|failure| matches!( assert!(wire.secondary.iter().any(|failure| matches!(
@@ -486,11 +955,15 @@ async fn real_session_record_denial_and_metadata_save_failure_remain_separate()
.diagnostics .diagnostics
.iter() .iter()
.filter_map(|d| d.startup.as_ref()) .filter_map(|d| d.startup.as_ref())
.find(|startup| startup.primary == wire.primary) .find(|startup| {
session_primary(&startup.primary)
== Some(SessionRegistrationCause::denied(SessionPrincipal::Subject))
})
.unwrap(); .unwrap();
assert_eq!( assert_startup_semantics(
host, wire, host,
"Remote Host preserves the refusal and independent save failure" wire,
SessionRegistrationCause::denied(SessionPrincipal::Subject),
); );
assert!( assert!(
open_subjektiv_store(&fixture.connected.api) open_subjektiv_store(&fixture.connected.api)
@@ -623,7 +1096,7 @@ async fn release_fence_refusal_survives_signed_registration_and_metadata_reopen(
let wire = &failures[0]; let wire = &failures[0];
let expected = let expected =
session_registration_attachment(SessionRegistrationCause::binding_mismatch()).unwrap(); session_registration_attachment(SessionRegistrationCause::binding_mismatch()).unwrap();
assert_eq!(wire.primary, expected); assert_session_primary(&wire.primary, SessionRegistrationCause::binding_mismatch());
assert_eq!( assert_eq!(
wire.persistence, wire.persistence,
Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted) Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted)
@@ -632,9 +1105,11 @@ async fn release_fence_refusal_survives_signed_registration_and_metadata_reopen(
.diagnostics .diagnostics
.iter() .iter()
.filter_map(|d| d.startup.as_ref()) .filter_map(|d| d.startup.as_ref())
.find(|failure| failure.primary == expected) .find(|failure| {
session_primary(&failure.primary) == Some(SessionRegistrationCause::binding_mismatch())
})
.expect("Remote Host retains known refusal"); .expect("Remote Host retains known refusal");
assert_eq!(host.primary, wire.primary); assert_startup_semantics(host, wire, SessionRegistrationCause::binding_mismatch());
assert!( assert!(
host.secondary.iter().any(|failure| matches!( host.secondary.iter().any(|failure| matches!(
failure, failure,
@@ -0,0 +1,803 @@
//! Backend result fixtures use real SQLite reservations, coordinator helpers, typed
//! reads, public ApiError responses and HTTP middleware observation. Publication
//! faults enter the production committed-summary response helper with a fixture
//! Runtime Worker; only the earlier Runtime startup is substituted. These cases do
//! not claim creation-route authorization or Session callback coverage (see
//! session_registration_connected_tests). Runs in the ordinary server lib suite;
//! each case owns its tempdir/DB and subscriber, with no external services.
use super::*;
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
use protocol::startup_failure::{StartupFailureAttachment, StartupPersistenceOutcome};
use runtime_api::startup::{
CompensationOutcome, CreationAcceptance, DecodeStartupProblem, DecodeStoredWorkerCreateFailure,
ExecutionOutcome, FinalizeFacts, FinalizeStage, StartupBoundary, StartupCause,
StartupOperationFacts, WorkerCreateFailure, decode_startup_problem,
};
use std::io::Write;
use std::sync::atomic::{AtomicUsize, Ordering};
use tracing::instrument::WithSubscriber;
const SECRETS: &[&str] = &[
"attachment-secret",
"context-secret",
"diagnostic-secret",
"source-secret",
"sqlite-secret",
"/private/startup-provider",
];
fn failure_fixture(compensation: CompensationOutcome) -> WorkerCreateFailure {
let clean = WorkerCreateFailure::from_cause(
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)),
StartupOperationFacts {
acceptance: CreationAcceptance::Accepted,
execution: ExecutionOutcome::Errored,
finalize: Some(FinalizeFacts {
stage: FinalizeStage::SessionRegistration,
compensation,
}),
backend_compensation: Some(compensation),
correlation: None,
},
)
.unwrap();
let (mut startup, operation) = clean.into_parts();
// Real untrusted attachment bytes, not a source string which the codec never
// sees. Projection must reconstruct the cause and discard received prose/context.
let encoded: String =
serde_json::from_value(serde_json::to_value(&startup.primary).unwrap()).unwrap();
let mut envelope: Value = serde_json::from_str(&encoded).unwrap();
envelope["explanation"] = json!("attachment-secret /private/startup-provider");
envelope["context"]["operations"] = json!([{
"operation": "context-secret", "stage": "context-secret",
"operation_ref": "context-secret"
}]);
envelope["context"]["diagnostics"] = json!([{
"code": "diagnostic-secret", "explanation": "diagnostic-secret"
}]);
startup.primary = serde_json::from_value::<StartupFailureAttachment>(json!(
serde_json::to_string(&envelope).unwrap()
))
.unwrap();
// Session metadata and Backend result persistence have independent owners.
startup.persistence = Some(StartupPersistenceOutcome::NotSaved);
let failure = WorkerCreateFailure::new(startup, operation).unwrap();
assert_eq!(
primary_cause(&failure),
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject))
);
failure
}
fn primary_cause(failure: &WorkerCreateFailure) -> StartupCause {
match failure.startup().primary.decode::<StartupCause>() {
error_contract::DecodeFailure::Known(received) => received.facts().cause().clone(),
_ => panic!("fixture/result must retain a known typed primary"),
}
}
fn assert_semantics(actual: &WorkerCreateFailure, expected: &WorkerCreateFailure) {
assert_eq!(primary_cause(actual), primary_cause(expected));
assert_eq!(actual.public_facts(), expected.public_facts());
}
fn reserve(api: &WorkspaceApi) -> RuntimeWorkerRef {
reserve_for_runtime(api, EMBEDDED_WORKER_RUNTIME_ID)
}
fn reserve_for_runtime(api: &WorkspaceApi, runtime_id: &str) -> RuntimeWorkerRef {
let settings = api
.config_store
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
.unwrap();
let reservation = api
.config_store
.reserve_worker_create(
TEST_WORKSPACE_ID,
runtime_id,
"manual:startup-result-fixture",
"sha256:startup-result-fixture",
None,
&settings,
)
.unwrap();
RuntimeWorkerRef {
runtime_id: runtime_id.into(),
worker_id: reservation.worker_id.to_string(),
}
}
fn retained(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> WorkerCreateFailure {
match api.worker_create_failure(worker).unwrap() {
Some(DecodeStoredWorkerCreateFailure::Known(failure)) => failure,
other => panic!("typed public read must return retained result: {other:?}"),
}
}
// Snapshot all reservation columns except the observation, plus allocation/lease
// rows. Saving a diagnostic must not complete/remove/reallocate the reservation.
fn lifecycle_snapshot(api: &WorkspaceApi) -> Vec<Vec<rusqlite::types::Value>> {
api.config_store.with_conn(|conn| {
let mut columns = conn.prepare("PRAGMA table_info(worker_create_reservations)")?;
let names = columns.query_map([], |row| row.get::<_, String>(1))?
.collect::<rusqlite::Result<Vec<_>>>()?;
let columns = names.into_iter().filter(|name| name != "startup_failure_json")
.map(|name| format!("\"{name}\"" )).collect::<Vec<_>>().join(",");
let queries = [
format!("SELECT {columns} FROM worker_create_reservations ORDER BY workspace_id,allocation_key"),
"SELECT * FROM workspace_resource_keys ORDER BY workspace_id,resource_kind,resource_id".into(),
"SELECT * FROM worker_singleton_owners ORDER BY workspace_id,singleton_key".into(),
"SELECT * FROM worker_registry ORDER BY workspace_id,runtime_id,worker_id".into(),
"SELECT * FROM ticket_worker_assignments ORDER BY workspace_id,ticket_id,assignment_id".into(),
"SELECT * FROM ticket_assignment_operations ORDER BY workspace_id,operation_id".into(),
];
let mut snapshot = Vec::new();
for query in queries {
let mut statement = conn.prepare(&query)?;
let count = statement.column_count();
snapshot.extend(statement.query_map([], |row| {
(0..count).map(|i| row.get(i)).collect::<rusqlite::Result<Vec<_>>>()
})?.collect::<rusqlite::Result<Vec<_>>>()?);
}
Ok(snapshot)
}).unwrap()
}
fn api_error(failure: WorkerCreateFailure) -> ApiError {
ApiError::with_diagnostics(
Error::WorkerCreateFailed {
failure: Box::new(failure),
source: Some(Box::new(Error::InvalidInput(
"source-secret /private/startup-provider".into(),
))),
},
Vec::new(),
)
}
fn result_router(failure: WorkerCreateFailure) -> Router {
Router::new()
.route(
"/api/startup-result-fixture",
axum::routing::get(move || {
let failure = failure.clone();
async move { api_error(failure).into_response() }
}),
)
.layer(middleware::from_fn(log_failed_api_response))
}
async fn public_result(app: Router) -> (StatusCode, Value, WorkerCreateFailure) {
let response = app
.oneshot(
Request::builder()
.uri("/api/startup-result-fixture")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
assert_eq!(
response.headers()[CONTENT_TYPE],
runtime_api::startup::PROBLEM_CONTENT_TYPE
);
let body = to_bytes(
response.into_body(),
runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
)
.await
.unwrap();
let decoded = decode_startup_problem(
&body,
status.as_u16(),
runtime_api::startup::PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser,
);
let DecodeStartupProblem::Known(failure) = decoded else {
panic!("actual HTTP response must satisfy shared bounded contract: {decoded:?}");
};
let value = serde_json::from_slice(&body).unwrap();
assert_no_secrets(std::str::from_utf8(&body).unwrap());
(status, value, failure)
}
#[derive(Clone, Default)]
struct Capture(Arc<std::sync::Mutex<Vec<u8>>>);
impl Write for Capture {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl Capture {
fn events(&self) -> Vec<Value> {
let bytes = self.0.lock().unwrap().clone();
let text = std::str::from_utf8(&bytes).unwrap();
assert_no_secrets(text);
text.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect()
}
}
fn subscriber(capture: Capture) -> impl tracing::Subscriber + Send + Sync {
tracing_subscriber::fmt()
.without_time()
.with_ansi(false)
.json()
.flatten_event(true)
.with_writer(move || capture.clone())
.finish()
}
fn assert_no_secrets(text: &str) {
for secret in SECRETS {
assert!(
!text.contains(secret),
"private value {secret} leaked: {text}"
);
}
}
fn assert_log(event: &Value, failure: &WorkerCreateFailure, status: StatusCode) {
assert_eq!(event["target"], "yoi::worker_startup");
assert_eq!(event["event"], "worker_create_result");
assert_eq!(event["domain"], "session_registration");
assert_eq!(event["reason"], "permission_denied");
assert_eq!(
event["correlation"],
failure.operation().correlation.as_ref().unwrap().as_str()
);
assert_eq!(event["status"], status.as_u16());
assert_eq!(
event["acceptance"],
format!("{:?}", failure.operation().acceptance)
);
assert_eq!(
event["execution"],
format!("{:?}", failure.operation().execution)
);
assert_eq!(
event["finalize"],
format!("{:?}", failure.operation().finalize)
);
assert_eq!(
event["backend_compensation"],
format!("{:?}", failure.operation().backend_compensation)
);
assert_eq!(
event["backend_result_persistence"],
format!("{:?}", failure.backend_result_persistence())
);
assert_eq!(
event["persistence"],
format!("{:?}", failure.startup().persistence)
);
}
#[tokio::test]
async fn persisted_result_public_problem_and_single_http_log_have_matching_semantics() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let worker = reserve(&api);
let before = lifecycle_snapshot(&api);
let original = failure_fixture(CompensationOutcome::Residual);
let returned = worker_startup::persist(&api, &worker, original.clone());
assert_eq!(
returned.backend_result_persistence(),
Some(StartupPersistenceOutcome::Persisted)
);
assert_eq!(
returned.startup().persistence,
Some(StartupPersistenceOutcome::NotSaved)
);
assert_eq!(primary_cause(&returned), primary_cause(&original));
assert_eq!(lifecycle_snapshot(&api), before);
let saved = retained(&api, &worker);
assert_semantics(&saved, &returned);
// Storage projection changes envelope bytes; equality must be semantic.
assert_ne!(saved.startup().primary, original.startup().primary);
let capture = Capture::default();
let (status, problem, public) = public_result(result_router(returned.clone()))
.with_subscriber(subscriber(capture.clone()))
.await;
assert_eq!(status, StatusCode::CONFLICT);
assert!(
problem["detail"]
.as_str()
.unwrap()
.contains("session.record")
);
assert!(problem["detail"].as_str().unwrap().contains("Subject"));
assert!(
problem["instance"]
.as_str()
.unwrap()
.contains(returned.operation().correlation.as_ref().unwrap().as_str())
);
assert_semantics(&public, &saved);
let events = capture.events();
assert_eq!(
events.len(),
1,
"only HTTP responsibility boundary records primary: {events:?}"
);
assert_log(&events[0], &public, status);
assert_eq!(lifecycle_snapshot(&api), before);
}
#[tokio::test]
async fn sqlite_result_write_failure_preserves_primary_facts_prior_bytes_and_lifecycle() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let worker = reserve(&api);
let prior = worker_startup::persist(
&api,
&worker,
failure_fixture(CompensationOutcome::Completed),
);
let prior_bytes = api
.config_store
.worker_create_failure(
TEST_WORKSPACE_ID,
&worker.runtime_id,
WorkerId::parse(&worker.worker_id).unwrap(),
)
.unwrap()
.unwrap();
let before = lifecycle_snapshot(&api);
api.config_store
.with_conn(|conn| {
conn.execute_batch(
"CREATE TRIGGER fail_startup_result BEFORE UPDATE OF startup_failure_json
ON worker_create_reservations BEGIN SELECT RAISE(FAIL, 'sqlite-secret'); END;",
)?;
Ok(())
})
.unwrap();
let attempted = failure_fixture(CompensationOutcome::Unknown);
let returned = worker_startup::persist(&api, &worker, attempted.clone());
assert_eq!(
returned.backend_result_persistence(),
Some(StartupPersistenceOutcome::OutcomeUnknown)
);
assert_eq!(primary_cause(&returned), primary_cause(&attempted));
let mut expected_operation = attempted.operation().clone();
expected_operation.correlation = returned.operation().correlation.clone();
assert_eq!(returned.operation(), &expected_operation);
assert_eq!(returned.startup(), attempted.startup());
assert_eq!(lifecycle_snapshot(&api), before);
assert_eq!(
api.config_store
.worker_create_failure(
TEST_WORKSPACE_ID,
&worker.runtime_id,
WorkerId::parse(&worker.worker_id).unwrap(),
)
.unwrap()
.unwrap(),
prior_bytes
);
assert_semantics(&retained(&api, &worker), &prior);
let capture = Capture::default();
let (status, _, public) = public_result(result_router(returned.clone()))
.with_subscriber(subscriber(capture.clone()))
.await;
assert_eq!(status, StatusCode::CONFLICT);
assert_semantics(&public, &returned);
let events = capture.events();
assert_eq!(events.len(), 1);
assert_log(&events[0], &returned, status);
}
#[tokio::test]
async fn finalize_compensation_does_not_replace_received_primary_or_acceptance() {
for compensation in [
CompensationOutcome::Completed,
CompensationOutcome::Residual,
CompensationOutcome::Unknown,
] {
let original = failure_fixture(CompensationOutcome::Residual);
let error = worker_startup::finalize_error(
api_error(original.clone()),
WorkerSpawnFinalizeStage::WorkdirAttachment,
compensation,
);
let Error::WorkerCreateFailed { failure, .. } = &error.error else {
panic!("finalize must keep a typed startup result");
};
assert_eq!(primary_cause(failure), primary_cause(&original));
assert_eq!(failure.startup(), original.startup());
assert_eq!(
failure.operation().acceptance,
original.operation().acceptance
);
assert_eq!(
failure.operation().execution,
original.operation().execution
);
assert_eq!(
failure.operation().finalize,
Some(FinalizeFacts {
stage: FinalizeStage::WorkdirAttachment,
compensation
})
);
assert_eq!(failure.operation().backend_compensation, Some(compensation));
let expected = failure.as_ref().clone();
let (_, _, public) = public_result(result_router(expected.clone())).await;
assert_semantics(&public, &expected);
}
}
#[tokio::test]
async fn context_and_public_projection_are_silent_until_explicit_http_observation() {
let capture = Capture::default();
let expected = async {
let worker = RuntimeWorkerRef {
runtime_id: "context-secret".into(),
worker_id: "source-secret".into(),
};
let failure =
worker_startup::correlate(failure_fixture(CompensationOutcome::Unknown), &worker);
let public = api_error(failure.clone()).into_response();
assert_eq!(public.status(), StatusCode::CONFLICT);
assert!(
capture.events().is_empty(),
"context/encoder must not log the primary"
);
worker_startup::observe_http(&failure, public.status());
failure
}
.with_subscriber(subscriber(capture.clone()))
.await;
let events = capture.events();
assert_eq!(events.len(), 1);
assert_log(&events[0], &expected, StatusCode::CONFLICT);
}
async fn assert_publication_fault_retains_committed_worker(query_failure: bool) {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
// Freeze unrelated observation writes before deliberately damaging the DB.
api.worker_projection.shutdown_async().await;
api.store.db_executor().drain().await.unwrap();
register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await;
let stops = Arc::new(AtomicUsize::new(0));
let stop_count = stops.clone();
let runtime = WorkdirlessFixtureRuntime {
stop_callback: Some(Arc::new(move || {
stop_count.fetch_add(1, Ordering::SeqCst);
})),
..Default::default()
};
api.runtime.register_or_replace(runtime.clone());
let worker = reserve_for_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID);
let spawned = crate::hosts::WorkspaceWorkerRuntime::spawn_worker(
&runtime,
WorkerCreateBinding {
plugin_execution: None,
worker_id: WorkerId::parse(&worker.worker_id).unwrap(),
create_fingerprint: "sha256:startup-result-fixture".into(),
},
WorkerSpawnRequest {
intent: WorkerSpawnIntent::WorkspaceCompanion,
requested_worker_name: Some("Publication fixture".into()),
singleton_key: None,
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
expected_segments: 0,
},
profile: ProfileSelector::Builtin("builtin:companion".into()),
ticket_assignment: None,
initial_submit: Vec::new(),
workdir_attachment_requests: Vec::new(),
resolved_workdir_attachment_requests: Vec::new(),
resolved_workdir_attachments: Vec::new(),
resolved_config_bundle: None,
resolved_worker_observation_enabled: false,
resolved_worker_observation_grants: Vec::new(),
resolved_workspace_api: None,
resolved_memory_settings: None,
resolved_subjektiv_attached: false,
resolved_control_operation: None,
resolved_execution_grant: None,
},
);
assert_eq!(spawned.state, InternalWorkerOperationState::Accepted);
let summary = spawned.worker.unwrap();
assert_eq!(summary.worker, worker);
let record = record_worker_summary(
&api,
&summary,
&summary.display_name,
summary.profile.clone(),
WorkerRegistryDisplayNamePolicy::UseProvided,
)
.unwrap();
api.config_store
.complete_worker_create_reservation(
TEST_WORKSPACE_ID,
WorkerId::parse(&worker.worker_id).unwrap(),
)
.unwrap();
let ticket = browser_ticket_backend(&api)
.unwrap()
.create(ticket::NewTicket::new("Publication retains assignment"))
.unwrap();
let assignment = TicketWorkerAssignmentRecord {
workspace_id: TEST_WORKSPACE_ID.into(),
ticket_id: ticket.id.clone(),
assignment_id: "publication-assignment".into(),
worker: worker.clone(),
assigned_by: "test-user".into(),
assigned_at: TEST_CREATED_AT.into(),
};
api.store
.set_current_ticket_worker_assignment(
&assignment,
None,
"publication-assignment-event",
"publication-assignment-operation",
false,
)
.unwrap();
// Establish a working publication read first; the fault changes only that
// final read, not reservation persistence or an earlier caller-identity check.
let key = api
.store
.worker_resource_key(TEST_WORKSPACE_ID, &worker)
.unwrap()
.unwrap();
api.config_store
.with_conn(|conn| {
if query_failure {
conn.execute_batch(
"ALTER TABLE workspace_resource_keys RENAME COLUMN resource_key TO \"sqlite-secret\";",
)?;
} else {
let deleted = conn.execute(
"DELETE FROM workspace_resource_keys WHERE workspace_id=?1 AND resource_kind='worker' AND resource_key=?2",
rusqlite::params![TEST_WORKSPACE_ID, key],
)?;
assert_eq!(deleted, 1);
}
Ok(())
})
.unwrap();
let before = lifecycle_snapshot(&api);
let capture = Capture::default();
let route_api = api.clone();
let app = Router::new()
.route(
"/api/startup-result-fixture",
axum::routing::get(move || {
let api = route_api.clone();
let summary = summary.clone();
async move {
let error = browser_worker_response_for_committed_summary(
&api,
summary,
"Publication fixture".into(),
Vec::new(),
)
.unwrap_err();
let Error::WorkerCreateFailed { source, .. } = &error.error else {
panic!("final publication read must return a typed result: {error:?}");
};
let source = source
.as_deref()
.expect("keep the actual read error internal");
if query_failure {
assert!(matches!(source, Error::Sqlite(_)), "{source:?}");
} else {
assert!(matches!(source, Error::Store(_)), "{source:?}");
assert!(source.to_string().contains("no resource key"));
}
error.into_response()
}
}),
)
.layer(middleware::from_fn(log_failed_api_response));
let (status, problem, public) = public_result(app)
.with_subscriber(subscriber(capture.clone()))
.await;
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
primary_cause(&public),
StartupCause::Boundary(runtime_api::startup::StartupBoundaryCause {
reason: runtime_api::startup::StartupBoundaryReason::FinalizeFailed,
})
);
assert_eq!(public.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(public.operation().execution, ExecutionOutcome::Accepted);
assert_eq!(
public.operation().finalize,
Some(FinalizeFacts {
stage: FinalizeStage::Publication,
compensation: CompensationOutcome::Residual,
})
);
assert_eq!(public.operation().backend_compensation, None);
assert_eq!(
public.startup().resources,
Some(protocol::startup_failure::StartupResourceFacts {
resources: protocol::startup_failure::StartupResources::WorkerStartup,
cleanup: protocol::startup_failure::StartupCleanup::NotNeeded,
durable_effects_possible: true,
})
);
assert_eq!(
public.backend_result_persistence(),
Some(StartupPersistenceOutcome::Persisted)
);
assert_eq!(
public.operation().correlation.as_ref().unwrap().as_str(),
worker_startup::correlation(&worker.runtime_id, &worker.worker_id)
);
assert!(
problem["instance"]
.as_str()
.unwrap()
.ends_with(public.operation().correlation.as_ref().unwrap().as_str())
);
assert_semantics(&retained(&api, &worker), &public);
let events = capture.events();
assert_eq!(events.len(), 1, "one primary HTTP event: {events:?}");
let event = &events[0];
assert_eq!(event["target"], "yoi::worker_startup");
assert_eq!(event["event"], "worker_create_result");
assert_eq!(event["domain"], "startup_boundary");
assert_eq!(event["reason"], "finalize_failed");
assert_eq!(event["acceptance"], "Accepted");
assert_eq!(event["execution"], "Accepted");
assert_eq!(
event["finalize"],
format!("{:?}", public.operation().finalize)
);
assert_eq!(event["backend_compensation"], "None");
assert_eq!(event["backend_result_persistence"], "Some(Persisted)");
assert_eq!(
event["resources"],
format!("{:?}", public.startup().resources)
);
assert_eq!(event["status"], status.as_u16());
assert_eq!(
event["correlation"],
problem["yoi_startup"]["operation"]["correlation"]
);
let exposed = format!("{problem} {events:?}");
for internal in [
worker.worker_id.as_str(),
"no such column",
"no resource key",
"workspace_resource_keys",
dir.path().to_str().unwrap(),
] {
assert!(
!exposed.contains(internal),
"internal source leaked: {exposed}"
);
}
assert_eq!(lifecycle_snapshot(&api), before);
assert_eq!(
api.store
.get_worker_registry(TEST_WORKSPACE_ID, &worker)
.unwrap(),
Some(record)
);
assert_eq!(
api.store
.get_current_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id)
.unwrap(),
Some(assignment)
);
assert_eq!(stops.load(Ordering::SeqCst), 0);
assert_eq!(runtime.spawn_requests().len(), 1, "no reexecution");
assert_eq!(
runtime.workers.lock().unwrap().len(),
1,
"no Runtime deletion"
);
}
#[tokio::test]
async fn publication_sqlite_read_failure_retains_committed_worker_and_typed_result() {
assert_publication_fault_retains_committed_worker(true).await;
}
#[tokio::test]
async fn publication_missing_resource_key_retains_committed_worker_and_typed_result() {
assert_publication_fault_retains_committed_worker(false).await;
}
#[tokio::test]
async fn publication_error_preserves_existing_primary_without_claiming_compensation() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let worker = reserve(&api);
let original = failure_fixture(CompensationOutcome::Completed);
let error = worker_startup::publication_error(&api, &worker, api_error(original.clone()));
let Error::WorkerCreateFailed { failure, source } = &error.error else {
panic!("publication must retain a typed startup result");
};
assert!(matches!(
source.as_deref(),
Some(Error::WorkerCreateFailed { .. })
));
assert_eq!(failure.startup().primary, original.startup().primary);
assert_eq!(failure.startup().secondary, original.startup().secondary);
assert_eq!(
failure.startup().persistence,
original.startup().persistence
);
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted);
assert_eq!(failure.operation().backend_compensation, None);
assert_eq!(
failure.operation().finalize.as_ref().unwrap().stage,
FinalizeStage::Publication
);
assert_semantics(&retained(&api, &worker), failure);
}
#[derive(Clone)]
struct FailingWriter(Arc<AtomicUsize>);
impl Write for FailingWriter {
fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
self.0.fetch_add(1, Ordering::SeqCst);
Err(std::io::Error::other("sink unavailable"))
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
#[tokio::test]
async fn failed_log_sink_still_returns_original_http_problem_and_retained_result() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let worker = reserve(&api);
let saved = worker_startup::persist(
&api,
&worker,
failure_fixture(CompensationOutcome::Residual),
);
let before = lifecycle_snapshot(&api);
let capture = Capture::default();
let baseline = public_result(result_router(saved.clone()))
.with_subscriber(subscriber(capture.clone()))
.await;
assert_eq!(capture.events().len(), 1);
let attempts = Arc::new(AtomicUsize::new(0));
let writer = FailingWriter(attempts.clone());
let failed_subscriber = tracing_subscriber::fmt()
.without_time()
.with_ansi(false)
.json()
.flatten_event(true)
.with_writer(move || writer.clone())
.finish();
let actual = public_result(result_router(saved.clone()))
.with_subscriber(failed_subscriber)
.await;
assert!(
attempts.load(Ordering::SeqCst) > 0,
"must exercise actual observation sink failure"
);
assert_eq!(actual.0, baseline.0);
assert_eq!(
actual.1, baseline.1,
"sink failure must not replace the outgoing Problem"
);
assert_semantics(&actual.2, &saved);
assert_semantics(&retained(&api, &worker), &saved);
assert_eq!(lifecycle_snapshot(&api), before);
}
@@ -2745,24 +2745,53 @@ pub(super) struct ConnectedWorkerHarness {
_dir: tempfile::TempDir, _dir: tempfile::TempDir,
} }
impl ConnectedWorkerHarness { impl ConnectedWorkerHarness {
pub(super) async fn human( /// Exercise the authenticated Browser TCP/HTTP boundary without discarding
/// media type or rewriting its body. Bound both elapsed time and streamed bytes;
/// redirects/retries must not turn a failed create response into another POST.
pub(super) async fn human_raw(
&self, &self,
method: reqwest::Method, method: reqwest::Method,
path: &str, path: &str,
body: Value, body: Value,
) -> (StatusCode, Value) { ) -> (StatusCode, HeaderMap, Vec<u8>) {
let response = reqwest::Client::builder() const RESPONSE_LIMIT: usize = 2 * 1024 * 1024;
let mut response = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30)) .timeout(std::time::Duration::from_secs(30))
.redirect(reqwest::redirect::Policy::none())
.retry(reqwest::retry::never())
.build() .build()
.unwrap() .unwrap()
.request(method.clone(), format!("{}{path}", self.backend_url)) .request(method, format!("{}{path}", self.backend_url))
.bearer_auth(&self.owner) .bearer_auth(&self.owner)
.json(&body) .json(&body)
.send() .send()
.await .await
.unwrap(); .unwrap();
let status = StatusCode::from_u16(response.status().as_u16()).unwrap(); let status = StatusCode::from_u16(response.status().as_u16()).unwrap();
let bytes = response.bytes().await.unwrap(); let headers = response.headers().clone();
assert!(
response
.content_length()
.is_none_or(|len| len <= RESPONSE_LIMIT as u64)
);
let mut bytes = Vec::new();
while let Some(chunk) = response.chunk().await.unwrap() {
assert!(
chunk.len() <= RESPONSE_LIMIT - bytes.len(),
"Browser response exceeds test limit"
);
bytes.extend_from_slice(&chunk);
}
(status, headers, bytes)
}
pub(super) async fn human(
&self,
method: reqwest::Method,
path: &str,
body: Value,
) -> (StatusCode, Value) {
let (status, _, bytes) = self.human_raw(method.clone(), path, body).await;
let body = if status == StatusCode::NO_CONTENT { let body = if status == StatusCode::NO_CONTENT {
assert!(bytes.is_empty(), "{method} {path}: HTTP204 must be empty"); assert!(bytes.is_empty(), "{method} {path}: HTTP204 must be empty");
Value::Null Value::Null
@@ -2970,20 +2999,36 @@ impl ConnectedWorkerHarness {
} }
let (parts, body) = response.into_parts(); let (parts, body) = response.into_parts();
let bytes = to_bytes(body, 2 * 1024 * 1024).await.unwrap(); let bytes = to_bytes(body, 2 * 1024 * 1024).await.unwrap();
let content_type = parts
.headers
.get(axum::http::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
let body: Value = serde_json::from_slice(&bytes).unwrap(); let body: Value = serde_json::from_slice(&bytes).unwrap();
if let Some(startup) = body["error"].get("startup").filter(|v| !v.is_null()) let message = if content_type == runtime_api::startup::PROBLEM_CONTENT_TYPE
{ {
startup_failures.lock().unwrap().push( use runtime_api::startup::{
serde_json::from_value(startup.clone()) DecodeStartupProblem, StartupBoundary, decode_startup_problem,
.expect("RuntimeHTTP structured startup failure"), };
); let DecodeStartupProblem::Known(failure) = decode_startup_problem(
} &bytes,
errors.lock().unwrap().push( parts.status.as_u16(),
content_type,
StartupBoundary::Runtime,
) else {
panic!("RuntimeHTTP must emit a valid typed startup Problem");
};
startup_failures
.lock()
.unwrap()
.push(failure.startup().clone());
body["detail"].as_str().expect("Problem detail")
} else {
body["error"]["message"] body["error"]["message"]
.as_str() .as_str()
.unwrap_or("unknown fixture error") .unwrap_or("unknown fixture error")
.to_owned(), };
); errors.lock().unwrap().push(message.to_owned());
Response::from_parts(parts, Body::from(bytes)) Response::from_parts(parts, Body::from(bytes))
} }
}, },
@@ -12,6 +12,8 @@ use worker_runtime::execution::{
// published Profile projection; leave Worker creation/control to the real Runtime. // published Profile projection; leave Worker creation/control to the real Runtime.
#[derive(Default)] #[derive(Default)]
struct LaunchExecutionBackend { struct LaunchExecutionBackend {
lose_initial_ack: std::sync::atomic::AtomicBool,
fail_stop: std::sync::atomic::AtomicBool,
execution: DeterministicExecutionBackend, execution: DeterministicExecutionBackend,
provider_close_gate: Mutex<Option<Arc<super::workdir_session_close_tests::ProviderCloseGate>>>, provider_close_gate: Mutex<Option<Arc<super::workdir_session_close_tests::ProviderCloseGate>>>,
bundles: Mutex<std::collections::HashMap<String, worker_runtime::config_bundle::ConfigBundle>>, bundles: Mutex<std::collections::HashMap<String, worker_runtime::config_bundle::ConfigBundle>>,
@@ -196,10 +198,45 @@ impl WorkerExecutionBackend for LaunchExecutionBackend {
.lock() .lock()
.unwrap() .unwrap()
.push((worker.clone(), input.clone())); .push((worker.clone(), input.clone()));
self.execution.dispatch_input(worker, input) let mut result = self.execution.dispatch_input(worker, input);
if self
.lose_initial_ack
.swap(false, std::sync::atomic::Ordering::SeqCst)
{
// Lose only the response acknowledgement, after the real fixture
// execution committed its durable receipt. Never redispatch it.
result.submission = None;
}
result
}
fn termination_receipt(
&self,
worker: &worker_runtime::identity::WorkerRef,
) -> std::result::Result<Option<worker_runtime::execution::WorkerExecutionTermination>, String>
{
self.execution.termination_receipt(worker)
}
fn stop_worker_operation(
&self,
request: worker_runtime::execution::WorkerExecutionStopRequest,
) -> WorkerExecutionResult {
if self.fail_stop.load(std::sync::atomic::Ordering::SeqCst) {
return self.stop_worker(&request.worker_ref);
}
// Preserve the fixture's actual joined execution and operation-bound
// termination receipt; an accepted legacy stop alone is not that evidence.
self.execution.stop_worker_operation(request)
} }
fn stop_worker(&self, worker: &worker_runtime::identity::WorkerRef) -> WorkerExecutionResult { fn stop_worker(&self, worker: &worker_runtime::identity::WorkerRef) -> WorkerExecutionResult {
if self.fail_stop.load(std::sync::atomic::Ordering::SeqCst) {
return WorkerExecutionResult::errored(
worker_runtime::execution::WorkerExecutionOperation::Stop,
"TEST_ONLY_PRIVATE_STOP_CREDENTIAL /private/path",
);
}
self.execution.stop_worker(worker) self.execution.stop_worker(worker)
} }
} }
@@ -1133,6 +1170,90 @@ async fn read_only_ticket_targets_allow_extra_external_grants_without_widening_c
); );
} }
/// Production create owner and Backend adapters share the same executor; only
/// its response acknowledgement and cleanup boundary are fault-injected.
#[tokio::test(flavor = "multi_thread")]
async fn initial_input_ack_loss_preserves_creation_facts_across_embedded_and_remote() {
use runtime_api::startup::*;
for failed_cleanup in [false, true] {
let mut observed = Vec::new();
for remote in [false, true] {
let fixture = LaunchFixture::new(remote).await;
let ticket = fixture.ticket();
let payload = fixture.payload(&ticket, "lost-initial-ack", "builtin:companion", false);
fixture
.backend
.lose_initial_ack
.store(true, std::sync::atomic::Ordering::SeqCst);
fixture
.backend
.fail_stop
.store(failed_cleanup, std::sync::atomic::Ordering::SeqCst);
let (status, body) = fixture.post(TEST_WORKSPACE_ID, payload).await;
assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}");
let DecodeStartupProblem::Known(failure) = decode_startup_problem(
&serde_json::to_vec(&body).unwrap(),
status.as_u16(),
PROBLEM_CONTENT_TYPE,
StartupBoundary::Browser,
) else {
panic!("must retain typed creation facts: {body}");
};
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown);
let finalize = failure.operation().finalize.unwrap();
assert_eq!(finalize.stage, FinalizeStage::InitialInput);
assert_eq!(
finalize.compensation,
if failed_cleanup {
CompensationOutcome::Unknown
} else {
CompensationOutcome::Completed
}
);
assert_eq!(body["yoi_failure"]["reason"], "invalid_response");
assert!(
!body
.to_string()
.contains("TEST_ONLY_PRIVATE_STOP_CREDENTIAL")
);
let creates = fixture.backend.creates.lock().unwrap().clone();
assert_eq!(creates.len(), 1);
let worker_id = creates[0].worker_id.to_string();
let worker = RuntimeWorkerRef::new(&fixture.runtime_id, &worker_id);
let Some(DecodeStoredWorkerCreateFailure::Known(saved)) =
fixture.api.worker_create_failure(&worker).unwrap()
else {
panic!("retain queryable create result");
};
assert_eq!(saved.public_facts(), failure.public_facts());
assert_eq!(
fixture.backend.submits_for(&worker_id).len(),
1,
"observation failure never redispatches"
);
assert_eq!(
fixture
.backend
.execution
.contexts
.lock()
.unwrap()
.keys()
.any(|worker| worker.worker_id.to_string() == worker_id),
failed_cleanup
);
let mut facts = failure.public_facts();
facts.operation.correlation = None;
observed.push((status, body["yoi_failure"]["reason"].clone(), facts));
}
assert_eq!(
observed[0], observed[1],
"same owner facts through Embedded and Remote"
);
}
}
#[tokio::test(flavor = "multi_thread")] #[tokio::test(flavor = "multi_thread")]
async fn worker_assignment_finalize_failure_compensates_real_embedded_runtime_and_preserves_open_case() async fn worker_assignment_finalize_failure_compensates_real_embedded_runtime_and_preserves_open_case()
{ {
@@ -1151,10 +1272,19 @@ async fn worker_assignment_finalize_failure_compensates_real_embedded_runtime_an
.unwrap(); .unwrap();
let payload = fixture.payload(&ticket_id, "compensated-worker", "builtin:companion", false); let payload = fixture.payload(&ticket_id, "compensated-worker", "builtin:companion", false);
let (status, body) = fixture.post(TEST_WORKSPACE_ID, payload).await; let (status, body) = fixture.post(TEST_WORKSPACE_ID, payload).await;
assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}");
assert_eq!(body["yoi_failure"]["domain"], "startup_boundary");
assert_eq!(body["yoi_failure"]["reason"], "finalize_failed");
let operation = &body["yoi_startup"]["operation"];
assert_eq!(operation["acceptance"], "accepted");
assert_eq!(operation["execution"], "accepted");
assert_eq!(operation["finalize"]["stage"], "ticket_assignment_bind");
assert_eq!(operation["finalize"]["compensation"], "completed");
assert!(body["detail"].as_str().unwrap().contains("finalization"));
assert!( assert!(
body["message"].as_str().unwrap().contains("compensat"), !body
"{body}" .to_string()
.contains("injected Worker assignment failure")
); );
let creates = fixture let creates = fixture
.backend .backend
@@ -1311,6 +1441,7 @@ impl crate::hosts::WorkspaceWorkerRuntime for LostSpawnResponseRuntime {
assert_eq!(result.state, InternalWorkerOperationState::Accepted); assert_eq!(result.state, InternalWorkerOperationState::Accepted);
assert!(result.worker.is_some()); assert!(result.worker.is_some());
WorkerSpawnResult { WorkerSpawnResult {
failure: None,
state: InternalWorkerOperationState::Rejected, state: InternalWorkerOperationState::Rejected,
worker: None, worker: None,
acceptance_evidence: Vec::new(), acceptance_evidence: Vec::new(),
@@ -1829,3 +1960,81 @@ async fn mr_open_retains_source_before_first_review_and_observed_result() {
) )
.unwrap(); .unwrap();
} }
// Real Runtime creation and SQLite finalization fail independently; only Stop is
// injected. Reconciliation must retain the create reservation while absence is unknown.
#[tokio::test(flavor = "multi_thread")]
async fn accepted_creation_finalize_and_failed_stop_retain_unknown_compensation_and_reservation() {
let fixture = LaunchFixture::new(false).await;
let ticket_id = fixture.ticket();
rusqlite::Connection::open(&fixture.api.config.database_path).unwrap().execute_batch(
"CREATE TRIGGER fail_startup_finalize BEFORE INSERT ON ticket_current_worker_assignments WHEN NEW.role='worker' BEGIN SELECT RAISE(ABORT, 'TEST_ONLY_PRIVATE_DB_CREDENTIAL'); END;"
).unwrap();
fixture
.backend
.fail_stop
.store(true, std::sync::atomic::Ordering::SeqCst);
let (status, body) = fixture
.post(
TEST_WORKSPACE_ID,
fixture.payload(
&ticket_id,
"unknown-compensation",
"builtin:companion",
false,
),
)
.await;
fixture
.backend
.fail_stop
.store(false, std::sync::atomic::Ordering::SeqCst);
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}");
assert_eq!(body["yoi_failure"]["reason"], "finalize_failed");
let operation = &body["yoi_startup"]["operation"];
assert_eq!(operation["acceptance"], "accepted");
assert_eq!(operation["execution"], "accepted");
assert_eq!(operation["finalize"]["compensation"], "unknown");
assert!(
operation["correlation"]
.as_str()
.unwrap()
.starts_with("worker-")
);
for secret in [
"TEST_ONLY_PRIVATE_STOP_CREDENTIAL",
"/private/path",
"TEST_ONLY_PRIVATE_DB_CREDENTIAL",
] {
assert!(!body.to_string().contains(secret), "{body}");
}
assert_eq!(
fixture
.api
.authority
.ticket(&ticket_id)
.unwrap()
.state
.as_str(),
"open"
);
let created = fixture.backend.creates.lock().unwrap();
let created: Vec<_> = created
.iter()
.filter(|request| request.display_name.as_deref() == Some("Explicit Ticket Worker"))
.collect();
assert_eq!(created.len(), 1);
let worker_id = created[0].worker_id.to_string();
let reserved: i64 = fixture.api.config_store.with_conn(|conn| conn.query_row(
"SELECT COUNT(*) FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='reserved'",
rusqlite::params![TEST_WORKSPACE_ID, fixture.runtime_id, worker_id], |row| row.get(0),
).map_err(Error::from)).unwrap();
// A reservation may already be completed by Runtime admission, but it must
// remain present (never released as absent) until Runtime absence is confirmed.
let retained: i64 = fixture.api.config_store.with_conn(|conn| conn.query_row(
"SELECT COUNT(*) FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
rusqlite::params![TEST_WORKSPACE_ID, fixture.runtime_id, worker_id], |row| row.get(0),
).map_err(Error::from)).unwrap();
assert_eq!(retained, 1);
assert!(reserved <= retained);
}
@@ -0,0 +1,286 @@
//! Backend creation boundary. Cause and independent owner observations are retained
//! before HTTP projection. Correlation fingerprints are lookup aids, not operation
//! identities, idempotency keys, execution grants, or permission to retry.
//!
//! # Result investigation
//!
//! Start with the Browser Problem's `yoi_startup.operation.correlation` (also its
//! `instance` suffix). For a known Runtime/Worker identity, `correlation` derives
//! the same path-free token. Find the single `yoi::worker_startup` event named
//! `worker_create_result`; do not use the token as a request or operation key.
//! Compare its typed domain/reason with `WorkspaceApi::worker_create_failure`'s
//! retained reservation evidence. Inspect acceptance/execution first, then finalize,
//! Backend compensation and residual resources independently. Unknown/corrupt or
//! absent history remains unknown/absent, not evidence of rejection or cleanup.
//!
//! The existing Session metadata retains the Runtime startup primary and its own
//! `persistence`; `backend_result_persistence` reports only this coordinator result
//! column's atomic write. A failed sink leaves the primary and observations intact.
//! Reopening the store reads evidence, never dispatches work. Current authorization
//! and a fresh domain projection are required before redisclosing loaded evidence.
//! Unsupported bodies, credentials, source errors and filesystem paths are not log
//! fields. Context wrapping, Remote decoding and persistence do not duplicate the
//! HTTP primary event; cleanup has a separate safe coordinator event. Non-HTTP
//! consumers receive the same typed result and own their eventual observation sink.
use super::*;
use runtime_api::startup::{
CreationAcceptance, ExecutionOutcome, StartupCause, StartupOperationFacts, WorkerCreateFailure,
};
pub(super) fn local_rejection() -> WorkerCreateFailure {
WorkerCreateFailure::from_cause(
StartupCause::Runtime(protocol::startup_failure::RuntimeStartupCause {
stage: protocol::startup_failure::StartupStage::Preflight,
reason: protocol::startup_failure::RuntimeStartupReason::StartupFailed,
}),
StartupOperationFacts {
acceptance: CreationAcceptance::Rejected,
execution: ExecutionOutcome::Unknown,
finalize: None,
correlation: None,
backend_compensation: None,
},
)
.expect("enum-only local cause fits the boundary budget")
}
pub(super) fn finalize_error(
source: ApiError,
stage: WorkerSpawnFinalizeStage,
compensation: runtime_api::startup::CompensationOutcome,
) -> ApiError {
let failure = finalize_failure(&source, stage, compensation);
ApiError::with_diagnostics(
Error::WorkerCreateFailed {
failure: Box::new(failure),
source: Some(Box::new(source.error)),
},
source.diagnostics,
)
}
pub(super) fn finalize_failure(
source: &ApiError,
stage: WorkerSpawnFinalizeStage,
compensation: runtime_api::startup::CompensationOutcome,
) -> WorkerCreateFailure {
use runtime_api::startup::{
FinalizeFacts, FinalizeStage, StartupBoundaryCause, StartupBoundaryReason,
};
let stage = match stage {
WorkerSpawnFinalizeStage::WorkerIdentity => FinalizeStage::WorkerIdentity,
WorkerSpawnFinalizeStage::WorkspaceApiBinding => FinalizeStage::WorkspaceApiBinding,
WorkerSpawnFinalizeStage::CreateReservation => FinalizeStage::CreateReservation,
WorkerSpawnFinalizeStage::WorkerRegistry => FinalizeStage::WorkerRegistry,
WorkerSpawnFinalizeStage::TicketAssignmentBind => FinalizeStage::TicketAssignmentBind,
WorkerSpawnFinalizeStage::TicketAssignmentCurrent => FinalizeStage::TicketAssignmentCurrent,
WorkerSpawnFinalizeStage::TicketStateAccept => FinalizeStage::TicketStateAccept,
WorkerSpawnFinalizeStage::WorkdirRegistry => FinalizeStage::WorkdirRegistry,
WorkerSpawnFinalizeStage::WorkdirAttachment => FinalizeStage::WorkdirAttachment,
};
let operation = StartupOperationFacts {
acceptance: CreationAcceptance::Accepted,
execution: ExecutionOutcome::Accepted,
finalize: Some(FinalizeFacts {
stage,
compensation,
}),
backend_compensation: Some(compensation),
correlation: None,
};
match &source.error {
Error::WorkerCreateFailed { failure, .. } => {
let (startup, mut previous) = failure.clone().into_parts();
// Adding finalization context must not rewrite the earlier owner result.
previous.finalize = operation.finalize;
previous.backend_compensation = Some(compensation);
WorkerCreateFailure::new(startup, previous)
}
_ => match source.session_registration_cause {
Some(cause) => WorkerCreateFailure::from_cause(StartupCause::Session(cause), operation),
None => WorkerCreateFailure::from_cause(
StartupCause::Boundary(StartupBoundaryCause {
reason: StartupBoundaryReason::FinalizeFailed,
}),
operation,
),
},
}
.expect("locally constructed finalization facts fit the shared boundary budget")
}
/// Publication failure cannot undo committed creation or authorize reexecution.
/// Keep the Worker and its assignment: `Residual` describes deliberate retention,
/// not attempted rollback; Backend compensation remains unattempted (`None`).
/// The original store error stays internal while the independent reservation sink
/// retains only the typed result and a path-free correlation.
pub(super) fn publication_error(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
source: ApiError,
) -> ApiError {
use protocol::startup_failure::{StartupCleanup, StartupResourceFacts, StartupResources};
use runtime_api::startup::{
CompensationOutcome, FinalizeFacts, FinalizeStage, StartupBoundaryCause,
StartupBoundaryReason,
};
let operation = StartupOperationFacts {
acceptance: CreationAcceptance::Accepted,
execution: ExecutionOutcome::Accepted,
finalize: Some(FinalizeFacts {
stage: FinalizeStage::Publication,
compensation: CompensationOutcome::Residual,
}),
backend_compensation: None,
correlation: None,
};
let failure = match &source.error {
Error::WorkerCreateFailed { failure, .. } => failure.as_ref().clone(),
_ => WorkerCreateFailure::from_cause(
StartupCause::Boundary(StartupBoundaryCause {
reason: StartupBoundaryReason::FinalizeFailed,
}),
operation.clone(),
)
.expect("enum-only publication cause fits the boundary budget"),
};
let (mut startup, _) = failure.into_parts();
startup.resources = Some(StartupResourceFacts {
resources: StartupResources::WorkerStartup,
cleanup: StartupCleanup::NotNeeded,
durable_effects_possible: true,
});
let failure = WorkerCreateFailure::new(startup, operation)
.expect("locally constructed publication facts fit the shared boundary budget");
let failure = persist(api, worker, failure);
ApiError::with_diagnostics(
Error::WorkerCreateFailed {
failure: Box::new(failure),
source: Some(Box::new(source.error)),
},
source.diagnostics,
)
}
pub(super) fn unknown_result() -> WorkerCreateFailure {
WorkerCreateFailure::from_cause(
StartupCause::Boundary(runtime_api::startup::StartupBoundaryCause {
reason: runtime_api::startup::StartupBoundaryReason::InvalidResponse,
}),
StartupOperationFacts {
acceptance: CreationAcceptance::Unknown,
execution: ExecutionOutcome::Unknown,
finalize: None,
correlation: None,
backend_compensation: None,
},
)
.expect("enum-only observation fits the boundary budget")
}
pub(super) fn correlate(
failure: WorkerCreateFailure,
worker: &RuntimeWorkerRef,
) -> WorkerCreateFailure {
let reference = correlation(&worker.runtime_id, &worker.worker_id);
let mut operation = failure.operation().clone();
operation.correlation = Some(
runtime_api::startup::StartupCorrelation::new(reference).expect("fixed digest is safe"),
);
failure.with_operation(operation)
}
pub(super) fn compensated(
failure: WorkerCreateFailure,
outcome: runtime_api::startup::CompensationOutcome,
) -> WorkerCreateFailure {
let mut operation = failure.operation().clone();
operation.backend_compensation = Some(outcome);
failure.with_operation(operation)
}
/// Save only observations on the existing reservation, never its lifecycle state.
/// An unavailable diagnostic sink cannot replace the original primary or authorize replay.
/// The prospective persisted marker commits in the same transaction as these facts.
/// Remote-supplied Backend persistence is discarded at this responsibility boundary.
pub(super) fn persist(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
failure: WorkerCreateFailure,
) -> WorkerCreateFailure {
use protocol::startup_failure::StartupPersistenceOutcome::{
Conflict, NotSaved, OutcomeUnknown, Persisted,
};
use rusqlite::OptionalExtension;
let failure = correlate(failure, worker).with_backend_result_persistence(None);
let Some(worker_id) = WorkerId::parse(&worker.worker_id) else {
return failure.with_backend_result_persistence(Some(NotSaved));
};
let observed = api.config_store.with_conn(|conn| {
conn.query_row(
"SELECT create_fingerprint, startup_failure_json FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
rusqlite::params![api.workspace_id(), worker.runtime_id, worker.worker_id],
|row| Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?)),
).optional().map_err(Error::from)
});
let (fingerprint, previous) = match observed {
Ok(Some(observed)) => observed,
Ok(None) => return failure.with_backend_result_persistence(Some(NotSaved)),
Err(_) => return failure.with_backend_result_persistence(Some(OutcomeUnknown)),
};
let saved = failure
.clone()
.with_backend_result_persistence(Some(Persisted));
let Ok(encoded) = saved.encode_storage() else {
return failure.with_backend_result_persistence(Some(NotSaved));
};
match api.config_store.save_worker_create_failure(
api.workspace_id(),
&worker.runtime_id,
worker_id,
&fingerprint,
previous.as_deref(),
&encoded,
) {
Ok(true) => saved,
Ok(false) => failure.with_backend_result_persistence(Some(Conflict)),
Err(_) => failure.with_backend_result_persistence(Some(OutcomeUnknown)),
}
}
pub(super) fn correlation(runtime_id: &str, worker_id: &str) -> String {
runtime_api::startup::StartupCorrelation::for_worker(runtime_id, worker_id)
.as_str()
.to_owned()
}
/// Only the HTTP result boundary records the primary failure. Context wrappers,
/// encoders, and Remote adapters do not emit duplicate failure records. The
/// independent coordinator cleanup event contains no exception/source text.
pub(super) fn observe_http(failure: &WorkerCreateFailure, status: StatusCode) {
let Ok(projected) = failure.telemetry() else {
tracing::error!(target: "yoi::worker_startup", event = "startup_projection_unavailable", status = status.as_u16());
return;
};
let fields = error_contract::observation_fields(&projected);
let operation = failure.operation();
tracing::warn!(
target: "yoi::worker_startup",
event = "worker_create_result",
domain = fields.domain,
reason = fields.reason,
operation = fields.operation,
stage = fields.stage,
correlation = operation.correlation.as_ref().map(|value| value.as_str()),
acceptance = ?operation.acceptance,
execution = ?operation.execution,
finalize = ?operation.finalize,
backend_compensation = ?operation.backend_compensation,
backend_result_persistence = ?failure.backend_result_persistence(),
persistence = ?failure.startup().persistence,
resources = ?failure.startup().resources,
status = status.as_u16(),
"Worker creation returned a typed failure"
);
}
+23 -2
View File
@@ -76,6 +76,26 @@ pub(super) fn verify_history(conn: &Connection, current: i64) -> Result<()> {
fn verify_history_records(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> { fn verify_history_records(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> {
verify_raw_capture_shape(conn, current, rows)?; verify_raw_capture_shape(conn, current, rows)?;
if current == 124 {
super::verify_worker_create_failure_schema(conn)?;
if rows == [(124, super::SCHEMA_BASELINE_NAME.to_owned())] {
verify_execution_shape(conn)?;
super::backend_job_upgrade::verify(conn)?;
verify_configuration_shape(conn)?;
require_shape(conn, Stage::Base)?;
return verify_103_features(conn, current, rows);
}
if rows.last().map(|(version, name)| (*version, name.as_str()))
!= Some((124, super::WORKER_CREATE_FAILURE_MIGRATION_NAME))
{
return Err(invalid("unrecognized Worker creation result receipt"));
}
let prefix = &rows[..rows.len() - 1];
if prefix.last().map(|row| row.0) != Some(123) {
return Err(invalid("missing Worker creation result predecessor"));
}
return verify_history_records(conn, 123, prefix);
}
if current == 123 { if current == 123 {
verify_execution_shape(conn)?; verify_execution_shape(conn)?;
super::backend_job_upgrade::verify(conn)?; super::backend_job_upgrade::verify(conn)?;
@@ -236,10 +256,11 @@ pub(super) fn published_plan(
verify_history(conn, current)?; verify_history(conn, current)?;
Ok(Some(super::WorkspaceSchemaMigrationPlan { Ok(Some(super::WorkspaceSchemaMigrationPlan {
current_schema_version: current, current_schema_version: current,
target_schema_version: 123, target_schema_version: 124,
migrations: [ migrations: [
(122, crate::config_adoption::MIGRATION_NAME), (122, crate::config_adoption::MIGRATION_NAME),
(123, super::backend_job_upgrade::MIGRATION_NAME), (123, super::backend_job_upgrade::MIGRATION_NAME),
(124, super::WORKER_CREATE_FAILURE_MIGRATION_NAME),
] ]
.into_iter() .into_iter()
.filter(|(version, _)| *version > current) .filter(|(version, _)| *version > current)
@@ -465,7 +486,7 @@ fn legacy_shape(conn: &Connection) -> Result<Shape> {
} }
fn upgraded_reference(conn: &Connection, mut shape: Shape) -> Result<Shape> { fn upgraded_reference(conn: &Connection, mut shape: Shape) -> Result<Shape> {
if super::current_schema_version(conn)? != 123 { if super::current_schema_version(conn)? < 123 {
return Ok(shape); return Ok(shape);
} }
for table in ["backend_jobs", "backend_job_attempts"] { for table in ["backend_jobs", "backend_job_attempts"] {
+67 -6
View File
@@ -137,7 +137,10 @@ pub(crate) mod backend_job_upgrade;
mod storage_formats; mod storage_formats;
pub use backend_job_recovery_store::BackendJobReconciliation; pub use backend_job_recovery_store::BackendJobReconciliation;
const LATEST_SCHEMA_VERSION: i64 = 123; const LATEST_SCHEMA_VERSION: i64 = 124;
const WORKER_CREATE_FAILURE_MIGRATION_NAME: &str =
"Worker create coordinator returned startup failure";
pub(crate) const WORKER_CREATE_FAILURE_MAX_BYTES: usize = 32 * 1024;
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
@@ -520,6 +523,11 @@ const MIGRATIONS: &[Migration] = &[
name: backend_job_upgrade::MIGRATION_NAME, name: backend_job_upgrade::MIGRATION_NAME,
apply: migrate_backend_job_execution_to_v123, apply: migrate_backend_job_execution_to_v123,
}, },
Migration {
version: 124,
name: WORKER_CREATE_FAILURE_MIGRATION_NAME,
apply: migrate_worker_create_failure_to_v124,
},
]; ];
// Both schema-103 identities shipped. Keep the alternate implementation and accept // Both schema-103 identities shipped. Keep the alternate implementation and accept
@@ -16388,6 +16396,50 @@ fn migrate_plugin_services_v111_to_v112(conn: &Connection) -> Result<()> {
Ok(()) Ok(())
} }
fn verify_worker_create_failure_schema(conn: &Connection) -> Result<()> {
let column = conn
.query_row(
"SELECT type, \"notnull\", dflt_value FROM pragma_table_info('worker_create_reservations') WHERE name='startup_failure_json'",
[],
|row| Ok((row.get::<_, String>(0)?, row.get::<_, bool>(1)?, row.get::<_, Option<String>>(2)?)),
)
.optional()?;
let sql: String = conn.query_row(
"SELECT sql FROM sqlite_schema WHERE type='table' AND name='worker_create_reservations'",
[],
|row| row.get(0),
)?;
let normalized: String = sql.chars().filter(|c| !c.is_whitespace()).collect();
let constraint = format!(
"CHECK(startup_failure_jsonISNULLOR(typeof(startup_failure_json)='text'ANDlength(CAST(startup_failure_jsonASBLOB))<={}))",
WORKER_CREATE_FAILURE_MAX_BYTES
);
if column != Some(("TEXT".into(), false, None)) || !normalized.contains(&constraint) {
return Err(Error::Store(
"Worker create startup failure column or byte bound is missing".into(),
));
}
Ok(())
}
fn migrate_worker_create_failure_to_v124(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 123 {
return Err(Error::Store(
"Worker create failure migration requires schema 123".into(),
));
}
storage_formats::verify_history(conn, 123)?;
let tx = conn.unchecked_transaction()?;
tx.execute_batch(include_str!("worker_create_failure_schema.sql"))?;
verify_worker_create_failure_schema(&tx)?;
tx.execute(
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(124,?1)",
[WORKER_CREATE_FAILURE_MIGRATION_NAME],
)?;
tx.commit()?;
Ok(())
}
fn migrate_backend_job_execution_to_v123(conn: &Connection) -> Result<()> { fn migrate_backend_job_execution_to_v123(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 122 { if current_schema_version(conn)? != 122 {
return Err(Error::Store( return Err(Error::Store(
@@ -16537,6 +16589,8 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
conn.execute_batch(content_history_store::SCHEMA)?; conn.execute_batch(content_history_store::SCHEMA)?;
conn.execute_batch(crate::config_adoption::SCHEMA)?; conn.execute_batch(crate::config_adoption::SCHEMA)?;
backend_job_upgrade::install_fresh(conn)?; backend_job_upgrade::install_fresh(conn)?;
conn.execute_batch(include_str!("worker_create_failure_schema.sql"))?;
verify_worker_create_failure_schema(conn)?;
Ok(()) Ok(())
} }
@@ -16574,11 +16628,14 @@ fn migrate_plugin_plans_v106_to_v107(conn: &Connection) -> Result<()> {
// Fresh composition uses the posts contract. Retained fixtures pass frozen host // Fresh composition uses the posts contract. Retained fixtures pass frozen host
// and component DDL to the shared cross-domain constraint composition below. // and component DDL to the shared cross-domain constraint composition below.
fn create_workspace_schema_structure(conn: &Connection) -> Result<()> { fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
create_workspace_schema_structure_from_sql( // The pre-124 reference composition must not already contain the column
conn, // added by migration 124. Fresh latest creation installs it after this prefix.
&backend_job_upgrade::retained_checks(include_str!("latest_schema.sql")), let retained_schema = backend_job_upgrade::retained_checks(include_str!("latest_schema.sql"))
true, .replace(
)?; " startup_failure_json TEXT CHECK (startup_failure_json IS NULL OR\n (typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768)),\n",
"",
);
create_workspace_schema_structure_from_sql(conn, &retained_schema, true)?;
conn.execute_batch(include_str!("activation_schema.sql"))?; conn.execute_batch(include_str!("activation_schema.sql"))?;
conn.execute_batch(include_str!("passkey_results_schema.sql"))?; conn.execute_batch(include_str!("passkey_results_schema.sql"))?;
conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?; conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?;
@@ -17672,6 +17729,10 @@ fn table_columns(conn: &Connection, table_name: &str) -> Result<Vec<String>> {
#[path = "storage_owner_tests.rs"] #[path = "storage_owner_tests.rs"]
mod storage_owner_tests; mod storage_owner_tests;
#[cfg(test)]
#[path = "worker_create_failure_store_tests.rs"]
mod worker_create_failure_store_tests;
#[cfg(test)] #[cfg(test)]
#[path = "store_test_support.rs"] #[path = "store_test_support.rs"]
mod test_support; mod test_support;
@@ -852,8 +852,6 @@ mod subject_assignment_migration_tests {
}; };
const PLAN_NAME: &str = "Immutable Worker Plugin plans and retained pins"; const PLAN_NAME: &str = "Immutable Worker Plugin plans and retained pins";
const SUBJECT_NAME: &str =
"Subject Ticket responsibility and current-body notification routing";
type Rows = Vec<Vec<rusqlite::types::Value>>; type Rows = Vec<Vec<rusqlite::types::Value>>;
fn raw(path: &Path) -> Connection { fn raw(path: &Path) -> Connection {
@@ -1038,8 +1036,23 @@ mod subject_assignment_migration_tests {
} else { } else {
String::new() String::new()
}; };
// Keep append identities for rowid tables; current content
// history tables are WITHOUT ROWID, so compare every column
// in deterministic value order instead of inventing rowids.
let without_rowid: bool = conn
.query_row(
"SELECT wr FROM pragma_table_list WHERE schema='main' AND name=?1",
[&table],
|row| row.get(0),
)
.unwrap();
let (projection, order) = if without_rowid {
(columns.clone(), columns)
} else {
(format!("rowid,{columns}"), "rowid".into())
};
let query = format!( let query = format!(
"SELECT rowid,{columns} FROM \"{table}\"{sequence_filter} ORDER BY rowid" "SELECT {projection} FROM \"{table}\"{sequence_filter} ORDER BY {order}"
); );
let rows = fixture_query_rows(conn, &query); let rows = fixture_query_rows(conn, &query);
(query, rows) (query, rows)
@@ -1120,49 +1133,15 @@ mod subject_assignment_migration_tests {
} }
fn expected_tail(version: i64) -> Vec<(i64, String)> { fn expected_tail(version: i64) -> Vec<(i64, String)> {
let mut tail = match version { assert!(
96 => vec![ matches!(version, 96 | 106 | 107),
( "unsupported fixture origin"
97, );
"MR notifications and authenticated implementation results".into(), MIGRATIONS
), .iter()
(98, "Assignment-independent implementation reports".into()), .filter(|migration| migration.version > version)
( .map(|migration| (migration.version, migration.name.into()))
99, .collect()
"Explicit hosting administrators and self-registration policy".into(),
),
(
100,
"Provisional users and scoped passkey activation".into(),
),
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into()),
(103, "Passkey metadata and atomic addition results".into()),
(
104,
"Converge Passkey results and Job input retention".into(),
),
(
105,
"Durable Subject deletion snapshots and Job admission fences".into(),
),
(106, crate::plugin_catalog::MIGRATION_NAME.into()),
(107, PLAN_NAME.into()),
(108, SUBJECT_NAME.into()),
],
106 => vec![(107, PLAN_NAME.into()), (108, SUBJECT_NAME.into())],
107 => vec![(108, SUBJECT_NAME.into())],
_ => panic!("unsupported fixture origin"),
};
tail.push((109, "Durable Plugin Host invocations and HTTPS effect receipts".into()));
tail.push((110, "Worker Plugin launch selection associations".into()));
tail.push((
111,
"Typed Plugin Ticket extension data and atomic mutation receipts".into(),
));
tail.push((112, "Ticket Service outbox and fenced consumers".into()));
tail.push((113, "Ticket work-decomposition parent links".into()));
tail
} }
fn plan_steps(version: i64) -> Vec<WorkspaceSchemaMigrationStep> { fn plan_steps(version: i64) -> Vec<WorkspaceSchemaMigrationStep> {
@@ -56,12 +56,17 @@ fn ticket_posts_schema93_preserves_initial_body_creator_comments_and_archived_au
drop(conn); drop(conn);
let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap(); let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap();
assert_eq!(plan.current_schema_version, 93); assert_eq!(plan.current_schema_version, 93);
assert_eq!(plan.target_schema_version, LATEST_SCHEMA_VERSION);
assert_eq!( assert_eq!(
plan.migrations plan.migrations,
MIGRATIONS
.iter() .iter()
.map(|step| step.version) .filter(|migration| migration.version > 93)
.collect::<Vec<_>>(), .map(|migration| WorkspaceSchemaMigrationStep {
(94..=LATEST_SCHEMA_VERSION).collect::<Vec<_>>() version: migration.version,
name: migration.name.into(),
})
.collect::<Vec<_>>()
); );
let conn = Connection::open(&path).unwrap(); let conn = Connection::open(&path).unwrap();
assert!( assert!(
@@ -0,0 +1,3 @@
ALTER TABLE worker_create_reservations ADD COLUMN startup_failure_json TEXT
CHECK (startup_failure_json IS NULL OR
(typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768));
@@ -0,0 +1,98 @@
//! Backend coordinator returned startup results belong to the existing create
//! reservation. Runtime primary failure remains in Session metadata. Saving or
//! reading this evidence neither transitions the reservation nor grants retry.
use runtime_api::startup::{DecodeStoredWorkerCreateFailure, decode_stored_worker_create_failure};
use rusqlite::{OptionalExtension, params};
use worker_runtime::identity::WorkerId;
use crate::store::{SqliteWorkspaceStore, WORKER_CREATE_FAILURE_MAX_BYTES};
use crate::{Error, Result};
impl SqliteWorkspaceStore {
#[allow(clippy::too_many_arguments)]
pub(crate) fn save_worker_create_failure(
&self,
workspace_id: &str,
runtime_id: &str,
worker_id: WorkerId,
expected_create_fingerprint: &str,
expected_previous: Option<&str>,
encoded: &str,
) -> Result<bool> {
// Bound the wire representation before the typed decoder allocates JSON.
if encoded.len() > WORKER_CREATE_FAILURE_MAX_BYTES
|| expected_previous.is_some_and(|value| value.len() > WORKER_CREATE_FAILURE_MAX_BYTES)
{
return Err(Error::InvalidInput(
"Worker create failure exceeds storage byte bound".into(),
));
}
if !matches!(
decode_stored_worker_create_failure(encoded),
DecodeStoredWorkerCreateFailure::Known(_)
) {
return Err(Error::InvalidInput(
"Worker create failure is not a known storage representation".into(),
));
}
self.with_immediate_transaction(|tx| {
// Identity, immutable create intent and exact prior bytes are checked
// in the same UPDATE. Removed reservations deliberately remain eligible.
let changed = tx.execute(
"UPDATE worker_create_reservations SET startup_failure_json=?6
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
AND create_fingerprint=?4 AND startup_failure_json IS ?5",
params![
workspace_id,
runtime_id,
worker_id.to_string(),
expected_create_fingerprint,
expected_previous,
encoded
],
)?;
Ok(changed == 1)
})
}
pub(crate) fn worker_create_failure(
&self,
workspace_id: &str,
runtime_id: &str,
worker_id: WorkerId,
) -> Result<Option<String>> {
self.with_conn(|conn| {
// Return raw bounded evidence, including unknown/corrupt versions;
// classification belongs to the typed reader, not legacy prose.
let saved = conn
.query_row(
"SELECT CASE WHEN length(CAST(startup_failure_json AS BLOB)) <= ?4
THEN startup_failure_json ELSE NULL END,
length(CAST(startup_failure_json AS BLOB))
FROM worker_create_reservations
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
params![
workspace_id,
runtime_id,
worker_id.to_string(),
WORKER_CREATE_FAILURE_MAX_BYTES as i64
],
|row| {
Ok((
row.get::<_, Option<String>>(0)?,
row.get::<_, Option<i64>>(1)?,
))
},
)
.optional()?;
match saved {
Some((_, Some(bytes))) if bytes > WORKER_CREATE_FAILURE_MAX_BYTES as i64 => Err(
Error::Store("Stored Worker create failure exceeds byte bound".into()),
),
Some((value, _)) => Ok(value),
None => Ok(None),
}
})
}
}
@@ -0,0 +1,475 @@
use super::*;
use runtime_api::startup::{
CompensationOutcome, CreationAcceptance, DecodeStoredWorkerCreateFailure, ExecutionOutcome,
FinalizeFacts, FinalizeStage, StartupBoundaryCause, StartupBoundaryReason, StartupCause,
StartupOperationFacts, WorkerCreateFailure, decode_stored_worker_create_failure,
};
use rusqlite::types::Value;
const WORKSPACE: &str = "failure-workspace";
const RUNTIME: &str = "failure-runtime";
fn encoded_failure(compensation: CompensationOutcome) -> String {
let failure = WorkerCreateFailure::from_cause(
StartupCause::Boundary(StartupBoundaryCause {
reason: StartupBoundaryReason::FinalizeFailed,
}),
StartupOperationFacts {
acceptance: CreationAcceptance::Accepted,
execution: ExecutionOutcome::Accepted,
finalize: Some(FinalizeFacts {
stage: FinalizeStage::CreateReservation,
compensation,
}),
backend_compensation: Some(compensation),
correlation: None,
},
)
.unwrap();
failure.encode_storage().unwrap()
}
fn reserve(store: &SqliteWorkspaceStore) -> WorkerCreateReservation {
store.with_conn(|conn| {
conn.execute_batch("INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at)
VALUES('failure-owner','user','failure-owner','Owner','1','1');
INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at)
VALUES('failure-workspace','failure-owner','Failure workspace','active','1','1');
INSERT INTO workspace_memory_settings(workspace_id,language,created_at,updated_at)
VALUES('failure-workspace','en','1','1');")?;
Ok(())
}).unwrap();
store
.reserve_worker_create(
WORKSPACE,
RUNTIME,
"manual:startup-failure",
"sha256:startup-failure",
Some("subjektiv:failed-create"),
&store.get_workspace_memory_settings(WORKSPACE).unwrap(),
)
.unwrap()
}
// Snapshot every non-result reservation field plus resource keys and leases, so
// a save cannot silently acquire lifecycle effects as new columns are added.
fn non_result_snapshot(store: &SqliteWorkspaceStore) -> Vec<Vec<Value>> {
store.with_conn(|conn| {
let columns = table_columns(conn, "worker_create_reservations")?
.into_iter()
.filter(|name| name != "startup_failure_json")
.map(|name| format!("\"{name}\""))
.collect::<Vec<_>>()
.join(",");
let queries = [
format!("SELECT {columns} FROM worker_create_reservations ORDER BY workspace_id,allocation_key"),
"SELECT * FROM workspace_resource_keys ORDER BY workspace_id,resource_kind,resource_id".into(),
"SELECT * FROM workspace_resource_key_counters ORDER BY workspace_id,resource_kind".into(),
"SELECT * FROM worker_singleton_owners ORDER BY workspace_id,singleton_key".into(),
];
let mut values = Vec::new();
for query in queries {
let mut statement = conn.prepare(&query)?;
let count = statement.column_count();
values.extend(statement.query_map([], |row| {
(0..count).map(|i| row.get::<_, Value>(i)).collect::<rusqlite::Result<Vec<_>>>()
})?.collect::<rusqlite::Result<Vec<_>>>()?);
}
Ok(values)
}).unwrap()
}
fn save(
store: &SqliteWorkspaceStore,
reservation: &WorkerCreateReservation,
previous: Option<&str>,
encoded: &str,
) -> Result<bool> {
store.save_worker_create_failure(
WORKSPACE,
RUNTIME,
reservation.worker_id,
&reservation.create_fingerprint,
previous,
encoded,
)
}
#[test]
fn worker_create_failure_survives_restart_and_compensation_without_changing_lifecycle() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("server.db");
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let reservation = reserve(&store);
assert_eq!(
store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap(),
None
);
let original = non_result_snapshot(&store);
let encoded = encoded_failure(CompensationOutcome::Unknown);
assert!(save(&store, &reservation, None, &encoded).unwrap());
assert_eq!(non_result_snapshot(&store), original);
drop(store);
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
assert_eq!(
store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap()
.as_deref(),
Some(encoded.as_str())
);
store
.fail_worker_create_reservation(
WORKSPACE,
RUNTIME,
reservation.worker_id,
&reservation.create_fingerprint,
)
.unwrap();
assert_eq!(
store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap()
.as_deref(),
Some(encoded.as_str())
);
let removed = non_result_snapshot(&store);
let compensated = encoded_failure(CompensationOutcome::Completed);
assert!(save(&store, &reservation, Some(&encoded), &compensated).unwrap());
assert_eq!(non_result_snapshot(&store), removed);
drop(store);
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let raw = store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap()
.unwrap();
assert_eq!(raw, compensated);
let DecodeStoredWorkerCreateFailure::Known(failure) = decode_stored_worker_create_failure(&raw)
else {
panic!("saved result must decode")
};
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
assert_eq!(
failure.operation().finalize.unwrap().compensation,
CompensationOutcome::Completed
);
}
#[test]
fn worker_create_failure_identity_and_exact_previous_bytes_fence_every_write() {
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
let reservation = reserve(&store);
let first = encoded_failure(CompensationOutcome::Unknown);
let next = encoded_failure(CompensationOutcome::Completed);
assert!(save(&store, &reservation, None, &first).unwrap());
let original = non_result_snapshot(&store);
for (workspace, runtime, worker, fingerprint, previous) in [
(
"other",
RUNTIME,
reservation.worker_id,
reservation.create_fingerprint.as_str(),
Some(first.as_str()),
),
(
WORKSPACE,
"other",
reservation.worker_id,
reservation.create_fingerprint.as_str(),
Some(first.as_str()),
),
(
WORKSPACE,
RUNTIME,
WorkerId::now_v7(),
reservation.create_fingerprint.as_str(),
Some(first.as_str()),
),
(
WORKSPACE,
RUNTIME,
reservation.worker_id,
"wrong",
Some(first.as_str()),
),
(
WORKSPACE,
RUNTIME,
reservation.worker_id,
reservation.create_fingerprint.as_str(),
None,
),
(
WORKSPACE,
RUNTIME,
reservation.worker_id,
reservation.create_fingerprint.as_str(),
Some(next.as_str()),
),
] {
assert!(
!store
.save_worker_create_failure(
workspace,
runtime,
worker,
fingerprint,
previous,
&next
)
.unwrap()
);
assert_eq!(
store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap()
.as_deref(),
Some(first.as_str())
);
}
let equivalent_but_different_bytes = format!("{first} ");
assert!(
!save(
&store,
&reservation,
Some(&equivalent_but_different_bytes),
&next
)
.unwrap()
);
for (workspace, runtime, worker) in [
("other", RUNTIME, reservation.worker_id),
(WORKSPACE, "other", reservation.worker_id),
(WORKSPACE, RUNTIME, WorkerId::now_v7()),
] {
assert_eq!(
store
.worker_create_failure(workspace, runtime, worker)
.unwrap(),
None
);
}
assert_eq!(non_result_snapshot(&store), original);
assert!(save(&store, &reservation, Some(&first), &next).unwrap());
}
#[test]
fn worker_create_failure_failed_write_rolls_back_result_and_all_other_fields() {
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
let reservation = reserve(&store);
let first = encoded_failure(CompensationOutcome::Unknown);
let next = encoded_failure(CompensationOutcome::Completed);
assert!(save(&store, &reservation, None, &first).unwrap());
let original = non_result_snapshot(&store);
store.with_conn(|conn| {
conn.execute_batch("CREATE TRIGGER interrupt_startup_failure AFTER UPDATE OF startup_failure_json ON worker_create_reservations
BEGIN UPDATE worker_create_reservations SET state='removed';
SELECT RAISE(ABORT,'injected startup failure write interruption'); END;")?;
Ok(())
}).unwrap();
assert!(save(&store, &reservation, Some(&first), &next).is_err());
assert_eq!(non_result_snapshot(&store), original);
assert_eq!(
store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap()
.as_deref(),
Some(first.as_str())
);
store
.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER interrupt_startup_failure")?;
Ok(())
})
.unwrap();
assert!(save(&store, &reservation, Some(&first), &next).unwrap());
}
#[test]
fn worker_create_failure_rejects_unknown_corrupt_and_oversized_writes_but_reads_raw_history() {
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
let reservation = reserve(&store);
let first = encoded_failure(CompensationOutcome::Unknown);
assert!(save(&store, &reservation, None, &first).unwrap());
let mut unknown: serde_json::Value = serde_json::from_str(&first).unwrap();
unknown["version"] = serde_json::json!(65535);
let unknown = serde_json::to_string(&unknown).unwrap();
assert!(matches!(
decode_stored_worker_create_failure(&unknown),
DecodeStoredWorkerCreateFailure::Unsupported
));
for invalid in [
unknown.as_str(),
"not JSON",
&" ".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1),
] {
assert!(save(&store, &reservation, Some(&first), invalid).is_err());
assert_eq!(
store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap()
.as_deref(),
Some(first.as_str())
);
}
// Simulate a retained newer writer/corrupt row, without interpreting prose.
for raw in [unknown.as_str(), "not JSON"] {
store
.with_conn(|conn| {
conn.execute(
"UPDATE worker_create_reservations SET startup_failure_json=?1",
[raw],
)?;
Ok(())
})
.unwrap();
assert_eq!(
store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap()
.as_deref(),
Some(raw)
);
}
assert!(matches!(
decode_stored_worker_create_failure("not JSON"),
DecodeStoredWorkerCreateFailure::Corrupt(_)
));
store
.with_conn(|conn| {
conn.execute_batch("PRAGMA ignore_check_constraints=ON;")?;
conn.execute(
"UPDATE worker_create_reservations SET startup_failure_json=?1",
["x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1)],
)?;
conn.execute_batch("PRAGMA ignore_check_constraints=OFF;")?;
Ok(())
})
.unwrap();
assert!(
store
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.is_err()
);
}
fn result_column_signature(conn: &Connection) -> Vec<(String, String, bool, Option<String>)> {
conn.prepare("SELECT name,type,\"notnull\",dflt_value FROM pragma_table_info('worker_create_reservations') ORDER BY name").unwrap()
.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))).unwrap()
.collect::<rusqlite::Result<Vec<_>>>().unwrap()
}
fn assert_result_constraints(conn: &Connection, worker: WorkerId) {
for accepted in [
"x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES),
"é".repeat(WORKER_CREATE_FAILURE_MAX_BYTES / 2),
] {
conn.execute(
"UPDATE worker_create_reservations SET startup_failure_json=?1 WHERE worker_id=?2",
params![accepted, worker.to_string()],
)
.unwrap();
}
for rejected in [
"x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1),
"é".repeat(WORKER_CREATE_FAILURE_MAX_BYTES / 2 + 1),
] {
assert!(
conn.execute(
"UPDATE worker_create_reservations SET startup_failure_json=?1 WHERE worker_id=?2",
params![rejected, worker.to_string()]
)
.is_err()
);
}
assert!(
conn.execute(
"UPDATE worker_create_reservations SET startup_failure_json=x'00' WHERE worker_id=?1",
[worker.to_string()]
)
.is_err()
);
conn.execute(
"UPDATE worker_create_reservations SET startup_failure_json=NULL WHERE worker_id=?1",
[worker.to_string()],
)
.unwrap();
verify_worker_create_failure_schema(conn).unwrap();
}
#[test]
fn worker_create_failure_oldest_upgrade_preserves_legacy_null_and_matches_fresh_constraints() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("oldest.db");
tests::prepare_schema_v50(&path, Some("workspace-a"));
let legacy_worker = WorkerId::now_v7();
let conn = Connection::open(&path).unwrap();
conn.execute("INSERT INTO worker_create_reservations(workspace_id,allocation_key,worker_id,runtime_id,create_fingerprint,state,created_at,updated_at)
VALUES('workspace-a','legacy-allocation',?1,'shared','legacy-fingerprint','reserved','1','1')", [legacy_worker.to_string()]).unwrap();
drop(conn);
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
assert_eq!(
upgraded
.worker_create_failure("workspace-a", "shared", legacy_worker)
.unwrap(),
None
);
upgraded.with_conn(|conn| {
let saved: (String, String, String) = conn.query_row("SELECT create_fingerprint,state,created_at FROM worker_create_reservations WHERE worker_id=?1", [legacy_worker.to_string()], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
assert_eq!(saved, ("legacy-fingerprint".into(), "reserved".into(), "1".into()));
assert_result_constraints(conn, legacy_worker);
Ok(())
}).unwrap();
let reservation = reserve(&upgraded);
let encoded = encoded_failure(CompensationOutcome::Residual);
assert!(save(&upgraded, &reservation, None, &encoded).unwrap());
drop(upgraded);
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
assert_eq!(
upgraded
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
.unwrap()
.as_deref(),
Some(encoded.as_str())
);
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
let reservation = reserve(&fresh);
fresh
.with_conn(|conn| {
assert_result_constraints(conn, reservation.worker_id);
Ok(())
})
.unwrap();
assert_eq!(
upgraded
.with_conn(|conn| Ok(result_column_signature(conn)))
.unwrap(),
fresh
.with_conn(|conn| Ok(result_column_signature(conn)))
.unwrap()
);
}
#[test]
fn worker_create_failure_migration_failure_rolls_back_column_and_receipt() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("upgrade.db");
tests::prepare_retained_schema(&path, 123);
let conn = Connection::open(&path).unwrap();
configure_sqlite(&conn).unwrap();
conn.execute_batch("CREATE TRIGGER interrupt_startup_failure_migration BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=124
BEGIN SELECT RAISE(ABORT,'injected startup failure migration interruption'); END;").unwrap();
assert!(migrate_worker_create_failure_to_v124(&conn).is_err());
assert_eq!(current_schema_version(&conn).unwrap(), 123);
assert!(!column_exists(&conn, "worker_create_reservations", "startup_failure_json").unwrap());
conn.execute_batch("DROP TRIGGER interrupt_startup_failure_migration")
.unwrap();
migrate_worker_create_failure_to_v124(&conn).unwrap();
verify_worker_create_failure_schema(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 124);
drop(conn);
SqliteWorkspaceStore::open_for_test(&path).unwrap();
}