diff --git a/Cargo.lock b/Cargo.lock index e288d73ea..958518a8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,6 +199,7 @@ dependencies = [ "api-macros-impl", "axum", "bytes", + "error-contract", "futures", "oas3", "reqwest", @@ -4325,12 +4326,15 @@ name = "runtime-api" version = "0.1.0" dependencies = [ "api-macros", + "error-contract", "manifest", "plugin-plan", "protocol", + "schemars", "serde", "serde_json", "session-store", + "sha2 0.11.0", "tokio", "workdir", ] @@ -4764,6 +4768,7 @@ name = "server-api" version = "0.1.0" dependencies = [ "api-macros", + "error-contract", "feature-permissions", "memory", "merge-request", diff --git a/crates/api-macros-impl/src/lib.rs b/crates/api-macros-impl/src/lib.rs index 655549a7a..30105ca0e 100644 --- a/crates/api-macros-impl/src/lib.rs +++ b/crates/api-macros-impl/src/lib.rs @@ -53,6 +53,7 @@ enum Method { #[derive(Clone, Copy, Default)] struct AdapterConfig { reqwest: bool, + client_error_hook: bool, axum: bool, openapi: bool, } @@ -64,9 +65,10 @@ impl Parse for AdapterConfig { for adapter in adapters { match adapter.to_string().as_str() { "reqwest" if !config.reqwest => config.reqwest = true, + "client_error_hook" if !config.client_error_hook => config.client_error_hook = true, "axum" if !config.axum => config.axum = true, "openapi" if !config.openapi => config.openapi = true, - "reqwest" | "axum" | "openapi" => { + "reqwest" | "axum" | "openapi" | "client_error_hook" => { return Err(syn::Error::new(adapter.span(), "duplicate API adapter")); } _ => { @@ -77,6 +79,12 @@ impl Parse for AdapterConfig { } } } + if config.client_error_hook && !config.reqwest { + return Err(syn::Error::new( + Span::call_site(), + "`client_error_hook` requires the `reqwest` adapter", + )); + } Ok(config) } } @@ -121,6 +129,8 @@ struct RouteArgs { responses: Vec, error_status: Option, additional_error_statuses: Vec, + additional_error_content_types: Option>, + client_error_hook: Option, bearer_auth: Option, browser_auth: Option, normalize_body_errors: Option, @@ -310,6 +320,8 @@ impl Parse for RouteArgs { responses: Vec::new(), error_status: None, additional_error_statuses: Vec::new(), + additional_error_content_types: None, + client_error_hook: None, bearer_auth: None, browser_auth: None, normalize_body_errors: None, @@ -379,6 +391,25 @@ impl Parse for RouteArgs { } } } + "additional_error_content_types" => { + let content; + syn::bracketed!(content in input); + let values = Punctuated::::parse_terminated(&content)? + .into_iter() + .collect(); + set_once( + &mut result.additional_error_content_types, + values, + &key, + "additional_error_content_types", + )?; + } + "client_error_hook" => set_once( + &mut result.client_error_hook, + input.parse::()?, + &key, + "client_error_hook", + )?, "bearer_auth" => set_once( &mut result.bearer_auth, input.parse::()?, @@ -406,7 +437,7 @@ impl Parse for RouteArgs { _ => { return Err(syn::Error::new( key.span(), - "unsupported route option; expected operation_id, status, alternate_status, responses, error_status, additional_error_statuses, bearer_auth, browser_auth, normalize_body_errors, or openapi", + "unsupported route option; expected operation_id, status, alternate_status, responses, error_status, additional_error_statuses, additional_error_content_types, client_error_hook, bearer_auth, browser_auth, normalize_body_errors, or openapi", )); } } @@ -502,6 +533,8 @@ struct Operation { alternate_status: Option, error_status: Option, additional_error_statuses: Vec, + additional_error_content_types: Vec, + client_error_hook: bool, bearer_auth: bool, browser_auth: bool, normalize_body_errors: bool, @@ -546,7 +579,7 @@ fn normalize_api( for mut trait_item in std::mem::take(&mut item.items) { let operation = match &mut trait_item { TraitItem::Fn(method) => { - let operation = normalize_operation(method)?; + let operation = normalize_operation(method, adapters)?; retained_items.push(trait_item); operation } @@ -778,6 +811,8 @@ fn normalize_websocket_operation(declaration: &mut syn::TraitItemType) -> syn::R alternate_status: None, error_status: None, additional_error_statuses: Vec::new(), + additional_error_content_types: Vec::new(), + client_error_hook: false, bearer_auth: false, browser_auth: false, normalize_body_errors: false, @@ -796,7 +831,10 @@ fn validate_frame_type(ty: &Type) -> syn::Result<()> { } } -fn normalize_operation(method: &mut TraitItemFn) -> syn::Result { +fn normalize_operation( + method: &mut TraitItemFn, + adapters: AdapterConfig, +) -> syn::Result { validate_signature(method)?; let mut route = None; @@ -845,6 +883,53 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result { validate_named_body_type(ty, "error response")?; } + let client_error_hook = route + .client_error_hook + .as_ref() + .is_some_and(|value| value.value); + if client_error_hook { + let option = route.client_error_hook.as_ref().expect("enabled hook"); + if !adapters.reqwest { + return Err(syn::Error::new_spanned( + option, + "client_error_hook requires the `reqwest` adapter", + )); + } + if error_body.is_none() { + return Err(syn::Error::new_spanned( + option, + "client_error_hook requires a public error response type", + )); + } + } + + let mut additional_error_content_types = Vec::new(); + if let Some(content_types) = &route.additional_error_content_types { + if error_body.is_none() { + return Err(syn::Error::new_spanned( + &route.path, + "additional_error_content_types require a public error response type", + )); + } + if content_types.is_empty() { + return Err(syn::Error::new_spanned( + &route.path, + "additional_error_content_types must not be empty", + )); + } + for content_type in content_types { + let value = validate_error_content_type(content_type)?; + if value == "application/json" || additional_error_content_types.contains(&value) { + return Err(syn::Error::new_spanned( + content_type, + "additional error content types must be distinct from each other and application/json", + )); + } + additional_error_content_types.push(value); + } + additional_error_content_types.sort(); + } + let declared_responses = !route.responses.is_empty(); if declared_responses && (route.status.is_some() || route.alternate_status.is_some()) { return Err(syn::Error::new_spanned( @@ -1196,6 +1281,8 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result { alternate_status, error_status, additional_error_statuses, + additional_error_content_types, + client_error_hook, bearer_auth: route.bearer_auth.as_ref().is_some_and(|value| value.value), browser_auth: route.browser_auth.as_ref().is_some_and(|value| value.value), normalize_body_errors: route @@ -1206,6 +1293,43 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result { }) } +// Route media declarations are concrete type/subtype tokens, not media ranges or parameters. +fn validate_error_content_type(literal: &LitStr) -> syn::Result { + let value = literal.value().to_ascii_lowercase(); + let valid_token = |token: &str| { + !token.is_empty() + && token.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' + ) + }) + }; + if !value + .split_once('/') + .is_some_and(|(kind, subtype)| valid_token(kind) && valid_token(subtype)) + { + return Err(syn::Error::new_spanned( + literal, + "error content type must be a concrete type/subtype without parameters", + )); + } + Ok(value) +} + fn validate_signature(method: &TraitItemFn) -> syn::Result<()> { let sig = &method.sig; if sig.asyncness.is_none() { @@ -1815,17 +1939,27 @@ fn reqwest_adapter_tokens( } } }; + let public_error_decode = if api.adapters.client_error_hook || operation.client_error_hook { + quote! { + <#error_type as #api_crate::reqwest::HttpClientError>::decode_error(&__response) + .map_err(#api_crate::reqwest::ClientError::failure)? + } + } else { + quote! { + __response + .decode_json::<#error_type>(#api_crate::reqwest::DecodeKind::PublicError) + .map_err(|_| #api_crate::reqwest::ClientError::failure( + #api_crate::reqwest::ClientFailure::ErrorResponseDecode { + status: __status.as_u16(), + }, + ))? + } + }; let public_error = match operation.error_body.as_ref() { - Some(error) => quote! { + Some(_) => quote! { if (400..=599).contains(&__actual) { let __status = __response.status(); - let __error = __response - .decode_json::<#error>(#api_crate::reqwest::DecodeKind::PublicError) - .map_err(|_| #api_crate::reqwest::ClientError::failure( - #api_crate::reqwest::ClientFailure::ErrorResponseDecode { - status: __status.as_u16(), - }, - ))?; + let __error = #public_error_decode; return ::core::result::Result::Err( #api_crate::reqwest::ClientError::public(__status, __error), ); @@ -2098,7 +2232,7 @@ fn axum_adapter_tokens( status, rejection.body_text(), ); - return #api_crate::axum::error_response( + return #api_crate::axum::http_error_response( #api_crate::axum::status(status), error, ); @@ -2132,7 +2266,7 @@ fn axum_adapter_tokens( status, rejection.body_text(), ); - return #api_crate::axum::error_response( + return #api_crate::axum::http_error_response( #api_crate::axum::status(status), error, ); @@ -2218,7 +2352,7 @@ fn axum_adapter_tokens( let error = match operation.error_status { Some(_) => quote!({ let status = #api_crate::HttpError::status_code(&error); - #api_crate::axum::error_response(#api_crate::axum::status(status), error) + #api_crate::axum::http_error_response(#api_crate::axum::status(status), error) }), None => quote!({ let _ = error; #api_crate::axum::empty_response(#api_crate::axum::framework::StatusCode::INTERNAL_SERVER_ERROR) }), }; @@ -2315,110 +2449,115 @@ fn openapi_adapter_tokens( matches!(operation.transport, Transport::Http) && !operation.openapi_skip }) .map(|operation| { - let method = method_name(operation.method); - let path = &operation.path; - let operation_id = &operation.operation_id; - let parameters = operation.parameters.iter().filter_map(|parameter| { - if matches!(parameter.location, Location::Body | Location::Binary | Location::Extension) { - return None; - } - let ty = ¶meter.ty; - let name = ¶meter.wire_name; - let location = match parameter.location { - Location::Path => "path", - Location::Query => "query", - Location::Header => "header", - Location::Body | Location::Binary | Location::Extension => unreachable!(), - }; - Some(quote! { - operation.parameter::<#ty>(#name, #location)?; - }) - }); - let request = operation.request_body.as_ref().map(|body| match body.wire_kind { - RequestWireKind::Json => { - let ty = &body.ty; - quote! { - operation.request_body::<#ty>("application/json")?; - } - } - RequestWireKind::Binary => quote! { - operation.binary_request_body()?; - }, - }); - let success_responses = operation.success_responses.iter().enumerate().map(|(index, response)| { - let status = response.status; - let description = if index == 0 { - "Successful response" - } else { - "Alternate successful response" - }; - let body = match response.body.as_ref() { - Some(ty) if is_binary_body_type(ty) => quote! { - operation.binary_response(#status, #description)?; - }, - Some(ty) => quote! { - operation.response::<#ty>(#status, "application/json", #description)?; - }, - None => quote! { - operation.empty_response(#status, #description)?; - }, - }; - let headers = response.headers.iter().map(|header| { - let ty = &header.ty; - let name = &header.wire_name; - quote! { - operation.response_header::<#ty>(#status, #name)?; + let method = method_name(operation.method); + let path = &operation.path; + let operation_id = &operation.operation_id; + let parameters = operation.parameters.iter().filter_map(|parameter| { + if matches!( + parameter.location, + Location::Body | Location::Binary | Location::Extension + ) { + return None; } + let ty = ¶meter.ty; + let name = ¶meter.wire_name; + let location = match parameter.location { + Location::Path => "path", + Location::Query => "query", + Location::Header => "header", + Location::Body | Location::Binary | Location::Extension => unreachable!(), + }; + Some(quote! { + operation.parameter::<#ty>(#name, #location)?; + }) }); - quote! { - #body - #(#headers)* - } - }); - let error = operation - .error_status - .zip(operation.error_body.as_ref()) - .map(|(status, ty)| { - quote! { - operation.response::<#ty>(#status, "application/json", "Error response")?; - } - }); - let additional_errors = operation - .error_body - .as_ref() - .map(|ty| { - operation - .additional_error_statuses - .iter() - .map(|status| { + let request = operation + .request_body + .as_ref() + .map(|body| match body.wire_kind { + RequestWireKind::Json => { + let ty = &body.ty; quote! { - operation.response::<#ty>(#status, "application/json", "Error response")?; + operation.request_body::<#ty>("application/json")?; } - }) - .collect::>() - }) - .unwrap_or_default(); - let bearer_auth = operation.bearer_auth.then(|| { - quote! { operation.bearer_authentication(); } - }); - let browser_auth = operation.browser_auth.then(|| { - quote! { operation.browser_authentication(); } - }); + } + RequestWireKind::Binary => quote! { + operation.binary_request_body()?; + }, + }); + let success_responses = operation.success_responses.iter().enumerate().map( + |(index, response)| { + let status = response.status; + let description = if index == 0 { + "Successful response" + } else { + "Alternate successful response" + }; + let body = match response.body.as_ref() { + Some(ty) if is_binary_body_type(ty) => quote! { + operation.binary_response(#status, #description)?; + }, + Some(ty) => quote! { + operation.response::<#ty>(#status, "application/json", #description)?; + }, + None => quote! { + operation.empty_response(#status, #description)?; + }, + }; + let headers = response.headers.iter().map(|header| { + let ty = &header.ty; + let name = &header.wire_name; + quote! { + operation.response_header::<#ty>(#status, #name)?; + } + }); + quote! { + #body + #(#headers)* + } + }, + ); + let errors = operation + .error_body + .as_ref() + .map(|ty| { + let additional_content_types = &operation.additional_error_content_types; + operation + .error_status + .into_iter() + .chain(operation.additional_error_statuses.iter().copied()) + .map(|status| { + quote! { + operation.response_content_types::<#ty>( + #status, + &["application/json", #(#additional_content_types),*], + "Error response", + )?; + } + }) + .collect::>() + }) + .unwrap_or_default(); + let bearer_auth = operation.bearer_auth.then(|| { + quote! { operation.bearer_authentication(); } + }); + let browser_auth = operation.browser_auth.then(|| { + quote! { operation.browser_authentication(); } + }); - quote! { - { - let mut operation = builder.operation(#method, #path, #operation_id)?; - #(#parameters)* - #bearer_auth - #browser_auth - #request - #(#success_responses)* - #error - #(#additional_errors)* - operation.finish()?; + quote! { + { + let mut operation = builder.operation(#method, #path, #operation_id)?; + #(#parameters)* + #bearer_auth + #browser_auth + #request + #(#success_responses)* + #(#errors)* + operation.finish()?; + } } - } - }); + }); quote! { #[doc = "Build this trait's deterministic, deployment-independent OpenAPI 3.1 document."] @@ -2660,6 +2799,13 @@ fn metadata_tokens( } else { quote!(#api_crate::WireKind::Empty) }; + let content_types = if response.body.as_ref().is_some_and(is_binary_body_type) { + quote!(&["application/octet-stream"]) + } else if response.body.is_some() { + quote!(&["application/json"]) + } else { + quote!(&[]) + }; let headers = response.headers.iter().map(|header| { let wire_name = &header.wire_name; let ty = &header.ty; @@ -2674,6 +2820,7 @@ fn metadata_tokens( #api_crate::ResponseMetadata { status: #status, body: #api_crate::BodyMetadata { wire_kind: #wire_kind }, + content_types: #content_types, headers: &[#(#headers),*], } } @@ -2683,10 +2830,12 @@ fn metadata_tokens( .into_iter() .chain(operation.additional_error_statuses.iter().copied()); let error_responses = error_statuses.map(|status| { + let additional_content_types = &operation.additional_error_content_types; quote! { #api_crate::ResponseMetadata { status: #status, body: #api_crate::BodyMetadata { wire_kind: #api_crate::WireKind::Json }, + content_types: &["application/json", #(#additional_content_types),*], headers: &[], } } diff --git a/crates/api-macros/Cargo.toml b/crates/api-macros/Cargo.toml index b56e64eb8..75c63af53 100644 --- a/crates/api-macros/Cargo.toml +++ b/crates/api-macros/Cargo.toml @@ -7,6 +7,7 @@ publish = false [features] default = [] +error-contract = ["dep:error-contract"] axum = ["dep:axum", "dep:serde"] openapi = ["dep:schemars", "dep:serde_json", "api-macros-impl/openapi"] reqwest = [ @@ -20,6 +21,7 @@ reqwest = [ [dependencies] api-macros-impl = { path = "../api-macros-impl" } +error-contract = { path = "../error-contract", optional = true } bytes.workspace = true axum = { workspace = true, optional = true } futures = { workspace = true, optional = true } diff --git a/crates/api-macros/README.md b/crates/api-macros/README.md index 7c8687e21..1078f0f49 100644 --- a/crates/api-macros/README.md +++ b/crates/api-macros/README.md @@ -6,4 +6,6 @@ Defines transport-independent API metadata from public traits. The same declarat - [`api`] — API declaration and validation - [`ApiContract`], [`Operation`], and [`WebSocketOperation`] — typed operation contracts +- [`HttpError`] — opt-in public error encoding, with unchanged JSON defaults - `axum`, `reqwest`, and `openapi` — adapters and document generation enabled by their respective features +- `http` — projected error-contract response conversion and bounded reception, enabled by `error-contract` without enabling a framework diff --git a/crates/api-macros/src/axum.rs b/crates/api-macros/src/axum.rs index e235031d9..deea195ab 100644 --- a/crates/api-macros/src/axum.rs +++ b/crates/api-macros/src/axum.rs @@ -96,6 +96,34 @@ pub fn error_response( response } +/// Apply the optional public encoding hook, retaining the original server-local error. +/// +/// Invalid hook metadata fails closed with an empty 500 response and no typed extension. +/// A hook failure must not expose a potentially private legacy serialization. +pub fn http_error_response( + status: StatusCode, + error: T, +) -> Response { + let Some(encoded) = error.custom_response() else { + return error_response(status, error); + }; + let Ok(status) = StatusCode::from_u16(encoded.status) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(content_type) = axum::http::HeaderValue::from_str(encoded.content_type) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + if !(status.is_client_error() || status.is_server_error()) || content_type.is_empty() { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + let mut response = (status, encoded.body.into_bytes()).into_response(); + response + .headers_mut() + .insert(axum::http::header::CONTENT_TYPE, content_type); + response.extensions_mut().insert(std::sync::Arc::new(error)); + response +} + /// Emit an exact binary response with the contract status and octet-stream content type. pub fn binary_response(status: StatusCode, value: crate::BinaryBody) -> Response { ( diff --git a/crates/api-macros/src/http.rs b/crates/api-macros/src/http.rs new file mode 100644 index 000000000..59697479f --- /dev/null +++ b/crates/api-macros/src/http.rs @@ -0,0 +1,56 @@ +//! Shared HTTP error-contract adapter, independent of Axum and Reqwest. +//! +//! Only Browser-projected failures can be encoded here. Receiving a valid failure does not +//! grant permission to forward it: domains must reproject with current authorization. + +pub use error_contract::problem::{ApiBoundary, ProblemResponse}; +use error_contract::{ + Browser, Budget, ContractError, DecodeFailure, ProjectedFailure, WireDetails, +}; + +/// Encode a projected failure or a static, non-domain 500 fallback under the same budget. +/// +/// `safe_instance` must be selected by the caller's disclosure policy (for example a local +/// opaque occurrence URI), never copied from a request URI containing credentials or queries. +/// URI syntax validation is not disclosure authorization. If even the static fallback exceeds +/// the budget, return an error instead of truncating JSON or silently exceeding the limit. +pub fn problem_response( + projected: &ProjectedFailure, + boundary: &impl ApiBoundary, + safe_instance: &str, + budget: Budget, +) -> Result { + error_contract::problem::to_problem_details(projected, boundary, safe_instance, budget) + .or_else(|_| error_contract::problem::problem_fallback(budget)) +} + +impl From for crate::HttpErrorResponse { + fn from(response: ProblemResponse) -> Self { + Self { + status: response.status, + content_type: response.content_type, + body: response.body().to_vec().into(), + } + } +} + +/// Normalize a bounded HTTP error using actual transport metadata, not a body-claimed status. +/// +/// Unsupported schema/reason, corrupt input, and `Corrupt(TooLarge)` remain distinct. None of +/// these results, including a known cause, establish remote execution or retry permission. +/// Streaming callers must also bound collection before calling this byte-slice adapter. +pub fn decode_problem_response( + input: &[u8], + actual_status: u16, + content_type: &str, + boundary: &impl ApiBoundary, + budget: Budget, +) -> DecodeFailure { + error_contract::problem::decode_problem_details( + input, + actual_status, + content_type, + boundary, + budget, + ) +} diff --git a/crates/api-macros/src/lib.rs b/crates/api-macros/src/lib.rs index 0c9bfa21f..2b74970e9 100644 --- a/crates/api-macros/src/lib.rs +++ b/crates/api-macros/src/lib.rs @@ -14,6 +14,16 @@ extern crate self as api_macros; /// leave authentication, authorization, body limits, and timeouts to the enclosing application. /// Declarations without adapters do not depend on an HTTP framework. /// +/// `#[api(reqwest, client_error_hook)]` delegates error decoding to the error type's +/// `reqwest::HttpClientError` implementation. A mixed-error API can instead set +/// `client_error_hook = true` on individual HTTP operations. Either flag enables the hook; +/// a method-level `false` does not override a trait-wide opt-in. Without opt-in, client-only +/// errors need only serde deserialization; server traits and codecs are not imposed on them. +/// +/// `additional_error_content_types = ["application/problem+json"]` on an HTTP operation +/// declares extra representations for every declared error status, alongside legacy JSON. +/// Each representation uses the error type's schema; the producer still owns its encoding. +/// /// This declaration checks that a Rust trait and typed operation markers can be defined together. /// /// ``` @@ -71,6 +81,8 @@ pub use api_macros_impl::api; #[cfg(feature = "axum")] pub mod axum; +#[cfg(feature = "error-contract")] +pub mod http; #[cfg(feature = "openapi")] pub mod openapi; #[cfg(feature = "reqwest")] @@ -213,6 +225,8 @@ pub struct ResponseHeaderMetadata { pub struct ResponseMetadata { pub status: u16, pub body: BodyMetadata, + /// Declared media types, empty for bodyless responses. JSON remains the default error media. + pub content_types: &'static [&'static str], pub headers: &'static [ResponseHeaderMetadata], } @@ -272,6 +286,24 @@ pub trait WebSocketOperation { /// service's typed 401/403/404/409 responses without introducing duplicate route handlers. pub trait HttpError { fn status_code(&self) -> u16; + + /// Opt in to an already encoded public response instead of the legacy JSON representation. + /// The adapter retains the original error as a server-local `Arc` extension. + /// Returning `None` preserves the existing JSON/status behavior exactly. + fn custom_response(&self) -> Option { + None + } +} + +/// An explicitly encoded HTTP error, independent of any server or client framework. +/// +/// Producers own disclosure and byte limits. Adapters reject non-error statuses and invalid +/// content types rather than substituting the legacy representation, which may not be public. +#[derive(Debug)] +pub struct HttpErrorResponse { + pub status: u16, + pub content_type: &'static str, + pub body: BinaryBody, } /// Successful JSON response that selects one of an operation's declared success statuses. diff --git a/crates/api-macros/src/openapi.rs b/crates/api-macros/src/openapi.rs index 58fc70e82..660c15703 100644 --- a/crates/api-macros/src/openapi.rs +++ b/crates/api-macros/src/openapi.rs @@ -39,6 +39,19 @@ pub trait OpenApiSchema: JsonSchema { fn openapi_schema_name() -> Cow<'static, str> { Self::schema_name() } + + /// Select the exact wire schema for an explicitly declared response media type. + /// Custom error adapters may have distinct JSON and Problem Details bodies; + /// this avoids claiming an ambiguous untagged body union for either medium. + fn response_schema( + builder: &mut OpenApiBuilder, + _content_type: &str, + ) -> Result + where + Self: Sized, + { + builder.schema_ref::() + } } macro_rules! impl_openapi_schema { @@ -256,7 +269,8 @@ impl OpenApiBuilder { Ok(OpenApiDocument(document)) } - fn schema_ref(&mut self) -> Result { + /// Register an explicitly asserted wire schema, retaining collision and bounds checks. + pub fn schema_ref(&mut self) -> Result { let name = T::openapi_schema_name().into_owned(); validate_component_name(&name)?; @@ -411,13 +425,39 @@ impl OpenApiOperation<'_> { content_type: &'static str, description: &'static str, ) -> Result<(), OpenApiError> { - let schema = self.parent.schema_ref::()?; + self.response_content_types::(status, &[content_type], description) + } + + /// Declare one response schema under several explicit representations, atomically. + /// This does not change conflict detection for repeated status declarations. + pub fn response_content_types( + &mut self, + status: u16, + content_types: &[&str], + description: &'static str, + ) -> Result<(), OpenApiError> { + let mut content = Map::new(); + for content_type in content_types { + let schema = T::response_schema(self.parent, content_type)?; + if content + .insert((*content_type).to_owned(), json!({ "schema": schema })) + .is_some() + { + return Err(OpenApiError::InvalidContract(format!( + "duplicate response content type for `{}`", + self.operation_id + ))); + } + } + if content.is_empty() { + return Err(OpenApiError::InvalidContract(format!( + "response content types are empty for `{}`", + self.operation_id + ))); + } self.insert_response( status, - json!({ - "description": description, - "content": { content_type: { "schema": schema } }, - }), + json!({ "description": description, "content": content }), ) } diff --git a/crates/api-macros/src/reqwest.rs b/crates/api-macros/src/reqwest.rs index 136c953bc..c817d60ee 100644 --- a/crates/api-macros/src/reqwest.rs +++ b/crates/api-macros/src/reqwest.rs @@ -482,6 +482,17 @@ pub fn encode_binary(value: crate::BinaryBody) -> EncodedBody { EncodedBody::binary(value) } +/// Explicit decoder for APIs declared with `#[api(reqwest, client_error_hook)]` or an +/// operation-level `client_error_hook = true` option. +/// +/// There is deliberately no blanket `Deserialize` or `HttpError` bound: legacy client-only +/// errors keep serde decoding without implementing this trait. Opted-in decoders receive the +/// actual status/headers and bytes collected under the client's configured limit. Returning +/// a failure does not imply anything about remote execution or authorize a new request. +pub trait HttpClientError: Sized { + fn decode_error(response: &ReceivedResponse) -> Result; +} + /// Response received under the configured byte limit. Its content is always redacted in Debug. pub struct ReceivedResponse { status: StatusCode, @@ -494,6 +505,48 @@ impl ReceivedResponse { self.status } + /// Borrow bytes already collected under the configured response limit. + pub fn body(&self) -> &[u8] { + &self.bytes + } + + /// Require exactly one valid Content-Type header; ambiguous metadata must not select a codec. + pub fn content_type(&self) -> Result<&str, ClientFailure> { + let name = "content-type"; + let mut values = self.headers.get_all(name).iter(); + let value = values + .next() + .ok_or(ClientFailure::MissingResponseHeader { name })?; + if values.next().is_some() { + return Err(ClientFailure::DuplicateResponseHeader { name }); + } + value + .to_str() + .map_err(|_| ClientFailure::InvalidResponseHeader { name }) + } + + /// Decode without promoting received data to a public projection or execution result. + /// Transport collection and contract budgets are enforced independently. + #[cfg(feature = "error-contract")] + pub fn decode_problem( + &self, + boundary: &impl crate::http::ApiBoundary, + budget: error_contract::Budget, + ) -> error_contract::DecodeFailure { + match self.content_type() { + Ok(content_type) => crate::http::decode_problem_response( + &self.bytes, + self.status.as_u16(), + content_type, + boundary, + budget, + ), + Err(_) => { + error_contract::DecodeFailure::Corrupt(error_contract::ContractError::Invalid) + } + } + } + pub fn parse_header(&self, name: &'static str) -> Result where T: std::str::FromStr, @@ -555,13 +608,32 @@ pub enum ClientFailure { Connect, Transport, Timeout, - ResponseTooLarge { limit: usize }, - UnexpectedStatus { expected: u16, actual: u16 }, - ErrorResponseDecode { status: u16 }, - Decode { kind: DecodeKind }, - MissingResponseHeader { name: &'static str }, - DuplicateResponseHeader { name: &'static str }, - InvalidResponseHeader { name: &'static str }, + ResponseTooLarge { + limit: usize, + }, + UnexpectedStatus { + expected: u16, + actual: u16, + }, + ErrorResponseDecode { + status: u16, + }, + /// A bounded, structurally valid error used an unsupported schema or reason. + UnsupportedErrorResponse { + status: u16, + }, + Decode { + kind: DecodeKind, + }, + MissingResponseHeader { + name: &'static str, + }, + DuplicateResponseHeader { + name: &'static str, + }, + InvalidResponseHeader { + name: &'static str, + }, UnexpectedBody, } @@ -588,6 +660,12 @@ impl fmt::Display for ClientFailure { "expected HTTP status {expected}, received {actual}" ) } + Self::UnsupportedErrorResponse { status } => { + write!( + formatter, + "HTTP {status} error response contract was unsupported" + ) + } Self::ErrorResponseDecode { status } => { write!(formatter, "HTTP {status} error response JSON was malformed") } diff --git a/crates/api-macros/tests/error_hooks.rs b/crates/api-macros/tests/error_hooks.rs new file mode 100644 index 000000000..f75176307 --- /dev/null +++ b/crates/api-macros/tests/error_hooks.rs @@ -0,0 +1,463 @@ +//! Connect generated Axum responses to generated Reqwest opt-in decoding over isolated loopback. +//! Domain disclosure is a fixture; HTTP encoding, bounded reads and metadata checks are production. +#![cfg(all(feature = "axum", feature = "reqwest", feature = "error-contract"))] + +use std::{sync::Arc, time::Duration}; + +use api_macros::{ + HttpErrorResponse, api, + reqwest::{ClientError, ClientFailure, HttpClientError, ReceivedResponse}, +}; +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, + response::IntoResponse, + routing::get, +}; +use error_contract::{ + Browser, Budget, Category, ContractError, DecodeFailure, Disclosure, FailureCause, + FailureFacts, FailureIdentity, ProjectFor, ProjectedFailure, WireDetails, project_failure, +}; +use serde::{Deserialize, Serialize}; +use tower::ServiceExt; + +#[derive(Serialize, Deserialize)] +pub struct Payload { + value: String, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct Denied { + required_operation: String, +} +impl FailureCause for Denied { + fn identity(&self) -> FailureIdentity { + FailureIdentity { + domain: "subject", + reason: "operation_denied", + category: Category::Authorization, + details_version: 1, + } + } + fn validate(&self) -> Result<(), ContractError> { + if self.required_operation == "session.record" { + Ok(()) + } else { + Err(ContractError::InvalidDetails) + } + } +} +impl WireDetails for Denied { + fn supports(domain: &str, reason: &str, version: u16) -> bool { + domain == "subject" && reason == "operation_denied" && version == 1 + } +} +impl ProjectFor for Denied { + type Authorization = (); + type Details = Self; + fn disclose(&self, _: &FailureFacts, _: &()) -> Result, ContractError> { + Ok(Disclosure::new( + self.clone(), + "Subject cannot record a session.", + )) + } +} +struct Boundary; +impl api_macros::http::ApiBoundary for Boundary { + fn classify(&self, _: &Denied) -> error_contract::problem::ProblemClass { + error_contract::problem::ProblemClass::authorization_denied() + } +} +fn budget() -> Budget { + Budget::new(4096, 4096) +} +fn projected() -> ProjectedFailure { + project_failure( + &FailureFacts::new(Denied { + required_operation: "session.record".into(), + }) + .unwrap(), + &(), + ) + .unwrap() +} +fn encoded() -> HttpErrorResponse { + api_macros::http::problem_response(&projected(), &Boundary, "urn:yoi:failure:local-1", budget()) + .unwrap() + .into() +} + +// Not Clone: the original must be retained without duplicating a domain error. +#[derive(Serialize)] +pub struct ServerError { + legacy: &'static str, + #[serde(skip)] + mode: u8, +} +impl api_macros::HttpError for ServerError { + fn status_code(&self) -> u16 { + 422 + } + fn custom_response(&self) -> Option { + match self.mode { + 0 => Some(encoded()), + 1 => None, + mode => Some(HttpErrorResponse { + status: if mode == 2 { 200 } else { 403 }, + content_type: if mode == 3 { + "bad\ncontent-type" + } else { + "application/problem+json" + }, + body: b"private-hook-body".to_vec().into(), + }), + } + } +} +impl api_macros::HttpRequestError for ServerError { + fn from_request_rejection(_: u16, _: String) -> Self { + Self { + legacy: "never serialize", + mode: 0, + } + } +} +#[api(axum)] +pub trait HookServer { + #[get("/problem", status = 200, error_status = 422)] + async fn problem(&self) -> Result; + #[post("/reject", error_status = 422, normalize_body_errors = true)] + async fn reject(&self, #[body] input: Payload) -> Result; + #[post("/reject-binary", error_status = 422, normalize_body_errors = true)] + async fn reject_binary( + &self, + #[binary] input: api_macros::BinaryBody, + ) -> Result; +} +pub struct Service; +impl HookServer for Service { + async fn problem(&self) -> Result { + Err(ServerError { + legacy: "never serialize", + mode: 0, + }) + } + async fn reject(&self, input: Payload) -> Result { + Ok(input) + } + async fn reject_binary(&self, _input: api_macros::BinaryBody) -> Result { + Ok(Payload { value: "ok".into() }) + } +} + +// Neither Deserialize nor HttpError: client hooks impose no server or serde error bounds. +pub struct DecodedError { + operation: String, +} +impl HttpClientError for DecodedError { + fn decode_error(response: &ReceivedResponse) -> Result { + let status = response.status().as_u16(); + match response.decode_problem::(&Boundary, budget()) { + DecodeFailure::Known(failure) => Ok(Self { + operation: failure.facts().cause().required_operation.clone(), + }), + DecodeFailure::Unsupported(_) => { + Err(ClientFailure::UnsupportedErrorResponse { status }) + } + DecodeFailure::Corrupt(ContractError::TooLarge) => { + Err(ClientFailure::ResponseTooLarge { + limit: budget().bytes(), + }) + } + DecodeFailure::Corrupt(_) => Err(ClientFailure::ErrorResponseDecode { status }), + } + } +} +#[api(reqwest)] +pub trait HookClient { + #[get("/problem", status = 200, error_status = 422, client_error_hook = true)] + async fn problem(&self) -> Result; + #[get( + "/wire/{mode}", + status = 200, + error_status = 422, + client_error_hook = true + )] + async fn wire(&self, mode: String) -> Result; + #[get("/legacy", error_status = 422)] + async fn legacy(&self) -> Result; +} + +// A false method flag cannot disable a trait-wide opt-in; the selection is logical OR. +#[api(reqwest, client_error_hook)] +pub trait TraitHookClient { + #[get("/problem", client_error_hook = false)] + async fn problem(&self) -> Result; +} + +// Client-only legacy errors must still compile and decode without either hook trait. +#[derive(Deserialize)] +pub struct LegacyClientError { + legacy: String, +} +#[api(reqwest)] +pub trait LegacyClient { + #[get("/legacy", error_status = 422)] + async fn legacy(&self) -> Result; +} + +#[tokio::test] +async fn generated_server_hook_overrides_wire_encoding_and_retains_arc_for_errors_and_rejections() { + let app = HookServerAxum::router(Service).layer(axum::extract::DefaultBodyLimit::max(32)); + for request in [ + Request::builder() + .uri("/problem") + .body(Body::empty()) + .unwrap(), + Request::builder() + .method("POST") + .uri("/reject") + .header("content-type", "application/json") + .body(Body::from("{")) + .unwrap(), + Request::builder() + .method("POST") + .uri("/reject-binary") + .body(Body::from(vec![0; 64])) + .unwrap(), + ] { + let response = app.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!( + response.headers()["content-type"], + "application/problem+json" + ); + assert_eq!( + response + .extensions() + .get::>() + .unwrap() + .mode, + 0 + ); + let bytes = to_bytes(response.into_body(), 4096).await.unwrap(); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(value["status"], 403); + assert_eq!( + value["yoi_failure"]["details"]["required_operation"], + "session.record" + ); + assert!(!String::from_utf8_lossy(&bytes).contains("never serialize")); + } +} + +#[tokio::test] +async fn absent_hook_preserves_json_and_invalid_hook_never_falls_back_to_legacy() { + let legacy = api_macros::axum::http_error_response( + StatusCode::UNPROCESSABLE_ENTITY, + ServerError { + legacy: "legacy-value", + mode: 1, + }, + ); + assert_eq!(legacy.status(), StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(legacy.headers()["content-type"], "application/json"); + assert!(legacy.extensions().get::>().is_some()); + let body = to_bytes(legacy.into_body(), 4096).await.unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["legacy"], + "legacy-value" + ); + for mode in [2, 3] { + let response = api_macros::axum::http_error_response( + StatusCode::UNPROCESSABLE_ENTITY, + ServerError { + legacy: "private", + mode, + }, + ); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(response.extensions().get::>().is_none()); + assert!( + to_bytes(response.into_body(), 4096) + .await + .unwrap() + .is_empty() + ); + } +} + +#[test] +fn problem_conversion_falls_back_without_exceeding_byte_or_depth_budgets() { + for (instance, budget) in [ + ("invalid URI", budget()), + ("", Budget::new(256, 256)), + ("", budget().with_depth(1, 1)), + ] { + let response = + api_macros::http::problem_response(&projected(), &Boundary, instance, budget).unwrap(); + assert_eq!(response.status, 500); + assert_eq!( + response.body(), + error_contract::problem::STATIC_PROBLEM_FALLBACK + ); + assert!(response.body().len() <= budget.bytes()); + } + assert!(matches!( + api_macros::http::problem_response(&projected(), &Boundary, "", Budget::new(1, 1)), + Err(ContractError::TooLarge) + )); +} + +async fn wire(axum::extract::Path(mode): axum::extract::Path) -> axum::response::Response { + let mut value: serde_json::Value = serde_json::from_slice(encoded().body.as_ref()).unwrap(); + let mut status = StatusCode::FORBIDDEN; + let mut content_type = "application/problem+json; charset=utf-8"; + match mode.as_str() { + "unknown" => value["yoi_failure"]["reason"] = "future_reason".into(), + "version" => value["yoi_failure"]["version"] = 99.into(), + "status" => status = StatusCode::CONFLICT, + "content-type" => content_type = "application/json", + "corrupt" => { + return ( + status, + [("content-type", content_type)], + "{private-raw-body", + ) + .into_response(); + } + "large" => { + return (status, [("content-type", content_type)], "x".repeat(8192)).into_response(); + } + _ => {} + } + let mut response = ( + status, + [("content-type", content_type)], + serde_json::to_vec(&value).unwrap(), + ) + .into_response(); + if mode == "duplicate" { + response + .headers_mut() + .append("content-type", "application/json".parse().unwrap()); + } + if mode == "missing" { + response.headers_mut().remove("content-type"); + } + response +} +struct ServerTask(tokio::task::JoinHandle<()>); +impl Drop for ServerTask { + fn drop(&mut self) { + self.0.abort(); + } +} +async fn serve() -> (String, ServerTask) { + let app = HookServerAxum::router(Service) + .route("/wire/{mode}", get(wire)) + .route( + "/legacy", + get(|| async { + api_macros::axum::http_error_response( + StatusCode::UNPROCESSABLE_ENTITY, + ServerError { + legacy: "legacy-value", + mode: 1, + }, + ) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let task = ServerTask(tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + })); + (url, task) +} + +#[tokio::test] +async fn generated_client_hook_uses_actual_metadata_and_preserves_unknown_corrupt_and_size_failures() + { + let (url, _server) = serve().await; + let client = HookClientClient::builder(&url) + .unwrap() + .request_timeout(Duration::from_secs(5)) + .response_body_limit(16384) + .build() + .unwrap(); + match client.problem().await { + Err(ClientError::Public { status, error }) => { + assert_eq!(status.as_u16(), 403); + assert_eq!(error.operation, "session.record"); + } + _ => panic!("expected a typed problem"), + } + for mode in ["unknown", "version"] { + assert!(matches!( + client.wire(mode.into()).await, + Err(ClientError::Failure( + ClientFailure::UnsupportedErrorResponse { status: 403 } + )) + )); + } + for (mode, expected) in [ + ("status", 409), + ("content-type", 403), + ("corrupt", 403), + ("duplicate", 403), + ("missing", 403), + ] { + let result = client.wire(mode.into()).await; + assert!( + matches!(result, Err(ClientError::Failure(ClientFailure::ErrorResponseDecode { status })) if status == expected), + "mode={mode}" + ); + } + // Contract budget and streaming transport budget are independent, bounded failures. + assert!(matches!( + client.wire("large".into()).await, + Err(ClientError::Failure(ClientFailure::ResponseTooLarge { + limit: 4096 + })) + )); + let small_client = HookClientClient::builder(&url) + .unwrap() + .request_timeout(Duration::from_secs(5)) + .response_body_limit(128) + .build() + .unwrap(); + assert!(matches!( + small_client.problem().await, + Err(ClientError::Failure(ClientFailure::ResponseTooLarge { + limit: 128 + })) + )); + match client.legacy().await { + Err(ClientError::Public { status, error }) => { + assert_eq!(status.as_u16(), 422); + assert_eq!(error.legacy, "legacy-value"); + } + _ => panic!("unrelated error on the same trait must keep serde behavior"), + } + let trait_client = TraitHookClientClient::builder(&url) + .unwrap() + .request_timeout(Duration::from_secs(5)) + .build() + .unwrap(); + assert!( + matches!(trait_client.problem().await, Err(ClientError::Public { error, .. }) if error.operation == "session.record") + ); + let legacy = LegacyClientClient::builder(&url) + .unwrap() + .request_timeout(Duration::from_secs(5)) + .build() + .unwrap(); + match legacy.legacy().await { + Err(ClientError::Public { status, error }) => { + assert_eq!(status.as_u16(), 422); + assert_eq!(error.legacy, "legacy-value"); + } + _ => panic!("legacy client-only error must keep serde behavior"), + } +} diff --git a/crates/api-macros/tests/openapi.rs b/crates/api-macros/tests/openapi.rs index db9bc5e74..1f2393f52 100644 --- a/crates/api-macros/tests/openapi.rs +++ b/crates/api-macros/tests/openapi.rs @@ -3,7 +3,7 @@ use api_macros::{ BinaryBody, api, - openapi::{OpenApiInfo, OpenApiSchema}, + openapi::{OpenApiBuilder, OpenApiInfo, OpenApiSchema}, }; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -465,3 +465,154 @@ fn file_export_writes_the_canonical_document() { std::fs::remove_file(path).expect("remove export"); assert_eq!(written, document.to_json().expect("canonical JSON")); } + +#[derive(Deserialize, Serialize)] +#[serde(untagged, deny_unknown_fields)] +pub enum ProblemOrLegacy { + Legacy { code: String }, + Problem { status: u16, detail: String }, +} +impl JsonSchema for ProblemOrLegacy { + fn schema_name() -> std::borrow::Cow<'static, str> { + "ProblemOrLegacy".into() + } + fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + // The owner explicitly attests disjoint closed object shapes instead of admitting anyOf. + schemars::json_schema!({ "oneOf": [ + { "type": "object", "required": ["code"], "additionalProperties": false, + "properties": { "code": { "type": "string" } } }, + { "type": "object", "required": ["status", "detail"], "additionalProperties": false, + "properties": { "status": { "type": "integer", "format": "uint16" }, "detail": { "type": "string" } } } + ] }) + } +} +impl OpenApiSchema for ProblemOrLegacy {} + +#[api(openapi)] +pub trait ErrorMediaApi { + #[post( + "/create", + error_status = 403, + additional_error_statuses = [409, 504], + additional_error_content_types = ["Application/Problem+JSON"] + )] + async fn create(&self) -> Result; + + #[get("/legacy", error_status = 422)] + async fn legacy(&self) -> Result; +} + +#[test] +fn additional_error_media_share_the_declared_schema_for_every_error_status_only() { + use api_macros::{ApiContract, Operation}; + let document = error_media_api_openapi(OpenApiInfo { + title: "Error media", + version: "1", + source_digest: "fixture:error-media", + }) + .unwrap(); + let value = document.as_value(); + let responses = &value["paths"]["/create"]["post"]["responses"]; + for status in ["403", "409", "504"] { + let content = responses[status]["content"].as_object().unwrap(); + assert_eq!(content.len(), 2); + assert_eq!( + content["application/json"]["schema"], + json!({ "$ref": "#/components/schemas/ProblemOrLegacy" }) + ); + assert_eq!( + content["application/problem+json"], + content["application/json"] + ); + } + assert_eq!(responses["200"]["content"].as_object().unwrap().len(), 1); + assert!( + responses["200"]["content"] + .get("application/problem+json") + .is_none() + ); + assert_eq!( + value["paths"]["/legacy"]["get"]["responses"]["422"]["content"] + .as_object() + .unwrap() + .len(), + 1 + ); + assert!(value["components"]["schemas"]["ProblemOrLegacy"]["oneOf"].is_array()); + let parsed = oas3::from_json(&document.to_json().unwrap()).unwrap(); + assert_eq!(parsed.validate_version().unwrap().major, 3); + + let operation = error_media_api_operations::Create::METADATA; + assert_eq!( + operation + .error_responses + .iter() + .map(|response| response.status) + .collect::>(), + [403, 409, 504] + ); + for response in operation.error_responses { + assert_eq!( + response.content_types, + ["application/json", "application/problem+json"] + ); + } + assert_eq!( + operation.success_responses[0].content_types, + ["application/json"] + ); + let legacy = ErrorMediaApiMetadata::OPERATIONS + .iter() + .find(|operation| operation.path == "/legacy") + .unwrap(); + assert_eq!( + legacy.error_responses[0].content_types, + ["application/json"] + ); +} + +#[derive(JsonSchema)] +pub struct MediaSpecificError; +impl OpenApiSchema for MediaSpecificError { + fn response_schema( + builder: &mut OpenApiBuilder, + content_type: &str, + ) -> Result { + match content_type { + "application/json" => builder.schema_ref::(), + "application/problem+json" => builder.schema_ref::(), + _ => unreachable!("only declared media are selected"), + } + } +} + +#[api(openapi)] +pub trait MediaSpecificApi { + #[post("/media", error_status = 409, additional_error_content_types = ["application/problem+json"])] + async fn create(&self) -> Result; +} + +#[test] +fn explicitly_different_error_representations_register_their_actual_schema_closures() { + let document = media_specific_api_openapi(OpenApiInfo { + title: "Media-specific", + version: "1", + source_digest: "fixture:media-specific", + }) + .unwrap(); + let value = document.as_value(); + let content = &value["paths"]["/media"]["post"]["responses"]["409"]["content"]; + assert_eq!( + content["application/json"]["schema"], + json!({"$ref":"#/components/schemas/PublicError"}) + ); + assert_eq!( + content["application/problem+json"]["schema"], + json!({"$ref":"#/components/schemas/ProblemOrLegacy"}) + ); + assert!(value["components"]["schemas"]["MediaSpecificError"].is_null()); + oas3::from_json(&document.to_json().unwrap()) + .unwrap() + .validate_version() + .unwrap(); +} diff --git a/crates/api-macros/tests/ui/client_hook_requires_reqwest.rs b/crates/api-macros/tests/ui/client_hook_requires_reqwest.rs new file mode 100644 index 000000000..1e6146b83 --- /dev/null +++ b/crates/api-macros/tests/ui/client_hook_requires_reqwest.rs @@ -0,0 +1,6 @@ +use api_macros::api; + +#[api(client_error_hook)] +pub trait Invalid {} + +fn main() {} diff --git a/crates/api-macros/tests/ui/client_hook_requires_reqwest.stderr b/crates/api-macros/tests/ui/client_hook_requires_reqwest.stderr new file mode 100644 index 000000000..d53f8c6ac --- /dev/null +++ b/crates/api-macros/tests/ui/client_hook_requires_reqwest.stderr @@ -0,0 +1,7 @@ +error: `client_error_hook` requires the `reqwest` adapter + --> tests/ui/client_hook_requires_reqwest.rs:3:1 + | +3 | #[api(client_error_hook)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the attribute macro `api` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/crates/api-macros/tests/ui/invalid_error_media.rs b/crates/api-macros/tests/ui/invalid_error_media.rs new file mode 100644 index 000000000..53a57ebe0 --- /dev/null +++ b/crates/api-macros/tests/ui/invalid_error_media.rs @@ -0,0 +1,47 @@ +use api_macros::api; + +pub struct Error; + +#[api] +pub trait Wildcard { + #[get("/wildcard", additional_error_content_types = ["application/*"])] + async fn wildcard(&self) -> Result<(), Error>; +} + +#[api] +pub trait DuplicateMedia { + #[get("/duplicate", additional_error_content_types = ["application/problem+json", "APPLICATION/PROBLEM+JSON"])] + async fn duplicate(&self) -> Result<(), Error>; +} + +#[api] +pub trait DefaultMedia { + #[get("/default", additional_error_content_types = ["application/json"])] + async fn default_media(&self) -> Result<(), Error>; +} + +#[api] +pub trait NoError { + #[get("/infallible", additional_error_content_types = ["application/problem+json"])] + async fn infallible(&self); +} + +#[api] +pub trait EmptyMedia { + #[get("/empty", additional_error_content_types = [])] + async fn empty(&self) -> Result<(), Error>; +} + +#[api] +pub trait DuplicateOption { + #[get("/duplicate-option", additional_error_content_types = [], additional_error_content_types = [])] + async fn duplicate_option(&self) -> Result<(), Error>; +} + +#[api] +pub trait ParameterizedMedia { + #[get("/parameterized", additional_error_content_types = ["application/problem+json; charset=utf-8"])] + async fn parameterized(&self) -> Result<(), Error>; +} + +fn main() {} diff --git a/crates/api-macros/tests/ui/invalid_error_media.stderr b/crates/api-macros/tests/ui/invalid_error_media.stderr new file mode 100644 index 000000000..6cb8a9cf1 --- /dev/null +++ b/crates/api-macros/tests/ui/invalid_error_media.stderr @@ -0,0 +1,41 @@ +error: error content type must be a concrete type/subtype without parameters + --> tests/ui/invalid_error_media.rs:7:58 + | +7 | #[get("/wildcard", additional_error_content_types = ["application/*"])] + | ^^^^^^^^^^^^^^^ + +error: additional error content types must be distinct from each other and application/json + --> tests/ui/invalid_error_media.rs:13:87 + | +13 | #[get("/duplicate", additional_error_content_types = ["application/problem+json", "APPLICATION/PROBLEM+JSON"])] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: additional error content types must be distinct from each other and application/json + --> tests/ui/invalid_error_media.rs:19:57 + | +19 | #[get("/default", additional_error_content_types = ["application/json"])] + | ^^^^^^^^^^^^^^^^^^ + +error: additional_error_content_types require a public error response type + --> tests/ui/invalid_error_media.rs:25:11 + | +25 | #[get("/infallible", additional_error_content_types = ["application/problem+json"])] + | ^^^^^^^^^^^^^ + +error: additional_error_content_types must not be empty + --> tests/ui/invalid_error_media.rs:31:11 + | +31 | #[get("/empty", additional_error_content_types = [])] + | ^^^^^^^^ + +error: duplicate `additional_error_content_types` route option + --> tests/ui/invalid_error_media.rs:37:69 + | +37 | #[get("/duplicate-option", additional_error_content_types = [], additional_error_content_types = [])] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: error content type must be a concrete type/subtype without parameters + --> tests/ui/invalid_error_media.rs:43:63 + | +43 | #[get("/parameterized", additional_error_content_types = ["application/problem+json; charset=utf-8"])] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/crates/api-macros/tests/ui/operation_hook_invalid_combinations.rs b/crates/api-macros/tests/ui/operation_hook_invalid_combinations.rs new file mode 100644 index 000000000..be2010a3c --- /dev/null +++ b/crates/api-macros/tests/ui/operation_hook_invalid_combinations.rs @@ -0,0 +1,23 @@ +use api_macros::api; + +pub struct Error; + +#[api] +pub trait NoReqwest { + #[get("/error", client_error_hook = true)] + async fn error(&self) -> Result<(), Error>; +} + +#[api(reqwest)] +pub trait NoError { + #[post("/infallible", client_error_hook = true)] + async fn infallible(&self); +} + +#[api(reqwest)] +pub trait DuplicateHook { + #[get("/duplicate", client_error_hook = true, client_error_hook = false)] + async fn duplicate(&self) -> Result<(), Error>; +} + +fn main() {} diff --git a/crates/api-macros/tests/ui/operation_hook_invalid_combinations.stderr b/crates/api-macros/tests/ui/operation_hook_invalid_combinations.stderr new file mode 100644 index 000000000..2099f87a9 --- /dev/null +++ b/crates/api-macros/tests/ui/operation_hook_invalid_combinations.stderr @@ -0,0 +1,17 @@ +error: client_error_hook requires the `reqwest` adapter + --> tests/ui/operation_hook_invalid_combinations.rs:7:41 + | +7 | #[get("/error", client_error_hook = true)] + | ^^^^ + +error: client_error_hook requires a public error response type + --> tests/ui/operation_hook_invalid_combinations.rs:13:47 + | +13 | #[post("/infallible", client_error_hook = true)] + | ^^^^ + +error: duplicate `client_error_hook` route option + --> tests/ui/operation_hook_invalid_combinations.rs:19:51 + | +19 | #[get("/duplicate", client_error_hook = true, client_error_hook = false)] + | ^^^^^^^^^^^^^^^^^ diff --git a/crates/client/src/backend_runtime.rs b/crates/client/src/backend_runtime.rs index 5be0d0eab..f9a51f06e 100644 --- a/crates/client/src/backend_runtime.rs +++ b/crates/client/src/backend_runtime.rs @@ -253,6 +253,7 @@ pub enum BackendRuntimeClientError { Api(BackendApiClientError), SessionApi(server_api::ServerApiClientError), ContractApi(server_api::client_support::ClientError), + WorkerCreateApi(server_api::client_support::ClientError), Http(reqwest::Error), Protocol(String), } @@ -264,6 +265,7 @@ impl fmt::Display for BackendRuntimeClientError { Self::Api(error) => write!(f, "{error}"), Self::SessionApi(error) => write!(f, "{error}"), Self::ContractApi(error) => write!(f, "{error}"), + Self::WorkerCreateApi(error) => write!(f, "{error}"), Self::Http(error) => write!(f, "{error}"), Self::Protocol(message) => f.write_str(message), } @@ -1016,3 +1018,13 @@ mod tests { assert!(serde_json::from_value::(stale).is_err()); } } + +impl From> + for BackendRuntimeClientError +{ + fn from( + error: server_api::client_support::ClientError, + ) -> Self { + Self::WorkerCreateApi(error) + } +} diff --git a/crates/client/src/backend_workspace.rs b/crates/client/src/backend_workspace.rs index 4cdb53b93..7c9c1699c 100644 --- a/crates/client/src/backend_workspace.rs +++ b/crates/client/src/backend_workspace.rs @@ -39,6 +39,7 @@ pub enum BackendWorkspaceClientError { Api(BackendApiClientError), Http(reqwest::Error), ServerApi(server_api::client_support::ClientError), + WorkerCreateApi(server_api::client_support::ClientError), RuntimeManagementApi( server_api::client_support::ClientError, ), @@ -51,6 +52,7 @@ impl fmt::Display for BackendWorkspaceClientError { Self::Api(error) => write!(f, "{error}"), Self::Http(error) => write!(f, "{error}"), Self::ServerApi(error) => write!(f, "{error}"), + Self::WorkerCreateApi(error) => write!(f, "{error}"), Self::RuntimeManagementApi(error) => write!(f, "{error}"), } } @@ -454,3 +456,31 @@ mod tests { assert_eq!(retry, request); } } + +pub(crate) fn worker_create_client_error( + backend: &BackendApiClient, + error: server_api::client_support::ClientError, +) -> BackendWorkspaceClientError { + use server_api::client_support::ClientError; + // Retain typed startup facts even for an operator denial. A Subject execution + // condition is not a refusal of this client's credential or a login prompt. + match &error { + ClientError::Public { + status, + error: response, + } if response.failure().is_none() && *status == reqwest::StatusCode::UNAUTHORIZED => { + BackendWorkspaceClientError::Api(BackendApiClientError::Unauthorized { + origin: backend.origin().clone(), + }) + } + ClientError::Public { + status, + error: response, + } if response.failure().is_none() && *status == reqwest::StatusCode::FORBIDDEN => { + BackendWorkspaceClientError::Api(BackendApiClientError::Forbidden { + origin: backend.origin().clone(), + }) + } + _ => BackendWorkspaceClientError::WorkerCreateApi(error), + } +} diff --git a/crates/client/src/workspace_product.rs b/crates/client/src/workspace_product.rs index ddf0cfb38..ea77d73f3 100644 --- a/crates/client/src/workspace_product.rs +++ b/crates/client/src/workspace_product.rs @@ -81,6 +81,23 @@ impl BackendWorkspaceProductClient { server_api::client_support::ClientError, >, >, + { + self.generated_normalized(operation, crate::backend_workspace::server_client_error) + } + + fn generated_normalized( + &self, + operation: F, + normalize: impl FnOnce( + &BackendApiClient, + server_api::client_support::ClientError, + ) -> BackendWorkspaceClientError, + ) -> Result + where + F: FnOnce( + server_api::ServerApiClient, + ) -> Fut, + Fut: std::future::Future>>, { let client = crate::backend_workspace::server_api_client(&self.api)?; let runtime = tokio::runtime::Builder::new_current_thread() @@ -89,7 +106,7 @@ impl BackendWorkspaceProductClient { .map_err(|error| BackendWorkspaceClientError::InvalidTarget(error.to_string()))?; runtime .block_on(operation(client)) - .map_err(|error| crate::backend_workspace::server_client_error(&self.api, error)) + .map_err(|error| normalize(&self.api, error)) } pub fn read_ticket_link_summary( @@ -584,9 +601,9 @@ impl BackendWorkspaceProductClient { execution_grant: None, }; let workspace_id = self.workspace_id.clone(); - let response: BrowserCreateWorkerResponse = self.generated(move |client| async move { + let response: BrowserCreateWorkerResponse = self.generated_normalized(move |client| async move { client.workspace_worker_create(workspace_id, request).await - })?; + }, crate::backend_workspace::worker_create_client_error)?; Ok(format!( "Started Intake Worker {}/{} for Ticket {ticket_id}", response.runtime_id, response.worker_id diff --git a/crates/error-contract/src/codec.rs b/crates/error-contract/src/codec.rs index 15d2f57e9..aa56f4ebc 100644 --- a/crates/error-contract/src/codec.rs +++ b/crates/error-contract/src/codec.rs @@ -162,6 +162,14 @@ fn validate_envelope_common(raw: &ReaderEnvelope) -> Result<(), ContractError> { raw.context.validate() } +/// Validate byte and lexical depth limits before parsing an enclosing JSON DTO. +/// +/// This is a resource guard only: callers must still use their typed JSON parser +/// to validate grammar and schema. It grants no disclosure or encoding authority. +pub fn validate_json_input(input: &[u8], budget: Budget) -> Result<(), ContractError> { + check_json_bounds(input, budget) +} + /// Linear lexical depth guard runs before serde allocates any JSON tree. Serde validates grammar. pub(crate) fn check_json_bounds(input: &[u8], budget: Budget) -> Result<(), ContractError> { if input.len() > budget.bytes() { diff --git a/crates/error-contract/src/facts.rs b/crates/error-contract/src/facts.rs index 83e65b8d8..6cc65102d 100644 --- a/crates/error-contract/src/facts.rs +++ b/crates/error-contract/src/facts.rs @@ -107,6 +107,7 @@ pub struct CausalLink { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", schemars(rename = "FailureDiagnostic"))] pub struct Diagnostic { #[cfg_attr(feature = "json-schema", schemars(length(min = 1, max = 128)))] pub code: String, diff --git a/crates/error-contract/src/lib.rs b/crates/error-contract/src/lib.rs index d4db17b6a..5b30f9f5a 100644 --- a/crates/error-contract/src/lib.rs +++ b/crates/error-contract/src/lib.rs @@ -10,7 +10,10 @@ pub mod schema; #[cfg(feature = "test-support")] pub mod test_support; -pub use codec::{Budget, DecodeFailure, ReceivedFailure, decode_failure, encode_projected_failure}; +pub use codec::{ + Budget, DecodeFailure, ReceivedFailure, decode_failure, encode_projected_failure, + validate_json_input, +}; pub use facts::*; pub use projection::*; diff --git a/crates/runtime-api/Cargo.toml b/crates/runtime-api/Cargo.toml index bf44b1d74..3f45ae2c6 100644 --- a/crates/runtime-api/Cargo.toml +++ b/crates/runtime-api/Cargo.toml @@ -6,10 +6,14 @@ edition.workspace = true license.workspace = true [dependencies] -api-macros = { workspace = true, features = ["reqwest", "axum"] } +api-macros = { workspace = true, features = ["reqwest", "axum", "openapi"] } manifest.workspace = true plugin-plan = { path = "../plugin-plan" } -protocol.workspace = true +protocol = { workspace = true, features = ["json-schema"] } +error-contract = { workspace = true, features = ["json-schema"] } +schemars.workspace = true +sha2.workspace = true +serde_json.workspace = true session-store.workspace = true serde = { workspace = true, features = ["derive"] } workdir.workspace = true diff --git a/crates/runtime-api/README.md b/crates/runtime-api/README.md index 79ebfd971..e7417e9c4 100644 --- a/crates/runtime-api/README.md +++ b/crates/runtime-api/README.md @@ -8,3 +8,4 @@ HTTP contract for Worker Runtime management. Owns REST DTOs and generated client - [`WorkerRestoreRequest`] / [`WorkerRestoreCoordinationRequest`] — Restore intent and Backend preparation contracts - [`LogicalWorkdirAttachment`] — Logical Workdir attachments determined by Workspace - [`BackendJobExecutionBinding`] — Job/attempt execution identity fixed by the Host +- [`startup`] — projected startup Problem Details, independent operation observations and bounded receiver normalization diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index cdb819085..b2e45d0c4 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -1,5 +1,7 @@ #![doc = include_str!("../README.md")] +pub mod startup; + use std::collections::BTreeMap; use api_macros::api; @@ -103,6 +105,9 @@ pub struct RuntimeApiError { pub error: RuntimeApiErrorDetail, #[serde(skip)] status: u16, + /// Local owner/normalized observations, never a second wire attachment. + #[serde(skip)] + create_failure: Option, } /// Transient, allowlisted startup observations copied from the execution owner. @@ -135,18 +140,37 @@ impl RuntimeApiError { startup: None, }, status, + create_failure: None, } } pub fn status(&self) -> u16 { self.status } + + /// Retain owner-selected facts locally; the HTTP hook projects them afresh. + pub fn with_create_failure(mut self, failure: startup::WorkerCreateFailure) -> Self { + self.create_failure = Some(failure); + self + } + + pub fn create_failure(&self) -> Option<&startup::WorkerCreateFailure> { + self.create_failure.as_ref() + } + + pub fn into_create_failure(self) -> Option { + self.create_failure + } } impl api_macros::HttpError for RuntimeApiError { fn status_code(&self) -> u16 { self.status } + + fn custom_response(&self) -> Option { + startup::http::runtime_error_response(self) + } } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -1305,7 +1329,7 @@ pub struct SubjectJobRemovalPurgeResponse { pub input_fingerprint: String, } -#[api(reqwest, axum)] +#[api(reqwest, axum, client_error_hook)] pub trait RuntimeApi { #[post("/v1/subject-job-removal-purge", status = 200, error_status = 400)] async fn purge_subject_job_removal( @@ -1924,6 +1948,25 @@ mod tests { } async fn get_worker(&self, worker_id: String) -> Result { + if worker_id == "startup" { + use startup::*; + let failure = WorkerCreateFailure::from_cause( + StartupCause::Session( + protocol::session_failure::SessionRegistrationCause::denied( + protocol::session_failure::SessionPrincipal::Subject, + ), + ), + StartupOperationFacts { + acceptance: CreationAcceptance::Unknown, + execution: ExecutionOutcome::Unknown, + finalize: None, + backend_compensation: None, + correlation: None, + }, + ) + .unwrap(); + return Err(test_error(500).with_create_failure(failure)); + } assert_eq!(worker_id, "missing worker"); Err(test_error(404)) } @@ -2233,6 +2276,34 @@ mod tests { assert!( matches!(error, client_support::ClientError::Public { status, .. } if status.as_u16() == 404) ); + let error = client.get_worker("startup".to_string()).await.unwrap_err(); + let client_support::ClientError::Public { status, error } = error else { + panic!("expected a normalized public startup failure"); + }; + assert_eq!(status.as_u16(), 403); + assert_eq!(error.status(), 403); + let failure = error + .create_failure() + .expect("hook retains non-HTTP observations"); + assert_eq!( + failure.operation().acceptance, + startup::CreationAcceptance::Unknown + ); + let browser = failure + .problem( + startup::StartupBoundary::Browser, + &startup::SessionFailureVisibility { + can_view_cause: true, + }, + "", + ) + .unwrap(); + assert_eq!(browser.status, 409); + assert!( + std::str::from_utf8(browser.body()) + .unwrap() + .contains("session.record") + ); server.abort(); } diff --git a/crates/runtime-api/src/startup.rs b/crates/runtime-api/src/startup.rs new file mode 100644 index 000000000..a7aff76ec --- /dev/null +++ b/crates/runtime-api/src/startup.rs @@ -0,0 +1,1092 @@ +//! Reusable startup HTTP boundary, independent of an HTTP framework. +//! +//! `WorkerCreateFailure` retains the execution owner's attachment and independent +//! observations. It is not another durable operation record and grants no retry rights. +//! Only `ProblemResponse::from_projected` encodes public responses. The public DTO is +//! schema/reader data, **not** a disclosure capability. Received explanations, labels, +//! diagnostic strings and attachment bytes are never copied to a public response. + +//! ## Startup boundary adapter +//! +//! The `startup` module is framework-independent. `WorkerCreateFailure::new` wraps the +//! existing `WorkerStartupFailure` without replacing its primary attachment, secondary +//! cleanup/persistence facts, resources or IO classification. `StartupOperationFacts` +//! requires callers to supply creation acceptance and execution outcome explicitly; +//! there is no HTTP-to-outcome conversion, default rejection or retry authorization. +//! These observations are **not** a second durable result authority. +//! +//! `StartupCause` is an untagged concrete union of the adopted Session and Runtime +//! causes and the enum-only `StartupBoundaryCause`. The existing `session_registration` +//! and `worker_startup` identities and details remain unchanged. Boundary observations +//! use `startup_boundary` and distinguish timeout, transport, decode, invalid response, +//! size limit, unsupported version/reason and finalize failure. The cause does not carry +//! raw error text. Finalization stage and compensation are separate from that cause and +//! from acceptance/execution. Completed compensation does not establish non-execution. +//! +//! ### Boundary integration API +//! +//! 1. Supply `StartupOperationFacts { acceptance, execution, finalize, backend_compensation, correlation }` +//! from actual owner observations. On response loss, preserve `Unknown` unless +//! separate authoritative evidence establishes another outcome. +//! 2. Build `WorkerCreateFailure::new(startup, operation)` or `::from_cause(cause, operation)`. +//! `startup()`, `operation()` and `into_parts()` support non-HTTP callers. +//! 3. Call `failure.problem(StartupBoundary::Browser, ¤t_visibility, instance)`. +//! The resulting `startup::ProblemResponse` provides `status`, `content_type`, +//! `body()` and `into_body()`. The response body includes the core `yoi_failure` +//! envelope plus `yoi_startup: StartupProblemFacts`. The latter never contains an +//! attachment. `StartupProblemDetails` is the public `schemars` source DTO; parsing +//! that DTO alone grants no permission to disclose it. +//! 4. Runtime response adapters select `StartupBoundary::Runtime`. Both boundaries +//! deliberately emit the **minimum public-safe schema**, using a fresh Browser/public +//! projection; Runtime does not relabel InternalTransfer output as Browser output. +//! Session disclosure requires an explicit current `SessionFailureVisibility` under +//! authenticated Runtime scope, never a received/public label. +//! `ProblemResponse::from_projected` supports callers already holding a +//! `ProjectedFailure` and typed facts. +//! 5. Remote clients call `decode_startup_problem(bytes, status, content_type, sender_boundary)`. +//! Its result is `Known(WorkerCreateFailure)`, `UnsupportedVersion`, +//! `UnsupportedReason`, `Invalid` or `TooLarge`. The boundary argument describes +//! the **sender**, so a Runtime Subject denial decodes as 403 and is later projected +//! as 409 at the Browser boundary. Do not classify the Browser response by forwarding +//! that 403. Unknown results never expose raw retained input. +//! +//! The normalizer reuses core byte/depth checks (32 KiB / 16 levels), envelope validation, +//! identity validation and Problem Details consistency checks. It distinguishes unknown +//! versions, details versions and required extensions from unknown reasons/domains using +//! validated metadata, without interpreting unknown details. Known input is projected +//! again through the actual concrete domain's InternalTransfer allowlist. Wire detail, +//! explanation, context diagnostics and references are not forwarded. A later Browser +//! projection applies current Session visibility again. Missing acceptance/execution is +//! invalid, not an implicit `Unknown` or `Rejected`. +//! +//! ### Classification and disclosure +//! +//! | Cause | Browser | Runtime | +//! | --- | --- | --- | +//! | Session operator denied | 403 | 403 | +//! | Session Subject/unknown-principal denied | 409 | 403 | +//! | Session attribution missing / binding mismatch | 409 | 409 | +//! | Session transport / unknown / invalid response | 502 | 502 | +//! | Boundary timeout | 504 | 504 | +//! | Boundary transport / decode / invalid / oversized / unsupported | 502 | 502 | +//! | Finalize failure | 500 | 500 | +//! | Runtime empty Segment / missing prompt | 400 | 400 | +//! | Runtime metadata pending/missing/inactive / manifest missing | 409 | 409 | +//! | Runtime allocation failure | 503 | 503 | +//! | Other closed Runtime startup reasons | 500 | 500 | +//! +//! Classification uses typed causes only, never severity, diagnostic order, explanation, +//! HTTP hints, compensation or resources. Browser Session explanations explicitly name +//! `session.record`, the principal class and the Worker impact. Domain projectors retain +//! only their allowlisted context and remove operation references; received text is not +//! a source of public explanations. +//! +//! `yoi_startup` retains independently: acceptance, execution, optional finalization stage +//! and compensation, optional **Backend compensation** even without finalization, optional +//! safe correlation, all secondary facts, resource facts, persistence and path-free IO +//! kind. Backend compensation is not inferred from Runtime cleanup or finalize facts. +//! Only these closed enum facts are copied into +//! this extension. `StartupCorrelation::new` accepts at most 128 ASCII letters/digits, +//! periods, underscores and hyphens. It is an opaque lookup token, not a credential or +//! operation identity. Callers must choose an actual safe token (lexical validation cannot +//! identify a secret) and a safe `instance` URI-reference, not a raw upstream URL. +//! +//! ### RuntimeApiError HTTP hooks +//! +//! `RuntimeApiError::with_create_failure(failure)` retains local typed facts in a +//! `serde(skip)` field. `create_failure()` and `into_create_failure()` expose them to +//! internal callers. The Runtime API's `HttpError::custom_response` resolves a fresh +//! public-safe projection for the Runtime boundary; invalid/unavailable projection uses +//! the core static Problem Details fallback, never the legacy attachment serialization. +//! `with_create_failure` and Runtime HTTP serving are trusted local adapters invoked +//! **after current Runtime scope authentication/authorization**; this minimum schema +//! policy does not authorize Browser disclosure. Ordinary errors without startup facts +//! retain their original JSON/status behavior. +//! +//! `RuntimeApi` opts into `client_error_hook`. The generated client passes the actual +//! HTTP status, unique Content-Type and bounded bytes to `RuntimeApiError`'s +//! `HttpClientError` implementation. Known Problems retain normalized typed facts. +//! Unsupported version and unsupported reason become distinct local boundary observations +//! with acceptance/execution `Unknown`; malformed/oversized Problems also remain local +//! observations rather than reports of remote execution failure. Collection failures +//! outside the hook remain the macro's precise `ClientFailure` values and carry no +//! execution verdict. +//! +//! Legacy `application/json` errors use a narrow typed schema under the same 32 KiB byte +//! limit; top-level/detail extension fields are rejected, and nested adopted facts are +//! parsed as typed values rather than allocated as arbitrary JSON trees. Valid startup attachments are projected again into InternalTransfer with +//! acceptance/execution left `Unknown` because the legacy shape does not report them. +//! A legacy unsupported attachment exposes no safe version/reason metadata, so it is +//! classified locally as invalid response rather than guessed from opaque bytes. +//! Non-startup legacy errors retain their code/message and actual HTTP status. +//! +//! The core currently exposes encoded responses, not a generic projected Problem DTO +//! constructor. The extension encoder therefore reuses core URI/classification/envelope +//! validation and reads that bounded core DTO before attaching typed facts. A core +//! `to_problem_dto` primitive could remove this in-memory round trip; implementing a +//! second URI/envelope encoder here would be less maintainable and is intentionally +//! avoided. +//! +//! ### Investigation / recording ownership +//! +//! This module does not write metadata, log, reconcile, retry or compensate. The execution +//! owner remains responsible for persistence and actual resource cleanup; the parent +//! boundary records one safe correlation at its existing operation/result/log boundary. +//! Use that correlation to locate the existing owner result, compare acceptance and +//! execution separately, then inspect finalize/compensation and resource/persistence +//! facts. Do not infer a new operation is safe from timeout, failed persistence or +//! completed compensation. Do not log raw unsupported HTTP bodies or attachments. +//! +//! `WorkerCreateFailure::telemetry()` returns a `ProjectedFailure` +//! for `error_contract::observation_fields`, not a log write. The projector invokes the +//! concrete domain allowlist, selects fixed operation/stage labels and drops all received +//! operation references and attachment correlation. It uses only the typed operation +//! correlation as a correlation reference, never as an operation identity. Backend callers +//! should replace any remote correlation with their fresh local reservation-bound token +//! before recording at the Backend responsibility boundary. +//! +//! The shared `tests/fixtures/` inputs include a constructor-generated denial and +//! exact Browser HTTP bodies captured by workspace-server's registered +//! `session_registration_connected` tests (Subject denial and committed-registration +//! response loss). `startup_contract` checks consumer normalization and preservation +//! through the HTTP-independent storage codec. These captured operation facts are not +//! synthesized from HTTP status. Fixture changes must be intentional source changes; +//! tests never rewrite fixtures or select behavior through ambient environment flags. +//! The `startup_http` integration tests exercise actual generated RuntimeApi clients +//! against HTTP responses, and the normal generated-router round trip covers startup +//! Problem responses as well as legacy errors. Actual Subject authorization, Runtime +//! execution/persistence and Backend integration remain the parent Ticket's responsibility; +//! these contract tests do not substitute for those connected checks. +//! + +pub(crate) mod http; +mod storage; + +pub use storage::{DecodeStoredWorkerCreateFailure, decode_stored_worker_create_failure}; + +use error_contract::codec::FailureEnvelope; +use error_contract::problem::{ + ApiBoundary, HttpFailureStatus, ProblemClass, ProblemDetails, decode_problem_details, + to_problem_details, +}; +use error_contract::{ + Browser, Budget, Category, ContractError, DecodeFailure, Disclosure, FailureCause, + FailureFacts, FailureIdentity, InternalTransfer, ProjectFor, ProjectedFailure, Telemetry, + WireDetails, project_failure, +}; +pub use protocol::session_failure::SessionFailureVisibility; +use protocol::session_failure::{ + SessionPrincipal, SessionRegistrationCause, SessionRegistrationReason, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + RuntimeStartupCause, RuntimeStartupReason, StartupFailureAttachment, StartupIoKind, + StartupPersistenceOutcome, StartupResourceFacts, StartupSecondaryFailure, WorkerStartupFailure, +}; + +pub use error_contract::problem::PROBLEM_CONTENT_TYPE; +pub const MAX_STARTUP_PROBLEM_BYTES: usize = error_contract::MAX_FAILURE_BYTES; +pub const STARTUP_FACTS_VERSION: u16 = 1; + +fn budget() -> Budget { + Budget::new(MAX_STARTUP_PROBLEM_BYTES, MAX_STARTUP_PROBLEM_BYTES) +} + +/// Concrete domain union: serialization is the original domain details object, +/// not an additional tagged wrapper. Envelope domain/reason identity is unchanged. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum StartupCause { + Session(SessionRegistrationCause), + Runtime(RuntimeStartupCause), + Boundary(StartupBoundaryCause), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum StartupBoundaryReason { + Timeout, + Transport, + Decode, + InvalidResponse, + TooLarge, + UnsupportedVersion, + UnsupportedReason, + FinalizeFailed, +} + +/// An observation at the receiving/finalizing boundary, not a reported Runtime +/// execution result. It deliberately cannot hold raw transport or error text. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StartupBoundaryCause { + pub reason: StartupBoundaryReason, +} + +impl FailureCause for StartupBoundaryCause { + fn identity(&self) -> FailureIdentity { + use StartupBoundaryReason::*; + let reason = match self.reason { + Timeout => "timeout", + Transport => "transport", + Decode => "decode", + InvalidResponse => "invalid_response", + TooLarge => "too_large", + UnsupportedVersion => "unsupported_version", + UnsupportedReason => "unsupported_reason", + FinalizeFailed => "finalize_failed", + }; + FailureIdentity { + domain: "startup_boundary", + reason, + category: if self.reason == FinalizeFailed { + Category::Internal + } else { + Category::Dependency + }, + details_version: 1, + } + } + + fn validate(&self) -> Result<(), ContractError> { + Ok(()) + } +} + +impl WireDetails for StartupBoundaryCause { + fn supports(domain: &str, reason: &str, version: u16) -> bool { + domain == "startup_boundary" + && version == 1 + && matches!( + reason, + "timeout" + | "transport" + | "decode" + | "invalid_response" + | "too_large" + | "unsupported_version" + | "unsupported_reason" + | "finalize_failed" + ) + } +} + +impl FailureCause for StartupCause { + fn identity(&self) -> FailureIdentity { + match self { + Self::Session(cause) => cause.identity(), + Self::Runtime(cause) => cause.identity(), + Self::Boundary(cause) => cause.identity(), + } + } + + fn validate(&self) -> Result<(), ContractError> { + match self { + Self::Session(cause) => cause.validate(), + Self::Runtime(cause) => cause.validate(), + Self::Boundary(cause) => cause.validate(), + } + } +} + +impl WireDetails for StartupCause { + fn supports(domain: &str, reason: &str, version: u16) -> bool { + SessionRegistrationCause::supports(domain, reason, version) + || RuntimeStartupCause::supports(domain, reason, version) + || StartupBoundaryCause::supports(domain, reason, version) + } +} + +// Reconstruct concrete domain facts, then invoke that domain's actual allowlist. +// No received diagnostic/cause/correlation context is implicitly redisclosed. +fn domain_facts( + cause: D, + facts: &FailureFacts, +) -> Result, ContractError> { + let mut concrete = FailureFacts::new(cause)?; + for operation in &facts.context().operations { + concrete.with_context(operation.clone())?; + } + Ok(concrete) +} + +fn wrap( + disclosure: Disclosure, + wrap_details: fn(D) -> StartupCause, + explanation: String, +) -> Disclosure { + Disclosure { + details: wrap_details(disclosure.details), + explanation, + context: disclosure.context, + } +} + +fn session_explanation(cause: &SessionRegistrationCause) -> String { + let principal = match cause.principal { + SessionPrincipal::Operator => "Operator", + SessionPrincipal::Subject => "Subject", + SessionPrincipal::Unknown => "Unknown principal", + }; + let why = match cause.reason { + SessionRegistrationReason::PermissionDenied => { + "is not authorized for required operation session.record" + } + SessionRegistrationReason::AttributionMissing => { + "attribution is missing for required operation session.record" + } + SessionRegistrationReason::BindingMismatch => { + "binding does not match for required operation session.record" + } + SessionRegistrationReason::Transport => { + "could not be confirmed because the session.record transport failed" + } + SessionRegistrationReason::OutcomeUnknown => "has an unknown session.record outcome", + SessionRegistrationReason::InvalidResponse => "received an invalid session.record response", + }; + format!( + "Worker creation could not complete Session registration: {principal} {why}. Creation acceptance and execution outcome are reported separately." + ) +} + +fn runtime_explanation(cause: &RuntimeStartupCause) -> &'static str { + use RuntimeStartupReason::*; + match cause.reason { + StartupFailed => "Worker startup failed at the Runtime.", + SegmentEmpty => "Worker startup requires a nonempty Segment.", + SystemPromptMissing => "Worker startup requires a system prompt.", + MetadataPending => "Worker startup metadata is pending reconciliation.", + MetadataMissing => "Worker startup metadata is missing.", + MetadataInactive => "Worker startup metadata is inactive.", + ManifestMissing => "Worker startup manifest is missing.", + AllocationFailed => "Worker startup resource allocation failed.", + SessionReadFailed => "Worker startup could not read the Session.", + MetadataReadFailed => "Worker startup could not read metadata.", + ProviderFailed => "Worker startup provider initialization failed.", + AttributionFailed => "Worker startup attribution failed.", + FeatureFailed => "Worker startup feature initialization failed.", + } +} + +fn boundary_explanation(cause: &StartupBoundaryCause) -> &'static str { + use StartupBoundaryReason::*; + match cause.reason { + Timeout => "Worker creation response timed out; this does not establish non-execution.", + Transport => { + "Worker creation response transport failed; this does not establish non-execution." + } + Decode => { + "Worker creation response could not be decoded; execution outcome remains independently reported." + } + InvalidResponse => { + "Worker creation returned an invalid response; execution outcome remains independently reported." + } + TooLarge => { + "Worker creation response exceeded the supported size limit; execution outcome remains independently reported." + } + UnsupportedVersion => { + "Worker creation response uses an unsupported contract version; execution outcome remains independently reported." + } + UnsupportedReason => { + "Worker creation response contains an unsupported failure reason; execution outcome remains independently reported." + } + FinalizeFailed => { + "Worker creation finalization failed; acceptance, execution and compensation are reported separately." + } + } +} + +impl ProjectFor for StartupCause { + type Authorization = SessionFailureVisibility; + type Details = Self; + + fn disclose( + &self, + facts: &FailureFacts, + visibility: &SessionFailureVisibility, + ) -> Result, ContractError> { + match self { + Self::Session(cause) => { + let concrete = domain_facts(*cause, facts)?; + let disclosed = >::disclose( + cause, &concrete, visibility, + )?; + Ok(wrap(disclosed, Self::Session, session_explanation(cause))) + } + Self::Runtime(cause) => { + let concrete = domain_facts(cause.clone(), facts)?; + let disclosed = + >::disclose(cause, &concrete, &())?; + Ok(wrap( + disclosed, + Self::Runtime, + runtime_explanation(cause).into(), + )) + } + Self::Boundary(cause) => Ok(Disclosure::new(self.clone(), boundary_explanation(cause))), + } + } +} + +impl ProjectFor for StartupCause { + type Authorization = (); + type Details = Self; + + fn disclose( + &self, + facts: &FailureFacts, + _: &(), + ) -> Result, ContractError> { + match self { + Self::Session(cause) => { + let concrete = domain_facts(*cause, facts)?; + let disclosed = + >::disclose( + cause, + &concrete, + &(), + )?; + let explanation = session_explanation(cause); + Ok(wrap(disclosed, Self::Session, explanation)) + } + Self::Runtime(cause) => { + let concrete = domain_facts(cause.clone(), facts)?; + let disclosed = >::disclose( + cause, + &concrete, + &(), + )?; + Ok(wrap( + disclosed, + Self::Runtime, + runtime_explanation(cause).into(), + )) + } + Self::Boundary(cause) => Ok(Disclosure::new(self.clone(), boundary_explanation(cause))), + } + } +} + +impl ProjectFor for StartupCause { + type Authorization = (); + type Details = Self; + + fn disclose( + &self, + facts: &FailureFacts, + _: &(), + ) -> Result, ContractError> { + // Reuse concrete domain allowlists, not arbitrary received context. + let mut disclosure = >::disclose(self, facts, &())?; + let stage = match self { + Self::Session(_) => "binding", + Self::Runtime(_) => "runtime", + Self::Boundary(StartupBoundaryCause { + reason: StartupBoundaryReason::FinalizeFailed, + }) => "finalize", + Self::Boundary(_) => "response", + }; + // Telemetry selects one typed responsibility context, not a received + // context ordering or arbitrary operation identity. + disclosure.context.operations.clear(); + disclosure + .context + .operations + .push(error_contract::OperationContext::new( + "worker_startup", + stage, + )?); + if let Some(correlation) = &facts.context().correlation { + let safe = StartupCorrelation::new(correlation.as_str())?; + disclosure.context.correlation = + Some(error_contract::CorrelationRef::new(safe.as_str())?); + } + Ok(disclosure) + } +} + +/// Both ends select a boundary explicitly. Never forwards an inner HTTP status. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StartupBoundary { + Browser, + Runtime, +} + +impl ApiBoundary for StartupBoundary { + fn classify(&self, cause: &StartupCause) -> ProblemClass { + use HttpFailureStatus::*; + let status = match (self, cause) { + (Self::Browser, StartupCause::Session(cause)) => { + match (cause.reason, cause.principal) { + (SessionRegistrationReason::PermissionDenied, SessionPrincipal::Operator) => { + Forbidden + } + ( + SessionRegistrationReason::PermissionDenied + | SessionRegistrationReason::AttributionMissing + | SessionRegistrationReason::BindingMismatch, + _, + ) => Conflict, + ( + SessionRegistrationReason::Transport + | SessionRegistrationReason::OutcomeUnknown + | SessionRegistrationReason::InvalidResponse, + _, + ) => BadGateway, + } + } + (Self::Runtime, StartupCause::Session(cause)) => match cause.reason { + SessionRegistrationReason::PermissionDenied => Forbidden, + SessionRegistrationReason::AttributionMissing + | SessionRegistrationReason::BindingMismatch => Conflict, + SessionRegistrationReason::Transport + | SessionRegistrationReason::OutcomeUnknown + | SessionRegistrationReason::InvalidResponse => BadGateway, + }, + (_, StartupCause::Runtime(cause)) => match cause.reason { + RuntimeStartupReason::SegmentEmpty | RuntimeStartupReason::SystemPromptMissing => { + BadRequest + } + RuntimeStartupReason::MetadataPending + | RuntimeStartupReason::MetadataMissing + | RuntimeStartupReason::MetadataInactive + | RuntimeStartupReason::ManifestMissing => Conflict, + RuntimeStartupReason::AllocationFailed => Unavailable, + RuntimeStartupReason::StartupFailed + | RuntimeStartupReason::SessionReadFailed + | RuntimeStartupReason::MetadataReadFailed + | RuntimeStartupReason::ProviderFailed + | RuntimeStartupReason::AttributionFailed + | RuntimeStartupReason::FeatureFailed => Internal, + }, + (_, StartupCause::Boundary(cause)) => match cause.reason { + StartupBoundaryReason::Timeout => GatewayTimeout, + StartupBoundaryReason::FinalizeFailed => Internal, + StartupBoundaryReason::Transport + | StartupBoundaryReason::Decode + | StartupBoundaryReason::InvalidResponse + | StartupBoundaryReason::TooLarge + | StartupBoundaryReason::UnsupportedVersion + | StartupBoundaryReason::UnsupportedReason => BadGateway, + }, + }; + let (type_uri, title) = match status { + Forbidden => ( + "urn:yoi:problem:authorization-denied", + "Authorization denied", + ), + Conflict => ("urn:yoi:problem:conflict", "Operation conflict"), + BadRequest => ("urn:yoi:problem:invalid-input", "Invalid input"), + GatewayTimeout => ( + "urn:yoi:problem:startup-timeout", + "Startup response timeout", + ), + BadGateway => ("urn:yoi:problem:dependency", "Dependency failure"), + Unavailable => ( + "urn:yoi:problem:startup-unavailable", + "Startup resources unavailable", + ), + _ => ("urn:yoi:problem:startup-failed", "Worker startup failed"), + }; + ProblemClass { + status, + type_uri, + title, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CreationAcceptance { + Rejected, + Accepted, + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionOutcome { + Accepted, + Busy, + Rejected, + Errored, + Unsupported, + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FinalizeStage { + WorkerIdentity, + WorkspaceApiBinding, + CreateReservation, + WorkerRegistry, + WorkdirAttachment, + WorkdirRegistry, + TicketAssignmentBind, + TicketAssignmentCurrent, + TicketStateAccept, + SessionRegistration, + InitialInput, + Publication, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CompensationOutcome { + Completed, + Residual, + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct FinalizeFacts { + pub stage: FinalizeStage, + pub compensation: CompensationOutcome, +} + +/// Opaque, caller-selected lookup token, not an operation identity or capability. +/// No whitespace, URI/path/query syntax or control characters are accepted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(transparent)] +pub struct StartupCorrelation( + #[schemars(length(min = 1, max = 128), regex(pattern = "^[A-Za-z0-9._-]+$"))] String, +); + +impl StartupCorrelation { + /// A shared path-free lookup token for an owner-bound Runtime/Worker pair. + /// This is not the create fingerprint, operation key, or replay authority. + pub fn for_worker(runtime_id: &str, worker_id: &str) -> Self { + use sha2::{Digest, Sha256}; + let mut hash = Sha256::new(); + hash.update(runtime_id.as_bytes()); + hash.update([0]); + hash.update(worker_id.as_bytes()); + let digest = hash + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + Self(format!("worker-{digest}")) + } + + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if value.is_empty() + || value.len() > 128 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + { + return Err(ContractError::Invalid); + } + Ok(Self(value)) + } + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl<'de> Deserialize<'de> for StartupCorrelation { + fn deserialize>(deserializer: D) -> Result { + Self::new(String::deserialize(deserializer)?).map_err(serde::de::Error::custom) + } +} + +/// Explicit owner observations. There is deliberately no Default and no conversion +/// from HTTP status, retry hints, resource facts or failure reason into these fields. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StartupOperationFacts { + pub acceptance: CreationAcceptance, + pub execution: ExecutionOutcome, + pub finalize: Option, + /// Backend compensation can occur before/without finalization; never infer + /// this observation from Runtime cleanup, resources or finalize compensation. + pub backend_compensation: Option, + pub correlation: Option, +} + +/// The extension contains only typed allowlisted observations; never the attachment. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct StartupProblemFacts { + #[schemars(range(min = 1, max = 1))] + pub version: u16, + pub operation: StartupOperationFacts, + #[schemars(length(max = 8))] + pub secondary: Vec, + pub resources: Option, + /// Session metadata persistence, independent of the Backend reservation result. + pub persistence: Option, + /// Backend-owned result write only; never evidence of Session metadata persistence. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backend_result_persistence: Option, + pub io_kind: Option, +} + +impl StartupProblemFacts { + pub fn validate(&self) -> Result<(), ContractError> { + if self.version != STARTUP_FACTS_VERSION + || self.secondary.len() > error_contract::MAX_DIAGNOSTICS + { + return Err(ContractError::InvalidDetails); + } + Ok(()) + } +} + +/// Public generator/reader DTO. Deserialization does not grant disclosure authority. +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct StartupProblemDetails { + #[serde(rename = "type")] + #[schemars(length(max = 2048))] + pub type_uri: String, + #[schemars(length(min = 1, max = 2048))] + pub title: String, + #[schemars(range(min = 400, max = 599))] + pub status: u16, + #[schemars(length(min = 1, max = 2048))] + pub detail: String, + #[schemars(length(max = 2048))] + pub instance: String, + #[schemars(schema_with = "startup_envelope_schema")] + pub yoi_failure: FailureEnvelope, + pub yoi_startup: StartupProblemFacts, +} + +// The envelope's existing domain is the wire discriminator. Describing only the +// inner untagged implementation union would lose this external contract and make +// generated clients guess between overlapping reason fields. +fn startup_envelope_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let domains = [ + ( + "session_registration", + FailureEnvelope::::json_schema(generator), + ), + ( + "worker_startup", + FailureEnvelope::::json_schema(generator), + ), + ( + "startup_boundary", + FailureEnvelope::::json_schema(generator), + ), + ]; + let branches = domains + .into_iter() + .map(|(domain, mut schema)| { + let properties = schema + .as_object_mut() + .and_then(|schema| schema.get_mut("properties")) + .and_then(serde_json::Value::as_object_mut) + .expect("FailureEnvelope is an object with typed properties"); + properties.insert( + "domain".into(), + serde_json::json!({"type": "string", "const": domain}), + ); + serde_json::Value::from(schema) + }) + .collect::>(); + schemars::json_schema!({ "oneOf": branches }) +} + +impl api_macros::openapi::OpenApiSchema for StartupProblemDetails {} + +/// HTTP-independent failure retaining the exact original startup attachment. +/// Not serializable to prevent accidentally publishing internal attachments. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkerCreateFailure { + startup: WorkerStartupFailure, + operation: StartupOperationFacts, + backend_result_persistence: Option, +} + +impl WorkerCreateFailure { + pub fn new( + startup: WorkerStartupFailure, + operation: StartupOperationFacts, + ) -> Result { + let failure = Self { + startup, + operation, + backend_result_persistence: None, + }; + failure.public_facts().validate()?; + Ok(failure) + } + + pub fn from_cause( + cause: StartupCause, + operation: StartupOperationFacts, + ) -> Result { + let facts = FailureFacts::new(cause)?; + let projected = project_failure::(&facts, &())?; + Self::new( + WorkerStartupFailure { + primary: StartupFailureAttachment::from_projected(&projected)?, + secondary: Vec::new(), + resources: None, + persistence: None, + io_kind: None, + }, + operation, + ) + } + + pub fn startup(&self) -> &WorkerStartupFailure { + &self.startup + } + pub fn operation(&self) -> &StartupOperationFacts { + &self.operation + } + /// Backend reservation result persistence only, not Session metadata persistence. + pub fn backend_result_persistence(&self) -> Option { + self.backend_result_persistence + } + + /// Set a local write observation. A Backend receiver must clear a remote marker + /// before its own save; an encoder writes the current marker without inferring it. + pub fn with_backend_result_persistence( + mut self, + outcome: Option, + ) -> Self { + self.backend_result_persistence = outcome; + self + } + + /// Replace boundary observations without discarding Backend persistence evidence. + pub fn with_operation(mut self, operation: StartupOperationFacts) -> Self { + self.operation = operation; + self + } + + /// Legacy two-part decomposition discards Backend result persistence. Extract + /// `backend_result_persistence()` first and restore it after `new`, or prefer + /// `with_operation` when only the operation observations need replacement. + pub fn into_parts(self) -> (WorkerStartupFailure, StartupOperationFacts) { + (self.startup, self.operation) + } + + pub fn public_facts(&self) -> StartupProblemFacts { + StartupProblemFacts { + version: STARTUP_FACTS_VERSION, + operation: self.operation.clone(), + secondary: self.startup.secondary.clone(), + resources: self.startup.resources, + persistence: self.startup.persistence, + backend_result_persistence: self.backend_result_persistence, + io_kind: self.startup.io_kind, + } + } + + /// Safe observation fields for an existing sink. This method never logs. + /// The operation correlation is only a lookup token, never an operation identity. + /// Backend callers should replace a remote correlation with their fresh local + /// token before observing at their own responsibility boundary. + pub fn telemetry(&self) -> Result, ContractError> { + let received = match self.startup.primary.decode::() { + DecodeFailure::Known(received) => received, + _ => return Err(ContractError::ProjectionUnavailable), + }; + let mut facts = FailureFacts::new(received.facts().cause().clone())?; + for context in &received.facts().context().operations { + facts.with_context(context.clone())?; + } + // Never copy correlation from the attachment's untrusted context. + if let Some(correlation) = &self.operation.correlation { + facts.set_correlation(error_contract::CorrelationRef::new(correlation.as_str())?)?; + } + project_failure::(&facts, &()) + } + + /// Both boundaries deliberately send the minimum public-safe cause schema. + /// Runtime callers must resolve visibility under their authenticated scope; + /// this is a fresh Browser/public-schema projection, never an audience relabel. + pub fn problem( + &self, + boundary: StartupBoundary, + visibility: &SessionFailureVisibility, + instance: &str, + ) -> Result { + let projected = self + .startup + .primary + .project_for::(visibility)?; + ProblemResponse::from_projected(&projected, boundary, instance, self.public_facts()) + } +} + +/// Encoded public response. The body cannot be replaced with raw attachment data. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProblemResponse { + pub status: u16, + pub content_type: &'static str, + body: Vec, +} + +impl ProblemResponse { + pub fn from_projected( + projected: &ProjectedFailure, + boundary: StartupBoundary, + instance: &str, + facts: StartupProblemFacts, + ) -> Result { + facts.validate()?; + // Reuse the core encoder for URI, classification and envelope validation. + let core = to_problem_details(projected, &boundary, instance, budget())?; + let dto: ProblemDetails = + serde_json::from_slice(core.body()).map_err(|_| ContractError::Encoding)?; + let dto = StartupProblemDetails { + type_uri: dto.type_uri, + title: dto.title, + status: dto.status, + detail: dto.detail, + instance: dto.instance, + yoi_failure: dto.yoi_failure, + yoi_startup: facts, + }; + let body = serde_json::to_vec(&dto).map_err(|_| ContractError::Encoding)?; + if body.len() > MAX_STARTUP_PROBLEM_BYTES { + return Err(ContractError::TooLarge); + } + // The complete extension is included in the same byte/depth budget as the core. + if let DecodeFailure::Corrupt(error) = decode_problem_details::( + &body, + core.status, + core.content_type, + &boundary, + budget(), + ) { + return Err(error); + } + Ok(Self { + status: core.status, + content_type: core.content_type, + body, + }) + } + + pub fn body(&self) -> &[u8] { + &self.body + } + pub fn into_body(self) -> Vec { + self.body + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DecodeStartupProblem { + Known(WorkerCreateFailure), + UnsupportedVersion, + UnsupportedReason, + Invalid, + TooLarge, +} + +// Original-byte reader for safe unsupported metadata, including duplicate rejection. +#[derive(Deserialize)] +struct Metadata { + yoi_failure: EnvelopeMetadata, + yoi_startup: ExtensionMetadata, +} +#[derive(Deserialize)] +struct EnvelopeMetadata { + version: u16, + details_version: u16, + required_extensions: Vec, + // Read all core fields from original bytes even on unsupported envelopes. + // The core validates their values; this pass additionally rejects duplicate + // metadata that could otherwise be collapsed by its tolerant JSON reader. + #[serde(rename = "domain")] + _domain: String, + #[serde(rename = "reason")] + _reason: String, + #[serde(rename = "category")] + _category: String, + #[serde(rename = "details")] + _details: serde::de::IgnoredAny, + #[serde(rename = "explanation")] + _explanation: String, + #[serde(rename = "context")] + _context: error_contract::FailureContext, +} +#[derive(Deserialize)] +struct ExtensionMetadata { + version: u16, +} + +/// Bounded receiver normalization. Never derives acceptance/execution from HTTP, +/// returns raw unsupported bytes, trusts wire explanations, or retries anything. +/// `boundary` is the sender's known API boundary, not the next public boundary. +/// A known result must still be projected with current visibility before publication. +pub fn decode_startup_problem( + bytes: &[u8], + status: u16, + content_type: &str, + boundary: StartupBoundary, +) -> DecodeStartupProblem { + use DecodeStartupProblem::*; + // Core performs byte/depth guards before JSON allocation and validates standard + // Problem Details coherence, envelope identity and domain details on known data. + let decoded = + decode_problem_details::(bytes, status, content_type, &boundary, budget()); + match decoded { + DecodeFailure::Corrupt(ContractError::TooLarge) => TooLarge, + DecodeFailure::Corrupt(_) => Invalid, + decoded => { + let Ok(metadata) = serde_json::from_slice::(bytes) else { + return Invalid; + }; + if metadata.yoi_startup.version == 0 { + return Invalid; + } + if metadata.yoi_failure.version != error_contract::SCHEMA_VERSION + || metadata.yoi_failure.details_version != 1 + || !metadata.yoi_failure.required_extensions.is_empty() + || metadata.yoi_startup.version != STARTUP_FACTS_VERSION + { + return UnsupportedVersion; + } + let received = match decoded { + DecodeFailure::Known(received) => received, + DecodeFailure::Unsupported(_) => return UnsupportedReason, + DecodeFailure::Corrupt(_) => unreachable!(), + }; + // Parse the original bytes, not Value, so duplicate typed fields fail. + let Ok(dto) = serde_json::from_slice::(bytes) else { + return Invalid; + }; + if dto.yoi_startup.validate().is_err() { + return Invalid; + } + let Ok(projected) = project_failure::(received.facts(), &()) + else { + return Invalid; + }; + let Ok(primary) = StartupFailureAttachment::from_projected(&projected) else { + return Invalid; + }; + let extension = dto.yoi_startup; + match WorkerCreateFailure::new( + WorkerStartupFailure { + primary, + secondary: extension.secondary, + resources: extension.resources, + persistence: extension.persistence, + io_kind: extension.io_kind, + }, + extension.operation, + ) { + Ok(failure) => Known( + failure.with_backend_result_persistence(extension.backend_result_persistence), + ), + Err(_) => Invalid, + } + } + } +} diff --git a/crates/runtime-api/src/startup/http.rs b/crates/runtime-api/src/startup/http.rs new file mode 100644 index 000000000..67ad01a47 --- /dev/null +++ b/crates/runtime-api/src/startup/http.rs @@ -0,0 +1,193 @@ +//! Runtime API HTTP integration. No HTTP status is converted into execution facts. +use api_macros::reqwest::{ClientFailure, HttpClientError, ReceivedResponse}; +use error_contract::{DecodeFailure, InternalTransfer}; +use serde::Deserialize; + +use super::*; +use crate::RuntimeApiError; + +fn unknown_operation() -> StartupOperationFacts { + StartupOperationFacts { + acceptance: CreationAcceptance::Unknown, + execution: ExecutionOutcome::Unknown, + finalize: None, + backend_compensation: None, + correlation: None, + } +} + +/// RuntimeManagementApi invokes this only after checking authenticated Runtime +/// scope. That scope permits the minimum public-safe cause schema, not arbitrary +/// internal-transfer details. Resolve the public projection afresh on each call; +/// no stored/received projection is relabeled as Browser authorization. +pub(crate) fn runtime_error_response( + error: &RuntimeApiError, +) -> Option { + let legacy; + let failure = match error.create_failure() { + Some(failure) => failure, + None => { + let startup = error.error.startup.as_ref()?; + legacy = WorkerCreateFailure::new(startup.clone(), unknown_operation()); + match &legacy { + Ok(failure) => failure, + Err(_) => return Some(static_fallback()), + } + } + }; + let visibility = SessionFailureVisibility { + can_view_cause: true, + }; + Some( + match failure.problem(StartupBoundary::Runtime, &visibility, "") { + Ok(response) => api_macros::HttpErrorResponse { + status: response.status, + content_type: response.content_type, + body: response.into_body().into(), + }, + // Never expose the legacy/raw attachment if projection cannot be made. + Err(_) => static_fallback(), + }, + ) +} + +fn static_fallback() -> api_macros::HttpErrorResponse { + api_macros::HttpErrorResponse { + status: 500, + content_type: PROBLEM_CONTENT_TYPE, + body: error_contract::problem::STATIC_PROBLEM_FALLBACK + .to_vec() + .into(), + } +} + +fn normalized_error( + status: u16, + failure: WorkerCreateFailure, +) -> Result { + // Received text and legacy labels are not sources of local explanations. + // InternalTransfer is an internal observation policy, not Browser authority. + let projected = failure + .startup() + .primary + .project_for::(&()) + .map_err(|_| ClientFailure::ErrorResponseDecode { status })?; + let identity = projected.identity(); + let mut error = RuntimeApiError::new( + status, + format!("{}.{}", identity.domain, identity.reason), + projected.explanation(), + ); + error.error.startup = Some(failure.startup().clone()); + Ok(error.with_create_failure(failure)) +} + +fn local_observation( + status: u16, + reason: StartupBoundaryReason, +) -> Result { + let failure = WorkerCreateFailure::from_cause( + StartupCause::Boundary(StartupBoundaryCause { reason }), + unknown_operation(), + ) + .map_err(|_| ClientFailure::ErrorResponseDecode { status })?; + normalized_error(status, failure) +} + +// A deliberately narrow legacy schema. Bounded typed parsing avoids arbitrary JSON +// allocation; startup attachment strings remain opaque until domain codec validation. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyError { + error: LegacyDetail, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyDetail { + code: String, + message: String, + startup: Option, +} + +impl HttpClientError for RuntimeApiError { + fn decode_error(response: &ReceivedResponse) -> Result { + let status = response.status().as_u16(); + if !(400..=599).contains(&status) { + return Err(ClientFailure::ErrorResponseDecode { status }); + } + // The generated client has already bounded collection; enforce this + // smaller contract limit independently before any schema allocation. + if response.body().len() > MAX_STARTUP_PROBLEM_BYTES { + return local_observation(status, StartupBoundaryReason::TooLarge); + } + let content_type = match response.content_type() { + Ok(content_type) => content_type, + Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse), + }; + let media_type = content_type.split(';').next().unwrap_or("").trim(); + if media_type.eq_ignore_ascii_case(PROBLEM_CONTENT_TYPE) { + return match decode_startup_problem( + response.body(), + status, + content_type, + StartupBoundary::Runtime, + ) { + DecodeStartupProblem::Known(failure) => normalized_error(status, failure), + DecodeStartupProblem::UnsupportedVersion => { + local_observation(status, StartupBoundaryReason::UnsupportedVersion) + } + DecodeStartupProblem::UnsupportedReason => { + local_observation(status, StartupBoundaryReason::UnsupportedReason) + } + DecodeStartupProblem::Invalid => { + local_observation(status, StartupBoundaryReason::InvalidResponse) + } + DecodeStartupProblem::TooLarge => { + local_observation(status, StartupBoundaryReason::TooLarge) + } + }; + } + if !media_type.eq_ignore_ascii_case("application/json") { + return local_observation(status, StartupBoundaryReason::InvalidResponse); + } + // The typed legacy parser has fixed nesting and rejects unknown fields. + // Its total allocation/opaque attachment size is capped by the byte limit. + let legacy = match serde_json::from_slice::(response.body()) { + Ok(legacy) => legacy.error, + Err(_) => return local_observation(status, StartupBoundaryReason::Decode), + }; + if error_contract::validate_identifier(&legacy.code).is_err() + || error_contract::validate_explanation(&legacy.message).is_err() + { + return local_observation(status, StartupBoundaryReason::InvalidResponse); + } + let Some(mut startup) = legacy.startup else { + // Preserve ordinary legacy errors exactly, including their actual status. + return Ok(RuntimeApiError::new(status, legacy.code, legacy.message)); + }; + let projected = match startup.primary.decode::() { + DecodeFailure::Known(received) => match error_contract::project_failure::< + InternalTransfer, + _, + >(received.facts(), &()) + { + Ok(projected) => projected, + Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse), + }, + DecodeFailure::Corrupt(error_contract::ContractError::TooLarge) => { + return local_observation(status, StartupBoundaryReason::TooLarge); + } + // The legacy attachment codec intentionally exposes no unsupported + // metadata. Do not guess a reason/version from that opaque evidence. + _ => return local_observation(status, StartupBoundaryReason::InvalidResponse), + }; + startup.primary = match StartupFailureAttachment::from_projected(&projected) { + Ok(primary) => primary, + Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse), + }; + match WorkerCreateFailure::new(startup, unknown_operation()) { + Ok(failure) => normalized_error(status, failure), + Err(_) => local_observation(status, StartupBoundaryReason::InvalidResponse), + } + } +} diff --git a/crates/runtime-api/src/startup/storage.rs b/crates/runtime-api/src/startup/storage.rs new file mode 100644 index 000000000..ee4cc22d1 --- /dev/null +++ b/crates/runtime-api/src/startup/storage.rs @@ -0,0 +1,162 @@ +//! HTTP-independent snapshot of a Backend coordinator failure. This is a codec +//! for the existing reservation result, not a separate ledger or retry authority. +use super::*; +use crate::StoredStartupFailureAttachment; +use error_contract::{Storage, validate_json_input}; + +const STORAGE_VERSION: u16 = 1; + +// Private DTO: only the concrete failure encoder may construct a stored snapshot. +// In particular no arbitrary error/source, HTTP response or private identity fits here. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StoredWorkerCreateFailure { + version: u16, + primary: StoredStartupFailureAttachment, + facts: StartupProblemFacts, +} + +// Inspect versions from original input before attempting to understand future facts. +// IgnoredAny still checks JSON grammar without allocating arbitrary extension trees. +#[derive(Deserialize)] +struct StorageMetadata { + version: u16, + #[serde(rename = "primary")] + _primary: serde::de::IgnoredAny, + facts: ExtensionMetadata, +} + +impl ProjectFor for StartupBoundaryCause { + type Authorization = (); + type Details = Self; + + fn disclose(&self, _: &FailureFacts, _: &()) -> Result, ContractError> { + // The boundary domain stores only its local enum, never received context. + Ok(Disclosure::new(*self, boundary_explanation(self))) + } +} + +impl ProjectFor for StartupCause { + type Authorization = (); + type Details = Self; + + fn disclose( + &self, + facts: &FailureFacts, + _: &(), + ) -> Result, ContractError> { + match self { + Self::Session(cause) => { + let concrete = domain_facts(*cause, facts)?; + let disclosed = >::disclose( + cause, + &concrete, + &(), + )?; + Ok(wrap(disclosed, Self::Session, session_explanation(cause))) + } + Self::Runtime(cause) => { + let concrete = domain_facts(cause.clone(), facts)?; + let disclosed = + >::disclose(cause, &concrete, &())?; + Ok(wrap( + disclosed, + Self::Runtime, + runtime_explanation(cause).into(), + )) + } + Self::Boundary(cause) => { + let concrete = domain_facts(*cause, facts)?; + let disclosed = + >::disclose(cause, &concrete, &())?; + Ok(wrap( + disclosed, + Self::Boundary, + boundary_explanation(cause).into(), + )) + } + } + } +} + +impl WorkerCreateFailure { + /// Encode a bounded version-1 storage snapshot through each concrete domain's + /// Storage allowlist. Never encodes or parses an HTTP body. The caller selects + /// the Backend persistence marker (including prospective Persisted for an + /// atomic reservation write); encoding alone never proves persistence. + pub fn encode_storage(&self) -> Result { + let facts = self.public_facts(); + facts.validate()?; + let projected = self + .startup + .primary + .project_for::(&())?; + let dto = StoredWorkerCreateFailure { + version: STORAGE_VERSION, + primary: StoredStartupFailureAttachment::from_projected(&projected)?, + facts, + }; + let encoded = serde_json::to_string(&dto).map_err(|_| ContractError::Encoding)?; + // Includes escaped attachment bytes and the entire enclosing snapshot. + validate_json_input(encoded.as_bytes(), budget())?; + Ok(encoded) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DecodeStoredWorkerCreateFailure { + Known(WorkerCreateFailure), + Unsupported, + Corrupt(ContractError), +} + +/// Read a bounded storage snapshot without an HTTP classification or boundary. +/// Unsupported/corrupt evidence never becomes a fabricated primary cause, and a +/// Known result still needs current visibility for any later public projection. +/// Missing legacy columns are represented by the store's Option, not this codec. +pub fn decode_stored_worker_create_failure(input: &str) -> DecodeStoredWorkerCreateFailure { + match decode_checked(input) { + Ok(value) => value, + Err(error) => DecodeStoredWorkerCreateFailure::Corrupt(error), + } +} + +fn decode_checked(input: &str) -> Result { + use DecodeStoredWorkerCreateFailure::*; + // Run the shared lexical guard over the whole input before any JSON parsing. + validate_json_input(input.as_bytes(), budget())?; + let metadata: StorageMetadata = + serde_json::from_str(input).map_err(|_| ContractError::Invalid)?; + if metadata.version == 0 || metadata.facts.version == 0 { + return Err(ContractError::Invalid); + } + if metadata.version != STORAGE_VERSION || metadata.facts.version != STARTUP_FACTS_VERSION { + return Ok(Unsupported); + } + // Original-byte typed decoding rejects duplicates rather than collapsing a Value. + let dto: StoredWorkerCreateFailure = + serde_json::from_str(input).map_err(|_| ContractError::InvalidDetails)?; + dto.facts.validate()?; + let received = match dto.primary.decode::() { + DecodeFailure::Known(received) => received, + DecodeFailure::Unsupported(_) => return Ok(Unsupported), + DecodeFailure::Corrupt(error) => return Err(error), + }; + // Deserialization is not a capability. Project again instead of relabeling or + // retaining raw loaded attachment bytes (including injected explanations/context). + let projected = project_failure::(received.facts(), &())?; + let primary = StartupFailureAttachment::from_projected(&projected)?; + let facts = dto.facts; + let failure = WorkerCreateFailure::new( + WorkerStartupFailure { + primary, + secondary: facts.secondary, + resources: facts.resources, + persistence: facts.persistence, + io_kind: facts.io_kind, + }, + facts.operation, + )? + .with_backend_result_persistence(facts.backend_result_persistence); + Ok(Known(failure)) +} diff --git a/crates/runtime-api/tests/fixtures/startup-session-record-denied.problem.json b/crates/runtime-api/tests/fixtures/startup-session-record-denied.problem.json new file mode 100644 index 000000000..182a33fca --- /dev/null +++ b/crates/runtime-api/tests/fixtures/startup-session-record-denied.problem.json @@ -0,0 +1,58 @@ +{ + "detail": "Worker creation could not complete Session registration: Subject is not authorized for required operation session.record. Creation acceptance and execution outcome are reported separately.", + "instance": "/workers/create/fixture-812", + "status": 409, + "title": "Operation conflict", + "type": "urn:yoi:problem:conflict", + "yoi_failure": { + "category": "authorization", + "context": { + "causes": [], + "correlation": null, + "diagnostics": [], + "operations": [], + "supplementary_omitted": false + }, + "details": { + "principal": "subject", + "reason": "permission_denied", + "required_operation": "session.record" + }, + "details_version": 1, + "domain": "session_registration", + "explanation": "Worker creation could not complete Session registration: Subject is not authorized for required operation session.record. Creation acceptance and execution outcome are reported separately.", + "reason": "permission_denied", + "required_extensions": [], + "version": 1 + }, + "yoi_startup": { + "io_kind": "permission_denied", + "operation": { + "acceptance": "rejected", + "backend_compensation": null, + "correlation": "startup-fixture-812", + "execution": "rejected", + "finalize": null + }, + "persistence": "outcome_unknown", + "resources": { + "cleanup": { + "failed": "workdir" + }, + "durable_effects_possible": true, + "resources": "worker_startup" + }, + "secondary": [ + { + "failure": "workdir", + "kind": "cleanup" + }, + { + "kind": "persistence", + "outcome": "outcome_unknown", + "phase": "metadata_save" + } + ], + "version": 1 + } +} diff --git a/crates/runtime-api/tests/startup_contract.rs b/crates/runtime-api/tests/startup_contract.rs new file mode 100644 index 000000000..00964f3fc --- /dev/null +++ b/crates/runtime-api/tests/startup_contract.rs @@ -0,0 +1,918 @@ +//! Producer/consumer checks against the production constructor and decoder. +use error_contract::problem::ApiBoundary as _; +use error_contract::{ + Browser, ContractError, DecodeFailure, FailureCause, FailureFacts, InternalTransfer, + OperationContext, ProjectFor, WireDetails, project_failure, +}; +use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause}; +use runtime_api::startup::*; +use runtime_api::{ + RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure, + StartupFailureAttachment, StartupFailurePhase, StartupIoKind, StartupPersistenceOutcome, + StartupResourceFacts, StartupResources, StartupSecondaryFailure, StartupStage, + WorkerStartupFailure, +}; +use serde_json::{Value, json}; + +fn visible() -> SessionFailureVisibility { + SessionFailureVisibility { + can_view_cause: true, + } +} +fn operation(acceptance: CreationAcceptance, execution: ExecutionOutcome) -> StartupOperationFacts { + StartupOperationFacts { + acceptance, + execution, + finalize: None, + backend_compensation: None, + correlation: Some(StartupCorrelation::new("startup-fixture-812").unwrap()), + } +} +fn session() -> StartupCause { + StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)) +} +fn failure(cause: StartupCause) -> WorkerCreateFailure { + WorkerCreateFailure::from_cause( + cause, + operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown), + ) + .unwrap() +} +fn decode(value: &Value, status: u16, boundary: StartupBoundary) -> DecodeStartupProblem { + decode_startup_problem( + &serde_json::to_vec(value).unwrap(), + status, + PROBLEM_CONTENT_TYPE, + boundary, + ) +} +fn known(result: DecodeStartupProblem) -> WorkerCreateFailure { + match result { + DecodeStartupProblem::Known(failure) => failure, + other => panic!("{other:?}"), + } +} +fn fixture_failure() -> WorkerCreateFailure { + let base = failure(session()); + let (mut startup, _) = base.into_parts(); + startup.secondary = vec![ + StartupSecondaryFailure::Cleanup { + failure: StartupCleanupFailure::Workdir, + }, + StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::MetadataSave, + outcome: StartupPersistenceOutcome::OutcomeUnknown, + }, + ]; + startup.resources = Some(StartupResourceFacts { + resources: StartupResources::WorkerStartup, + cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir), + durable_effects_possible: true, + }); + startup.persistence = Some(StartupPersistenceOutcome::OutcomeUnknown); + startup.io_kind = Some(StartupIoKind::PermissionDenied); + WorkerCreateFailure::new( + startup, + operation(CreationAcceptance::Rejected, ExecutionOutcome::Rejected), + ) + .unwrap() +} + +#[test] +fn production_public_fixture_is_stable_and_consumable() { + let failure = fixture_failure(); + let response = failure + .problem( + StartupBoundary::Browser, + &visible(), + "/workers/create/fixture-812", + ) + .unwrap(); + assert_eq!(response.status, 409); + assert_eq!(response.content_type, PROBLEM_CONTENT_TYPE); + let value: Value = serde_json::from_slice(response.body()).unwrap(); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/startup-session-record-denied.problem.json"); + let fixture: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + assert_eq!( + value, fixture, + "fixture must be generated by the public production constructor" + ); + let decoded = known(decode(&fixture, 409, StartupBoundary::Browser)); + assert_eq!(decoded.operation(), failure.operation()); + assert_eq!(decoded.public_facts(), failure.public_facts()); + assert_eq!( + decoded + .startup() + .primary + .decode::() + .known_cause(), + session() + ); + assert!(fixture["detail"].as_str().unwrap().contains("Subject")); + assert!( + fixture["detail"] + .as_str() + .unwrap() + .contains("session.record") + ); + assert!(fixture["detail"].as_str().unwrap().contains("Worker")); + assert!(fixture["yoi_startup"].get("primary").is_none()); +} + +/// Captured from the real signed Session registration → Runtime → Browser tests +/// in workspace-server. These are observed operation facts, not a declaration +/// that an HTTP conflict rejected creation or that successful cleanup erased effects. +#[test] +fn connected_browser_fixtures_preserve_observed_operation_and_storage_facts() { + for (body, status, cause) in [ + ( + include_bytes!("fixtures/startup-browser-session-record-denied.problem.json") + .as_slice(), + 409, + SessionRegistrationCause::denied(SessionPrincipal::Subject), + ), + ( + include_bytes!("fixtures/startup-browser-registration-response-lost.problem.json") + .as_slice(), + 502, + SessionRegistrationCause::outcome_unknown(), + ), + ] { + let failure = known(decode_startup_problem( + body, + status, + PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser, + )); + assert_eq!( + failure + .startup() + .primary + .decode::() + .known_cause(), + StartupCause::Session(cause) + ); + assert_eq!(failure.operation().acceptance, CreationAcceptance::Unknown); + assert_eq!(failure.operation().execution, ExecutionOutcome::Errored); + assert_eq!( + failure.operation().backend_compensation, + Some(CompensationOutcome::Unknown) + ); + assert_eq!( + failure.startup().persistence, + Some(StartupPersistenceOutcome::Persisted) + ); + assert_eq!( + failure.backend_result_persistence(), + Some(StartupPersistenceOutcome::Persisted) + ); + let encoded = failure.encode_storage().unwrap(); + let DecodeStoredWorkerCreateFailure::Known(saved) = + decode_stored_worker_create_failure(&encoded) + else { + panic!("connected result must survive HTTP-independent storage"); + }; + assert_eq!(saved.public_facts(), failure.public_facts()); + let response = saved + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + assert_eq!(response.status, status); + let original: Value = serde_json::from_slice(body).unwrap(); + let projected: Value = serde_json::from_slice(response.body()).unwrap(); + assert_eq!(projected["detail"], original["detail"]); + assert_eq!(projected["yoi_startup"], original["yoi_startup"]); + } +} + +// A tiny test-only adapter so assertions never format raw received data. +trait CauseFromDecode { + fn known_cause(self) -> StartupCause; +} +impl CauseFromDecode for DecodeFailure { + fn known_cause(self) -> StartupCause { + match self { + DecodeFailure::Known(received) => received.facts().cause().clone(), + _ => panic!("expected known cause"), + } + } +} + +#[test] +fn original_domain_wire_identity_and_details_have_no_union_wrapper() { + for cause in [ + session(), + StartupCause::Runtime(RuntimeStartupCause { + stage: StartupStage::Binding, + reason: RuntimeStartupReason::AttributionFailed, + }), + StartupCause::Boundary(StartupBoundaryCause { + reason: StartupBoundaryReason::Transport, + }), + ] { + let response = failure(cause.clone()) + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + let value: Value = serde_json::from_slice(response.body()).unwrap(); + let id = cause.identity(); + assert_eq!(value["yoi_failure"]["domain"], id.domain); + assert_eq!(value["yoi_failure"]["reason"], id.reason); + assert_eq!(value["yoi_failure"]["details_version"], id.details_version); + assert_eq!( + value["yoi_failure"]["details"], + serde_json::to_value(&cause).unwrap() + ); + assert!(value["yoi_failure"]["details"].get("Session").is_none()); + assert!(StartupCause::supports( + id.domain, + id.reason, + id.details_version + )); + let decoded = known(decode(&value, response.status, StartupBoundary::Browser)); + assert_eq!( + decoded + .startup() + .primary + .decode::() + .known_cause(), + cause + ); + } +} + +#[test] +fn browser_session_projection_requires_current_visibility_even_after_receipt() { + let failure = failure(session()); + assert_eq!( + failure + .problem( + StartupBoundary::Browser, + &SessionFailureVisibility::default(), + "" + ) + .unwrap_err(), + ContractError::InvalidDetails + ); + let wire = failure + .problem(StartupBoundary::Runtime, &visible(), "") + .unwrap(); + let received = known(decode_startup_problem( + wire.body(), + wire.status, + wire.content_type, + StartupBoundary::Runtime, + )); + assert!( + received + .problem( + StartupBoundary::Browser, + &SessionFailureVisibility::default(), + "" + ) + .is_err() + ); + assert_eq!( + received + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap() + .status, + 409 + ); +} + +#[test] +fn boundary_classification_is_explicit_not_category_text_or_inner_status() { + let cases = [ + ( + StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Operator)), + 403, + 403, + ), + (session(), 409, 403), + ( + StartupCause::Session(SessionRegistrationCause::attribution_missing()), + 409, + 409, + ), + ( + StartupCause::Session(SessionRegistrationCause::binding_mismatch()), + 409, + 409, + ), + ( + StartupCause::Session(SessionRegistrationCause::transport()), + 502, + 502, + ), + ( + StartupCause::Session(SessionRegistrationCause::outcome_unknown()), + 502, + 502, + ), + ( + StartupCause::Session(SessionRegistrationCause::invalid_response()), + 502, + 502, + ), + ]; + for (cause, browser, runtime) in cases { + assert_eq!( + StartupBoundary::Browser.classify(&cause).status.code(), + browser + ); + assert_eq!( + StartupBoundary::Runtime.classify(&cause).status.code(), + runtime + ); + } + for (reason, status) in [ + (StartupBoundaryReason::Timeout, 504), + (StartupBoundaryReason::Transport, 502), + (StartupBoundaryReason::Decode, 502), + (StartupBoundaryReason::InvalidResponse, 502), + (StartupBoundaryReason::TooLarge, 502), + (StartupBoundaryReason::UnsupportedVersion, 502), + (StartupBoundaryReason::UnsupportedReason, 502), + (StartupBoundaryReason::FinalizeFailed, 500), + ] { + let cause = StartupCause::Boundary(StartupBoundaryCause { reason }); + for boundary in [StartupBoundary::Browser, StartupBoundary::Runtime] { + assert_eq!(boundary.classify(&cause).status.code(), status); + } + } + for (reason, status) in [ + (RuntimeStartupReason::SegmentEmpty, 400), + (RuntimeStartupReason::SystemPromptMissing, 400), + (RuntimeStartupReason::MetadataPending, 409), + (RuntimeStartupReason::MetadataMissing, 409), + (RuntimeStartupReason::MetadataInactive, 409), + (RuntimeStartupReason::ManifestMissing, 409), + (RuntimeStartupReason::AllocationFailed, 503), + (RuntimeStartupReason::SessionReadFailed, 500), + (RuntimeStartupReason::MetadataReadFailed, 500), + (RuntimeStartupReason::ProviderFailed, 500), + (RuntimeStartupReason::AttributionFailed, 500), + (RuntimeStartupReason::FeatureFailed, 500), + (RuntimeStartupReason::StartupFailed, 500), + ] { + let cause = StartupCause::Runtime(RuntimeStartupCause { + stage: StartupStage::Controller, + reason, + }); + for boundary in [StartupBoundary::Browser, StartupBoundary::Runtime] { + assert_eq!(boundary.classify(&cause).status.code(), status); + } + } +} + +#[test] +fn all_operation_facts_remain_independent_including_unknown_and_finalize() { + for acceptance in [ + CreationAcceptance::Rejected, + CreationAcceptance::Accepted, + CreationAcceptance::Unknown, + ] { + for execution in [ + ExecutionOutcome::Accepted, + ExecutionOutcome::Busy, + ExecutionOutcome::Rejected, + ExecutionOutcome::Errored, + ExecutionOutcome::Unsupported, + ExecutionOutcome::Unknown, + ] { + for compensation in [ + CompensationOutcome::Completed, + CompensationOutcome::Residual, + CompensationOutcome::Unknown, + ] { + let (startup, _) = fixture_failure().into_parts(); + let mut operation = operation(acceptance, execution); + operation.finalize = Some(FinalizeFacts { + stage: FinalizeStage::WorkerRegistry, + compensation, + }); + let original = WorkerCreateFailure::new(startup, operation).unwrap(); + let response = original + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + assert_eq!(response.status, 409); // primary cause only, not outcomes/cleanup + let received = known(decode_startup_problem( + response.body(), + response.status, + response.content_type, + StartupBoundary::Browser, + )); + assert_eq!(received.public_facts(), original.public_facts()); + } + } + } +} + +#[test] +fn transport_unknown_never_becomes_rejected_or_accepted_from_http_or_hints() { + let response = failure(StartupCause::Boundary(StartupBoundaryCause { + reason: StartupBoundaryReason::Timeout, + })) + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + assert_eq!(response.status, 504); + let mut value: Value = serde_json::from_slice(response.body()).unwrap(); + value["retry_after"] = json!(0); + value["accepted"] = json!(false); // unrecognized RFC extension is not outcome authority + let decoded = known(decode(&value, 504, StartupBoundary::Browser)); + assert_eq!(decoded.operation().acceptance, CreationAcceptance::Unknown); + assert_eq!(decoded.operation().execution, ExecutionOutcome::Unknown); + value["yoi_startup"]["operation"] + .as_object_mut() + .unwrap() + .remove("acceptance"); + assert_eq!( + decode(&value, 504, StartupBoundary::Browser), + DecodeStartupProblem::Invalid + ); +} + +#[test] +fn received_text_and_context_are_reprojected_not_forwarded() { + let original = failure(session()) + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + let mut value: Value = serde_json::from_slice(original.body()).unwrap(); + let secret = "RAW_UPSTREAM_CREDENTIAL_OR_PATH"; + value["detail"] = json!(secret); + value["yoi_failure"]["explanation"] = json!(secret); + value["yoi_failure"]["context"]["diagnostics"] = + json!([{ "code":"raw", "explanation":secret }]); + value["yoi_failure"]["context"]["operations"] = json!([ + {"operation":"session.record","stage":"registration","operation_ref":secret}, + {"operation":secret,"stage":"raw","operation_ref":secret}]); + value["yoi_failure"]["context"]["correlation"] = json!(secret); + let received = known(decode(&value, 409, StartupBoundary::Browser)); + let public = received + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + let text = std::str::from_utf8(public.body()).unwrap(); + assert!(!text.contains(secret)); + assert!(text.contains("Subject")); + assert!(text.contains("session.record")); + assert_eq!(public.status, original.status); + let (mut startup, operation) = received.into_parts(); + startup.secondary.reverse(); + assert_eq!( + WorkerCreateFailure::new(startup, operation) + .unwrap() + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap() + .status, + 409 + ); +} + +#[test] +fn concrete_domain_allowlists_filter_projection_context() { + let mut facts = FailureFacts::new(session()).unwrap(); + let mut allowed = OperationContext::new("session.record", "authorization").unwrap(); + allowed.operation_ref = Some("DO_NOT_PUBLISH".into()); + facts.with_context(allowed).unwrap(); + facts + .with_context(OperationContext::new("DO_NOT_PUBLISH", "raw").unwrap()) + .unwrap(); + let projected = project_failure::(&facts, &visible()).unwrap(); + assert_eq!(projected.context().operations.len(), 1); + assert_eq!(projected.context().operations[0].operation_ref, None); + assert!( + >::disclose(facts.cause(), &facts, &()) + .is_ok() + ); +} + +#[test] +fn unknown_version_reason_and_bad_envelopes_are_distinct() { + let response = fixture_failure() + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + let original: Value = serde_json::from_slice(response.body()).unwrap(); + for field in ["version", "details_version"] { + let mut value = original.clone(); + value["yoi_failure"][field] = json!(77); + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::UnsupportedVersion + ); + } + let mut value = original.clone(); + value["yoi_startup"]["version"] = json!(77); + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::UnsupportedVersion + ); + value = original.clone(); + value["yoi_failure"]["required_extensions"] = json!(["future-contract"]); + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::UnsupportedVersion + ); + value = original.clone(); + value["yoi_failure"]["reason"] = json!("future_reason"); + value["yoi_failure"]["category"] = json!("future_category"); + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::UnsupportedReason + ); + value = original.clone(); + value["yoi_failure"]["domain"] = json!("future_domain"); + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::UnsupportedReason + ); + for mutate in ["category", "domain", "reason"] { + let mut value = original.clone(); + // Another KNOWN identity with mismatching typed details must be invalid. + value["yoi_failure"][mutate] = match mutate { + "category" => json!("internal"), + "domain" => json!("startup_boundary"), + _ => json!("transport"), + }; + let result = decode(&value, 409, StartupBoundary::Browser); + if mutate != "domain" { + assert_eq!(result, DecodeStartupProblem::Invalid); + } else { + assert_eq!(result, DecodeStartupProblem::UnsupportedReason); + } + } + for field in ["version", "details_version"] { + let mut value = original.clone(); + value["yoi_failure"][field] = json!(0); + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::Invalid + ); + } +} + +#[test] +fn decoder_rejects_incoherence_bad_shapes_duplicate_fields_and_limits() { + let response = failure(session()) + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + let original: Value = serde_json::from_slice(response.body()).unwrap(); + assert_eq!( + decode_startup_problem( + response.body(), + 403, + PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser + ), + DecodeStartupProblem::Invalid + ); + assert_eq!( + decode_startup_problem( + response.body(), + 409, + "application/json", + StartupBoundary::Browser + ), + DecodeStartupProblem::Invalid + ); + assert!(matches!( + decode_startup_problem( + response.body(), + 409, + "Application/Problem+Json; charset=utf-8", + StartupBoundary::Browser + ), + DecodeStartupProblem::Known(_) + )); + assert_eq!( + decode_startup_problem( + response.body(), + 409, + PROBLEM_CONTENT_TYPE, + StartupBoundary::Runtime + ), + DecodeStartupProblem::Invalid + ); + for (field, replacement) in [ + ("status", json!(502)), + ("type", json!("urn:wrong")), + ("title", json!("raw upstream")), + ("detail", json!("different")), + ("instance", json!("%zz")), + ] { + let mut value = original.clone(); + value[field] = replacement; + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::Invalid + ); + } + for field in ["yoi_failure", "yoi_startup"] { + let mut value = original.clone(); + value.as_object_mut().unwrap().remove(field); + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::Invalid + ); + } + let text = std::str::from_utf8(response.body()).unwrap(); + for (needle, duplicated) in [ + ("\"status\":409", "\"status\":409,\"status\":409"), + ( + "\"acceptance\":\"unknown\"", + "\"acceptance\":\"unknown\",\"acceptance\":\"rejected\"", + ), + ( + "\"principal\":\"subject\"", + "\"principal\":\"subject\",\"principal\":\"operator\"", + ), + ] { + let duplicate = text.replace(needle, duplicated); + assert_ne!(duplicate, text); + assert_eq!( + decode_startup_problem( + duplicate.as_bytes(), + 409, + PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser + ), + DecodeStartupProblem::Invalid + ); + } + let oversized = vec![b' '; MAX_STARTUP_PROBLEM_BYTES + 1]; + assert_eq!( + decode_startup_problem( + &oversized, + 409, + PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser + ), + DecodeStartupProblem::TooLarge + ); + let deep = format!("{}0{}", "[".repeat(1000), "]".repeat(1000)); + assert_eq!( + decode_startup_problem( + deep.as_bytes(), + 409, + PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser + ), + DecodeStartupProblem::Invalid + ); + for bytes in [b"not json".as_slice(), b"\xff".as_slice(), b"{}".as_slice()] { + assert_eq!( + decode_startup_problem(bytes, 409, PROBLEM_CONTENT_TYPE, StartupBoundary::Browser), + DecodeStartupProblem::Invalid + ); + } +} + +#[test] +fn safe_correlation_and_extension_validation_cannot_be_bypassed_by_serde() { + for unsafe_value in [ + "", + "path/to/file", + "https://host", + "Bearer secret", + "query?token=secret", + "control\n", + ] { + assert!(StartupCorrelation::new(unsafe_value).is_err()); + assert!(serde_json::from_value::(json!(unsafe_value)).is_err()); + } + assert!(StartupCorrelation::new("a".repeat(129)).is_err()); + let failure = fixture_failure(); + let mut facts = failure.public_facts(); + facts.secondary = vec![ + StartupSecondaryFailure::Cleanup { + failure: StartupCleanupFailure::Workdir + }; + 9 + ]; + assert!(facts.validate().is_err()); + let (mut startup, operation) = failure.into_parts(); + startup.secondary = facts.secondary; + assert!(WorkerCreateFailure::new(startup, operation).is_err()); + let response = fixture_failure() + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + let mut value: Value = serde_json::from_slice(response.body()).unwrap(); + value["yoi_startup"]["operation"]["correlation"] = json!("/private/path"); + assert_eq!( + decode(&value, 409, StartupBoundary::Browser), + DecodeStartupProblem::Invalid + ); + assert!( + fixture_failure() + .problem(StartupBoundary::Browser, &visible(), "invalid%uri") + .is_err() + ); +} + +#[test] +fn unknown_or_corrupt_internal_attachments_are_retained_but_not_publishable() { + for raw in ["{invalid SECRET", "{}"] { + let primary: StartupFailureAttachment = serde_json::from_value(json!(raw)).unwrap(); + let failure = WorkerCreateFailure::new( + WorkerStartupFailure { + primary, + secondary: vec![], + resources: None, + persistence: None, + io_kind: None, + }, + operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown), + ) + .unwrap(); + assert!( + failure + .problem(StartupBoundary::Browser, &visible(), "") + .is_err() + ); + assert!(!format!("{failure:?}").contains("SECRET")); + } +} + +#[test] +fn unsupported_envelopes_still_reject_duplicate_safe_metadata() { + let response = failure(session()) + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + let text = std::str::from_utf8(response.body()) + .unwrap() + .replace("\"version\":1", "\"version\":77"); + assert_eq!( + decode_startup_problem( + text.as_bytes(), + 409, + PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser + ), + DecodeStartupProblem::UnsupportedVersion + ); + for (needle, duplicate) in [ + ( + "\"reason\":\"permission_denied\"", + "\"reason\":\"permission_denied\",\"reason\":\"future\"", + ), + ( + "\"domain\":\"session_registration\"", + "\"domain\":\"session_registration\",\"domain\":\"future\"", + ), + ( + "\"category\":\"authorization\"", + "\"category\":\"authorization\",\"category\":\"future\"", + ), + ] { + let bytes = text.replace(needle, duplicate); + assert_ne!(bytes, text); + assert_eq!( + decode_startup_problem( + bytes.as_bytes(), + 409, + PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser + ), + DecodeStartupProblem::Invalid + ); + } +} + +#[test] +fn backend_compensation_is_independent_of_finalize_runtime_cleanup_and_outcomes() { + for backend in [ + None, + Some(CompensationOutcome::Completed), + Some(CompensationOutcome::Residual), + Some(CompensationOutcome::Unknown), + ] { + for finalize in [ + None, + Some(FinalizeFacts { + stage: FinalizeStage::WorkdirRegistry, + compensation: CompensationOutcome::Residual, + }), + ] { + let (startup, mut operation) = fixture_failure().into_parts(); + operation.backend_compensation = backend; + operation.finalize = finalize; + let original = WorkerCreateFailure::new(startup, operation).unwrap(); + let response = original + .problem(StartupBoundary::Browser, &visible(), "") + .unwrap(); + let received = known(decode_startup_problem( + response.body(), + response.status, + response.content_type, + StartupBoundary::Browser, + )); + assert_eq!(received.public_facts(), original.public_facts()); + assert_eq!(received.operation().backend_compensation, backend); + assert_eq!(received.operation().finalize, finalize); + } + } + for stage in [ + FinalizeStage::TicketAssignmentBind, + FinalizeStage::TicketAssignmentCurrent, + FinalizeStage::TicketStateAccept, + FinalizeStage::WorkdirRegistry, + ] { + assert_eq!( + serde_json::from_value::(serde_json::to_value(stage).unwrap()).unwrap(), + stage + ); + } +} + +#[test] +fn telemetry_uses_typed_context_and_operation_correlation_not_attachment_identity() { + use error_contract::{CorrelationRef, Telemetry, observation_fields}; + let mut facts = FailureFacts::new(session()).unwrap(); + let mut context = OperationContext::new("session.record", "registration").unwrap(); + context.operation_ref = Some("PRIVATE_OPERATION_ID".into()); + facts.with_context(context).unwrap(); + facts + .with_context(OperationContext::new("SECRET_CONTEXT", "SECRET_STAGE").unwrap()) + .unwrap(); + facts + .set_correlation(CorrelationRef::new("UNTRUSTED_ATTACHMENT_ID").unwrap()) + .unwrap(); + let projected = project_failure::(&facts, &()).unwrap(); + let fields = observation_fields(&projected); + assert_eq!(fields.domain, "session_registration"); + assert_eq!(fields.reason, "permission_denied"); + assert_eq!(fields.operation, Some("worker_startup")); + assert_eq!(fields.stage, Some("binding")); + assert_eq!(fields.correlation, Some("UNTRUSTED_ATTACHMENT_ID")); + assert!( + projected + .context() + .operations + .iter() + .all(|op| op.operation_ref.is_none()) + ); + + let transfer = project_failure::(&facts, &()).unwrap(); + let startup = WorkerStartupFailure { + primary: StartupFailureAttachment::from_projected(&transfer).unwrap(), + secondary: vec![], + resources: None, + persistence: None, + io_kind: None, + }; + let mut operation = operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown); + operation.correlation = Some(StartupCorrelation::new("fresh-backend-correlation").unwrap()); + let failure = WorkerCreateFailure::new(startup, operation).unwrap(); + let telemetry = failure.telemetry().unwrap(); + assert_eq!( + observation_fields(&telemetry).correlation, + Some("fresh-backend-correlation") + ); + assert_eq!(telemetry.identity(), session().identity()); + let encoded = error_contract::encode_projected_failure( + &telemetry, + error_contract::Budget::new(32768, 32768), + ) + .unwrap(); + let text = std::str::from_utf8(encoded.as_bytes()).unwrap(); + for secret in [ + "PRIVATE_OPERATION_ID", + "SECRET_CONTEXT", + "SECRET_STAGE", + "UNTRUSTED_ATTACHMENT_ID", + ] { + assert!(!text.contains(secret)); + } + let (startup, mut operation) = failure.into_parts(); + operation.correlation = None; + assert_eq!( + observation_fields( + &WorkerCreateFailure::new(startup, operation) + .unwrap() + .telemetry() + .unwrap() + ) + .correlation, + None + ); +} + +#[test] +fn public_dto_schema_is_accessible_for_generator() { + let schema = schemars::schema_for!(StartupProblemDetails); + let value = serde_json::to_value(schema).unwrap(); + for field in [ + "type", + "title", + "status", + "detail", + "instance", + "yoi_failure", + "yoi_startup", + ] { + assert!(value["properties"].get(field).is_some(), "{field}"); + } + let schema_text = serde_json::to_string(&value).unwrap(); + assert!(!schema_text.contains("StartupFailureAttachment")); + assert!(schema_text.contains("CreationAcceptance")); + assert!(schema_text.contains("ExecutionOutcome")); +} diff --git a/crates/runtime-api/tests/startup_http.rs b/crates/runtime-api/tests/startup_http.rs new file mode 100644 index 000000000..45ea2f348 --- /dev/null +++ b/crates/runtime-api/tests/startup_http.rs @@ -0,0 +1,313 @@ +//! Actual generated RuntimeApi client error hook with bounded HTTP metadata/bytes. +use api_macros::HttpError as _; +use error_contract::{DecodeFailure, FailureCause}; +use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause}; +use runtime_api::startup::*; +use runtime_api::{RuntimeApiClient, RuntimeApiError, client_support, server_support}; +use serde_json::{Value, json}; + +fn failure() -> WorkerCreateFailure { + WorkerCreateFailure::from_cause( + StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)), + StartupOperationFacts { + acceptance: CreationAcceptance::Accepted, + execution: ExecutionOutcome::Errored, + finalize: Some(FinalizeFacts { + stage: FinalizeStage::TicketAssignmentBind, + compensation: CompensationOutcome::Unknown, + }), + backend_compensation: Some(CompensationOutcome::Residual), + correlation: Some(StartupCorrelation::new("backend-local-812").unwrap()), + }, + ) + .unwrap() +} +fn cause(error: &RuntimeApiError) -> StartupCause { + match error + .create_failure() + .unwrap() + .startup() + .primary + .decode::() + { + DecodeFailure::Known(received) => received.facts().cause().clone(), + _ => panic!("expected locally typed cause"), + } +} +fn assert_unknown(error: &RuntimeApiError, reason: StartupBoundaryReason) { + let operation = error.create_failure().unwrap().operation(); + assert_eq!(operation.acceptance, CreationAcceptance::Unknown); + assert_eq!(operation.execution, ExecutionOutcome::Unknown); + assert_eq!(operation.finalize, None); + assert_eq!(operation.backend_compensation, None); + assert_eq!( + cause(error), + StartupCause::Boundary(StartupBoundaryCause { reason }) + ); +} + +async fn receive(status: u16, types: Vec<&'static str>, body: Vec) -> RuntimeApiError { + use server_support::framework::{Body, Response, Router, get}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let router = Router::new().route( + "/v1/workers/{worker_id}", + get(move || { + let types = types.clone(); + let body = body.clone(); + async move { + let mut response = Response::builder().status(status); + for content_type in types { + response = response.header("content-type", content_type); + } + response.body(Body::from(body)).unwrap() + } + }), + ); + let server = tokio::spawn(async move { + server_support::framework::serve(listener, router) + .await + .unwrap(); + }); + let client = RuntimeApiClient::builder(&format!("http://{address}")) + .unwrap() + .request_timeout(std::time::Duration::from_secs(3)) + .build() + .unwrap(); + let result = client.get_worker("test".into()).await; + server.abort(); + match result { + Err(client_support::ClientError::Public { + status: actual, + error, + }) => { + assert_eq!(actual.as_u16(), status); + assert_eq!(error.status(), status); + error + } + _ => panic!("expected normalized public error"), + } +} + +#[test] +fn runtime_error_projects_public_schema_fresh_and_keeps_local_facts_off_serde() { + let failure = failure(); + let error = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE") + .with_create_failure(failure.clone()); + let legacy = serde_json::to_value(&error).unwrap(); + assert!(legacy.get("create_failure").is_none()); + let reparsed: RuntimeApiError = serde_json::from_value(legacy).unwrap(); + assert!(reparsed.create_failure().is_none()); + let response = error.custom_response().unwrap(); + assert_eq!(response.status, 403); // runtime boundary is not Browser's 409 + assert_eq!(response.content_type, PROBLEM_CONTENT_TYPE); + let text = std::str::from_utf8(response.body.as_ref()).unwrap(); + assert!(!text.contains("PRIVATE_CODE")); + assert!(!text.contains("PRIVATE_MESSAGE")); + let decoded = match decode_startup_problem( + response.body.as_ref(), + response.status, + response.content_type, + StartupBoundary::Runtime, + ) { + DecodeStartupProblem::Known(failure) => failure, + _ => panic!("expected known public-safe Runtime failure"), + }; + assert_eq!(decoded.public_facts(), failure.public_facts()); + assert_eq!(error.into_create_failure(), Some(failure)); + let ordinary = RuntimeApiError::new(404, "not_found", "not found"); + assert!(ordinary.custom_response().is_none()); +} + +#[test] +fn invalid_projection_uses_static_fallback_not_legacy_raw_serialization() { + let (mut startup, operation) = failure().into_parts(); + startup.primary = serde_json::from_value(json!("RAW_PRIVATE_CREDENTIAL")).unwrap(); + let error = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE") + .with_create_failure(WorkerCreateFailure::new(startup, operation).unwrap()); + let response = error.custom_response().unwrap(); + assert_eq!(response.status, 500); + assert_eq!( + response.body.as_ref(), + error_contract::problem::STATIC_PROBLEM_FALLBACK + ); + let mut legacy = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE"); + legacy.error.startup = error + .create_failure() + .map(|failure| failure.startup().clone()); + assert_eq!( + legacy.custom_response().unwrap().body.as_ref(), + error_contract::problem::STATIC_PROBLEM_FALLBACK + ); +} + +#[tokio::test] +async fn generated_client_normalizes_known_problem_and_actual_metadata() { + let original = failure(); + let response = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE") + .with_create_failure(original.clone()) + .custom_response() + .unwrap(); + let mut value: Value = serde_json::from_slice(response.body.as_ref()).unwrap(); + value["detail"] = json!("WIRE_SECRET_EXPLANATION"); + value["yoi_failure"]["explanation"] = json!("WIRE_SECRET_EXPLANATION"); + let error = receive( + response.status, + vec![PROBLEM_CONTENT_TYPE], + serde_json::to_vec(&value).unwrap(), + ) + .await; + assert_eq!( + error.create_failure().unwrap().public_facts(), + original.public_facts() + ); + assert_eq!( + cause(&error).identity(), + StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)) + .identity() + ); + assert!(!error.error.message.contains("WIRE_SECRET")); + let browser = error + .create_failure() + .unwrap() + .problem( + StartupBoundary::Browser, + &SessionFailureVisibility { + can_view_cause: true, + }, + "", + ) + .unwrap(); + assert_eq!(browser.status, 409); + let wrong_status = receive( + 409, + vec![PROBLEM_CONTENT_TYPE], + serde_json::to_vec(&value).unwrap(), + ) + .await; + assert_unknown(&wrong_status, StartupBoundaryReason::InvalidResponse); +} + +#[tokio::test] +async fn unknown_versions_reasons_and_invalid_responses_remain_local_unknown_observations() { + let response = failure() + .problem( + StartupBoundary::Runtime, + &SessionFailureVisibility { + can_view_cause: true, + }, + "", + ) + .unwrap(); + let original: Value = serde_json::from_slice(response.body()).unwrap(); + for (field, replacement, reason) in [ + ( + "version", + json!(99), + StartupBoundaryReason::UnsupportedVersion, + ), + ( + "details_version", + json!(99), + StartupBoundaryReason::UnsupportedVersion, + ), + ( + "reason", + json!("future_reason"), + StartupBoundaryReason::UnsupportedReason, + ), + ( + "category", + json!("invalid_category"), + StartupBoundaryReason::InvalidResponse, + ), + ] { + let mut value = original.clone(); + value["yoi_failure"][field] = replacement; + let error = receive( + 403, + vec![PROBLEM_CONTENT_TYPE], + serde_json::to_vec(&value).unwrap(), + ) + .await; + assert_unknown(&error, reason); + } + for content_types in [ + vec![], + vec!["text/plain"], + vec![PROBLEM_CONTENT_TYPE, "application/json"], + ] { + let error = receive(403, content_types, response.body().to_vec()).await; + assert_unknown(&error, StartupBoundaryReason::InvalidResponse); + } + let error = receive( + 502, + vec![PROBLEM_CONTENT_TYPE], + vec![b'x'; MAX_STARTUP_PROBLEM_BYTES + 1], + ) + .await; + assert_unknown(&error, StartupBoundaryReason::TooLarge); + let error = receive( + 502, + vec![PROBLEM_CONTENT_TYPE], + b"not JSON PRIVATE_SECRET".to_vec(), + ) + .await; + assert_unknown(&error, StartupBoundaryReason::InvalidResponse); + assert!(!error.error.message.contains("PRIVATE_SECRET")); +} + +#[tokio::test] +async fn legacy_errors_are_bounded_and_startup_attachments_reprojected_without_outcome_inference() { + let ordinary = RuntimeApiError::new(404, "old_not_found", "old not found"); + let error = receive( + 404, + vec!["application/json"], + serde_json::to_vec(&ordinary).unwrap(), + ) + .await; + assert_eq!(error.error, ordinary.error); + assert!(error.create_failure().is_none()); + let original = failure(); + let mut legacy = RuntimeApiError::new(502, "OLD_PRIVATE_CODE", "OLD_PRIVATE_MESSAGE"); + legacy.error.startup = Some(original.startup().clone()); + let error = receive( + 502, + vec!["application/json; charset=utf-8"], + serde_json::to_vec(&legacy).unwrap(), + ) + .await; + assert_eq!( + cause(&error).identity(), + cause(&RuntimeApiError::new(500, "unused", "unused").with_create_failure(original.clone())) + .identity() + ); + let failure = error.create_failure().unwrap(); + assert_eq!(failure.operation().acceptance, CreationAcceptance::Unknown); + assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown); + assert_eq!(failure.startup().secondary, original.startup().secondary); + assert!(!error.error.message.contains("OLD_PRIVATE")); + let mut value = serde_json::to_value(&legacy).unwrap(); + value["error"]["startup"]["primary"] = json!("PRIVATE_MALFORMED_ATTACHMENT"); + let malformed = receive( + 502, + vec!["application/json"], + serde_json::to_vec(&value).unwrap(), + ) + .await; + assert_unknown(&malformed, StartupBoundaryReason::InvalidResponse); + let invalid = receive( + 502, + vec!["application/json"], + b"{invalid PRIVATE_SECRET".to_vec(), + ) + .await; + assert_unknown(&invalid, StartupBoundaryReason::Decode); + let oversized = receive( + 502, + vec!["application/json"], + vec![b'x'; MAX_STARTUP_PROBLEM_BYTES + 1], + ) + .await; + assert_unknown(&oversized, StartupBoundaryReason::TooLarge); +} diff --git a/crates/runtime-api/tests/startup_storage.rs b/crates/runtime-api/tests/startup_storage.rs new file mode 100644 index 000000000..32bcd4ae5 --- /dev/null +++ b/crates/runtime-api/tests/startup_storage.rs @@ -0,0 +1,341 @@ +//! Storage codecs use the production attachment projector, never an HTTP body. +use error_contract::{Budget, ContractError, DecodeFailure, validate_json_input}; +use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause}; +use runtime_api::startup::*; +use runtime_api::{ + RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure, + StartupFailureAttachment, StartupIoKind, StartupPersistenceOutcome, StartupResourceFacts, + StartupResources, StartupSecondaryFailure, StartupStage, WorkerStartupFailure, +}; +use serde_json::{Value, json}; + +fn operation() -> StartupOperationFacts { + StartupOperationFacts { + acceptance: CreationAcceptance::Accepted, + execution: ExecutionOutcome::Errored, + finalize: Some(FinalizeFacts { + stage: FinalizeStage::Publication, + compensation: CompensationOutcome::Residual, + }), + backend_compensation: Some(CompensationOutcome::Unknown), + correlation: Some(StartupCorrelation::new("storage-test-812").unwrap()), + } +} + +fn failure(cause: StartupCause) -> WorkerCreateFailure { + let base = WorkerCreateFailure::from_cause(cause, operation()).unwrap(); + let (mut startup, operation) = base.into_parts(); + startup.secondary.push(StartupSecondaryFailure::Cleanup { + failure: StartupCleanupFailure::Workdir, + }); + startup.resources = Some(StartupResourceFacts { + resources: StartupResources::WorkerStartup, + cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir), + durable_effects_possible: true, + }); + // Session metadata write and Backend result write deliberately disagree. + startup.persistence = Some(StartupPersistenceOutcome::NotSaved); + startup.io_kind = Some(StartupIoKind::PermissionDenied); + WorkerCreateFailure::new(startup, operation) + .unwrap() + .with_backend_result_persistence(Some(StartupPersistenceOutcome::Persisted)) +} + +fn session() -> StartupCause { + StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)) +} + +fn known(input: &str) -> WorkerCreateFailure { + match decode_stored_worker_create_failure(input) { + DecodeStoredWorkerCreateFailure::Known(failure) => failure, + other => panic!("expected known storage snapshot, got {other:?}"), + } +} + +fn change_primary(value: &mut Value, change: impl FnOnce(&mut Value)) { + let mut primary: Value = serde_json::from_str(value["primary"].as_str().unwrap()).unwrap(); + change(&mut primary); + value["primary"] = Value::String(serde_json::to_string(&primary).unwrap()); +} + +#[test] +fn storage_semantic_roundtrip_covers_domains_and_independent_facts() { + for cause in [ + session(), + StartupCause::Runtime(RuntimeStartupCause { + stage: StartupStage::Feature, + reason: RuntimeStartupReason::FeatureFailed, + }), + StartupCause::Boundary(StartupBoundaryCause { + reason: StartupBoundaryReason::FinalizeFailed, + }), + ] { + let original = failure(cause.clone()); + let encoded = original.encode_storage().unwrap(); + let value: Value = serde_json::from_str(&encoded).unwrap(); + assert_eq!(value["version"], 1); + assert!(value.get("status").is_none()); + assert!(value.get("yoi_failure").is_none()); + let loaded = known(&encoded); + assert_eq!(loaded.public_facts(), original.public_facts()); + assert_eq!(loaded.operation(), original.operation()); + assert_eq!( + loaded.startup().persistence, + Some(StartupPersistenceOutcome::NotSaved) + ); + assert_eq!( + loaded.backend_result_persistence(), + Some(StartupPersistenceOutcome::Persisted) + ); + match loaded.startup().primary.decode::() { + DecodeFailure::Known(received) => assert_eq!(received.facts().cause(), &cause), + _ => panic!("primary cause lost"), + } + assert_eq!(loaded.encode_storage().unwrap(), encoded); + } +} + +#[test] +fn encoding_uses_current_marker_without_replacing_primary_or_inferring_persistence() { + let original = failure(session()); + for marker in [ + None, + Some(StartupPersistenceOutcome::Persisted), + Some(StartupPersistenceOutcome::NotSaved), + Some(StartupPersistenceOutcome::Conflict), + Some(StartupPersistenceOutcome::OutcomeUnknown), + ] { + let updated = original.clone().with_backend_result_persistence(marker); + assert_eq!(updated.startup(), original.startup()); + assert_eq!(updated.operation(), original.operation()); + let loaded = known(&updated.encode_storage().unwrap()); + assert_eq!(loaded.backend_result_persistence(), marker); + assert_eq!(loaded.startup().persistence, original.startup().persistence); + assert_eq!(loaded.public_facts(), updated.public_facts()); + } +} + +#[test] +fn legacy_absent_marker_is_none_in_storage_and_public_reader() { + let original = failure(session()).with_backend_result_persistence(None); + let mut value: Value = serde_json::from_str(&original.encode_storage().unwrap()).unwrap(); + value["facts"] + .as_object_mut() + .unwrap() + .remove("backend_result_persistence"); + assert_eq!(known(&value.to_string()).backend_result_persistence(), None); + let response = original + .problem( + StartupBoundary::Browser, + &SessionFailureVisibility { + can_view_cause: true, + }, + "", + ) + .unwrap(); + let public: Value = serde_json::from_slice(response.body()).unwrap(); + assert!( + public["yoi_startup"] + .get("backend_result_persistence") + .is_none() + ); + match decode_startup_problem( + response.body(), + response.status, + response.content_type, + StartupBoundary::Browser, + ) { + DecodeStartupProblem::Known(failure) => { + assert_eq!(failure.backend_result_persistence(), None) + } + other => panic!("legacy public response failed: {other:?}"), + } +} + +#[test] +fn public_receiver_restores_backend_marker_and_current_visibility_is_required() { + let original = failure(session()); + let loaded = known(&original.encode_storage().unwrap()); + assert!( + loaded + .problem( + StartupBoundary::Browser, + &SessionFailureVisibility { + can_view_cause: false + }, + "" + ) + .is_err() + ); + let response = loaded + .problem( + StartupBoundary::Browser, + &SessionFailureVisibility { + can_view_cause: true, + }, + "", + ) + .unwrap(); + let received = match decode_startup_problem( + response.body(), + response.status, + response.content_type, + StartupBoundary::Browser, + ) { + DecodeStartupProblem::Known(failure) => failure, + other => panic!("public response failed: {other:?}"), + }; + assert_eq!(received.public_facts(), original.public_facts()); + let mut updated = operation(); + updated.acceptance = CreationAcceptance::Unknown; + let received = received.with_operation(updated.clone()); + assert_eq!(received.operation(), &updated); + assert_eq!( + received.backend_result_persistence(), + Some(StartupPersistenceOutcome::Persisted) + ); + // The receiving Backend clears remote evidence before its own write. + assert_eq!( + received + .with_backend_result_persistence(None) + .backend_result_persistence(), + None + ); +} + +#[test] +fn unknown_versions_and_primary_identities_are_unsupported() { + let base: Value = serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap(); + for mutate in [ + |v: &mut Value| v["version"] = json!(2), + |v: &mut Value| v["facts"]["version"] = json!(2), + |v: &mut Value| change_primary(v, |p| p["version"] = json!(2)), + |v: &mut Value| change_primary(v, |p| p["reason"] = json!("future_reason")), + |v: &mut Value| { + change_primary(v, |p| { + p["required_extensions"] = json!(["future_extension"]) + }) + }, + ] { + let mut value = base.clone(); + mutate(&mut value); + assert_eq!( + decode_stored_worker_create_failure(&value.to_string()), + DecodeStoredWorkerCreateFailure::Unsupported + ); + } +} + +#[test] +fn malformed_and_duplicate_known_records_are_corrupt() { + let base: Value = serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap(); + for input in [ + "".to_owned(), + "null".to_owned(), + "{}".to_owned(), + "{".to_owned(), + base.to_string() + .replacen("\"version\":1", "\"version\":1,\"version\":1", 1), + ] { + assert!(matches!( + decode_stored_worker_create_failure(&input), + DecodeStoredWorkerCreateFailure::Corrupt(_) + )); + } + for mutate in [ + |v: &mut Value| v["version"] = json!(0), + |v: &mut Value| v["primary"] = json!("{broken"), + |v: &mut Value| v["facts"]["backend_result_persistence"] = json!("invented"), + |v: &mut Value| { + v["facts"]["secondary"] = json!(vec![json!({"kind":"cleanup","failure":"workdir"}); 9]) + }, + |v: &mut Value| change_primary(v, |p| p["category"] = json!("internal")), + ] { + let mut value = base.clone(); + mutate(&mut value); + assert!(matches!( + decode_stored_worker_create_failure(&value.to_string()), + DecodeStoredWorkerCreateFailure::Corrupt(_) + )); + } +} + +#[test] +fn complete_input_and_embedded_attachment_are_bounded_before_parsing() { + let oversized = " ".repeat(32 * 1024 + 1); + assert_eq!( + decode_stored_worker_create_failure(&oversized), + DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooLarge) + ); + let deep = format!("{}0{}", "[".repeat(17), "]".repeat(17)); + assert_eq!( + decode_stored_worker_create_failure(&deep), + DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep) + ); + // A supported-looking prefix must not bypass the whole-input guard. + let deep = format!("{{\"version\":2,\"extra\":{deep}}}"); + assert_eq!( + decode_stored_worker_create_failure(&deep), + DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep) + ); + let mut value: Value = + serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap(); + change_primary(&mut value, |p| { + p["extra"] = + serde_json::from_str(&format!("{}0{}", "[".repeat(17), "]".repeat(17))).unwrap() + }); + assert_eq!( + decode_stored_worker_create_failure(&value.to_string()), + DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep) + ); + // Shared wrapper is a bounds check, not a grammar validator or encoder. + assert_eq!(validate_json_input(b"[", Budget::new(1024, 1024)), Ok(())); +} + +#[test] +fn storage_encoder_and_reader_strip_untrusted_context_and_explanations() { + let original = failure(session()); + let mut value: Value = serde_json::from_str(&original.encode_storage().unwrap()).unwrap(); + change_primary(&mut value, |p| { + p["explanation"] = json!("TEST_ONLY_SECRET_RAW_HTTP"); + p["context"]["correlation"] = json!("private-worker-id"); + p["context"]["operations"] = json!([{ + "operation":"session.record", "stage":"binding", "operation_ref":"TEST_ONLY_SECRET_RAW_HTTP" + }, { + "operation":"/private/path", "stage":"http-body", "operation_ref":null + }]); + }); + // Simulate a received attachment without granting a projection capability. + let primary: StartupFailureAttachment = + serde_json::from_value(value["primary"].clone()).unwrap(); + let (startup, operation) = original.clone().into_parts(); + let incoming = + WorkerCreateFailure::new(WorkerStartupFailure { primary, ..startup }, operation).unwrap(); + for encoded in [ + incoming.encode_storage().unwrap(), + known(&value.to_string()).encode_storage().unwrap(), + ] { + for secret in [ + "TEST_ONLY_SECRET_RAW_HTTP", + "private-worker-id", + "/private/path", + "http-body", + ] { + assert!( + !encoded.contains(secret), + "unexpected secret in stored snapshot" + ); + } + let loaded = known(&encoded); + let response = loaded + .problem( + StartupBoundary::Browser, + &SessionFailureVisibility { + can_view_cause: true, + }, + "", + ) + .unwrap(); + assert!(!String::from_utf8_lossy(response.body()).contains("TEST_ONLY_SECRET")); + } +} diff --git a/crates/server-api/Cargo.toml b/crates/server-api/Cargo.toml index 5af04009e..9cc50a6d3 100644 --- a/crates/server-api/Cargo.toml +++ b/crates/server-api/Cargo.toml @@ -10,8 +10,9 @@ default = [] typescript = ["dep:ts-rs", "protocol/typescript"] [dependencies] +error-contract = { workspace = true, features = ["json-schema"] } feature-permissions.workspace = true -api-macros = { workspace = true, features = ["reqwest", "axum", "openapi"] } +api-macros = { workspace = true, features = ["reqwest", "axum", "openapi", "error-contract"] } memory.workspace = true plugin-plan.workspace = true merge-request.workspace = true diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index fcc212361..bffc349b8 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -1,4 +1,6 @@ #![doc = include_str!("../README.md")] +mod worker_create_error; +pub use worker_create_error::WorkerCreateApiError; use std::collections::BTreeMap; @@ -1907,7 +1909,9 @@ pub trait ServerApi { "/api/workers", status = 200, error_status = 400, - additional_error_statuses = [401, 403, 404, 409, 500, 502, 503], + additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504], + additional_error_content_types = ["application/problem+json"], + client_error_hook = true, bearer_auth = true, browser_auth = true )] @@ -1915,7 +1919,7 @@ pub trait ServerApi { &self, #[extension] context: ServerRequestContext, #[body] request: CreateWorkspaceWorkerRequest, - ) -> Result; + ) -> Result; #[get( "/api/w/{workspace_id}/orchestrator", @@ -1973,7 +1977,8 @@ pub trait ServerApi { "/api/w/{workspace_id}/worker-control/workers", status = 200, error_status = 400, - additional_error_statuses = [401, 403, 404, 409, 500, 502, 503], + additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504], + client_error_hook = true, openapi = false, )] async fn worker_control_spawn( @@ -1981,7 +1986,7 @@ pub trait ServerApi { #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[body] request: CreateWorkspaceWorkerRequest, - ) -> Result; + ) -> Result; #[post( "/api/w/{workspace_id}/worker-control/workers/{runtime_id}/{worker_id}/input", status = 200, @@ -2098,7 +2103,9 @@ pub trait ServerApi { "/api/w/{workspace_id}/workers", status = 200, error_status = 400, - additional_error_statuses = [401, 403, 404, 409, 500, 502, 503], + additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504], + additional_error_content_types = ["application/problem+json"], + client_error_hook = true, bearer_auth = true, browser_auth = true )] @@ -2107,7 +2114,7 @@ pub trait ServerApi { #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[body] request: CreateWorkspaceWorkerRequest, - ) -> Result; + ) -> Result; #[get( "/api/w/{workspace_id}/workers/{worker_ref}", @@ -11099,6 +11106,7 @@ pub struct BrowserWorkspaceOrchestratorResponse { #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum WorkerOperationState { + Unknown, Accepted, Unsupported, Rejected, diff --git a/crates/server-api/src/worker_create_error.rs b/crates/server-api/src/worker_create_error.rs new file mode 100644 index 000000000..4d1e9f282 --- /dev/null +++ b/crates/server-api/src/worker_create_error.rs @@ -0,0 +1,200 @@ +//! Worker creation's HTTP projection is separate from its HTTP-independent result. +//! A decoded DTO is never an encoder capability: only a current visibility decision +//! can populate the cached public response. Legacy preflight routes keep their body. +use api_macros::{HttpError, HttpErrorResponse}; +use runtime_api::startup::{ + DecodeStartupProblem, SessionFailureVisibility, StartupBoundary, StartupProblemDetails, + WorkerCreateFailure, decode_startup_problem, +}; +use schemars::{JsonSchema, Schema, SchemaGenerator}; +use serde::{Deserialize, Serialize}; + +use crate::RepositoryApiError; + +#[derive(Debug)] +pub struct WorkerCreateApiError { + legacy: RepositoryApiError, + failure: Option, + projected: Option, +} + +impl WorkerCreateApiError { + /// The service has authenticated and authorized the current creation caller. + /// Visibility is a local decision; neither wire labels nor saved results supply it. + pub fn startup(failure: WorkerCreateFailure, visibility: &SessionFailureVisibility) -> Self { + let instance = failure + .operation() + .correlation + .as_ref() + .map(|correlation| format!("urn:yoi:worker-startup:{}", correlation.as_str())) + .unwrap_or_default(); + let projected = failure + .problem(StartupBoundary::Browser, visibility, &instance) + .ok(); + let status = projected.as_ref().map_or(500, |response| response.status); + Self { + legacy: RepositoryApiError::new( + status, + "worker_startup", + "Worker creation did not complete; inspect the typed result.", + Vec::new(), + ), + failure: Some(failure), + projected, + } + } + + pub fn failure(&self) -> Option<&WorkerCreateFailure> { + self.failure.as_ref() + } + pub fn legacy(&self) -> &RepositoryApiError { + &self.legacy + } +} + +impl From for WorkerCreateApiError { + fn from(legacy: RepositoryApiError) -> Self { + Self { + legacy, + failure: None, + projected: None, + } + } +} + +impl HttpError for WorkerCreateApiError { + fn status_code(&self) -> u16 { + self.projected + .as_ref() + .map_or_else(|| self.legacy.status_code(), |response| response.status) + } + + fn custom_response(&self) -> Option { + if let Some(projected) = &self.projected { + return Some(HttpErrorResponse { + status: projected.status, + content_type: projected.content_type, + body: projected.body().to_vec().into(), + }); + } + // Projection failure does not serialize a possibly private internal result. + self.failure.as_ref().map(|_| { + error_contract::problem::problem_fallback(error_contract::Budget::new( + error_contract::MAX_FAILURE_BYTES, + error_contract::MAX_FAILURE_BYTES, + )) + .expect("static fallback fits the fixed boundary budget") + .into() + }) + } +} + +impl api_macros::HttpRequestError for WorkerCreateApiError { + fn from_request_rejection(status: u16, message: String) -> Self { + ::from_request_rejection( + status, message, + ) + .into() + } +} + +impl Serialize for WorkerCreateApiError { + fn serialize(&self, serializer: S) -> Result { + // Generated HTTP adapters use custom_response. The ordinary serializer remains + // legacy-only and cannot serialize the retained internal startup attachment. + if let Some(projected) = &self.projected { + let dto: StartupProblemDetails = + serde_json::from_slice(projected.body()).map_err(serde::ser::Error::custom)?; + dto.serialize(serializer) + } else { + self.legacy.serialize(serializer) + } + } +} +impl<'de> Deserialize<'de> for WorkerCreateApiError { + fn deserialize>(deserializer: D) -> Result { + RepositoryApiError::deserialize(deserializer).map(Into::into) + } +} + +#[allow(dead_code)] +#[derive(JsonSchema)] +#[serde(untagged)] +enum WorkerCreateErrorWire { + Problem(StartupProblemDetails), + Legacy(RepositoryApiError), +} +impl JsonSchema for WorkerCreateApiError { + fn schema_name() -> std::borrow::Cow<'static, str> { + "WorkerCreateApiError".into() + } + fn json_schema(generator: &mut SchemaGenerator) -> Schema { + WorkerCreateErrorWire::json_schema(generator) + } +} + +impl api_macros::openapi::OpenApiSchema for WorkerCreateApiError { + fn response_schema( + builder: &mut api_macros::openapi::OpenApiBuilder, + content_type: &str, + ) -> Result { + match content_type { + "application/json" => builder.schema_ref::(), + runtime_api::startup::PROBLEM_CONTENT_TYPE => { + builder.schema_ref::() + } + _ => Err(api_macros::openapi::OpenApiError::InvalidContract( + "unsupported Worker creation error representation".into(), + )), + } + } +} + +impl api_macros::reqwest::HttpClientError for WorkerCreateApiError { + fn decode_error( + response: &api_macros::reqwest::ReceivedResponse, + ) -> Result { + use api_macros::reqwest::ClientFailure; + let content_type = response.content_type()?; + if content_type + .split(';') + .next() + .unwrap_or("") + .trim() + .eq_ignore_ascii_case(runtime_api::startup::PROBLEM_CONTENT_TYPE) + { + return match decode_startup_problem( + response.body(), + response.status().as_u16(), + content_type, + StartupBoundary::Browser, + ) { + DecodeStartupProblem::Known(failure) => Ok(Self { + legacy: RepositoryApiError::new( + response.status().as_u16(), + "worker_startup", + "Worker creation returned a typed failure.", + Vec::new(), + ), + failure: Some(failure), + projected: None, + }), + DecodeStartupProblem::TooLarge => Err(ClientFailure::ResponseTooLarge { + limit: runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES, + }), + DecodeStartupProblem::UnsupportedVersion + | DecodeStartupProblem::UnsupportedReason => { + Err(ClientFailure::UnsupportedErrorResponse { + status: response.status().as_u16(), + }) + } + DecodeStartupProblem::Invalid => Err(ClientFailure::ErrorResponseDecode { + status: response.status().as_u16(), + }), + }; + } + let legacy = response + .decode_json::(api_macros::reqwest::DecodeKind::PublicError)?; + Ok(legacy.into()) + } +} diff --git a/crates/server-api/tests/worker_create_contract.rs b/crates/server-api/tests/worker_create_contract.rs new file mode 100644 index 000000000..7f3503fd4 --- /dev/null +++ b/crates/server-api/tests/worker_create_contract.rs @@ -0,0 +1,96 @@ +//! Generated Backend client → isolated loopback HTTP response → typed result. +//! The server response producer is substituted; authorization and persistence are +//! exercised by workspace-server's connected tests, not by this codec connection. +use runtime_api::startup::*; +use server_api::{CreateWorkspaceWorkerRequest, ServerApiClient, client_support, server_support}; +use server_support::framework::{Body, Response, Router, post}; + +struct ServerTask(tokio::task::JoinHandle<()>); +impl Drop for ServerTask { + fn drop(&mut self) { + self.0.abort(); + } +} + +fn request() -> CreateWorkspaceWorkerRequest { + serde_json::from_value(serde_json::json!({ + "runtime_id": "runtime", "display_name": "Worker", "feature_connections": {} + })) + .unwrap() +} + +#[tokio::test] +async fn every_creation_client_decodes_actual_problem_status_and_operation_facts() { + let timeout = WorkerCreateFailure::from_cause( + StartupCause::Boundary(StartupBoundaryCause { + reason: StartupBoundaryReason::Timeout, + }), + StartupOperationFacts { + acceptance: CreationAcceptance::Unknown, + execution: ExecutionOutcome::Unknown, + finalize: None, + backend_compensation: None, + correlation: None, + }, + ) + .unwrap() + .problem( + StartupBoundary::Browser, + &SessionFailureVisibility::default(), + "", + ) + .unwrap(); + for (status, body) in [ + (409, include_bytes!("../../runtime-api/tests/fixtures/startup-browser-session-record-denied.problem.json").to_vec()), + (502, include_bytes!("../../runtime-api/tests/fixtures/startup-browser-registration-response-lost.problem.json").to_vec()), + (504, timeout.into_body()), + ] { + let DecodeStartupProblem::Known(expected) = decode_startup_problem(&body, status, PROBLEM_CONTENT_TYPE, StartupBoundary::Browser) else { + panic!("producer fixture must have a known typed cause"); + }; + let router = ["/api/workers", "/api/w/workspace/workers", "/api/w/workspace/worker-control/workers"] + .into_iter().fold(Router::new(), |router, path| { + let body = body.clone(); + router.route(path, post(move || { + let body = body.clone(); + async move { Response::builder().status(status).header("content-type", PROBLEM_CONTENT_TYPE).body(Body::from(body)).unwrap() } + })) + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let _server = ServerTask(tokio::spawn(async move { + server_support::framework::serve(listener, router).await.unwrap(); + })); + let client = ServerApiClient::builder(format!("http://{address}")) + .unwrap().request_timeout(std::time::Duration::from_secs(3)).build().unwrap(); + for result in [ + client.workspace_worker_create("workspace".into(), request()).await, + client.workspace_worker_create_alias(request()).await, + client.worker_control_spawn("workspace".into(), request()).await, + ] { + let Err(client_support::ClientError::Public { status: actual_status, error }) = result else { + panic!("generated route must invoke the Problem decoder, not legacy Deserialize"); + }; + assert_eq!(actual_status.as_u16(), status); + let actual = error.failure().expect("retain typed non-HTTP failure"); + assert_eq!(actual.startup().primary, expected.startup().primary); + assert_eq!(actual.public_facts(), expected.public_facts()); + } + } +} + +#[test] +fn creation_openapi_declares_gateway_timeout_with_both_error_representations() { + let document: serde_json::Value = serde_json::from_str( + &server_api::canonical_openapi_document() + .unwrap() + .to_json() + .unwrap(), + ) + .unwrap(); + for path in ["/api/workers", "/api/w/{workspace_id}/workers"] { + let content = &document["paths"][path]["post"]["responses"]["504"]["content"]; + assert!(content.get("application/problem+json").is_some()); + assert!(content.get("application/json").is_some()); + } +} diff --git a/crates/worker-runtime/src/error.rs b/crates/worker-runtime/src/error.rs index 5b6e4e5af..abfd3692e 100644 --- a/crates/worker-runtime/src/error.rs +++ b/crates/worker-runtime/src/error.rs @@ -63,6 +63,15 @@ pub enum RuntimeError { result: WorkerExecutionResult, }, + /// Creation failed after connection; the original error remains internal source + /// evidence. Display deliberately never renders it or the startup attachment. + #[error("worker creation failed")] + WorkerCreateFailed { + failure: Box, + #[source] + source: Box, + }, + #[error("limit {requested} exceeds maximum {max}")] LimitTooLarge { requested: usize, max: usize }, @@ -149,11 +158,26 @@ pub enum RuntimeError { } impl RuntimeError { + /// Only an owning create boundary or a Spawn result can identify a creation + /// failure. Independent Input errors retain their ordinary API contract. + pub fn create_failure(&self) -> Option { + match self { + Self::WorkerCreateFailed { failure, .. } => Some((**failure).clone()), + Self::WorkerExecutionRejected { result, .. } => { + crate::execution::startup_create_failure(result) + .ok() + .flatten() + } + _ => None, + } + } + /// The original execution result, including primary facts and independent secondary /// failures. Display is presentation only and must not be parsed to recover causes. pub fn execution_result(&self) -> Option<&WorkerExecutionResult> { match self { Self::WorkerExecutionRejected { result, .. } => Some(result), + Self::WorkerCreateFailed { source, .. } => source.execution_result(), _ => None, } } diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index cd76170d3..078a73c31 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -1690,3 +1690,59 @@ impl fmt::Debug for WorkerExecutionBackendRef { .finish_non_exhaustive() } } + +/// Transfer a spawn result without deriving operation acceptance from its error text, +/// HTTP status, cleanup disposition, or persistence outcome. Other operation kinds +/// keep their existing result contracts and are not reclassified as creation. +pub fn startup_create_failure( + result: &WorkerExecutionResult, +) -> Result, error_contract::ContractError> { + use runtime_api::startup::{ + CreationAcceptance, ExecutionOutcome, StartupOperationFacts, WorkerCreateFailure, + }; + if result.operation != WorkerExecutionOperation::Spawn { + return Ok(None); + } + let execution = match result.outcome { + WorkerExecutionOutcome::Accepted => ExecutionOutcome::Accepted, + WorkerExecutionOutcome::Busy => ExecutionOutcome::Busy, + WorkerExecutionOutcome::Rejected => ExecutionOutcome::Rejected, + WorkerExecutionOutcome::Errored => ExecutionOutcome::Errored, + WorkerExecutionOutcome::Unsupported => ExecutionOutcome::Unsupported, + }; + let acceptance = match result.outcome { + WorkerExecutionOutcome::Accepted => CreationAcceptance::Accepted, + WorkerExecutionOutcome::Rejected | WorkerExecutionOutcome::Unsupported => { + CreationAcceptance::Rejected + } + WorkerExecutionOutcome::Busy | WorkerExecutionOutcome::Errored => { + CreationAcceptance::Unknown + } + }; + let primary = match result.startup_failure.as_ref() { + Some(primary) => primary.clone(), + None => worker::startup_failure::runtime_startup_attachment( + protocol::startup_failure::RuntimeStartupCause { + stage: protocol::startup_failure::StartupStage::Controller, + reason: protocol::startup_failure::RuntimeStartupReason::StartupFailed, + }, + )?, + }; + WorkerCreateFailure::new( + runtime_api::WorkerStartupFailure { + primary, + secondary: result.secondary_failures.clone(), + resources: result.startup_resources, + persistence: result.startup_failure_persistence, + io_kind: result.startup_io_kind, + }, + StartupOperationFacts { + acceptance, + execution, + finalize: None, + correlation: None, + backend_compensation: None, + }, + ) + .map(Some) +} diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 2d59cee2b..1c04322d8 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -2326,6 +2326,7 @@ impl IntoResponse for RuntimeHttpWorkdirError { #[derive(Debug)] struct RuntimeHttpRestError { + create_failure: Option, status: StatusCode, code: String, message: String, @@ -2339,12 +2340,14 @@ impl RuntimeHttpRestError { code: code.into(), message: message.into(), startup: None, + create_failure: None, } } fn runtime(error: RuntimeError) -> Self { let status = status_for_runtime_error(&error); let code = code_for_runtime_error(&error); + let create_failure = error.create_failure(); let startup = match &error { RuntimeError::WorkerExecutionRejected { result, .. } => startup_transfer(result), _ => None, @@ -2359,6 +2362,7 @@ impl RuntimeHttpRestError { code, message, startup, + create_failure, } } @@ -2391,6 +2395,13 @@ impl RuntimeHttpRestError { impl IntoResponse for RuntimeHttpRestError { fn into_response(self) -> Response { + if let Some(failure) = self.create_failure { + return runtime_api::server_support::http_error_response( + self.status, + runtime_api::RuntimeApiError::new(self.status.as_u16(), self.code, self.message) + .with_create_failure(failure), + ); + } let body = RuntimeHttpErrorResponse { error: RuntimeHttpErrorDetail { code: self.code, @@ -2422,6 +2433,7 @@ fn startup_transfer( fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { match error { + RuntimeError::WorkerCreateFailed { .. } => StatusCode::INTERNAL_SERVER_ERROR, RuntimeError::WorkerNotFound { .. } | RuntimeError::ConfigBundleMissing { .. } => { StatusCode::NOT_FOUND } @@ -2531,6 +2543,7 @@ fn code_for_runtime_error(error: &RuntimeError) -> String { RuntimeError::ExecutionBackendUnavailable { .. } => { "execution_backend_unavailable".to_string() } + RuntimeError::WorkerCreateFailed { .. } => "worker_create_failed".to_string(), RuntimeError::WorkerExecutionRejected { .. } => "worker_execution_rejected".to_string(), RuntimeError::WorkspaceOwnerMismatch { .. } => "workspace_owner_mismatch".to_string(), RuntimeError::LimitTooLarge { .. } => "limit_too_large".to_string(), diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index cac820371..fd4205099 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -227,6 +227,9 @@ fn api_error(error: RuntimeHttpRestError) -> runtime_api::RuntimeApiError { let mut response = runtime_api::RuntimeApiError::new(error.status.as_u16(), error.code, error.message); response.error.startup = error.startup; + if let Some(failure) = error.create_failure { + return response.with_create_failure(failure); + } response } diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index 637d56b6b..9fa6d9143 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -31,6 +31,7 @@ mod session_material; mod session_public; mod session_retention; pub mod ssh_host_key_probe; +mod startup_observation; pub mod worker_backend; pub mod worker_source; pub mod working_directory; diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index d172db44a..823d26a7b 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -56,6 +56,11 @@ use protocol::subscription::{ SubscriptionWorkerState, SubscriptionWorkerWorkdirAttachment, }; use protocol::{Event, Method}; +use runtime_api::startup::{ + CompensationOutcome, CreationAcceptance, ExecutionOutcome, FinalizeFacts, FinalizeStage, + StartupBoundaryCause, StartupBoundaryReason, StartupCause, StartupCorrelation, + StartupOperationFacts, WorkerCreateFailure, +}; use server_api::WorkerRestoreState; use std::collections::BTreeMap; #[cfg(feature = "ws-server")] @@ -1381,7 +1386,34 @@ impl Runtime { let worker_id = request.worker_id; let workspace_id = scope.map(|scope| scope.workspace_id.as_str()); let started_at = std::time::Instant::now(); - let result = self.create_worker_with_workspace_inner(request, scope); + let result = self + .create_worker_with_workspace_inner(request, scope) + .map_err(|error| { + // Use only the host-bound local identity, never received correlation. + // A poisoned/unbound Runtime cannot invent correlation authority. + let correlation = self.lock().ok().and_then(|state| { + state.runtime_identity.as_deref().map(|runtime_id| { + StartupCorrelation::for_worker(runtime_id, &worker_id.to_string()) + }) + }); + match (error.create_failure(), correlation) { + (Some(failure), Some(correlation)) => { + let mut operation = failure.operation().clone(); + operation.correlation = Some(correlation); + let failure = Box::new(failure.with_operation(operation)); + match error { + RuntimeError::WorkerCreateFailed { source, .. } => { + RuntimeError::WorkerCreateFailed { failure, source } + } + source => RuntimeError::WorkerCreateFailed { + failure, + source: Box::new(source), + }, + } + } + _ => error, + } + }); if let Err(error) = &result { write_runtime_worker_create_failure( worker_id, @@ -1528,13 +1560,13 @@ impl Runtime { WorkerExecutionSpawnResult::Rejected(mut result) | WorkerExecutionSpawnResult::RolledBack(mut result) | WorkerExecutionSpawnResult::Errored(mut result) => { - if let Err((error, phase, outcome)) = + if let Err((_error, phase, outcome)) = self.rollback_failed_create_with_phase(&worker_ref) { tracing::error!( worker_id = %worker_ref.worker_id, ?phase, - %error, + ?outcome, "failed to roll back Worker creation" ); // A later failure must not make a confirmed record deletion @@ -1601,7 +1633,8 @@ impl Runtime { }); tracing::error!( worker_id = %worker_ref.worker_id, - %error, + ?phase, + ?outcome, "failed to persist Worker create reconciliation evidence" ); } else { @@ -1620,80 +1653,119 @@ impl Runtime { } }; - if let Some(mut initial_input) = { - let state = self.lock()?; - state - .worker(&worker_ref)? - .request - .as_ref() - .and_then(|request| request.initial_input.clone()) - } { - let expected_submission_id = initial_input - .submission_request_id - .clone() - .filter(|request_id| !request_id.trim().is_empty()) - .unwrap_or_else(|| Uuid::now_v7().to_string()); - initial_input.submission_request_id = Some(expected_submission_id.clone()); - let dispatch_result = backend.dispatch_input(&worker_ref, initial_input.clone()); - if !dispatch_result.is_accepted() { - self.cleanup_connected_failed_create(&backend, &worker_ref)?; - return Err(RuntimeError::WorkerExecutionRejected { - worker_id: worker_ref.worker_id.clone(), - operation: dispatch_result.operation, - outcome: dispatch_result.outcome, - message: dispatch_result.message_or_default(), - result: dispatch_result, - }); - } - let has_durable_acceptance = dispatch_result - .submission - .as_ref() - .is_some_and(|ack| ack.submission_request_id == expected_submission_id); - if !has_durable_acceptance { - self.cleanup_connected_failed_create(&backend, &worker_ref)?; - let result = WorkerExecutionResult::rejected( - WorkerExecutionOperation::Input, - "execution backend accepted initial input without a durable submission acknowledgement", - ); - return Err(RuntimeError::WorkerExecutionRejected { - worker_id: worker_ref.worker_id.clone(), - operation: result.operation, - outcome: result.outcome, - message: result.message_or_default(), - result, - }); - } - let detail = match self.commit_created_worker( - &worker_ref, - initial_worker_state.clone(), - workdir_attachments, - dispatch_result, - ) { - Ok(detail) => detail, - Err(error) => { - self.cleanup_connected_failed_create(&backend, &worker_ref)?; - return Err(error); - } + // From Connected onward this owner, not the independent Input API, + // supplies creation context. Later cleanup must not replace the source. + let mut execution = ExecutionOutcome::Unknown; + let mut stage = FinalizeStage::InitialInput; + let mut reason = StartupBoundaryReason::FinalizeFailed; + let mut persistence_phase = None; + let mut input_result = None; + let result = (|| { + let initial_input = { + let state = self.lock()?; + state + .worker(&worker_ref)? + .request + .as_ref() + .and_then(|request| request.initial_input.clone()) }; - if let Err(error) = self.record_input_observation(&worker_ref, initial_input) { - self.cleanup_connected_failed_create(&backend, &worker_ref)?; - return Err(error); - } - Ok(detail) - } else { - match self.commit_created_worker( - &worker_ref, - initial_worker_state, - workdir_attachments, - WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn), - ) { - Ok(detail) => Ok(detail), - Err(error) => { - self.cleanup_connected_failed_create(&backend, &worker_ref)?; - Err(error) + if let Some(mut initial_input) = initial_input { + let expected_submission_id = initial_input + .submission_request_id + .clone() + .filter(|request_id| !request_id.trim().is_empty()) + .unwrap_or_else(|| Uuid::now_v7().to_string()); + initial_input.submission_request_id = Some(expected_submission_id.clone()); + let dispatch_result = backend.dispatch_input(&worker_ref, initial_input.clone()); + let has_durable_acceptance = dispatch_result + .submission + .as_ref() + .is_some_and(|ack| ack.submission_request_id == expected_submission_id); + input_result = Some(dispatch_result.clone()); + if !dispatch_result.is_accepted() { + // Confirmed rejection is failed initial execution. A lost + // acknowledgement does not establish non-execution. + execution = match dispatch_result.outcome { + crate::execution::WorkerExecutionOutcome::Errored + if !has_durable_acceptance => + { + ExecutionOutcome::Unknown + } + _ => ExecutionOutcome::Errored, + }; + return Err(RuntimeError::WorkerExecutionRejected { + worker_id: worker_ref.worker_id, + operation: dispatch_result.operation, + outcome: dispatch_result.outcome, + message: dispatch_result.message_or_default(), + result: dispatch_result, + }); } + if !has_durable_acceptance { + reason = StartupBoundaryReason::InvalidResponse; + return Err(RuntimeError::WorkerExecutionRejected { + worker_id: worker_ref.worker_id, + operation: dispatch_result.operation, + outcome: dispatch_result.outcome, + message: "initial input has no matching durable acknowledgement".into(), + result: dispatch_result, + }); + } + execution = ExecutionOutcome::Accepted; + stage = FinalizeStage::Publication; + let detail = self.commit_created_worker( + &worker_ref, + initial_worker_state, + workdir_attachments, + dispatch_result, + &mut persistence_phase, + )?; + self.record_input_observation(&worker_ref, initial_input)?; + Ok(detail) + } else { + execution = ExecutionOutcome::Accepted; + stage = FinalizeStage::Publication; + self.commit_created_worker( + &worker_ref, + initial_worker_state, + workdir_attachments, + WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn), + &mut persistence_phase, + ) } - } + })(); + result.map_err(|source| { + let mut startup = connected_create_startup(&source, input_result.as_ref(), reason); + if let Some(phase) = persistence_phase { + push_create_secondary( + &mut startup, + StartupSecondaryFailure::Persistence { + phase, + outcome: create_persistence_outcome(&source), + }, + ); + } + let compensation = + self.cleanup_connected_failed_create(&backend, &worker_ref, &mut startup); + let failure = WorkerCreateFailure::new( + startup, + StartupOperationFacts { + acceptance: CreationAcceptance::Accepted, + execution, + finalize: Some(FinalizeFacts { + stage, + compensation, + }), + backend_compensation: None, + correlation: None, + }, + ) + .expect("connected create facts have bounded secondary observations"); + RuntimeError::WorkerCreateFailed { + failure: Box::new(failure), + source: Box::new(source), + } + }) } /// List Workers known to this Runtime. @@ -2267,7 +2339,7 @@ impl Runtime { Some(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_canonical_write_unknown", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), )), None, )); @@ -2322,7 +2394,7 @@ impl Runtime { Some(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_authority_changed", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), )), None, )); @@ -2432,7 +2504,7 @@ impl Runtime { Some(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_admission_commit_unknown", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), )), None, )); @@ -2665,7 +2737,7 @@ impl Runtime { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_saved_binding_not_ready", - sanitize_worker_create_failure_message(&message), + sanitize_worker_restore_failure_message(&message), )); } let result = RuntimeWorkerRestoreResult::accepted(detail); @@ -2813,7 +2885,7 @@ impl Runtime { WorkerRestoreState::Rejected }, "worker_restore_preparation_rejected", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), )); } }; @@ -2922,7 +2994,7 @@ impl Runtime { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_observation_activation_pending", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), )); } } @@ -2933,7 +3005,7 @@ impl Runtime { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::Rejected, "worker_restore_preflight_rejected", - sanitize_worker_create_failure_message(&result.message_or_default()), + sanitize_worker_restore_failure_message(&result.message_or_default()), )); } @@ -3001,7 +3073,7 @@ impl Runtime { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_activation_pending", - sanitize_worker_create_failure_message(&error), + sanitize_worker_restore_failure_message(&error), )); } #[cfg(feature = "restore-checkpoints")] @@ -3021,7 +3093,7 @@ impl Runtime { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_observation_activation_pending", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), )); } let worker = self.lock()?.worker(worker_ref)?.detail(); @@ -3030,7 +3102,7 @@ impl Runtime { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_receipt_commit_pending", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), )); } Ok(result) @@ -3057,7 +3129,7 @@ impl Runtime { let receipt = RuntimeWorkerRestoreResult::failed( WorkerRestoreState::RolledBack, "worker_restore_decision_not_committed", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), ); if self.settle_restore_rollback(worker_ref, &receipt).is_ok() { return Ok(receipt); @@ -3084,7 +3156,7 @@ impl Runtime { Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_commit_reconciliation_required", - sanitize_worker_create_failure_message(&error.to_string()), + sanitize_worker_restore_failure_message(&error.to_string()), )) } } @@ -3094,14 +3166,14 @@ impl Runtime { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_reconciliation_rejected", - sanitize_worker_create_failure_message(&result.message_or_default()), + sanitize_worker_restore_failure_message(&result.message_or_default()), )); } self.discard_restore_candidate_context(worker_ref, &candidate_context)?; let receipt = RuntimeWorkerRestoreResult::failed( WorkerRestoreState::Rejected, "worker_restore_rejected", - sanitize_worker_create_failure_message(&result.message_or_default()), + sanitize_worker_restore_failure_message(&result.message_or_default()), ); if let Err(error) = self.settle_restore_rollback(worker_ref, &receipt) { tracing::error!( @@ -3121,7 +3193,7 @@ impl Runtime { let receipt = RuntimeWorkerRestoreResult::failed( WorkerRestoreState::RolledBack, "worker_restore_rolled_back", - sanitize_worker_create_failure_message(&_result.message_or_default()), + sanitize_worker_restore_failure_message(&_result.message_or_default()), ); self.discard_restore_candidate_context(worker_ref, &candidate_context)?; #[cfg(feature = "fs-store")] @@ -3174,7 +3246,7 @@ impl Runtime { Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_reconciliation_required", - sanitize_worker_create_failure_message(&format!( + sanitize_worker_restore_failure_message(&format!( "Worker restore requires reconciliation ({:?}): {}", result.outcome, result.message_or_default(), @@ -3194,7 +3266,7 @@ impl Runtime { Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_backend_outcome_unknown", - sanitize_worker_create_failure_message(&_result.message_or_default()), + sanitize_worker_restore_failure_message(&_result.message_or_default()), )) } } @@ -4007,6 +4079,7 @@ impl Runtime { initial_worker_state: protocol::WorkerStateSnapshot, workdir_attachments: Vec, result: WorkerExecutionResult, + persistence_phase: &mut Option, ) -> Result { let mut state = self.lock()?; let detail = { @@ -4030,41 +4103,155 @@ impl Runtime { worker.workdir_attachments = workdir_attachments; worker.detail() }; + *persistence_phase = Some(StartupFailurePhase::RuntimeSnapshotSave); state.persist_runtime_snapshot()?; + *persistence_phase = Some(StartupFailurePhase::RuntimeReconciliationRecordSave); state.persist_worker(&worker_ref.worker_id)?; + *persistence_phase = None; state.publish_worker_upsert(worker_ref.worker_id)?; Ok(detail) } + /// Observe compensation separately from the original create failure. This + /// performs the same single stop and cleanup as before; no receipt lookup, + /// retry, reexecution or new durable lifecycle operation is introduced here. fn cleanup_connected_failed_create( &self, backend: &WorkerExecutionBackendRef, worker_ref: &WorkerRef, - ) -> Result<(), RuntimeError> { + startup: &mut runtime_api::WorkerStartupFailure, + ) -> CompensationOutcome { + use protocol::startup_failure::{ + StartupCleanup, StartupCleanupFailure, StartupResourceFacts, StartupResources, + }; let operation_id = WorkerLifecycleOperationId::new(); let stop_result = backend.stop_worker_operation(WorkerExecutionStopRequest { operation_id, worker_ref: worker_ref.clone(), }); - if stop_result.is_accepted() + for secondary in &stop_result.secondary_failures { + push_create_secondary(startup, secondary.clone()); + } + if startup.io_kind.is_none() { + startup.io_kind = stop_result.startup_io_kind; + } + // Preserve the original owner's resource facts, if present. In their + // absence, connection establishes startup resources but not their release. + let had_resources = startup.resources.is_some() || stop_result.startup_resources.is_some(); + startup + .resources + .get_or_insert( + stop_result + .startup_resources + .unwrap_or(StartupResourceFacts { + resources: StartupResources::WorkerStartup, + cleanup: StartupCleanup::Required, + durable_effects_possible: true, + }), + ); + if let Some(outcome) = stop_result.startup_failure_persistence { + push_create_secondary( + startup, + StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::MetadataSave, + outcome, + }, + ); + } + let terminated = stop_result.is_accepted() && stop_result.termination.as_ref().is_some_and(|termination| { termination.confirms_scope_shutdown(worker_ref, operation_id) - }) - { - return self.rollback_failed_create(worker_ref); + }); + if terminated { + if !had_resources { + // Confirms scope shutdown only, not removal of Workdir contents + // or undoing already accepted initial-input effects. + startup.resources.as_mut().unwrap().cleanup = StartupCleanup::Confirmed; + } + match self.rollback_failed_create_with_phase(worker_ref) { + Ok(()) => CompensationOutcome::Completed, + Err((error, phase, outcome)) => { + if matches!( + phase, + StartupFailurePhase::RuntimeSnapshotSave + | StartupFailurePhase::RuntimeRemovalPublish + ) { + push_create_secondary( + startup, + StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::RuntimeRecordDelete, + outcome: StartupPersistenceOutcome::Persisted, + }, + ); + } + if phase == StartupFailurePhase::RuntimeRemovalPublish { + push_create_secondary( + startup, + StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::RuntimeSnapshotSave, + outcome: StartupPersistenceOutcome::Persisted, + }, + ); + } + record_create_persistence_failure(startup, &error, phase, outcome); + if outcome == StartupPersistenceOutcome::OutcomeUnknown { + CompensationOutcome::Unknown + } else { + CompensationOutcome::Residual + } + } + } + } else { + let failed = !stop_result.is_accepted() + && stop_result.outcome != crate::execution::WorkerExecutionOutcome::Errored; + let feature_failed = stop_result.termination.as_ref().is_some_and(|receipt| { + receipt.worker_ref == *worker_ref + && receipt.stop_operation_id == operation_id + && !receipt.feature_failures.is_empty() + }); + if failed || feature_failed { + let failure = if feature_failed { + StartupCleanupFailure::Feature + } else if stop_result.outcome + == crate::execution::WorkerExecutionOutcome::Unsupported + { + StartupCleanupFailure::Unsupported + } else { + StartupCleanupFailure::Ownership + }; + push_create_secondary(startup, StartupSecondaryFailure::Cleanup { failure }); + if !had_resources { + startup.resources.as_mut().unwrap().cleanup = StartupCleanup::Failed(failure); + } + } + let mut phase = StartupFailurePhase::RuntimeSnapshotSave; + let retained = (|| { + let mut state = self.lock()?; + let record = state.worker_mut(worker_ref)?; + record.worker_state = stop_result.worker_state.clone(); + state.persist_runtime_snapshot()?; + phase = StartupFailurePhase::RuntimeReconciliationRecordSave; + state.persist_worker(&worker_ref.worker_id) + })(); + match retained { + Ok(()) => push_create_secondary( + startup, + StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::RuntimeReconciliationRecordSave, + outcome: StartupPersistenceOutcome::Persisted, + }, + ), + Err(error) => { + let outcome = create_persistence_outcome(&error); + record_create_persistence_failure(startup, &error, phase, outcome); + } + } + if failed || feature_failed { + CompensationOutcome::Residual + } else { + CompensationOutcome::Unknown + } } - let mut state = self.lock()?; - let record = state.worker_mut(worker_ref)?; - - record.worker_state = stop_result.worker_state.clone(); - state.persist_runtime_snapshot()?; - state.persist_worker(&worker_ref.worker_id)?; - Ok(()) - } - - fn rollback_failed_create(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> { - self.rollback_failed_create_with_phase(worker_ref) - .map_err(|(error, _, _)| error) } // The legacy caller still receives its RuntimeError. Spawn failures instead @@ -5110,7 +5297,7 @@ impl Runtime { let mut state = self.lock()?; let diagnostic_id = state.next_diagnostic_id; state.next_diagnostic_id = state.next_diagnostic_id.saturating_add(1); - let message = sanitize_worker_create_failure_message( + let message = sanitize_worker_restore_failure_message( result .message .as_deref() @@ -7148,7 +7335,7 @@ impl RuntimeState { result: &WorkerExecutionResult, receipt: &RuntimeWorkerRestoreResult, ) -> Result<(), RuntimeError> { - let message = sanitize_worker_create_failure_message(&result.message_or_default()); + let message = sanitize_worker_restore_failure_message(&result.message_or_default()); let diagnostic_id = self.next_diagnostic_id; let workspace_id = self .workers @@ -8381,25 +8568,90 @@ fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), R Ok(()) } -fn worker_execution_failure_code(result: &WorkerExecutionResult) -> &'static str { - let message = result.message.as_deref().unwrap_or_default(); - if message.contains("timed out waiting for durable Worker Submit acceptance") { - "input_commit_timeout" - } else if message.contains("worker adapter task did not complete within") { - "adapter_task_timeout" - } else if message.contains("failed to send Worker method") { - "worker_method_send_failed" - } else if message.contains("worker rejected Submit") { - "worker_submit_rejected" - } else if message.contains("before durable acceptance") { - "worker_failed_before_input_commit" - } else if message.contains("event stream closed") { - "worker_event_stream_closed" - } else { - "worker_execution_failed" +/// Keep the original result's typed facts even when publication is the failing +/// boundary. No message/path parsing is used to choose the fallback cause. +fn connected_create_startup( + source: &RuntimeError, + result: Option<&WorkerExecutionResult>, + reason: StartupBoundaryReason, +) -> runtime_api::WorkerStartupFailure { + let fallback = WorkerCreateFailure::from_cause( + StartupCause::Boundary(StartupBoundaryCause { reason }), + StartupOperationFacts { + acceptance: CreationAcceptance::Accepted, + execution: ExecutionOutcome::Unknown, + finalize: None, + backend_compensation: None, + correlation: None, + }, + ) + .expect("closed startup boundary cause is valid"); + let (mut startup, _) = fallback.into_parts(); + if let Some(result) = result { + if let Some(primary) = &result.startup_failure { + startup.primary = primary.clone(); + } + for secondary in &result.secondary_failures { + push_create_secondary(&mut startup, secondary.clone()); + } + startup.resources = result.startup_resources; + startup.persistence = result.startup_failure_persistence; + startup.io_kind = result.startup_io_kind; + } + if startup.io_kind.is_none() { + startup.io_kind = create_io_kind(source); + } + startup +} + +fn create_io_kind(error: &RuntimeError) -> Option { + match error { + RuntimeError::StoreIo { source, .. } + | RuntimeError::StoreCommitOutcomeUnknown { source, .. } => Some(source.kind().into()), + _ => None, } } +fn create_persistence_outcome(error: &RuntimeError) -> StartupPersistenceOutcome { + if matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) { + StartupPersistenceOutcome::OutcomeUnknown + } else { + StartupPersistenceOutcome::NotSaved + } +} + +fn push_create_secondary( + startup: &mut runtime_api::WorkerStartupFailure, + secondary: StartupSecondaryFailure, +) { + // The create DTO has the core diagnostic bound; this is an observation, + // not another durable ledger. The original error remains boxed as source. + if startup.secondary.len() < error_contract::MAX_DIAGNOSTICS { + startup.secondary.push(secondary); + } +} + +fn record_create_persistence_failure( + startup: &mut runtime_api::WorkerStartupFailure, + error: &RuntimeError, + phase: StartupFailurePhase, + outcome: StartupPersistenceOutcome, +) { + push_create_secondary( + startup, + StartupSecondaryFailure::Persistence { phase, outcome }, + ); + if startup.io_kind.is_none() { + startup.io_kind = create_io_kind(error); + } + tracing::error!( + ?phase, + ?outcome, + "Worker create compensation persistence failed" + ); +} + +#[cfg(test)] fn runtime_worker_create_failure_fields( error: &RuntimeError, ) -> (&'static str, Option, Option) { @@ -8425,6 +8677,7 @@ fn runtime_worker_create_failure_fields( RuntimeError::ExecutionBackendUnavailable { .. } => { ("execution_backend_unavailable", None, None) } + RuntimeError::WorkerCreateFailed { .. } => ("worker_create_failed", None, None), RuntimeError::WorkerExecutionRejected { operation, outcome, .. } => ( @@ -8454,9 +8707,11 @@ fn runtime_worker_create_failure_fields( } } -const WORKER_CREATE_FAILURE_MESSAGE_LIMIT: usize = 512; +// Legacy Restore diagnostic presentation only. This is not a create failure +// projector or telemetry allowlist; creation uses startup_observation instead. +const WORKER_RESTORE_FAILURE_MESSAGE_LIMIT: usize = 512; -fn sanitize_worker_create_failure_message(message: &str) -> String { +fn sanitize_worker_restore_failure_message(message: &str) -> String { let mut redact_next = false; let detail = message .split_whitespace() @@ -8496,10 +8751,10 @@ fn sanitize_worker_create_failure_message(message: &str) -> String { }) .collect::>() .join(" "); - let truncated = detail.chars().count() > WORKER_CREATE_FAILURE_MESSAGE_LIMIT; + let truncated = detail.chars().count() > WORKER_RESTORE_FAILURE_MESSAGE_LIMIT; let mut detail = detail .chars() - .take(WORKER_CREATE_FAILURE_MESSAGE_LIMIT) + .take(WORKER_RESTORE_FAILURE_MESSAGE_LIMIT) .collect::(); if truncated { detail.push('…'); @@ -8507,43 +8762,13 @@ fn sanitize_worker_create_failure_message(message: &str) -> String { detail } -fn runtime_worker_create_failure_message(error: &RuntimeError) -> Option { - match error { - RuntimeError::WorkerExecutionRejected { message, .. } => { - Some(sanitize_worker_create_failure_message(message)) - } - _ => None, - } -} - fn write_runtime_worker_create_failure( worker_id: WorkerId, workspace_id: Option<&str>, error: &RuntimeError, elapsed: std::time::Duration, ) { - let (error_kind, operation, outcome) = runtime_worker_create_failure_fields(error); - let execution_failure_code = match error { - RuntimeError::WorkerExecutionRejected { result, .. } => { - worker_execution_failure_code(result) - } - _ => "", - }; - let execution_failure_message = runtime_worker_create_failure_message(error); - tracing::error!( - target: "yoi::worker_create", - event = "worker_create_failed", - component = "runtime", - workspace_id = workspace_id.unwrap_or(""), - worker_id = %worker_id, - error_kind, - operation = operation.as_deref().unwrap_or(""), - outcome = outcome.as_deref().unwrap_or(""), - execution_failure_code, - execution_failure_message = execution_failure_message.as_deref().unwrap_or(""), - duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX), - "Worker creation failed" - ); + crate::startup_observation::write_create_failure(worker_id, workspace_id, error, elapsed); } fn validate_create_workspace_scope( @@ -8692,6 +8917,11 @@ mod tests { use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Barrier, Condvar, Mutex}; + mod create_failure_tests { + use super::*; + include!("runtime/create_failure_tests.rs"); + } + #[cfg(feature = "fs-store")] mod create_rollback_tests { //! Connect the real create/rollback path to FsRuntimeStore and reopen its disk state. @@ -8911,57 +9141,6 @@ mod tests { } } - #[test] - fn worker_create_failure_fields_only_include_sanitized_execution_detail() { - let invalid_request = RuntimeError::InvalidRequest("private-token".to_string()); - let (error_kind, operation, outcome) = - runtime_worker_create_failure_fields(&invalid_request); - assert_eq!(error_kind, "invalid_request"); - assert_eq!(operation, None); - assert_eq!(outcome, None); - assert_eq!( - runtime_worker_create_failure_message(&invalid_request), - None - ); - - let result = WorkerExecutionResult::errored( - WorkerExecutionOperation::Spawn, - "failed to spawn controller in /private/path: File exists; token super-secret", - ); - let error = RuntimeError::WorkerExecutionRejected { - worker_id: WorkerId::now_v7(), - operation: result.operation, - outcome: result.outcome, - message: result.message_or_default(), - result, - }; - assert_eq!( - runtime_worker_create_failure_message(&error).as_deref(), - Some("failed to spawn controller in [redacted] File exists; token [redacted]") - ); - - assert_eq!( - sanitize_worker_create_failure_message("backend private-token failed"), - "backend [redacted] failed" - ); - - let truncated = sanitize_worker_create_failure_message(&"x".repeat(513)); - assert_eq!(truncated.chars().count(), 513); - assert!(truncated.ends_with('…')); - } - - #[test] - fn worker_create_failure_code_classifies_durable_acceptance_timeout() { - let timeout = WorkerExecutionResult::errored( - WorkerExecutionOperation::Input, - "timed out waiting for durable Worker Submit acceptance; private-token", - ); - assert_eq!( - worker_execution_failure_code(&timeout), - "input_commit_timeout" - ); - } - fn test_command() -> protocol::WorkerCommandEnvelope { protocol::WorkerCommandEnvelope { command_id: 1 } } @@ -9950,7 +10129,6 @@ mod tests { recovery_absent: AtomicBool, recovery_checks: std::sync::atomic::AtomicUsize, spawn_result: Mutex>, - #[cfg(feature = "fs-store")] before_spawn_result: Mutex>>, job_bindings: Mutex>>, dispatch_result: Mutex>, @@ -10197,7 +10375,6 @@ mod tests { .lock() .unwrap() .push(request.request.backend_job.clone()); - #[cfg(feature = "fs-store")] if let Some(before_result) = self.before_spawn_result.lock().unwrap().take() { before_result(); } @@ -15503,13 +15680,7 @@ mod tests { let error = runtime.create_worker(request).unwrap_err(); - assert!(matches!( - error, - RuntimeError::WorkerExecutionRejected { - outcome: crate::execution::WorkerExecutionOutcome::Rejected, - .. - } - )); + create_failure_tests::assert_ack_failure(&error); assert!(runtime.list_workers().unwrap().is_empty()); } @@ -15524,13 +15695,7 @@ mod tests { let error = runtime.create_worker(request).unwrap_err(); - assert!(matches!( - error, - RuntimeError::WorkerExecutionRejected { - outcome: crate::execution::WorkerExecutionOutcome::Rejected, - .. - } - )); + create_failure_tests::assert_ack_failure(&error); assert!(runtime.list_workers().unwrap().is_empty()); } @@ -17160,7 +17325,7 @@ mod tests { ); assert!(!diagnostic.message.contains("credential-value")); assert!(!diagnostic.message.contains("/private/path")); - assert!(diagnostic.message.chars().count() <= WORKER_CREATE_FAILURE_MESSAGE_LIMIT + 1); + assert!(diagnostic.message.chars().count() <= WORKER_RESTORE_FAILURE_MESSAGE_LIMIT + 1); let before = std::fs::read( dir.path() .join("runtime/workers") @@ -17346,7 +17511,7 @@ mod tests { assert!(!message.contains("/private"), "{message}"); assert!(!message.contains("restore-token"), "{message}"); assert!(!message.contains("hidden-value"), "{message}"); - assert!(message.chars().count() <= WORKER_CREATE_FAILURE_MESSAGE_LIMIT + 1); + assert!(message.chars().count() <= WORKER_RESTORE_FAILURE_MESSAGE_LIMIT + 1); assert!( runtime .lock() diff --git a/crates/worker-runtime/src/runtime/create_failure_tests.rs b/crates/worker-runtime/src/runtime/create_failure_tests.rs new file mode 100644 index 000000000..6e62d877f --- /dev/null +++ b/crates/worker-runtime/src/runtime/create_failure_tests.rs @@ -0,0 +1,626 @@ +// Registered by runtime.rs's normal library test module. Execution responses are +// fixtures; creation, persistence, projection and compensation use the real owner. +use protocol::startup_failure::{ + RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure, + StartupIoKind, StartupResourceFacts, StartupResources, StartupStage, +}; + +fn boundary_reason(failure: &WorkerCreateFailure) -> StartupBoundaryReason { + let projected = failure + .startup() + .primary + .project_for::(&()) + .unwrap(); + let StartupCause::Boundary(cause) = projected.details() else { + panic!("expected a closed boundary cause"); + }; + cause.reason +} + +pub(super) fn assert_ack_failure(error: &RuntimeError) { + assert!(matches!(error, RuntimeError::WorkerCreateFailed { .. })); + let failure = error.create_failure().unwrap(); + assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown); + assert_eq!( + failure.operation().finalize, + Some(FinalizeFacts { + stage: FinalizeStage::InitialInput, + compensation: CompensationOutcome::Completed, + }) + ); + assert_eq!( + boundary_reason(&failure), + StartupBoundaryReason::InvalidResponse + ); + // The missing/foreign receipt is not rewritten into a confirmed rejection. + assert!(error.execution_result().unwrap().is_accepted()); + assert_eq!(error.to_string(), "worker creation failed"); +} + +fn rejected_input() -> WorkerExecutionResult { + let attachment = worker::startup_failure::runtime_startup_attachment(RuntimeStartupCause { + stage: StartupStage::Controller, + reason: RuntimeStartupReason::StartupFailed, + }) + .unwrap(); + let mut result = WorkerExecutionResult::rejected( + WorkerExecutionOperation::Input, + "TEST_ONLY_SECRET original rejection /private/owner/path", + ) + .with_startup_failure(attachment); + result + .secondary_failures + .push(StartupSecondaryFailure::Cleanup { + failure: StartupCleanupFailure::Workdir, + }); + result.startup_resources = Some(StartupResourceFacts { + resources: StartupResources::Unproven, + cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir), + durable_effects_possible: true, + }); + result.startup_failure_persistence = Some(StartupPersistenceOutcome::NotSaved); + result.startup_io_kind = Some(StartupIoKind::PermissionDenied); + result +} + +#[derive(Clone, Copy, Debug)] +enum DispatchFault { + DroppedAck, + MismatchedAck, + ConfirmedRejection, + LostResponse, +} + +#[derive(Clone, Copy, Debug)] +enum CleanupFault { + None, + MissingTermination, + Rejected, + LostResponse, + MismatchedTermination, +} + +#[test] +fn connected_create_dispatch_and_cleanup_fault_matrix() { + for dispatch in [ + DispatchFault::DroppedAck, + DispatchFault::MismatchedAck, + DispatchFault::ConfirmedRejection, + DispatchFault::LostResponse, + ] { + for cleanup in [ + CleanupFault::None, + CleanupFault::MissingTermination, + CleanupFault::Rejected, + CleanupFault::LostResponse, + CleanupFault::MismatchedTermination, + ] { + let (runtime, backend) = runtime_and_backend(); + let original = match dispatch { + DispatchFault::DroppedAck => { + WorkerExecutionResult::accepted(WorkerExecutionOperation::Input) + } + DispatchFault::MismatchedAck => { + backend.preserve_submission_acknowledgement_id(); + WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + "foreign-request", + "foreign-submission", + protocol::SubmissionDisposition::Started, + ) + } + DispatchFault::ConfirmedRejection => rejected_input(), + DispatchFault::LostResponse => WorkerExecutionResult::errored( + WorkerExecutionOperation::Input, + "response lost; TEST_ONLY_SECRET", + ), + }; + backend.set_dispatch_result(original.clone()); + let compensation = match cleanup { + CleanupFault::None => CompensationOutcome::Completed, + CleanupFault::MissingTermination => { + backend.set_stop_result(WorkerExecutionResult::accepted( + WorkerExecutionOperation::Stop, + )); + CompensationOutcome::Unknown + } + CleanupFault::Rejected => { + backend.set_stop_result(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Stop, + "TEST_ONLY_SECRET stop rejected", + )); + CompensationOutcome::Residual + } + CleanupFault::LostResponse => { + backend.set_stop_result(WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "TEST_ONLY_SECRET stop response lost", + )); + CompensationOutcome::Unknown + } + CleanupFault::MismatchedTermination => { + let mut stop = WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop); + stop.termination = Some(crate::execution::WorkerExecutionTermination { + scope: crate::execution::WorkerExecutionTerminationScope::Controller, + worker_ref: WorkerRef::new(WorkerId::now_v7()), + stop_operation_id: WorkerLifecycleOperationId::new(), + execution_id: Uuid::now_v7(), + shutdown_command_id: None, + submissions: Vec::new(), + feature_failures: Vec::new(), + }); + backend.set_stop_result(stop); + CompensationOutcome::Unknown + } + }; + let mut request = task_request("connected create fault matrix"); + request.initial_input = Some(WorkerInput::user("start once")); + let error = runtime.create_worker(request).unwrap_err(); + let failure = error.create_failure().unwrap(); + assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!( + failure.operation().execution, + if matches!(dispatch, DispatchFault::ConfirmedRejection) { + ExecutionOutcome::Errored + } else { + ExecutionOutcome::Unknown + }, + "{dispatch:?}/{cleanup:?}" + ); + assert_eq!( + failure.operation().finalize, + Some(FinalizeFacts { + stage: FinalizeStage::InitialInput, + compensation, + }) + ); + assert_eq!(failure.operation().backend_compensation, None); + assert_eq!(failure.operation().correlation, None); + assert_eq!(error.execution_result(), Some(&original)); + assert_eq!(error.to_string(), "worker creation failed"); + if let Some(primary) = original.startup_failure.as_ref() { + assert_eq!(&failure.startup().primary, primary); + assert!( + failure + .startup() + .secondary + .starts_with(&original.secondary_failures) + ); + assert_eq!(failure.startup().resources, original.startup_resources); + assert_eq!( + failure.startup().persistence, + original.startup_failure_persistence + ); + assert_eq!(failure.startup().io_kind, original.startup_io_kind); + assert!( + std::error::Error::source(&error) + .unwrap() + .to_string() + .contains("TEST_ONLY_SECRET") + ); + } else { + assert_eq!( + boundary_reason(&failure), + if matches!(dispatch, DispatchFault::LostResponse) { + StartupBoundaryReason::FinalizeFailed + } else { + StartupBoundaryReason::InvalidResponse + } + ); + let resources = failure.startup().resources.unwrap(); + assert!(resources.durable_effects_possible); + assert_eq!( + resources.cleanup, + match cleanup { + CleanupFault::None => StartupCleanup::Confirmed, + CleanupFault::Rejected => + StartupCleanup::Failed(StartupCleanupFailure::Ownership), + _ => StartupCleanup::Required, + } + ); + } + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1); + assert!( + backend + .submission_receipt_queries + .lock() + .unwrap() + .is_empty() + ); + assert_eq!( + runtime.list_workers().unwrap().is_empty(), + matches!(cleanup, CleanupFault::None) + ); + } + } +} + +#[test] +fn independent_input_error_is_not_a_create_failure() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("independent input")) + .unwrap(); + let original = rejected_input(); + backend.set_dispatch_result(original.clone()); + let error = runtime + .send_input(&worker.worker_ref, WorkerInput::user("ordinary input")) + .unwrap_err(); + assert!(matches!( + &error, + RuntimeError::WorkerExecutionRejected { .. } + )); + assert_eq!(error.create_failure(), None); + assert_eq!(error.execution_result(), Some(&original)); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); +} + +#[test] +fn connected_create_initial_input_read_failure_keeps_create_context() { + let (runtime, backend) = runtime_and_backend(); + let mut request = task_request("initial input read failure"); + request.initial_input = Some(WorkerInput::user("never dispatched")); + let worker_id = request.worker_id; + *backend.before_spawn_result.lock().unwrap() = Some(Box::new({ + let runtime = runtime.clone(); + move || { + runtime.lock().unwrap().workers.remove(&worker_id); + } + })); + let error = runtime.create_worker(request).unwrap_err(); + let RuntimeError::WorkerCreateFailed { failure, source } = error else { + panic!("post-Connected read failure must remain a create failure"); + }; + assert!(matches!(*source, RuntimeError::WorkerNotFound { .. })); + assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown); + assert_eq!( + failure.operation().finalize.unwrap().stage, + FinalizeStage::InitialInput + ); + assert_eq!( + boundary_reason(&failure), + StartupBoundaryReason::FinalizeFailed + ); + assert!(backend.dispatched_inputs.lock().unwrap().is_empty()); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); +} + +#[test] +fn connected_create_publication_failure_preserves_known_input_acceptance() { + let (runtime, backend) = runtime_and_backend(); + let mut request = task_request("publication fails after known acceptance"); + request.initial_input = Some(WorkerInput::user("accepted once")); + let worker_ref = WorkerRef::new(request.worker_id); + // Substitute only Connected data. The invalid backend Workdir identifier + // reaches the real subscription projection after the owner commit. + backend + .contexts + .lock() + .unwrap() + .insert(request.worker_id, runtime.execution_context(worker_ref)); + *backend.spawn_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStatus::Idle.into(), + workdir_attachments: vec![WorkingDirectoryAttachmentStatus { + alias: workdir::WorkdirAttachmentAlias::new("checkout").unwrap(), + working_directory: serde_json::from_value(serde_json::json!({ + "summary": { + "working_directory_id": "", + "repository_id": "repo", + "materializer_kind": "runtime_git_clone", + "status": "active" + } + })) + .unwrap(), + }], + }); + let error = runtime.create_worker(request).unwrap_err(); + let failure = error.create_failure().unwrap(); + assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted); + assert_eq!( + failure.operation().finalize, + Some(FinalizeFacts { + stage: FinalizeStage::Publication, + compensation: CompensationOutcome::Completed, + }) + ); + assert_eq!( + boundary_reason(&failure), + StartupBoundaryReason::FinalizeFailed + ); + let RuntimeError::WorkerCreateFailed { source, .. } = error else { + unreachable!() + }; + assert!(matches!(*source, RuntimeError::InvalidRequest(_))); + assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); +} + +#[test] +fn create_correlation_uses_bound_runtime_identity_for_connected_and_spawn_failures() { + for connected in [false, true] { + let (runtime, backend) = runtime_and_backend(); + runtime.bind_runtime_identity("runtime-identity").unwrap(); + let mut request = task_request("safe correlation"); + let worker_id = request.worker_id; + if connected { + request.initial_input = Some(WorkerInput::user("start once")); + backend.set_dispatch_result(rejected_input()); + } else { + let mut result = rejected_input(); + result.operation = WorkerExecutionOperation::Spawn; + *backend.spawn_result.lock().unwrap() = + Some(WorkerExecutionSpawnResult::Rejected(result)); + } + let error = runtime.create_worker(request).unwrap_err(); + assert_eq!( + error.create_failure().unwrap().operation().correlation, + Some(StartupCorrelation::for_worker( + "runtime-identity", + &worker_id.to_string() + )) + ); + assert_eq!(error.to_string(), "worker creation failed"); + } +} + +#[cfg(feature = "fs-store")] +#[test] +fn connected_create_commit_write_faults_preserve_acceptance_and_io_facts() { + use crate::fs_store::AtomicWriteFault; + for fault in [ + AtomicWriteFault::BeforeRename, + AtomicWriteFault::AfterRename, + ] { + for has_initial_input in [false, true] { + let root = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(root.path().join("runtime")) + .with_runtime_id("create-commit-fault"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + *backend.before_spawn_result.lock().unwrap() = Some(Box::new({ + let runtime = runtime.clone(); + move || { + runtime + .lock() + .unwrap() + .fs_store() + .unwrap() + .fail_next_worker_write(fault) + } + })); + let mut request = task_request("create commit fault"); + if has_initial_input { + request.initial_input = Some(WorkerInput::user("accepted once")); + } + let worker_id = request.worker_id; + let error = runtime.create_worker(request).unwrap_err(); + let failure = error.create_failure().unwrap(); + assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted); + assert_eq!( + failure.operation().finalize, + Some(FinalizeFacts { + stage: FinalizeStage::Publication, + compensation: CompensationOutcome::Completed, + }) + ); + assert_eq!( + boundary_reason(&failure), + StartupBoundaryReason::FinalizeFailed + ); + assert_eq!(failure.startup().io_kind, Some(StartupIoKind::Other)); + assert_eq!(failure.startup().persistence, None); + assert!( + failure + .startup() + .secondary + .contains(&StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::RuntimeReconciliationRecordSave, + outcome: if fault == AtomicWriteFault::AfterRename { + StartupPersistenceOutcome::OutcomeUnknown + } else { + StartupPersistenceOutcome::NotSaved + }, + }) + ); + let RuntimeError::WorkerCreateFailed { source, .. } = error else { + unreachable!() + }; + assert!(matches!( + *source, + RuntimeError::StoreIo { .. } | RuntimeError::StoreCommitOutcomeUnknown { .. } + )); + assert_eq!( + backend.dispatched_inputs.lock().unwrap().len(), + usize::from(has_initial_input) + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + assert!( + !options + .root + .join("workers") + .join(worker_id.to_string()) + .exists() + ); + drop(runtime); + drop(backend); + let reopened = Runtime::with_fs_store(options).unwrap(); + assert!(reopened.list_workers().unwrap().is_empty()); + } + } +} + +#[cfg(feature = "fs-store")] +#[test] +fn connected_create_cleanup_persistence_faults_never_replace_primary() { + for delete_fault in [false, true] { + let root = tempfile::tempdir().unwrap(); + let store_root = root.path().join("runtime"); + let options = + FsRuntimeStoreOptions::new(&store_root).with_runtime_id("create-cleanup-fault"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let mut request = task_request("create cleanup persistence fault"); + request.initial_input = Some(WorkerInput::user("rejected once")); + let worker_dir = store_root + .join("workers") + .join(request.worker_id.to_string()); + let runtime_path = store_root.join("runtime.json"); + *backend.before_spawn_result.lock().unwrap() = Some(Box::new(move || { + if delete_fault { + std::fs::create_dir(worker_dir.join("metadata.json")).unwrap(); + } else { + std::fs::rename(&runtime_path, runtime_path.with_extension("saved")).unwrap(); + std::fs::create_dir(&runtime_path).unwrap(); + } + })); + let original = rejected_input(); + backend.set_dispatch_result(original.clone()); + let error = runtime.create_worker(request).unwrap_err(); + let failure = error.create_failure().unwrap(); + assert_eq!(error.execution_result(), Some(&original)); + assert_eq!(failure.startup().primary, original.startup_failure.unwrap()); + assert_eq!(failure.startup().io_kind, original.startup_io_kind); + assert_eq!(failure.operation().execution, ExecutionOutcome::Errored); + assert_eq!( + failure.operation().finalize.unwrap().compensation, + CompensationOutcome::Unknown + ); + assert!( + failure + .startup() + .secondary + .contains(&StartupSecondaryFailure::Persistence { + phase: if delete_fault { + StartupFailurePhase::RuntimeRecordDelete + } else { + StartupFailurePhase::RuntimeSnapshotSave + }, + outcome: StartupPersistenceOutcome::OutcomeUnknown, + }) + ); + if !delete_fault { + assert!( + failure + .startup() + .secondary + .contains(&StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::RuntimeRecordDelete, + outcome: StartupPersistenceOutcome::Persisted, + }) + ); + } + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1); + } +} + +#[test] +fn connected_create_retains_cleanup_owner_resource_and_metadata_observations() { + let (runtime, backend) = runtime_and_backend(); + backend.set_dispatch_result(WorkerExecutionResult::accepted( + WorkerExecutionOperation::Input, + )); + let mut stop = + WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, "TEST_ONLY_SECRET"); + let resources = StartupResourceFacts { + resources: StartupResources::Unproven, + cleanup: StartupCleanup::Failed(StartupCleanupFailure::Feature), + durable_effects_possible: true, + }; + stop.startup_resources = Some(resources); + stop.startup_io_kind = Some(StartupIoKind::TimedOut); + stop.startup_failure_persistence = Some(StartupPersistenceOutcome::OutcomeUnknown); + backend.set_stop_result(stop); + let mut request = task_request("cleanup owner observations"); + request.initial_input = Some(WorkerInput::user("start once")); + let error = runtime.create_worker(request).unwrap_err(); + let failure = error.create_failure().unwrap(); + assert_eq!(failure.startup().resources, Some(resources)); + assert_eq!(failure.startup().io_kind, Some(StartupIoKind::TimedOut)); + assert_eq!(failure.startup().persistence, None); + assert!( + failure + .startup() + .secondary + .contains(&StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::MetadataSave, + outcome: StartupPersistenceOutcome::OutcomeUnknown, + }) + ); + assert_eq!( + failure.operation().finalize.unwrap().compensation, + CompensationOutcome::Unknown + ); + assert_eq!( + boundary_reason(&failure), + StartupBoundaryReason::InvalidResponse + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); +} + +#[cfg(feature = "fs-store")] +#[test] +fn connected_create_unknown_cleanup_retention_write_failure_preserves_primary() { + use crate::fs_store::AtomicWriteFault; + let root = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(root.path().join("runtime")) + .with_runtime_id("create-retain-fault"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + *backend.before_spawn_result.lock().unwrap() = Some(Box::new({ + let runtime = runtime.clone(); + move || { + runtime + .lock() + .unwrap() + .fs_store() + .unwrap() + .fail_next_worker_write(AtomicWriteFault::BeforeRename) + } + })); + let original = rejected_input(); + backend.set_dispatch_result(original.clone()); + // Accepted stop without a termination receipt cannot authorize deletion. + backend.set_stop_result(WorkerExecutionResult::accepted( + WorkerExecutionOperation::Stop, + )); + let mut request = task_request("unknown cleanup retention persistence failure"); + request.initial_input = Some(WorkerInput::user("rejected once")); + let error = runtime.create_worker(request).unwrap_err(); + let failure = error.create_failure().unwrap(); + assert_eq!(error.execution_result(), Some(&original)); + assert_eq!( + failure.operation().finalize.unwrap().compensation, + CompensationOutcome::Unknown + ); + assert!( + failure + .startup() + .secondary + .contains(&StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::RuntimeReconciliationRecordSave, + outcome: StartupPersistenceOutcome::NotSaved, + }) + ); + assert_eq!(runtime.list_workers().unwrap().len(), 1); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1); + assert!( + backend + .submission_receipt_queries + .lock() + .unwrap() + .is_empty() + ); +} diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs index ba8e884c7..0ad6d69ae 100644 --- a/crates/worker-runtime/src/runtime/restore_continuation.rs +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -479,7 +479,7 @@ fn run( } Ok(Step::Waiting) | Ok(Step::Contended) => None, Ok(Step::Retry(reason)) => Some(reason), - Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())), + Err(error) => Some(sanitize_worker_restore_failure_message(&error.to_string())), }; if let Some(reason) = &reason { repeated = repeated.saturating_add(1); diff --git a/crates/worker-runtime/src/startup_observation.rs b/crates/worker-runtime/src/startup_observation.rs new file mode 100644 index 000000000..dbec359cd --- /dev/null +++ b/crates/worker-runtime/src/startup_observation.rs @@ -0,0 +1,205 @@ +//! The Runtime creation responsibility sink records only projected typed fields. +//! Source/provider strings are never safe merely because a blacklist did not match. +//! Wrappers/projectors do not write events; a failing sink cannot change the result. +use crate::{error::RuntimeError, identity::WorkerId}; + +pub(crate) fn write_create_failure( + worker_id: WorkerId, + workspace_id: Option<&str>, + error: &RuntimeError, + elapsed: std::time::Duration, +) { + let failure = error.create_failure(); + let projected = failure + .as_ref() + .and_then(|failure| failure.telemetry().ok()); + let fields = projected.as_ref().map(error_contract::observation_fields); + let operation = failure.as_ref().map(|failure| failure.operation()); + tracing::error!( + target: "yoi::worker_create", + event = "worker_create_failed", + component = "runtime", + workspace_id = workspace_id.unwrap_or(""), + worker_id = %worker_id, + domain = fields.as_ref().map_or("startup_boundary", |fields| fields.domain), + reason = fields.as_ref().map_or("observation_unavailable", |fields| fields.reason), + operation = fields.as_ref().and_then(|fields| fields.operation), + stage = fields.as_ref().and_then(|fields| fields.stage), + correlation = operation.and_then(|operation| operation.correlation.as_ref()).map(|value| value.as_str()), + acceptance = ?operation.map(|operation| operation.acceptance), + execution = ?operation.map(|operation| operation.execution), + finalize = ?operation.and_then(|operation| operation.finalize), + persistence = ?failure.as_ref().and_then(|failure| failure.startup().persistence), + resources = ?failure.as_ref().and_then(|failure| failure.startup().resources), + duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX), + "Worker creation returned a typed failure" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::{CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveSource}; + use crate::execution::{ + WorkerExecutionBackend, WorkerExecutionOperation, WorkerExecutionResult, + WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + }; + use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveInput}; + use crate::{Runtime, RuntimeOptions}; + use runtime_api::startup::*; + use std::collections::BTreeMap; + use std::sync::{Arc, Mutex}; + use tracing_subscriber::prelude::*; + + const CANARIES: &[&str] = &[ + "aQ7xV2zN9mL5", + "/private/startup/provider", + "Bearer q3fV8aJ6", + "token=K7sM2pQ4", + ]; + struct FailingBackend; + impl WorkerExecutionBackend for FailingBackend { + fn backend_id(&self) -> &str { + "log-owner-fixture" + } + fn dispatch_input( + &self, + _: &crate::identity::WorkerRef, + _: crate::interaction::WorkerInput, + ) -> WorkerExecutionResult { + panic!("rejected creation never dispatches input") + } + fn spawn_worker(&self, _: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { + let result = + WorkerExecutionResult::errored(WorkerExecutionOperation::Spawn, CANARIES.join(" ")) + .with_startup_failure( + worker::startup_failure::runtime_startup_attachment( + runtime_api::RuntimeStartupCause { + stage: runtime_api::StartupStage::Controller, + reason: runtime_api::RuntimeStartupReason::StartupFailed, + }, + ) + .unwrap(), + ); + WorkerExecutionSpawnResult::Errored(result) + } + } + #[derive(Clone, Default)] + struct Events(Arc>>>); + struct Fields(BTreeMap); + impl tracing::field::Visit for Fields { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.0.insert(field.name().into(), value.into()); + } + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0.insert(field.name().into(), format!("{value:?}")); + } + } + impl tracing_subscriber::Layer for Events { + fn on_event( + &self, + event: &tracing::Event<'_>, + _: tracing_subscriber::layer::Context<'_, S>, + ) { + if event.metadata().target() == "yoi::worker_create" { + let mut fields = Fields(BTreeMap::new()); + event.record(&mut fields); + self.0.lock().unwrap().push(fields.0); + } + } + } + fn request() -> CreateWorkerRequest { + let archive = ProfileSourceArchive::build(ProfileSourceArchiveInput { + id: "log-profile".into(), + entrypoints: BTreeMap::from([("builtin:coder".into(), "profiles/coder.dcdl".into())]), + imports: BTreeMap::new(), + sources: BTreeMap::from([("profiles/coder.dcdl".into(), "{}".into())]), + }) + .unwrap(); + CreateWorkerRequest { + worker_id: WorkerId::now_v7(), + create_fingerprint: "log-create".into(), + profile: ProfileSelector::Builtin("builtin:coder".into()), + display_name: None, + profile_source: ProfileSourceArchiveSource::Embedded { archive }, + config_bundle: None, + initial_input: None, + workdir_attachment_requests: Vec::new(), + workdir_attachments: Vec::new(), + worker_observation_enabled: false, + worker_observation_grants: Vec::new(), + workspace_api: None, + memory_settings: None, + plugin_execution: None, + subjektiv_attached: false, + backend_job: None, + } + } + #[test] + fn real_runtime_creation_sink_omits_unlabelled_provider_canaries_and_matches_typed_result() { + let runtime = + Runtime::with_execution_backend(RuntimeOptions::default(), Arc::new(FailingBackend)) + .unwrap(); + runtime + .bind_runtime_identity("runtime-log-fixture") + .unwrap(); + let request = request(); + let worker_id = request.worker_id; + let events = Events::default(); + let subscriber = tracing_subscriber::registry().with(events.clone()); + let error = + tracing::subscriber::with_default(subscriber, || runtime.create_worker(request)) + .unwrap_err(); + let failure = error + .create_failure() + .expect("creation owner must retain a typed cause"); + let wire = failure + .problem( + StartupBoundary::Runtime, + &SessionFailureVisibility { + can_view_cause: true, + }, + "", + ) + .unwrap(); + let encoded = failure.encode_storage().unwrap(); + let DecodeStoredWorkerCreateFailure::Known(saved) = + decode_stored_worker_create_failure(&encoded) + else { + panic!("known storage"); + }; + assert_eq!(saved.public_facts(), failure.public_facts()); + let captured = events.0.lock().unwrap(); + assert_eq!(captured.len(), 1); + let event = &captured[0]; + let projection = failure.telemetry().unwrap(); + let fields = error_contract::observation_fields(&projection); + assert_eq!(event["domain"], fields.domain); + assert_eq!(event["reason"], fields.reason); + assert_eq!( + event["correlation"], + StartupCorrelation::for_worker("runtime-log-fixture", &worker_id.to_string()).as_str() + ); + for canary in CANARIES { + assert!(!format!("{captured:?}").contains(canary)); + assert!(!String::from_utf8_lossy(wire.body()).contains(canary)); + assert!(!encoded.contains(canary)); + } + assert!(!event.contains_key("execution_failure_message")); + } + #[test] + fn unsupported_failure_logs_only_fixed_fallback_not_source_text() { + let error = RuntimeError::InvalidRequest(CANARIES.join(" ")); + let events = Events::default(); + let subscriber = tracing_subscriber::registry().with(events.clone()); + tracing::subscriber::with_default(subscriber, || { + write_create_failure(WorkerId::now_v7(), None, &error, std::time::Duration::ZERO) + }); + let captured = events.0.lock().unwrap(); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0]["reason"], "observation_unavailable"); + for canary in CANARIES { + assert!(!format!("{captured:?}").contains(canary)); + } + } +} diff --git a/crates/worker-runtime/src/worker_source.rs b/crates/worker-runtime/src/worker_source.rs index 7366fcc5b..a315a102f 100644 --- a/crates/worker-runtime/src/worker_source.rs +++ b/crates/worker-runtime/src/worker_source.rs @@ -529,8 +529,8 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient { } #[cfg(feature = "http-server")] -fn generated_workspace_response( - response: Result>, +fn generated_workspace_response( + response: Result>, ) -> Result { match response { Ok(response) => Ok(WorkspaceResponse { @@ -721,7 +721,19 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient { ), WorkspaceServerOperation::WorkerControlSpawn(request) => { generated_workspace_response( - client.worker_control_spawn(workspace_id, request).await, + client.worker_control_spawn(workspace_id, request).await.map_err(|error| match error { + server_api::client_support::ClientError::Public { status, error } => { + let error = match error.failure().cloned() { + // This already authenticated Worker control caller may + // receive the enum-only public startup cause. The domain + // allowlist reprojects it, never forwards upstream text. + Some(failure) => server_api::WorkerCreateApiError::startup(failure, &protocol::session_failure::SessionFailureVisibility { can_view_cause: true }), + None => error, + }; + server_api::client_support::ClientError::Public { status, error } + } + other => other, + }), ) } WorkspaceServerOperation::WorkerControlInput { diff --git a/crates/workspace-server/Cargo.toml b/crates/workspace-server/Cargo.toml index 4f14a0de4..a254794e0 100644 --- a/crates/workspace-server/Cargo.toml +++ b/crates/workspace-server/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" test-support = [] [dependencies] +error-contract.workspace = true api-macros = { workspace = true, features = ["axum"] } async-trait.workspace = true axum = { workspace = true, features = ["ws"] } @@ -70,7 +71,6 @@ webauthn-rs = { workspace = true, features = ["conditional-ui"] } webauthn-rs-proto = "0.5.5" [dev-dependencies] -error-contract.workspace = true rusqlite = { workspace = true, features = ["hooks"] } feature-storage = { workspace = true, features = ["test-support"] } native-tls = "0.2" diff --git a/crates/workspace-server/src/backend_job_upgrade_tests.rs b/crates/workspace-server/src/backend_job_upgrade_tests.rs index fe445887e..6df37b38a 100644 --- a/crates/workspace-server/src/backend_job_upgrade_tests.rs +++ b/crates/workspace-server/src/backend_job_upgrade_tests.rs @@ -1476,6 +1476,21 @@ fn captured_history_is_accepted_at_each_real_retained_prefix_and_cannot_be_forge let dir = tempfile::tempdir().unwrap(); let conn = legacy_fixture_to_85(&dir.path().join("prefixes.sqlite"), 85); let original = pre_content_values(&conn, false); + let mut expected_receipts = rows( + &conn, + "SELECT version,name FROM __yoi_schema_migrations ORDER BY version", + ); + expected_receipts.extend( + store_owner::MIGRATIONS + .iter() + .filter(|migration| migration.version > 85) + .map(|migration| { + vec![ + Value::Integer(migration.version), + Value::Text(migration.name.into()), + ] + }), + ); for migration in store_owner::MIGRATIONS .iter() .filter(|migration| migration.version >= 86 && migration.version <= 122) @@ -1496,10 +1511,13 @@ fn captured_history_is_accepted_at_each_real_retained_prefix_and_cannot_be_forge &conn, "SELECT version,name FROM __yoi_schema_migrations ORDER BY version", ); + assert_eq!(receipts, expected_receipts); assert_eq!( - receipts.last().unwrap(), - &vec![Value::Integer(123), Value::Text(MIGRATION_NAME.into())] + store_owner::current_schema_version(&conn).unwrap(), + store_owner::LATEST_SCHEMA_VERSION ); + super::super::storage_formats::verify_history(&conn, store_owner::LATEST_SCHEMA_VERSION) + .unwrap(); assert_eq!(original, pre_content_values(&conn, true)); assert!(conn.execute_batch("INSERT INTO backend_job_pre_content_history SELECT * FROM backend_job_pre_content_history LIMIT 1").is_err()); } diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index b341c617b..509ba4dad 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -628,6 +628,10 @@ fn initial_worker_input(segments: &[Segment]) -> Option { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct WorkerSpawnResult { + /// Typed failure for internal consumers, independent of diagnostic prose. + /// HTTP publication requires a fresh audience projection, not serialization here. + #[serde(skip)] + pub failure: Option, pub state: InternalWorkerOperationState, #[serde(skip_serializing_if = "Option::is_none")] pub worker: Option, @@ -672,6 +676,8 @@ fn required_worker_workspace_api( #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum InternalWorkerOperationState { + /// The boundary did not observe a conclusive operation result. + Unknown, Accepted, Unsupported, Rejected, @@ -680,6 +686,7 @@ pub enum InternalWorkerOperationState { impl From for server_api::WorkerOperationState { fn from(state: InternalWorkerOperationState) -> Self { match state { + InternalWorkerOperationState::Unknown => Self::Unknown, InternalWorkerOperationState::Accepted => Self::Accepted, InternalWorkerOperationState::Unsupported => Self::Unsupported, InternalWorkerOperationState::Rejected => Self::Rejected, @@ -1137,6 +1144,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { request: WorkerSpawnRequest, ) -> WorkerSpawnResult { WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Unsupported, worker: None, acceptance_evidence: Vec::new(), @@ -3210,6 +3218,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { { diagnostics.push(embedded_workdir_unsupported_diagnostic()); return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -3226,6 +3235,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { "Embedded backend Runtime is transportless; use run_accepted/create acceptance for backend-internal Workers".to_string(), )); return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -3251,6 +3261,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { error, )); return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -3263,6 +3274,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { Err(diagnostic) => { diagnostics.push(diagnostic); return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -3285,6 +3297,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { .create_worker_scoped(&workspace_scope, create_request) { Ok(detail) => WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Accepted, worker: Some(self.map_worker_detail(detail)), acceptance_evidence: vec![ @@ -3303,9 +3316,15 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { diagnostics, }, Err(err) => { + let failure = err.create_failure(); diagnostics.push(embedded_runtime_diagnostic(&err)); + let state = failure.as_ref().map_or( + InternalWorkerOperationState::Rejected, + startup_failure_operation_state, + ); WorkerSpawnResult { - state: InternalWorkerOperationState::Rejected, + failure, + state, worker: None, acceptance_evidence: Vec::new(), diagnostics, @@ -5750,6 +5769,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { WorkerSpawnAcceptanceRequirement::SocketReady ) { return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -5765,6 +5785,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { Ok(archive) => archive, Err(error) => { return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -5783,6 +5804,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { Ok(workspace_api) => workspace_api, Err(diagnostic) => { return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -5802,6 +5824,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { Ok(create) => create, Err(diagnostic) => { return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -5809,30 +5832,70 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { }; } }; - match self - .run_runtime_api( - REMOTE_WORKER_CREATE_TIMEOUT, - MAX_REMOTE_RUNTIME_RESPONSE_BYTES, - move |client| async move { client.create_worker(create).await }, - ) - .and_then(|value| { - runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>(value.worker) - }) { - Ok(response) => WorkerSpawnResult { - state: InternalWorkerOperationState::Accepted, - worker: Some(self.map_worker_detail(response)), - acceptance_evidence: vec![WorkerSpawnAcceptanceEvidence { - kind: "remote_runtime_worker_created".to_string(), - detail: "worker-runtime REST create endpoint accepted the Worker".to_string(), - }], - diagnostics: Vec::new(), - }, - Err(diagnostic) => WorkerSpawnResult { - state: InternalWorkerOperationState::Rejected, - worker: None, - acceptance_evidence: Vec::new(), - diagnostics: vec![diagnostic], - }, + let client = match self.runtime_api_client( + REMOTE_WORKER_CREATE_TIMEOUT, + MAX_REMOTE_RUNTIME_RESPONSE_BYTES, + ) { + Ok(client) => client, + Err(diagnostic) => { + return WorkerSpawnResult { + failure: None, + state: InternalWorkerOperationState::Rejected, + worker: None, + acceptance_evidence: Vec::new(), + diagnostics: vec![diagnostic], + }; + } + }; + let received = run_blocking_http(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| { + runtime_api::client_support::ClientError::failure( + runtime_api::client_support::ClientFailure::Transport, + ) + })? + .block_on(client.create_worker(create)) + }); + match received { + Ok(response) => { + match runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>( + response.worker, + ) { + Ok(worker) => WorkerSpawnResult { + failure: None, + state: InternalWorkerOperationState::Accepted, + worker: Some(self.map_worker_detail(worker)), + acceptance_evidence: vec![WorkerSpawnAcceptanceEvidence { + kind: "remote_runtime_worker_created".into(), + detail: "Runtime create returned a Worker summary".into(), + }], + diagnostics: Vec::new(), + }, + Err(diagnostic) => WorkerSpawnResult { + failure: Some(startup_observation_failure( + runtime_api::startup::StartupBoundaryReason::Decode, + )), + state: InternalWorkerOperationState::Unknown, + worker: None, + acceptance_evidence: Vec::new(), + diagnostics: vec![diagnostic], + }, + } + } + Err(error) => { + let failure = runtime_create_client_failure(&error); + let state = startup_failure_operation_state(&failure); + let diagnostic = runtime_api_diagnostic(&self.runtime_id, error); + WorkerSpawnResult { + failure: Some(failure), + state, + worker: None, + acceptance_evidence: Vec::new(), + diagnostics: vec![diagnostic], + } + } } } @@ -6884,6 +6947,11 @@ fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnosti message, ), ), + EmbeddedRuntimeError::WorkerCreateFailed { failure, .. } => diagnostic( + "embedded_worker_create_failed", + HostDiagnosticSeverity::Warning, + worker::startup_failure::internal_startup_failure_summary(&failure.startup().primary), + ), EmbeddedRuntimeError::WorkerExecutionRejected { operation, outcome, @@ -6966,6 +7034,9 @@ fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnosti }) }); } + if let EmbeddedRuntimeError::WorkerCreateFailed { failure, .. } = error { + projection.startup = internal_startup_transfer(failure.startup().clone()); + } projection } @@ -7367,6 +7438,7 @@ pub fn placeholder_worker(host_id: impl Into) -> InternalWorkerSummary { pub fn placeholder_spawn_response(host_id: impl Into) -> WorkerSpawnResult { WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Unsupported, worker: Some(placeholder_worker(host_id)), acceptance_evidence: Vec::new(), @@ -8811,7 +8883,7 @@ mod tests { } #[test] - fn embedded_runtime_spawn_execution_failure_is_rejected_and_not_input_capable() { + fn embedded_runtime_spawn_execution_error_preserves_unknown_acceptance_and_no_input() { let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend( "local:test", Arc::new(FailingSpawnBackend), @@ -8820,14 +8892,22 @@ mod tests { let mut request = embedded_spawn_request(); request.requested_worker_name = Some("friendly-name".into()); let spawned = runtime.spawn_worker(test_create_binding(), request); - assert_eq!(spawned.state, InternalWorkerOperationState::Rejected); + assert_eq!(spawned.state, InternalWorkerOperationState::Unknown); + let operation = spawned.failure.as_ref().unwrap().operation(); + assert_eq!( + operation.acceptance, + runtime_api::startup::CreationAcceptance::Unknown + ); + assert_eq!( + operation.execution, + runtime_api::startup::ExecutionOutcome::Errored + ); assert!(spawned.acceptance_evidence.is_empty()); assert_eq!(spawned.diagnostics.len(), 1); assert!(spawned.diagnostics.iter().any(|diagnostic| { - diagnostic.code == "embedded_worker_execution_rejected" - && diagnostic - .message - .contains("provider setup failed at [redacted]") + diagnostic.code == "embedded_worker_create_failed" + && diagnostic.startup.is_some() + && !diagnostic.message.contains("provider setup failed") && !diagnostic.message.contains("/tmp/secret-provider-config") && !diagnostic.message.contains("secret-value") && !diagnostic.message.contains("session-42") @@ -9977,3 +10057,69 @@ mod tests { }) } } + +/// A local observation failure is not a received execution failure. Its operation +/// facts remain unknown even if HTTP or compatibility diagnostics say "rejected". +fn startup_failure_operation_state( + failure: &runtime_api::startup::WorkerCreateFailure, +) -> InternalWorkerOperationState { + use runtime_api::startup::{CreationAcceptance, ExecutionOutcome}; + if failure.operation().acceptance == CreationAcceptance::Unknown + || failure.operation().execution == ExecutionOutcome::Unknown + { + InternalWorkerOperationState::Unknown + } else if failure.operation().execution == ExecutionOutcome::Unsupported { + InternalWorkerOperationState::Unsupported + } else { + InternalWorkerOperationState::Rejected + } +} + +fn startup_observation_failure( + reason: runtime_api::startup::StartupBoundaryReason, +) -> runtime_api::startup::WorkerCreateFailure { + use runtime_api::startup::*; + WorkerCreateFailure::from_cause( + StartupCause::Boundary(StartupBoundaryCause { reason }), + StartupOperationFacts { + acceptance: CreationAcceptance::Unknown, + execution: ExecutionOutcome::Unknown, + finalize: None, + correlation: None, + backend_compensation: None, + }, + ) + .expect("fixed enum-only observation fits the startup boundary") +} + +fn runtime_create_client_failure( + error: &runtime_api::client_support::ClientError, +) -> runtime_api::startup::WorkerCreateFailure { + use runtime_api::client_support::{ClientError, ClientFailure}; + use runtime_api::startup::StartupBoundaryReason; + match error { + ClientError::Public { error, .. } => error + .create_failure() + .cloned() + .unwrap_or_else(|| startup_observation_failure(StartupBoundaryReason::InvalidResponse)), + ClientError::Failure(failure) => startup_observation_failure(match failure { + ClientFailure::Timeout => StartupBoundaryReason::Timeout, + ClientFailure::ResponseTooLarge { .. } => StartupBoundaryReason::TooLarge, + ClientFailure::Decode { .. } | ClientFailure::ErrorResponseDecode { .. } => { + StartupBoundaryReason::Decode + } + ClientFailure::UnsupportedErrorResponse { .. } => { + StartupBoundaryReason::UnsupportedVersion + } + ClientFailure::UnexpectedStatus { .. } + | ClientFailure::MissingResponseHeader { .. } + | ClientFailure::DuplicateResponseHeader { .. } + | ClientFailure::InvalidResponseHeader { .. } + | ClientFailure::UnexpectedBody => StartupBoundaryReason::InvalidResponse, + ClientFailure::Authorization + | ClientFailure::RequestEncoding + | ClientFailure::Connect + | ClientFailure::Transport => StartupBoundaryReason::Transport, + }), + } +} diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 5c5b35a96..df4dca27c 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -695,6 +695,8 @@ CREATE TABLE worker_create_reservations ( updated_at TEXT NOT NULL, request_fingerprint TEXT, memory_language TEXT, singleton_key TEXT, singleton_generation INTEGER CHECK (singleton_generation IS NULL OR singleton_generation > 0), + startup_failure_json TEXT CHECK (startup_failure_json IS NULL OR + (typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768)), PRIMARY KEY (workspace_id, allocation_key), UNIQUE (workspace_id, worker_id), FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 324569a6f..338640144 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -54,6 +54,7 @@ mod ticket_browse_performance_tests; mod ticket_item_checker; pub mod workdir_create_operations; mod workdir_removal; +mod worker_create_failure_store; mod worker_projection; pub mod worker_source; pub mod workspace_catalog; @@ -124,6 +125,15 @@ pub enum Error { InvalidRuntimeIdentifier { kind: String, value: String }, #[error("worker name is reserved for a dedicated Workspace service: {0}")] ReservedWorkerName(String), + /// Typed startup result; HTTP and other callers project this without parsing Display. + #[error( + "Worker creation failed; the typed result retains its cause and operation observations" + )] + WorkerCreateFailed { + failure: Box, + #[source] + source: Option>, + }, #[error("runtime `{runtime_id}` operation failed ({code}): {message}")] RuntimeOperationFailed { runtime_id: String, diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index fe3097cf4..0dc4f12b5 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -40,6 +40,7 @@ mod subjektiv_memory_journal; mod workdir_authorization; mod worker_operations; mod worker_protocol; +mod worker_startup; pub(crate) use worker_operations::{ WorkerOperationContext, WorkspaceWorker, WorkspaceWorkerMethodSender, }; @@ -5996,6 +5997,23 @@ impl WorkspaceApi { } } + /// Read retained creation evidence for this Workspace without changing lifecycle + /// state. Trusted internal callers must enforce their current authority before + /// redisclosure; a saved record never supplies visibility or retry permission. + pub fn worker_create_failure( + &self, + worker: &RuntimeWorkerRef, + ) -> Result> { + let worker_id = WorkerId::parse(&worker.worker_id) + .ok_or_else(|| Error::InvalidInput("invalid Worker identity".into()))?; + self.config_store + .worker_create_failure(self.workspace_id(), &worker.runtime_id, worker_id) + .map(|encoded| { + encoded + .map(|value| runtime_api::startup::decode_stored_worker_create_failure(&value)) + }) + } + fn spawn_workspace_worker( &self, runtime_id: &str, @@ -6012,7 +6030,46 @@ impl WorkspaceApi { ) -> ApiResult { let assignment = request.ticket_assignment.clone(); let mut dispatched = false; - let mut result = self.spawn_workspace_worker_inner(runtime_id, request, &mut dispatched); + let mut reserved_worker = None; + let mut result = self.spawn_workspace_worker_inner( + runtime_id, + request, + &mut dispatched, + &mut reserved_worker, + ); + if let Some(worker) = reserved_worker.as_ref() { + result = result + .map(|mut result| { + if let Some(failure) = result.failure.take() { + result.failure = Some(worker_startup::persist(self, worker, failure)); + } + result + }) + .map_err(|mut error| { + error.error = match error.error { + Error::WorkerCreateFailed { failure, source } => { + Error::WorkerCreateFailed { + failure: Box::new(worker_startup::persist(self, worker, *failure)), + source, + } + } + source if dispatched => { + // Dispatch is not proof of acceptance or execution. + let failure = worker_startup::persist( + self, + worker, + worker_startup::unknown_result(), + ); + Error::WorkerCreateFailed { + failure: Box::new(failure), + source: Some(Box::new(source)), + } + } + source => source, + }; + error + }); + } // Only a newly inserted durable receipt proves there was no earlier // dispatch. A local rejection on replay says nothing about a prior // unknown outcome; keep its fence until absence is confirmed. @@ -6040,6 +6097,7 @@ impl WorkspaceApi { runtime_id: &str, mut request: WorkerSpawnRequest, dispatched: &mut bool, + reserved_identity: &mut Option, ) -> ApiResult { if let Some(singleton_key) = request.singleton_key.as_deref() { crate::store::validate_worker_singleton_key(singleton_key)?; @@ -6192,6 +6250,7 @@ impl WorkspaceApi { .worker(&reserved_worker) .unwrap_or_else(|_| worker_summary_from_registry(&record)); return Ok(WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Accepted, worker: Some(summary), acceptance_evidence: vec![crate::hosts::WorkerSpawnAcceptanceEvidence { @@ -6230,6 +6289,7 @@ impl WorkspaceApi { }; request.resolved_memory_settings = Some(reservation.memory_settings); let reservation_fingerprint = create_binding.create_fingerprint.clone(); + *reserved_identity = Some(reserved_worker.clone()); let lifecycle_assignment = request.ticket_assignment.clone(); let compensation_context = WorkerSpawnCompensationContext { assignment: lifecycle_assignment.as_ref(), @@ -6372,19 +6432,35 @@ impl WorkspaceApi { return Err(ApiError::with_diagnostics(error.into_error(), diagnostics)); } }; + if result.state == InternalWorkerOperationState::Accepted && result.worker.is_none() { + // A malformed success is not conclusive evidence about the reserved Worker. + result.state = InternalWorkerOperationState::Unknown; + } + if result.state != InternalWorkerOperationState::Accepted && result.failure.is_none() { + result.failure = Some(if result.state == InternalWorkerOperationState::Unknown { + worker_startup::unknown_result() + } else { + worker_startup::local_rejection() + }); + } + if let Some(failure) = result.failure.take() { + result.failure = Some(worker_startup::correlate(failure, &reserved_worker)); + } let Some(worker) = result.worker.as_mut() else { - result - .diagnostics - .extend(compensate_failed_workspace_worker_create( - self, - runtime_id, - worker_id, - &reservation_fingerprint, - "runtime_spawn_rejected", - None, - &compensation_context, - &reserved_attachments, - )); + let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed( + self, + runtime_id, + worker_id, + &reservation_fingerprint, + "runtime_spawn_failure", + None, + &compensation_context, + &reserved_attachments, + ); + if let Some(failure) = result.failure.take() { + result.failure = Some(worker_startup::compensated(failure, compensation)); + } + result.diagnostics.extend(diagnostics); return Ok(result); }; // Runtime metadata is display-only. The lease returned by the same DB transaction that @@ -6437,21 +6513,26 @@ impl WorkspaceApi { { Ok(replacement) => replacement, Err(error) => { - let diagnostics = compensate_failed_workspace_worker_create( - self, - runtime_id, - worker_id, - &reservation_fingerprint, - "workspace_api_transport", - Some(worker), - &compensation_context, - &reserved_attachments, - ); - return Err(ApiError::with_diagnostics(error.into_error(), diagnostics)); + let (diagnostics, compensation) = + compensate_failed_workspace_worker_create_observed( + self, + runtime_id, + worker_id, + &reservation_fingerprint, + "workspace_api_transport", + Some(worker), + &compensation_context, + &reserved_attachments, + ); + return Err(worker_startup::finalize_error( + ApiError::with_diagnostics(error.into_error(), diagnostics), + WorkerSpawnFinalizeStage::WorkspaceApiBinding, + compensation, + )); } }; if replacement.state != InternalWorkerOperationState::Accepted { - let diagnostics = compensate_failed_workspace_worker_create( + let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed( self, runtime_id, worker_id, @@ -6461,23 +6542,27 @@ impl WorkspaceApi { &compensation_context, &reserved_attachments, ); - return Err(ApiError::with_diagnostics( - Error::RuntimeOperationFailed { - runtime_id: runtime_id.to_string(), - code: "worker_workspace_api_replace_failed".to_string(), - message: replacement - .diagnostics - .first() - .map(|diagnostic| diagnostic.message.clone()) - .unwrap_or_else(|| { - "Runtime rejected Workspace API replacement after spawn".to_string() - }), - }, - diagnostics, + return Err(worker_startup::finalize_error( + ApiError::with_diagnostics( + Error::RuntimeOperationFailed { + runtime_id: runtime_id.to_string(), + code: "worker_workspace_api_replace_failed".to_string(), + message: replacement + .diagnostics + .first() + .map(|diagnostic| diagnostic.message.clone()) + .unwrap_or_else(|| { + "Runtime rejected Workspace API replacement after spawn".to_string() + }), + }, + diagnostics, + ), + WorkerSpawnFinalizeStage::WorkspaceApiBinding, + compensation, )); } if let Err(error) = parse_runtime_worker_id_for_registry(&worker.worker.worker_id) { - let diagnostics = compensate_failed_workspace_worker_create( + let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed( self, runtime_id, worker_id, @@ -6487,7 +6572,11 @@ impl WorkspaceApi { &compensation_context, &reserved_attachments, ); - return Err(api_error_with_additional_diagnostics(error, diagnostics)); + return Err(worker_startup::finalize_error( + api_error_with_additional_diagnostics(error, diagnostics), + WorkerSpawnFinalizeStage::WorkerIdentity, + compensation, + )); } let registry_result = record_worker_summary( self, @@ -6556,7 +6645,7 @@ impl WorkspaceApi { .config_store .complete_worker_create_reservation(&self.config.workspace_id, worker_id) { - let diagnostics = compensate_failed_workspace_worker_create( + let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed( self, runtime_id, worker_id, @@ -6566,9 +6655,10 @@ impl WorkspaceApi { &compensation_context, &reserved_attachments, ); - return Err(ApiError::with_diagnostics( - Error::Config(error.to_string()), - diagnostics, + return Err(worker_startup::finalize_error( + ApiError::with_diagnostics(Error::Config(error.to_string()), diagnostics), + WorkerSpawnFinalizeStage::CreateReservation, + compensation, )); } if let Err(error) = subjektiv_journal::reconcile_created_worker(self, &worker_ref) { @@ -10762,8 +10852,10 @@ impl server_api::ServerApi for ServerApiContractService { &self, context: server_api::ServerRequestContext, request: server_api::CreateWorkspaceWorkerRequest, - ) -> std::result::Result - { + ) -> std::result::Result< + server_api::BrowserCreateWorkerResponse, + server_api::WorkerCreateApiError, + > { create_workspace_worker_authenticated( self.workspace_api()?.clone(), contract_request_headers(&context)?, @@ -10773,7 +10865,7 @@ impl server_api::ServerApi for ServerApiContractService { ) .await .map(|Json(response)| response) - .map_err(ApiError::into_repository_api_error) + .map_err(ApiError::into_worker_create_api_error) } async fn workspace_orchestrator_status( @@ -10869,8 +10961,10 @@ impl server_api::ServerApi for ServerApiContractService { context: server_api::ServerRequestContext, workspace_id: String, request: server_api::CreateWorkspaceWorkerRequest, - ) -> std::result::Result - { + ) -> std::result::Result< + server_api::BrowserCreateWorkerResponse, + server_api::WorkerCreateApiError, + > { spawn_known_worker_authenticated( State(self.workspace_api()?.clone()), AxumPath(ScopedWorkspacePath { workspace_id }), @@ -10880,7 +10974,7 @@ impl server_api::ServerApi for ServerApiContractService { ) .await .map(|Json(response)| response) - .map_err(ApiError::into_repository_api_error) + .map_err(ApiError::into_worker_create_api_error) } async fn worker_control_input( @@ -11048,8 +11142,10 @@ impl server_api::ServerApi for ServerApiContractService { context: server_api::ServerRequestContext, workspace_id: String, request: server_api::CreateWorkspaceWorkerRequest, - ) -> std::result::Result - { + ) -> std::result::Result< + server_api::BrowserCreateWorkerResponse, + server_api::WorkerCreateApiError, + > { validate_workspace_scope(self.workspace_api()?, &workspace_id) .map_err(|error| ApiError::from(error).into_repository_api_error())?; create_workspace_worker_authenticated( @@ -11061,7 +11157,7 @@ impl server_api::ServerApi for ServerApiContractService { ) .await .map(|Json(response)| response) - .map_err(ApiError::into_repository_api_error) + .map_err(ApiError::into_worker_create_api_error) } async fn workspace_worker_detail( @@ -15015,6 +15111,15 @@ async fn log_failed_api_response(request: Request, next: Next) -> Response { .and_then(|source| source.token_id_hash.clone()); let response = next.run(request).await; let status = response.status(); + if let Some(error) = response + .extensions() + .get::>() + { + if let Some(failure) = error.failure() { + worker_startup::observe_http(failure, status); + return response; + } + } let is_api = uri.path().starts_with("/api/") || canonical_ticket_operation_workspace_id(uri.path()).is_some(); @@ -15024,6 +15129,12 @@ async fn log_failed_api_response(request: Request, next: Next) -> Response { .extensions() .get::>() .map(|error| ApiErrorLog::from_repository_error(error)) + .or_else(|| { + response + .extensions() + .get::>() + .map(|error| ApiErrorLog::from_repository_error(error.legacy())) + }) .or_else(|| { response .extensions() @@ -21673,6 +21784,7 @@ fn ticket_notification_delivery_error_category( InternalWorkerOperationState::Accepted => None, InternalWorkerOperationState::Rejected => Some("runtime_rejected"), InternalWorkerOperationState::Unsupported => Some("runtime_unsupported"), + InternalWorkerOperationState::Unknown => Some("runtime_outcome_unknown"), }, Err(RuntimeRegistryError::InvalidIdentifier { .. }) => Some("invalid_identifier"), Err(RuntimeRegistryError::UnknownRuntime(_)) => Some("unknown_runtime"), @@ -34894,6 +35006,7 @@ fn record_browser_worker_spawn( if result.state != InternalWorkerOperationState::Accepted { return Err(worker_create_not_accepted_error( requested_runtime_id.clone(), + result.failure, result.diagnostics, )); } @@ -34912,20 +35025,24 @@ fn browser_worker_response_from_summary( diagnostics: Vec, assignment: Option<&WorkerTicketAssignmentRequest>, ) -> ApiResult { - let worker_record = match record_worker_summary( - api, - &worker, - display_name.as_str(), - worker.profile.clone(), - WorkerRegistryDisplayNamePolicy::UseProvided, - ) { - Ok(record) => record, - Err(error) => return Err(error.into()), - }; let compensation = WorkerSpawnCompensationContext { assignment, spawned_workdir_ids: &[], }; + let worker_record = finalize_worker_spawn_stage( + api, + &worker, + &compensation, + WorkerSpawnFinalizeStage::WorkerRegistry, + record_worker_summary( + api, + &worker, + display_name.as_str(), + worker.profile.clone(), + WorkerRegistryDisplayNamePolicy::UseProvided, + ) + .map_err(ApiError::from), + )?; if let Some(assignment) = assignment { if let Err(error) = api.store.bind_ticket_assignment_operation_worker( &api.config.workspace_id, @@ -34979,12 +35096,15 @@ fn browser_worker_response_for_committed_summary( let workspace_id = api.workspace_id().to_string(); let resource_key = api .store - .worker_resource_key(&workspace_id, &worker.worker)? - .ok_or_else(|| { - Error::Store(format!( - "Workspace Worker `{worker_id}` has no resource key after registration" - )) - })?; + .worker_resource_key(&workspace_id, &worker.worker) + .and_then(|key| { + key.ok_or_else(|| { + Error::Store(format!( + "Workspace Worker `{worker_id}` has no resource key after registration" + )) + }) + }) + .map_err(|error| worker_startup::publication_error(api, &worker.worker, error.into()))?; let worker = worker_launch_worker_summary(worker); let console_href = browser_worker_console_href(&workspace_id, &resource_key); Ok(BrowserCreateWorkerResponse { @@ -35594,6 +35714,9 @@ async fn list_runtime_workers( #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum WorkerSpawnFinalizeStage { + WorkerIdentity, + WorkspaceApiBinding, + CreateReservation, WorkerRegistry, TicketAssignmentBind, TicketAssignmentCurrent, @@ -35602,19 +35725,6 @@ enum WorkerSpawnFinalizeStage { WorkdirAttachment, } -impl WorkerSpawnFinalizeStage { - fn as_str(self) -> &'static str { - match self { - Self::WorkerRegistry => "worker_registry", - Self::TicketAssignmentBind => "ticket_assignment_bind", - Self::TicketAssignmentCurrent => "ticket_assignment_current", - Self::TicketStateAccept => "ticket_state_accept", - Self::WorkdirRegistry => "workdir_registry", - Self::WorkdirAttachment => "workdir_attachment", - } - } -} - struct WorkerSpawnCompensationContext<'a> { assignment: Option<&'a crate::hosts::WorkerTicketAssignmentRequest>, spawned_workdir_ids: &'a [String], @@ -35630,52 +35740,19 @@ fn finalize_worker_spawn_stage( let Err(source) = result else { return result; }; - let source_message = sanitize_backend_error(&source.error.to_string()); - let operation_id = context - .assignment - .map(|assignment| assignment.operation_id.as_str()) - .unwrap_or("none"); - let mut diagnostics = vec![RuntimeDiagnostic { - startup: None, - code: format!("worker_spawn_finalize_{}_failed", stage.as_str()), - severity: HostDiagnosticSeverity::Error, - message: format!( - "Worker spawn finalize failed at stage `{}` for Runtime Worker {}:{} (operation `{operation_id}`): {source_message}", - stage.as_str(), - worker.worker.runtime_id, - worker.worker.worker_id - ), - }]; - diagnostics.extend(source.diagnostics); - let compensation_errors = compensate_failed_worker_spawn(api, worker, context); - let compensation_failed = !compensation_errors.is_empty(); - diagnostics.extend(compensation_errors); - if !compensation_failed { - diagnostics.push(RuntimeDiagnostic { - startup: None, - code: "worker_spawn_compensated".to_string(), - severity: HostDiagnosticSeverity::Info, - message: format!( - "Removed Runtime Worker {}:{} and rolled back Backend spawn state", - worker.worker.runtime_id, worker.worker.worker_id - ), - }); - } - let compensation = if compensation_failed { - "compensation left residual state; inspect diagnostics" - } else { - "compensation completed" - }; + let (compensation_diagnostics, compensation) = + compensate_failed_worker_spawn_observed(api, worker, context); + let failure = worker_startup::persist( + api, + &worker.worker, + worker_startup::finalize_failure(&source, stage, compensation), + ); + let mut diagnostics = source.diagnostics; + diagnostics.extend(compensation_diagnostics); Err(ApiError::with_diagnostics( - Error::RuntimeOperationFailed { - runtime_id: worker.worker.runtime_id.clone(), - code: format!("worker_spawn_finalize_{}_failed", stage.as_str()), - message: format!( - "Worker spawn finalize failed at stage `{}` for Runtime Worker {}:{} (operation `{operation_id}`): {source_message}; {compensation}", - stage.as_str(), - worker.worker.runtime_id, - worker.worker.worker_id - ), + Error::WorkerCreateFailed { + failure: Box::new(failure), + source: Some(Box::new(source.error)), }, diagnostics, )) @@ -35687,16 +35764,14 @@ fn write_workspace_worker_create_failure( phase: &str, diagnostics: &[RuntimeDiagnostic], ) { - tracing::error!( + tracing::warn!( target: "yoi::worker_create", - event = "worker_create_failed", + event = "worker_create_compensation", component = "workspace_server", - runtime_id, - worker_id = %worker_id, + correlation = %worker_startup::correlation(runtime_id, &worker_id.to_string()), phase, - cleanup_succeeded = diagnostics.is_empty(), - cleanup_diagnostics = ?diagnostics, - "Worker creation failed" + cleanup_completed = diagnostics.is_empty(), + "Worker creation compensation observed" ); } @@ -35771,6 +35846,32 @@ fn compensate_failed_workspace_worker_create( context: &WorkerSpawnCompensationContext<'_>, attachment_reservations: &[(String, String, String, workdir::WorkdirSessionCapabilities)], ) -> Vec { + compensate_failed_workspace_worker_create_observed( + api, + runtime_id, + reservation_worker_id, + create_fingerprint, + failure_phase, + worker, + context, + attachment_reservations, + ) + .0 +} + +fn compensate_failed_workspace_worker_create_observed( + api: &WorkspaceApi, + runtime_id: &str, + reservation_worker_id: WorkerId, + create_fingerprint: &str, + failure_phase: &str, + worker: Option<&InternalWorkerSummary>, + context: &WorkerSpawnCompensationContext<'_>, + attachment_reservations: &[(String, String, String, workdir::WorkdirSessionCapabilities)], +) -> ( + Vec, + runtime_api::startup::CompensationOutcome, +) { let mut diagnostics = Vec::new(); let reserved_worker_ref = RuntimeWorkerRef::new(runtime_id.to_string(), reservation_worker_id.to_string()); @@ -35840,18 +35941,31 @@ fn compensate_failed_workspace_worker_create( failure_phase, &diagnostics, ); - diagnostics + let outcome = if !reserved_worker_absent { + runtime_api::startup::CompensationOutcome::Unknown + } else if diagnostics.is_empty() { + runtime_api::startup::CompensationOutcome::Completed + } else { + runtime_api::startup::CompensationOutcome::Residual + }; + (diagnostics, outcome) } -fn compensate_failed_worker_spawn( +fn compensate_failed_worker_spawn_observed( api: &WorkspaceApi, worker: &InternalWorkerSummary, context: &WorkerSpawnCompensationContext<'_>, -) -> Vec { +) -> ( + Vec, + runtime_api::startup::CompensationOutcome, +) { let (runtime_deleted, mut diagnostics) = WorkerRemovalService::new(api).delete_spawn_compensation_runtime_worker(&worker.worker); if !runtime_deleted { - return diagnostics; + return ( + diagnostics, + runtime_api::startup::CompensationOutcome::Unknown, + ); } diagnostics.extend(finalize_spawn_compensation_after_worker_delete( api, @@ -35872,7 +35986,12 @@ fn compensate_failed_worker_spawn( ), )); } - diagnostics + let outcome = if diagnostics.is_empty() { + runtime_api::startup::CompensationOutcome::Completed + } else { + runtime_api::startup::CompensationOutcome::Residual + }; + (diagnostics, outcome) } fn finalize_spawn_compensation_after_worker_delete( @@ -36148,6 +36267,7 @@ async fn create_runtime_worker_authenticated_on_host( { accept_queued_ticket_after_worker_spawn(&api, assignment)?; return Ok(Json(worker_spawn_result_to_api(WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Accepted, worker: Some(worker), acceptance_evidence: Vec::new(), @@ -39159,19 +39279,15 @@ fn encode_path_segment(value: &str) -> String { } fn worker_create_not_accepted_error( - runtime_id: String, - mut diagnostics: Vec, + _runtime_id: String, + failure: Option, + diagnostics: Vec, ) -> ApiError { - diagnostics.push(settings_diagnostic( - "workspace_worker_create_not_accepted", - HostDiagnosticSeverity::Error, - "Runtime did not accept worker creation; see diagnostics for sanitized Runtime compatibility details.", - )); + let failure = failure.unwrap_or_else(worker_startup::local_rejection); ApiError::with_diagnostics( - Error::RuntimeOperationFailed { - runtime_id, - code: "workspace_worker_create_failed".to_string(), - message: "Runtime did not accept worker creation".to_string(), + Error::WorkerCreateFailed { + failure: Box::new(failure), + source: None, }, diagnostics, ) @@ -39666,6 +39782,18 @@ fn subjektiv_diagnostic_code(message: &str) -> Option<&'static str> { } impl ApiError { + fn into_worker_create_api_error(self) -> server_api::WorkerCreateApiError { + match self.error { + Error::WorkerCreateFailed { failure, .. } => server_api::WorkerCreateApiError::startup( + *failure, + &protocol::session_failure::SessionFailureVisibility { + can_view_cause: true, + }, + ), + _ => self.into_repository_api_error().into(), + } + } + fn with_diagnostics(error: Error, diagnostics: Vec) -> Self { Self { error, @@ -39852,6 +39980,13 @@ fn api_error_response_message(error: &Error) -> String { impl IntoResponse for ApiError { fn into_response(self) -> Response { + if matches!(&self.error, Error::WorkerCreateFailed { .. }) { + let error = self.into_worker_create_api_error(); + return api_macros::axum::http_error_response( + api_macros::axum::status(api_macros::HttpError::status_code(&error)), + error, + ); + } let status = api_error_status(&self.error); let response_message = api_error_response_message(&self.error); let log = ApiErrorLog { @@ -39953,6 +40088,7 @@ mod tests { mod protocol_backpressure_tests; mod restore_workdir_tests; mod restored_parent_control_tests; + mod startup_result_tests; mod subject_authorization_tests; mod subject_deletion_tests; mod notification_source_authorization_tests { @@ -43038,6 +43174,7 @@ mod tests { .unwrap_or_else(std::sync::PoisonError::into_inner), ) { return WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -43069,6 +43206,7 @@ mod tests { .unwrap_or_else(std::sync::PoisonError::into_inner) .push(worker.clone()); WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Accepted, worker: Some(worker), acceptance_evidence: Vec::new(), @@ -47164,7 +47302,18 @@ mod tests { guarded_spawn_payload(&fixture, "guarded-runtime-rejection"), ) .await; - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json"); + let bytes = to_bytes( + response.into_body(), + runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES, + ) + .await + .unwrap(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["yoi_failure"]["domain"], "worker_startup"); + assert_eq!(body["yoi_failure"]["reason"], "failed"); + assert_eq!(body["yoi_startup"]["operation"]["acceptance"], "rejected"); assert_eq!(fixture.runtime.spawn_requests().len(), 1); assert_guarded_spawn_rolled_back(&fixture); } @@ -47191,7 +47340,17 @@ mod tests { guarded_spawn_payload(&fixture, "guarded-post-create-failure"), ) .await; - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json"); + let bytes = to_bytes( + response.into_body(), + runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES, + ) + .await + .unwrap(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["yoi_startup"]["operation"]["acceptance"], "accepted"); + assert_eq!(body["yoi_failure"]["reason"], "finalize_failed"); assert_eq!(fixture.runtime.spawn_requests().len(), 1); assert_guarded_spawn_rolled_back(&fixture); } @@ -61792,7 +61951,7 @@ mod tests { sync_worker_observation(&api, &summary).unwrap(); let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); let _ = subscriber.take_snapshot(); - let diagnostics = compensate_failed_worker_spawn( + let (diagnostics, _) = compensate_failed_worker_spawn_observed( &api, &summary, &WorkerSpawnCompensationContext { @@ -68042,24 +68201,32 @@ mod tests { ); let response = worker_create_not_accepted_error( - "unsupported-runtime".to_string(), + "unsupported-runtime".into(), + None, vec![settings_diagnostic( "remote_runtime_unsupported", HostDiagnosticSeverity::Warning, - "Remote Runtime provisioning is unsupported by this v0 worker launch path.", + "TEST_ONLY_DIAGNOSTIC_TEXT", )], ) .into_response(); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json"); + let bytes = to_bytes( + response.into_body(), + runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES, + ) + .await + .unwrap(); let response: Value = serde_json::from_slice(&bytes).unwrap(); - let diagnostics = response["diagnostics"].as_array().unwrap(); - assert!(diagnostics.len() >= 2); + assert_eq!(response["yoi_failure"]["domain"], "worker_startup"); + assert!(response.get("diagnostics").is_none()); assert!( - diagnostics - .iter() - .any(|diagnostic| { diagnostic["code"] == "workspace_worker_create_not_accepted" }) + !response + .to_string() + .contains("workspace_worker_create_not_accepted") ); + assert!(!response.to_string().contains("TEST_ONLY_DIAGNOSTIC_TEXT")); let projected = serde_json::to_string(&response).unwrap(); assert!(!projected.contains("http://")); } diff --git a/crates/workspace-server/src/server/tests/session_registration_connected_tests.rs b/crates/workspace-server/src/server/tests/session_registration_connected_tests.rs index 8faf4f226..165c660b0 100644 --- a/crates/workspace-server/src/server/tests/session_registration_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/session_registration_connected_tests.rs @@ -1,14 +1,18 @@ -//! Focused T840 connection: real Subject policy -> signed Session POST -> real -//! ProfileRuntimeWorkerFactory startup -> RuntimeHTTP/Host diagnostic -> saved -//! Worker metadata. Only the unused external model is replaced. No restore, -//! operation replay, singleton-release, or Browser rendering contract is asserted. +//! Connected startup acceptance: real Subject policy -> signed Session POST -> +//! ProfileRuntimeWorkerFactory -> Runtime HTTP -> Backend, including the real +//! authenticated Browser worker-create HTTP route and saved Worker metadata. +//! Only the unused external model is replaced. Read-only reconciliation is not +//! restore, operation replay, or a Browser rendering contract. use super::*; use api_macros::HttpError as _; use protocol::session_failure::{ SessionPrincipal, SessionRegistrationCause, session_registration_attachment, }; use session_store::WorkerMetadataStore as _; +use std::collections::BTreeMap; use std::sync::atomic::{AtomicUsize, Ordering}; +use tracing::instrument::WithSubscriber as _; +use tracing_subscriber::{Layer as _, layer::SubscriberExt as _}; use worker_runtime::worker_backend::RuntimeWorkerFactory as _; #[derive(Clone, Default)] @@ -30,6 +34,57 @@ impl agen::llm_client::LlmClient for UnusedModel { } } +#[derive(Clone, Default)] +struct StartupEvents(Arc>>>); + +#[derive(Default)] +struct StartupEventFields(BTreeMap); + +impl tracing::field::Visit for StartupEventFields { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.0.insert(field.name().into(), value.into()); + } + + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0.insert(field.name().into(), format!("{value:?}")); + } +} + +impl tracing_subscriber::Layer for StartupEvents { + fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + if event.metadata().target() == "yoi::worker_startup" { + let mut fields = StartupEventFields::default(); + event.record(&mut fields); + self.0.lock().unwrap().push(fields.0); + } + } +} + +/// A real formatter writer failure, not a replacement result/observer function. +/// The request-scoped subscriber avoids process-global test interference. +#[derive(Clone, Default)] +struct FailedStartupSink(Arc); + +impl std::io::Write for FailedStartupSink { + fn write(&mut self, _: &[u8]) -> std::io::Result { + self.0.fetch_add(1, Ordering::SeqCst); + Err(std::io::Error::other( + "test: startup observation sink unavailable", + )) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for FailedStartupSink { + type Writer = Self; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } +} + type ReleaseFence = ( crate::store::WorkerSingletonLease, subjektiv::journal::JournalWriterBinding, @@ -41,6 +96,8 @@ struct RegistrationFixture { callbacks: Arc>>, model: UnusedModel, release_fence: Arc>>, + startup_events: StartupEvents, + failed_sink: FailedStartupSink, } #[derive(Clone, Copy)] @@ -49,6 +106,7 @@ enum RegistrationFault { Deny, DenyWithMetadataReadFailure, ReleasePending, + CommittedResponseLost, } impl RegistrationFixture { @@ -63,13 +121,31 @@ impl RegistrationFixture { let model = UnusedModel::default(); let release_fence = Arc::new(Mutex::new(None)); let reserved = release_fence.clone(); + let startup_events = StartupEvents::default(); + let failed_sink = FailedStartupSink::default(); + let events = startup_events.clone(); + let sink = failed_sink.clone(); let connected = ConnectedWorkerHarness::start_with_layer(model.clone(), move |router| { router.layer(axum::middleware::from_fn( move |request: axum::extract::Request, next: axum::middleware::Next| { let hook = hook.clone(); let observed = observed.clone(); let reserved = reserved.clone(); + let events = events.clone(); + let sink = sink.clone(); async move { + if request.method() == axum::http::Method::POST + && request.uri().path() == format!("/api/w/{TEST_WORKSPACE_ID}/workers") + { + let subscriber = tracing_subscriber::registry() + .with(events) + .with(tracing_subscriber::fmt::layer() + .with_writer(sink) + .with_filter(tracing_subscriber::filter::filter_fn(|metadata| { + metadata.target() == "yoi::worker_startup" + }))); + return next.run(request).with_subscriber(subscriber).await; + } if request.method() != axum::http::Method::POST || request.uri().path() != format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions") @@ -108,7 +184,7 @@ impl RegistrationFixture { assert!(store.journal_writer_active(&writer).unwrap()); assert_eq!(api.store.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key).unwrap(), Some(lease.clone())); *reserved.lock().unwrap() = Some((lease, writer)); - } else { + } else if !matches!(fault, RegistrationFault::CommittedResponseLost) { store .set_subject_permissions( &subject, @@ -163,6 +239,20 @@ impl RegistrationFixture { std::fs::create_dir(&metadata).unwrap(); } observed.lock().unwrap().push((response.status(), input)); + if matches!(fault, RegistrationFault::CommittedResponseLost) { + assert_eq!(response.status(), StatusCode::OK); + // The genuine signed handler committed before its response is + // lost. Do not replace its authorization, attribution, or write. + // A broken body fails the actual Runtime HTTP client rather + // than fabricating a typed Runtime/domain failure in this layer. + let (parts, _) = response.into_parts(); + let body = Body::from_stream(futures::stream::once(async { + Err::, _>(std::io::Error::other( + "test: committed registration response lost", + )) + })); + return Response::from_parts(parts, body); + } response } }, @@ -200,6 +290,8 @@ impl RegistrationFixture { callbacks, model, release_fence, + startup_events, + failed_sink, } } @@ -222,6 +314,178 @@ impl RegistrationFixture { } } + async fn browser_failure( + &self, + expected_status: StatusCode, + ) -> ( + runtime_api::startup::StartupProblemDetails, + runtime_api::startup::WorkerCreateFailure, + ) { + use runtime_api::startup::{ + DecodeStartupProblem, PROBLEM_CONTENT_TYPE, StartupBoundary, decode_startup_problem, + }; + let (status, headers, bytes) = self + .connected + .human_raw( + reqwest::Method::POST, + &format!("/api/w/{TEST_WORKSPACE_ID}/workers"), + serde_json::to_value(self.request()).unwrap(), + ) + .await; + assert_eq!(status, expected_status); + let content_type = headers[axum::http::header::CONTENT_TYPE].to_str().unwrap(); + assert_eq!(content_type, PROBLEM_CONTENT_TYPE); + let DecodeStartupProblem::Known(failure) = decode_startup_problem( + &bytes, + status.as_u16(), + content_type, + StartupBoundary::Browser, + ) else { + panic!("real Browser HTTP response must decode as a bounded startup Problem"); + }; + let problem: runtime_api::startup::StartupProblemDetails = + serde_json::from_slice(&bytes).unwrap(); + assert_eq!(problem.status, status.as_u16()); + // Neither raw callback diagnostics nor authority-bearing identifiers may + // be copied into a public explanation/extension by the adapters. + let body = std::str::from_utf8(&bytes).unwrap(); + for secret in [ + &self.connected.owner, + &self.subject, + &self.connected.backend_url, + ] { + assert!( + !body.contains(secret), + "Browser Problem disclosed private fixture data" + ); + } + assert!(!body.contains("test: committed registration response lost")); + assert!(!body.contains("worker-runtime-")); + // Public, bounded handler bytes, after the non-disclosure checks above. + // These provide the actual shared Browser fixture source under --nocapture; + // no metadata, callback body, credential, or private upstream text is logged. + eprintln!( + "T812_BROWSER_STARTUP_PROBLEM status={status} content_type={content_type} body={body}" + ); + (problem, failure) + } + + fn assert_backend_failure( + &self, + worker: &str, + session: &str, + problem: &runtime_api::startup::StartupProblemDetails, + browser: &runtime_api::startup::WorkerCreateFailure, + expected: SessionRegistrationCause, + ) { + use protocol::startup_failure::StartupPersistenceOutcome::Persisted; + use runtime_api::startup::DecodeStoredWorkerCreateFailure; + let reference = RuntimeWorkerRef::new(CONNECTED_RUNTIME, worker); + let Some(DecodeStoredWorkerCreateFailure::Known(saved)) = self + .connected + .api + .worker_create_failure(&reference) + .unwrap() + else { + panic!("real Browser failure must be readable from the canonical Backend reservation"); + }; + assert_startup_semantics(saved.startup(), browser.startup(), expected); + assert_eq!( + saved.operation(), + browser.operation(), + "HTTP cannot rewrite acceptance/execution/finalization/compensation/correlation" + ); + assert_eq!(saved.backend_result_persistence(), Some(Persisted)); + assert_eq!(browser.backend_result_persistence(), Some(Persisted)); + assert_eq!( + problem.yoi_startup.backend_result_persistence, + Some(Persisted) + ); + assert_eq!(saved.public_facts(), problem.yoi_startup); + let correlation = saved + .operation() + .correlation + .as_ref() + .expect("Backend-selected lookup token"); + assert_eq!( + correlation.as_str(), + worker_startup::correlation(CONNECTED_RUNTIME, worker) + ); + assert_eq!( + problem.instance, + format!("urn:yoi:worker-startup:{}", correlation.as_str()) + ); + + let events = self.startup_events.0.lock().unwrap(); + let result_events = events + .iter() + .filter(|event| { + event + .get("event") + .is_some_and(|value| value == "worker_create_result") + }) + .collect::>(); + assert_eq!( + result_events.len(), + 1, + "one HTTP responsibility boundary records the primary, not every adapter" + ); + let event = result_events[0]; + assert_eq!(event["domain"], problem.yoi_failure.domain); + assert_eq!(event["reason"], problem.yoi_failure.reason); + assert_eq!(event["correlation"], correlation.as_str()); + assert_eq!(event["status"], problem.status.to_string()); + assert_eq!( + event["acceptance"], + format!("{:?}", saved.operation().acceptance) + ); + assert_eq!( + event["execution"], + format!("{:?}", saved.operation().execution) + ); + let logged = serde_json::to_string(&*events).unwrap(); + for secret in [ + self.connected.owner.as_str(), + self.subject.as_str(), + self.connected.backend_url.as_str(), + session, + worker, + "test: committed registration response lost", + ] { + assert!( + !logged.contains(secret), + "startup observation leaked private fixture data" + ); + } + assert!(!logged.contains(self.connected.runtime_root.to_str().unwrap())); + drop(events); + assert!( + self.failed_sink.0.load(Ordering::SeqCst) > 0, + "actual tracing formatter sink failed while HTTP/Backend/metadata results were preserved" + ); + assert_eq!( + self.connected + .api + .worker_create_failure(&reference) + .unwrap(), + Some(DecodeStoredWorkerCreateFailure::Known(saved)), + "lookup is read-only, never a create retry" + ); + } + + fn attempted_worker(&self) -> String { + let workers = std::fs::read_dir(self.connected.runtime_root.join("workers")) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect::>(); + assert_eq!( + workers.len(), + 1, + "one real factory attempt, never a new create/replay" + ); + workers[0].clone() + } + fn assert_callback(&self, expected: StatusCode) -> String { let callbacks = self.callbacks.lock().unwrap(); assert!(!callbacks.is_empty(), "real signed startup callback"); @@ -250,6 +514,42 @@ fn contains_saved_value(saved: &Value, expected: &Value) -> bool { } } +// Runtime/Browser normalization reprojects for the receiving boundary, including +// its local explanation. Compare domain enums, not byte identity of envelopes; +// saved metadata below still checks the original Storage attachment independently. +fn session_primary( + primary: &protocol::startup_failure::StartupFailureAttachment, +) -> Option { + match primary.decode::() { + error_contract::DecodeFailure::Known(received) => Some(*received.facts().cause()), + _ => None, + } +} + +fn assert_session_primary( + primary: &protocol::startup_failure::StartupFailureAttachment, + expected: SessionRegistrationCause, +) { + assert_eq!( + session_primary(primary), + Some(expected), + "Session failure must remain a known typed domain cause" + ); +} + +fn assert_startup_semantics( + host: &runtime_api::WorkerStartupFailure, + wire: &runtime_api::WorkerStartupFailure, + expected: SessionRegistrationCause, +) { + assert_session_primary(&host.primary, expected); + assert_session_primary(&wire.primary, expected); + assert_eq!(host.secondary, wire.secondary); + assert_eq!(host.persistence, wire.persistence); + assert_eq!(host.resources, wire.resources); + assert_eq!(host.io_kind, wire.io_kind); +} + fn read_metadata(root: &std::path::Path, worker: &str) -> Value { // Reopen the canonical metadata store, not the separate Runtime catalog. let name = format!("worker-runtime-{worker}"); @@ -346,6 +646,168 @@ fn assert_registration_cause(body: Value, cause: SessionRegistrationCause) { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn browser_http_session_record_denial_retains_subject_cause_and_saved_resources() { + use protocol::startup_failure::{ + StartupFailurePhase, StartupPersistenceOutcome, StartupSecondaryFailure, + }; + use runtime_api::startup::{ExecutionOutcome, StartupCause}; + + let fixture = RegistrationFixture::new(RegistrationFault::Deny).await; + // This POST must pass the real Browser authentication/preflight and reach + // the signed callback. Only then does the fixture remove session.record. + let (problem, browser) = fixture.browser_failure(StatusCode::CONFLICT).await; + let session = fixture.assert_callback(StatusCode::FORBIDDEN); + assert_eq!(fixture.callbacks.lock().unwrap().len(), 1); + let expected = SessionRegistrationCause::denied(SessionPrincipal::Subject); + assert_eq!(problem.type_uri, "urn:yoi:problem:conflict"); + assert_eq!(problem.yoi_failure.domain, "session_registration"); + assert_eq!(problem.yoi_failure.reason, "permission_denied"); + assert_eq!(problem.yoi_failure.details, StartupCause::Session(expected)); + assert!(problem.detail.contains("Subject"), "{}", problem.detail); + assert!( + problem.detail.contains("session.record"), + "{}", + problem.detail + ); + assert_eq!(browser.operation().execution, ExecutionOutcome::Errored); + assert_session_primary(&browser.startup().primary, expected); + + let runtime = fixture.connected.runtime_startup_failures.lock().unwrap(); + assert_eq!(runtime.len(), 1, "one actual Runtime HTTP failure"); + assert_session_primary(&runtime[0].primary, expected); + assert_eq!( + browser.startup().persistence, + Some(StartupPersistenceOutcome::Persisted) + ); + assert_eq!(browser.startup().persistence, runtime[0].persistence); + assert!( + runtime[0].resources.is_some(), + "resource disposition is independent of the denial" + ); + assert_eq!(browser.startup().resources, runtime[0].resources); + assert!( + browser.startup().secondary.iter().any(|secondary| matches!( + secondary, + StartupSecondaryFailure::Persistence { + phase: StartupFailurePhase::RuntimeRecordDelete, + .. + } + )), + "real retention pin prevents Runtime record deletion without replacing the denial" + ); + drop(runtime); + + let store = open_subjektiv_store(&fixture.connected.api).unwrap(); + assert!(store.session_attribution(&session).unwrap().is_none()); + assert!( + store + .subject_session_attributions(&fixture.subject) + .unwrap() + .is_empty() + ); + let worker = fixture.attempted_worker(); + fixture.assert_backend_failure(&worker, &session, &problem, &browser, expected); + let saved = read_metadata(&fixture.connected.runtime_root, &worker); + assert!(contains_saved_value( + &saved, + &serde_json::to_value(session_registration_attachment(expected).unwrap()).unwrap() + )); + assert_eq!( + fixture.connected.runtime.list_workers().unwrap().len(), + 1, + "retained Runtime resource is observed separately from persisted cause and HTTP409" + ); + assert_eq!( + read_metadata(&fixture.connected.runtime_root, &worker), + saved, + "ordinary read-only reopen, not create/restore/retry" + ); + assert_eq!(fixture.callbacks.lock().unwrap().len(), 1); + assert_eq!(fixture.model.0.load(Ordering::SeqCst), 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn browser_http_committed_registration_response_loss_stays_unknown_without_reexecution() { + use protocol::startup_failure::StartupPersistenceOutcome; + use runtime_api::startup::StartupCause; + + let fixture = RegistrationFixture::new(RegistrationFault::CommittedResponseLost).await; + let (problem, browser) = fixture.browser_failure(StatusCode::BAD_GATEWAY).await; + // This is the status of the original signed handler, before transport loss. + let session = fixture.assert_callback(StatusCode::OK); + assert_eq!( + fixture.callbacks.lock().unwrap().len(), + 1, + "response loss must not automatically reexecute Session registration" + ); + let expected = SessionRegistrationCause::outcome_unknown(); + assert_eq!(problem.yoi_failure.domain, "session_registration"); + assert_eq!(problem.yoi_failure.reason, "outcome_unknown"); + assert_eq!(problem.yoi_failure.details, StartupCause::Session(expected)); + assert!( + problem.detail.contains("session.record"), + "{}", + problem.detail + ); + assert_session_primary(&browser.startup().primary, expected); + assert_eq!( + browser.startup().persistence, + Some(StartupPersistenceOutcome::Persisted) + ); + let runtime = fixture.connected.runtime_startup_failures.lock().unwrap(); + assert_eq!(runtime.len(), 1); + assert_session_primary(&runtime[0].primary, expected); + assert_eq!(browser.startup().resources, runtime[0].resources); + assert_eq!(browser.startup().persistence, runtime[0].persistence); + drop(runtime); + + let store = open_subjektiv_store(&fixture.connected.api).unwrap(); + let committed = store.session_attribution(&session).unwrap().unwrap(); + assert_eq!(committed.subject_id, fixture.subject); + assert_eq!(committed.runtime_id, CONNECTED_RUNTIME); + let worker = fixture.attempted_worker(); + fixture.assert_backend_failure(&worker, &session, &problem, &browser, expected); + assert_eq!(committed.worker_id, worker); + assert_eq!( + store + .subject_session_attributions(&fixture.subject) + .unwrap() + .len(), + 1, + "HTTP failure is not proof of no Backend write" + ); + let saved = read_metadata(&fixture.connected.runtime_root, &worker); + assert_eq!( + saved["subjektiv_session_attribution"]["state"], "outcome_unknown", + "saved Runtime observation must not claim confirmed registration or rejection" + ); + assert_eq!( + saved["subjektiv_session_attribution"]["session_id"], + session + ); + assert!(contains_saved_value( + &saved, + &serde_json::to_value(session_registration_attachment(expected).unwrap()).unwrap() + )); + assert_eq!( + read_metadata(&fixture.connected.runtime_root, &worker), + saved + ); + assert_eq!( + store.session_attribution(&session).unwrap(), + Some(committed) + ); + assert_eq!(fixture.connected.runtime.list_workers().unwrap().len(), 1); + assert_eq!(fixture.attempted_worker(), worker); + assert_eq!( + fixture.callbacks.lock().unwrap().len(), + 1, + "Problem decoding, attribution lookup and metadata reopen are read-only, never replay authority" + ); + assert_eq!(fixture.model.0.load(Ordering::SeqCst), 0); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 6)] async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen() { let fixture = RegistrationFixture::new(RegistrationFault::Deny).await; @@ -390,7 +852,10 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen() SessionPrincipal::Subject, )) .unwrap(); - assert_eq!(wire.primary, expected_primary); + assert_session_primary( + &wire.primary, + SessionRegistrationCause::denied(SessionPrincipal::Subject), + ); assert_eq!( wire.persistence, Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted) @@ -399,9 +864,16 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen() .diagnostics .iter() .filter_map(|d| d.startup.as_ref()) - .find(|startup| startup.primary == wire.primary) + .find(|startup| { + session_primary(&startup.primary) + == Some(SessionRegistrationCause::denied(SessionPrincipal::Subject)) + }) .expect("RemoteWorkerRuntime must preserve the typed startup primary in Host diagnostics"); - assert_eq!(host.primary, wire.primary); + assert_startup_semantics( + host, + wire, + SessionRegistrationCause::denied(SessionPrincipal::Subject), + ); assert!( host.secondary.iter().any(|failure| matches!( failure, @@ -412,7 +884,7 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen() )), "a real Session retention pin rejects outer Runtime deletion without hiding the primary" ); - let expected = serde_json::to_value(&wire.primary).unwrap(); + let expected = serde_json::to_value(&expected_primary).unwrap(); assert!( expected.is_string(), "the shared attachment contains a codec envelope" @@ -467,12 +939,9 @@ async fn real_session_record_denial_and_metadata_save_failure_remain_separate() "one real startup attempt, not an operation replay" ); let wire = &failures[0]; - assert_eq!( - wire.primary, - session_registration_attachment(SessionRegistrationCause::denied( - SessionPrincipal::Subject - ),) - .unwrap() + assert_session_primary( + &wire.primary, + SessionRegistrationCause::denied(SessionPrincipal::Subject), ); assert_eq!(wire.persistence, Some(StartupPersistenceOutcome::NotSaved)); assert!(wire.secondary.iter().any(|failure| matches!( @@ -486,11 +955,15 @@ async fn real_session_record_denial_and_metadata_save_failure_remain_separate() .diagnostics .iter() .filter_map(|d| d.startup.as_ref()) - .find(|startup| startup.primary == wire.primary) + .find(|startup| { + session_primary(&startup.primary) + == Some(SessionRegistrationCause::denied(SessionPrincipal::Subject)) + }) .unwrap(); - assert_eq!( - host, wire, - "Remote Host preserves the refusal and independent save failure" + assert_startup_semantics( + host, + wire, + SessionRegistrationCause::denied(SessionPrincipal::Subject), ); assert!( open_subjektiv_store(&fixture.connected.api) @@ -623,7 +1096,7 @@ async fn release_fence_refusal_survives_signed_registration_and_metadata_reopen( let wire = &failures[0]; let expected = session_registration_attachment(SessionRegistrationCause::binding_mismatch()).unwrap(); - assert_eq!(wire.primary, expected); + assert_session_primary(&wire.primary, SessionRegistrationCause::binding_mismatch()); assert_eq!( wire.persistence, Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted) @@ -632,9 +1105,11 @@ async fn release_fence_refusal_survives_signed_registration_and_metadata_reopen( .diagnostics .iter() .filter_map(|d| d.startup.as_ref()) - .find(|failure| failure.primary == expected) + .find(|failure| { + session_primary(&failure.primary) == Some(SessionRegistrationCause::binding_mismatch()) + }) .expect("Remote Host retains known refusal"); - assert_eq!(host.primary, wire.primary); + assert_startup_semantics(host, wire, SessionRegistrationCause::binding_mismatch()); assert!( host.secondary.iter().any(|failure| matches!( failure, diff --git a/crates/workspace-server/src/server/tests/startup_result_tests.rs b/crates/workspace-server/src/server/tests/startup_result_tests.rs new file mode 100644 index 000000000..731b806fb --- /dev/null +++ b/crates/workspace-server/src/server/tests/startup_result_tests.rs @@ -0,0 +1,803 @@ +//! Backend result fixtures use real SQLite reservations, coordinator helpers, typed +//! reads, public ApiError responses and HTTP middleware observation. Publication +//! faults enter the production committed-summary response helper with a fixture +//! Runtime Worker; only the earlier Runtime startup is substituted. These cases do +//! not claim creation-route authorization or Session callback coverage (see +//! session_registration_connected_tests). Runs in the ordinary server lib suite; +//! each case owns its tempdir/DB and subscriber, with no external services. + +use super::*; +use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause}; +use protocol::startup_failure::{StartupFailureAttachment, StartupPersistenceOutcome}; +use runtime_api::startup::{ + CompensationOutcome, CreationAcceptance, DecodeStartupProblem, DecodeStoredWorkerCreateFailure, + ExecutionOutcome, FinalizeFacts, FinalizeStage, StartupBoundary, StartupCause, + StartupOperationFacts, WorkerCreateFailure, decode_startup_problem, +}; +use std::io::Write; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tracing::instrument::WithSubscriber; + +const SECRETS: &[&str] = &[ + "attachment-secret", + "context-secret", + "diagnostic-secret", + "source-secret", + "sqlite-secret", + "/private/startup-provider", +]; + +fn failure_fixture(compensation: CompensationOutcome) -> WorkerCreateFailure { + let clean = WorkerCreateFailure::from_cause( + StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)), + StartupOperationFacts { + acceptance: CreationAcceptance::Accepted, + execution: ExecutionOutcome::Errored, + finalize: Some(FinalizeFacts { + stage: FinalizeStage::SessionRegistration, + compensation, + }), + backend_compensation: Some(compensation), + correlation: None, + }, + ) + .unwrap(); + let (mut startup, operation) = clean.into_parts(); + // Real untrusted attachment bytes, not a source string which the codec never + // sees. Projection must reconstruct the cause and discard received prose/context. + let encoded: String = + serde_json::from_value(serde_json::to_value(&startup.primary).unwrap()).unwrap(); + let mut envelope: Value = serde_json::from_str(&encoded).unwrap(); + envelope["explanation"] = json!("attachment-secret /private/startup-provider"); + envelope["context"]["operations"] = json!([{ + "operation": "context-secret", "stage": "context-secret", + "operation_ref": "context-secret" + }]); + envelope["context"]["diagnostics"] = json!([{ + "code": "diagnostic-secret", "explanation": "diagnostic-secret" + }]); + startup.primary = serde_json::from_value::(json!( + serde_json::to_string(&envelope).unwrap() + )) + .unwrap(); + // Session metadata and Backend result persistence have independent owners. + startup.persistence = Some(StartupPersistenceOutcome::NotSaved); + let failure = WorkerCreateFailure::new(startup, operation).unwrap(); + assert_eq!( + primary_cause(&failure), + StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)) + ); + failure +} + +fn primary_cause(failure: &WorkerCreateFailure) -> StartupCause { + match failure.startup().primary.decode::() { + error_contract::DecodeFailure::Known(received) => received.facts().cause().clone(), + _ => panic!("fixture/result must retain a known typed primary"), + } +} + +fn assert_semantics(actual: &WorkerCreateFailure, expected: &WorkerCreateFailure) { + assert_eq!(primary_cause(actual), primary_cause(expected)); + assert_eq!(actual.public_facts(), expected.public_facts()); +} + +fn reserve(api: &WorkspaceApi) -> RuntimeWorkerRef { + reserve_for_runtime(api, EMBEDDED_WORKER_RUNTIME_ID) +} + +fn reserve_for_runtime(api: &WorkspaceApi, runtime_id: &str) -> RuntimeWorkerRef { + let settings = api + .config_store + .get_workspace_memory_settings(TEST_WORKSPACE_ID) + .unwrap(); + let reservation = api + .config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + runtime_id, + "manual:startup-result-fixture", + "sha256:startup-result-fixture", + None, + &settings, + ) + .unwrap(); + RuntimeWorkerRef { + runtime_id: runtime_id.into(), + worker_id: reservation.worker_id.to_string(), + } +} + +fn retained(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> WorkerCreateFailure { + match api.worker_create_failure(worker).unwrap() { + Some(DecodeStoredWorkerCreateFailure::Known(failure)) => failure, + other => panic!("typed public read must return retained result: {other:?}"), + } +} + +// Snapshot all reservation columns except the observation, plus allocation/lease +// rows. Saving a diagnostic must not complete/remove/reallocate the reservation. +fn lifecycle_snapshot(api: &WorkspaceApi) -> Vec> { + api.config_store.with_conn(|conn| { + let mut columns = conn.prepare("PRAGMA table_info(worker_create_reservations)")?; + let names = columns.query_map([], |row| row.get::<_, String>(1))? + .collect::>>()?; + let columns = names.into_iter().filter(|name| name != "startup_failure_json") + .map(|name| format!("\"{name}\"" )).collect::>().join(","); + let queries = [ + format!("SELECT {columns} FROM worker_create_reservations ORDER BY workspace_id,allocation_key"), + "SELECT * FROM workspace_resource_keys ORDER BY workspace_id,resource_kind,resource_id".into(), + "SELECT * FROM worker_singleton_owners ORDER BY workspace_id,singleton_key".into(), + "SELECT * FROM worker_registry ORDER BY workspace_id,runtime_id,worker_id".into(), + "SELECT * FROM ticket_worker_assignments ORDER BY workspace_id,ticket_id,assignment_id".into(), + "SELECT * FROM ticket_assignment_operations ORDER BY workspace_id,operation_id".into(), + ]; + let mut snapshot = Vec::new(); + for query in queries { + let mut statement = conn.prepare(&query)?; + let count = statement.column_count(); + snapshot.extend(statement.query_map([], |row| { + (0..count).map(|i| row.get(i)).collect::>>() + })?.collect::>>()?); + } + Ok(snapshot) + }).unwrap() +} + +fn api_error(failure: WorkerCreateFailure) -> ApiError { + ApiError::with_diagnostics( + Error::WorkerCreateFailed { + failure: Box::new(failure), + source: Some(Box::new(Error::InvalidInput( + "source-secret /private/startup-provider".into(), + ))), + }, + Vec::new(), + ) +} + +fn result_router(failure: WorkerCreateFailure) -> Router { + Router::new() + .route( + "/api/startup-result-fixture", + axum::routing::get(move || { + let failure = failure.clone(); + async move { api_error(failure).into_response() } + }), + ) + .layer(middleware::from_fn(log_failed_api_response)) +} + +async fn public_result(app: Router) -> (StatusCode, Value, WorkerCreateFailure) { + let response = app + .oneshot( + Request::builder() + .uri("/api/startup-result-fixture") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + assert_eq!( + response.headers()[CONTENT_TYPE], + runtime_api::startup::PROBLEM_CONTENT_TYPE + ); + let body = to_bytes( + response.into_body(), + runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES, + ) + .await + .unwrap(); + let decoded = decode_startup_problem( + &body, + status.as_u16(), + runtime_api::startup::PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser, + ); + let DecodeStartupProblem::Known(failure) = decoded else { + panic!("actual HTTP response must satisfy shared bounded contract: {decoded:?}"); + }; + let value = serde_json::from_slice(&body).unwrap(); + assert_no_secrets(std::str::from_utf8(&body).unwrap()); + (status, value, failure) +} + +#[derive(Clone, Default)] +struct Capture(Arc>>); + +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl Capture { + fn events(&self) -> Vec { + let bytes = self.0.lock().unwrap().clone(); + let text = std::str::from_utf8(&bytes).unwrap(); + assert_no_secrets(text); + text.lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } +} + +fn subscriber(capture: Capture) -> impl tracing::Subscriber + Send + Sync { + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .json() + .flatten_event(true) + .with_writer(move || capture.clone()) + .finish() +} + +fn assert_no_secrets(text: &str) { + for secret in SECRETS { + assert!( + !text.contains(secret), + "private value {secret} leaked: {text}" + ); + } +} + +fn assert_log(event: &Value, failure: &WorkerCreateFailure, status: StatusCode) { + assert_eq!(event["target"], "yoi::worker_startup"); + assert_eq!(event["event"], "worker_create_result"); + assert_eq!(event["domain"], "session_registration"); + assert_eq!(event["reason"], "permission_denied"); + assert_eq!( + event["correlation"], + failure.operation().correlation.as_ref().unwrap().as_str() + ); + assert_eq!(event["status"], status.as_u16()); + assert_eq!( + event["acceptance"], + format!("{:?}", failure.operation().acceptance) + ); + assert_eq!( + event["execution"], + format!("{:?}", failure.operation().execution) + ); + assert_eq!( + event["finalize"], + format!("{:?}", failure.operation().finalize) + ); + assert_eq!( + event["backend_compensation"], + format!("{:?}", failure.operation().backend_compensation) + ); + assert_eq!( + event["backend_result_persistence"], + format!("{:?}", failure.backend_result_persistence()) + ); + assert_eq!( + event["persistence"], + format!("{:?}", failure.startup().persistence) + ); +} + +#[tokio::test] +async fn persisted_result_public_problem_and_single_http_log_have_matching_semantics() { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let worker = reserve(&api); + let before = lifecycle_snapshot(&api); + let original = failure_fixture(CompensationOutcome::Residual); + let returned = worker_startup::persist(&api, &worker, original.clone()); + assert_eq!( + returned.backend_result_persistence(), + Some(StartupPersistenceOutcome::Persisted) + ); + assert_eq!( + returned.startup().persistence, + Some(StartupPersistenceOutcome::NotSaved) + ); + assert_eq!(primary_cause(&returned), primary_cause(&original)); + assert_eq!(lifecycle_snapshot(&api), before); + let saved = retained(&api, &worker); + assert_semantics(&saved, &returned); + // Storage projection changes envelope bytes; equality must be semantic. + assert_ne!(saved.startup().primary, original.startup().primary); + let capture = Capture::default(); + let (status, problem, public) = public_result(result_router(returned.clone())) + .with_subscriber(subscriber(capture.clone())) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert!( + problem["detail"] + .as_str() + .unwrap() + .contains("session.record") + ); + assert!(problem["detail"].as_str().unwrap().contains("Subject")); + assert!( + problem["instance"] + .as_str() + .unwrap() + .contains(returned.operation().correlation.as_ref().unwrap().as_str()) + ); + assert_semantics(&public, &saved); + let events = capture.events(); + assert_eq!( + events.len(), + 1, + "only HTTP responsibility boundary records primary: {events:?}" + ); + assert_log(&events[0], &public, status); + assert_eq!(lifecycle_snapshot(&api), before); +} + +#[tokio::test] +async fn sqlite_result_write_failure_preserves_primary_facts_prior_bytes_and_lifecycle() { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let worker = reserve(&api); + let prior = worker_startup::persist( + &api, + &worker, + failure_fixture(CompensationOutcome::Completed), + ); + let prior_bytes = api + .config_store + .worker_create_failure( + TEST_WORKSPACE_ID, + &worker.runtime_id, + WorkerId::parse(&worker.worker_id).unwrap(), + ) + .unwrap() + .unwrap(); + let before = lifecycle_snapshot(&api); + api.config_store + .with_conn(|conn| { + conn.execute_batch( + "CREATE TRIGGER fail_startup_result BEFORE UPDATE OF startup_failure_json + ON worker_create_reservations BEGIN SELECT RAISE(FAIL, 'sqlite-secret'); END;", + )?; + Ok(()) + }) + .unwrap(); + let attempted = failure_fixture(CompensationOutcome::Unknown); + let returned = worker_startup::persist(&api, &worker, attempted.clone()); + assert_eq!( + returned.backend_result_persistence(), + Some(StartupPersistenceOutcome::OutcomeUnknown) + ); + assert_eq!(primary_cause(&returned), primary_cause(&attempted)); + let mut expected_operation = attempted.operation().clone(); + expected_operation.correlation = returned.operation().correlation.clone(); + assert_eq!(returned.operation(), &expected_operation); + assert_eq!(returned.startup(), attempted.startup()); + assert_eq!(lifecycle_snapshot(&api), before); + assert_eq!( + api.config_store + .worker_create_failure( + TEST_WORKSPACE_ID, + &worker.runtime_id, + WorkerId::parse(&worker.worker_id).unwrap(), + ) + .unwrap() + .unwrap(), + prior_bytes + ); + assert_semantics(&retained(&api, &worker), &prior); + let capture = Capture::default(); + let (status, _, public) = public_result(result_router(returned.clone())) + .with_subscriber(subscriber(capture.clone())) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert_semantics(&public, &returned); + let events = capture.events(); + assert_eq!(events.len(), 1); + assert_log(&events[0], &returned, status); +} + +#[tokio::test] +async fn finalize_compensation_does_not_replace_received_primary_or_acceptance() { + for compensation in [ + CompensationOutcome::Completed, + CompensationOutcome::Residual, + CompensationOutcome::Unknown, + ] { + let original = failure_fixture(CompensationOutcome::Residual); + let error = worker_startup::finalize_error( + api_error(original.clone()), + WorkerSpawnFinalizeStage::WorkdirAttachment, + compensation, + ); + let Error::WorkerCreateFailed { failure, .. } = &error.error else { + panic!("finalize must keep a typed startup result"); + }; + assert_eq!(primary_cause(failure), primary_cause(&original)); + assert_eq!(failure.startup(), original.startup()); + assert_eq!( + failure.operation().acceptance, + original.operation().acceptance + ); + assert_eq!( + failure.operation().execution, + original.operation().execution + ); + assert_eq!( + failure.operation().finalize, + Some(FinalizeFacts { + stage: FinalizeStage::WorkdirAttachment, + compensation + }) + ); + assert_eq!(failure.operation().backend_compensation, Some(compensation)); + let expected = failure.as_ref().clone(); + let (_, _, public) = public_result(result_router(expected.clone())).await; + assert_semantics(&public, &expected); + } +} + +#[tokio::test] +async fn context_and_public_projection_are_silent_until_explicit_http_observation() { + let capture = Capture::default(); + let expected = async { + let worker = RuntimeWorkerRef { + runtime_id: "context-secret".into(), + worker_id: "source-secret".into(), + }; + let failure = + worker_startup::correlate(failure_fixture(CompensationOutcome::Unknown), &worker); + let public = api_error(failure.clone()).into_response(); + assert_eq!(public.status(), StatusCode::CONFLICT); + assert!( + capture.events().is_empty(), + "context/encoder must not log the primary" + ); + worker_startup::observe_http(&failure, public.status()); + failure + } + .with_subscriber(subscriber(capture.clone())) + .await; + let events = capture.events(); + assert_eq!(events.len(), 1); + assert_log(&events[0], &expected, StatusCode::CONFLICT); +} + +async fn assert_publication_fault_retains_committed_worker(query_failure: bool) { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + // Freeze unrelated observation writes before deliberately damaging the DB. + api.worker_projection.shutdown_async().await; + api.store.db_executor().drain().await.unwrap(); + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let stops = Arc::new(AtomicUsize::new(0)); + let stop_count = stops.clone(); + let runtime = WorkdirlessFixtureRuntime { + stop_callback: Some(Arc::new(move || { + stop_count.fetch_add(1, Ordering::SeqCst); + })), + ..Default::default() + }; + api.runtime.register_or_replace(runtime.clone()); + let worker = reserve_for_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID); + let spawned = crate::hosts::WorkspaceWorkerRuntime::spawn_worker( + &runtime, + WorkerCreateBinding { + plugin_execution: None, + worker_id: WorkerId::parse(&worker.worker_id).unwrap(), + create_fingerprint: "sha256:startup-result-fixture".into(), + }, + WorkerSpawnRequest { + intent: WorkerSpawnIntent::WorkspaceCompanion, + requested_worker_name: Some("Publication fixture".into()), + singleton_key: None, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile: ProfileSelector::Builtin("builtin:companion".into()), + ticket_assignment: None, + initial_submit: Vec::new(), + workdir_attachment_requests: Vec::new(), + resolved_workdir_attachment_requests: Vec::new(), + resolved_workdir_attachments: Vec::new(), + resolved_config_bundle: None, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_control_operation: None, + resolved_execution_grant: None, + }, + ); + assert_eq!(spawned.state, InternalWorkerOperationState::Accepted); + let summary = spawned.worker.unwrap(); + assert_eq!(summary.worker, worker); + let record = record_worker_summary( + &api, + &summary, + &summary.display_name, + summary.profile.clone(), + WorkerRegistryDisplayNamePolicy::UseProvided, + ) + .unwrap(); + api.config_store + .complete_worker_create_reservation( + TEST_WORKSPACE_ID, + WorkerId::parse(&worker.worker_id).unwrap(), + ) + .unwrap(); + let ticket = browser_ticket_backend(&api) + .unwrap() + .create(ticket::NewTicket::new("Publication retains assignment")) + .unwrap(); + let assignment = TicketWorkerAssignmentRecord { + workspace_id: TEST_WORKSPACE_ID.into(), + ticket_id: ticket.id.clone(), + assignment_id: "publication-assignment".into(), + worker: worker.clone(), + assigned_by: "test-user".into(), + assigned_at: TEST_CREATED_AT.into(), + }; + api.store + .set_current_ticket_worker_assignment( + &assignment, + None, + "publication-assignment-event", + "publication-assignment-operation", + false, + ) + .unwrap(); + // Establish a working publication read first; the fault changes only that + // final read, not reservation persistence or an earlier caller-identity check. + let key = api + .store + .worker_resource_key(TEST_WORKSPACE_ID, &worker) + .unwrap() + .unwrap(); + api.config_store + .with_conn(|conn| { + if query_failure { + conn.execute_batch( + "ALTER TABLE workspace_resource_keys RENAME COLUMN resource_key TO \"sqlite-secret\";", + )?; + } else { + let deleted = conn.execute( + "DELETE FROM workspace_resource_keys WHERE workspace_id=?1 AND resource_kind='worker' AND resource_key=?2", + rusqlite::params![TEST_WORKSPACE_ID, key], + )?; + assert_eq!(deleted, 1); + } + Ok(()) + }) + .unwrap(); + let before = lifecycle_snapshot(&api); + let capture = Capture::default(); + let route_api = api.clone(); + let app = Router::new() + .route( + "/api/startup-result-fixture", + axum::routing::get(move || { + let api = route_api.clone(); + let summary = summary.clone(); + async move { + let error = browser_worker_response_for_committed_summary( + &api, + summary, + "Publication fixture".into(), + Vec::new(), + ) + .unwrap_err(); + let Error::WorkerCreateFailed { source, .. } = &error.error else { + panic!("final publication read must return a typed result: {error:?}"); + }; + let source = source + .as_deref() + .expect("keep the actual read error internal"); + if query_failure { + assert!(matches!(source, Error::Sqlite(_)), "{source:?}"); + } else { + assert!(matches!(source, Error::Store(_)), "{source:?}"); + assert!(source.to_string().contains("no resource key")); + } + error.into_response() + } + }), + ) + .layer(middleware::from_fn(log_failed_api_response)); + let (status, problem, public) = public_result(app) + .with_subscriber(subscriber(capture.clone())) + .await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + primary_cause(&public), + StartupCause::Boundary(runtime_api::startup::StartupBoundaryCause { + reason: runtime_api::startup::StartupBoundaryReason::FinalizeFailed, + }) + ); + assert_eq!(public.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!(public.operation().execution, ExecutionOutcome::Accepted); + assert_eq!( + public.operation().finalize, + Some(FinalizeFacts { + stage: FinalizeStage::Publication, + compensation: CompensationOutcome::Residual, + }) + ); + assert_eq!(public.operation().backend_compensation, None); + assert_eq!( + public.startup().resources, + Some(protocol::startup_failure::StartupResourceFacts { + resources: protocol::startup_failure::StartupResources::WorkerStartup, + cleanup: protocol::startup_failure::StartupCleanup::NotNeeded, + durable_effects_possible: true, + }) + ); + assert_eq!( + public.backend_result_persistence(), + Some(StartupPersistenceOutcome::Persisted) + ); + assert_eq!( + public.operation().correlation.as_ref().unwrap().as_str(), + worker_startup::correlation(&worker.runtime_id, &worker.worker_id) + ); + assert!( + problem["instance"] + .as_str() + .unwrap() + .ends_with(public.operation().correlation.as_ref().unwrap().as_str()) + ); + assert_semantics(&retained(&api, &worker), &public); + let events = capture.events(); + assert_eq!(events.len(), 1, "one primary HTTP event: {events:?}"); + let event = &events[0]; + assert_eq!(event["target"], "yoi::worker_startup"); + assert_eq!(event["event"], "worker_create_result"); + assert_eq!(event["domain"], "startup_boundary"); + assert_eq!(event["reason"], "finalize_failed"); + assert_eq!(event["acceptance"], "Accepted"); + assert_eq!(event["execution"], "Accepted"); + assert_eq!( + event["finalize"], + format!("{:?}", public.operation().finalize) + ); + assert_eq!(event["backend_compensation"], "None"); + assert_eq!(event["backend_result_persistence"], "Some(Persisted)"); + assert_eq!( + event["resources"], + format!("{:?}", public.startup().resources) + ); + assert_eq!(event["status"], status.as_u16()); + assert_eq!( + event["correlation"], + problem["yoi_startup"]["operation"]["correlation"] + ); + let exposed = format!("{problem} {events:?}"); + for internal in [ + worker.worker_id.as_str(), + "no such column", + "no resource key", + "workspace_resource_keys", + dir.path().to_str().unwrap(), + ] { + assert!( + !exposed.contains(internal), + "internal source leaked: {exposed}" + ); + } + assert_eq!(lifecycle_snapshot(&api), before); + assert_eq!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &worker) + .unwrap(), + Some(record) + ); + assert_eq!( + api.store + .get_current_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id) + .unwrap(), + Some(assignment) + ); + assert_eq!(stops.load(Ordering::SeqCst), 0); + assert_eq!(runtime.spawn_requests().len(), 1, "no reexecution"); + assert_eq!( + runtime.workers.lock().unwrap().len(), + 1, + "no Runtime deletion" + ); +} + +#[tokio::test] +async fn publication_sqlite_read_failure_retains_committed_worker_and_typed_result() { + assert_publication_fault_retains_committed_worker(true).await; +} + +#[tokio::test] +async fn publication_missing_resource_key_retains_committed_worker_and_typed_result() { + assert_publication_fault_retains_committed_worker(false).await; +} + +#[tokio::test] +async fn publication_error_preserves_existing_primary_without_claiming_compensation() { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let worker = reserve(&api); + let original = failure_fixture(CompensationOutcome::Completed); + let error = worker_startup::publication_error(&api, &worker, api_error(original.clone())); + let Error::WorkerCreateFailed { failure, source } = &error.error else { + panic!("publication must retain a typed startup result"); + }; + assert!(matches!( + source.as_deref(), + Some(Error::WorkerCreateFailed { .. }) + )); + assert_eq!(failure.startup().primary, original.startup().primary); + assert_eq!(failure.startup().secondary, original.startup().secondary); + assert_eq!( + failure.startup().persistence, + original.startup().persistence + ); + assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted); + assert_eq!(failure.operation().backend_compensation, None); + assert_eq!( + failure.operation().finalize.as_ref().unwrap().stage, + FinalizeStage::Publication + ); + assert_semantics(&retained(&api, &worker), failure); +} + +#[derive(Clone)] +struct FailingWriter(Arc); + +impl Write for FailingWriter { + fn write(&mut self, _: &[u8]) -> std::io::Result { + self.0.fetch_add(1, Ordering::SeqCst); + Err(std::io::Error::other("sink unavailable")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[tokio::test] +async fn failed_log_sink_still_returns_original_http_problem_and_retained_result() { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let worker = reserve(&api); + let saved = worker_startup::persist( + &api, + &worker, + failure_fixture(CompensationOutcome::Residual), + ); + let before = lifecycle_snapshot(&api); + let capture = Capture::default(); + let baseline = public_result(result_router(saved.clone())) + .with_subscriber(subscriber(capture.clone())) + .await; + assert_eq!(capture.events().len(), 1); + let attempts = Arc::new(AtomicUsize::new(0)); + let writer = FailingWriter(attempts.clone()); + let failed_subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .json() + .flatten_event(true) + .with_writer(move || writer.clone()) + .finish(); + let actual = public_result(result_router(saved.clone())) + .with_subscriber(failed_subscriber) + .await; + assert!( + attempts.load(Ordering::SeqCst) > 0, + "must exercise actual observation sink failure" + ); + assert_eq!(actual.0, baseline.0); + assert_eq!( + actual.1, baseline.1, + "sink failure must not replace the outgoing Problem" + ); + assert_semantics(&actual.2, &saved); + assert_semantics(&retained(&api, &worker), &saved); + assert_eq!(lifecycle_snapshot(&api), before); +} diff --git a/crates/workspace-server/src/server/tests/subject_authorization_tests.rs b/crates/workspace-server/src/server/tests/subject_authorization_tests.rs index 514fed0de..0ec36128d 100644 --- a/crates/workspace-server/src/server/tests/subject_authorization_tests.rs +++ b/crates/workspace-server/src/server/tests/subject_authorization_tests.rs @@ -2745,24 +2745,53 @@ pub(super) struct ConnectedWorkerHarness { _dir: tempfile::TempDir, } impl ConnectedWorkerHarness { - pub(super) async fn human( + /// Exercise the authenticated Browser TCP/HTTP boundary without discarding + /// media type or rewriting its body. Bound both elapsed time and streamed bytes; + /// redirects/retries must not turn a failed create response into another POST. + pub(super) async fn human_raw( &self, method: reqwest::Method, path: &str, body: Value, - ) -> (StatusCode, Value) { - let response = reqwest::Client::builder() + ) -> (StatusCode, HeaderMap, Vec) { + const RESPONSE_LIMIT: usize = 2 * 1024 * 1024; + let mut response = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(30)) + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) .build() .unwrap() - .request(method.clone(), format!("{}{path}", self.backend_url)) + .request(method, format!("{}{path}", self.backend_url)) .bearer_auth(&self.owner) .json(&body) .send() .await .unwrap(); let status = StatusCode::from_u16(response.status().as_u16()).unwrap(); - let bytes = response.bytes().await.unwrap(); + let headers = response.headers().clone(); + assert!( + response + .content_length() + .is_none_or(|len| len <= RESPONSE_LIMIT as u64) + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await.unwrap() { + assert!( + chunk.len() <= RESPONSE_LIMIT - bytes.len(), + "Browser response exceeds test limit" + ); + bytes.extend_from_slice(&chunk); + } + (status, headers, bytes) + } + + pub(super) async fn human( + &self, + method: reqwest::Method, + path: &str, + body: Value, + ) -> (StatusCode, Value) { + let (status, _, bytes) = self.human_raw(method.clone(), path, body).await; let body = if status == StatusCode::NO_CONTENT { assert!(bytes.is_empty(), "{method} {path}: HTTP204 must be empty"); Value::Null @@ -2970,20 +2999,36 @@ impl ConnectedWorkerHarness { } let (parts, body) = response.into_parts(); let bytes = to_bytes(body, 2 * 1024 * 1024).await.unwrap(); + let content_type = parts + .headers + .get(axum::http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or(""); let body: Value = serde_json::from_slice(&bytes).unwrap(); - if let Some(startup) = body["error"].get("startup").filter(|v| !v.is_null()) + let message = if content_type == runtime_api::startup::PROBLEM_CONTENT_TYPE { - startup_failures.lock().unwrap().push( - serde_json::from_value(startup.clone()) - .expect("RuntimeHTTP structured startup failure"), - ); - } - errors.lock().unwrap().push( + use runtime_api::startup::{ + DecodeStartupProblem, StartupBoundary, decode_startup_problem, + }; + let DecodeStartupProblem::Known(failure) = decode_startup_problem( + &bytes, + parts.status.as_u16(), + content_type, + StartupBoundary::Runtime, + ) else { + panic!("RuntimeHTTP must emit a valid typed startup Problem"); + }; + startup_failures + .lock() + .unwrap() + .push(failure.startup().clone()); + body["detail"].as_str().expect("Problem detail") + } else { body["error"]["message"] .as_str() .unwrap_or("unknown fixture error") - .to_owned(), - ); + }; + errors.lock().unwrap().push(message.to_owned()); Response::from_parts(parts, Body::from(bytes)) } }, diff --git a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs index 21275c2f8..2615c5d19 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs @@ -12,6 +12,8 @@ use worker_runtime::execution::{ // published Profile projection; leave Worker creation/control to the real Runtime. #[derive(Default)] struct LaunchExecutionBackend { + lose_initial_ack: std::sync::atomic::AtomicBool, + fail_stop: std::sync::atomic::AtomicBool, execution: DeterministicExecutionBackend, provider_close_gate: Mutex>>, bundles: Mutex>, @@ -196,10 +198,45 @@ impl WorkerExecutionBackend for LaunchExecutionBackend { .lock() .unwrap() .push((worker.clone(), input.clone())); - self.execution.dispatch_input(worker, input) + let mut result = self.execution.dispatch_input(worker, input); + if self + .lose_initial_ack + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + // Lose only the response acknowledgement, after the real fixture + // execution committed its durable receipt. Never redispatch it. + result.submission = None; + } + result + } + + fn termination_receipt( + &self, + worker: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result, String> + { + self.execution.termination_receipt(worker) + } + + fn stop_worker_operation( + &self, + request: worker_runtime::execution::WorkerExecutionStopRequest, + ) -> WorkerExecutionResult { + if self.fail_stop.load(std::sync::atomic::Ordering::SeqCst) { + return self.stop_worker(&request.worker_ref); + } + // Preserve the fixture's actual joined execution and operation-bound + // termination receipt; an accepted legacy stop alone is not that evidence. + self.execution.stop_worker_operation(request) } fn stop_worker(&self, worker: &worker_runtime::identity::WorkerRef) -> WorkerExecutionResult { + if self.fail_stop.load(std::sync::atomic::Ordering::SeqCst) { + return WorkerExecutionResult::errored( + worker_runtime::execution::WorkerExecutionOperation::Stop, + "TEST_ONLY_PRIVATE_STOP_CREDENTIAL /private/path", + ); + } self.execution.stop_worker(worker) } } @@ -1133,6 +1170,90 @@ async fn read_only_ticket_targets_allow_extra_external_grants_without_widening_c ); } +/// Production create owner and Backend adapters share the same executor; only +/// its response acknowledgement and cleanup boundary are fault-injected. +#[tokio::test(flavor = "multi_thread")] +async fn initial_input_ack_loss_preserves_creation_facts_across_embedded_and_remote() { + use runtime_api::startup::*; + for failed_cleanup in [false, true] { + let mut observed = Vec::new(); + for remote in [false, true] { + let fixture = LaunchFixture::new(remote).await; + let ticket = fixture.ticket(); + let payload = fixture.payload(&ticket, "lost-initial-ack", "builtin:companion", false); + fixture + .backend + .lose_initial_ack + .store(true, std::sync::atomic::Ordering::SeqCst); + fixture + .backend + .fail_stop + .store(failed_cleanup, std::sync::atomic::Ordering::SeqCst); + let (status, body) = fixture.post(TEST_WORKSPACE_ID, payload).await; + assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); + let DecodeStartupProblem::Known(failure) = decode_startup_problem( + &serde_json::to_vec(&body).unwrap(), + status.as_u16(), + PROBLEM_CONTENT_TYPE, + StartupBoundary::Browser, + ) else { + panic!("must retain typed creation facts: {body}"); + }; + assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown); + let finalize = failure.operation().finalize.unwrap(); + assert_eq!(finalize.stage, FinalizeStage::InitialInput); + assert_eq!( + finalize.compensation, + if failed_cleanup { + CompensationOutcome::Unknown + } else { + CompensationOutcome::Completed + } + ); + assert_eq!(body["yoi_failure"]["reason"], "invalid_response"); + assert!( + !body + .to_string() + .contains("TEST_ONLY_PRIVATE_STOP_CREDENTIAL") + ); + let creates = fixture.backend.creates.lock().unwrap().clone(); + assert_eq!(creates.len(), 1); + let worker_id = creates[0].worker_id.to_string(); + let worker = RuntimeWorkerRef::new(&fixture.runtime_id, &worker_id); + let Some(DecodeStoredWorkerCreateFailure::Known(saved)) = + fixture.api.worker_create_failure(&worker).unwrap() + else { + panic!("retain queryable create result"); + }; + assert_eq!(saved.public_facts(), failure.public_facts()); + assert_eq!( + fixture.backend.submits_for(&worker_id).len(), + 1, + "observation failure never redispatches" + ); + assert_eq!( + fixture + .backend + .execution + .contexts + .lock() + .unwrap() + .keys() + .any(|worker| worker.worker_id.to_string() == worker_id), + failed_cleanup + ); + let mut facts = failure.public_facts(); + facts.operation.correlation = None; + observed.push((status, body["yoi_failure"]["reason"].clone(), facts)); + } + assert_eq!( + observed[0], observed[1], + "same owner facts through Embedded and Remote" + ); + } +} + #[tokio::test(flavor = "multi_thread")] async fn worker_assignment_finalize_failure_compensates_real_embedded_runtime_and_preserves_open_case() { @@ -1151,10 +1272,19 @@ async fn worker_assignment_finalize_failure_compensates_real_embedded_runtime_an .unwrap(); let payload = fixture.payload(&ticket_id, "compensated-worker", "builtin:companion", false); let (status, body) = fixture.post(TEST_WORKSPACE_ID, payload).await; - assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}"); + assert_eq!(body["yoi_failure"]["domain"], "startup_boundary"); + assert_eq!(body["yoi_failure"]["reason"], "finalize_failed"); + let operation = &body["yoi_startup"]["operation"]; + assert_eq!(operation["acceptance"], "accepted"); + assert_eq!(operation["execution"], "accepted"); + assert_eq!(operation["finalize"]["stage"], "ticket_assignment_bind"); + assert_eq!(operation["finalize"]["compensation"], "completed"); + assert!(body["detail"].as_str().unwrap().contains("finalization")); assert!( - body["message"].as_str().unwrap().contains("compensat"), - "{body}" + !body + .to_string() + .contains("injected Worker assignment failure") ); let creates = fixture .backend @@ -1311,6 +1441,7 @@ impl crate::hosts::WorkspaceWorkerRuntime for LostSpawnResponseRuntime { assert_eq!(result.state, InternalWorkerOperationState::Accepted); assert!(result.worker.is_some()); WorkerSpawnResult { + failure: None, state: InternalWorkerOperationState::Rejected, worker: None, acceptance_evidence: Vec::new(), @@ -1829,3 +1960,81 @@ async fn mr_open_retains_source_before_first_review_and_observed_result() { ) .unwrap(); } + +// Real Runtime creation and SQLite finalization fail independently; only Stop is +// injected. Reconciliation must retain the create reservation while absence is unknown. +#[tokio::test(flavor = "multi_thread")] +async fn accepted_creation_finalize_and_failed_stop_retain_unknown_compensation_and_reservation() { + let fixture = LaunchFixture::new(false).await; + let ticket_id = fixture.ticket(); + rusqlite::Connection::open(&fixture.api.config.database_path).unwrap().execute_batch( + "CREATE TRIGGER fail_startup_finalize BEFORE INSERT ON ticket_current_worker_assignments WHEN NEW.role='worker' BEGIN SELECT RAISE(ABORT, 'TEST_ONLY_PRIVATE_DB_CREDENTIAL'); END;" + ).unwrap(); + fixture + .backend + .fail_stop + .store(true, std::sync::atomic::Ordering::SeqCst); + let (status, body) = fixture + .post( + TEST_WORKSPACE_ID, + fixture.payload( + &ticket_id, + "unknown-compensation", + "builtin:companion", + false, + ), + ) + .await; + fixture + .backend + .fail_stop + .store(false, std::sync::atomic::Ordering::SeqCst); + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}"); + assert_eq!(body["yoi_failure"]["reason"], "finalize_failed"); + let operation = &body["yoi_startup"]["operation"]; + assert_eq!(operation["acceptance"], "accepted"); + assert_eq!(operation["execution"], "accepted"); + assert_eq!(operation["finalize"]["compensation"], "unknown"); + assert!( + operation["correlation"] + .as_str() + .unwrap() + .starts_with("worker-") + ); + for secret in [ + "TEST_ONLY_PRIVATE_STOP_CREDENTIAL", + "/private/path", + "TEST_ONLY_PRIVATE_DB_CREDENTIAL", + ] { + assert!(!body.to_string().contains(secret), "{body}"); + } + assert_eq!( + fixture + .api + .authority + .ticket(&ticket_id) + .unwrap() + .state + .as_str(), + "open" + ); + let created = fixture.backend.creates.lock().unwrap(); + let created: Vec<_> = created + .iter() + .filter(|request| request.display_name.as_deref() == Some("Explicit Ticket Worker")) + .collect(); + assert_eq!(created.len(), 1); + let worker_id = created[0].worker_id.to_string(); + let reserved: i64 = fixture.api.config_store.with_conn(|conn| conn.query_row( + "SELECT COUNT(*) FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='reserved'", + rusqlite::params![TEST_WORKSPACE_ID, fixture.runtime_id, worker_id], |row| row.get(0), + ).map_err(Error::from)).unwrap(); + // A reservation may already be completed by Runtime admission, but it must + // remain present (never released as absent) until Runtime absence is confirmed. + let retained: i64 = fixture.api.config_store.with_conn(|conn| conn.query_row( + "SELECT COUNT(*) FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3", + rusqlite::params![TEST_WORKSPACE_ID, fixture.runtime_id, worker_id], |row| row.get(0), + ).map_err(Error::from)).unwrap(); + assert_eq!(retained, 1); + assert!(reserved <= retained); +} diff --git a/crates/workspace-server/src/server/worker_startup.rs b/crates/workspace-server/src/server/worker_startup.rs new file mode 100644 index 000000000..0b92ca985 --- /dev/null +++ b/crates/workspace-server/src/server/worker_startup.rs @@ -0,0 +1,286 @@ +//! Backend creation boundary. Cause and independent owner observations are retained +//! before HTTP projection. Correlation fingerprints are lookup aids, not operation +//! identities, idempotency keys, execution grants, or permission to retry. +//! +//! # Result investigation +//! +//! Start with the Browser Problem's `yoi_startup.operation.correlation` (also its +//! `instance` suffix). For a known Runtime/Worker identity, `correlation` derives +//! the same path-free token. Find the single `yoi::worker_startup` event named +//! `worker_create_result`; do not use the token as a request or operation key. +//! Compare its typed domain/reason with `WorkspaceApi::worker_create_failure`'s +//! retained reservation evidence. Inspect acceptance/execution first, then finalize, +//! Backend compensation and residual resources independently. Unknown/corrupt or +//! absent history remains unknown/absent, not evidence of rejection or cleanup. +//! +//! The existing Session metadata retains the Runtime startup primary and its own +//! `persistence`; `backend_result_persistence` reports only this coordinator result +//! column's atomic write. A failed sink leaves the primary and observations intact. +//! Reopening the store reads evidence, never dispatches work. Current authorization +//! and a fresh domain projection are required before redisclosing loaded evidence. +//! Unsupported bodies, credentials, source errors and filesystem paths are not log +//! fields. Context wrapping, Remote decoding and persistence do not duplicate the +//! HTTP primary event; cleanup has a separate safe coordinator event. Non-HTTP +//! consumers receive the same typed result and own their eventual observation sink. +use super::*; +use runtime_api::startup::{ + CreationAcceptance, ExecutionOutcome, StartupCause, StartupOperationFacts, WorkerCreateFailure, +}; + +pub(super) fn local_rejection() -> WorkerCreateFailure { + WorkerCreateFailure::from_cause( + StartupCause::Runtime(protocol::startup_failure::RuntimeStartupCause { + stage: protocol::startup_failure::StartupStage::Preflight, + reason: protocol::startup_failure::RuntimeStartupReason::StartupFailed, + }), + StartupOperationFacts { + acceptance: CreationAcceptance::Rejected, + execution: ExecutionOutcome::Unknown, + finalize: None, + correlation: None, + backend_compensation: None, + }, + ) + .expect("enum-only local cause fits the boundary budget") +} + +pub(super) fn finalize_error( + source: ApiError, + stage: WorkerSpawnFinalizeStage, + compensation: runtime_api::startup::CompensationOutcome, +) -> ApiError { + let failure = finalize_failure(&source, stage, compensation); + ApiError::with_diagnostics( + Error::WorkerCreateFailed { + failure: Box::new(failure), + source: Some(Box::new(source.error)), + }, + source.diagnostics, + ) +} + +pub(super) fn finalize_failure( + source: &ApiError, + stage: WorkerSpawnFinalizeStage, + compensation: runtime_api::startup::CompensationOutcome, +) -> WorkerCreateFailure { + use runtime_api::startup::{ + FinalizeFacts, FinalizeStage, StartupBoundaryCause, StartupBoundaryReason, + }; + let stage = match stage { + WorkerSpawnFinalizeStage::WorkerIdentity => FinalizeStage::WorkerIdentity, + WorkerSpawnFinalizeStage::WorkspaceApiBinding => FinalizeStage::WorkspaceApiBinding, + WorkerSpawnFinalizeStage::CreateReservation => FinalizeStage::CreateReservation, + WorkerSpawnFinalizeStage::WorkerRegistry => FinalizeStage::WorkerRegistry, + WorkerSpawnFinalizeStage::TicketAssignmentBind => FinalizeStage::TicketAssignmentBind, + WorkerSpawnFinalizeStage::TicketAssignmentCurrent => FinalizeStage::TicketAssignmentCurrent, + WorkerSpawnFinalizeStage::TicketStateAccept => FinalizeStage::TicketStateAccept, + WorkerSpawnFinalizeStage::WorkdirRegistry => FinalizeStage::WorkdirRegistry, + WorkerSpawnFinalizeStage::WorkdirAttachment => FinalizeStage::WorkdirAttachment, + }; + let operation = StartupOperationFacts { + acceptance: CreationAcceptance::Accepted, + execution: ExecutionOutcome::Accepted, + finalize: Some(FinalizeFacts { + stage, + compensation, + }), + backend_compensation: Some(compensation), + correlation: None, + }; + match &source.error { + Error::WorkerCreateFailed { failure, .. } => { + let (startup, mut previous) = failure.clone().into_parts(); + // Adding finalization context must not rewrite the earlier owner result. + previous.finalize = operation.finalize; + previous.backend_compensation = Some(compensation); + WorkerCreateFailure::new(startup, previous) + } + _ => match source.session_registration_cause { + Some(cause) => WorkerCreateFailure::from_cause(StartupCause::Session(cause), operation), + None => WorkerCreateFailure::from_cause( + StartupCause::Boundary(StartupBoundaryCause { + reason: StartupBoundaryReason::FinalizeFailed, + }), + operation, + ), + }, + } + .expect("locally constructed finalization facts fit the shared boundary budget") +} + +/// Publication failure cannot undo committed creation or authorize reexecution. +/// Keep the Worker and its assignment: `Residual` describes deliberate retention, +/// not attempted rollback; Backend compensation remains unattempted (`None`). +/// The original store error stays internal while the independent reservation sink +/// retains only the typed result and a path-free correlation. +pub(super) fn publication_error( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + source: ApiError, +) -> ApiError { + use protocol::startup_failure::{StartupCleanup, StartupResourceFacts, StartupResources}; + use runtime_api::startup::{ + CompensationOutcome, FinalizeFacts, FinalizeStage, StartupBoundaryCause, + StartupBoundaryReason, + }; + + let operation = StartupOperationFacts { + acceptance: CreationAcceptance::Accepted, + execution: ExecutionOutcome::Accepted, + finalize: Some(FinalizeFacts { + stage: FinalizeStage::Publication, + compensation: CompensationOutcome::Residual, + }), + backend_compensation: None, + correlation: None, + }; + let failure = match &source.error { + Error::WorkerCreateFailed { failure, .. } => failure.as_ref().clone(), + _ => WorkerCreateFailure::from_cause( + StartupCause::Boundary(StartupBoundaryCause { + reason: StartupBoundaryReason::FinalizeFailed, + }), + operation.clone(), + ) + .expect("enum-only publication cause fits the boundary budget"), + }; + let (mut startup, _) = failure.into_parts(); + startup.resources = Some(StartupResourceFacts { + resources: StartupResources::WorkerStartup, + cleanup: StartupCleanup::NotNeeded, + durable_effects_possible: true, + }); + let failure = WorkerCreateFailure::new(startup, operation) + .expect("locally constructed publication facts fit the shared boundary budget"); + let failure = persist(api, worker, failure); + ApiError::with_diagnostics( + Error::WorkerCreateFailed { + failure: Box::new(failure), + source: Some(Box::new(source.error)), + }, + source.diagnostics, + ) +} + +pub(super) fn unknown_result() -> WorkerCreateFailure { + WorkerCreateFailure::from_cause( + StartupCause::Boundary(runtime_api::startup::StartupBoundaryCause { + reason: runtime_api::startup::StartupBoundaryReason::InvalidResponse, + }), + StartupOperationFacts { + acceptance: CreationAcceptance::Unknown, + execution: ExecutionOutcome::Unknown, + finalize: None, + correlation: None, + backend_compensation: None, + }, + ) + .expect("enum-only observation fits the boundary budget") +} + +pub(super) fn correlate( + failure: WorkerCreateFailure, + worker: &RuntimeWorkerRef, +) -> WorkerCreateFailure { + let reference = correlation(&worker.runtime_id, &worker.worker_id); + let mut operation = failure.operation().clone(); + operation.correlation = Some( + runtime_api::startup::StartupCorrelation::new(reference).expect("fixed digest is safe"), + ); + failure.with_operation(operation) +} + +pub(super) fn compensated( + failure: WorkerCreateFailure, + outcome: runtime_api::startup::CompensationOutcome, +) -> WorkerCreateFailure { + let mut operation = failure.operation().clone(); + operation.backend_compensation = Some(outcome); + failure.with_operation(operation) +} + +/// Save only observations on the existing reservation, never its lifecycle state. +/// An unavailable diagnostic sink cannot replace the original primary or authorize replay. +/// The prospective persisted marker commits in the same transaction as these facts. +/// Remote-supplied Backend persistence is discarded at this responsibility boundary. +pub(super) fn persist( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + failure: WorkerCreateFailure, +) -> WorkerCreateFailure { + use protocol::startup_failure::StartupPersistenceOutcome::{ + Conflict, NotSaved, OutcomeUnknown, Persisted, + }; + use rusqlite::OptionalExtension; + let failure = correlate(failure, worker).with_backend_result_persistence(None); + let Some(worker_id) = WorkerId::parse(&worker.worker_id) else { + return failure.with_backend_result_persistence(Some(NotSaved)); + }; + let observed = api.config_store.with_conn(|conn| { + conn.query_row( + "SELECT create_fingerprint, startup_failure_json FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3", + rusqlite::params![api.workspace_id(), worker.runtime_id, worker.worker_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)), + ).optional().map_err(Error::from) + }); + let (fingerprint, previous) = match observed { + Ok(Some(observed)) => observed, + Ok(None) => return failure.with_backend_result_persistence(Some(NotSaved)), + Err(_) => return failure.with_backend_result_persistence(Some(OutcomeUnknown)), + }; + let saved = failure + .clone() + .with_backend_result_persistence(Some(Persisted)); + let Ok(encoded) = saved.encode_storage() else { + return failure.with_backend_result_persistence(Some(NotSaved)); + }; + match api.config_store.save_worker_create_failure( + api.workspace_id(), + &worker.runtime_id, + worker_id, + &fingerprint, + previous.as_deref(), + &encoded, + ) { + Ok(true) => saved, + Ok(false) => failure.with_backend_result_persistence(Some(Conflict)), + Err(_) => failure.with_backend_result_persistence(Some(OutcomeUnknown)), + } +} + +pub(super) fn correlation(runtime_id: &str, worker_id: &str) -> String { + runtime_api::startup::StartupCorrelation::for_worker(runtime_id, worker_id) + .as_str() + .to_owned() +} + +/// Only the HTTP result boundary records the primary failure. Context wrappers, +/// encoders, and Remote adapters do not emit duplicate failure records. The +/// independent coordinator cleanup event contains no exception/source text. +pub(super) fn observe_http(failure: &WorkerCreateFailure, status: StatusCode) { + let Ok(projected) = failure.telemetry() else { + tracing::error!(target: "yoi::worker_startup", event = "startup_projection_unavailable", status = status.as_u16()); + return; + }; + let fields = error_contract::observation_fields(&projected); + let operation = failure.operation(); + tracing::warn!( + target: "yoi::worker_startup", + event = "worker_create_result", + domain = fields.domain, + reason = fields.reason, + operation = fields.operation, + stage = fields.stage, + correlation = operation.correlation.as_ref().map(|value| value.as_str()), + acceptance = ?operation.acceptance, + execution = ?operation.execution, + finalize = ?operation.finalize, + backend_compensation = ?operation.backend_compensation, + backend_result_persistence = ?failure.backend_result_persistence(), + persistence = ?failure.startup().persistence, + resources = ?failure.startup().resources, + status = status.as_u16(), + "Worker creation returned a typed failure" + ); +} diff --git a/crates/workspace-server/src/storage_formats.rs b/crates/workspace-server/src/storage_formats.rs index 32d187d20..60a768b2c 100644 --- a/crates/workspace-server/src/storage_formats.rs +++ b/crates/workspace-server/src/storage_formats.rs @@ -76,6 +76,26 @@ pub(super) fn verify_history(conn: &Connection, current: i64) -> Result<()> { fn verify_history_records(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> { verify_raw_capture_shape(conn, current, rows)?; + if current == 124 { + super::verify_worker_create_failure_schema(conn)?; + if rows == [(124, super::SCHEMA_BASELINE_NAME.to_owned())] { + verify_execution_shape(conn)?; + super::backend_job_upgrade::verify(conn)?; + verify_configuration_shape(conn)?; + require_shape(conn, Stage::Base)?; + return verify_103_features(conn, current, rows); + } + if rows.last().map(|(version, name)| (*version, name.as_str())) + != Some((124, super::WORKER_CREATE_FAILURE_MIGRATION_NAME)) + { + return Err(invalid("unrecognized Worker creation result receipt")); + } + let prefix = &rows[..rows.len() - 1]; + if prefix.last().map(|row| row.0) != Some(123) { + return Err(invalid("missing Worker creation result predecessor")); + } + return verify_history_records(conn, 123, prefix); + } if current == 123 { verify_execution_shape(conn)?; super::backend_job_upgrade::verify(conn)?; @@ -236,10 +256,11 @@ pub(super) fn published_plan( verify_history(conn, current)?; Ok(Some(super::WorkspaceSchemaMigrationPlan { current_schema_version: current, - target_schema_version: 123, + target_schema_version: 124, migrations: [ (122, crate::config_adoption::MIGRATION_NAME), (123, super::backend_job_upgrade::MIGRATION_NAME), + (124, super::WORKER_CREATE_FAILURE_MIGRATION_NAME), ] .into_iter() .filter(|(version, _)| *version > current) @@ -465,7 +486,7 @@ fn legacy_shape(conn: &Connection) -> Result { } fn upgraded_reference(conn: &Connection, mut shape: Shape) -> Result { - if super::current_schema_version(conn)? != 123 { + if super::current_schema_version(conn)? < 123 { return Ok(shape); } for table in ["backend_jobs", "backend_job_attempts"] { diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 4e20ec416..c0b56f55b 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -137,7 +137,10 @@ pub(crate) mod backend_job_upgrade; mod storage_formats; pub use backend_job_recovery_store::BackendJobReconciliation; -const LATEST_SCHEMA_VERSION: i64 = 123; +const LATEST_SCHEMA_VERSION: i64 = 124; +const WORKER_CREATE_FAILURE_MIGRATION_NAME: &str = + "Worker create coordinator returned startup failure"; +pub(crate) const WORKER_CREATE_FAILURE_MAX_BYTES: usize = 32 * 1024; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; @@ -520,6 +523,11 @@ const MIGRATIONS: &[Migration] = &[ name: backend_job_upgrade::MIGRATION_NAME, apply: migrate_backend_job_execution_to_v123, }, + Migration { + version: 124, + name: WORKER_CREATE_FAILURE_MIGRATION_NAME, + apply: migrate_worker_create_failure_to_v124, + }, ]; // Both schema-103 identities shipped. Keep the alternate implementation and accept @@ -16388,6 +16396,50 @@ fn migrate_plugin_services_v111_to_v112(conn: &Connection) -> Result<()> { Ok(()) } +fn verify_worker_create_failure_schema(conn: &Connection) -> Result<()> { + let column = conn + .query_row( + "SELECT type, \"notnull\", dflt_value FROM pragma_table_info('worker_create_reservations') WHERE name='startup_failure_json'", + [], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, bool>(1)?, row.get::<_, Option>(2)?)), + ) + .optional()?; + let sql: String = conn.query_row( + "SELECT sql FROM sqlite_schema WHERE type='table' AND name='worker_create_reservations'", + [], + |row| row.get(0), + )?; + let normalized: String = sql.chars().filter(|c| !c.is_whitespace()).collect(); + let constraint = format!( + "CHECK(startup_failure_jsonISNULLOR(typeof(startup_failure_json)='text'ANDlength(CAST(startup_failure_jsonASBLOB))<={}))", + WORKER_CREATE_FAILURE_MAX_BYTES + ); + if column != Some(("TEXT".into(), false, None)) || !normalized.contains(&constraint) { + return Err(Error::Store( + "Worker create startup failure column or byte bound is missing".into(), + )); + } + Ok(()) +} + +fn migrate_worker_create_failure_to_v124(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 123 { + return Err(Error::Store( + "Worker create failure migration requires schema 123".into(), + )); + } + storage_formats::verify_history(conn, 123)?; + let tx = conn.unchecked_transaction()?; + tx.execute_batch(include_str!("worker_create_failure_schema.sql"))?; + verify_worker_create_failure_schema(&tx)?; + tx.execute( + "INSERT INTO __yoi_schema_migrations(version,name) VALUES(124,?1)", + [WORKER_CREATE_FAILURE_MIGRATION_NAME], + )?; + tx.commit()?; + Ok(()) +} + fn migrate_backend_job_execution_to_v123(conn: &Connection) -> Result<()> { if current_schema_version(conn)? != 122 { return Err(Error::Store( @@ -16537,6 +16589,8 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { conn.execute_batch(content_history_store::SCHEMA)?; conn.execute_batch(crate::config_adoption::SCHEMA)?; backend_job_upgrade::install_fresh(conn)?; + conn.execute_batch(include_str!("worker_create_failure_schema.sql"))?; + verify_worker_create_failure_schema(conn)?; Ok(()) } @@ -16574,11 +16628,14 @@ fn migrate_plugin_plans_v106_to_v107(conn: &Connection) -> Result<()> { // Fresh composition uses the posts contract. Retained fixtures pass frozen host // and component DDL to the shared cross-domain constraint composition below. fn create_workspace_schema_structure(conn: &Connection) -> Result<()> { - create_workspace_schema_structure_from_sql( - conn, - &backend_job_upgrade::retained_checks(include_str!("latest_schema.sql")), - true, - )?; + // The pre-124 reference composition must not already contain the column + // added by migration 124. Fresh latest creation installs it after this prefix. + let retained_schema = backend_job_upgrade::retained_checks(include_str!("latest_schema.sql")) + .replace( + " startup_failure_json TEXT CHECK (startup_failure_json IS NULL OR\n (typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768)),\n", + "", + ); + create_workspace_schema_structure_from_sql(conn, &retained_schema, true)?; conn.execute_batch(include_str!("activation_schema.sql"))?; conn.execute_batch(include_str!("passkey_results_schema.sql"))?; conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?; @@ -17672,6 +17729,10 @@ fn table_columns(conn: &Connection, table_name: &str) -> Result> { #[path = "storage_owner_tests.rs"] mod storage_owner_tests; +#[cfg(test)] +#[path = "worker_create_failure_store_tests.rs"] +mod worker_create_failure_store_tests; + #[cfg(test)] #[path = "store_test_support.rs"] mod test_support; diff --git a/crates/workspace-server/src/subject_assignment_migration_tests.rs b/crates/workspace-server/src/subject_assignment_migration_tests.rs index 4323c1604..d8a758519 100644 --- a/crates/workspace-server/src/subject_assignment_migration_tests.rs +++ b/crates/workspace-server/src/subject_assignment_migration_tests.rs @@ -852,8 +852,6 @@ mod subject_assignment_migration_tests { }; const PLAN_NAME: &str = "Immutable Worker Plugin plans and retained pins"; - const SUBJECT_NAME: &str = - "Subject Ticket responsibility and current-body notification routing"; type Rows = Vec>; fn raw(path: &Path) -> Connection { @@ -1038,8 +1036,23 @@ mod subject_assignment_migration_tests { } else { String::new() }; + // Keep append identities for rowid tables; current content + // history tables are WITHOUT ROWID, so compare every column + // in deterministic value order instead of inventing rowids. + let without_rowid: bool = conn + .query_row( + "SELECT wr FROM pragma_table_list WHERE schema='main' AND name=?1", + [&table], + |row| row.get(0), + ) + .unwrap(); + let (projection, order) = if without_rowid { + (columns.clone(), columns) + } else { + (format!("rowid,{columns}"), "rowid".into()) + }; let query = format!( - "SELECT rowid,{columns} FROM \"{table}\"{sequence_filter} ORDER BY rowid" + "SELECT {projection} FROM \"{table}\"{sequence_filter} ORDER BY {order}" ); let rows = fixture_query_rows(conn, &query); (query, rows) @@ -1120,49 +1133,15 @@ mod subject_assignment_migration_tests { } fn expected_tail(version: i64) -> Vec<(i64, String)> { - let mut tail = match version { - 96 => vec![ - ( - 97, - "MR notifications and authenticated implementation results".into(), - ), - (98, "Assignment-independent implementation reports".into()), - ( - 99, - "Explicit hosting administrators and self-registration policy".into(), - ), - ( - 100, - "Provisional users and scoped passkey activation".into(), - ), - (101, "Durable explicit Subject body release".into()), - (102, "Durable failed Subject creation recovery".into()), - (103, "Passkey metadata and atomic addition results".into()), - ( - 104, - "Converge Passkey results and Job input retention".into(), - ), - ( - 105, - "Durable Subject deletion snapshots and Job admission fences".into(), - ), - (106, crate::plugin_catalog::MIGRATION_NAME.into()), - (107, PLAN_NAME.into()), - (108, SUBJECT_NAME.into()), - ], - 106 => vec![(107, PLAN_NAME.into()), (108, SUBJECT_NAME.into())], - 107 => vec![(108, SUBJECT_NAME.into())], - _ => panic!("unsupported fixture origin"), - }; - tail.push((109, "Durable Plugin Host invocations and HTTPS effect receipts".into())); - tail.push((110, "Worker Plugin launch selection associations".into())); - tail.push(( - 111, - "Typed Plugin Ticket extension data and atomic mutation receipts".into(), - )); - tail.push((112, "Ticket Service outbox and fenced consumers".into())); - tail.push((113, "Ticket work-decomposition parent links".into())); - tail + assert!( + matches!(version, 96 | 106 | 107), + "unsupported fixture origin" + ); + MIGRATIONS + .iter() + .filter(|migration| migration.version > version) + .map(|migration| (migration.version, migration.name.into())) + .collect() } fn plan_steps(version: i64) -> Vec { diff --git a/crates/workspace-server/src/ticket_post_upgrade_tests.rs b/crates/workspace-server/src/ticket_post_upgrade_tests.rs index db803191c..4a9ce8a89 100644 --- a/crates/workspace-server/src/ticket_post_upgrade_tests.rs +++ b/crates/workspace-server/src/ticket_post_upgrade_tests.rs @@ -56,12 +56,17 @@ fn ticket_posts_schema93_preserves_initial_body_creator_comments_and_archived_au drop(conn); let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap(); assert_eq!(plan.current_schema_version, 93); + assert_eq!(plan.target_schema_version, LATEST_SCHEMA_VERSION); assert_eq!( - plan.migrations + plan.migrations, + MIGRATIONS .iter() - .map(|step| step.version) - .collect::>(), - (94..=LATEST_SCHEMA_VERSION).collect::>() + .filter(|migration| migration.version > 93) + .map(|migration| WorkspaceSchemaMigrationStep { + version: migration.version, + name: migration.name.into(), + }) + .collect::>() ); let conn = Connection::open(&path).unwrap(); assert!( diff --git a/crates/workspace-server/src/worker_create_failure_schema.sql b/crates/workspace-server/src/worker_create_failure_schema.sql new file mode 100644 index 000000000..5aa9a1255 --- /dev/null +++ b/crates/workspace-server/src/worker_create_failure_schema.sql @@ -0,0 +1,3 @@ +ALTER TABLE worker_create_reservations ADD COLUMN startup_failure_json TEXT + CHECK (startup_failure_json IS NULL OR + (typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768)); diff --git a/crates/workspace-server/src/worker_create_failure_store.rs b/crates/workspace-server/src/worker_create_failure_store.rs new file mode 100644 index 000000000..4e81d8196 --- /dev/null +++ b/crates/workspace-server/src/worker_create_failure_store.rs @@ -0,0 +1,98 @@ +//! Backend coordinator returned startup results belong to the existing create +//! reservation. Runtime primary failure remains in Session metadata. Saving or +//! reading this evidence neither transitions the reservation nor grants retry. + +use runtime_api::startup::{DecodeStoredWorkerCreateFailure, decode_stored_worker_create_failure}; +use rusqlite::{OptionalExtension, params}; +use worker_runtime::identity::WorkerId; + +use crate::store::{SqliteWorkspaceStore, WORKER_CREATE_FAILURE_MAX_BYTES}; +use crate::{Error, Result}; + +impl SqliteWorkspaceStore { + #[allow(clippy::too_many_arguments)] + pub(crate) fn save_worker_create_failure( + &self, + workspace_id: &str, + runtime_id: &str, + worker_id: WorkerId, + expected_create_fingerprint: &str, + expected_previous: Option<&str>, + encoded: &str, + ) -> Result { + // Bound the wire representation before the typed decoder allocates JSON. + if encoded.len() > WORKER_CREATE_FAILURE_MAX_BYTES + || expected_previous.is_some_and(|value| value.len() > WORKER_CREATE_FAILURE_MAX_BYTES) + { + return Err(Error::InvalidInput( + "Worker create failure exceeds storage byte bound".into(), + )); + } + if !matches!( + decode_stored_worker_create_failure(encoded), + DecodeStoredWorkerCreateFailure::Known(_) + ) { + return Err(Error::InvalidInput( + "Worker create failure is not a known storage representation".into(), + )); + } + self.with_immediate_transaction(|tx| { + // Identity, immutable create intent and exact prior bytes are checked + // in the same UPDATE. Removed reservations deliberately remain eligible. + let changed = tx.execute( + "UPDATE worker_create_reservations SET startup_failure_json=?6 + WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 + AND create_fingerprint=?4 AND startup_failure_json IS ?5", + params![ + workspace_id, + runtime_id, + worker_id.to_string(), + expected_create_fingerprint, + expected_previous, + encoded + ], + )?; + Ok(changed == 1) + }) + } + + pub(crate) fn worker_create_failure( + &self, + workspace_id: &str, + runtime_id: &str, + worker_id: WorkerId, + ) -> Result> { + self.with_conn(|conn| { + // Return raw bounded evidence, including unknown/corrupt versions; + // classification belongs to the typed reader, not legacy prose. + let saved = conn + .query_row( + "SELECT CASE WHEN length(CAST(startup_failure_json AS BLOB)) <= ?4 + THEN startup_failure_json ELSE NULL END, + length(CAST(startup_failure_json AS BLOB)) + FROM worker_create_reservations + WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3", + params![ + workspace_id, + runtime_id, + worker_id.to_string(), + WORKER_CREATE_FAILURE_MAX_BYTES as i64 + ], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + )) + }, + ) + .optional()?; + match saved { + Some((_, Some(bytes))) if bytes > WORKER_CREATE_FAILURE_MAX_BYTES as i64 => Err( + Error::Store("Stored Worker create failure exceeds byte bound".into()), + ), + Some((value, _)) => Ok(value), + None => Ok(None), + } + }) + } +} diff --git a/crates/workspace-server/src/worker_create_failure_store_tests.rs b/crates/workspace-server/src/worker_create_failure_store_tests.rs new file mode 100644 index 000000000..f95873508 --- /dev/null +++ b/crates/workspace-server/src/worker_create_failure_store_tests.rs @@ -0,0 +1,475 @@ +use super::*; +use runtime_api::startup::{ + CompensationOutcome, CreationAcceptance, DecodeStoredWorkerCreateFailure, ExecutionOutcome, + FinalizeFacts, FinalizeStage, StartupBoundaryCause, StartupBoundaryReason, StartupCause, + StartupOperationFacts, WorkerCreateFailure, decode_stored_worker_create_failure, +}; +use rusqlite::types::Value; + +const WORKSPACE: &str = "failure-workspace"; +const RUNTIME: &str = "failure-runtime"; + +fn encoded_failure(compensation: CompensationOutcome) -> String { + let failure = WorkerCreateFailure::from_cause( + StartupCause::Boundary(StartupBoundaryCause { + reason: StartupBoundaryReason::FinalizeFailed, + }), + StartupOperationFacts { + acceptance: CreationAcceptance::Accepted, + execution: ExecutionOutcome::Accepted, + finalize: Some(FinalizeFacts { + stage: FinalizeStage::CreateReservation, + compensation, + }), + backend_compensation: Some(compensation), + correlation: None, + }, + ) + .unwrap(); + failure.encode_storage().unwrap() +} + +fn reserve(store: &SqliteWorkspaceStore) -> WorkerCreateReservation { + store.with_conn(|conn| { + conn.execute_batch("INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) + VALUES('failure-owner','user','failure-owner','Owner','1','1'); + INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) + VALUES('failure-workspace','failure-owner','Failure workspace','active','1','1'); + INSERT INTO workspace_memory_settings(workspace_id,language,created_at,updated_at) + VALUES('failure-workspace','en','1','1');")?; + Ok(()) + }).unwrap(); + store + .reserve_worker_create( + WORKSPACE, + RUNTIME, + "manual:startup-failure", + "sha256:startup-failure", + Some("subjektiv:failed-create"), + &store.get_workspace_memory_settings(WORKSPACE).unwrap(), + ) + .unwrap() +} + +// Snapshot every non-result reservation field plus resource keys and leases, so +// a save cannot silently acquire lifecycle effects as new columns are added. +fn non_result_snapshot(store: &SqliteWorkspaceStore) -> Vec> { + store.with_conn(|conn| { + let columns = table_columns(conn, "worker_create_reservations")? + .into_iter() + .filter(|name| name != "startup_failure_json") + .map(|name| format!("\"{name}\"")) + .collect::>() + .join(","); + let queries = [ + format!("SELECT {columns} FROM worker_create_reservations ORDER BY workspace_id,allocation_key"), + "SELECT * FROM workspace_resource_keys ORDER BY workspace_id,resource_kind,resource_id".into(), + "SELECT * FROM workspace_resource_key_counters ORDER BY workspace_id,resource_kind".into(), + "SELECT * FROM worker_singleton_owners ORDER BY workspace_id,singleton_key".into(), + ]; + let mut values = Vec::new(); + for query in queries { + let mut statement = conn.prepare(&query)?; + let count = statement.column_count(); + values.extend(statement.query_map([], |row| { + (0..count).map(|i| row.get::<_, Value>(i)).collect::>>() + })?.collect::>>()?); + } + Ok(values) + }).unwrap() +} + +fn save( + store: &SqliteWorkspaceStore, + reservation: &WorkerCreateReservation, + previous: Option<&str>, + encoded: &str, +) -> Result { + store.save_worker_create_failure( + WORKSPACE, + RUNTIME, + reservation.worker_id, + &reservation.create_fingerprint, + previous, + encoded, + ) +} + +#[test] +fn worker_create_failure_survives_restart_and_compensation_without_changing_lifecycle() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("server.db"); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let reservation = reserve(&store); + assert_eq!( + store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap(), + None + ); + let original = non_result_snapshot(&store); + let encoded = encoded_failure(CompensationOutcome::Unknown); + assert!(save(&store, &reservation, None, &encoded).unwrap()); + assert_eq!(non_result_snapshot(&store), original); + drop(store); + + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + assert_eq!( + store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap() + .as_deref(), + Some(encoded.as_str()) + ); + store + .fail_worker_create_reservation( + WORKSPACE, + RUNTIME, + reservation.worker_id, + &reservation.create_fingerprint, + ) + .unwrap(); + assert_eq!( + store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap() + .as_deref(), + Some(encoded.as_str()) + ); + let removed = non_result_snapshot(&store); + let compensated = encoded_failure(CompensationOutcome::Completed); + assert!(save(&store, &reservation, Some(&encoded), &compensated).unwrap()); + assert_eq!(non_result_snapshot(&store), removed); + drop(store); + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let raw = store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap() + .unwrap(); + assert_eq!(raw, compensated); + let DecodeStoredWorkerCreateFailure::Known(failure) = decode_stored_worker_create_failure(&raw) + else { + panic!("saved result must decode") + }; + assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted); + assert_eq!( + failure.operation().finalize.unwrap().compensation, + CompensationOutcome::Completed + ); +} + +#[test] +fn worker_create_failure_identity_and_exact_previous_bytes_fence_every_write() { + let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + let reservation = reserve(&store); + let first = encoded_failure(CompensationOutcome::Unknown); + let next = encoded_failure(CompensationOutcome::Completed); + assert!(save(&store, &reservation, None, &first).unwrap()); + let original = non_result_snapshot(&store); + for (workspace, runtime, worker, fingerprint, previous) in [ + ( + "other", + RUNTIME, + reservation.worker_id, + reservation.create_fingerprint.as_str(), + Some(first.as_str()), + ), + ( + WORKSPACE, + "other", + reservation.worker_id, + reservation.create_fingerprint.as_str(), + Some(first.as_str()), + ), + ( + WORKSPACE, + RUNTIME, + WorkerId::now_v7(), + reservation.create_fingerprint.as_str(), + Some(first.as_str()), + ), + ( + WORKSPACE, + RUNTIME, + reservation.worker_id, + "wrong", + Some(first.as_str()), + ), + ( + WORKSPACE, + RUNTIME, + reservation.worker_id, + reservation.create_fingerprint.as_str(), + None, + ), + ( + WORKSPACE, + RUNTIME, + reservation.worker_id, + reservation.create_fingerprint.as_str(), + Some(next.as_str()), + ), + ] { + assert!( + !store + .save_worker_create_failure( + workspace, + runtime, + worker, + fingerprint, + previous, + &next + ) + .unwrap() + ); + assert_eq!( + store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap() + .as_deref(), + Some(first.as_str()) + ); + } + let equivalent_but_different_bytes = format!("{first} "); + assert!( + !save( + &store, + &reservation, + Some(&equivalent_but_different_bytes), + &next + ) + .unwrap() + ); + for (workspace, runtime, worker) in [ + ("other", RUNTIME, reservation.worker_id), + (WORKSPACE, "other", reservation.worker_id), + (WORKSPACE, RUNTIME, WorkerId::now_v7()), + ] { + assert_eq!( + store + .worker_create_failure(workspace, runtime, worker) + .unwrap(), + None + ); + } + assert_eq!(non_result_snapshot(&store), original); + assert!(save(&store, &reservation, Some(&first), &next).unwrap()); +} + +#[test] +fn worker_create_failure_failed_write_rolls_back_result_and_all_other_fields() { + let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + let reservation = reserve(&store); + let first = encoded_failure(CompensationOutcome::Unknown); + let next = encoded_failure(CompensationOutcome::Completed); + assert!(save(&store, &reservation, None, &first).unwrap()); + let original = non_result_snapshot(&store); + store.with_conn(|conn| { + conn.execute_batch("CREATE TRIGGER interrupt_startup_failure AFTER UPDATE OF startup_failure_json ON worker_create_reservations + BEGIN UPDATE worker_create_reservations SET state='removed'; + SELECT RAISE(ABORT,'injected startup failure write interruption'); END;")?; + Ok(()) + }).unwrap(); + assert!(save(&store, &reservation, Some(&first), &next).is_err()); + assert_eq!(non_result_snapshot(&store), original); + assert_eq!( + store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap() + .as_deref(), + Some(first.as_str()) + ); + store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER interrupt_startup_failure")?; + Ok(()) + }) + .unwrap(); + assert!(save(&store, &reservation, Some(&first), &next).unwrap()); +} + +#[test] +fn worker_create_failure_rejects_unknown_corrupt_and_oversized_writes_but_reads_raw_history() { + let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + let reservation = reserve(&store); + let first = encoded_failure(CompensationOutcome::Unknown); + assert!(save(&store, &reservation, None, &first).unwrap()); + let mut unknown: serde_json::Value = serde_json::from_str(&first).unwrap(); + unknown["version"] = serde_json::json!(65535); + let unknown = serde_json::to_string(&unknown).unwrap(); + assert!(matches!( + decode_stored_worker_create_failure(&unknown), + DecodeStoredWorkerCreateFailure::Unsupported + )); + for invalid in [ + unknown.as_str(), + "not JSON", + &" ".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1), + ] { + assert!(save(&store, &reservation, Some(&first), invalid).is_err()); + assert_eq!( + store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap() + .as_deref(), + Some(first.as_str()) + ); + } + // Simulate a retained newer writer/corrupt row, without interpreting prose. + for raw in [unknown.as_str(), "not JSON"] { + store + .with_conn(|conn| { + conn.execute( + "UPDATE worker_create_reservations SET startup_failure_json=?1", + [raw], + )?; + Ok(()) + }) + .unwrap(); + assert_eq!( + store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap() + .as_deref(), + Some(raw) + ); + } + assert!(matches!( + decode_stored_worker_create_failure("not JSON"), + DecodeStoredWorkerCreateFailure::Corrupt(_) + )); + store + .with_conn(|conn| { + conn.execute_batch("PRAGMA ignore_check_constraints=ON;")?; + conn.execute( + "UPDATE worker_create_reservations SET startup_failure_json=?1", + ["x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1)], + )?; + conn.execute_batch("PRAGMA ignore_check_constraints=OFF;")?; + Ok(()) + }) + .unwrap(); + assert!( + store + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .is_err() + ); +} + +fn result_column_signature(conn: &Connection) -> Vec<(String, String, bool, Option)> { + conn.prepare("SELECT name,type,\"notnull\",dflt_value FROM pragma_table_info('worker_create_reservations') ORDER BY name").unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))).unwrap() + .collect::>>().unwrap() +} + +fn assert_result_constraints(conn: &Connection, worker: WorkerId) { + for accepted in [ + "x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES), + "é".repeat(WORKER_CREATE_FAILURE_MAX_BYTES / 2), + ] { + conn.execute( + "UPDATE worker_create_reservations SET startup_failure_json=?1 WHERE worker_id=?2", + params![accepted, worker.to_string()], + ) + .unwrap(); + } + for rejected in [ + "x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1), + "é".repeat(WORKER_CREATE_FAILURE_MAX_BYTES / 2 + 1), + ] { + assert!( + conn.execute( + "UPDATE worker_create_reservations SET startup_failure_json=?1 WHERE worker_id=?2", + params![rejected, worker.to_string()] + ) + .is_err() + ); + } + assert!( + conn.execute( + "UPDATE worker_create_reservations SET startup_failure_json=x'00' WHERE worker_id=?1", + [worker.to_string()] + ) + .is_err() + ); + conn.execute( + "UPDATE worker_create_reservations SET startup_failure_json=NULL WHERE worker_id=?1", + [worker.to_string()], + ) + .unwrap(); + verify_worker_create_failure_schema(conn).unwrap(); +} + +#[test] +fn worker_create_failure_oldest_upgrade_preserves_legacy_null_and_matches_fresh_constraints() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("oldest.db"); + tests::prepare_schema_v50(&path, Some("workspace-a")); + let legacy_worker = WorkerId::now_v7(); + let conn = Connection::open(&path).unwrap(); + conn.execute("INSERT INTO worker_create_reservations(workspace_id,allocation_key,worker_id,runtime_id,create_fingerprint,state,created_at,updated_at) + VALUES('workspace-a','legacy-allocation',?1,'shared','legacy-fingerprint','reserved','1','1')", [legacy_worker.to_string()]).unwrap(); + drop(conn); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + assert_eq!( + upgraded + .worker_create_failure("workspace-a", "shared", legacy_worker) + .unwrap(), + None + ); + upgraded.with_conn(|conn| { + let saved: (String, String, String) = conn.query_row("SELECT create_fingerprint,state,created_at FROM worker_create_reservations WHERE worker_id=?1", [legacy_worker.to_string()], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?; + assert_eq!(saved, ("legacy-fingerprint".into(), "reserved".into(), "1".into())); + assert_result_constraints(conn, legacy_worker); + Ok(()) + }).unwrap(); + let reservation = reserve(&upgraded); + let encoded = encoded_failure(CompensationOutcome::Residual); + assert!(save(&upgraded, &reservation, None, &encoded).unwrap()); + drop(upgraded); + let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + assert_eq!( + upgraded + .worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id) + .unwrap() + .as_deref(), + Some(encoded.as_str()) + ); + let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let reservation = reserve(&fresh); + fresh + .with_conn(|conn| { + assert_result_constraints(conn, reservation.worker_id); + Ok(()) + }) + .unwrap(); + assert_eq!( + upgraded + .with_conn(|conn| Ok(result_column_signature(conn))) + .unwrap(), + fresh + .with_conn(|conn| Ok(result_column_signature(conn))) + .unwrap() + ); +} + +#[test] +fn worker_create_failure_migration_failure_rolls_back_column_and_receipt() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("upgrade.db"); + tests::prepare_retained_schema(&path, 123); + let conn = Connection::open(&path).unwrap(); + configure_sqlite(&conn).unwrap(); + conn.execute_batch("CREATE TRIGGER interrupt_startup_failure_migration BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=124 + BEGIN SELECT RAISE(ABORT,'injected startup failure migration interruption'); END;").unwrap(); + assert!(migrate_worker_create_failure_to_v124(&conn).is_err()); + assert_eq!(current_schema_version(&conn).unwrap(), 123); + assert!(!column_exists(&conn, "worker_create_reservations", "startup_failure_json").unwrap()); + conn.execute_batch("DROP TRIGGER interrupt_startup_failure_migration") + .unwrap(); + migrate_worker_create_failure_to_v124(&conn).unwrap(); + verify_worker_create_failure_schema(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 124); + drop(conn); + SqliteWorkspaceStore::open_for_test(&path).unwrap(); +}