fix(web): align startup consumer with corrected T812 Rust boundary
This commit is contained in:
Generated
+5
@@ -199,6 +199,7 @@ dependencies = [
|
|||||||
"api-macros-impl",
|
"api-macros-impl",
|
||||||
"axum",
|
"axum",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
"error-contract",
|
||||||
"futures",
|
"futures",
|
||||||
"oas3",
|
"oas3",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
@@ -4325,12 +4326,15 @@ name = "runtime-api"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"api-macros",
|
"api-macros",
|
||||||
|
"error-contract",
|
||||||
"manifest",
|
"manifest",
|
||||||
"plugin-plan",
|
"plugin-plan",
|
||||||
"protocol",
|
"protocol",
|
||||||
|
"schemars",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"session-store",
|
"session-store",
|
||||||
|
"sha2 0.11.0",
|
||||||
"tokio",
|
"tokio",
|
||||||
"workdir",
|
"workdir",
|
||||||
]
|
]
|
||||||
@@ -4764,6 +4768,7 @@ name = "server-api"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"api-macros",
|
"api-macros",
|
||||||
|
"error-contract",
|
||||||
"feature-permissions",
|
"feature-permissions",
|
||||||
"memory",
|
"memory",
|
||||||
"merge-request",
|
"merge-request",
|
||||||
|
|||||||
+262
-113
@@ -53,6 +53,7 @@ enum Method {
|
|||||||
#[derive(Clone, Copy, Default)]
|
#[derive(Clone, Copy, Default)]
|
||||||
struct AdapterConfig {
|
struct AdapterConfig {
|
||||||
reqwest: bool,
|
reqwest: bool,
|
||||||
|
client_error_hook: bool,
|
||||||
axum: bool,
|
axum: bool,
|
||||||
openapi: bool,
|
openapi: bool,
|
||||||
}
|
}
|
||||||
@@ -64,9 +65,10 @@ impl Parse for AdapterConfig {
|
|||||||
for adapter in adapters {
|
for adapter in adapters {
|
||||||
match adapter.to_string().as_str() {
|
match adapter.to_string().as_str() {
|
||||||
"reqwest" if !config.reqwest => config.reqwest = true,
|
"reqwest" if !config.reqwest => config.reqwest = true,
|
||||||
|
"client_error_hook" if !config.client_error_hook => config.client_error_hook = true,
|
||||||
"axum" if !config.axum => config.axum = true,
|
"axum" if !config.axum => config.axum = true,
|
||||||
"openapi" if !config.openapi => config.openapi = true,
|
"openapi" if !config.openapi => config.openapi = true,
|
||||||
"reqwest" | "axum" | "openapi" => {
|
"reqwest" | "axum" | "openapi" | "client_error_hook" => {
|
||||||
return Err(syn::Error::new(adapter.span(), "duplicate API adapter"));
|
return Err(syn::Error::new(adapter.span(), "duplicate API adapter"));
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
@@ -77,6 +79,12 @@ impl Parse for AdapterConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if config.client_error_hook && !config.reqwest {
|
||||||
|
return Err(syn::Error::new(
|
||||||
|
Span::call_site(),
|
||||||
|
"`client_error_hook` requires the `reqwest` adapter",
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(config)
|
Ok(config)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -121,6 +129,8 @@ struct RouteArgs {
|
|||||||
responses: Vec<DeclaredResponse>,
|
responses: Vec<DeclaredResponse>,
|
||||||
error_status: Option<LitInt>,
|
error_status: Option<LitInt>,
|
||||||
additional_error_statuses: Vec<LitInt>,
|
additional_error_statuses: Vec<LitInt>,
|
||||||
|
additional_error_content_types: Option<Vec<LitStr>>,
|
||||||
|
client_error_hook: Option<syn::LitBool>,
|
||||||
bearer_auth: Option<syn::LitBool>,
|
bearer_auth: Option<syn::LitBool>,
|
||||||
browser_auth: Option<syn::LitBool>,
|
browser_auth: Option<syn::LitBool>,
|
||||||
normalize_body_errors: Option<syn::LitBool>,
|
normalize_body_errors: Option<syn::LitBool>,
|
||||||
@@ -310,6 +320,8 @@ impl Parse for RouteArgs {
|
|||||||
responses: Vec::new(),
|
responses: Vec::new(),
|
||||||
error_status: None,
|
error_status: None,
|
||||||
additional_error_statuses: Vec::new(),
|
additional_error_statuses: Vec::new(),
|
||||||
|
additional_error_content_types: None,
|
||||||
|
client_error_hook: None,
|
||||||
bearer_auth: None,
|
bearer_auth: None,
|
||||||
browser_auth: None,
|
browser_auth: None,
|
||||||
normalize_body_errors: None,
|
normalize_body_errors: None,
|
||||||
@@ -379,6 +391,25 @@ impl Parse for RouteArgs {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
"additional_error_content_types" => {
|
||||||
|
let content;
|
||||||
|
syn::bracketed!(content in input);
|
||||||
|
let values = Punctuated::<LitStr, Token![,]>::parse_terminated(&content)?
|
||||||
|
.into_iter()
|
||||||
|
.collect();
|
||||||
|
set_once(
|
||||||
|
&mut result.additional_error_content_types,
|
||||||
|
values,
|
||||||
|
&key,
|
||||||
|
"additional_error_content_types",
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
"client_error_hook" => set_once(
|
||||||
|
&mut result.client_error_hook,
|
||||||
|
input.parse::<syn::LitBool>()?,
|
||||||
|
&key,
|
||||||
|
"client_error_hook",
|
||||||
|
)?,
|
||||||
"bearer_auth" => set_once(
|
"bearer_auth" => set_once(
|
||||||
&mut result.bearer_auth,
|
&mut result.bearer_auth,
|
||||||
input.parse::<syn::LitBool>()?,
|
input.parse::<syn::LitBool>()?,
|
||||||
@@ -406,7 +437,7 @@ impl Parse for RouteArgs {
|
|||||||
_ => {
|
_ => {
|
||||||
return Err(syn::Error::new(
|
return Err(syn::Error::new(
|
||||||
key.span(),
|
key.span(),
|
||||||
"unsupported route option; expected operation_id, status, alternate_status, responses, error_status, additional_error_statuses, bearer_auth, browser_auth, normalize_body_errors, or openapi",
|
"unsupported route option; expected operation_id, status, alternate_status, responses, error_status, additional_error_statuses, additional_error_content_types, client_error_hook, bearer_auth, browser_auth, normalize_body_errors, or openapi",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -502,6 +533,8 @@ struct Operation {
|
|||||||
alternate_status: Option<u16>,
|
alternate_status: Option<u16>,
|
||||||
error_status: Option<u16>,
|
error_status: Option<u16>,
|
||||||
additional_error_statuses: Vec<u16>,
|
additional_error_statuses: Vec<u16>,
|
||||||
|
additional_error_content_types: Vec<String>,
|
||||||
|
client_error_hook: bool,
|
||||||
bearer_auth: bool,
|
bearer_auth: bool,
|
||||||
browser_auth: bool,
|
browser_auth: bool,
|
||||||
normalize_body_errors: bool,
|
normalize_body_errors: bool,
|
||||||
@@ -546,7 +579,7 @@ fn normalize_api(
|
|||||||
for mut trait_item in std::mem::take(&mut item.items) {
|
for mut trait_item in std::mem::take(&mut item.items) {
|
||||||
let operation = match &mut trait_item {
|
let operation = match &mut trait_item {
|
||||||
TraitItem::Fn(method) => {
|
TraitItem::Fn(method) => {
|
||||||
let operation = normalize_operation(method)?;
|
let operation = normalize_operation(method, adapters)?;
|
||||||
retained_items.push(trait_item);
|
retained_items.push(trait_item);
|
||||||
operation
|
operation
|
||||||
}
|
}
|
||||||
@@ -778,6 +811,8 @@ fn normalize_websocket_operation(declaration: &mut syn::TraitItemType) -> syn::R
|
|||||||
alternate_status: None,
|
alternate_status: None,
|
||||||
error_status: None,
|
error_status: None,
|
||||||
additional_error_statuses: Vec::new(),
|
additional_error_statuses: Vec::new(),
|
||||||
|
additional_error_content_types: Vec::new(),
|
||||||
|
client_error_hook: false,
|
||||||
bearer_auth: false,
|
bearer_auth: false,
|
||||||
browser_auth: false,
|
browser_auth: false,
|
||||||
normalize_body_errors: false,
|
normalize_body_errors: false,
|
||||||
@@ -796,7 +831,10 @@ fn validate_frame_type(ty: &Type) -> syn::Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalize_operation(method: &mut TraitItemFn) -> syn::Result<Operation> {
|
fn normalize_operation(
|
||||||
|
method: &mut TraitItemFn,
|
||||||
|
adapters: AdapterConfig,
|
||||||
|
) -> syn::Result<Operation> {
|
||||||
validate_signature(method)?;
|
validate_signature(method)?;
|
||||||
|
|
||||||
let mut route = None;
|
let mut route = None;
|
||||||
@@ -845,6 +883,53 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result<Operation> {
|
|||||||
validate_named_body_type(ty, "error response")?;
|
validate_named_body_type(ty, "error response")?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let client_error_hook = route
|
||||||
|
.client_error_hook
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|value| value.value);
|
||||||
|
if client_error_hook {
|
||||||
|
let option = route.client_error_hook.as_ref().expect("enabled hook");
|
||||||
|
if !adapters.reqwest {
|
||||||
|
return Err(syn::Error::new_spanned(
|
||||||
|
option,
|
||||||
|
"client_error_hook requires the `reqwest` adapter",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if error_body.is_none() {
|
||||||
|
return Err(syn::Error::new_spanned(
|
||||||
|
option,
|
||||||
|
"client_error_hook requires a public error response type",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut additional_error_content_types = Vec::new();
|
||||||
|
if let Some(content_types) = &route.additional_error_content_types {
|
||||||
|
if error_body.is_none() {
|
||||||
|
return Err(syn::Error::new_spanned(
|
||||||
|
&route.path,
|
||||||
|
"additional_error_content_types require a public error response type",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if content_types.is_empty() {
|
||||||
|
return Err(syn::Error::new_spanned(
|
||||||
|
&route.path,
|
||||||
|
"additional_error_content_types must not be empty",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for content_type in content_types {
|
||||||
|
let value = validate_error_content_type(content_type)?;
|
||||||
|
if value == "application/json" || additional_error_content_types.contains(&value) {
|
||||||
|
return Err(syn::Error::new_spanned(
|
||||||
|
content_type,
|
||||||
|
"additional error content types must be distinct from each other and application/json",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
additional_error_content_types.push(value);
|
||||||
|
}
|
||||||
|
additional_error_content_types.sort();
|
||||||
|
}
|
||||||
|
|
||||||
let declared_responses = !route.responses.is_empty();
|
let declared_responses = !route.responses.is_empty();
|
||||||
if declared_responses && (route.status.is_some() || route.alternate_status.is_some()) {
|
if declared_responses && (route.status.is_some() || route.alternate_status.is_some()) {
|
||||||
return Err(syn::Error::new_spanned(
|
return Err(syn::Error::new_spanned(
|
||||||
@@ -1196,6 +1281,8 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result<Operation> {
|
|||||||
alternate_status,
|
alternate_status,
|
||||||
error_status,
|
error_status,
|
||||||
additional_error_statuses,
|
additional_error_statuses,
|
||||||
|
additional_error_content_types,
|
||||||
|
client_error_hook,
|
||||||
bearer_auth: route.bearer_auth.as_ref().is_some_and(|value| value.value),
|
bearer_auth: route.bearer_auth.as_ref().is_some_and(|value| value.value),
|
||||||
browser_auth: route.browser_auth.as_ref().is_some_and(|value| value.value),
|
browser_auth: route.browser_auth.as_ref().is_some_and(|value| value.value),
|
||||||
normalize_body_errors: route
|
normalize_body_errors: route
|
||||||
@@ -1206,6 +1293,43 @@ fn normalize_operation(method: &mut TraitItemFn) -> syn::Result<Operation> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Route media declarations are concrete type/subtype tokens, not media ranges or parameters.
|
||||||
|
fn validate_error_content_type(literal: &LitStr) -> syn::Result<String> {
|
||||||
|
let value = literal.value().to_ascii_lowercase();
|
||||||
|
let valid_token = |token: &str| {
|
||||||
|
!token.is_empty()
|
||||||
|
&& token.bytes().all(|byte| {
|
||||||
|
byte.is_ascii_alphanumeric()
|
||||||
|
|| matches!(
|
||||||
|
byte,
|
||||||
|
b'!' | b'#'
|
||||||
|
| b'$'
|
||||||
|
| b'%'
|
||||||
|
| b'&'
|
||||||
|
| b'\''
|
||||||
|
| b'+'
|
||||||
|
| b'-'
|
||||||
|
| b'.'
|
||||||
|
| b'^'
|
||||||
|
| b'_'
|
||||||
|
| b'`'
|
||||||
|
| b'|'
|
||||||
|
| b'~'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
};
|
||||||
|
if !value
|
||||||
|
.split_once('/')
|
||||||
|
.is_some_and(|(kind, subtype)| valid_token(kind) && valid_token(subtype))
|
||||||
|
{
|
||||||
|
return Err(syn::Error::new_spanned(
|
||||||
|
literal,
|
||||||
|
"error content type must be a concrete type/subtype without parameters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_signature(method: &TraitItemFn) -> syn::Result<()> {
|
fn validate_signature(method: &TraitItemFn) -> syn::Result<()> {
|
||||||
let sig = &method.sig;
|
let sig = &method.sig;
|
||||||
if sig.asyncness.is_none() {
|
if sig.asyncness.is_none() {
|
||||||
@@ -1815,17 +1939,27 @@ fn reqwest_adapter_tokens(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let public_error_decode = if api.adapters.client_error_hook || operation.client_error_hook {
|
||||||
|
quote! {
|
||||||
|
<#error_type as #api_crate::reqwest::HttpClientError>::decode_error(&__response)
|
||||||
|
.map_err(#api_crate::reqwest::ClientError::failure)?
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
quote! {
|
||||||
|
__response
|
||||||
|
.decode_json::<#error_type>(#api_crate::reqwest::DecodeKind::PublicError)
|
||||||
|
.map_err(|_| #api_crate::reqwest::ClientError::failure(
|
||||||
|
#api_crate::reqwest::ClientFailure::ErrorResponseDecode {
|
||||||
|
status: __status.as_u16(),
|
||||||
|
},
|
||||||
|
))?
|
||||||
|
}
|
||||||
|
};
|
||||||
let public_error = match operation.error_body.as_ref() {
|
let public_error = match operation.error_body.as_ref() {
|
||||||
Some(error) => quote! {
|
Some(_) => quote! {
|
||||||
if (400..=599).contains(&__actual) {
|
if (400..=599).contains(&__actual) {
|
||||||
let __status = __response.status();
|
let __status = __response.status();
|
||||||
let __error = __response
|
let __error = #public_error_decode;
|
||||||
.decode_json::<#error>(#api_crate::reqwest::DecodeKind::PublicError)
|
|
||||||
.map_err(|_| #api_crate::reqwest::ClientError::failure(
|
|
||||||
#api_crate::reqwest::ClientFailure::ErrorResponseDecode {
|
|
||||||
status: __status.as_u16(),
|
|
||||||
},
|
|
||||||
))?;
|
|
||||||
return ::core::result::Result::Err(
|
return ::core::result::Result::Err(
|
||||||
#api_crate::reqwest::ClientError::public(__status, __error),
|
#api_crate::reqwest::ClientError::public(__status, __error),
|
||||||
);
|
);
|
||||||
@@ -2098,7 +2232,7 @@ fn axum_adapter_tokens(
|
|||||||
status,
|
status,
|
||||||
rejection.body_text(),
|
rejection.body_text(),
|
||||||
);
|
);
|
||||||
return #api_crate::axum::error_response(
|
return #api_crate::axum::http_error_response(
|
||||||
#api_crate::axum::status(status),
|
#api_crate::axum::status(status),
|
||||||
error,
|
error,
|
||||||
);
|
);
|
||||||
@@ -2132,7 +2266,7 @@ fn axum_adapter_tokens(
|
|||||||
status,
|
status,
|
||||||
rejection.body_text(),
|
rejection.body_text(),
|
||||||
);
|
);
|
||||||
return #api_crate::axum::error_response(
|
return #api_crate::axum::http_error_response(
|
||||||
#api_crate::axum::status(status),
|
#api_crate::axum::status(status),
|
||||||
error,
|
error,
|
||||||
);
|
);
|
||||||
@@ -2218,7 +2352,7 @@ fn axum_adapter_tokens(
|
|||||||
let error = match operation.error_status {
|
let error = match operation.error_status {
|
||||||
Some(_) => quote!({
|
Some(_) => quote!({
|
||||||
let status = #api_crate::HttpError::status_code(&error);
|
let status = #api_crate::HttpError::status_code(&error);
|
||||||
#api_crate::axum::error_response(#api_crate::axum::status(status), error)
|
#api_crate::axum::http_error_response(#api_crate::axum::status(status), error)
|
||||||
}),
|
}),
|
||||||
None => quote!({ let _ = error; #api_crate::axum::empty_response(#api_crate::axum::framework::StatusCode::INTERNAL_SERVER_ERROR) }),
|
None => quote!({ let _ = error; #api_crate::axum::empty_response(#api_crate::axum::framework::StatusCode::INTERNAL_SERVER_ERROR) }),
|
||||||
};
|
};
|
||||||
@@ -2315,110 +2449,115 @@ fn openapi_adapter_tokens(
|
|||||||
matches!(operation.transport, Transport::Http) && !operation.openapi_skip
|
matches!(operation.transport, Transport::Http) && !operation.openapi_skip
|
||||||
})
|
})
|
||||||
.map(|operation| {
|
.map(|operation| {
|
||||||
let method = method_name(operation.method);
|
let method = method_name(operation.method);
|
||||||
let path = &operation.path;
|
let path = &operation.path;
|
||||||
let operation_id = &operation.operation_id;
|
let operation_id = &operation.operation_id;
|
||||||
let parameters = operation.parameters.iter().filter_map(|parameter| {
|
let parameters = operation.parameters.iter().filter_map(|parameter| {
|
||||||
if matches!(parameter.location, Location::Body | Location::Binary | Location::Extension) {
|
if matches!(
|
||||||
return None;
|
parameter.location,
|
||||||
}
|
Location::Body | Location::Binary | Location::Extension
|
||||||
let ty = ¶meter.ty;
|
) {
|
||||||
let name = ¶meter.wire_name;
|
return None;
|
||||||
let location = match parameter.location {
|
|
||||||
Location::Path => "path",
|
|
||||||
Location::Query => "query",
|
|
||||||
Location::Header => "header",
|
|
||||||
Location::Body | Location::Binary | Location::Extension => unreachable!(),
|
|
||||||
};
|
|
||||||
Some(quote! {
|
|
||||||
operation.parameter::<#ty>(#name, #location)?;
|
|
||||||
})
|
|
||||||
});
|
|
||||||
let request = operation.request_body.as_ref().map(|body| match body.wire_kind {
|
|
||||||
RequestWireKind::Json => {
|
|
||||||
let ty = &body.ty;
|
|
||||||
quote! {
|
|
||||||
operation.request_body::<#ty>("application/json")?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
RequestWireKind::Binary => quote! {
|
|
||||||
operation.binary_request_body()?;
|
|
||||||
},
|
|
||||||
});
|
|
||||||
let success_responses = operation.success_responses.iter().enumerate().map(|(index, response)| {
|
|
||||||
let status = response.status;
|
|
||||||
let description = if index == 0 {
|
|
||||||
"Successful response"
|
|
||||||
} else {
|
|
||||||
"Alternate successful response"
|
|
||||||
};
|
|
||||||
let body = match response.body.as_ref() {
|
|
||||||
Some(ty) if is_binary_body_type(ty) => quote! {
|
|
||||||
operation.binary_response(#status, #description)?;
|
|
||||||
},
|
|
||||||
Some(ty) => quote! {
|
|
||||||
operation.response::<#ty>(#status, "application/json", #description)?;
|
|
||||||
},
|
|
||||||
None => quote! {
|
|
||||||
operation.empty_response(#status, #description)?;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
let headers = response.headers.iter().map(|header| {
|
|
||||||
let ty = &header.ty;
|
|
||||||
let name = &header.wire_name;
|
|
||||||
quote! {
|
|
||||||
operation.response_header::<#ty>(#status, #name)?;
|
|
||||||
}
|
}
|
||||||
|
let ty = ¶meter.ty;
|
||||||
|
let name = ¶meter.wire_name;
|
||||||
|
let location = match parameter.location {
|
||||||
|
Location::Path => "path",
|
||||||
|
Location::Query => "query",
|
||||||
|
Location::Header => "header",
|
||||||
|
Location::Body | Location::Binary | Location::Extension => unreachable!(),
|
||||||
|
};
|
||||||
|
Some(quote! {
|
||||||
|
operation.parameter::<#ty>(#name, #location)?;
|
||||||
|
})
|
||||||
});
|
});
|
||||||
quote! {
|
let request = operation
|
||||||
#body
|
.request_body
|
||||||
#(#headers)*
|
.as_ref()
|
||||||
}
|
.map(|body| match body.wire_kind {
|
||||||
});
|
RequestWireKind::Json => {
|
||||||
let error = operation
|
let ty = &body.ty;
|
||||||
.error_status
|
|
||||||
.zip(operation.error_body.as_ref())
|
|
||||||
.map(|(status, ty)| {
|
|
||||||
quote! {
|
|
||||||
operation.response::<#ty>(#status, "application/json", "Error response")?;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let additional_errors = operation
|
|
||||||
.error_body
|
|
||||||
.as_ref()
|
|
||||||
.map(|ty| {
|
|
||||||
operation
|
|
||||||
.additional_error_statuses
|
|
||||||
.iter()
|
|
||||||
.map(|status| {
|
|
||||||
quote! {
|
quote! {
|
||||||
operation.response::<#ty>(#status, "application/json", "Error response")?;
|
operation.request_body::<#ty>("application/json")?;
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
.collect::<Vec<_>>()
|
RequestWireKind::Binary => quote! {
|
||||||
})
|
operation.binary_request_body()?;
|
||||||
.unwrap_or_default();
|
},
|
||||||
let bearer_auth = operation.bearer_auth.then(|| {
|
});
|
||||||
quote! { operation.bearer_authentication(); }
|
let success_responses = operation.success_responses.iter().enumerate().map(
|
||||||
});
|
|(index, response)| {
|
||||||
let browser_auth = operation.browser_auth.then(|| {
|
let status = response.status;
|
||||||
quote! { operation.browser_authentication(); }
|
let description = if index == 0 {
|
||||||
});
|
"Successful response"
|
||||||
|
} else {
|
||||||
|
"Alternate successful response"
|
||||||
|
};
|
||||||
|
let body = match response.body.as_ref() {
|
||||||
|
Some(ty) if is_binary_body_type(ty) => quote! {
|
||||||
|
operation.binary_response(#status, #description)?;
|
||||||
|
},
|
||||||
|
Some(ty) => quote! {
|
||||||
|
operation.response::<#ty>(#status, "application/json", #description)?;
|
||||||
|
},
|
||||||
|
None => quote! {
|
||||||
|
operation.empty_response(#status, #description)?;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let headers = response.headers.iter().map(|header| {
|
||||||
|
let ty = &header.ty;
|
||||||
|
let name = &header.wire_name;
|
||||||
|
quote! {
|
||||||
|
operation.response_header::<#ty>(#status, #name)?;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
quote! {
|
||||||
|
#body
|
||||||
|
#(#headers)*
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let errors = operation
|
||||||
|
.error_body
|
||||||
|
.as_ref()
|
||||||
|
.map(|ty| {
|
||||||
|
let additional_content_types = &operation.additional_error_content_types;
|
||||||
|
operation
|
||||||
|
.error_status
|
||||||
|
.into_iter()
|
||||||
|
.chain(operation.additional_error_statuses.iter().copied())
|
||||||
|
.map(|status| {
|
||||||
|
quote! {
|
||||||
|
operation.response_content_types::<#ty>(
|
||||||
|
#status,
|
||||||
|
&["application/json", #(#additional_content_types),*],
|
||||||
|
"Error response",
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let bearer_auth = operation.bearer_auth.then(|| {
|
||||||
|
quote! { operation.bearer_authentication(); }
|
||||||
|
});
|
||||||
|
let browser_auth = operation.browser_auth.then(|| {
|
||||||
|
quote! { operation.browser_authentication(); }
|
||||||
|
});
|
||||||
|
|
||||||
quote! {
|
quote! {
|
||||||
{
|
{
|
||||||
let mut operation = builder.operation(#method, #path, #operation_id)?;
|
let mut operation = builder.operation(#method, #path, #operation_id)?;
|
||||||
#(#parameters)*
|
#(#parameters)*
|
||||||
#bearer_auth
|
#bearer_auth
|
||||||
#browser_auth
|
#browser_auth
|
||||||
#request
|
#request
|
||||||
#(#success_responses)*
|
#(#success_responses)*
|
||||||
#error
|
#(#errors)*
|
||||||
#(#additional_errors)*
|
operation.finish()?;
|
||||||
operation.finish()?;
|
}
|
||||||
}
|
}
|
||||||
}
|
});
|
||||||
});
|
|
||||||
|
|
||||||
quote! {
|
quote! {
|
||||||
#[doc = "Build this trait's deterministic, deployment-independent OpenAPI 3.1 document."]
|
#[doc = "Build this trait's deterministic, deployment-independent OpenAPI 3.1 document."]
|
||||||
@@ -2660,6 +2799,13 @@ fn metadata_tokens(
|
|||||||
} else {
|
} else {
|
||||||
quote!(#api_crate::WireKind::Empty)
|
quote!(#api_crate::WireKind::Empty)
|
||||||
};
|
};
|
||||||
|
let content_types = if response.body.as_ref().is_some_and(is_binary_body_type) {
|
||||||
|
quote!(&["application/octet-stream"])
|
||||||
|
} else if response.body.is_some() {
|
||||||
|
quote!(&["application/json"])
|
||||||
|
} else {
|
||||||
|
quote!(&[])
|
||||||
|
};
|
||||||
let headers = response.headers.iter().map(|header| {
|
let headers = response.headers.iter().map(|header| {
|
||||||
let wire_name = &header.wire_name;
|
let wire_name = &header.wire_name;
|
||||||
let ty = &header.ty;
|
let ty = &header.ty;
|
||||||
@@ -2674,6 +2820,7 @@ fn metadata_tokens(
|
|||||||
#api_crate::ResponseMetadata {
|
#api_crate::ResponseMetadata {
|
||||||
status: #status,
|
status: #status,
|
||||||
body: #api_crate::BodyMetadata { wire_kind: #wire_kind },
|
body: #api_crate::BodyMetadata { wire_kind: #wire_kind },
|
||||||
|
content_types: #content_types,
|
||||||
headers: &[#(#headers),*],
|
headers: &[#(#headers),*],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2683,10 +2830,12 @@ fn metadata_tokens(
|
|||||||
.into_iter()
|
.into_iter()
|
||||||
.chain(operation.additional_error_statuses.iter().copied());
|
.chain(operation.additional_error_statuses.iter().copied());
|
||||||
let error_responses = error_statuses.map(|status| {
|
let error_responses = error_statuses.map(|status| {
|
||||||
|
let additional_content_types = &operation.additional_error_content_types;
|
||||||
quote! {
|
quote! {
|
||||||
#api_crate::ResponseMetadata {
|
#api_crate::ResponseMetadata {
|
||||||
status: #status,
|
status: #status,
|
||||||
body: #api_crate::BodyMetadata { wire_kind: #api_crate::WireKind::Json },
|
body: #api_crate::BodyMetadata { wire_kind: #api_crate::WireKind::Json },
|
||||||
|
content_types: &["application/json", #(#additional_content_types),*],
|
||||||
headers: &[],
|
headers: &[],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ publish = false
|
|||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
|
error-contract = ["dep:error-contract"]
|
||||||
axum = ["dep:axum", "dep:serde"]
|
axum = ["dep:axum", "dep:serde"]
|
||||||
openapi = ["dep:schemars", "dep:serde_json", "api-macros-impl/openapi"]
|
openapi = ["dep:schemars", "dep:serde_json", "api-macros-impl/openapi"]
|
||||||
reqwest = [
|
reqwest = [
|
||||||
@@ -20,6 +21,7 @@ reqwest = [
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
api-macros-impl = { path = "../api-macros-impl" }
|
api-macros-impl = { path = "../api-macros-impl" }
|
||||||
|
error-contract = { path = "../error-contract", optional = true }
|
||||||
bytes.workspace = true
|
bytes.workspace = true
|
||||||
axum = { workspace = true, optional = true }
|
axum = { workspace = true, optional = true }
|
||||||
futures = { workspace = true, optional = true }
|
futures = { workspace = true, optional = true }
|
||||||
|
|||||||
@@ -6,4 +6,6 @@ Defines transport-independent API metadata from public traits. The same declarat
|
|||||||
|
|
||||||
- [`api`] — API declaration and validation
|
- [`api`] — API declaration and validation
|
||||||
- [`ApiContract`], [`Operation`], and [`WebSocketOperation`] — typed operation contracts
|
- [`ApiContract`], [`Operation`], and [`WebSocketOperation`] — typed operation contracts
|
||||||
|
- [`HttpError`] — opt-in public error encoding, with unchanged JSON defaults
|
||||||
- `axum`, `reqwest`, and `openapi` — adapters and document generation enabled by their respective features
|
- `axum`, `reqwest`, and `openapi` — adapters and document generation enabled by their respective features
|
||||||
|
- `http` — projected error-contract response conversion and bounded reception, enabled by `error-contract` without enabling a framework
|
||||||
|
|||||||
@@ -96,6 +96,34 @@ pub fn error_response<T: Serialize + Send + Sync + 'static>(
|
|||||||
response
|
response
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Apply the optional public encoding hook, retaining the original server-local error.
|
||||||
|
///
|
||||||
|
/// Invalid hook metadata fails closed with an empty 500 response and no typed extension.
|
||||||
|
/// A hook failure must not expose a potentially private legacy serialization.
|
||||||
|
pub fn http_error_response<T: crate::HttpError + Serialize + Send + Sync + 'static>(
|
||||||
|
status: StatusCode,
|
||||||
|
error: T,
|
||||||
|
) -> Response {
|
||||||
|
let Some(encoded) = error.custom_response() else {
|
||||||
|
return error_response(status, error);
|
||||||
|
};
|
||||||
|
let Ok(status) = StatusCode::from_u16(encoded.status) else {
|
||||||
|
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
|
||||||
|
};
|
||||||
|
let Ok(content_type) = axum::http::HeaderValue::from_str(encoded.content_type) else {
|
||||||
|
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
|
||||||
|
};
|
||||||
|
if !(status.is_client_error() || status.is_server_error()) || content_type.is_empty() {
|
||||||
|
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
|
||||||
|
}
|
||||||
|
let mut response = (status, encoded.body.into_bytes()).into_response();
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||||
|
response.extensions_mut().insert(std::sync::Arc::new(error));
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
/// Emit an exact binary response with the contract status and octet-stream content type.
|
/// Emit an exact binary response with the contract status and octet-stream content type.
|
||||||
pub fn binary_response(status: StatusCode, value: crate::BinaryBody) -> Response {
|
pub fn binary_response(status: StatusCode, value: crate::BinaryBody) -> Response {
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
//! Shared HTTP error-contract adapter, independent of Axum and Reqwest.
|
||||||
|
//!
|
||||||
|
//! Only Browser-projected failures can be encoded here. Receiving a valid failure does not
|
||||||
|
//! grant permission to forward it: domains must reproject with current authorization.
|
||||||
|
|
||||||
|
pub use error_contract::problem::{ApiBoundary, ProblemResponse};
|
||||||
|
use error_contract::{
|
||||||
|
Browser, Budget, ContractError, DecodeFailure, ProjectedFailure, WireDetails,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Encode a projected failure or a static, non-domain 500 fallback under the same budget.
|
||||||
|
///
|
||||||
|
/// `safe_instance` must be selected by the caller's disclosure policy (for example a local
|
||||||
|
/// opaque occurrence URI), never copied from a request URI containing credentials or queries.
|
||||||
|
/// URI syntax validation is not disclosure authorization. If even the static fallback exceeds
|
||||||
|
/// the budget, return an error instead of truncating JSON or silently exceeding the limit.
|
||||||
|
pub fn problem_response<D: WireDetails>(
|
||||||
|
projected: &ProjectedFailure<Browser, D>,
|
||||||
|
boundary: &impl ApiBoundary<D>,
|
||||||
|
safe_instance: &str,
|
||||||
|
budget: Budget,
|
||||||
|
) -> Result<ProblemResponse, ContractError> {
|
||||||
|
error_contract::problem::to_problem_details(projected, boundary, safe_instance, budget)
|
||||||
|
.or_else(|_| error_contract::problem::problem_fallback(budget))
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ProblemResponse> for crate::HttpErrorResponse {
|
||||||
|
fn from(response: ProblemResponse) -> Self {
|
||||||
|
Self {
|
||||||
|
status: response.status,
|
||||||
|
content_type: response.content_type,
|
||||||
|
body: response.body().to_vec().into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Normalize a bounded HTTP error using actual transport metadata, not a body-claimed status.
|
||||||
|
///
|
||||||
|
/// Unsupported schema/reason, corrupt input, and `Corrupt(TooLarge)` remain distinct. None of
|
||||||
|
/// these results, including a known cause, establish remote execution or retry permission.
|
||||||
|
/// Streaming callers must also bound collection before calling this byte-slice adapter.
|
||||||
|
pub fn decode_problem_response<D: WireDetails>(
|
||||||
|
input: &[u8],
|
||||||
|
actual_status: u16,
|
||||||
|
content_type: &str,
|
||||||
|
boundary: &impl ApiBoundary<D>,
|
||||||
|
budget: Budget,
|
||||||
|
) -> DecodeFailure<D> {
|
||||||
|
error_contract::problem::decode_problem_details(
|
||||||
|
input,
|
||||||
|
actual_status,
|
||||||
|
content_type,
|
||||||
|
boundary,
|
||||||
|
budget,
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -14,6 +14,16 @@ extern crate self as api_macros;
|
|||||||
/// leave authentication, authorization, body limits, and timeouts to the enclosing application.
|
/// leave authentication, authorization, body limits, and timeouts to the enclosing application.
|
||||||
/// Declarations without adapters do not depend on an HTTP framework.
|
/// Declarations without adapters do not depend on an HTTP framework.
|
||||||
///
|
///
|
||||||
|
/// `#[api(reqwest, client_error_hook)]` delegates error decoding to the error type's
|
||||||
|
/// `reqwest::HttpClientError` implementation. A mixed-error API can instead set
|
||||||
|
/// `client_error_hook = true` on individual HTTP operations. Either flag enables the hook;
|
||||||
|
/// a method-level `false` does not override a trait-wide opt-in. Without opt-in, client-only
|
||||||
|
/// errors need only serde deserialization; server traits and codecs are not imposed on them.
|
||||||
|
///
|
||||||
|
/// `additional_error_content_types = ["application/problem+json"]` on an HTTP operation
|
||||||
|
/// declares extra representations for every declared error status, alongside legacy JSON.
|
||||||
|
/// Each representation uses the error type's schema; the producer still owns its encoding.
|
||||||
|
///
|
||||||
/// This declaration checks that a Rust trait and typed operation markers can be defined together.
|
/// This declaration checks that a Rust trait and typed operation markers can be defined together.
|
||||||
///
|
///
|
||||||
/// ```
|
/// ```
|
||||||
@@ -71,6 +81,8 @@ pub use api_macros_impl::api;
|
|||||||
|
|
||||||
#[cfg(feature = "axum")]
|
#[cfg(feature = "axum")]
|
||||||
pub mod axum;
|
pub mod axum;
|
||||||
|
#[cfg(feature = "error-contract")]
|
||||||
|
pub mod http;
|
||||||
#[cfg(feature = "openapi")]
|
#[cfg(feature = "openapi")]
|
||||||
pub mod openapi;
|
pub mod openapi;
|
||||||
#[cfg(feature = "reqwest")]
|
#[cfg(feature = "reqwest")]
|
||||||
@@ -213,6 +225,8 @@ pub struct ResponseHeaderMetadata {
|
|||||||
pub struct ResponseMetadata {
|
pub struct ResponseMetadata {
|
||||||
pub status: u16,
|
pub status: u16,
|
||||||
pub body: BodyMetadata,
|
pub body: BodyMetadata,
|
||||||
|
/// Declared media types, empty for bodyless responses. JSON remains the default error media.
|
||||||
|
pub content_types: &'static [&'static str],
|
||||||
pub headers: &'static [ResponseHeaderMetadata],
|
pub headers: &'static [ResponseHeaderMetadata],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -272,6 +286,24 @@ pub trait WebSocketOperation {
|
|||||||
/// service's typed 401/403/404/409 responses without introducing duplicate route handlers.
|
/// service's typed 401/403/404/409 responses without introducing duplicate route handlers.
|
||||||
pub trait HttpError {
|
pub trait HttpError {
|
||||||
fn status_code(&self) -> u16;
|
fn status_code(&self) -> u16;
|
||||||
|
|
||||||
|
/// Opt in to an already encoded public response instead of the legacy JSON representation.
|
||||||
|
/// The adapter retains the original error as a server-local `Arc<Self>` extension.
|
||||||
|
/// Returning `None` preserves the existing JSON/status behavior exactly.
|
||||||
|
fn custom_response(&self) -> Option<HttpErrorResponse> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An explicitly encoded HTTP error, independent of any server or client framework.
|
||||||
|
///
|
||||||
|
/// Producers own disclosure and byte limits. Adapters reject non-error statuses and invalid
|
||||||
|
/// content types rather than substituting the legacy representation, which may not be public.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct HttpErrorResponse {
|
||||||
|
pub status: u16,
|
||||||
|
pub content_type: &'static str,
|
||||||
|
pub body: BinaryBody,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Successful JSON response that selects one of an operation's declared success statuses.
|
/// Successful JSON response that selects one of an operation's declared success statuses.
|
||||||
|
|||||||
@@ -39,6 +39,19 @@ pub trait OpenApiSchema: JsonSchema {
|
|||||||
fn openapi_schema_name() -> Cow<'static, str> {
|
fn openapi_schema_name() -> Cow<'static, str> {
|
||||||
Self::schema_name()
|
Self::schema_name()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Select the exact wire schema for an explicitly declared response media type.
|
||||||
|
/// Custom error adapters may have distinct JSON and Problem Details bodies;
|
||||||
|
/// this avoids claiming an ambiguous untagged body union for either medium.
|
||||||
|
fn response_schema(
|
||||||
|
builder: &mut OpenApiBuilder,
|
||||||
|
_content_type: &str,
|
||||||
|
) -> Result<Value, OpenApiError>
|
||||||
|
where
|
||||||
|
Self: Sized,
|
||||||
|
{
|
||||||
|
builder.schema_ref::<Self>()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
macro_rules! impl_openapi_schema {
|
macro_rules! impl_openapi_schema {
|
||||||
@@ -256,7 +269,8 @@ impl OpenApiBuilder {
|
|||||||
Ok(OpenApiDocument(document))
|
Ok(OpenApiDocument(document))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn schema_ref<T: OpenApiSchema>(&mut self) -> Result<Value, OpenApiError> {
|
/// Register an explicitly asserted wire schema, retaining collision and bounds checks.
|
||||||
|
pub fn schema_ref<T: OpenApiSchema>(&mut self) -> Result<Value, OpenApiError> {
|
||||||
let name = T::openapi_schema_name().into_owned();
|
let name = T::openapi_schema_name().into_owned();
|
||||||
validate_component_name(&name)?;
|
validate_component_name(&name)?;
|
||||||
|
|
||||||
@@ -411,13 +425,39 @@ impl OpenApiOperation<'_> {
|
|||||||
content_type: &'static str,
|
content_type: &'static str,
|
||||||
description: &'static str,
|
description: &'static str,
|
||||||
) -> Result<(), OpenApiError> {
|
) -> Result<(), OpenApiError> {
|
||||||
let schema = self.parent.schema_ref::<T>()?;
|
self.response_content_types::<T>(status, &[content_type], description)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Declare one response schema under several explicit representations, atomically.
|
||||||
|
/// This does not change conflict detection for repeated status declarations.
|
||||||
|
pub fn response_content_types<T: OpenApiSchema>(
|
||||||
|
&mut self,
|
||||||
|
status: u16,
|
||||||
|
content_types: &[&str],
|
||||||
|
description: &'static str,
|
||||||
|
) -> Result<(), OpenApiError> {
|
||||||
|
let mut content = Map::new();
|
||||||
|
for content_type in content_types {
|
||||||
|
let schema = T::response_schema(self.parent, content_type)?;
|
||||||
|
if content
|
||||||
|
.insert((*content_type).to_owned(), json!({ "schema": schema }))
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
return Err(OpenApiError::InvalidContract(format!(
|
||||||
|
"duplicate response content type for `{}`",
|
||||||
|
self.operation_id
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if content.is_empty() {
|
||||||
|
return Err(OpenApiError::InvalidContract(format!(
|
||||||
|
"response content types are empty for `{}`",
|
||||||
|
self.operation_id
|
||||||
|
)));
|
||||||
|
}
|
||||||
self.insert_response(
|
self.insert_response(
|
||||||
status,
|
status,
|
||||||
json!({
|
json!({ "description": description, "content": content }),
|
||||||
"description": description,
|
|
||||||
"content": { content_type: { "schema": schema } },
|
|
||||||
}),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -482,6 +482,17 @@ pub fn encode_binary(value: crate::BinaryBody) -> EncodedBody {
|
|||||||
EncodedBody::binary(value)
|
EncodedBody::binary(value)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Explicit decoder for APIs declared with `#[api(reqwest, client_error_hook)]` or an
|
||||||
|
/// operation-level `client_error_hook = true` option.
|
||||||
|
///
|
||||||
|
/// There is deliberately no blanket `Deserialize` or `HttpError` bound: legacy client-only
|
||||||
|
/// errors keep serde decoding without implementing this trait. Opted-in decoders receive the
|
||||||
|
/// actual status/headers and bytes collected under the client's configured limit. Returning
|
||||||
|
/// a failure does not imply anything about remote execution or authorize a new request.
|
||||||
|
pub trait HttpClientError: Sized {
|
||||||
|
fn decode_error(response: &ReceivedResponse) -> Result<Self, ClientFailure>;
|
||||||
|
}
|
||||||
|
|
||||||
/// Response received under the configured byte limit. Its content is always redacted in Debug.
|
/// Response received under the configured byte limit. Its content is always redacted in Debug.
|
||||||
pub struct ReceivedResponse {
|
pub struct ReceivedResponse {
|
||||||
status: StatusCode,
|
status: StatusCode,
|
||||||
@@ -494,6 +505,48 @@ impl ReceivedResponse {
|
|||||||
self.status
|
self.status
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Borrow bytes already collected under the configured response limit.
|
||||||
|
pub fn body(&self) -> &[u8] {
|
||||||
|
&self.bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Require exactly one valid Content-Type header; ambiguous metadata must not select a codec.
|
||||||
|
pub fn content_type(&self) -> Result<&str, ClientFailure> {
|
||||||
|
let name = "content-type";
|
||||||
|
let mut values = self.headers.get_all(name).iter();
|
||||||
|
let value = values
|
||||||
|
.next()
|
||||||
|
.ok_or(ClientFailure::MissingResponseHeader { name })?;
|
||||||
|
if values.next().is_some() {
|
||||||
|
return Err(ClientFailure::DuplicateResponseHeader { name });
|
||||||
|
}
|
||||||
|
value
|
||||||
|
.to_str()
|
||||||
|
.map_err(|_| ClientFailure::InvalidResponseHeader { name })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode without promoting received data to a public projection or execution result.
|
||||||
|
/// Transport collection and contract budgets are enforced independently.
|
||||||
|
#[cfg(feature = "error-contract")]
|
||||||
|
pub fn decode_problem<D: error_contract::WireDetails>(
|
||||||
|
&self,
|
||||||
|
boundary: &impl crate::http::ApiBoundary<D>,
|
||||||
|
budget: error_contract::Budget,
|
||||||
|
) -> error_contract::DecodeFailure<D> {
|
||||||
|
match self.content_type() {
|
||||||
|
Ok(content_type) => crate::http::decode_problem_response(
|
||||||
|
&self.bytes,
|
||||||
|
self.status.as_u16(),
|
||||||
|
content_type,
|
||||||
|
boundary,
|
||||||
|
budget,
|
||||||
|
),
|
||||||
|
Err(_) => {
|
||||||
|
error_contract::DecodeFailure::Corrupt(error_contract::ContractError::Invalid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn parse_header<T>(&self, name: &'static str) -> Result<T, ClientFailure>
|
pub fn parse_header<T>(&self, name: &'static str) -> Result<T, ClientFailure>
|
||||||
where
|
where
|
||||||
T: std::str::FromStr,
|
T: std::str::FromStr,
|
||||||
@@ -555,13 +608,32 @@ pub enum ClientFailure {
|
|||||||
Connect,
|
Connect,
|
||||||
Transport,
|
Transport,
|
||||||
Timeout,
|
Timeout,
|
||||||
ResponseTooLarge { limit: usize },
|
ResponseTooLarge {
|
||||||
UnexpectedStatus { expected: u16, actual: u16 },
|
limit: usize,
|
||||||
ErrorResponseDecode { status: u16 },
|
},
|
||||||
Decode { kind: DecodeKind },
|
UnexpectedStatus {
|
||||||
MissingResponseHeader { name: &'static str },
|
expected: u16,
|
||||||
DuplicateResponseHeader { name: &'static str },
|
actual: u16,
|
||||||
InvalidResponseHeader { name: &'static str },
|
},
|
||||||
|
ErrorResponseDecode {
|
||||||
|
status: u16,
|
||||||
|
},
|
||||||
|
/// A bounded, structurally valid error used an unsupported schema or reason.
|
||||||
|
UnsupportedErrorResponse {
|
||||||
|
status: u16,
|
||||||
|
},
|
||||||
|
Decode {
|
||||||
|
kind: DecodeKind,
|
||||||
|
},
|
||||||
|
MissingResponseHeader {
|
||||||
|
name: &'static str,
|
||||||
|
},
|
||||||
|
DuplicateResponseHeader {
|
||||||
|
name: &'static str,
|
||||||
|
},
|
||||||
|
InvalidResponseHeader {
|
||||||
|
name: &'static str,
|
||||||
|
},
|
||||||
UnexpectedBody,
|
UnexpectedBody,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -588,6 +660,12 @@ impl fmt::Display for ClientFailure {
|
|||||||
"expected HTTP status {expected}, received {actual}"
|
"expected HTTP status {expected}, received {actual}"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
Self::UnsupportedErrorResponse { status } => {
|
||||||
|
write!(
|
||||||
|
formatter,
|
||||||
|
"HTTP {status} error response contract was unsupported"
|
||||||
|
)
|
||||||
|
}
|
||||||
Self::ErrorResponseDecode { status } => {
|
Self::ErrorResponseDecode { status } => {
|
||||||
write!(formatter, "HTTP {status} error response JSON was malformed")
|
write!(formatter, "HTTP {status} error response JSON was malformed")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,463 @@
|
|||||||
|
//! Connect generated Axum responses to generated Reqwest opt-in decoding over isolated loopback.
|
||||||
|
//! Domain disclosure is a fixture; HTTP encoding, bounded reads and metadata checks are production.
|
||||||
|
#![cfg(all(feature = "axum", feature = "reqwest", feature = "error-contract"))]
|
||||||
|
|
||||||
|
use std::{sync::Arc, time::Duration};
|
||||||
|
|
||||||
|
use api_macros::{
|
||||||
|
HttpErrorResponse, api,
|
||||||
|
reqwest::{ClientError, ClientFailure, HttpClientError, ReceivedResponse},
|
||||||
|
};
|
||||||
|
use axum::{
|
||||||
|
body::{Body, to_bytes},
|
||||||
|
http::{Request, StatusCode},
|
||||||
|
response::IntoResponse,
|
||||||
|
routing::get,
|
||||||
|
};
|
||||||
|
use error_contract::{
|
||||||
|
Browser, Budget, Category, ContractError, DecodeFailure, Disclosure, FailureCause,
|
||||||
|
FailureFacts, FailureIdentity, ProjectFor, ProjectedFailure, WireDetails, project_failure,
|
||||||
|
};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tower::ServiceExt;
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
pub struct Payload {
|
||||||
|
value: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize, Deserialize)]
|
||||||
|
pub struct Denied {
|
||||||
|
required_operation: String,
|
||||||
|
}
|
||||||
|
impl FailureCause for Denied {
|
||||||
|
fn identity(&self) -> FailureIdentity {
|
||||||
|
FailureIdentity {
|
||||||
|
domain: "subject",
|
||||||
|
reason: "operation_denied",
|
||||||
|
category: Category::Authorization,
|
||||||
|
details_version: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn validate(&self) -> Result<(), ContractError> {
|
||||||
|
if self.required_operation == "session.record" {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(ContractError::InvalidDetails)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl WireDetails for Denied {
|
||||||
|
fn supports(domain: &str, reason: &str, version: u16) -> bool {
|
||||||
|
domain == "subject" && reason == "operation_denied" && version == 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl ProjectFor<Browser> for Denied {
|
||||||
|
type Authorization = ();
|
||||||
|
type Details = Self;
|
||||||
|
fn disclose(&self, _: &FailureFacts<Self>, _: &()) -> Result<Disclosure<Self>, ContractError> {
|
||||||
|
Ok(Disclosure::new(
|
||||||
|
self.clone(),
|
||||||
|
"Subject cannot record a session.",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
struct Boundary;
|
||||||
|
impl api_macros::http::ApiBoundary<Denied> for Boundary {
|
||||||
|
fn classify(&self, _: &Denied) -> error_contract::problem::ProblemClass {
|
||||||
|
error_contract::problem::ProblemClass::authorization_denied()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn budget() -> Budget {
|
||||||
|
Budget::new(4096, 4096)
|
||||||
|
}
|
||||||
|
fn projected() -> ProjectedFailure<Browser, Denied> {
|
||||||
|
project_failure(
|
||||||
|
&FailureFacts::new(Denied {
|
||||||
|
required_operation: "session.record".into(),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
&(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
fn encoded() -> HttpErrorResponse {
|
||||||
|
api_macros::http::problem_response(&projected(), &Boundary, "urn:yoi:failure:local-1", budget())
|
||||||
|
.unwrap()
|
||||||
|
.into()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not Clone: the original must be retained without duplicating a domain error.
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct ServerError {
|
||||||
|
legacy: &'static str,
|
||||||
|
#[serde(skip)]
|
||||||
|
mode: u8,
|
||||||
|
}
|
||||||
|
impl api_macros::HttpError for ServerError {
|
||||||
|
fn status_code(&self) -> u16 {
|
||||||
|
422
|
||||||
|
}
|
||||||
|
fn custom_response(&self) -> Option<HttpErrorResponse> {
|
||||||
|
match self.mode {
|
||||||
|
0 => Some(encoded()),
|
||||||
|
1 => None,
|
||||||
|
mode => Some(HttpErrorResponse {
|
||||||
|
status: if mode == 2 { 200 } else { 403 },
|
||||||
|
content_type: if mode == 3 {
|
||||||
|
"bad\ncontent-type"
|
||||||
|
} else {
|
||||||
|
"application/problem+json"
|
||||||
|
},
|
||||||
|
body: b"private-hook-body".to_vec().into(),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl api_macros::HttpRequestError for ServerError {
|
||||||
|
fn from_request_rejection(_: u16, _: String) -> Self {
|
||||||
|
Self {
|
||||||
|
legacy: "never serialize",
|
||||||
|
mode: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[api(axum)]
|
||||||
|
pub trait HookServer {
|
||||||
|
#[get("/problem", status = 200, error_status = 422)]
|
||||||
|
async fn problem(&self) -> Result<Payload, ServerError>;
|
||||||
|
#[post("/reject", error_status = 422, normalize_body_errors = true)]
|
||||||
|
async fn reject(&self, #[body] input: Payload) -> Result<Payload, ServerError>;
|
||||||
|
#[post("/reject-binary", error_status = 422, normalize_body_errors = true)]
|
||||||
|
async fn reject_binary(
|
||||||
|
&self,
|
||||||
|
#[binary] input: api_macros::BinaryBody,
|
||||||
|
) -> Result<Payload, ServerError>;
|
||||||
|
}
|
||||||
|
pub struct Service;
|
||||||
|
impl HookServer for Service {
|
||||||
|
async fn problem(&self) -> Result<Payload, ServerError> {
|
||||||
|
Err(ServerError {
|
||||||
|
legacy: "never serialize",
|
||||||
|
mode: 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
async fn reject(&self, input: Payload) -> Result<Payload, ServerError> {
|
||||||
|
Ok(input)
|
||||||
|
}
|
||||||
|
async fn reject_binary(&self, _input: api_macros::BinaryBody) -> Result<Payload, ServerError> {
|
||||||
|
Ok(Payload { value: "ok".into() })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Neither Deserialize nor HttpError: client hooks impose no server or serde error bounds.
|
||||||
|
pub struct DecodedError {
|
||||||
|
operation: String,
|
||||||
|
}
|
||||||
|
impl HttpClientError for DecodedError {
|
||||||
|
fn decode_error(response: &ReceivedResponse) -> Result<Self, ClientFailure> {
|
||||||
|
let status = response.status().as_u16();
|
||||||
|
match response.decode_problem::<Denied>(&Boundary, budget()) {
|
||||||
|
DecodeFailure::Known(failure) => Ok(Self {
|
||||||
|
operation: failure.facts().cause().required_operation.clone(),
|
||||||
|
}),
|
||||||
|
DecodeFailure::Unsupported(_) => {
|
||||||
|
Err(ClientFailure::UnsupportedErrorResponse { status })
|
||||||
|
}
|
||||||
|
DecodeFailure::Corrupt(ContractError::TooLarge) => {
|
||||||
|
Err(ClientFailure::ResponseTooLarge {
|
||||||
|
limit: budget().bytes(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
DecodeFailure::Corrupt(_) => Err(ClientFailure::ErrorResponseDecode { status }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[api(reqwest)]
|
||||||
|
pub trait HookClient {
|
||||||
|
#[get("/problem", status = 200, error_status = 422, client_error_hook = true)]
|
||||||
|
async fn problem(&self) -> Result<Payload, DecodedError>;
|
||||||
|
#[get(
|
||||||
|
"/wire/{mode}",
|
||||||
|
status = 200,
|
||||||
|
error_status = 422,
|
||||||
|
client_error_hook = true
|
||||||
|
)]
|
||||||
|
async fn wire(&self, mode: String) -> Result<Payload, DecodedError>;
|
||||||
|
#[get("/legacy", error_status = 422)]
|
||||||
|
async fn legacy(&self) -> Result<Payload, LegacyClientError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A false method flag cannot disable a trait-wide opt-in; the selection is logical OR.
|
||||||
|
#[api(reqwest, client_error_hook)]
|
||||||
|
pub trait TraitHookClient {
|
||||||
|
#[get("/problem", client_error_hook = false)]
|
||||||
|
async fn problem(&self) -> Result<Payload, DecodedError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Client-only legacy errors must still compile and decode without either hook trait.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct LegacyClientError {
|
||||||
|
legacy: String,
|
||||||
|
}
|
||||||
|
#[api(reqwest)]
|
||||||
|
pub trait LegacyClient {
|
||||||
|
#[get("/legacy", error_status = 422)]
|
||||||
|
async fn legacy(&self) -> Result<Payload, LegacyClientError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn generated_server_hook_overrides_wire_encoding_and_retains_arc_for_errors_and_rejections() {
|
||||||
|
let app = HookServerAxum::router(Service).layer(axum::extract::DefaultBodyLimit::max(32));
|
||||||
|
for request in [
|
||||||
|
Request::builder()
|
||||||
|
.uri("/problem")
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/reject")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from("{"))
|
||||||
|
.unwrap(),
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/reject-binary")
|
||||||
|
.body(Body::from(vec![0; 64]))
|
||||||
|
.unwrap(),
|
||||||
|
] {
|
||||||
|
let response = app.clone().oneshot(request).await.unwrap();
|
||||||
|
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()["content-type"],
|
||||||
|
"application/problem+json"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.extensions()
|
||||||
|
.get::<Arc<ServerError>>()
|
||||||
|
.unwrap()
|
||||||
|
.mode,
|
||||||
|
0
|
||||||
|
);
|
||||||
|
let bytes = to_bytes(response.into_body(), 4096).await.unwrap();
|
||||||
|
let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
|
||||||
|
assert_eq!(value["status"], 403);
|
||||||
|
assert_eq!(
|
||||||
|
value["yoi_failure"]["details"]["required_operation"],
|
||||||
|
"session.record"
|
||||||
|
);
|
||||||
|
assert!(!String::from_utf8_lossy(&bytes).contains("never serialize"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn absent_hook_preserves_json_and_invalid_hook_never_falls_back_to_legacy() {
|
||||||
|
let legacy = api_macros::axum::http_error_response(
|
||||||
|
StatusCode::UNPROCESSABLE_ENTITY,
|
||||||
|
ServerError {
|
||||||
|
legacy: "legacy-value",
|
||||||
|
mode: 1,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert_eq!(legacy.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||||
|
assert_eq!(legacy.headers()["content-type"], "application/json");
|
||||||
|
assert!(legacy.extensions().get::<Arc<ServerError>>().is_some());
|
||||||
|
let body = to_bytes(legacy.into_body(), 4096).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_slice::<serde_json::Value>(&body).unwrap()["legacy"],
|
||||||
|
"legacy-value"
|
||||||
|
);
|
||||||
|
for mode in [2, 3] {
|
||||||
|
let response = api_macros::axum::http_error_response(
|
||||||
|
StatusCode::UNPROCESSABLE_ENTITY,
|
||||||
|
ServerError {
|
||||||
|
legacy: "private",
|
||||||
|
mode,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||||
|
assert!(response.extensions().get::<Arc<ServerError>>().is_none());
|
||||||
|
assert!(
|
||||||
|
to_bytes(response.into_body(), 4096)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn problem_conversion_falls_back_without_exceeding_byte_or_depth_budgets() {
|
||||||
|
for (instance, budget) in [
|
||||||
|
("invalid URI", budget()),
|
||||||
|
("", Budget::new(256, 256)),
|
||||||
|
("", budget().with_depth(1, 1)),
|
||||||
|
] {
|
||||||
|
let response =
|
||||||
|
api_macros::http::problem_response(&projected(), &Boundary, instance, budget).unwrap();
|
||||||
|
assert_eq!(response.status, 500);
|
||||||
|
assert_eq!(
|
||||||
|
response.body(),
|
||||||
|
error_contract::problem::STATIC_PROBLEM_FALLBACK
|
||||||
|
);
|
||||||
|
assert!(response.body().len() <= budget.bytes());
|
||||||
|
}
|
||||||
|
assert!(matches!(
|
||||||
|
api_macros::http::problem_response(&projected(), &Boundary, "", Budget::new(1, 1)),
|
||||||
|
Err(ContractError::TooLarge)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wire(axum::extract::Path(mode): axum::extract::Path<String>) -> axum::response::Response {
|
||||||
|
let mut value: serde_json::Value = serde_json::from_slice(encoded().body.as_ref()).unwrap();
|
||||||
|
let mut status = StatusCode::FORBIDDEN;
|
||||||
|
let mut content_type = "application/problem+json; charset=utf-8";
|
||||||
|
match mode.as_str() {
|
||||||
|
"unknown" => value["yoi_failure"]["reason"] = "future_reason".into(),
|
||||||
|
"version" => value["yoi_failure"]["version"] = 99.into(),
|
||||||
|
"status" => status = StatusCode::CONFLICT,
|
||||||
|
"content-type" => content_type = "application/json",
|
||||||
|
"corrupt" => {
|
||||||
|
return (
|
||||||
|
status,
|
||||||
|
[("content-type", content_type)],
|
||||||
|
"{private-raw-body",
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
"large" => {
|
||||||
|
return (status, [("content-type", content_type)], "x".repeat(8192)).into_response();
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
let mut response = (
|
||||||
|
status,
|
||||||
|
[("content-type", content_type)],
|
||||||
|
serde_json::to_vec(&value).unwrap(),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
if mode == "duplicate" {
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.append("content-type", "application/json".parse().unwrap());
|
||||||
|
}
|
||||||
|
if mode == "missing" {
|
||||||
|
response.headers_mut().remove("content-type");
|
||||||
|
}
|
||||||
|
response
|
||||||
|
}
|
||||||
|
struct ServerTask(tokio::task::JoinHandle<()>);
|
||||||
|
impl Drop for ServerTask {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async fn serve() -> (String, ServerTask) {
|
||||||
|
let app = HookServerAxum::router(Service)
|
||||||
|
.route("/wire/{mode}", get(wire))
|
||||||
|
.route(
|
||||||
|
"/legacy",
|
||||||
|
get(|| async {
|
||||||
|
api_macros::axum::http_error_response(
|
||||||
|
StatusCode::UNPROCESSABLE_ENTITY,
|
||||||
|
ServerError {
|
||||||
|
legacy: "legacy-value",
|
||||||
|
mode: 1,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
let task = ServerTask(tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
}));
|
||||||
|
(url, task)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn generated_client_hook_uses_actual_metadata_and_preserves_unknown_corrupt_and_size_failures()
|
||||||
|
{
|
||||||
|
let (url, _server) = serve().await;
|
||||||
|
let client = HookClientClient::builder(&url)
|
||||||
|
.unwrap()
|
||||||
|
.request_timeout(Duration::from_secs(5))
|
||||||
|
.response_body_limit(16384)
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
match client.problem().await {
|
||||||
|
Err(ClientError::Public { status, error }) => {
|
||||||
|
assert_eq!(status.as_u16(), 403);
|
||||||
|
assert_eq!(error.operation, "session.record");
|
||||||
|
}
|
||||||
|
_ => panic!("expected a typed problem"),
|
||||||
|
}
|
||||||
|
for mode in ["unknown", "version"] {
|
||||||
|
assert!(matches!(
|
||||||
|
client.wire(mode.into()).await,
|
||||||
|
Err(ClientError::Failure(
|
||||||
|
ClientFailure::UnsupportedErrorResponse { status: 403 }
|
||||||
|
))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for (mode, expected) in [
|
||||||
|
("status", 409),
|
||||||
|
("content-type", 403),
|
||||||
|
("corrupt", 403),
|
||||||
|
("duplicate", 403),
|
||||||
|
("missing", 403),
|
||||||
|
] {
|
||||||
|
let result = client.wire(mode.into()).await;
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(ClientError::Failure(ClientFailure::ErrorResponseDecode { status })) if status == expected),
|
||||||
|
"mode={mode}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Contract budget and streaming transport budget are independent, bounded failures.
|
||||||
|
assert!(matches!(
|
||||||
|
client.wire("large".into()).await,
|
||||||
|
Err(ClientError::Failure(ClientFailure::ResponseTooLarge {
|
||||||
|
limit: 4096
|
||||||
|
}))
|
||||||
|
));
|
||||||
|
let small_client = HookClientClient::builder(&url)
|
||||||
|
.unwrap()
|
||||||
|
.request_timeout(Duration::from_secs(5))
|
||||||
|
.response_body_limit(128)
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
small_client.problem().await,
|
||||||
|
Err(ClientError::Failure(ClientFailure::ResponseTooLarge {
|
||||||
|
limit: 128
|
||||||
|
}))
|
||||||
|
));
|
||||||
|
match client.legacy().await {
|
||||||
|
Err(ClientError::Public { status, error }) => {
|
||||||
|
assert_eq!(status.as_u16(), 422);
|
||||||
|
assert_eq!(error.legacy, "legacy-value");
|
||||||
|
}
|
||||||
|
_ => panic!("unrelated error on the same trait must keep serde behavior"),
|
||||||
|
}
|
||||||
|
let trait_client = TraitHookClientClient::builder(&url)
|
||||||
|
.unwrap()
|
||||||
|
.request_timeout(Duration::from_secs(5))
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
matches!(trait_client.problem().await, Err(ClientError::Public { error, .. }) if error.operation == "session.record")
|
||||||
|
);
|
||||||
|
let legacy = LegacyClientClient::builder(&url)
|
||||||
|
.unwrap()
|
||||||
|
.request_timeout(Duration::from_secs(5))
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
match legacy.legacy().await {
|
||||||
|
Err(ClientError::Public { status, error }) => {
|
||||||
|
assert_eq!(status.as_u16(), 422);
|
||||||
|
assert_eq!(error.legacy, "legacy-value");
|
||||||
|
}
|
||||||
|
_ => panic!("legacy client-only error must keep serde behavior"),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
use api_macros::{
|
use api_macros::{
|
||||||
BinaryBody, api,
|
BinaryBody, api,
|
||||||
openapi::{OpenApiInfo, OpenApiSchema},
|
openapi::{OpenApiBuilder, OpenApiInfo, OpenApiSchema},
|
||||||
};
|
};
|
||||||
use schemars::JsonSchema;
|
use schemars::JsonSchema;
|
||||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||||
@@ -465,3 +465,154 @@ fn file_export_writes_the_canonical_document() {
|
|||||||
std::fs::remove_file(path).expect("remove export");
|
std::fs::remove_file(path).expect("remove export");
|
||||||
assert_eq!(written, document.to_json().expect("canonical JSON"));
|
assert_eq!(written, document.to_json().expect("canonical JSON"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Serialize)]
|
||||||
|
#[serde(untagged, deny_unknown_fields)]
|
||||||
|
pub enum ProblemOrLegacy {
|
||||||
|
Legacy { code: String },
|
||||||
|
Problem { status: u16, detail: String },
|
||||||
|
}
|
||||||
|
impl JsonSchema for ProblemOrLegacy {
|
||||||
|
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||||
|
"ProblemOrLegacy".into()
|
||||||
|
}
|
||||||
|
fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
|
||||||
|
// The owner explicitly attests disjoint closed object shapes instead of admitting anyOf.
|
||||||
|
schemars::json_schema!({ "oneOf": [
|
||||||
|
{ "type": "object", "required": ["code"], "additionalProperties": false,
|
||||||
|
"properties": { "code": { "type": "string" } } },
|
||||||
|
{ "type": "object", "required": ["status", "detail"], "additionalProperties": false,
|
||||||
|
"properties": { "status": { "type": "integer", "format": "uint16" }, "detail": { "type": "string" } } }
|
||||||
|
] })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl OpenApiSchema for ProblemOrLegacy {}
|
||||||
|
|
||||||
|
#[api(openapi)]
|
||||||
|
pub trait ErrorMediaApi {
|
||||||
|
#[post(
|
||||||
|
"/create",
|
||||||
|
error_status = 403,
|
||||||
|
additional_error_statuses = [409, 504],
|
||||||
|
additional_error_content_types = ["Application/Problem+JSON"]
|
||||||
|
)]
|
||||||
|
async fn create(&self) -> Result<Widget, ProblemOrLegacy>;
|
||||||
|
|
||||||
|
#[get("/legacy", error_status = 422)]
|
||||||
|
async fn legacy(&self) -> Result<Widget, PublicError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn additional_error_media_share_the_declared_schema_for_every_error_status_only() {
|
||||||
|
use api_macros::{ApiContract, Operation};
|
||||||
|
let document = error_media_api_openapi(OpenApiInfo {
|
||||||
|
title: "Error media",
|
||||||
|
version: "1",
|
||||||
|
source_digest: "fixture:error-media",
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let value = document.as_value();
|
||||||
|
let responses = &value["paths"]["/create"]["post"]["responses"];
|
||||||
|
for status in ["403", "409", "504"] {
|
||||||
|
let content = responses[status]["content"].as_object().unwrap();
|
||||||
|
assert_eq!(content.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
content["application/json"]["schema"],
|
||||||
|
json!({ "$ref": "#/components/schemas/ProblemOrLegacy" })
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
content["application/problem+json"],
|
||||||
|
content["application/json"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(responses["200"]["content"].as_object().unwrap().len(), 1);
|
||||||
|
assert!(
|
||||||
|
responses["200"]["content"]
|
||||||
|
.get("application/problem+json")
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
value["paths"]["/legacy"]["get"]["responses"]["422"]["content"]
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert!(value["components"]["schemas"]["ProblemOrLegacy"]["oneOf"].is_array());
|
||||||
|
let parsed = oas3::from_json(&document.to_json().unwrap()).unwrap();
|
||||||
|
assert_eq!(parsed.validate_version().unwrap().major, 3);
|
||||||
|
|
||||||
|
let operation = error_media_api_operations::Create::METADATA;
|
||||||
|
assert_eq!(
|
||||||
|
operation
|
||||||
|
.error_responses
|
||||||
|
.iter()
|
||||||
|
.map(|response| response.status)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
[403, 409, 504]
|
||||||
|
);
|
||||||
|
for response in operation.error_responses {
|
||||||
|
assert_eq!(
|
||||||
|
response.content_types,
|
||||||
|
["application/json", "application/problem+json"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
operation.success_responses[0].content_types,
|
||||||
|
["application/json"]
|
||||||
|
);
|
||||||
|
let legacy = ErrorMediaApiMetadata::OPERATIONS
|
||||||
|
.iter()
|
||||||
|
.find(|operation| operation.path == "/legacy")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
legacy.error_responses[0].content_types,
|
||||||
|
["application/json"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(JsonSchema)]
|
||||||
|
pub struct MediaSpecificError;
|
||||||
|
impl OpenApiSchema for MediaSpecificError {
|
||||||
|
fn response_schema(
|
||||||
|
builder: &mut OpenApiBuilder,
|
||||||
|
content_type: &str,
|
||||||
|
) -> Result<Value, api_macros::openapi::OpenApiError> {
|
||||||
|
match content_type {
|
||||||
|
"application/json" => builder.schema_ref::<PublicError>(),
|
||||||
|
"application/problem+json" => builder.schema_ref::<ProblemOrLegacy>(),
|
||||||
|
_ => unreachable!("only declared media are selected"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api(openapi)]
|
||||||
|
pub trait MediaSpecificApi {
|
||||||
|
#[post("/media", error_status = 409, additional_error_content_types = ["application/problem+json"])]
|
||||||
|
async fn create(&self) -> Result<Widget, MediaSpecificError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicitly_different_error_representations_register_their_actual_schema_closures() {
|
||||||
|
let document = media_specific_api_openapi(OpenApiInfo {
|
||||||
|
title: "Media-specific",
|
||||||
|
version: "1",
|
||||||
|
source_digest: "fixture:media-specific",
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let value = document.as_value();
|
||||||
|
let content = &value["paths"]["/media"]["post"]["responses"]["409"]["content"];
|
||||||
|
assert_eq!(
|
||||||
|
content["application/json"]["schema"],
|
||||||
|
json!({"$ref":"#/components/schemas/PublicError"})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
content["application/problem+json"]["schema"],
|
||||||
|
json!({"$ref":"#/components/schemas/ProblemOrLegacy"})
|
||||||
|
);
|
||||||
|
assert!(value["components"]["schemas"]["MediaSpecificError"].is_null());
|
||||||
|
oas3::from_json(&document.to_json().unwrap())
|
||||||
|
.unwrap()
|
||||||
|
.validate_version()
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
use api_macros::api;
|
||||||
|
|
||||||
|
#[api(client_error_hook)]
|
||||||
|
pub trait Invalid {}
|
||||||
|
|
||||||
|
fn main() {}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
error: `client_error_hook` requires the `reqwest` adapter
|
||||||
|
--> tests/ui/client_hook_requires_reqwest.rs:3:1
|
||||||
|
|
|
||||||
|
3 | #[api(client_error_hook)]
|
||||||
|
| ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
|
||||||
|
= note: this error originates in the attribute macro `api` (in Nightly builds, run with -Z macro-backtrace for more info)
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
use api_macros::api;
|
||||||
|
|
||||||
|
pub struct Error;
|
||||||
|
|
||||||
|
#[api]
|
||||||
|
pub trait Wildcard {
|
||||||
|
#[get("/wildcard", additional_error_content_types = ["application/*"])]
|
||||||
|
async fn wildcard(&self) -> Result<(), Error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api]
|
||||||
|
pub trait DuplicateMedia {
|
||||||
|
#[get("/duplicate", additional_error_content_types = ["application/problem+json", "APPLICATION/PROBLEM+JSON"])]
|
||||||
|
async fn duplicate(&self) -> Result<(), Error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api]
|
||||||
|
pub trait DefaultMedia {
|
||||||
|
#[get("/default", additional_error_content_types = ["application/json"])]
|
||||||
|
async fn default_media(&self) -> Result<(), Error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api]
|
||||||
|
pub trait NoError {
|
||||||
|
#[get("/infallible", additional_error_content_types = ["application/problem+json"])]
|
||||||
|
async fn infallible(&self);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api]
|
||||||
|
pub trait EmptyMedia {
|
||||||
|
#[get("/empty", additional_error_content_types = [])]
|
||||||
|
async fn empty(&self) -> Result<(), Error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api]
|
||||||
|
pub trait DuplicateOption {
|
||||||
|
#[get("/duplicate-option", additional_error_content_types = [], additional_error_content_types = [])]
|
||||||
|
async fn duplicate_option(&self) -> Result<(), Error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api]
|
||||||
|
pub trait ParameterizedMedia {
|
||||||
|
#[get("/parameterized", additional_error_content_types = ["application/problem+json; charset=utf-8"])]
|
||||||
|
async fn parameterized(&self) -> Result<(), Error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
error: error content type must be a concrete type/subtype without parameters
|
||||||
|
--> tests/ui/invalid_error_media.rs:7:58
|
||||||
|
|
|
||||||
|
7 | #[get("/wildcard", additional_error_content_types = ["application/*"])]
|
||||||
|
| ^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
error: additional error content types must be distinct from each other and application/json
|
||||||
|
--> tests/ui/invalid_error_media.rs:13:87
|
||||||
|
|
|
||||||
|
13 | #[get("/duplicate", additional_error_content_types = ["application/problem+json", "APPLICATION/PROBLEM+JSON"])]
|
||||||
|
| ^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
error: additional error content types must be distinct from each other and application/json
|
||||||
|
--> tests/ui/invalid_error_media.rs:19:57
|
||||||
|
|
|
||||||
|
19 | #[get("/default", additional_error_content_types = ["application/json"])]
|
||||||
|
| ^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
error: additional_error_content_types require a public error response type
|
||||||
|
--> tests/ui/invalid_error_media.rs:25:11
|
||||||
|
|
|
||||||
|
25 | #[get("/infallible", additional_error_content_types = ["application/problem+json"])]
|
||||||
|
| ^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
error: additional_error_content_types must not be empty
|
||||||
|
--> tests/ui/invalid_error_media.rs:31:11
|
||||||
|
|
|
||||||
|
31 | #[get("/empty", additional_error_content_types = [])]
|
||||||
|
| ^^^^^^^^
|
||||||
|
|
||||||
|
error: duplicate `additional_error_content_types` route option
|
||||||
|
--> tests/ui/invalid_error_media.rs:37:69
|
||||||
|
|
|
||||||
|
37 | #[get("/duplicate-option", additional_error_content_types = [], additional_error_content_types = [])]
|
||||||
|
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
error: error content type must be a concrete type/subtype without parameters
|
||||||
|
--> tests/ui/invalid_error_media.rs:43:63
|
||||||
|
|
|
||||||
|
43 | #[get("/parameterized", additional_error_content_types = ["application/problem+json; charset=utf-8"])]
|
||||||
|
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
use api_macros::api;
|
||||||
|
|
||||||
|
pub struct Error;
|
||||||
|
|
||||||
|
#[api]
|
||||||
|
pub trait NoReqwest {
|
||||||
|
#[get("/error", client_error_hook = true)]
|
||||||
|
async fn error(&self) -> Result<(), Error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api(reqwest)]
|
||||||
|
pub trait NoError {
|
||||||
|
#[post("/infallible", client_error_hook = true)]
|
||||||
|
async fn infallible(&self);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[api(reqwest)]
|
||||||
|
pub trait DuplicateHook {
|
||||||
|
#[get("/duplicate", client_error_hook = true, client_error_hook = false)]
|
||||||
|
async fn duplicate(&self) -> Result<(), Error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
error: client_error_hook requires the `reqwest` adapter
|
||||||
|
--> tests/ui/operation_hook_invalid_combinations.rs:7:41
|
||||||
|
|
|
||||||
|
7 | #[get("/error", client_error_hook = true)]
|
||||||
|
| ^^^^
|
||||||
|
|
||||||
|
error: client_error_hook requires a public error response type
|
||||||
|
--> tests/ui/operation_hook_invalid_combinations.rs:13:47
|
||||||
|
|
|
||||||
|
13 | #[post("/infallible", client_error_hook = true)]
|
||||||
|
| ^^^^
|
||||||
|
|
||||||
|
error: duplicate `client_error_hook` route option
|
||||||
|
--> tests/ui/operation_hook_invalid_combinations.rs:19:51
|
||||||
|
|
|
||||||
|
19 | #[get("/duplicate", client_error_hook = true, client_error_hook = false)]
|
||||||
|
| ^^^^^^^^^^^^^^^^^
|
||||||
@@ -253,6 +253,7 @@ pub enum BackendRuntimeClientError {
|
|||||||
Api(BackendApiClientError),
|
Api(BackendApiClientError),
|
||||||
SessionApi(server_api::ServerApiClientError),
|
SessionApi(server_api::ServerApiClientError),
|
||||||
ContractApi(server_api::client_support::ClientError<server_api::RepositoryApiError>),
|
ContractApi(server_api::client_support::ClientError<server_api::RepositoryApiError>),
|
||||||
|
WorkerCreateApi(server_api::client_support::ClientError<server_api::WorkerCreateApiError>),
|
||||||
Http(reqwest::Error),
|
Http(reqwest::Error),
|
||||||
Protocol(String),
|
Protocol(String),
|
||||||
}
|
}
|
||||||
@@ -264,6 +265,7 @@ impl fmt::Display for BackendRuntimeClientError {
|
|||||||
Self::Api(error) => write!(f, "{error}"),
|
Self::Api(error) => write!(f, "{error}"),
|
||||||
Self::SessionApi(error) => write!(f, "{error}"),
|
Self::SessionApi(error) => write!(f, "{error}"),
|
||||||
Self::ContractApi(error) => write!(f, "{error}"),
|
Self::ContractApi(error) => write!(f, "{error}"),
|
||||||
|
Self::WorkerCreateApi(error) => write!(f, "{error}"),
|
||||||
Self::Http(error) => write!(f, "{error}"),
|
Self::Http(error) => write!(f, "{error}"),
|
||||||
Self::Protocol(message) => f.write_str(message),
|
Self::Protocol(message) => f.write_str(message),
|
||||||
}
|
}
|
||||||
@@ -1016,3 +1018,13 @@ mod tests {
|
|||||||
assert!(serde_json::from_value::<BackendWorkerSummary>(stale).is_err());
|
assert!(serde_json::from_value::<BackendWorkerSummary>(stale).is_err());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<server_api::client_support::ClientError<server_api::WorkerCreateApiError>>
|
||||||
|
for BackendRuntimeClientError
|
||||||
|
{
|
||||||
|
fn from(
|
||||||
|
error: server_api::client_support::ClientError<server_api::WorkerCreateApiError>,
|
||||||
|
) -> Self {
|
||||||
|
Self::WorkerCreateApi(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ pub enum BackendWorkspaceClientError {
|
|||||||
Api(BackendApiClientError),
|
Api(BackendApiClientError),
|
||||||
Http(reqwest::Error),
|
Http(reqwest::Error),
|
||||||
ServerApi(server_api::client_support::ClientError<server_api::RepositoryApiError>),
|
ServerApi(server_api::client_support::ClientError<server_api::RepositoryApiError>),
|
||||||
|
WorkerCreateApi(server_api::client_support::ClientError<server_api::WorkerCreateApiError>),
|
||||||
RuntimeManagementApi(
|
RuntimeManagementApi(
|
||||||
server_api::client_support::ClientError<server_api::RuntimeManagementApiError>,
|
server_api::client_support::ClientError<server_api::RuntimeManagementApiError>,
|
||||||
),
|
),
|
||||||
@@ -51,6 +52,7 @@ impl fmt::Display for BackendWorkspaceClientError {
|
|||||||
Self::Api(error) => write!(f, "{error}"),
|
Self::Api(error) => write!(f, "{error}"),
|
||||||
Self::Http(error) => write!(f, "{error}"),
|
Self::Http(error) => write!(f, "{error}"),
|
||||||
Self::ServerApi(error) => write!(f, "{error}"),
|
Self::ServerApi(error) => write!(f, "{error}"),
|
||||||
|
Self::WorkerCreateApi(error) => write!(f, "{error}"),
|
||||||
Self::RuntimeManagementApi(error) => write!(f, "{error}"),
|
Self::RuntimeManagementApi(error) => write!(f, "{error}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -454,3 +456,31 @@ mod tests {
|
|||||||
assert_eq!(retry, request);
|
assert_eq!(retry, request);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn worker_create_client_error(
|
||||||
|
backend: &BackendApiClient,
|
||||||
|
error: server_api::client_support::ClientError<server_api::WorkerCreateApiError>,
|
||||||
|
) -> BackendWorkspaceClientError {
|
||||||
|
use server_api::client_support::ClientError;
|
||||||
|
// Retain typed startup facts even for an operator denial. A Subject execution
|
||||||
|
// condition is not a refusal of this client's credential or a login prompt.
|
||||||
|
match &error {
|
||||||
|
ClientError::Public {
|
||||||
|
status,
|
||||||
|
error: response,
|
||||||
|
} if response.failure().is_none() && *status == reqwest::StatusCode::UNAUTHORIZED => {
|
||||||
|
BackendWorkspaceClientError::Api(BackendApiClientError::Unauthorized {
|
||||||
|
origin: backend.origin().clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
ClientError::Public {
|
||||||
|
status,
|
||||||
|
error: response,
|
||||||
|
} if response.failure().is_none() && *status == reqwest::StatusCode::FORBIDDEN => {
|
||||||
|
BackendWorkspaceClientError::Api(BackendApiClientError::Forbidden {
|
||||||
|
origin: backend.origin().clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
_ => BackendWorkspaceClientError::WorkerCreateApi(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -81,6 +81,23 @@ impl BackendWorkspaceProductClient {
|
|||||||
server_api::client_support::ClientError<server_api::RepositoryApiError>,
|
server_api::client_support::ClientError<server_api::RepositoryApiError>,
|
||||||
>,
|
>,
|
||||||
>,
|
>,
|
||||||
|
{
|
||||||
|
self.generated_normalized(operation, crate::backend_workspace::server_client_error)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn generated_normalized<T, E, F, Fut>(
|
||||||
|
&self,
|
||||||
|
operation: F,
|
||||||
|
normalize: impl FnOnce(
|
||||||
|
&BackendApiClient,
|
||||||
|
server_api::client_support::ClientError<E>,
|
||||||
|
) -> BackendWorkspaceClientError,
|
||||||
|
) -> Result<T, BackendWorkspaceClientError>
|
||||||
|
where
|
||||||
|
F: FnOnce(
|
||||||
|
server_api::ServerApiClient<crate::backend_workspace::ServerBearerAuthorizer>,
|
||||||
|
) -> Fut,
|
||||||
|
Fut: std::future::Future<Output = Result<T, server_api::client_support::ClientError<E>>>,
|
||||||
{
|
{
|
||||||
let client = crate::backend_workspace::server_api_client(&self.api)?;
|
let client = crate::backend_workspace::server_api_client(&self.api)?;
|
||||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||||
@@ -89,7 +106,7 @@ impl BackendWorkspaceProductClient {
|
|||||||
.map_err(|error| BackendWorkspaceClientError::InvalidTarget(error.to_string()))?;
|
.map_err(|error| BackendWorkspaceClientError::InvalidTarget(error.to_string()))?;
|
||||||
runtime
|
runtime
|
||||||
.block_on(operation(client))
|
.block_on(operation(client))
|
||||||
.map_err(|error| crate::backend_workspace::server_client_error(&self.api, error))
|
.map_err(|error| normalize(&self.api, error))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn read_ticket_link_summary(
|
pub fn read_ticket_link_summary(
|
||||||
@@ -584,9 +601,9 @@ impl BackendWorkspaceProductClient {
|
|||||||
execution_grant: None,
|
execution_grant: None,
|
||||||
};
|
};
|
||||||
let workspace_id = self.workspace_id.clone();
|
let workspace_id = self.workspace_id.clone();
|
||||||
let response: BrowserCreateWorkerResponse = self.generated(move |client| async move {
|
let response: BrowserCreateWorkerResponse = self.generated_normalized(move |client| async move {
|
||||||
client.workspace_worker_create(workspace_id, request).await
|
client.workspace_worker_create(workspace_id, request).await
|
||||||
})?;
|
}, crate::backend_workspace::worker_create_client_error)?;
|
||||||
Ok(format!(
|
Ok(format!(
|
||||||
"Started Intake Worker {}/{} for Ticket {ticket_id}",
|
"Started Intake Worker {}/{} for Ticket {ticket_id}",
|
||||||
response.runtime_id, response.worker_id
|
response.runtime_id, response.worker_id
|
||||||
|
|||||||
@@ -162,6 +162,14 @@ fn validate_envelope_common(raw: &ReaderEnvelope) -> Result<(), ContractError> {
|
|||||||
raw.context.validate()
|
raw.context.validate()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Validate byte and lexical depth limits before parsing an enclosing JSON DTO.
|
||||||
|
///
|
||||||
|
/// This is a resource guard only: callers must still use their typed JSON parser
|
||||||
|
/// to validate grammar and schema. It grants no disclosure or encoding authority.
|
||||||
|
pub fn validate_json_input(input: &[u8], budget: Budget) -> Result<(), ContractError> {
|
||||||
|
check_json_bounds(input, budget)
|
||||||
|
}
|
||||||
|
|
||||||
/// Linear lexical depth guard runs before serde allocates any JSON tree. Serde validates grammar.
|
/// Linear lexical depth guard runs before serde allocates any JSON tree. Serde validates grammar.
|
||||||
pub(crate) fn check_json_bounds(input: &[u8], budget: Budget) -> Result<(), ContractError> {
|
pub(crate) fn check_json_bounds(input: &[u8], budget: Budget) -> Result<(), ContractError> {
|
||||||
if input.len() > budget.bytes() {
|
if input.len() > budget.bytes() {
|
||||||
|
|||||||
@@ -107,6 +107,7 @@ pub struct CausalLink {
|
|||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[cfg_attr(feature = "json-schema", schemars(rename = "FailureDiagnostic"))]
|
||||||
pub struct Diagnostic {
|
pub struct Diagnostic {
|
||||||
#[cfg_attr(feature = "json-schema", schemars(length(min = 1, max = 128)))]
|
#[cfg_attr(feature = "json-schema", schemars(length(min = 1, max = 128)))]
|
||||||
pub code: String,
|
pub code: String,
|
||||||
|
|||||||
@@ -10,7 +10,10 @@ pub mod schema;
|
|||||||
#[cfg(feature = "test-support")]
|
#[cfg(feature = "test-support")]
|
||||||
pub mod test_support;
|
pub mod test_support;
|
||||||
|
|
||||||
pub use codec::{Budget, DecodeFailure, ReceivedFailure, decode_failure, encode_projected_failure};
|
pub use codec::{
|
||||||
|
Budget, DecodeFailure, ReceivedFailure, decode_failure, encode_projected_failure,
|
||||||
|
validate_json_input,
|
||||||
|
};
|
||||||
pub use facts::*;
|
pub use facts::*;
|
||||||
pub use projection::*;
|
pub use projection::*;
|
||||||
|
|
||||||
|
|||||||
@@ -6,10 +6,14 @@ edition.workspace = true
|
|||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
api-macros = { workspace = true, features = ["reqwest", "axum"] }
|
api-macros = { workspace = true, features = ["reqwest", "axum", "openapi"] }
|
||||||
manifest.workspace = true
|
manifest.workspace = true
|
||||||
plugin-plan = { path = "../plugin-plan" }
|
plugin-plan = { path = "../plugin-plan" }
|
||||||
protocol.workspace = true
|
protocol = { workspace = true, features = ["json-schema"] }
|
||||||
|
error-contract = { workspace = true, features = ["json-schema"] }
|
||||||
|
schemars.workspace = true
|
||||||
|
sha2.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
session-store.workspace = true
|
session-store.workspace = true
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
workdir.workspace = true
|
workdir.workspace = true
|
||||||
|
|||||||
@@ -8,3 +8,4 @@ HTTP contract for Worker Runtime management. Owns REST DTOs and generated client
|
|||||||
- [`WorkerRestoreRequest`] / [`WorkerRestoreCoordinationRequest`] — Restore intent and Backend preparation contracts
|
- [`WorkerRestoreRequest`] / [`WorkerRestoreCoordinationRequest`] — Restore intent and Backend preparation contracts
|
||||||
- [`LogicalWorkdirAttachment`] — Logical Workdir attachments determined by Workspace
|
- [`LogicalWorkdirAttachment`] — Logical Workdir attachments determined by Workspace
|
||||||
- [`BackendJobExecutionBinding`] — Job/attempt execution identity fixed by the Host
|
- [`BackendJobExecutionBinding`] — Job/attempt execution identity fixed by the Host
|
||||||
|
- [`startup`] — projected startup Problem Details, independent operation observations and bounded receiver normalization
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
#![doc = include_str!("../README.md")]
|
#![doc = include_str!("../README.md")]
|
||||||
|
|
||||||
|
pub mod startup;
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use api_macros::api;
|
use api_macros::api;
|
||||||
@@ -103,6 +105,9 @@ pub struct RuntimeApiError {
|
|||||||
pub error: RuntimeApiErrorDetail,
|
pub error: RuntimeApiErrorDetail,
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
status: u16,
|
status: u16,
|
||||||
|
/// Local owner/normalized observations, never a second wire attachment.
|
||||||
|
#[serde(skip)]
|
||||||
|
create_failure: Option<startup::WorkerCreateFailure>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Transient, allowlisted startup observations copied from the execution owner.
|
/// Transient, allowlisted startup observations copied from the execution owner.
|
||||||
@@ -135,18 +140,37 @@ impl RuntimeApiError {
|
|||||||
startup: None,
|
startup: None,
|
||||||
},
|
},
|
||||||
status,
|
status,
|
||||||
|
create_failure: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn status(&self) -> u16 {
|
pub fn status(&self) -> u16 {
|
||||||
self.status
|
self.status
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Retain owner-selected facts locally; the HTTP hook projects them afresh.
|
||||||
|
pub fn with_create_failure(mut self, failure: startup::WorkerCreateFailure) -> Self {
|
||||||
|
self.create_failure = Some(failure);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn create_failure(&self) -> Option<&startup::WorkerCreateFailure> {
|
||||||
|
self.create_failure.as_ref()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn into_create_failure(self) -> Option<startup::WorkerCreateFailure> {
|
||||||
|
self.create_failure
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl api_macros::HttpError for RuntimeApiError {
|
impl api_macros::HttpError for RuntimeApiError {
|
||||||
fn status_code(&self) -> u16 {
|
fn status_code(&self) -> u16 {
|
||||||
self.status
|
self.status
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn custom_response(&self) -> Option<api_macros::HttpErrorResponse> {
|
||||||
|
startup::http::runtime_error_response(self)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
@@ -1305,7 +1329,7 @@ pub struct SubjectJobRemovalPurgeResponse {
|
|||||||
pub input_fingerprint: String,
|
pub input_fingerprint: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[api(reqwest, axum)]
|
#[api(reqwest, axum, client_error_hook)]
|
||||||
pub trait RuntimeApi {
|
pub trait RuntimeApi {
|
||||||
#[post("/v1/subject-job-removal-purge", status = 200, error_status = 400)]
|
#[post("/v1/subject-job-removal-purge", status = 200, error_status = 400)]
|
||||||
async fn purge_subject_job_removal(
|
async fn purge_subject_job_removal(
|
||||||
@@ -1924,6 +1948,25 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn get_worker(&self, worker_id: String) -> Result<WorkerResponse, RuntimeApiError> {
|
async fn get_worker(&self, worker_id: String) -> Result<WorkerResponse, RuntimeApiError> {
|
||||||
|
if worker_id == "startup" {
|
||||||
|
use startup::*;
|
||||||
|
let failure = WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Session(
|
||||||
|
protocol::session_failure::SessionRegistrationCause::denied(
|
||||||
|
protocol::session_failure::SessionPrincipal::Subject,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Unknown,
|
||||||
|
execution: ExecutionOutcome::Unknown,
|
||||||
|
finalize: None,
|
||||||
|
backend_compensation: None,
|
||||||
|
correlation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
return Err(test_error(500).with_create_failure(failure));
|
||||||
|
}
|
||||||
assert_eq!(worker_id, "missing worker");
|
assert_eq!(worker_id, "missing worker");
|
||||||
Err(test_error(404))
|
Err(test_error(404))
|
||||||
}
|
}
|
||||||
@@ -2233,6 +2276,34 @@ mod tests {
|
|||||||
assert!(
|
assert!(
|
||||||
matches!(error, client_support::ClientError::Public { status, .. } if status.as_u16() == 404)
|
matches!(error, client_support::ClientError::Public { status, .. } if status.as_u16() == 404)
|
||||||
);
|
);
|
||||||
|
let error = client.get_worker("startup".to_string()).await.unwrap_err();
|
||||||
|
let client_support::ClientError::Public { status, error } = error else {
|
||||||
|
panic!("expected a normalized public startup failure");
|
||||||
|
};
|
||||||
|
assert_eq!(status.as_u16(), 403);
|
||||||
|
assert_eq!(error.status(), 403);
|
||||||
|
let failure = error
|
||||||
|
.create_failure()
|
||||||
|
.expect("hook retains non-HTTP observations");
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().acceptance,
|
||||||
|
startup::CreationAcceptance::Unknown
|
||||||
|
);
|
||||||
|
let browser = failure
|
||||||
|
.problem(
|
||||||
|
startup::StartupBoundary::Browser,
|
||||||
|
&startup::SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(browser.status, 409);
|
||||||
|
assert!(
|
||||||
|
std::str::from_utf8(browser.body())
|
||||||
|
.unwrap()
|
||||||
|
.contains("session.record")
|
||||||
|
);
|
||||||
server.abort();
|
server.abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,193 @@
|
|||||||
|
//! Runtime API HTTP integration. No HTTP status is converted into execution facts.
|
||||||
|
use api_macros::reqwest::{ClientFailure, HttpClientError, ReceivedResponse};
|
||||||
|
use error_contract::{DecodeFailure, InternalTransfer};
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::RuntimeApiError;
|
||||||
|
|
||||||
|
fn unknown_operation() -> StartupOperationFacts {
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Unknown,
|
||||||
|
execution: ExecutionOutcome::Unknown,
|
||||||
|
finalize: None,
|
||||||
|
backend_compensation: None,
|
||||||
|
correlation: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// RuntimeManagementApi invokes this only after checking authenticated Runtime
|
||||||
|
/// scope. That scope permits the minimum public-safe cause schema, not arbitrary
|
||||||
|
/// internal-transfer details. Resolve the public projection afresh on each call;
|
||||||
|
/// no stored/received projection is relabeled as Browser authorization.
|
||||||
|
pub(crate) fn runtime_error_response(
|
||||||
|
error: &RuntimeApiError,
|
||||||
|
) -> Option<api_macros::HttpErrorResponse> {
|
||||||
|
let legacy;
|
||||||
|
let failure = match error.create_failure() {
|
||||||
|
Some(failure) => failure,
|
||||||
|
None => {
|
||||||
|
let startup = error.error.startup.as_ref()?;
|
||||||
|
legacy = WorkerCreateFailure::new(startup.clone(), unknown_operation());
|
||||||
|
match &legacy {
|
||||||
|
Ok(failure) => failure,
|
||||||
|
Err(_) => return Some(static_fallback()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let visibility = SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
};
|
||||||
|
Some(
|
||||||
|
match failure.problem(StartupBoundary::Runtime, &visibility, "") {
|
||||||
|
Ok(response) => api_macros::HttpErrorResponse {
|
||||||
|
status: response.status,
|
||||||
|
content_type: response.content_type,
|
||||||
|
body: response.into_body().into(),
|
||||||
|
},
|
||||||
|
// Never expose the legacy/raw attachment if projection cannot be made.
|
||||||
|
Err(_) => static_fallback(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn static_fallback() -> api_macros::HttpErrorResponse {
|
||||||
|
api_macros::HttpErrorResponse {
|
||||||
|
status: 500,
|
||||||
|
content_type: PROBLEM_CONTENT_TYPE,
|
||||||
|
body: error_contract::problem::STATIC_PROBLEM_FALLBACK
|
||||||
|
.to_vec()
|
||||||
|
.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_error(
|
||||||
|
status: u16,
|
||||||
|
failure: WorkerCreateFailure,
|
||||||
|
) -> Result<RuntimeApiError, ClientFailure> {
|
||||||
|
// Received text and legacy labels are not sources of local explanations.
|
||||||
|
// InternalTransfer is an internal observation policy, not Browser authority.
|
||||||
|
let projected = failure
|
||||||
|
.startup()
|
||||||
|
.primary
|
||||||
|
.project_for::<InternalTransfer, StartupCause>(&())
|
||||||
|
.map_err(|_| ClientFailure::ErrorResponseDecode { status })?;
|
||||||
|
let identity = projected.identity();
|
||||||
|
let mut error = RuntimeApiError::new(
|
||||||
|
status,
|
||||||
|
format!("{}.{}", identity.domain, identity.reason),
|
||||||
|
projected.explanation(),
|
||||||
|
);
|
||||||
|
error.error.startup = Some(failure.startup().clone());
|
||||||
|
Ok(error.with_create_failure(failure))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn local_observation(
|
||||||
|
status: u16,
|
||||||
|
reason: StartupBoundaryReason,
|
||||||
|
) -> Result<RuntimeApiError, ClientFailure> {
|
||||||
|
let failure = WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause { reason }),
|
||||||
|
unknown_operation(),
|
||||||
|
)
|
||||||
|
.map_err(|_| ClientFailure::ErrorResponseDecode { status })?;
|
||||||
|
normalized_error(status, failure)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A deliberately narrow legacy schema. Bounded typed parsing avoids arbitrary JSON
|
||||||
|
// allocation; startup attachment strings remain opaque until domain codec validation.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct LegacyError {
|
||||||
|
error: LegacyDetail,
|
||||||
|
}
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct LegacyDetail {
|
||||||
|
code: String,
|
||||||
|
message: String,
|
||||||
|
startup: Option<WorkerStartupFailure>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpClientError for RuntimeApiError {
|
||||||
|
fn decode_error(response: &ReceivedResponse) -> Result<Self, ClientFailure> {
|
||||||
|
let status = response.status().as_u16();
|
||||||
|
if !(400..=599).contains(&status) {
|
||||||
|
return Err(ClientFailure::ErrorResponseDecode { status });
|
||||||
|
}
|
||||||
|
// The generated client has already bounded collection; enforce this
|
||||||
|
// smaller contract limit independently before any schema allocation.
|
||||||
|
if response.body().len() > MAX_STARTUP_PROBLEM_BYTES {
|
||||||
|
return local_observation(status, StartupBoundaryReason::TooLarge);
|
||||||
|
}
|
||||||
|
let content_type = match response.content_type() {
|
||||||
|
Ok(content_type) => content_type,
|
||||||
|
Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse),
|
||||||
|
};
|
||||||
|
let media_type = content_type.split(';').next().unwrap_or("").trim();
|
||||||
|
if media_type.eq_ignore_ascii_case(PROBLEM_CONTENT_TYPE) {
|
||||||
|
return match decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
status,
|
||||||
|
content_type,
|
||||||
|
StartupBoundary::Runtime,
|
||||||
|
) {
|
||||||
|
DecodeStartupProblem::Known(failure) => normalized_error(status, failure),
|
||||||
|
DecodeStartupProblem::UnsupportedVersion => {
|
||||||
|
local_observation(status, StartupBoundaryReason::UnsupportedVersion)
|
||||||
|
}
|
||||||
|
DecodeStartupProblem::UnsupportedReason => {
|
||||||
|
local_observation(status, StartupBoundaryReason::UnsupportedReason)
|
||||||
|
}
|
||||||
|
DecodeStartupProblem::Invalid => {
|
||||||
|
local_observation(status, StartupBoundaryReason::InvalidResponse)
|
||||||
|
}
|
||||||
|
DecodeStartupProblem::TooLarge => {
|
||||||
|
local_observation(status, StartupBoundaryReason::TooLarge)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if !media_type.eq_ignore_ascii_case("application/json") {
|
||||||
|
return local_observation(status, StartupBoundaryReason::InvalidResponse);
|
||||||
|
}
|
||||||
|
// The typed legacy parser has fixed nesting and rejects unknown fields.
|
||||||
|
// Its total allocation/opaque attachment size is capped by the byte limit.
|
||||||
|
let legacy = match serde_json::from_slice::<LegacyError>(response.body()) {
|
||||||
|
Ok(legacy) => legacy.error,
|
||||||
|
Err(_) => return local_observation(status, StartupBoundaryReason::Decode),
|
||||||
|
};
|
||||||
|
if error_contract::validate_identifier(&legacy.code).is_err()
|
||||||
|
|| error_contract::validate_explanation(&legacy.message).is_err()
|
||||||
|
{
|
||||||
|
return local_observation(status, StartupBoundaryReason::InvalidResponse);
|
||||||
|
}
|
||||||
|
let Some(mut startup) = legacy.startup else {
|
||||||
|
// Preserve ordinary legacy errors exactly, including their actual status.
|
||||||
|
return Ok(RuntimeApiError::new(status, legacy.code, legacy.message));
|
||||||
|
};
|
||||||
|
let projected = match startup.primary.decode::<StartupCause>() {
|
||||||
|
DecodeFailure::Known(received) => match error_contract::project_failure::<
|
||||||
|
InternalTransfer,
|
||||||
|
_,
|
||||||
|
>(received.facts(), &())
|
||||||
|
{
|
||||||
|
Ok(projected) => projected,
|
||||||
|
Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse),
|
||||||
|
},
|
||||||
|
DecodeFailure::Corrupt(error_contract::ContractError::TooLarge) => {
|
||||||
|
return local_observation(status, StartupBoundaryReason::TooLarge);
|
||||||
|
}
|
||||||
|
// The legacy attachment codec intentionally exposes no unsupported
|
||||||
|
// metadata. Do not guess a reason/version from that opaque evidence.
|
||||||
|
_ => return local_observation(status, StartupBoundaryReason::InvalidResponse),
|
||||||
|
};
|
||||||
|
startup.primary = match StartupFailureAttachment::from_projected(&projected) {
|
||||||
|
Ok(primary) => primary,
|
||||||
|
Err(_) => return local_observation(status, StartupBoundaryReason::InvalidResponse),
|
||||||
|
};
|
||||||
|
match WorkerCreateFailure::new(startup, unknown_operation()) {
|
||||||
|
Ok(failure) => normalized_error(status, failure),
|
||||||
|
Err(_) => local_observation(status, StartupBoundaryReason::InvalidResponse),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
//! HTTP-independent snapshot of a Backend coordinator failure. This is a codec
|
||||||
|
//! for the existing reservation result, not a separate ledger or retry authority.
|
||||||
|
use super::*;
|
||||||
|
use crate::StoredStartupFailureAttachment;
|
||||||
|
use error_contract::{Storage, validate_json_input};
|
||||||
|
|
||||||
|
const STORAGE_VERSION: u16 = 1;
|
||||||
|
|
||||||
|
// Private DTO: only the concrete failure encoder may construct a stored snapshot.
|
||||||
|
// In particular no arbitrary error/source, HTTP response or private identity fits here.
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct StoredWorkerCreateFailure {
|
||||||
|
version: u16,
|
||||||
|
primary: StoredStartupFailureAttachment,
|
||||||
|
facts: StartupProblemFacts,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inspect versions from original input before attempting to understand future facts.
|
||||||
|
// IgnoredAny still checks JSON grammar without allocating arbitrary extension trees.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct StorageMetadata {
|
||||||
|
version: u16,
|
||||||
|
#[serde(rename = "primary")]
|
||||||
|
_primary: serde::de::IgnoredAny,
|
||||||
|
facts: ExtensionMetadata,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProjectFor<Storage> for StartupBoundaryCause {
|
||||||
|
type Authorization = ();
|
||||||
|
type Details = Self;
|
||||||
|
|
||||||
|
fn disclose(&self, _: &FailureFacts<Self>, _: &()) -> Result<Disclosure<Self>, ContractError> {
|
||||||
|
// The boundary domain stores only its local enum, never received context.
|
||||||
|
Ok(Disclosure::new(*self, boundary_explanation(self)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProjectFor<Storage> for StartupCause {
|
||||||
|
type Authorization = ();
|
||||||
|
type Details = Self;
|
||||||
|
|
||||||
|
fn disclose(
|
||||||
|
&self,
|
||||||
|
facts: &FailureFacts<Self>,
|
||||||
|
_: &(),
|
||||||
|
) -> Result<Disclosure<Self>, ContractError> {
|
||||||
|
match self {
|
||||||
|
Self::Session(cause) => {
|
||||||
|
let concrete = domain_facts(*cause, facts)?;
|
||||||
|
let disclosed = <SessionRegistrationCause as ProjectFor<Storage>>::disclose(
|
||||||
|
cause,
|
||||||
|
&concrete,
|
||||||
|
&(),
|
||||||
|
)?;
|
||||||
|
Ok(wrap(disclosed, Self::Session, session_explanation(cause)))
|
||||||
|
}
|
||||||
|
Self::Runtime(cause) => {
|
||||||
|
let concrete = domain_facts(cause.clone(), facts)?;
|
||||||
|
let disclosed =
|
||||||
|
<RuntimeStartupCause as ProjectFor<Storage>>::disclose(cause, &concrete, &())?;
|
||||||
|
Ok(wrap(
|
||||||
|
disclosed,
|
||||||
|
Self::Runtime,
|
||||||
|
runtime_explanation(cause).into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
Self::Boundary(cause) => {
|
||||||
|
let concrete = domain_facts(*cause, facts)?;
|
||||||
|
let disclosed =
|
||||||
|
<StartupBoundaryCause as ProjectFor<Storage>>::disclose(cause, &concrete, &())?;
|
||||||
|
Ok(wrap(
|
||||||
|
disclosed,
|
||||||
|
Self::Boundary,
|
||||||
|
boundary_explanation(cause).into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerCreateFailure {
|
||||||
|
/// Encode a bounded version-1 storage snapshot through each concrete domain's
|
||||||
|
/// Storage allowlist. Never encodes or parses an HTTP body. The caller selects
|
||||||
|
/// the Backend persistence marker (including prospective Persisted for an
|
||||||
|
/// atomic reservation write); encoding alone never proves persistence.
|
||||||
|
pub fn encode_storage(&self) -> Result<String, ContractError> {
|
||||||
|
let facts = self.public_facts();
|
||||||
|
facts.validate()?;
|
||||||
|
let projected = self
|
||||||
|
.startup
|
||||||
|
.primary
|
||||||
|
.project_for::<Storage, StartupCause>(&())?;
|
||||||
|
let dto = StoredWorkerCreateFailure {
|
||||||
|
version: STORAGE_VERSION,
|
||||||
|
primary: StoredStartupFailureAttachment::from_projected(&projected)?,
|
||||||
|
facts,
|
||||||
|
};
|
||||||
|
let encoded = serde_json::to_string(&dto).map_err(|_| ContractError::Encoding)?;
|
||||||
|
// Includes escaped attachment bytes and the entire enclosing snapshot.
|
||||||
|
validate_json_input(encoded.as_bytes(), budget())?;
|
||||||
|
Ok(encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub enum DecodeStoredWorkerCreateFailure {
|
||||||
|
Known(WorkerCreateFailure),
|
||||||
|
Unsupported,
|
||||||
|
Corrupt(ContractError),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a bounded storage snapshot without an HTTP classification or boundary.
|
||||||
|
/// Unsupported/corrupt evidence never becomes a fabricated primary cause, and a
|
||||||
|
/// Known result still needs current visibility for any later public projection.
|
||||||
|
/// Missing legacy columns are represented by the store's Option, not this codec.
|
||||||
|
pub fn decode_stored_worker_create_failure(input: &str) -> DecodeStoredWorkerCreateFailure {
|
||||||
|
match decode_checked(input) {
|
||||||
|
Ok(value) => value,
|
||||||
|
Err(error) => DecodeStoredWorkerCreateFailure::Corrupt(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_checked(input: &str) -> Result<DecodeStoredWorkerCreateFailure, ContractError> {
|
||||||
|
use DecodeStoredWorkerCreateFailure::*;
|
||||||
|
// Run the shared lexical guard over the whole input before any JSON parsing.
|
||||||
|
validate_json_input(input.as_bytes(), budget())?;
|
||||||
|
let metadata: StorageMetadata =
|
||||||
|
serde_json::from_str(input).map_err(|_| ContractError::Invalid)?;
|
||||||
|
if metadata.version == 0 || metadata.facts.version == 0 {
|
||||||
|
return Err(ContractError::Invalid);
|
||||||
|
}
|
||||||
|
if metadata.version != STORAGE_VERSION || metadata.facts.version != STARTUP_FACTS_VERSION {
|
||||||
|
return Ok(Unsupported);
|
||||||
|
}
|
||||||
|
// Original-byte typed decoding rejects duplicates rather than collapsing a Value.
|
||||||
|
let dto: StoredWorkerCreateFailure =
|
||||||
|
serde_json::from_str(input).map_err(|_| ContractError::InvalidDetails)?;
|
||||||
|
dto.facts.validate()?;
|
||||||
|
let received = match dto.primary.decode::<StartupCause>() {
|
||||||
|
DecodeFailure::Known(received) => received,
|
||||||
|
DecodeFailure::Unsupported(_) => return Ok(Unsupported),
|
||||||
|
DecodeFailure::Corrupt(error) => return Err(error),
|
||||||
|
};
|
||||||
|
// Deserialization is not a capability. Project again instead of relabeling or
|
||||||
|
// retaining raw loaded attachment bytes (including injected explanations/context).
|
||||||
|
let projected = project_failure::<Storage, _>(received.facts(), &())?;
|
||||||
|
let primary = StartupFailureAttachment::from_projected(&projected)?;
|
||||||
|
let facts = dto.facts;
|
||||||
|
let failure = WorkerCreateFailure::new(
|
||||||
|
WorkerStartupFailure {
|
||||||
|
primary,
|
||||||
|
secondary: facts.secondary,
|
||||||
|
resources: facts.resources,
|
||||||
|
persistence: facts.persistence,
|
||||||
|
io_kind: facts.io_kind,
|
||||||
|
},
|
||||||
|
facts.operation,
|
||||||
|
)?
|
||||||
|
.with_backend_result_persistence(facts.backend_result_persistence);
|
||||||
|
Ok(Known(failure))
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
{
|
||||||
|
"detail": "Worker creation could not complete Session registration: Subject is not authorized for required operation session.record. Creation acceptance and execution outcome are reported separately.",
|
||||||
|
"instance": "/workers/create/fixture-812",
|
||||||
|
"status": 409,
|
||||||
|
"title": "Operation conflict",
|
||||||
|
"type": "urn:yoi:problem:conflict",
|
||||||
|
"yoi_failure": {
|
||||||
|
"category": "authorization",
|
||||||
|
"context": {
|
||||||
|
"causes": [],
|
||||||
|
"correlation": null,
|
||||||
|
"diagnostics": [],
|
||||||
|
"operations": [],
|
||||||
|
"supplementary_omitted": false
|
||||||
|
},
|
||||||
|
"details": {
|
||||||
|
"principal": "subject",
|
||||||
|
"reason": "permission_denied",
|
||||||
|
"required_operation": "session.record"
|
||||||
|
},
|
||||||
|
"details_version": 1,
|
||||||
|
"domain": "session_registration",
|
||||||
|
"explanation": "Worker creation could not complete Session registration: Subject is not authorized for required operation session.record. Creation acceptance and execution outcome are reported separately.",
|
||||||
|
"reason": "permission_denied",
|
||||||
|
"required_extensions": [],
|
||||||
|
"version": 1
|
||||||
|
},
|
||||||
|
"yoi_startup": {
|
||||||
|
"io_kind": "permission_denied",
|
||||||
|
"operation": {
|
||||||
|
"acceptance": "rejected",
|
||||||
|
"backend_compensation": null,
|
||||||
|
"correlation": "startup-fixture-812",
|
||||||
|
"execution": "rejected",
|
||||||
|
"finalize": null
|
||||||
|
},
|
||||||
|
"persistence": "outcome_unknown",
|
||||||
|
"resources": {
|
||||||
|
"cleanup": {
|
||||||
|
"failed": "workdir"
|
||||||
|
},
|
||||||
|
"durable_effects_possible": true,
|
||||||
|
"resources": "worker_startup"
|
||||||
|
},
|
||||||
|
"secondary": [
|
||||||
|
{
|
||||||
|
"failure": "workdir",
|
||||||
|
"kind": "cleanup"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "persistence",
|
||||||
|
"outcome": "outcome_unknown",
|
||||||
|
"phase": "metadata_save"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,918 @@
|
|||||||
|
//! Producer/consumer checks against the production constructor and decoder.
|
||||||
|
use error_contract::problem::ApiBoundary as _;
|
||||||
|
use error_contract::{
|
||||||
|
Browser, ContractError, DecodeFailure, FailureCause, FailureFacts, InternalTransfer,
|
||||||
|
OperationContext, ProjectFor, WireDetails, project_failure,
|
||||||
|
};
|
||||||
|
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
|
||||||
|
use runtime_api::startup::*;
|
||||||
|
use runtime_api::{
|
||||||
|
RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure,
|
||||||
|
StartupFailureAttachment, StartupFailurePhase, StartupIoKind, StartupPersistenceOutcome,
|
||||||
|
StartupResourceFacts, StartupResources, StartupSecondaryFailure, StartupStage,
|
||||||
|
WorkerStartupFailure,
|
||||||
|
};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
fn visible() -> SessionFailureVisibility {
|
||||||
|
SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn operation(acceptance: CreationAcceptance, execution: ExecutionOutcome) -> StartupOperationFacts {
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance,
|
||||||
|
execution,
|
||||||
|
finalize: None,
|
||||||
|
backend_compensation: None,
|
||||||
|
correlation: Some(StartupCorrelation::new("startup-fixture-812").unwrap()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn session() -> StartupCause {
|
||||||
|
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject))
|
||||||
|
}
|
||||||
|
fn failure(cause: StartupCause) -> WorkerCreateFailure {
|
||||||
|
WorkerCreateFailure::from_cause(
|
||||||
|
cause,
|
||||||
|
operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
fn decode(value: &Value, status: u16, boundary: StartupBoundary) -> DecodeStartupProblem {
|
||||||
|
decode_startup_problem(
|
||||||
|
&serde_json::to_vec(value).unwrap(),
|
||||||
|
status,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
boundary,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
fn known(result: DecodeStartupProblem) -> WorkerCreateFailure {
|
||||||
|
match result {
|
||||||
|
DecodeStartupProblem::Known(failure) => failure,
|
||||||
|
other => panic!("{other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn fixture_failure() -> WorkerCreateFailure {
|
||||||
|
let base = failure(session());
|
||||||
|
let (mut startup, _) = base.into_parts();
|
||||||
|
startup.secondary = vec![
|
||||||
|
StartupSecondaryFailure::Cleanup {
|
||||||
|
failure: StartupCleanupFailure::Workdir,
|
||||||
|
},
|
||||||
|
StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::MetadataSave,
|
||||||
|
outcome: StartupPersistenceOutcome::OutcomeUnknown,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
startup.resources = Some(StartupResourceFacts {
|
||||||
|
resources: StartupResources::WorkerStartup,
|
||||||
|
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir),
|
||||||
|
durable_effects_possible: true,
|
||||||
|
});
|
||||||
|
startup.persistence = Some(StartupPersistenceOutcome::OutcomeUnknown);
|
||||||
|
startup.io_kind = Some(StartupIoKind::PermissionDenied);
|
||||||
|
WorkerCreateFailure::new(
|
||||||
|
startup,
|
||||||
|
operation(CreationAcceptance::Rejected, ExecutionOutcome::Rejected),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn production_public_fixture_is_stable_and_consumable() {
|
||||||
|
let failure = fixture_failure();
|
||||||
|
let response = failure
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&visible(),
|
||||||
|
"/workers/create/fixture-812",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status, 409);
|
||||||
|
assert_eq!(response.content_type, PROBLEM_CONTENT_TYPE);
|
||||||
|
let value: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("tests/fixtures/startup-session-record-denied.problem.json");
|
||||||
|
let fixture: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
value, fixture,
|
||||||
|
"fixture must be generated by the public production constructor"
|
||||||
|
);
|
||||||
|
let decoded = known(decode(&fixture, 409, StartupBoundary::Browser));
|
||||||
|
assert_eq!(decoded.operation(), failure.operation());
|
||||||
|
assert_eq!(decoded.public_facts(), failure.public_facts());
|
||||||
|
assert_eq!(
|
||||||
|
decoded
|
||||||
|
.startup()
|
||||||
|
.primary
|
||||||
|
.decode::<StartupCause>()
|
||||||
|
.known_cause(),
|
||||||
|
session()
|
||||||
|
);
|
||||||
|
assert!(fixture["detail"].as_str().unwrap().contains("Subject"));
|
||||||
|
assert!(
|
||||||
|
fixture["detail"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("session.record")
|
||||||
|
);
|
||||||
|
assert!(fixture["detail"].as_str().unwrap().contains("Worker"));
|
||||||
|
assert!(fixture["yoi_startup"].get("primary").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Captured from the real signed Session registration → Runtime → Browser tests
|
||||||
|
/// in workspace-server. These are observed operation facts, not a declaration
|
||||||
|
/// that an HTTP conflict rejected creation or that successful cleanup erased effects.
|
||||||
|
#[test]
|
||||||
|
fn connected_browser_fixtures_preserve_observed_operation_and_storage_facts() {
|
||||||
|
for (body, status, cause) in [
|
||||||
|
(
|
||||||
|
include_bytes!("fixtures/startup-browser-session-record-denied.problem.json")
|
||||||
|
.as_slice(),
|
||||||
|
409,
|
||||||
|
SessionRegistrationCause::denied(SessionPrincipal::Subject),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
include_bytes!("fixtures/startup-browser-registration-response-lost.problem.json")
|
||||||
|
.as_slice(),
|
||||||
|
502,
|
||||||
|
SessionRegistrationCause::outcome_unknown(),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let failure = known(decode_startup_problem(
|
||||||
|
body,
|
||||||
|
status,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
failure
|
||||||
|
.startup()
|
||||||
|
.primary
|
||||||
|
.decode::<StartupCause>()
|
||||||
|
.known_cause(),
|
||||||
|
StartupCause::Session(cause)
|
||||||
|
);
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Unknown);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Errored);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().backend_compensation,
|
||||||
|
Some(CompensationOutcome::Unknown)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
failure.startup().persistence,
|
||||||
|
Some(StartupPersistenceOutcome::Persisted)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
failure.backend_result_persistence(),
|
||||||
|
Some(StartupPersistenceOutcome::Persisted)
|
||||||
|
);
|
||||||
|
let encoded = failure.encode_storage().unwrap();
|
||||||
|
let DecodeStoredWorkerCreateFailure::Known(saved) =
|
||||||
|
decode_stored_worker_create_failure(&encoded)
|
||||||
|
else {
|
||||||
|
panic!("connected result must survive HTTP-independent storage");
|
||||||
|
};
|
||||||
|
assert_eq!(saved.public_facts(), failure.public_facts());
|
||||||
|
let response = saved
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status, status);
|
||||||
|
let original: Value = serde_json::from_slice(body).unwrap();
|
||||||
|
let projected: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
assert_eq!(projected["detail"], original["detail"]);
|
||||||
|
assert_eq!(projected["yoi_startup"], original["yoi_startup"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A tiny test-only adapter so assertions never format raw received data.
|
||||||
|
trait CauseFromDecode {
|
||||||
|
fn known_cause(self) -> StartupCause;
|
||||||
|
}
|
||||||
|
impl CauseFromDecode for DecodeFailure<StartupCause> {
|
||||||
|
fn known_cause(self) -> StartupCause {
|
||||||
|
match self {
|
||||||
|
DecodeFailure::Known(received) => received.facts().cause().clone(),
|
||||||
|
_ => panic!("expected known cause"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn original_domain_wire_identity_and_details_have_no_union_wrapper() {
|
||||||
|
for cause in [
|
||||||
|
session(),
|
||||||
|
StartupCause::Runtime(RuntimeStartupCause {
|
||||||
|
stage: StartupStage::Binding,
|
||||||
|
reason: RuntimeStartupReason::AttributionFailed,
|
||||||
|
}),
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause {
|
||||||
|
reason: StartupBoundaryReason::Transport,
|
||||||
|
}),
|
||||||
|
] {
|
||||||
|
let response = failure(cause.clone())
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let value: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
let id = cause.identity();
|
||||||
|
assert_eq!(value["yoi_failure"]["domain"], id.domain);
|
||||||
|
assert_eq!(value["yoi_failure"]["reason"], id.reason);
|
||||||
|
assert_eq!(value["yoi_failure"]["details_version"], id.details_version);
|
||||||
|
assert_eq!(
|
||||||
|
value["yoi_failure"]["details"],
|
||||||
|
serde_json::to_value(&cause).unwrap()
|
||||||
|
);
|
||||||
|
assert!(value["yoi_failure"]["details"].get("Session").is_none());
|
||||||
|
assert!(StartupCause::supports(
|
||||||
|
id.domain,
|
||||||
|
id.reason,
|
||||||
|
id.details_version
|
||||||
|
));
|
||||||
|
let decoded = known(decode(&value, response.status, StartupBoundary::Browser));
|
||||||
|
assert_eq!(
|
||||||
|
decoded
|
||||||
|
.startup()
|
||||||
|
.primary
|
||||||
|
.decode::<StartupCause>()
|
||||||
|
.known_cause(),
|
||||||
|
cause
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn browser_session_projection_requires_current_visibility_even_after_receipt() {
|
||||||
|
let failure = failure(session());
|
||||||
|
assert_eq!(
|
||||||
|
failure
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&SessionFailureVisibility::default(),
|
||||||
|
""
|
||||||
|
)
|
||||||
|
.unwrap_err(),
|
||||||
|
ContractError::InvalidDetails
|
||||||
|
);
|
||||||
|
let wire = failure
|
||||||
|
.problem(StartupBoundary::Runtime, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let received = known(decode_startup_problem(
|
||||||
|
wire.body(),
|
||||||
|
wire.status,
|
||||||
|
wire.content_type,
|
||||||
|
StartupBoundary::Runtime,
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
received
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&SessionFailureVisibility::default(),
|
||||||
|
""
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
received
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap()
|
||||||
|
.status,
|
||||||
|
409
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn boundary_classification_is_explicit_not_category_text_or_inner_status() {
|
||||||
|
let cases = [
|
||||||
|
(
|
||||||
|
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Operator)),
|
||||||
|
403,
|
||||||
|
403,
|
||||||
|
),
|
||||||
|
(session(), 409, 403),
|
||||||
|
(
|
||||||
|
StartupCause::Session(SessionRegistrationCause::attribution_missing()),
|
||||||
|
409,
|
||||||
|
409,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
StartupCause::Session(SessionRegistrationCause::binding_mismatch()),
|
||||||
|
409,
|
||||||
|
409,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
StartupCause::Session(SessionRegistrationCause::transport()),
|
||||||
|
502,
|
||||||
|
502,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
StartupCause::Session(SessionRegistrationCause::outcome_unknown()),
|
||||||
|
502,
|
||||||
|
502,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
StartupCause::Session(SessionRegistrationCause::invalid_response()),
|
||||||
|
502,
|
||||||
|
502,
|
||||||
|
),
|
||||||
|
];
|
||||||
|
for (cause, browser, runtime) in cases {
|
||||||
|
assert_eq!(
|
||||||
|
StartupBoundary::Browser.classify(&cause).status.code(),
|
||||||
|
browser
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
StartupBoundary::Runtime.classify(&cause).status.code(),
|
||||||
|
runtime
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (reason, status) in [
|
||||||
|
(StartupBoundaryReason::Timeout, 504),
|
||||||
|
(StartupBoundaryReason::Transport, 502),
|
||||||
|
(StartupBoundaryReason::Decode, 502),
|
||||||
|
(StartupBoundaryReason::InvalidResponse, 502),
|
||||||
|
(StartupBoundaryReason::TooLarge, 502),
|
||||||
|
(StartupBoundaryReason::UnsupportedVersion, 502),
|
||||||
|
(StartupBoundaryReason::UnsupportedReason, 502),
|
||||||
|
(StartupBoundaryReason::FinalizeFailed, 500),
|
||||||
|
] {
|
||||||
|
let cause = StartupCause::Boundary(StartupBoundaryCause { reason });
|
||||||
|
for boundary in [StartupBoundary::Browser, StartupBoundary::Runtime] {
|
||||||
|
assert_eq!(boundary.classify(&cause).status.code(), status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (reason, status) in [
|
||||||
|
(RuntimeStartupReason::SegmentEmpty, 400),
|
||||||
|
(RuntimeStartupReason::SystemPromptMissing, 400),
|
||||||
|
(RuntimeStartupReason::MetadataPending, 409),
|
||||||
|
(RuntimeStartupReason::MetadataMissing, 409),
|
||||||
|
(RuntimeStartupReason::MetadataInactive, 409),
|
||||||
|
(RuntimeStartupReason::ManifestMissing, 409),
|
||||||
|
(RuntimeStartupReason::AllocationFailed, 503),
|
||||||
|
(RuntimeStartupReason::SessionReadFailed, 500),
|
||||||
|
(RuntimeStartupReason::MetadataReadFailed, 500),
|
||||||
|
(RuntimeStartupReason::ProviderFailed, 500),
|
||||||
|
(RuntimeStartupReason::AttributionFailed, 500),
|
||||||
|
(RuntimeStartupReason::FeatureFailed, 500),
|
||||||
|
(RuntimeStartupReason::StartupFailed, 500),
|
||||||
|
] {
|
||||||
|
let cause = StartupCause::Runtime(RuntimeStartupCause {
|
||||||
|
stage: StartupStage::Controller,
|
||||||
|
reason,
|
||||||
|
});
|
||||||
|
for boundary in [StartupBoundary::Browser, StartupBoundary::Runtime] {
|
||||||
|
assert_eq!(boundary.classify(&cause).status.code(), status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn all_operation_facts_remain_independent_including_unknown_and_finalize() {
|
||||||
|
for acceptance in [
|
||||||
|
CreationAcceptance::Rejected,
|
||||||
|
CreationAcceptance::Accepted,
|
||||||
|
CreationAcceptance::Unknown,
|
||||||
|
] {
|
||||||
|
for execution in [
|
||||||
|
ExecutionOutcome::Accepted,
|
||||||
|
ExecutionOutcome::Busy,
|
||||||
|
ExecutionOutcome::Rejected,
|
||||||
|
ExecutionOutcome::Errored,
|
||||||
|
ExecutionOutcome::Unsupported,
|
||||||
|
ExecutionOutcome::Unknown,
|
||||||
|
] {
|
||||||
|
for compensation in [
|
||||||
|
CompensationOutcome::Completed,
|
||||||
|
CompensationOutcome::Residual,
|
||||||
|
CompensationOutcome::Unknown,
|
||||||
|
] {
|
||||||
|
let (startup, _) = fixture_failure().into_parts();
|
||||||
|
let mut operation = operation(acceptance, execution);
|
||||||
|
operation.finalize = Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::WorkerRegistry,
|
||||||
|
compensation,
|
||||||
|
});
|
||||||
|
let original = WorkerCreateFailure::new(startup, operation).unwrap();
|
||||||
|
let response = original
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status, 409); // primary cause only, not outcomes/cleanup
|
||||||
|
let received = known(decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
response.status,
|
||||||
|
response.content_type,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
));
|
||||||
|
assert_eq!(received.public_facts(), original.public_facts());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn transport_unknown_never_becomes_rejected_or_accepted_from_http_or_hints() {
|
||||||
|
let response = failure(StartupCause::Boundary(StartupBoundaryCause {
|
||||||
|
reason: StartupBoundaryReason::Timeout,
|
||||||
|
}))
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status, 504);
|
||||||
|
let mut value: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
value["retry_after"] = json!(0);
|
||||||
|
value["accepted"] = json!(false); // unrecognized RFC extension is not outcome authority
|
||||||
|
let decoded = known(decode(&value, 504, StartupBoundary::Browser));
|
||||||
|
assert_eq!(decoded.operation().acceptance, CreationAcceptance::Unknown);
|
||||||
|
assert_eq!(decoded.operation().execution, ExecutionOutcome::Unknown);
|
||||||
|
value["yoi_startup"]["operation"]
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.remove("acceptance");
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 504, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn received_text_and_context_are_reprojected_not_forwarded() {
|
||||||
|
let original = failure(session())
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let mut value: Value = serde_json::from_slice(original.body()).unwrap();
|
||||||
|
let secret = "RAW_UPSTREAM_CREDENTIAL_OR_PATH";
|
||||||
|
value["detail"] = json!(secret);
|
||||||
|
value["yoi_failure"]["explanation"] = json!(secret);
|
||||||
|
value["yoi_failure"]["context"]["diagnostics"] =
|
||||||
|
json!([{ "code":"raw", "explanation":secret }]);
|
||||||
|
value["yoi_failure"]["context"]["operations"] = json!([
|
||||||
|
{"operation":"session.record","stage":"registration","operation_ref":secret},
|
||||||
|
{"operation":secret,"stage":"raw","operation_ref":secret}]);
|
||||||
|
value["yoi_failure"]["context"]["correlation"] = json!(secret);
|
||||||
|
let received = known(decode(&value, 409, StartupBoundary::Browser));
|
||||||
|
let public = received
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let text = std::str::from_utf8(public.body()).unwrap();
|
||||||
|
assert!(!text.contains(secret));
|
||||||
|
assert!(text.contains("Subject"));
|
||||||
|
assert!(text.contains("session.record"));
|
||||||
|
assert_eq!(public.status, original.status);
|
||||||
|
let (mut startup, operation) = received.into_parts();
|
||||||
|
startup.secondary.reverse();
|
||||||
|
assert_eq!(
|
||||||
|
WorkerCreateFailure::new(startup, operation)
|
||||||
|
.unwrap()
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap()
|
||||||
|
.status,
|
||||||
|
409
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn concrete_domain_allowlists_filter_projection_context() {
|
||||||
|
let mut facts = FailureFacts::new(session()).unwrap();
|
||||||
|
let mut allowed = OperationContext::new("session.record", "authorization").unwrap();
|
||||||
|
allowed.operation_ref = Some("DO_NOT_PUBLISH".into());
|
||||||
|
facts.with_context(allowed).unwrap();
|
||||||
|
facts
|
||||||
|
.with_context(OperationContext::new("DO_NOT_PUBLISH", "raw").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let projected = project_failure::<Browser, _>(&facts, &visible()).unwrap();
|
||||||
|
assert_eq!(projected.context().operations.len(), 1);
|
||||||
|
assert_eq!(projected.context().operations[0].operation_ref, None);
|
||||||
|
assert!(
|
||||||
|
<StartupCause as ProjectFor<InternalTransfer>>::disclose(facts.cause(), &facts, &())
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_version_reason_and_bad_envelopes_are_distinct() {
|
||||||
|
let response = fixture_failure()
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let original: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
for field in ["version", "details_version"] {
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["yoi_failure"][field] = json!(77);
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::UnsupportedVersion
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["yoi_startup"]["version"] = json!(77);
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::UnsupportedVersion
|
||||||
|
);
|
||||||
|
value = original.clone();
|
||||||
|
value["yoi_failure"]["required_extensions"] = json!(["future-contract"]);
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::UnsupportedVersion
|
||||||
|
);
|
||||||
|
value = original.clone();
|
||||||
|
value["yoi_failure"]["reason"] = json!("future_reason");
|
||||||
|
value["yoi_failure"]["category"] = json!("future_category");
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::UnsupportedReason
|
||||||
|
);
|
||||||
|
value = original.clone();
|
||||||
|
value["yoi_failure"]["domain"] = json!("future_domain");
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::UnsupportedReason
|
||||||
|
);
|
||||||
|
for mutate in ["category", "domain", "reason"] {
|
||||||
|
let mut value = original.clone();
|
||||||
|
// Another KNOWN identity with mismatching typed details must be invalid.
|
||||||
|
value["yoi_failure"][mutate] = match mutate {
|
||||||
|
"category" => json!("internal"),
|
||||||
|
"domain" => json!("startup_boundary"),
|
||||||
|
_ => json!("transport"),
|
||||||
|
};
|
||||||
|
let result = decode(&value, 409, StartupBoundary::Browser);
|
||||||
|
if mutate != "domain" {
|
||||||
|
assert_eq!(result, DecodeStartupProblem::Invalid);
|
||||||
|
} else {
|
||||||
|
assert_eq!(result, DecodeStartupProblem::UnsupportedReason);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for field in ["version", "details_version"] {
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["yoi_failure"][field] = json!(0);
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoder_rejects_incoherence_bad_shapes_duplicate_fields_and_limits() {
|
||||||
|
let response = failure(session())
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let original: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
403,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
409,
|
||||||
|
"application/json",
|
||||||
|
StartupBoundary::Browser
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
409,
|
||||||
|
"Application/Problem+Json; charset=utf-8",
|
||||||
|
StartupBoundary::Browser
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::Known(_)
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
409,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Runtime
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
for (field, replacement) in [
|
||||||
|
("status", json!(502)),
|
||||||
|
("type", json!("urn:wrong")),
|
||||||
|
("title", json!("raw upstream")),
|
||||||
|
("detail", json!("different")),
|
||||||
|
("instance", json!("%zz")),
|
||||||
|
] {
|
||||||
|
let mut value = original.clone();
|
||||||
|
value[field] = replacement;
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for field in ["yoi_failure", "yoi_startup"] {
|
||||||
|
let mut value = original.clone();
|
||||||
|
value.as_object_mut().unwrap().remove(field);
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let text = std::str::from_utf8(response.body()).unwrap();
|
||||||
|
for (needle, duplicated) in [
|
||||||
|
("\"status\":409", "\"status\":409,\"status\":409"),
|
||||||
|
(
|
||||||
|
"\"acceptance\":\"unknown\"",
|
||||||
|
"\"acceptance\":\"unknown\",\"acceptance\":\"rejected\"",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"\"principal\":\"subject\"",
|
||||||
|
"\"principal\":\"subject\",\"principal\":\"operator\"",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let duplicate = text.replace(needle, duplicated);
|
||||||
|
assert_ne!(duplicate, text);
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(
|
||||||
|
duplicate.as_bytes(),
|
||||||
|
409,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let oversized = vec![b' '; MAX_STARTUP_PROBLEM_BYTES + 1];
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(
|
||||||
|
&oversized,
|
||||||
|
409,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::TooLarge
|
||||||
|
);
|
||||||
|
let deep = format!("{}0{}", "[".repeat(1000), "]".repeat(1000));
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(
|
||||||
|
deep.as_bytes(),
|
||||||
|
409,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
for bytes in [b"not json".as_slice(), b"\xff".as_slice(), b"{}".as_slice()] {
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(bytes, 409, PROBLEM_CONTENT_TYPE, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn safe_correlation_and_extension_validation_cannot_be_bypassed_by_serde() {
|
||||||
|
for unsafe_value in [
|
||||||
|
"",
|
||||||
|
"path/to/file",
|
||||||
|
"https://host",
|
||||||
|
"Bearer secret",
|
||||||
|
"query?token=secret",
|
||||||
|
"control\n",
|
||||||
|
] {
|
||||||
|
assert!(StartupCorrelation::new(unsafe_value).is_err());
|
||||||
|
assert!(serde_json::from_value::<StartupCorrelation>(json!(unsafe_value)).is_err());
|
||||||
|
}
|
||||||
|
assert!(StartupCorrelation::new("a".repeat(129)).is_err());
|
||||||
|
let failure = fixture_failure();
|
||||||
|
let mut facts = failure.public_facts();
|
||||||
|
facts.secondary = vec![
|
||||||
|
StartupSecondaryFailure::Cleanup {
|
||||||
|
failure: StartupCleanupFailure::Workdir
|
||||||
|
};
|
||||||
|
9
|
||||||
|
];
|
||||||
|
assert!(facts.validate().is_err());
|
||||||
|
let (mut startup, operation) = failure.into_parts();
|
||||||
|
startup.secondary = facts.secondary;
|
||||||
|
assert!(WorkerCreateFailure::new(startup, operation).is_err());
|
||||||
|
let response = fixture_failure()
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let mut value: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
value["yoi_startup"]["operation"]["correlation"] = json!("/private/path");
|
||||||
|
assert_eq!(
|
||||||
|
decode(&value, 409, StartupBoundary::Browser),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture_failure()
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "invalid%uri")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_or_corrupt_internal_attachments_are_retained_but_not_publishable() {
|
||||||
|
for raw in ["{invalid SECRET", "{}"] {
|
||||||
|
let primary: StartupFailureAttachment = serde_json::from_value(json!(raw)).unwrap();
|
||||||
|
let failure = WorkerCreateFailure::new(
|
||||||
|
WorkerStartupFailure {
|
||||||
|
primary,
|
||||||
|
secondary: vec![],
|
||||||
|
resources: None,
|
||||||
|
persistence: None,
|
||||||
|
io_kind: None,
|
||||||
|
},
|
||||||
|
operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
failure
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(!format!("{failure:?}").contains("SECRET"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unsupported_envelopes_still_reject_duplicate_safe_metadata() {
|
||||||
|
let response = failure(session())
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let text = std::str::from_utf8(response.body())
|
||||||
|
.unwrap()
|
||||||
|
.replace("\"version\":1", "\"version\":77");
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(
|
||||||
|
text.as_bytes(),
|
||||||
|
409,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::UnsupportedVersion
|
||||||
|
);
|
||||||
|
for (needle, duplicate) in [
|
||||||
|
(
|
||||||
|
"\"reason\":\"permission_denied\"",
|
||||||
|
"\"reason\":\"permission_denied\",\"reason\":\"future\"",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"\"domain\":\"session_registration\"",
|
||||||
|
"\"domain\":\"session_registration\",\"domain\":\"future\"",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"\"category\":\"authorization\"",
|
||||||
|
"\"category\":\"authorization\",\"category\":\"future\"",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let bytes = text.replace(needle, duplicate);
|
||||||
|
assert_ne!(bytes, text);
|
||||||
|
assert_eq!(
|
||||||
|
decode_startup_problem(
|
||||||
|
bytes.as_bytes(),
|
||||||
|
409,
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser
|
||||||
|
),
|
||||||
|
DecodeStartupProblem::Invalid
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn backend_compensation_is_independent_of_finalize_runtime_cleanup_and_outcomes() {
|
||||||
|
for backend in [
|
||||||
|
None,
|
||||||
|
Some(CompensationOutcome::Completed),
|
||||||
|
Some(CompensationOutcome::Residual),
|
||||||
|
Some(CompensationOutcome::Unknown),
|
||||||
|
] {
|
||||||
|
for finalize in [
|
||||||
|
None,
|
||||||
|
Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::WorkdirRegistry,
|
||||||
|
compensation: CompensationOutcome::Residual,
|
||||||
|
}),
|
||||||
|
] {
|
||||||
|
let (startup, mut operation) = fixture_failure().into_parts();
|
||||||
|
operation.backend_compensation = backend;
|
||||||
|
operation.finalize = finalize;
|
||||||
|
let original = WorkerCreateFailure::new(startup, operation).unwrap();
|
||||||
|
let response = original
|
||||||
|
.problem(StartupBoundary::Browser, &visible(), "")
|
||||||
|
.unwrap();
|
||||||
|
let received = known(decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
response.status,
|
||||||
|
response.content_type,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
));
|
||||||
|
assert_eq!(received.public_facts(), original.public_facts());
|
||||||
|
assert_eq!(received.operation().backend_compensation, backend);
|
||||||
|
assert_eq!(received.operation().finalize, finalize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for stage in [
|
||||||
|
FinalizeStage::TicketAssignmentBind,
|
||||||
|
FinalizeStage::TicketAssignmentCurrent,
|
||||||
|
FinalizeStage::TicketStateAccept,
|
||||||
|
FinalizeStage::WorkdirRegistry,
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_value::<FinalizeStage>(serde_json::to_value(stage).unwrap()).unwrap(),
|
||||||
|
stage
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn telemetry_uses_typed_context_and_operation_correlation_not_attachment_identity() {
|
||||||
|
use error_contract::{CorrelationRef, Telemetry, observation_fields};
|
||||||
|
let mut facts = FailureFacts::new(session()).unwrap();
|
||||||
|
let mut context = OperationContext::new("session.record", "registration").unwrap();
|
||||||
|
context.operation_ref = Some("PRIVATE_OPERATION_ID".into());
|
||||||
|
facts.with_context(context).unwrap();
|
||||||
|
facts
|
||||||
|
.with_context(OperationContext::new("SECRET_CONTEXT", "SECRET_STAGE").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
facts
|
||||||
|
.set_correlation(CorrelationRef::new("UNTRUSTED_ATTACHMENT_ID").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let projected = project_failure::<Telemetry, _>(&facts, &()).unwrap();
|
||||||
|
let fields = observation_fields(&projected);
|
||||||
|
assert_eq!(fields.domain, "session_registration");
|
||||||
|
assert_eq!(fields.reason, "permission_denied");
|
||||||
|
assert_eq!(fields.operation, Some("worker_startup"));
|
||||||
|
assert_eq!(fields.stage, Some("binding"));
|
||||||
|
assert_eq!(fields.correlation, Some("UNTRUSTED_ATTACHMENT_ID"));
|
||||||
|
assert!(
|
||||||
|
projected
|
||||||
|
.context()
|
||||||
|
.operations
|
||||||
|
.iter()
|
||||||
|
.all(|op| op.operation_ref.is_none())
|
||||||
|
);
|
||||||
|
|
||||||
|
let transfer = project_failure::<InternalTransfer, _>(&facts, &()).unwrap();
|
||||||
|
let startup = WorkerStartupFailure {
|
||||||
|
primary: StartupFailureAttachment::from_projected(&transfer).unwrap(),
|
||||||
|
secondary: vec![],
|
||||||
|
resources: None,
|
||||||
|
persistence: None,
|
||||||
|
io_kind: None,
|
||||||
|
};
|
||||||
|
let mut operation = operation(CreationAcceptance::Unknown, ExecutionOutcome::Unknown);
|
||||||
|
operation.correlation = Some(StartupCorrelation::new("fresh-backend-correlation").unwrap());
|
||||||
|
let failure = WorkerCreateFailure::new(startup, operation).unwrap();
|
||||||
|
let telemetry = failure.telemetry().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
observation_fields(&telemetry).correlation,
|
||||||
|
Some("fresh-backend-correlation")
|
||||||
|
);
|
||||||
|
assert_eq!(telemetry.identity(), session().identity());
|
||||||
|
let encoded = error_contract::encode_projected_failure(
|
||||||
|
&telemetry,
|
||||||
|
error_contract::Budget::new(32768, 32768),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let text = std::str::from_utf8(encoded.as_bytes()).unwrap();
|
||||||
|
for secret in [
|
||||||
|
"PRIVATE_OPERATION_ID",
|
||||||
|
"SECRET_CONTEXT",
|
||||||
|
"SECRET_STAGE",
|
||||||
|
"UNTRUSTED_ATTACHMENT_ID",
|
||||||
|
] {
|
||||||
|
assert!(!text.contains(secret));
|
||||||
|
}
|
||||||
|
let (startup, mut operation) = failure.into_parts();
|
||||||
|
operation.correlation = None;
|
||||||
|
assert_eq!(
|
||||||
|
observation_fields(
|
||||||
|
&WorkerCreateFailure::new(startup, operation)
|
||||||
|
.unwrap()
|
||||||
|
.telemetry()
|
||||||
|
.unwrap()
|
||||||
|
)
|
||||||
|
.correlation,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn public_dto_schema_is_accessible_for_generator() {
|
||||||
|
let schema = schemars::schema_for!(StartupProblemDetails);
|
||||||
|
let value = serde_json::to_value(schema).unwrap();
|
||||||
|
for field in [
|
||||||
|
"type",
|
||||||
|
"title",
|
||||||
|
"status",
|
||||||
|
"detail",
|
||||||
|
"instance",
|
||||||
|
"yoi_failure",
|
||||||
|
"yoi_startup",
|
||||||
|
] {
|
||||||
|
assert!(value["properties"].get(field).is_some(), "{field}");
|
||||||
|
}
|
||||||
|
let schema_text = serde_json::to_string(&value).unwrap();
|
||||||
|
assert!(!schema_text.contains("StartupFailureAttachment"));
|
||||||
|
assert!(schema_text.contains("CreationAcceptance"));
|
||||||
|
assert!(schema_text.contains("ExecutionOutcome"));
|
||||||
|
}
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
//! Actual generated RuntimeApi client error hook with bounded HTTP metadata/bytes.
|
||||||
|
use api_macros::HttpError as _;
|
||||||
|
use error_contract::{DecodeFailure, FailureCause};
|
||||||
|
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
|
||||||
|
use runtime_api::startup::*;
|
||||||
|
use runtime_api::{RuntimeApiClient, RuntimeApiError, client_support, server_support};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
fn failure() -> WorkerCreateFailure {
|
||||||
|
WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)),
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Accepted,
|
||||||
|
execution: ExecutionOutcome::Errored,
|
||||||
|
finalize: Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::TicketAssignmentBind,
|
||||||
|
compensation: CompensationOutcome::Unknown,
|
||||||
|
}),
|
||||||
|
backend_compensation: Some(CompensationOutcome::Residual),
|
||||||
|
correlation: Some(StartupCorrelation::new("backend-local-812").unwrap()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
fn cause(error: &RuntimeApiError) -> StartupCause {
|
||||||
|
match error
|
||||||
|
.create_failure()
|
||||||
|
.unwrap()
|
||||||
|
.startup()
|
||||||
|
.primary
|
||||||
|
.decode::<StartupCause>()
|
||||||
|
{
|
||||||
|
DecodeFailure::Known(received) => received.facts().cause().clone(),
|
||||||
|
_ => panic!("expected locally typed cause"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn assert_unknown(error: &RuntimeApiError, reason: StartupBoundaryReason) {
|
||||||
|
let operation = error.create_failure().unwrap().operation();
|
||||||
|
assert_eq!(operation.acceptance, CreationAcceptance::Unknown);
|
||||||
|
assert_eq!(operation.execution, ExecutionOutcome::Unknown);
|
||||||
|
assert_eq!(operation.finalize, None);
|
||||||
|
assert_eq!(operation.backend_compensation, None);
|
||||||
|
assert_eq!(
|
||||||
|
cause(error),
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause { reason })
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn receive(status: u16, types: Vec<&'static str>, body: Vec<u8>) -> RuntimeApiError {
|
||||||
|
use server_support::framework::{Body, Response, Router, get};
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
let router = Router::new().route(
|
||||||
|
"/v1/workers/{worker_id}",
|
||||||
|
get(move || {
|
||||||
|
let types = types.clone();
|
||||||
|
let body = body.clone();
|
||||||
|
async move {
|
||||||
|
let mut response = Response::builder().status(status);
|
||||||
|
for content_type in types {
|
||||||
|
response = response.header("content-type", content_type);
|
||||||
|
}
|
||||||
|
response.body(Body::from(body)).unwrap()
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
server_support::framework::serve(listener, router)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
});
|
||||||
|
let client = RuntimeApiClient::builder(&format!("http://{address}"))
|
||||||
|
.unwrap()
|
||||||
|
.request_timeout(std::time::Duration::from_secs(3))
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
let result = client.get_worker("test".into()).await;
|
||||||
|
server.abort();
|
||||||
|
match result {
|
||||||
|
Err(client_support::ClientError::Public {
|
||||||
|
status: actual,
|
||||||
|
error,
|
||||||
|
}) => {
|
||||||
|
assert_eq!(actual.as_u16(), status);
|
||||||
|
assert_eq!(error.status(), status);
|
||||||
|
error
|
||||||
|
}
|
||||||
|
_ => panic!("expected normalized public error"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_error_projects_public_schema_fresh_and_keeps_local_facts_off_serde() {
|
||||||
|
let failure = failure();
|
||||||
|
let error = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE")
|
||||||
|
.with_create_failure(failure.clone());
|
||||||
|
let legacy = serde_json::to_value(&error).unwrap();
|
||||||
|
assert!(legacy.get("create_failure").is_none());
|
||||||
|
let reparsed: RuntimeApiError = serde_json::from_value(legacy).unwrap();
|
||||||
|
assert!(reparsed.create_failure().is_none());
|
||||||
|
let response = error.custom_response().unwrap();
|
||||||
|
assert_eq!(response.status, 403); // runtime boundary is not Browser's 409
|
||||||
|
assert_eq!(response.content_type, PROBLEM_CONTENT_TYPE);
|
||||||
|
let text = std::str::from_utf8(response.body.as_ref()).unwrap();
|
||||||
|
assert!(!text.contains("PRIVATE_CODE"));
|
||||||
|
assert!(!text.contains("PRIVATE_MESSAGE"));
|
||||||
|
let decoded = match decode_startup_problem(
|
||||||
|
response.body.as_ref(),
|
||||||
|
response.status,
|
||||||
|
response.content_type,
|
||||||
|
StartupBoundary::Runtime,
|
||||||
|
) {
|
||||||
|
DecodeStartupProblem::Known(failure) => failure,
|
||||||
|
_ => panic!("expected known public-safe Runtime failure"),
|
||||||
|
};
|
||||||
|
assert_eq!(decoded.public_facts(), failure.public_facts());
|
||||||
|
assert_eq!(error.into_create_failure(), Some(failure));
|
||||||
|
let ordinary = RuntimeApiError::new(404, "not_found", "not found");
|
||||||
|
assert!(ordinary.custom_response().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn invalid_projection_uses_static_fallback_not_legacy_raw_serialization() {
|
||||||
|
let (mut startup, operation) = failure().into_parts();
|
||||||
|
startup.primary = serde_json::from_value(json!("RAW_PRIVATE_CREDENTIAL")).unwrap();
|
||||||
|
let error = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE")
|
||||||
|
.with_create_failure(WorkerCreateFailure::new(startup, operation).unwrap());
|
||||||
|
let response = error.custom_response().unwrap();
|
||||||
|
assert_eq!(response.status, 500);
|
||||||
|
assert_eq!(
|
||||||
|
response.body.as_ref(),
|
||||||
|
error_contract::problem::STATIC_PROBLEM_FALLBACK
|
||||||
|
);
|
||||||
|
let mut legacy = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE");
|
||||||
|
legacy.error.startup = error
|
||||||
|
.create_failure()
|
||||||
|
.map(|failure| failure.startup().clone());
|
||||||
|
assert_eq!(
|
||||||
|
legacy.custom_response().unwrap().body.as_ref(),
|
||||||
|
error_contract::problem::STATIC_PROBLEM_FALLBACK
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn generated_client_normalizes_known_problem_and_actual_metadata() {
|
||||||
|
let original = failure();
|
||||||
|
let response = RuntimeApiError::new(500, "PRIVATE_CODE", "PRIVATE_MESSAGE")
|
||||||
|
.with_create_failure(original.clone())
|
||||||
|
.custom_response()
|
||||||
|
.unwrap();
|
||||||
|
let mut value: Value = serde_json::from_slice(response.body.as_ref()).unwrap();
|
||||||
|
value["detail"] = json!("WIRE_SECRET_EXPLANATION");
|
||||||
|
value["yoi_failure"]["explanation"] = json!("WIRE_SECRET_EXPLANATION");
|
||||||
|
let error = receive(
|
||||||
|
response.status,
|
||||||
|
vec![PROBLEM_CONTENT_TYPE],
|
||||||
|
serde_json::to_vec(&value).unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
error.create_failure().unwrap().public_facts(),
|
||||||
|
original.public_facts()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cause(&error).identity(),
|
||||||
|
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject))
|
||||||
|
.identity()
|
||||||
|
);
|
||||||
|
assert!(!error.error.message.contains("WIRE_SECRET"));
|
||||||
|
let browser = error
|
||||||
|
.create_failure()
|
||||||
|
.unwrap()
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(browser.status, 409);
|
||||||
|
let wrong_status = receive(
|
||||||
|
409,
|
||||||
|
vec![PROBLEM_CONTENT_TYPE],
|
||||||
|
serde_json::to_vec(&value).unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_unknown(&wrong_status, StartupBoundaryReason::InvalidResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unknown_versions_reasons_and_invalid_responses_remain_local_unknown_observations() {
|
||||||
|
let response = failure()
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Runtime,
|
||||||
|
&SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let original: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
for (field, replacement, reason) in [
|
||||||
|
(
|
||||||
|
"version",
|
||||||
|
json!(99),
|
||||||
|
StartupBoundaryReason::UnsupportedVersion,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"details_version",
|
||||||
|
json!(99),
|
||||||
|
StartupBoundaryReason::UnsupportedVersion,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"reason",
|
||||||
|
json!("future_reason"),
|
||||||
|
StartupBoundaryReason::UnsupportedReason,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"category",
|
||||||
|
json!("invalid_category"),
|
||||||
|
StartupBoundaryReason::InvalidResponse,
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["yoi_failure"][field] = replacement;
|
||||||
|
let error = receive(
|
||||||
|
403,
|
||||||
|
vec![PROBLEM_CONTENT_TYPE],
|
||||||
|
serde_json::to_vec(&value).unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_unknown(&error, reason);
|
||||||
|
}
|
||||||
|
for content_types in [
|
||||||
|
vec![],
|
||||||
|
vec!["text/plain"],
|
||||||
|
vec![PROBLEM_CONTENT_TYPE, "application/json"],
|
||||||
|
] {
|
||||||
|
let error = receive(403, content_types, response.body().to_vec()).await;
|
||||||
|
assert_unknown(&error, StartupBoundaryReason::InvalidResponse);
|
||||||
|
}
|
||||||
|
let error = receive(
|
||||||
|
502,
|
||||||
|
vec![PROBLEM_CONTENT_TYPE],
|
||||||
|
vec![b'x'; MAX_STARTUP_PROBLEM_BYTES + 1],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_unknown(&error, StartupBoundaryReason::TooLarge);
|
||||||
|
let error = receive(
|
||||||
|
502,
|
||||||
|
vec![PROBLEM_CONTENT_TYPE],
|
||||||
|
b"not JSON PRIVATE_SECRET".to_vec(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_unknown(&error, StartupBoundaryReason::InvalidResponse);
|
||||||
|
assert!(!error.error.message.contains("PRIVATE_SECRET"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn legacy_errors_are_bounded_and_startup_attachments_reprojected_without_outcome_inference() {
|
||||||
|
let ordinary = RuntimeApiError::new(404, "old_not_found", "old not found");
|
||||||
|
let error = receive(
|
||||||
|
404,
|
||||||
|
vec!["application/json"],
|
||||||
|
serde_json::to_vec(&ordinary).unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(error.error, ordinary.error);
|
||||||
|
assert!(error.create_failure().is_none());
|
||||||
|
let original = failure();
|
||||||
|
let mut legacy = RuntimeApiError::new(502, "OLD_PRIVATE_CODE", "OLD_PRIVATE_MESSAGE");
|
||||||
|
legacy.error.startup = Some(original.startup().clone());
|
||||||
|
let error = receive(
|
||||||
|
502,
|
||||||
|
vec!["application/json; charset=utf-8"],
|
||||||
|
serde_json::to_vec(&legacy).unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
cause(&error).identity(),
|
||||||
|
cause(&RuntimeApiError::new(500, "unused", "unused").with_create_failure(original.clone()))
|
||||||
|
.identity()
|
||||||
|
);
|
||||||
|
let failure = error.create_failure().unwrap();
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Unknown);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown);
|
||||||
|
assert_eq!(failure.startup().secondary, original.startup().secondary);
|
||||||
|
assert!(!error.error.message.contains("OLD_PRIVATE"));
|
||||||
|
let mut value = serde_json::to_value(&legacy).unwrap();
|
||||||
|
value["error"]["startup"]["primary"] = json!("PRIVATE_MALFORMED_ATTACHMENT");
|
||||||
|
let malformed = receive(
|
||||||
|
502,
|
||||||
|
vec!["application/json"],
|
||||||
|
serde_json::to_vec(&value).unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_unknown(&malformed, StartupBoundaryReason::InvalidResponse);
|
||||||
|
let invalid = receive(
|
||||||
|
502,
|
||||||
|
vec!["application/json"],
|
||||||
|
b"{invalid PRIVATE_SECRET".to_vec(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_unknown(&invalid, StartupBoundaryReason::Decode);
|
||||||
|
let oversized = receive(
|
||||||
|
502,
|
||||||
|
vec!["application/json"],
|
||||||
|
vec![b'x'; MAX_STARTUP_PROBLEM_BYTES + 1],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_unknown(&oversized, StartupBoundaryReason::TooLarge);
|
||||||
|
}
|
||||||
@@ -0,0 +1,341 @@
|
|||||||
|
//! Storage codecs use the production attachment projector, never an HTTP body.
|
||||||
|
use error_contract::{Budget, ContractError, DecodeFailure, validate_json_input};
|
||||||
|
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
|
||||||
|
use runtime_api::startup::*;
|
||||||
|
use runtime_api::{
|
||||||
|
RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure,
|
||||||
|
StartupFailureAttachment, StartupIoKind, StartupPersistenceOutcome, StartupResourceFacts,
|
||||||
|
StartupResources, StartupSecondaryFailure, StartupStage, WorkerStartupFailure,
|
||||||
|
};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
fn operation() -> StartupOperationFacts {
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Accepted,
|
||||||
|
execution: ExecutionOutcome::Errored,
|
||||||
|
finalize: Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::Publication,
|
||||||
|
compensation: CompensationOutcome::Residual,
|
||||||
|
}),
|
||||||
|
backend_compensation: Some(CompensationOutcome::Unknown),
|
||||||
|
correlation: Some(StartupCorrelation::new("storage-test-812").unwrap()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn failure(cause: StartupCause) -> WorkerCreateFailure {
|
||||||
|
let base = WorkerCreateFailure::from_cause(cause, operation()).unwrap();
|
||||||
|
let (mut startup, operation) = base.into_parts();
|
||||||
|
startup.secondary.push(StartupSecondaryFailure::Cleanup {
|
||||||
|
failure: StartupCleanupFailure::Workdir,
|
||||||
|
});
|
||||||
|
startup.resources = Some(StartupResourceFacts {
|
||||||
|
resources: StartupResources::WorkerStartup,
|
||||||
|
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir),
|
||||||
|
durable_effects_possible: true,
|
||||||
|
});
|
||||||
|
// Session metadata write and Backend result write deliberately disagree.
|
||||||
|
startup.persistence = Some(StartupPersistenceOutcome::NotSaved);
|
||||||
|
startup.io_kind = Some(StartupIoKind::PermissionDenied);
|
||||||
|
WorkerCreateFailure::new(startup, operation)
|
||||||
|
.unwrap()
|
||||||
|
.with_backend_result_persistence(Some(StartupPersistenceOutcome::Persisted))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn session() -> StartupCause {
|
||||||
|
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn known(input: &str) -> WorkerCreateFailure {
|
||||||
|
match decode_stored_worker_create_failure(input) {
|
||||||
|
DecodeStoredWorkerCreateFailure::Known(failure) => failure,
|
||||||
|
other => panic!("expected known storage snapshot, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn change_primary(value: &mut Value, change: impl FnOnce(&mut Value)) {
|
||||||
|
let mut primary: Value = serde_json::from_str(value["primary"].as_str().unwrap()).unwrap();
|
||||||
|
change(&mut primary);
|
||||||
|
value["primary"] = Value::String(serde_json::to_string(&primary).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn storage_semantic_roundtrip_covers_domains_and_independent_facts() {
|
||||||
|
for cause in [
|
||||||
|
session(),
|
||||||
|
StartupCause::Runtime(RuntimeStartupCause {
|
||||||
|
stage: StartupStage::Feature,
|
||||||
|
reason: RuntimeStartupReason::FeatureFailed,
|
||||||
|
}),
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause {
|
||||||
|
reason: StartupBoundaryReason::FinalizeFailed,
|
||||||
|
}),
|
||||||
|
] {
|
||||||
|
let original = failure(cause.clone());
|
||||||
|
let encoded = original.encode_storage().unwrap();
|
||||||
|
let value: Value = serde_json::from_str(&encoded).unwrap();
|
||||||
|
assert_eq!(value["version"], 1);
|
||||||
|
assert!(value.get("status").is_none());
|
||||||
|
assert!(value.get("yoi_failure").is_none());
|
||||||
|
let loaded = known(&encoded);
|
||||||
|
assert_eq!(loaded.public_facts(), original.public_facts());
|
||||||
|
assert_eq!(loaded.operation(), original.operation());
|
||||||
|
assert_eq!(
|
||||||
|
loaded.startup().persistence,
|
||||||
|
Some(StartupPersistenceOutcome::NotSaved)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
loaded.backend_result_persistence(),
|
||||||
|
Some(StartupPersistenceOutcome::Persisted)
|
||||||
|
);
|
||||||
|
match loaded.startup().primary.decode::<StartupCause>() {
|
||||||
|
DecodeFailure::Known(received) => assert_eq!(received.facts().cause(), &cause),
|
||||||
|
_ => panic!("primary cause lost"),
|
||||||
|
}
|
||||||
|
assert_eq!(loaded.encode_storage().unwrap(), encoded);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn encoding_uses_current_marker_without_replacing_primary_or_inferring_persistence() {
|
||||||
|
let original = failure(session());
|
||||||
|
for marker in [
|
||||||
|
None,
|
||||||
|
Some(StartupPersistenceOutcome::Persisted),
|
||||||
|
Some(StartupPersistenceOutcome::NotSaved),
|
||||||
|
Some(StartupPersistenceOutcome::Conflict),
|
||||||
|
Some(StartupPersistenceOutcome::OutcomeUnknown),
|
||||||
|
] {
|
||||||
|
let updated = original.clone().with_backend_result_persistence(marker);
|
||||||
|
assert_eq!(updated.startup(), original.startup());
|
||||||
|
assert_eq!(updated.operation(), original.operation());
|
||||||
|
let loaded = known(&updated.encode_storage().unwrap());
|
||||||
|
assert_eq!(loaded.backend_result_persistence(), marker);
|
||||||
|
assert_eq!(loaded.startup().persistence, original.startup().persistence);
|
||||||
|
assert_eq!(loaded.public_facts(), updated.public_facts());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_absent_marker_is_none_in_storage_and_public_reader() {
|
||||||
|
let original = failure(session()).with_backend_result_persistence(None);
|
||||||
|
let mut value: Value = serde_json::from_str(&original.encode_storage().unwrap()).unwrap();
|
||||||
|
value["facts"]
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.remove("backend_result_persistence");
|
||||||
|
assert_eq!(known(&value.to_string()).backend_result_persistence(), None);
|
||||||
|
let response = original
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let public: Value = serde_json::from_slice(response.body()).unwrap();
|
||||||
|
assert!(
|
||||||
|
public["yoi_startup"]
|
||||||
|
.get("backend_result_persistence")
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
match decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
response.status,
|
||||||
|
response.content_type,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
) {
|
||||||
|
DecodeStartupProblem::Known(failure) => {
|
||||||
|
assert_eq!(failure.backend_result_persistence(), None)
|
||||||
|
}
|
||||||
|
other => panic!("legacy public response failed: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn public_receiver_restores_backend_marker_and_current_visibility_is_required() {
|
||||||
|
let original = failure(session());
|
||||||
|
let loaded = known(&original.encode_storage().unwrap());
|
||||||
|
assert!(
|
||||||
|
loaded
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&SessionFailureVisibility {
|
||||||
|
can_view_cause: false
|
||||||
|
},
|
||||||
|
""
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let response = loaded
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let received = match decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
response.status,
|
||||||
|
response.content_type,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
) {
|
||||||
|
DecodeStartupProblem::Known(failure) => failure,
|
||||||
|
other => panic!("public response failed: {other:?}"),
|
||||||
|
};
|
||||||
|
assert_eq!(received.public_facts(), original.public_facts());
|
||||||
|
let mut updated = operation();
|
||||||
|
updated.acceptance = CreationAcceptance::Unknown;
|
||||||
|
let received = received.with_operation(updated.clone());
|
||||||
|
assert_eq!(received.operation(), &updated);
|
||||||
|
assert_eq!(
|
||||||
|
received.backend_result_persistence(),
|
||||||
|
Some(StartupPersistenceOutcome::Persisted)
|
||||||
|
);
|
||||||
|
// The receiving Backend clears remote evidence before its own write.
|
||||||
|
assert_eq!(
|
||||||
|
received
|
||||||
|
.with_backend_result_persistence(None)
|
||||||
|
.backend_result_persistence(),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_versions_and_primary_identities_are_unsupported() {
|
||||||
|
let base: Value = serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap();
|
||||||
|
for mutate in [
|
||||||
|
|v: &mut Value| v["version"] = json!(2),
|
||||||
|
|v: &mut Value| v["facts"]["version"] = json!(2),
|
||||||
|
|v: &mut Value| change_primary(v, |p| p["version"] = json!(2)),
|
||||||
|
|v: &mut Value| change_primary(v, |p| p["reason"] = json!("future_reason")),
|
||||||
|
|v: &mut Value| {
|
||||||
|
change_primary(v, |p| {
|
||||||
|
p["required_extensions"] = json!(["future_extension"])
|
||||||
|
})
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
let mut value = base.clone();
|
||||||
|
mutate(&mut value);
|
||||||
|
assert_eq!(
|
||||||
|
decode_stored_worker_create_failure(&value.to_string()),
|
||||||
|
DecodeStoredWorkerCreateFailure::Unsupported
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_and_duplicate_known_records_are_corrupt() {
|
||||||
|
let base: Value = serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap();
|
||||||
|
for input in [
|
||||||
|
"".to_owned(),
|
||||||
|
"null".to_owned(),
|
||||||
|
"{}".to_owned(),
|
||||||
|
"{".to_owned(),
|
||||||
|
base.to_string()
|
||||||
|
.replacen("\"version\":1", "\"version\":1,\"version\":1", 1),
|
||||||
|
] {
|
||||||
|
assert!(matches!(
|
||||||
|
decode_stored_worker_create_failure(&input),
|
||||||
|
DecodeStoredWorkerCreateFailure::Corrupt(_)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for mutate in [
|
||||||
|
|v: &mut Value| v["version"] = json!(0),
|
||||||
|
|v: &mut Value| v["primary"] = json!("{broken"),
|
||||||
|
|v: &mut Value| v["facts"]["backend_result_persistence"] = json!("invented"),
|
||||||
|
|v: &mut Value| {
|
||||||
|
v["facts"]["secondary"] = json!(vec![json!({"kind":"cleanup","failure":"workdir"}); 9])
|
||||||
|
},
|
||||||
|
|v: &mut Value| change_primary(v, |p| p["category"] = json!("internal")),
|
||||||
|
] {
|
||||||
|
let mut value = base.clone();
|
||||||
|
mutate(&mut value);
|
||||||
|
assert!(matches!(
|
||||||
|
decode_stored_worker_create_failure(&value.to_string()),
|
||||||
|
DecodeStoredWorkerCreateFailure::Corrupt(_)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn complete_input_and_embedded_attachment_are_bounded_before_parsing() {
|
||||||
|
let oversized = " ".repeat(32 * 1024 + 1);
|
||||||
|
assert_eq!(
|
||||||
|
decode_stored_worker_create_failure(&oversized),
|
||||||
|
DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooLarge)
|
||||||
|
);
|
||||||
|
let deep = format!("{}0{}", "[".repeat(17), "]".repeat(17));
|
||||||
|
assert_eq!(
|
||||||
|
decode_stored_worker_create_failure(&deep),
|
||||||
|
DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep)
|
||||||
|
);
|
||||||
|
// A supported-looking prefix must not bypass the whole-input guard.
|
||||||
|
let deep = format!("{{\"version\":2,\"extra\":{deep}}}");
|
||||||
|
assert_eq!(
|
||||||
|
decode_stored_worker_create_failure(&deep),
|
||||||
|
DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep)
|
||||||
|
);
|
||||||
|
let mut value: Value =
|
||||||
|
serde_json::from_str(&failure(session()).encode_storage().unwrap()).unwrap();
|
||||||
|
change_primary(&mut value, |p| {
|
||||||
|
p["extra"] =
|
||||||
|
serde_json::from_str(&format!("{}0{}", "[".repeat(17), "]".repeat(17))).unwrap()
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
decode_stored_worker_create_failure(&value.to_string()),
|
||||||
|
DecodeStoredWorkerCreateFailure::Corrupt(ContractError::TooDeep)
|
||||||
|
);
|
||||||
|
// Shared wrapper is a bounds check, not a grammar validator or encoder.
|
||||||
|
assert_eq!(validate_json_input(b"[", Budget::new(1024, 1024)), Ok(()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn storage_encoder_and_reader_strip_untrusted_context_and_explanations() {
|
||||||
|
let original = failure(session());
|
||||||
|
let mut value: Value = serde_json::from_str(&original.encode_storage().unwrap()).unwrap();
|
||||||
|
change_primary(&mut value, |p| {
|
||||||
|
p["explanation"] = json!("TEST_ONLY_SECRET_RAW_HTTP");
|
||||||
|
p["context"]["correlation"] = json!("private-worker-id");
|
||||||
|
p["context"]["operations"] = json!([{
|
||||||
|
"operation":"session.record", "stage":"binding", "operation_ref":"TEST_ONLY_SECRET_RAW_HTTP"
|
||||||
|
}, {
|
||||||
|
"operation":"/private/path", "stage":"http-body", "operation_ref":null
|
||||||
|
}]);
|
||||||
|
});
|
||||||
|
// Simulate a received attachment without granting a projection capability.
|
||||||
|
let primary: StartupFailureAttachment =
|
||||||
|
serde_json::from_value(value["primary"].clone()).unwrap();
|
||||||
|
let (startup, operation) = original.clone().into_parts();
|
||||||
|
let incoming =
|
||||||
|
WorkerCreateFailure::new(WorkerStartupFailure { primary, ..startup }, operation).unwrap();
|
||||||
|
for encoded in [
|
||||||
|
incoming.encode_storage().unwrap(),
|
||||||
|
known(&value.to_string()).encode_storage().unwrap(),
|
||||||
|
] {
|
||||||
|
for secret in [
|
||||||
|
"TEST_ONLY_SECRET_RAW_HTTP",
|
||||||
|
"private-worker-id",
|
||||||
|
"/private/path",
|
||||||
|
"http-body",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!encoded.contains(secret),
|
||||||
|
"unexpected secret in stored snapshot"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let loaded = known(&encoded);
|
||||||
|
let response = loaded
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(!String::from_utf8_lossy(response.body()).contains("TEST_ONLY_SECRET"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,8 +10,9 @@ default = []
|
|||||||
typescript = ["dep:ts-rs", "protocol/typescript"]
|
typescript = ["dep:ts-rs", "protocol/typescript"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
error-contract = { workspace = true, features = ["json-schema"] }
|
||||||
feature-permissions.workspace = true
|
feature-permissions.workspace = true
|
||||||
api-macros = { workspace = true, features = ["reqwest", "axum", "openapi"] }
|
api-macros = { workspace = true, features = ["reqwest", "axum", "openapi", "error-contract"] }
|
||||||
memory.workspace = true
|
memory.workspace = true
|
||||||
plugin-plan.workspace = true
|
plugin-plan.workspace = true
|
||||||
merge-request.workspace = true
|
merge-request.workspace = true
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
#![doc = include_str!("../README.md")]
|
#![doc = include_str!("../README.md")]
|
||||||
|
mod worker_create_error;
|
||||||
|
pub use worker_create_error::WorkerCreateApiError;
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
@@ -1907,7 +1909,9 @@ pub trait ServerApi {
|
|||||||
"/api/workers",
|
"/api/workers",
|
||||||
status = 200,
|
status = 200,
|
||||||
error_status = 400,
|
error_status = 400,
|
||||||
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503],
|
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504],
|
||||||
|
additional_error_content_types = ["application/problem+json"],
|
||||||
|
client_error_hook = true,
|
||||||
bearer_auth = true,
|
bearer_auth = true,
|
||||||
browser_auth = true
|
browser_auth = true
|
||||||
)]
|
)]
|
||||||
@@ -1915,7 +1919,7 @@ pub trait ServerApi {
|
|||||||
&self,
|
&self,
|
||||||
#[extension] context: ServerRequestContext,
|
#[extension] context: ServerRequestContext,
|
||||||
#[body] request: CreateWorkspaceWorkerRequest,
|
#[body] request: CreateWorkspaceWorkerRequest,
|
||||||
) -> Result<BrowserCreateWorkerResponse, RepositoryApiError>;
|
) -> Result<BrowserCreateWorkerResponse, WorkerCreateApiError>;
|
||||||
|
|
||||||
#[get(
|
#[get(
|
||||||
"/api/w/{workspace_id}/orchestrator",
|
"/api/w/{workspace_id}/orchestrator",
|
||||||
@@ -1973,7 +1977,8 @@ pub trait ServerApi {
|
|||||||
"/api/w/{workspace_id}/worker-control/workers",
|
"/api/w/{workspace_id}/worker-control/workers",
|
||||||
status = 200,
|
status = 200,
|
||||||
error_status = 400,
|
error_status = 400,
|
||||||
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503],
|
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504],
|
||||||
|
client_error_hook = true,
|
||||||
openapi = false,
|
openapi = false,
|
||||||
)]
|
)]
|
||||||
async fn worker_control_spawn(
|
async fn worker_control_spawn(
|
||||||
@@ -1981,7 +1986,7 @@ pub trait ServerApi {
|
|||||||
#[extension] context: ServerRequestContext,
|
#[extension] context: ServerRequestContext,
|
||||||
#[path] workspace_id: String,
|
#[path] workspace_id: String,
|
||||||
#[body] request: CreateWorkspaceWorkerRequest,
|
#[body] request: CreateWorkspaceWorkerRequest,
|
||||||
) -> Result<BrowserCreateWorkerResponse, RepositoryApiError>;
|
) -> Result<BrowserCreateWorkerResponse, WorkerCreateApiError>;
|
||||||
#[post(
|
#[post(
|
||||||
"/api/w/{workspace_id}/worker-control/workers/{runtime_id}/{worker_id}/input",
|
"/api/w/{workspace_id}/worker-control/workers/{runtime_id}/{worker_id}/input",
|
||||||
status = 200,
|
status = 200,
|
||||||
@@ -2098,7 +2103,9 @@ pub trait ServerApi {
|
|||||||
"/api/w/{workspace_id}/workers",
|
"/api/w/{workspace_id}/workers",
|
||||||
status = 200,
|
status = 200,
|
||||||
error_status = 400,
|
error_status = 400,
|
||||||
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503],
|
additional_error_statuses = [401, 403, 404, 409, 500, 502, 503, 504],
|
||||||
|
additional_error_content_types = ["application/problem+json"],
|
||||||
|
client_error_hook = true,
|
||||||
bearer_auth = true,
|
bearer_auth = true,
|
||||||
browser_auth = true
|
browser_auth = true
|
||||||
)]
|
)]
|
||||||
@@ -2107,7 +2114,7 @@ pub trait ServerApi {
|
|||||||
#[extension] context: ServerRequestContext,
|
#[extension] context: ServerRequestContext,
|
||||||
#[path] workspace_id: String,
|
#[path] workspace_id: String,
|
||||||
#[body] request: CreateWorkspaceWorkerRequest,
|
#[body] request: CreateWorkspaceWorkerRequest,
|
||||||
) -> Result<BrowserCreateWorkerResponse, RepositoryApiError>;
|
) -> Result<BrowserCreateWorkerResponse, WorkerCreateApiError>;
|
||||||
|
|
||||||
#[get(
|
#[get(
|
||||||
"/api/w/{workspace_id}/workers/{worker_ref}",
|
"/api/w/{workspace_id}/workers/{worker_ref}",
|
||||||
@@ -11099,6 +11106,7 @@ pub struct BrowserWorkspaceOrchestratorResponse {
|
|||||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum WorkerOperationState {
|
pub enum WorkerOperationState {
|
||||||
|
Unknown,
|
||||||
Accepted,
|
Accepted,
|
||||||
Unsupported,
|
Unsupported,
|
||||||
Rejected,
|
Rejected,
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
//! Worker creation's HTTP projection is separate from its HTTP-independent result.
|
||||||
|
//! A decoded DTO is never an encoder capability: only a current visibility decision
|
||||||
|
//! can populate the cached public response. Legacy preflight routes keep their body.
|
||||||
|
use api_macros::{HttpError, HttpErrorResponse};
|
||||||
|
use runtime_api::startup::{
|
||||||
|
DecodeStartupProblem, SessionFailureVisibility, StartupBoundary, StartupProblemDetails,
|
||||||
|
WorkerCreateFailure, decode_startup_problem,
|
||||||
|
};
|
||||||
|
use schemars::{JsonSchema, Schema, SchemaGenerator};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::RepositoryApiError;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct WorkerCreateApiError {
|
||||||
|
legacy: RepositoryApiError,
|
||||||
|
failure: Option<WorkerCreateFailure>,
|
||||||
|
projected: Option<runtime_api::startup::ProblemResponse>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerCreateApiError {
|
||||||
|
/// The service has authenticated and authorized the current creation caller.
|
||||||
|
/// Visibility is a local decision; neither wire labels nor saved results supply it.
|
||||||
|
pub fn startup(failure: WorkerCreateFailure, visibility: &SessionFailureVisibility) -> Self {
|
||||||
|
let instance = failure
|
||||||
|
.operation()
|
||||||
|
.correlation
|
||||||
|
.as_ref()
|
||||||
|
.map(|correlation| format!("urn:yoi:worker-startup:{}", correlation.as_str()))
|
||||||
|
.unwrap_or_default();
|
||||||
|
let projected = failure
|
||||||
|
.problem(StartupBoundary::Browser, visibility, &instance)
|
||||||
|
.ok();
|
||||||
|
let status = projected.as_ref().map_or(500, |response| response.status);
|
||||||
|
Self {
|
||||||
|
legacy: RepositoryApiError::new(
|
||||||
|
status,
|
||||||
|
"worker_startup",
|
||||||
|
"Worker creation did not complete; inspect the typed result.",
|
||||||
|
Vec::new(),
|
||||||
|
),
|
||||||
|
failure: Some(failure),
|
||||||
|
projected,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn failure(&self) -> Option<&WorkerCreateFailure> {
|
||||||
|
self.failure.as_ref()
|
||||||
|
}
|
||||||
|
pub fn legacy(&self) -> &RepositoryApiError {
|
||||||
|
&self.legacy
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<RepositoryApiError> for WorkerCreateApiError {
|
||||||
|
fn from(legacy: RepositoryApiError) -> Self {
|
||||||
|
Self {
|
||||||
|
legacy,
|
||||||
|
failure: None,
|
||||||
|
projected: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpError for WorkerCreateApiError {
|
||||||
|
fn status_code(&self) -> u16 {
|
||||||
|
self.projected
|
||||||
|
.as_ref()
|
||||||
|
.map_or_else(|| self.legacy.status_code(), |response| response.status)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn custom_response(&self) -> Option<HttpErrorResponse> {
|
||||||
|
if let Some(projected) = &self.projected {
|
||||||
|
return Some(HttpErrorResponse {
|
||||||
|
status: projected.status,
|
||||||
|
content_type: projected.content_type,
|
||||||
|
body: projected.body().to_vec().into(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Projection failure does not serialize a possibly private internal result.
|
||||||
|
self.failure.as_ref().map(|_| {
|
||||||
|
error_contract::problem::problem_fallback(error_contract::Budget::new(
|
||||||
|
error_contract::MAX_FAILURE_BYTES,
|
||||||
|
error_contract::MAX_FAILURE_BYTES,
|
||||||
|
))
|
||||||
|
.expect("static fallback fits the fixed boundary budget")
|
||||||
|
.into()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl api_macros::HttpRequestError for WorkerCreateApiError {
|
||||||
|
fn from_request_rejection(status: u16, message: String) -> Self {
|
||||||
|
<RepositoryApiError as api_macros::HttpRequestError>::from_request_rejection(
|
||||||
|
status, message,
|
||||||
|
)
|
||||||
|
.into()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Serialize for WorkerCreateApiError {
|
||||||
|
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||||
|
// Generated HTTP adapters use custom_response. The ordinary serializer remains
|
||||||
|
// legacy-only and cannot serialize the retained internal startup attachment.
|
||||||
|
if let Some(projected) = &self.projected {
|
||||||
|
let dto: StartupProblemDetails =
|
||||||
|
serde_json::from_slice(projected.body()).map_err(serde::ser::Error::custom)?;
|
||||||
|
dto.serialize(serializer)
|
||||||
|
} else {
|
||||||
|
self.legacy.serialize(serializer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<'de> Deserialize<'de> for WorkerCreateApiError {
|
||||||
|
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||||
|
RepositoryApiError::deserialize(deserializer).map(Into::into)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
#[derive(JsonSchema)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum WorkerCreateErrorWire {
|
||||||
|
Problem(StartupProblemDetails),
|
||||||
|
Legacy(RepositoryApiError),
|
||||||
|
}
|
||||||
|
impl JsonSchema for WorkerCreateApiError {
|
||||||
|
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||||
|
"WorkerCreateApiError".into()
|
||||||
|
}
|
||||||
|
fn json_schema(generator: &mut SchemaGenerator) -> Schema {
|
||||||
|
WorkerCreateErrorWire::json_schema(generator)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl api_macros::openapi::OpenApiSchema for WorkerCreateApiError {
|
||||||
|
fn response_schema(
|
||||||
|
builder: &mut api_macros::openapi::OpenApiBuilder,
|
||||||
|
content_type: &str,
|
||||||
|
) -> Result<serde_json::Value, api_macros::openapi::OpenApiError> {
|
||||||
|
match content_type {
|
||||||
|
"application/json" => builder.schema_ref::<RepositoryApiError>(),
|
||||||
|
runtime_api::startup::PROBLEM_CONTENT_TYPE => {
|
||||||
|
builder.schema_ref::<StartupProblemDetails>()
|
||||||
|
}
|
||||||
|
_ => Err(api_macros::openapi::OpenApiError::InvalidContract(
|
||||||
|
"unsupported Worker creation error representation".into(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl api_macros::reqwest::HttpClientError for WorkerCreateApiError {
|
||||||
|
fn decode_error(
|
||||||
|
response: &api_macros::reqwest::ReceivedResponse,
|
||||||
|
) -> Result<Self, api_macros::reqwest::ClientFailure> {
|
||||||
|
use api_macros::reqwest::ClientFailure;
|
||||||
|
let content_type = response.content_type()?;
|
||||||
|
if content_type
|
||||||
|
.split(';')
|
||||||
|
.next()
|
||||||
|
.unwrap_or("")
|
||||||
|
.trim()
|
||||||
|
.eq_ignore_ascii_case(runtime_api::startup::PROBLEM_CONTENT_TYPE)
|
||||||
|
{
|
||||||
|
return match decode_startup_problem(
|
||||||
|
response.body(),
|
||||||
|
response.status().as_u16(),
|
||||||
|
content_type,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
) {
|
||||||
|
DecodeStartupProblem::Known(failure) => Ok(Self {
|
||||||
|
legacy: RepositoryApiError::new(
|
||||||
|
response.status().as_u16(),
|
||||||
|
"worker_startup",
|
||||||
|
"Worker creation returned a typed failure.",
|
||||||
|
Vec::new(),
|
||||||
|
),
|
||||||
|
failure: Some(failure),
|
||||||
|
projected: None,
|
||||||
|
}),
|
||||||
|
DecodeStartupProblem::TooLarge => Err(ClientFailure::ResponseTooLarge {
|
||||||
|
limit: runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
|
||||||
|
}),
|
||||||
|
DecodeStartupProblem::UnsupportedVersion
|
||||||
|
| DecodeStartupProblem::UnsupportedReason => {
|
||||||
|
Err(ClientFailure::UnsupportedErrorResponse {
|
||||||
|
status: response.status().as_u16(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
DecodeStartupProblem::Invalid => Err(ClientFailure::ErrorResponseDecode {
|
||||||
|
status: response.status().as_u16(),
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let legacy = response
|
||||||
|
.decode_json::<RepositoryApiError>(api_macros::reqwest::DecodeKind::PublicError)?;
|
||||||
|
Ok(legacy.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
//! Generated Backend client → isolated loopback HTTP response → typed result.
|
||||||
|
//! The server response producer is substituted; authorization and persistence are
|
||||||
|
//! exercised by workspace-server's connected tests, not by this codec connection.
|
||||||
|
use runtime_api::startup::*;
|
||||||
|
use server_api::{CreateWorkspaceWorkerRequest, ServerApiClient, client_support, server_support};
|
||||||
|
use server_support::framework::{Body, Response, Router, post};
|
||||||
|
|
||||||
|
struct ServerTask(tokio::task::JoinHandle<()>);
|
||||||
|
impl Drop for ServerTask {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request() -> CreateWorkspaceWorkerRequest {
|
||||||
|
serde_json::from_value(serde_json::json!({
|
||||||
|
"runtime_id": "runtime", "display_name": "Worker", "feature_connections": {}
|
||||||
|
}))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn every_creation_client_decodes_actual_problem_status_and_operation_facts() {
|
||||||
|
let timeout = WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause {
|
||||||
|
reason: StartupBoundaryReason::Timeout,
|
||||||
|
}),
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Unknown,
|
||||||
|
execution: ExecutionOutcome::Unknown,
|
||||||
|
finalize: None,
|
||||||
|
backend_compensation: None,
|
||||||
|
correlation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
&SessionFailureVisibility::default(),
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for (status, body) in [
|
||||||
|
(409, include_bytes!("../../runtime-api/tests/fixtures/startup-browser-session-record-denied.problem.json").to_vec()),
|
||||||
|
(502, include_bytes!("../../runtime-api/tests/fixtures/startup-browser-registration-response-lost.problem.json").to_vec()),
|
||||||
|
(504, timeout.into_body()),
|
||||||
|
] {
|
||||||
|
let DecodeStartupProblem::Known(expected) = decode_startup_problem(&body, status, PROBLEM_CONTENT_TYPE, StartupBoundary::Browser) else {
|
||||||
|
panic!("producer fixture must have a known typed cause");
|
||||||
|
};
|
||||||
|
let router = ["/api/workers", "/api/w/workspace/workers", "/api/w/workspace/worker-control/workers"]
|
||||||
|
.into_iter().fold(Router::new(), |router, path| {
|
||||||
|
let body = body.clone();
|
||||||
|
router.route(path, post(move || {
|
||||||
|
let body = body.clone();
|
||||||
|
async move { Response::builder().status(status).header("content-type", PROBLEM_CONTENT_TYPE).body(Body::from(body)).unwrap() }
|
||||||
|
}))
|
||||||
|
});
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
let _server = ServerTask(tokio::spawn(async move {
|
||||||
|
server_support::framework::serve(listener, router).await.unwrap();
|
||||||
|
}));
|
||||||
|
let client = ServerApiClient::builder(format!("http://{address}"))
|
||||||
|
.unwrap().request_timeout(std::time::Duration::from_secs(3)).build().unwrap();
|
||||||
|
for result in [
|
||||||
|
client.workspace_worker_create("workspace".into(), request()).await,
|
||||||
|
client.workspace_worker_create_alias(request()).await,
|
||||||
|
client.worker_control_spawn("workspace".into(), request()).await,
|
||||||
|
] {
|
||||||
|
let Err(client_support::ClientError::Public { status: actual_status, error }) = result else {
|
||||||
|
panic!("generated route must invoke the Problem decoder, not legacy Deserialize");
|
||||||
|
};
|
||||||
|
assert_eq!(actual_status.as_u16(), status);
|
||||||
|
let actual = error.failure().expect("retain typed non-HTTP failure");
|
||||||
|
assert_eq!(actual.startup().primary, expected.startup().primary);
|
||||||
|
assert_eq!(actual.public_facts(), expected.public_facts());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn creation_openapi_declares_gateway_timeout_with_both_error_representations() {
|
||||||
|
let document: serde_json::Value = serde_json::from_str(
|
||||||
|
&server_api::canonical_openapi_document()
|
||||||
|
.unwrap()
|
||||||
|
.to_json()
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for path in ["/api/workers", "/api/w/{workspace_id}/workers"] {
|
||||||
|
let content = &document["paths"][path]["post"]["responses"]["504"]["content"];
|
||||||
|
assert!(content.get("application/problem+json").is_some());
|
||||||
|
assert!(content.get("application/json").is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -63,6 +63,15 @@ pub enum RuntimeError {
|
|||||||
result: WorkerExecutionResult,
|
result: WorkerExecutionResult,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/// Creation failed after connection; the original error remains internal source
|
||||||
|
/// evidence. Display deliberately never renders it or the startup attachment.
|
||||||
|
#[error("worker creation failed")]
|
||||||
|
WorkerCreateFailed {
|
||||||
|
failure: Box<runtime_api::startup::WorkerCreateFailure>,
|
||||||
|
#[source]
|
||||||
|
source: Box<RuntimeError>,
|
||||||
|
},
|
||||||
|
|
||||||
#[error("limit {requested} exceeds maximum {max}")]
|
#[error("limit {requested} exceeds maximum {max}")]
|
||||||
LimitTooLarge { requested: usize, max: usize },
|
LimitTooLarge { requested: usize, max: usize },
|
||||||
|
|
||||||
@@ -149,11 +158,26 @@ pub enum RuntimeError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RuntimeError {
|
impl RuntimeError {
|
||||||
|
/// Only an owning create boundary or a Spawn result can identify a creation
|
||||||
|
/// failure. Independent Input errors retain their ordinary API contract.
|
||||||
|
pub fn create_failure(&self) -> Option<runtime_api::startup::WorkerCreateFailure> {
|
||||||
|
match self {
|
||||||
|
Self::WorkerCreateFailed { failure, .. } => Some((**failure).clone()),
|
||||||
|
Self::WorkerExecutionRejected { result, .. } => {
|
||||||
|
crate::execution::startup_create_failure(result)
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The original execution result, including primary facts and independent secondary
|
/// The original execution result, including primary facts and independent secondary
|
||||||
/// failures. Display is presentation only and must not be parsed to recover causes.
|
/// failures. Display is presentation only and must not be parsed to recover causes.
|
||||||
pub fn execution_result(&self) -> Option<&WorkerExecutionResult> {
|
pub fn execution_result(&self) -> Option<&WorkerExecutionResult> {
|
||||||
match self {
|
match self {
|
||||||
Self::WorkerExecutionRejected { result, .. } => Some(result),
|
Self::WorkerExecutionRejected { result, .. } => Some(result),
|
||||||
|
Self::WorkerCreateFailed { source, .. } => source.execution_result(),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1690,3 +1690,59 @@ impl fmt::Debug for WorkerExecutionBackendRef {
|
|||||||
.finish_non_exhaustive()
|
.finish_non_exhaustive()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Transfer a spawn result without deriving operation acceptance from its error text,
|
||||||
|
/// HTTP status, cleanup disposition, or persistence outcome. Other operation kinds
|
||||||
|
/// keep their existing result contracts and are not reclassified as creation.
|
||||||
|
pub fn startup_create_failure(
|
||||||
|
result: &WorkerExecutionResult,
|
||||||
|
) -> Result<Option<runtime_api::startup::WorkerCreateFailure>, error_contract::ContractError> {
|
||||||
|
use runtime_api::startup::{
|
||||||
|
CreationAcceptance, ExecutionOutcome, StartupOperationFacts, WorkerCreateFailure,
|
||||||
|
};
|
||||||
|
if result.operation != WorkerExecutionOperation::Spawn {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let execution = match result.outcome {
|
||||||
|
WorkerExecutionOutcome::Accepted => ExecutionOutcome::Accepted,
|
||||||
|
WorkerExecutionOutcome::Busy => ExecutionOutcome::Busy,
|
||||||
|
WorkerExecutionOutcome::Rejected => ExecutionOutcome::Rejected,
|
||||||
|
WorkerExecutionOutcome::Errored => ExecutionOutcome::Errored,
|
||||||
|
WorkerExecutionOutcome::Unsupported => ExecutionOutcome::Unsupported,
|
||||||
|
};
|
||||||
|
let acceptance = match result.outcome {
|
||||||
|
WorkerExecutionOutcome::Accepted => CreationAcceptance::Accepted,
|
||||||
|
WorkerExecutionOutcome::Rejected | WorkerExecutionOutcome::Unsupported => {
|
||||||
|
CreationAcceptance::Rejected
|
||||||
|
}
|
||||||
|
WorkerExecutionOutcome::Busy | WorkerExecutionOutcome::Errored => {
|
||||||
|
CreationAcceptance::Unknown
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let primary = match result.startup_failure.as_ref() {
|
||||||
|
Some(primary) => primary.clone(),
|
||||||
|
None => worker::startup_failure::runtime_startup_attachment(
|
||||||
|
protocol::startup_failure::RuntimeStartupCause {
|
||||||
|
stage: protocol::startup_failure::StartupStage::Controller,
|
||||||
|
reason: protocol::startup_failure::RuntimeStartupReason::StartupFailed,
|
||||||
|
},
|
||||||
|
)?,
|
||||||
|
};
|
||||||
|
WorkerCreateFailure::new(
|
||||||
|
runtime_api::WorkerStartupFailure {
|
||||||
|
primary,
|
||||||
|
secondary: result.secondary_failures.clone(),
|
||||||
|
resources: result.startup_resources,
|
||||||
|
persistence: result.startup_failure_persistence,
|
||||||
|
io_kind: result.startup_io_kind,
|
||||||
|
},
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance,
|
||||||
|
execution,
|
||||||
|
finalize: None,
|
||||||
|
correlation: None,
|
||||||
|
backend_compensation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map(Some)
|
||||||
|
}
|
||||||
|
|||||||
@@ -2326,6 +2326,7 @@ impl IntoResponse for RuntimeHttpWorkdirError {
|
|||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct RuntimeHttpRestError {
|
struct RuntimeHttpRestError {
|
||||||
|
create_failure: Option<runtime_api::startup::WorkerCreateFailure>,
|
||||||
status: StatusCode,
|
status: StatusCode,
|
||||||
code: String,
|
code: String,
|
||||||
message: String,
|
message: String,
|
||||||
@@ -2339,12 +2340,14 @@ impl RuntimeHttpRestError {
|
|||||||
code: code.into(),
|
code: code.into(),
|
||||||
message: message.into(),
|
message: message.into(),
|
||||||
startup: None,
|
startup: None,
|
||||||
|
create_failure: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn runtime(error: RuntimeError) -> Self {
|
fn runtime(error: RuntimeError) -> Self {
|
||||||
let status = status_for_runtime_error(&error);
|
let status = status_for_runtime_error(&error);
|
||||||
let code = code_for_runtime_error(&error);
|
let code = code_for_runtime_error(&error);
|
||||||
|
let create_failure = error.create_failure();
|
||||||
let startup = match &error {
|
let startup = match &error {
|
||||||
RuntimeError::WorkerExecutionRejected { result, .. } => startup_transfer(result),
|
RuntimeError::WorkerExecutionRejected { result, .. } => startup_transfer(result),
|
||||||
_ => None,
|
_ => None,
|
||||||
@@ -2359,6 +2362,7 @@ impl RuntimeHttpRestError {
|
|||||||
code,
|
code,
|
||||||
message,
|
message,
|
||||||
startup,
|
startup,
|
||||||
|
create_failure,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2391,6 +2395,13 @@ impl RuntimeHttpRestError {
|
|||||||
|
|
||||||
impl IntoResponse for RuntimeHttpRestError {
|
impl IntoResponse for RuntimeHttpRestError {
|
||||||
fn into_response(self) -> Response {
|
fn into_response(self) -> Response {
|
||||||
|
if let Some(failure) = self.create_failure {
|
||||||
|
return runtime_api::server_support::http_error_response(
|
||||||
|
self.status,
|
||||||
|
runtime_api::RuntimeApiError::new(self.status.as_u16(), self.code, self.message)
|
||||||
|
.with_create_failure(failure),
|
||||||
|
);
|
||||||
|
}
|
||||||
let body = RuntimeHttpErrorResponse {
|
let body = RuntimeHttpErrorResponse {
|
||||||
error: RuntimeHttpErrorDetail {
|
error: RuntimeHttpErrorDetail {
|
||||||
code: self.code,
|
code: self.code,
|
||||||
@@ -2422,6 +2433,7 @@ fn startup_transfer(
|
|||||||
|
|
||||||
fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
|
fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
|
||||||
match error {
|
match error {
|
||||||
|
RuntimeError::WorkerCreateFailed { .. } => StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
RuntimeError::WorkerNotFound { .. } | RuntimeError::ConfigBundleMissing { .. } => {
|
RuntimeError::WorkerNotFound { .. } | RuntimeError::ConfigBundleMissing { .. } => {
|
||||||
StatusCode::NOT_FOUND
|
StatusCode::NOT_FOUND
|
||||||
}
|
}
|
||||||
@@ -2531,6 +2543,7 @@ fn code_for_runtime_error(error: &RuntimeError) -> String {
|
|||||||
RuntimeError::ExecutionBackendUnavailable { .. } => {
|
RuntimeError::ExecutionBackendUnavailable { .. } => {
|
||||||
"execution_backend_unavailable".to_string()
|
"execution_backend_unavailable".to_string()
|
||||||
}
|
}
|
||||||
|
RuntimeError::WorkerCreateFailed { .. } => "worker_create_failed".to_string(),
|
||||||
RuntimeError::WorkerExecutionRejected { .. } => "worker_execution_rejected".to_string(),
|
RuntimeError::WorkerExecutionRejected { .. } => "worker_execution_rejected".to_string(),
|
||||||
RuntimeError::WorkspaceOwnerMismatch { .. } => "workspace_owner_mismatch".to_string(),
|
RuntimeError::WorkspaceOwnerMismatch { .. } => "workspace_owner_mismatch".to_string(),
|
||||||
RuntimeError::LimitTooLarge { .. } => "limit_too_large".to_string(),
|
RuntimeError::LimitTooLarge { .. } => "limit_too_large".to_string(),
|
||||||
|
|||||||
@@ -227,6 +227,9 @@ fn api_error(error: RuntimeHttpRestError) -> runtime_api::RuntimeApiError {
|
|||||||
let mut response =
|
let mut response =
|
||||||
runtime_api::RuntimeApiError::new(error.status.as_u16(), error.code, error.message);
|
runtime_api::RuntimeApiError::new(error.status.as_u16(), error.code, error.message);
|
||||||
response.error.startup = error.startup;
|
response.error.startup = error.startup;
|
||||||
|
if let Some(failure) = error.create_failure {
|
||||||
|
return response.with_create_failure(failure);
|
||||||
|
}
|
||||||
response
|
response
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ mod session_material;
|
|||||||
mod session_public;
|
mod session_public;
|
||||||
mod session_retention;
|
mod session_retention;
|
||||||
pub mod ssh_host_key_probe;
|
pub mod ssh_host_key_probe;
|
||||||
|
mod startup_observation;
|
||||||
pub mod worker_backend;
|
pub mod worker_backend;
|
||||||
pub mod worker_source;
|
pub mod worker_source;
|
||||||
pub mod working_directory;
|
pub mod working_directory;
|
||||||
|
|||||||
@@ -56,6 +56,11 @@ use protocol::subscription::{
|
|||||||
SubscriptionWorkerState, SubscriptionWorkerWorkdirAttachment,
|
SubscriptionWorkerState, SubscriptionWorkerWorkdirAttachment,
|
||||||
};
|
};
|
||||||
use protocol::{Event, Method};
|
use protocol::{Event, Method};
|
||||||
|
use runtime_api::startup::{
|
||||||
|
CompensationOutcome, CreationAcceptance, ExecutionOutcome, FinalizeFacts, FinalizeStage,
|
||||||
|
StartupBoundaryCause, StartupBoundaryReason, StartupCause, StartupCorrelation,
|
||||||
|
StartupOperationFacts, WorkerCreateFailure,
|
||||||
|
};
|
||||||
use server_api::WorkerRestoreState;
|
use server_api::WorkerRestoreState;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
@@ -1381,7 +1386,34 @@ impl Runtime {
|
|||||||
let worker_id = request.worker_id;
|
let worker_id = request.worker_id;
|
||||||
let workspace_id = scope.map(|scope| scope.workspace_id.as_str());
|
let workspace_id = scope.map(|scope| scope.workspace_id.as_str());
|
||||||
let started_at = std::time::Instant::now();
|
let started_at = std::time::Instant::now();
|
||||||
let result = self.create_worker_with_workspace_inner(request, scope);
|
let result = self
|
||||||
|
.create_worker_with_workspace_inner(request, scope)
|
||||||
|
.map_err(|error| {
|
||||||
|
// Use only the host-bound local identity, never received correlation.
|
||||||
|
// A poisoned/unbound Runtime cannot invent correlation authority.
|
||||||
|
let correlation = self.lock().ok().and_then(|state| {
|
||||||
|
state.runtime_identity.as_deref().map(|runtime_id| {
|
||||||
|
StartupCorrelation::for_worker(runtime_id, &worker_id.to_string())
|
||||||
|
})
|
||||||
|
});
|
||||||
|
match (error.create_failure(), correlation) {
|
||||||
|
(Some(failure), Some(correlation)) => {
|
||||||
|
let mut operation = failure.operation().clone();
|
||||||
|
operation.correlation = Some(correlation);
|
||||||
|
let failure = Box::new(failure.with_operation(operation));
|
||||||
|
match error {
|
||||||
|
RuntimeError::WorkerCreateFailed { source, .. } => {
|
||||||
|
RuntimeError::WorkerCreateFailed { failure, source }
|
||||||
|
}
|
||||||
|
source => RuntimeError::WorkerCreateFailed {
|
||||||
|
failure,
|
||||||
|
source: Box::new(source),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => error,
|
||||||
|
}
|
||||||
|
});
|
||||||
if let Err(error) = &result {
|
if let Err(error) = &result {
|
||||||
write_runtime_worker_create_failure(
|
write_runtime_worker_create_failure(
|
||||||
worker_id,
|
worker_id,
|
||||||
@@ -1528,13 +1560,13 @@ impl Runtime {
|
|||||||
WorkerExecutionSpawnResult::Rejected(mut result)
|
WorkerExecutionSpawnResult::Rejected(mut result)
|
||||||
| WorkerExecutionSpawnResult::RolledBack(mut result)
|
| WorkerExecutionSpawnResult::RolledBack(mut result)
|
||||||
| WorkerExecutionSpawnResult::Errored(mut result) => {
|
| WorkerExecutionSpawnResult::Errored(mut result) => {
|
||||||
if let Err((error, phase, outcome)) =
|
if let Err((_error, phase, outcome)) =
|
||||||
self.rollback_failed_create_with_phase(&worker_ref)
|
self.rollback_failed_create_with_phase(&worker_ref)
|
||||||
{
|
{
|
||||||
tracing::error!(
|
tracing::error!(
|
||||||
worker_id = %worker_ref.worker_id,
|
worker_id = %worker_ref.worker_id,
|
||||||
?phase,
|
?phase,
|
||||||
%error,
|
?outcome,
|
||||||
"failed to roll back Worker creation"
|
"failed to roll back Worker creation"
|
||||||
);
|
);
|
||||||
// A later failure must not make a confirmed record deletion
|
// A later failure must not make a confirmed record deletion
|
||||||
@@ -1601,7 +1633,8 @@ impl Runtime {
|
|||||||
});
|
});
|
||||||
tracing::error!(
|
tracing::error!(
|
||||||
worker_id = %worker_ref.worker_id,
|
worker_id = %worker_ref.worker_id,
|
||||||
%error,
|
?phase,
|
||||||
|
?outcome,
|
||||||
"failed to persist Worker create reconciliation evidence"
|
"failed to persist Worker create reconciliation evidence"
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
@@ -1620,80 +1653,119 @@ impl Runtime {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(mut initial_input) = {
|
// From Connected onward this owner, not the independent Input API,
|
||||||
let state = self.lock()?;
|
// supplies creation context. Later cleanup must not replace the source.
|
||||||
state
|
let mut execution = ExecutionOutcome::Unknown;
|
||||||
.worker(&worker_ref)?
|
let mut stage = FinalizeStage::InitialInput;
|
||||||
.request
|
let mut reason = StartupBoundaryReason::FinalizeFailed;
|
||||||
.as_ref()
|
let mut persistence_phase = None;
|
||||||
.and_then(|request| request.initial_input.clone())
|
let mut input_result = None;
|
||||||
} {
|
let result = (|| {
|
||||||
let expected_submission_id = initial_input
|
let initial_input = {
|
||||||
.submission_request_id
|
let state = self.lock()?;
|
||||||
.clone()
|
state
|
||||||
.filter(|request_id| !request_id.trim().is_empty())
|
.worker(&worker_ref)?
|
||||||
.unwrap_or_else(|| Uuid::now_v7().to_string());
|
.request
|
||||||
initial_input.submission_request_id = Some(expected_submission_id.clone());
|
.as_ref()
|
||||||
let dispatch_result = backend.dispatch_input(&worker_ref, initial_input.clone());
|
.and_then(|request| request.initial_input.clone())
|
||||||
if !dispatch_result.is_accepted() {
|
|
||||||
self.cleanup_connected_failed_create(&backend, &worker_ref)?;
|
|
||||||
return Err(RuntimeError::WorkerExecutionRejected {
|
|
||||||
worker_id: worker_ref.worker_id.clone(),
|
|
||||||
operation: dispatch_result.operation,
|
|
||||||
outcome: dispatch_result.outcome,
|
|
||||||
message: dispatch_result.message_or_default(),
|
|
||||||
result: dispatch_result,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
let has_durable_acceptance = dispatch_result
|
|
||||||
.submission
|
|
||||||
.as_ref()
|
|
||||||
.is_some_and(|ack| ack.submission_request_id == expected_submission_id);
|
|
||||||
if !has_durable_acceptance {
|
|
||||||
self.cleanup_connected_failed_create(&backend, &worker_ref)?;
|
|
||||||
let result = WorkerExecutionResult::rejected(
|
|
||||||
WorkerExecutionOperation::Input,
|
|
||||||
"execution backend accepted initial input without a durable submission acknowledgement",
|
|
||||||
);
|
|
||||||
return Err(RuntimeError::WorkerExecutionRejected {
|
|
||||||
worker_id: worker_ref.worker_id.clone(),
|
|
||||||
operation: result.operation,
|
|
||||||
outcome: result.outcome,
|
|
||||||
message: result.message_or_default(),
|
|
||||||
result,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
let detail = match self.commit_created_worker(
|
|
||||||
&worker_ref,
|
|
||||||
initial_worker_state.clone(),
|
|
||||||
workdir_attachments,
|
|
||||||
dispatch_result,
|
|
||||||
) {
|
|
||||||
Ok(detail) => detail,
|
|
||||||
Err(error) => {
|
|
||||||
self.cleanup_connected_failed_create(&backend, &worker_ref)?;
|
|
||||||
return Err(error);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
if let Err(error) = self.record_input_observation(&worker_ref, initial_input) {
|
if let Some(mut initial_input) = initial_input {
|
||||||
self.cleanup_connected_failed_create(&backend, &worker_ref)?;
|
let expected_submission_id = initial_input
|
||||||
return Err(error);
|
.submission_request_id
|
||||||
}
|
.clone()
|
||||||
Ok(detail)
|
.filter(|request_id| !request_id.trim().is_empty())
|
||||||
} else {
|
.unwrap_or_else(|| Uuid::now_v7().to_string());
|
||||||
match self.commit_created_worker(
|
initial_input.submission_request_id = Some(expected_submission_id.clone());
|
||||||
&worker_ref,
|
let dispatch_result = backend.dispatch_input(&worker_ref, initial_input.clone());
|
||||||
initial_worker_state,
|
let has_durable_acceptance = dispatch_result
|
||||||
workdir_attachments,
|
.submission
|
||||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
|
.as_ref()
|
||||||
) {
|
.is_some_and(|ack| ack.submission_request_id == expected_submission_id);
|
||||||
Ok(detail) => Ok(detail),
|
input_result = Some(dispatch_result.clone());
|
||||||
Err(error) => {
|
if !dispatch_result.is_accepted() {
|
||||||
self.cleanup_connected_failed_create(&backend, &worker_ref)?;
|
// Confirmed rejection is failed initial execution. A lost
|
||||||
Err(error)
|
// acknowledgement does not establish non-execution.
|
||||||
|
execution = match dispatch_result.outcome {
|
||||||
|
crate::execution::WorkerExecutionOutcome::Errored
|
||||||
|
if !has_durable_acceptance =>
|
||||||
|
{
|
||||||
|
ExecutionOutcome::Unknown
|
||||||
|
}
|
||||||
|
_ => ExecutionOutcome::Errored,
|
||||||
|
};
|
||||||
|
return Err(RuntimeError::WorkerExecutionRejected {
|
||||||
|
worker_id: worker_ref.worker_id,
|
||||||
|
operation: dispatch_result.operation,
|
||||||
|
outcome: dispatch_result.outcome,
|
||||||
|
message: dispatch_result.message_or_default(),
|
||||||
|
result: dispatch_result,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
if !has_durable_acceptance {
|
||||||
|
reason = StartupBoundaryReason::InvalidResponse;
|
||||||
|
return Err(RuntimeError::WorkerExecutionRejected {
|
||||||
|
worker_id: worker_ref.worker_id,
|
||||||
|
operation: dispatch_result.operation,
|
||||||
|
outcome: dispatch_result.outcome,
|
||||||
|
message: "initial input has no matching durable acknowledgement".into(),
|
||||||
|
result: dispatch_result,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
execution = ExecutionOutcome::Accepted;
|
||||||
|
stage = FinalizeStage::Publication;
|
||||||
|
let detail = self.commit_created_worker(
|
||||||
|
&worker_ref,
|
||||||
|
initial_worker_state,
|
||||||
|
workdir_attachments,
|
||||||
|
dispatch_result,
|
||||||
|
&mut persistence_phase,
|
||||||
|
)?;
|
||||||
|
self.record_input_observation(&worker_ref, initial_input)?;
|
||||||
|
Ok(detail)
|
||||||
|
} else {
|
||||||
|
execution = ExecutionOutcome::Accepted;
|
||||||
|
stage = FinalizeStage::Publication;
|
||||||
|
self.commit_created_worker(
|
||||||
|
&worker_ref,
|
||||||
|
initial_worker_state,
|
||||||
|
workdir_attachments,
|
||||||
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
|
||||||
|
&mut persistence_phase,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
})();
|
||||||
|
result.map_err(|source| {
|
||||||
|
let mut startup = connected_create_startup(&source, input_result.as_ref(), reason);
|
||||||
|
if let Some(phase) = persistence_phase {
|
||||||
|
push_create_secondary(
|
||||||
|
&mut startup,
|
||||||
|
StartupSecondaryFailure::Persistence {
|
||||||
|
phase,
|
||||||
|
outcome: create_persistence_outcome(&source),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let compensation =
|
||||||
|
self.cleanup_connected_failed_create(&backend, &worker_ref, &mut startup);
|
||||||
|
let failure = WorkerCreateFailure::new(
|
||||||
|
startup,
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Accepted,
|
||||||
|
execution,
|
||||||
|
finalize: Some(FinalizeFacts {
|
||||||
|
stage,
|
||||||
|
compensation,
|
||||||
|
}),
|
||||||
|
backend_compensation: None,
|
||||||
|
correlation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("connected create facts have bounded secondary observations");
|
||||||
|
RuntimeError::WorkerCreateFailed {
|
||||||
|
failure: Box::new(failure),
|
||||||
|
source: Box::new(source),
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// List Workers known to this Runtime.
|
/// List Workers known to this Runtime.
|
||||||
@@ -2267,7 +2339,7 @@ impl Runtime {
|
|||||||
Some(RuntimeWorkerRestoreResult::failed(
|
Some(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_canonical_write_unknown",
|
"worker_restore_canonical_write_unknown",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
)),
|
)),
|
||||||
None,
|
None,
|
||||||
));
|
));
|
||||||
@@ -2322,7 +2394,7 @@ impl Runtime {
|
|||||||
Some(RuntimeWorkerRestoreResult::failed(
|
Some(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_authority_changed",
|
"worker_restore_authority_changed",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
)),
|
)),
|
||||||
None,
|
None,
|
||||||
));
|
));
|
||||||
@@ -2432,7 +2504,7 @@ impl Runtime {
|
|||||||
Some(RuntimeWorkerRestoreResult::failed(
|
Some(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_admission_commit_unknown",
|
"worker_restore_admission_commit_unknown",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
)),
|
)),
|
||||||
None,
|
None,
|
||||||
));
|
));
|
||||||
@@ -2665,7 +2737,7 @@ impl Runtime {
|
|||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_saved_binding_not_ready",
|
"worker_restore_saved_binding_not_ready",
|
||||||
sanitize_worker_create_failure_message(&message),
|
sanitize_worker_restore_failure_message(&message),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let result = RuntimeWorkerRestoreResult::accepted(detail);
|
let result = RuntimeWorkerRestoreResult::accepted(detail);
|
||||||
@@ -2813,7 +2885,7 @@ impl Runtime {
|
|||||||
WorkerRestoreState::Rejected
|
WorkerRestoreState::Rejected
|
||||||
},
|
},
|
||||||
"worker_restore_preparation_rejected",
|
"worker_restore_preparation_rejected",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -2922,7 +2994,7 @@ impl Runtime {
|
|||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_observation_activation_pending",
|
"worker_restore_observation_activation_pending",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2933,7 +3005,7 @@ impl Runtime {
|
|||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::Rejected,
|
WorkerRestoreState::Rejected,
|
||||||
"worker_restore_preflight_rejected",
|
"worker_restore_preflight_rejected",
|
||||||
sanitize_worker_create_failure_message(&result.message_or_default()),
|
sanitize_worker_restore_failure_message(&result.message_or_default()),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3001,7 +3073,7 @@ impl Runtime {
|
|||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_activation_pending",
|
"worker_restore_activation_pending",
|
||||||
sanitize_worker_create_failure_message(&error),
|
sanitize_worker_restore_failure_message(&error),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
#[cfg(feature = "restore-checkpoints")]
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
@@ -3021,7 +3093,7 @@ impl Runtime {
|
|||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_observation_activation_pending",
|
"worker_restore_observation_activation_pending",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let worker = self.lock()?.worker(worker_ref)?.detail();
|
let worker = self.lock()?.worker(worker_ref)?.detail();
|
||||||
@@ -3030,7 +3102,7 @@ impl Runtime {
|
|||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_receipt_commit_pending",
|
"worker_restore_receipt_commit_pending",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
Ok(result)
|
Ok(result)
|
||||||
@@ -3057,7 +3129,7 @@ impl Runtime {
|
|||||||
let receipt = RuntimeWorkerRestoreResult::failed(
|
let receipt = RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::RolledBack,
|
WorkerRestoreState::RolledBack,
|
||||||
"worker_restore_decision_not_committed",
|
"worker_restore_decision_not_committed",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
);
|
);
|
||||||
if self.settle_restore_rollback(worker_ref, &receipt).is_ok() {
|
if self.settle_restore_rollback(worker_ref, &receipt).is_ok() {
|
||||||
return Ok(receipt);
|
return Ok(receipt);
|
||||||
@@ -3084,7 +3156,7 @@ impl Runtime {
|
|||||||
Ok(RuntimeWorkerRestoreResult::failed(
|
Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_commit_reconciliation_required",
|
"worker_restore_commit_reconciliation_required",
|
||||||
sanitize_worker_create_failure_message(&error.to_string()),
|
sanitize_worker_restore_failure_message(&error.to_string()),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3094,14 +3166,14 @@ impl Runtime {
|
|||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_reconciliation_rejected",
|
"worker_restore_reconciliation_rejected",
|
||||||
sanitize_worker_create_failure_message(&result.message_or_default()),
|
sanitize_worker_restore_failure_message(&result.message_or_default()),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
self.discard_restore_candidate_context(worker_ref, &candidate_context)?;
|
self.discard_restore_candidate_context(worker_ref, &candidate_context)?;
|
||||||
let receipt = RuntimeWorkerRestoreResult::failed(
|
let receipt = RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::Rejected,
|
WorkerRestoreState::Rejected,
|
||||||
"worker_restore_rejected",
|
"worker_restore_rejected",
|
||||||
sanitize_worker_create_failure_message(&result.message_or_default()),
|
sanitize_worker_restore_failure_message(&result.message_or_default()),
|
||||||
);
|
);
|
||||||
if let Err(error) = self.settle_restore_rollback(worker_ref, &receipt) {
|
if let Err(error) = self.settle_restore_rollback(worker_ref, &receipt) {
|
||||||
tracing::error!(
|
tracing::error!(
|
||||||
@@ -3121,7 +3193,7 @@ impl Runtime {
|
|||||||
let receipt = RuntimeWorkerRestoreResult::failed(
|
let receipt = RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::RolledBack,
|
WorkerRestoreState::RolledBack,
|
||||||
"worker_restore_rolled_back",
|
"worker_restore_rolled_back",
|
||||||
sanitize_worker_create_failure_message(&_result.message_or_default()),
|
sanitize_worker_restore_failure_message(&_result.message_or_default()),
|
||||||
);
|
);
|
||||||
self.discard_restore_candidate_context(worker_ref, &candidate_context)?;
|
self.discard_restore_candidate_context(worker_ref, &candidate_context)?;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
@@ -3174,7 +3246,7 @@ impl Runtime {
|
|||||||
Ok(RuntimeWorkerRestoreResult::failed(
|
Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_reconciliation_required",
|
"worker_restore_reconciliation_required",
|
||||||
sanitize_worker_create_failure_message(&format!(
|
sanitize_worker_restore_failure_message(&format!(
|
||||||
"Worker restore requires reconciliation ({:?}): {}",
|
"Worker restore requires reconciliation ({:?}): {}",
|
||||||
result.outcome,
|
result.outcome,
|
||||||
result.message_or_default(),
|
result.message_or_default(),
|
||||||
@@ -3194,7 +3266,7 @@ impl Runtime {
|
|||||||
Ok(RuntimeWorkerRestoreResult::failed(
|
Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_backend_outcome_unknown",
|
"worker_restore_backend_outcome_unknown",
|
||||||
sanitize_worker_create_failure_message(&_result.message_or_default()),
|
sanitize_worker_restore_failure_message(&_result.message_or_default()),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4007,6 +4079,7 @@ impl Runtime {
|
|||||||
initial_worker_state: protocol::WorkerStateSnapshot,
|
initial_worker_state: protocol::WorkerStateSnapshot,
|
||||||
workdir_attachments: Vec<crate::catalog::WorkingDirectoryAttachmentStatus>,
|
workdir_attachments: Vec<crate::catalog::WorkingDirectoryAttachmentStatus>,
|
||||||
result: WorkerExecutionResult,
|
result: WorkerExecutionResult,
|
||||||
|
persistence_phase: &mut Option<StartupFailurePhase>,
|
||||||
) -> Result<WorkerDetail, RuntimeError> {
|
) -> Result<WorkerDetail, RuntimeError> {
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
let detail = {
|
let detail = {
|
||||||
@@ -4030,41 +4103,155 @@ impl Runtime {
|
|||||||
worker.workdir_attachments = workdir_attachments;
|
worker.workdir_attachments = workdir_attachments;
|
||||||
worker.detail()
|
worker.detail()
|
||||||
};
|
};
|
||||||
|
*persistence_phase = Some(StartupFailurePhase::RuntimeSnapshotSave);
|
||||||
state.persist_runtime_snapshot()?;
|
state.persist_runtime_snapshot()?;
|
||||||
|
*persistence_phase = Some(StartupFailurePhase::RuntimeReconciliationRecordSave);
|
||||||
state.persist_worker(&worker_ref.worker_id)?;
|
state.persist_worker(&worker_ref.worker_id)?;
|
||||||
|
*persistence_phase = None;
|
||||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
state.publish_worker_upsert(worker_ref.worker_id)?;
|
||||||
Ok(detail)
|
Ok(detail)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Observe compensation separately from the original create failure. This
|
||||||
|
/// performs the same single stop and cleanup as before; no receipt lookup,
|
||||||
|
/// retry, reexecution or new durable lifecycle operation is introduced here.
|
||||||
fn cleanup_connected_failed_create(
|
fn cleanup_connected_failed_create(
|
||||||
&self,
|
&self,
|
||||||
backend: &WorkerExecutionBackendRef,
|
backend: &WorkerExecutionBackendRef,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
) -> Result<(), RuntimeError> {
|
startup: &mut runtime_api::WorkerStartupFailure,
|
||||||
|
) -> CompensationOutcome {
|
||||||
|
use protocol::startup_failure::{
|
||||||
|
StartupCleanup, StartupCleanupFailure, StartupResourceFacts, StartupResources,
|
||||||
|
};
|
||||||
let operation_id = WorkerLifecycleOperationId::new();
|
let operation_id = WorkerLifecycleOperationId::new();
|
||||||
let stop_result = backend.stop_worker_operation(WorkerExecutionStopRequest {
|
let stop_result = backend.stop_worker_operation(WorkerExecutionStopRequest {
|
||||||
operation_id,
|
operation_id,
|
||||||
worker_ref: worker_ref.clone(),
|
worker_ref: worker_ref.clone(),
|
||||||
});
|
});
|
||||||
if stop_result.is_accepted()
|
for secondary in &stop_result.secondary_failures {
|
||||||
|
push_create_secondary(startup, secondary.clone());
|
||||||
|
}
|
||||||
|
if startup.io_kind.is_none() {
|
||||||
|
startup.io_kind = stop_result.startup_io_kind;
|
||||||
|
}
|
||||||
|
// Preserve the original owner's resource facts, if present. In their
|
||||||
|
// absence, connection establishes startup resources but not their release.
|
||||||
|
let had_resources = startup.resources.is_some() || stop_result.startup_resources.is_some();
|
||||||
|
startup
|
||||||
|
.resources
|
||||||
|
.get_or_insert(
|
||||||
|
stop_result
|
||||||
|
.startup_resources
|
||||||
|
.unwrap_or(StartupResourceFacts {
|
||||||
|
resources: StartupResources::WorkerStartup,
|
||||||
|
cleanup: StartupCleanup::Required,
|
||||||
|
durable_effects_possible: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
if let Some(outcome) = stop_result.startup_failure_persistence {
|
||||||
|
push_create_secondary(
|
||||||
|
startup,
|
||||||
|
StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::MetadataSave,
|
||||||
|
outcome,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let terminated = stop_result.is_accepted()
|
||||||
&& stop_result.termination.as_ref().is_some_and(|termination| {
|
&& stop_result.termination.as_ref().is_some_and(|termination| {
|
||||||
termination.confirms_scope_shutdown(worker_ref, operation_id)
|
termination.confirms_scope_shutdown(worker_ref, operation_id)
|
||||||
})
|
});
|
||||||
{
|
if terminated {
|
||||||
return self.rollback_failed_create(worker_ref);
|
if !had_resources {
|
||||||
|
// Confirms scope shutdown only, not removal of Workdir contents
|
||||||
|
// or undoing already accepted initial-input effects.
|
||||||
|
startup.resources.as_mut().unwrap().cleanup = StartupCleanup::Confirmed;
|
||||||
|
}
|
||||||
|
match self.rollback_failed_create_with_phase(worker_ref) {
|
||||||
|
Ok(()) => CompensationOutcome::Completed,
|
||||||
|
Err((error, phase, outcome)) => {
|
||||||
|
if matches!(
|
||||||
|
phase,
|
||||||
|
StartupFailurePhase::RuntimeSnapshotSave
|
||||||
|
| StartupFailurePhase::RuntimeRemovalPublish
|
||||||
|
) {
|
||||||
|
push_create_secondary(
|
||||||
|
startup,
|
||||||
|
StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::RuntimeRecordDelete,
|
||||||
|
outcome: StartupPersistenceOutcome::Persisted,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if phase == StartupFailurePhase::RuntimeRemovalPublish {
|
||||||
|
push_create_secondary(
|
||||||
|
startup,
|
||||||
|
StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::RuntimeSnapshotSave,
|
||||||
|
outcome: StartupPersistenceOutcome::Persisted,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
record_create_persistence_failure(startup, &error, phase, outcome);
|
||||||
|
if outcome == StartupPersistenceOutcome::OutcomeUnknown {
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
} else {
|
||||||
|
CompensationOutcome::Residual
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let failed = !stop_result.is_accepted()
|
||||||
|
&& stop_result.outcome != crate::execution::WorkerExecutionOutcome::Errored;
|
||||||
|
let feature_failed = stop_result.termination.as_ref().is_some_and(|receipt| {
|
||||||
|
receipt.worker_ref == *worker_ref
|
||||||
|
&& receipt.stop_operation_id == operation_id
|
||||||
|
&& !receipt.feature_failures.is_empty()
|
||||||
|
});
|
||||||
|
if failed || feature_failed {
|
||||||
|
let failure = if feature_failed {
|
||||||
|
StartupCleanupFailure::Feature
|
||||||
|
} else if stop_result.outcome
|
||||||
|
== crate::execution::WorkerExecutionOutcome::Unsupported
|
||||||
|
{
|
||||||
|
StartupCleanupFailure::Unsupported
|
||||||
|
} else {
|
||||||
|
StartupCleanupFailure::Ownership
|
||||||
|
};
|
||||||
|
push_create_secondary(startup, StartupSecondaryFailure::Cleanup { failure });
|
||||||
|
if !had_resources {
|
||||||
|
startup.resources.as_mut().unwrap().cleanup = StartupCleanup::Failed(failure);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut phase = StartupFailurePhase::RuntimeSnapshotSave;
|
||||||
|
let retained = (|| {
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
let record = state.worker_mut(worker_ref)?;
|
||||||
|
record.worker_state = stop_result.worker_state.clone();
|
||||||
|
state.persist_runtime_snapshot()?;
|
||||||
|
phase = StartupFailurePhase::RuntimeReconciliationRecordSave;
|
||||||
|
state.persist_worker(&worker_ref.worker_id)
|
||||||
|
})();
|
||||||
|
match retained {
|
||||||
|
Ok(()) => push_create_secondary(
|
||||||
|
startup,
|
||||||
|
StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::RuntimeReconciliationRecordSave,
|
||||||
|
outcome: StartupPersistenceOutcome::Persisted,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
Err(error) => {
|
||||||
|
let outcome = create_persistence_outcome(&error);
|
||||||
|
record_create_persistence_failure(startup, &error, phase, outcome);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failed || feature_failed {
|
||||||
|
CompensationOutcome::Residual
|
||||||
|
} else {
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
}
|
||||||
}
|
}
|
||||||
let mut state = self.lock()?;
|
|
||||||
let record = state.worker_mut(worker_ref)?;
|
|
||||||
|
|
||||||
record.worker_state = stop_result.worker_state.clone();
|
|
||||||
state.persist_runtime_snapshot()?;
|
|
||||||
state.persist_worker(&worker_ref.worker_id)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn rollback_failed_create(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
|
|
||||||
self.rollback_failed_create_with_phase(worker_ref)
|
|
||||||
.map_err(|(error, _, _)| error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The legacy caller still receives its RuntimeError. Spawn failures instead
|
// The legacy caller still receives its RuntimeError. Spawn failures instead
|
||||||
@@ -5110,7 +5297,7 @@ impl Runtime {
|
|||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
let diagnostic_id = state.next_diagnostic_id;
|
let diagnostic_id = state.next_diagnostic_id;
|
||||||
state.next_diagnostic_id = state.next_diagnostic_id.saturating_add(1);
|
state.next_diagnostic_id = state.next_diagnostic_id.saturating_add(1);
|
||||||
let message = sanitize_worker_create_failure_message(
|
let message = sanitize_worker_restore_failure_message(
|
||||||
result
|
result
|
||||||
.message
|
.message
|
||||||
.as_deref()
|
.as_deref()
|
||||||
@@ -7148,7 +7335,7 @@ impl RuntimeState {
|
|||||||
result: &WorkerExecutionResult,
|
result: &WorkerExecutionResult,
|
||||||
receipt: &RuntimeWorkerRestoreResult,
|
receipt: &RuntimeWorkerRestoreResult,
|
||||||
) -> Result<(), RuntimeError> {
|
) -> Result<(), RuntimeError> {
|
||||||
let message = sanitize_worker_create_failure_message(&result.message_or_default());
|
let message = sanitize_worker_restore_failure_message(&result.message_or_default());
|
||||||
let diagnostic_id = self.next_diagnostic_id;
|
let diagnostic_id = self.next_diagnostic_id;
|
||||||
let workspace_id = self
|
let workspace_id = self
|
||||||
.workers
|
.workers
|
||||||
@@ -8381,25 +8568,90 @@ fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), R
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn worker_execution_failure_code(result: &WorkerExecutionResult) -> &'static str {
|
/// Keep the original result's typed facts even when publication is the failing
|
||||||
let message = result.message.as_deref().unwrap_or_default();
|
/// boundary. No message/path parsing is used to choose the fallback cause.
|
||||||
if message.contains("timed out waiting for durable Worker Submit acceptance") {
|
fn connected_create_startup(
|
||||||
"input_commit_timeout"
|
source: &RuntimeError,
|
||||||
} else if message.contains("worker adapter task did not complete within") {
|
result: Option<&WorkerExecutionResult>,
|
||||||
"adapter_task_timeout"
|
reason: StartupBoundaryReason,
|
||||||
} else if message.contains("failed to send Worker method") {
|
) -> runtime_api::WorkerStartupFailure {
|
||||||
"worker_method_send_failed"
|
let fallback = WorkerCreateFailure::from_cause(
|
||||||
} else if message.contains("worker rejected Submit") {
|
StartupCause::Boundary(StartupBoundaryCause { reason }),
|
||||||
"worker_submit_rejected"
|
StartupOperationFacts {
|
||||||
} else if message.contains("before durable acceptance") {
|
acceptance: CreationAcceptance::Accepted,
|
||||||
"worker_failed_before_input_commit"
|
execution: ExecutionOutcome::Unknown,
|
||||||
} else if message.contains("event stream closed") {
|
finalize: None,
|
||||||
"worker_event_stream_closed"
|
backend_compensation: None,
|
||||||
} else {
|
correlation: None,
|
||||||
"worker_execution_failed"
|
},
|
||||||
|
)
|
||||||
|
.expect("closed startup boundary cause is valid");
|
||||||
|
let (mut startup, _) = fallback.into_parts();
|
||||||
|
if let Some(result) = result {
|
||||||
|
if let Some(primary) = &result.startup_failure {
|
||||||
|
startup.primary = primary.clone();
|
||||||
|
}
|
||||||
|
for secondary in &result.secondary_failures {
|
||||||
|
push_create_secondary(&mut startup, secondary.clone());
|
||||||
|
}
|
||||||
|
startup.resources = result.startup_resources;
|
||||||
|
startup.persistence = result.startup_failure_persistence;
|
||||||
|
startup.io_kind = result.startup_io_kind;
|
||||||
|
}
|
||||||
|
if startup.io_kind.is_none() {
|
||||||
|
startup.io_kind = create_io_kind(source);
|
||||||
|
}
|
||||||
|
startup
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_io_kind(error: &RuntimeError) -> Option<protocol::startup_failure::StartupIoKind> {
|
||||||
|
match error {
|
||||||
|
RuntimeError::StoreIo { source, .. }
|
||||||
|
| RuntimeError::StoreCommitOutcomeUnknown { source, .. } => Some(source.kind().into()),
|
||||||
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn create_persistence_outcome(error: &RuntimeError) -> StartupPersistenceOutcome {
|
||||||
|
if matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) {
|
||||||
|
StartupPersistenceOutcome::OutcomeUnknown
|
||||||
|
} else {
|
||||||
|
StartupPersistenceOutcome::NotSaved
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_create_secondary(
|
||||||
|
startup: &mut runtime_api::WorkerStartupFailure,
|
||||||
|
secondary: StartupSecondaryFailure,
|
||||||
|
) {
|
||||||
|
// The create DTO has the core diagnostic bound; this is an observation,
|
||||||
|
// not another durable ledger. The original error remains boxed as source.
|
||||||
|
if startup.secondary.len() < error_contract::MAX_DIAGNOSTICS {
|
||||||
|
startup.secondary.push(secondary);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_create_persistence_failure(
|
||||||
|
startup: &mut runtime_api::WorkerStartupFailure,
|
||||||
|
error: &RuntimeError,
|
||||||
|
phase: StartupFailurePhase,
|
||||||
|
outcome: StartupPersistenceOutcome,
|
||||||
|
) {
|
||||||
|
push_create_secondary(
|
||||||
|
startup,
|
||||||
|
StartupSecondaryFailure::Persistence { phase, outcome },
|
||||||
|
);
|
||||||
|
if startup.io_kind.is_none() {
|
||||||
|
startup.io_kind = create_io_kind(error);
|
||||||
|
}
|
||||||
|
tracing::error!(
|
||||||
|
?phase,
|
||||||
|
?outcome,
|
||||||
|
"Worker create compensation persistence failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
fn runtime_worker_create_failure_fields(
|
fn runtime_worker_create_failure_fields(
|
||||||
error: &RuntimeError,
|
error: &RuntimeError,
|
||||||
) -> (&'static str, Option<String>, Option<String>) {
|
) -> (&'static str, Option<String>, Option<String>) {
|
||||||
@@ -8425,6 +8677,7 @@ fn runtime_worker_create_failure_fields(
|
|||||||
RuntimeError::ExecutionBackendUnavailable { .. } => {
|
RuntimeError::ExecutionBackendUnavailable { .. } => {
|
||||||
("execution_backend_unavailable", None, None)
|
("execution_backend_unavailable", None, None)
|
||||||
}
|
}
|
||||||
|
RuntimeError::WorkerCreateFailed { .. } => ("worker_create_failed", None, None),
|
||||||
RuntimeError::WorkerExecutionRejected {
|
RuntimeError::WorkerExecutionRejected {
|
||||||
operation, outcome, ..
|
operation, outcome, ..
|
||||||
} => (
|
} => (
|
||||||
@@ -8454,9 +8707,11 @@ fn runtime_worker_create_failure_fields(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const WORKER_CREATE_FAILURE_MESSAGE_LIMIT: usize = 512;
|
// Legacy Restore diagnostic presentation only. This is not a create failure
|
||||||
|
// projector or telemetry allowlist; creation uses startup_observation instead.
|
||||||
|
const WORKER_RESTORE_FAILURE_MESSAGE_LIMIT: usize = 512;
|
||||||
|
|
||||||
fn sanitize_worker_create_failure_message(message: &str) -> String {
|
fn sanitize_worker_restore_failure_message(message: &str) -> String {
|
||||||
let mut redact_next = false;
|
let mut redact_next = false;
|
||||||
let detail = message
|
let detail = message
|
||||||
.split_whitespace()
|
.split_whitespace()
|
||||||
@@ -8496,10 +8751,10 @@ fn sanitize_worker_create_failure_message(message: &str) -> String {
|
|||||||
})
|
})
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(" ");
|
.join(" ");
|
||||||
let truncated = detail.chars().count() > WORKER_CREATE_FAILURE_MESSAGE_LIMIT;
|
let truncated = detail.chars().count() > WORKER_RESTORE_FAILURE_MESSAGE_LIMIT;
|
||||||
let mut detail = detail
|
let mut detail = detail
|
||||||
.chars()
|
.chars()
|
||||||
.take(WORKER_CREATE_FAILURE_MESSAGE_LIMIT)
|
.take(WORKER_RESTORE_FAILURE_MESSAGE_LIMIT)
|
||||||
.collect::<String>();
|
.collect::<String>();
|
||||||
if truncated {
|
if truncated {
|
||||||
detail.push('…');
|
detail.push('…');
|
||||||
@@ -8507,43 +8762,13 @@ fn sanitize_worker_create_failure_message(message: &str) -> String {
|
|||||||
detail
|
detail
|
||||||
}
|
}
|
||||||
|
|
||||||
fn runtime_worker_create_failure_message(error: &RuntimeError) -> Option<String> {
|
|
||||||
match error {
|
|
||||||
RuntimeError::WorkerExecutionRejected { message, .. } => {
|
|
||||||
Some(sanitize_worker_create_failure_message(message))
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_runtime_worker_create_failure(
|
fn write_runtime_worker_create_failure(
|
||||||
worker_id: WorkerId,
|
worker_id: WorkerId,
|
||||||
workspace_id: Option<&str>,
|
workspace_id: Option<&str>,
|
||||||
error: &RuntimeError,
|
error: &RuntimeError,
|
||||||
elapsed: std::time::Duration,
|
elapsed: std::time::Duration,
|
||||||
) {
|
) {
|
||||||
let (error_kind, operation, outcome) = runtime_worker_create_failure_fields(error);
|
crate::startup_observation::write_create_failure(worker_id, workspace_id, error, elapsed);
|
||||||
let execution_failure_code = match error {
|
|
||||||
RuntimeError::WorkerExecutionRejected { result, .. } => {
|
|
||||||
worker_execution_failure_code(result)
|
|
||||||
}
|
|
||||||
_ => "",
|
|
||||||
};
|
|
||||||
let execution_failure_message = runtime_worker_create_failure_message(error);
|
|
||||||
tracing::error!(
|
|
||||||
target: "yoi::worker_create",
|
|
||||||
event = "worker_create_failed",
|
|
||||||
component = "runtime",
|
|
||||||
workspace_id = workspace_id.unwrap_or(""),
|
|
||||||
worker_id = %worker_id,
|
|
||||||
error_kind,
|
|
||||||
operation = operation.as_deref().unwrap_or(""),
|
|
||||||
outcome = outcome.as_deref().unwrap_or(""),
|
|
||||||
execution_failure_code,
|
|
||||||
execution_failure_message = execution_failure_message.as_deref().unwrap_or(""),
|
|
||||||
duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
|
|
||||||
"Worker creation failed"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn validate_create_workspace_scope(
|
fn validate_create_workspace_scope(
|
||||||
@@ -8692,6 +8917,11 @@ mod tests {
|
|||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, Barrier, Condvar, Mutex};
|
use std::sync::{Arc, Barrier, Condvar, Mutex};
|
||||||
|
|
||||||
|
mod create_failure_tests {
|
||||||
|
use super::*;
|
||||||
|
include!("runtime/create_failure_tests.rs");
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
mod create_rollback_tests {
|
mod create_rollback_tests {
|
||||||
//! Connect the real create/rollback path to FsRuntimeStore and reopen its disk state.
|
//! Connect the real create/rollback path to FsRuntimeStore and reopen its disk state.
|
||||||
@@ -8911,57 +9141,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn worker_create_failure_fields_only_include_sanitized_execution_detail() {
|
|
||||||
let invalid_request = RuntimeError::InvalidRequest("private-token".to_string());
|
|
||||||
let (error_kind, operation, outcome) =
|
|
||||||
runtime_worker_create_failure_fields(&invalid_request);
|
|
||||||
assert_eq!(error_kind, "invalid_request");
|
|
||||||
assert_eq!(operation, None);
|
|
||||||
assert_eq!(outcome, None);
|
|
||||||
assert_eq!(
|
|
||||||
runtime_worker_create_failure_message(&invalid_request),
|
|
||||||
None
|
|
||||||
);
|
|
||||||
|
|
||||||
let result = WorkerExecutionResult::errored(
|
|
||||||
WorkerExecutionOperation::Spawn,
|
|
||||||
"failed to spawn controller in /private/path: File exists; token super-secret",
|
|
||||||
);
|
|
||||||
let error = RuntimeError::WorkerExecutionRejected {
|
|
||||||
worker_id: WorkerId::now_v7(),
|
|
||||||
operation: result.operation,
|
|
||||||
outcome: result.outcome,
|
|
||||||
message: result.message_or_default(),
|
|
||||||
result,
|
|
||||||
};
|
|
||||||
assert_eq!(
|
|
||||||
runtime_worker_create_failure_message(&error).as_deref(),
|
|
||||||
Some("failed to spawn controller in [redacted] File exists; token [redacted]")
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
sanitize_worker_create_failure_message("backend private-token failed"),
|
|
||||||
"backend [redacted] failed"
|
|
||||||
);
|
|
||||||
|
|
||||||
let truncated = sanitize_worker_create_failure_message(&"x".repeat(513));
|
|
||||||
assert_eq!(truncated.chars().count(), 513);
|
|
||||||
assert!(truncated.ends_with('…'));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn worker_create_failure_code_classifies_durable_acceptance_timeout() {
|
|
||||||
let timeout = WorkerExecutionResult::errored(
|
|
||||||
WorkerExecutionOperation::Input,
|
|
||||||
"timed out waiting for durable Worker Submit acceptance; private-token",
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
worker_execution_failure_code(&timeout),
|
|
||||||
"input_commit_timeout"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn test_command() -> protocol::WorkerCommandEnvelope {
|
fn test_command() -> protocol::WorkerCommandEnvelope {
|
||||||
protocol::WorkerCommandEnvelope { command_id: 1 }
|
protocol::WorkerCommandEnvelope { command_id: 1 }
|
||||||
}
|
}
|
||||||
@@ -9950,7 +10129,6 @@ mod tests {
|
|||||||
recovery_absent: AtomicBool,
|
recovery_absent: AtomicBool,
|
||||||
recovery_checks: std::sync::atomic::AtomicUsize,
|
recovery_checks: std::sync::atomic::AtomicUsize,
|
||||||
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
|
||||||
#[cfg(feature = "fs-store")]
|
|
||||||
before_spawn_result: Mutex<Option<Box<dyn FnOnce() + Send>>>,
|
before_spawn_result: Mutex<Option<Box<dyn FnOnce() + Send>>>,
|
||||||
job_bindings: Mutex<Vec<Option<crate::catalog::BackendJobExecutionBinding>>>,
|
job_bindings: Mutex<Vec<Option<crate::catalog::BackendJobExecutionBinding>>>,
|
||||||
dispatch_result: Mutex<Option<WorkerExecutionResult>>,
|
dispatch_result: Mutex<Option<WorkerExecutionResult>>,
|
||||||
@@ -10197,7 +10375,6 @@ mod tests {
|
|||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.push(request.request.backend_job.clone());
|
.push(request.request.backend_job.clone());
|
||||||
#[cfg(feature = "fs-store")]
|
|
||||||
if let Some(before_result) = self.before_spawn_result.lock().unwrap().take() {
|
if let Some(before_result) = self.before_spawn_result.lock().unwrap().take() {
|
||||||
before_result();
|
before_result();
|
||||||
}
|
}
|
||||||
@@ -15503,13 +15680,7 @@ mod tests {
|
|||||||
|
|
||||||
let error = runtime.create_worker(request).unwrap_err();
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
|
||||||
assert!(matches!(
|
create_failure_tests::assert_ack_failure(&error);
|
||||||
error,
|
|
||||||
RuntimeError::WorkerExecutionRejected {
|
|
||||||
outcome: crate::execution::WorkerExecutionOutcome::Rejected,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
assert!(runtime.list_workers().unwrap().is_empty());
|
assert!(runtime.list_workers().unwrap().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -15524,13 +15695,7 @@ mod tests {
|
|||||||
|
|
||||||
let error = runtime.create_worker(request).unwrap_err();
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
|
||||||
assert!(matches!(
|
create_failure_tests::assert_ack_failure(&error);
|
||||||
error,
|
|
||||||
RuntimeError::WorkerExecutionRejected {
|
|
||||||
outcome: crate::execution::WorkerExecutionOutcome::Rejected,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
assert!(runtime.list_workers().unwrap().is_empty());
|
assert!(runtime.list_workers().unwrap().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -17160,7 +17325,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert!(!diagnostic.message.contains("credential-value"));
|
assert!(!diagnostic.message.contains("credential-value"));
|
||||||
assert!(!diagnostic.message.contains("/private/path"));
|
assert!(!diagnostic.message.contains("/private/path"));
|
||||||
assert!(diagnostic.message.chars().count() <= WORKER_CREATE_FAILURE_MESSAGE_LIMIT + 1);
|
assert!(diagnostic.message.chars().count() <= WORKER_RESTORE_FAILURE_MESSAGE_LIMIT + 1);
|
||||||
let before = std::fs::read(
|
let before = std::fs::read(
|
||||||
dir.path()
|
dir.path()
|
||||||
.join("runtime/workers")
|
.join("runtime/workers")
|
||||||
@@ -17346,7 +17511,7 @@ mod tests {
|
|||||||
assert!(!message.contains("/private"), "{message}");
|
assert!(!message.contains("/private"), "{message}");
|
||||||
assert!(!message.contains("restore-token"), "{message}");
|
assert!(!message.contains("restore-token"), "{message}");
|
||||||
assert!(!message.contains("hidden-value"), "{message}");
|
assert!(!message.contains("hidden-value"), "{message}");
|
||||||
assert!(message.chars().count() <= WORKER_CREATE_FAILURE_MESSAGE_LIMIT + 1);
|
assert!(message.chars().count() <= WORKER_RESTORE_FAILURE_MESSAGE_LIMIT + 1);
|
||||||
assert!(
|
assert!(
|
||||||
runtime
|
runtime
|
||||||
.lock()
|
.lock()
|
||||||
|
|||||||
@@ -0,0 +1,626 @@
|
|||||||
|
// Registered by runtime.rs's normal library test module. Execution responses are
|
||||||
|
// fixtures; creation, persistence, projection and compensation use the real owner.
|
||||||
|
use protocol::startup_failure::{
|
||||||
|
RuntimeStartupCause, RuntimeStartupReason, StartupCleanup, StartupCleanupFailure,
|
||||||
|
StartupIoKind, StartupResourceFacts, StartupResources, StartupStage,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn boundary_reason(failure: &WorkerCreateFailure) -> StartupBoundaryReason {
|
||||||
|
let projected = failure
|
||||||
|
.startup()
|
||||||
|
.primary
|
||||||
|
.project_for::<error_contract::InternalTransfer, StartupCause>(&())
|
||||||
|
.unwrap();
|
||||||
|
let StartupCause::Boundary(cause) = projected.details() else {
|
||||||
|
panic!("expected a closed boundary cause");
|
||||||
|
};
|
||||||
|
cause.reason
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn assert_ack_failure(error: &RuntimeError) {
|
||||||
|
assert!(matches!(error, RuntimeError::WorkerCreateFailed { .. }));
|
||||||
|
let failure = error.create_failure().unwrap();
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize,
|
||||||
|
Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::InitialInput,
|
||||||
|
compensation: CompensationOutcome::Completed,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
boundary_reason(&failure),
|
||||||
|
StartupBoundaryReason::InvalidResponse
|
||||||
|
);
|
||||||
|
// The missing/foreign receipt is not rewritten into a confirmed rejection.
|
||||||
|
assert!(error.execution_result().unwrap().is_accepted());
|
||||||
|
assert_eq!(error.to_string(), "worker creation failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rejected_input() -> WorkerExecutionResult {
|
||||||
|
let attachment = worker::startup_failure::runtime_startup_attachment(RuntimeStartupCause {
|
||||||
|
stage: StartupStage::Controller,
|
||||||
|
reason: RuntimeStartupReason::StartupFailed,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let mut result = WorkerExecutionResult::rejected(
|
||||||
|
WorkerExecutionOperation::Input,
|
||||||
|
"TEST_ONLY_SECRET original rejection /private/owner/path",
|
||||||
|
)
|
||||||
|
.with_startup_failure(attachment);
|
||||||
|
result
|
||||||
|
.secondary_failures
|
||||||
|
.push(StartupSecondaryFailure::Cleanup {
|
||||||
|
failure: StartupCleanupFailure::Workdir,
|
||||||
|
});
|
||||||
|
result.startup_resources = Some(StartupResourceFacts {
|
||||||
|
resources: StartupResources::Unproven,
|
||||||
|
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Workdir),
|
||||||
|
durable_effects_possible: true,
|
||||||
|
});
|
||||||
|
result.startup_failure_persistence = Some(StartupPersistenceOutcome::NotSaved);
|
||||||
|
result.startup_io_kind = Some(StartupIoKind::PermissionDenied);
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
enum DispatchFault {
|
||||||
|
DroppedAck,
|
||||||
|
MismatchedAck,
|
||||||
|
ConfirmedRejection,
|
||||||
|
LostResponse,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
enum CleanupFault {
|
||||||
|
None,
|
||||||
|
MissingTermination,
|
||||||
|
Rejected,
|
||||||
|
LostResponse,
|
||||||
|
MismatchedTermination,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn connected_create_dispatch_and_cleanup_fault_matrix() {
|
||||||
|
for dispatch in [
|
||||||
|
DispatchFault::DroppedAck,
|
||||||
|
DispatchFault::MismatchedAck,
|
||||||
|
DispatchFault::ConfirmedRejection,
|
||||||
|
DispatchFault::LostResponse,
|
||||||
|
] {
|
||||||
|
for cleanup in [
|
||||||
|
CleanupFault::None,
|
||||||
|
CleanupFault::MissingTermination,
|
||||||
|
CleanupFault::Rejected,
|
||||||
|
CleanupFault::LostResponse,
|
||||||
|
CleanupFault::MismatchedTermination,
|
||||||
|
] {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let original = match dispatch {
|
||||||
|
DispatchFault::DroppedAck => {
|
||||||
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
|
||||||
|
}
|
||||||
|
DispatchFault::MismatchedAck => {
|
||||||
|
backend.preserve_submission_acknowledgement_id();
|
||||||
|
WorkerExecutionResult::accepted_submission(
|
||||||
|
WorkerExecutionOperation::Input,
|
||||||
|
"foreign-request",
|
||||||
|
"foreign-submission",
|
||||||
|
protocol::SubmissionDisposition::Started,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
DispatchFault::ConfirmedRejection => rejected_input(),
|
||||||
|
DispatchFault::LostResponse => WorkerExecutionResult::errored(
|
||||||
|
WorkerExecutionOperation::Input,
|
||||||
|
"response lost; TEST_ONLY_SECRET",
|
||||||
|
),
|
||||||
|
};
|
||||||
|
backend.set_dispatch_result(original.clone());
|
||||||
|
let compensation = match cleanup {
|
||||||
|
CleanupFault::None => CompensationOutcome::Completed,
|
||||||
|
CleanupFault::MissingTermination => {
|
||||||
|
backend.set_stop_result(WorkerExecutionResult::accepted(
|
||||||
|
WorkerExecutionOperation::Stop,
|
||||||
|
));
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
}
|
||||||
|
CleanupFault::Rejected => {
|
||||||
|
backend.set_stop_result(WorkerExecutionResult::rejected(
|
||||||
|
WorkerExecutionOperation::Stop,
|
||||||
|
"TEST_ONLY_SECRET stop rejected",
|
||||||
|
));
|
||||||
|
CompensationOutcome::Residual
|
||||||
|
}
|
||||||
|
CleanupFault::LostResponse => {
|
||||||
|
backend.set_stop_result(WorkerExecutionResult::errored(
|
||||||
|
WorkerExecutionOperation::Stop,
|
||||||
|
"TEST_ONLY_SECRET stop response lost",
|
||||||
|
));
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
}
|
||||||
|
CleanupFault::MismatchedTermination => {
|
||||||
|
let mut stop = WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop);
|
||||||
|
stop.termination = Some(crate::execution::WorkerExecutionTermination {
|
||||||
|
scope: crate::execution::WorkerExecutionTerminationScope::Controller,
|
||||||
|
worker_ref: WorkerRef::new(WorkerId::now_v7()),
|
||||||
|
stop_operation_id: WorkerLifecycleOperationId::new(),
|
||||||
|
execution_id: Uuid::now_v7(),
|
||||||
|
shutdown_command_id: None,
|
||||||
|
submissions: Vec::new(),
|
||||||
|
feature_failures: Vec::new(),
|
||||||
|
});
|
||||||
|
backend.set_stop_result(stop);
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut request = task_request("connected create fault matrix");
|
||||||
|
request.initial_input = Some(WorkerInput::user("start once"));
|
||||||
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
let failure = error.create_failure().unwrap();
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().execution,
|
||||||
|
if matches!(dispatch, DispatchFault::ConfirmedRejection) {
|
||||||
|
ExecutionOutcome::Errored
|
||||||
|
} else {
|
||||||
|
ExecutionOutcome::Unknown
|
||||||
|
},
|
||||||
|
"{dispatch:?}/{cleanup:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize,
|
||||||
|
Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::InitialInput,
|
||||||
|
compensation,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(failure.operation().backend_compensation, None);
|
||||||
|
assert_eq!(failure.operation().correlation, None);
|
||||||
|
assert_eq!(error.execution_result(), Some(&original));
|
||||||
|
assert_eq!(error.to_string(), "worker creation failed");
|
||||||
|
if let Some(primary) = original.startup_failure.as_ref() {
|
||||||
|
assert_eq!(&failure.startup().primary, primary);
|
||||||
|
assert!(
|
||||||
|
failure
|
||||||
|
.startup()
|
||||||
|
.secondary
|
||||||
|
.starts_with(&original.secondary_failures)
|
||||||
|
);
|
||||||
|
assert_eq!(failure.startup().resources, original.startup_resources);
|
||||||
|
assert_eq!(
|
||||||
|
failure.startup().persistence,
|
||||||
|
original.startup_failure_persistence
|
||||||
|
);
|
||||||
|
assert_eq!(failure.startup().io_kind, original.startup_io_kind);
|
||||||
|
assert!(
|
||||||
|
std::error::Error::source(&error)
|
||||||
|
.unwrap()
|
||||||
|
.to_string()
|
||||||
|
.contains("TEST_ONLY_SECRET")
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assert_eq!(
|
||||||
|
boundary_reason(&failure),
|
||||||
|
if matches!(dispatch, DispatchFault::LostResponse) {
|
||||||
|
StartupBoundaryReason::FinalizeFailed
|
||||||
|
} else {
|
||||||
|
StartupBoundaryReason::InvalidResponse
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let resources = failure.startup().resources.unwrap();
|
||||||
|
assert!(resources.durable_effects_possible);
|
||||||
|
assert_eq!(
|
||||||
|
resources.cleanup,
|
||||||
|
match cleanup {
|
||||||
|
CleanupFault::None => StartupCleanup::Confirmed,
|
||||||
|
CleanupFault::Rejected =>
|
||||||
|
StartupCleanup::Failed(StartupCleanupFailure::Ownership),
|
||||||
|
_ => StartupCleanup::Required,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
|
||||||
|
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.submission_receipt_queries
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.list_workers().unwrap().is_empty(),
|
||||||
|
matches!(cleanup, CleanupFault::None)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn independent_input_error_is_not_a_create_failure() {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("independent input"))
|
||||||
|
.unwrap();
|
||||||
|
let original = rejected_input();
|
||||||
|
backend.set_dispatch_result(original.clone());
|
||||||
|
let error = runtime
|
||||||
|
.send_input(&worker.worker_ref, WorkerInput::user("ordinary input"))
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
&error,
|
||||||
|
RuntimeError::WorkerExecutionRejected { .. }
|
||||||
|
));
|
||||||
|
assert_eq!(error.create_failure(), None);
|
||||||
|
assert_eq!(error.execution_result(), Some(&original));
|
||||||
|
assert_eq!(*backend.stop_count.lock().unwrap(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn connected_create_initial_input_read_failure_keeps_create_context() {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let mut request = task_request("initial input read failure");
|
||||||
|
request.initial_input = Some(WorkerInput::user("never dispatched"));
|
||||||
|
let worker_id = request.worker_id;
|
||||||
|
*backend.before_spawn_result.lock().unwrap() = Some(Box::new({
|
||||||
|
let runtime = runtime.clone();
|
||||||
|
move || {
|
||||||
|
runtime.lock().unwrap().workers.remove(&worker_id);
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
let RuntimeError::WorkerCreateFailed { failure, source } = error else {
|
||||||
|
panic!("post-Connected read failure must remain a create failure");
|
||||||
|
};
|
||||||
|
assert!(matches!(*source, RuntimeError::WorkerNotFound { .. }));
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize.unwrap().stage,
|
||||||
|
FinalizeStage::InitialInput
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
boundary_reason(&failure),
|
||||||
|
StartupBoundaryReason::FinalizeFailed
|
||||||
|
);
|
||||||
|
assert!(backend.dispatched_inputs.lock().unwrap().is_empty());
|
||||||
|
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn connected_create_publication_failure_preserves_known_input_acceptance() {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
let mut request = task_request("publication fails after known acceptance");
|
||||||
|
request.initial_input = Some(WorkerInput::user("accepted once"));
|
||||||
|
let worker_ref = WorkerRef::new(request.worker_id);
|
||||||
|
// Substitute only Connected data. The invalid backend Workdir identifier
|
||||||
|
// reaches the real subscription projection after the owner commit.
|
||||||
|
backend
|
||||||
|
.contexts
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.insert(request.worker_id, runtime.execution_context(worker_ref));
|
||||||
|
*backend.spawn_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Connected {
|
||||||
|
worker_state: protocol::WorkerStatus::Idle.into(),
|
||||||
|
workdir_attachments: vec![WorkingDirectoryAttachmentStatus {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||||
|
working_directory: serde_json::from_value(serde_json::json!({
|
||||||
|
"summary": {
|
||||||
|
"working_directory_id": "",
|
||||||
|
"repository_id": "repo",
|
||||||
|
"materializer_kind": "runtime_git_clone",
|
||||||
|
"status": "active"
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.unwrap(),
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
let failure = error.create_failure().unwrap();
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize,
|
||||||
|
Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::Publication,
|
||||||
|
compensation: CompensationOutcome::Completed,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
boundary_reason(&failure),
|
||||||
|
StartupBoundaryReason::FinalizeFailed
|
||||||
|
);
|
||||||
|
let RuntimeError::WorkerCreateFailed { source, .. } = error else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
assert!(matches!(*source, RuntimeError::InvalidRequest(_)));
|
||||||
|
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
|
||||||
|
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn create_correlation_uses_bound_runtime_identity_for_connected_and_spawn_failures() {
|
||||||
|
for connected in [false, true] {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
runtime.bind_runtime_identity("runtime-identity").unwrap();
|
||||||
|
let mut request = task_request("safe correlation");
|
||||||
|
let worker_id = request.worker_id;
|
||||||
|
if connected {
|
||||||
|
request.initial_input = Some(WorkerInput::user("start once"));
|
||||||
|
backend.set_dispatch_result(rejected_input());
|
||||||
|
} else {
|
||||||
|
let mut result = rejected_input();
|
||||||
|
result.operation = WorkerExecutionOperation::Spawn;
|
||||||
|
*backend.spawn_result.lock().unwrap() =
|
||||||
|
Some(WorkerExecutionSpawnResult::Rejected(result));
|
||||||
|
}
|
||||||
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
error.create_failure().unwrap().operation().correlation,
|
||||||
|
Some(StartupCorrelation::for_worker(
|
||||||
|
"runtime-identity",
|
||||||
|
&worker_id.to_string()
|
||||||
|
))
|
||||||
|
);
|
||||||
|
assert_eq!(error.to_string(), "worker creation failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
#[test]
|
||||||
|
fn connected_create_commit_write_faults_preserve_acceptance_and_io_facts() {
|
||||||
|
use crate::fs_store::AtomicWriteFault;
|
||||||
|
for fault in [
|
||||||
|
AtomicWriteFault::BeforeRename,
|
||||||
|
AtomicWriteFault::AfterRename,
|
||||||
|
] {
|
||||||
|
for has_initial_input in [false, true] {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let options = FsRuntimeStoreOptions::new(root.path().join("runtime"))
|
||||||
|
.with_runtime_id("create-commit-fault");
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime =
|
||||||
|
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone())
|
||||||
|
.unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
*backend.before_spawn_result.lock().unwrap() = Some(Box::new({
|
||||||
|
let runtime = runtime.clone();
|
||||||
|
move || {
|
||||||
|
runtime
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.fs_store()
|
||||||
|
.unwrap()
|
||||||
|
.fail_next_worker_write(fault)
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let mut request = task_request("create commit fault");
|
||||||
|
if has_initial_input {
|
||||||
|
request.initial_input = Some(WorkerInput::user("accepted once"));
|
||||||
|
}
|
||||||
|
let worker_id = request.worker_id;
|
||||||
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
let failure = error.create_failure().unwrap();
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize,
|
||||||
|
Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::Publication,
|
||||||
|
compensation: CompensationOutcome::Completed,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
boundary_reason(&failure),
|
||||||
|
StartupBoundaryReason::FinalizeFailed
|
||||||
|
);
|
||||||
|
assert_eq!(failure.startup().io_kind, Some(StartupIoKind::Other));
|
||||||
|
assert_eq!(failure.startup().persistence, None);
|
||||||
|
assert!(
|
||||||
|
failure
|
||||||
|
.startup()
|
||||||
|
.secondary
|
||||||
|
.contains(&StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::RuntimeReconciliationRecordSave,
|
||||||
|
outcome: if fault == AtomicWriteFault::AfterRename {
|
||||||
|
StartupPersistenceOutcome::OutcomeUnknown
|
||||||
|
} else {
|
||||||
|
StartupPersistenceOutcome::NotSaved
|
||||||
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
|
let RuntimeError::WorkerCreateFailed { source, .. } = error else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
*source,
|
||||||
|
RuntimeError::StoreIo { .. } | RuntimeError::StoreCommitOutcomeUnknown { .. }
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
backend.dispatched_inputs.lock().unwrap().len(),
|
||||||
|
usize::from(has_initial_input)
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
|
||||||
|
assert!(
|
||||||
|
!options
|
||||||
|
.root
|
||||||
|
.join("workers")
|
||||||
|
.join(worker_id.to_string())
|
||||||
|
.exists()
|
||||||
|
);
|
||||||
|
drop(runtime);
|
||||||
|
drop(backend);
|
||||||
|
let reopened = Runtime::with_fs_store(options).unwrap();
|
||||||
|
assert!(reopened.list_workers().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
#[test]
|
||||||
|
fn connected_create_cleanup_persistence_faults_never_replace_primary() {
|
||||||
|
for delete_fault in [false, true] {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let store_root = root.path().join("runtime");
|
||||||
|
let options =
|
||||||
|
FsRuntimeStoreOptions::new(&store_root).with_runtime_id("create-cleanup-fault");
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime =
|
||||||
|
Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let mut request = task_request("create cleanup persistence fault");
|
||||||
|
request.initial_input = Some(WorkerInput::user("rejected once"));
|
||||||
|
let worker_dir = store_root
|
||||||
|
.join("workers")
|
||||||
|
.join(request.worker_id.to_string());
|
||||||
|
let runtime_path = store_root.join("runtime.json");
|
||||||
|
*backend.before_spawn_result.lock().unwrap() = Some(Box::new(move || {
|
||||||
|
if delete_fault {
|
||||||
|
std::fs::create_dir(worker_dir.join("metadata.json")).unwrap();
|
||||||
|
} else {
|
||||||
|
std::fs::rename(&runtime_path, runtime_path.with_extension("saved")).unwrap();
|
||||||
|
std::fs::create_dir(&runtime_path).unwrap();
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let original = rejected_input();
|
||||||
|
backend.set_dispatch_result(original.clone());
|
||||||
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
let failure = error.create_failure().unwrap();
|
||||||
|
assert_eq!(error.execution_result(), Some(&original));
|
||||||
|
assert_eq!(failure.startup().primary, original.startup_failure.unwrap());
|
||||||
|
assert_eq!(failure.startup().io_kind, original.startup_io_kind);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Errored);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize.unwrap().compensation,
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
failure
|
||||||
|
.startup()
|
||||||
|
.secondary
|
||||||
|
.contains(&StartupSecondaryFailure::Persistence {
|
||||||
|
phase: if delete_fault {
|
||||||
|
StartupFailurePhase::RuntimeRecordDelete
|
||||||
|
} else {
|
||||||
|
StartupFailurePhase::RuntimeSnapshotSave
|
||||||
|
},
|
||||||
|
outcome: StartupPersistenceOutcome::OutcomeUnknown,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
if !delete_fault {
|
||||||
|
assert!(
|
||||||
|
failure
|
||||||
|
.startup()
|
||||||
|
.secondary
|
||||||
|
.contains(&StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::RuntimeRecordDelete,
|
||||||
|
outcome: StartupPersistenceOutcome::Persisted,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
|
||||||
|
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn connected_create_retains_cleanup_owner_resource_and_metadata_observations() {
|
||||||
|
let (runtime, backend) = runtime_and_backend();
|
||||||
|
backend.set_dispatch_result(WorkerExecutionResult::accepted(
|
||||||
|
WorkerExecutionOperation::Input,
|
||||||
|
));
|
||||||
|
let mut stop =
|
||||||
|
WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, "TEST_ONLY_SECRET");
|
||||||
|
let resources = StartupResourceFacts {
|
||||||
|
resources: StartupResources::Unproven,
|
||||||
|
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Feature),
|
||||||
|
durable_effects_possible: true,
|
||||||
|
};
|
||||||
|
stop.startup_resources = Some(resources);
|
||||||
|
stop.startup_io_kind = Some(StartupIoKind::TimedOut);
|
||||||
|
stop.startup_failure_persistence = Some(StartupPersistenceOutcome::OutcomeUnknown);
|
||||||
|
backend.set_stop_result(stop);
|
||||||
|
let mut request = task_request("cleanup owner observations");
|
||||||
|
request.initial_input = Some(WorkerInput::user("start once"));
|
||||||
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
let failure = error.create_failure().unwrap();
|
||||||
|
assert_eq!(failure.startup().resources, Some(resources));
|
||||||
|
assert_eq!(failure.startup().io_kind, Some(StartupIoKind::TimedOut));
|
||||||
|
assert_eq!(failure.startup().persistence, None);
|
||||||
|
assert!(
|
||||||
|
failure
|
||||||
|
.startup()
|
||||||
|
.secondary
|
||||||
|
.contains(&StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::MetadataSave,
|
||||||
|
outcome: StartupPersistenceOutcome::OutcomeUnknown,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize.unwrap().compensation,
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
boundary_reason(&failure),
|
||||||
|
StartupBoundaryReason::InvalidResponse
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
#[test]
|
||||||
|
fn connected_create_unknown_cleanup_retention_write_failure_preserves_primary() {
|
||||||
|
use crate::fs_store::AtomicWriteFault;
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let options = FsRuntimeStoreOptions::new(root.path().join("runtime"))
|
||||||
|
.with_runtime_id("create-retain-fault");
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
*backend.before_spawn_result.lock().unwrap() = Some(Box::new({
|
||||||
|
let runtime = runtime.clone();
|
||||||
|
move || {
|
||||||
|
runtime
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.fs_store()
|
||||||
|
.unwrap()
|
||||||
|
.fail_next_worker_write(AtomicWriteFault::BeforeRename)
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let original = rejected_input();
|
||||||
|
backend.set_dispatch_result(original.clone());
|
||||||
|
// Accepted stop without a termination receipt cannot authorize deletion.
|
||||||
|
backend.set_stop_result(WorkerExecutionResult::accepted(
|
||||||
|
WorkerExecutionOperation::Stop,
|
||||||
|
));
|
||||||
|
let mut request = task_request("unknown cleanup retention persistence failure");
|
||||||
|
request.initial_input = Some(WorkerInput::user("rejected once"));
|
||||||
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
|
let failure = error.create_failure().unwrap();
|
||||||
|
assert_eq!(error.execution_result(), Some(&original));
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize.unwrap().compensation,
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
failure
|
||||||
|
.startup()
|
||||||
|
.secondary
|
||||||
|
.contains(&StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::RuntimeReconciliationRecordSave,
|
||||||
|
outcome: StartupPersistenceOutcome::NotSaved,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||||
|
assert_eq!(*backend.stop_count.lock().unwrap(), 1);
|
||||||
|
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.submission_receipt_queries
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -479,7 +479,7 @@ fn run(
|
|||||||
}
|
}
|
||||||
Ok(Step::Waiting) | Ok(Step::Contended) => None,
|
Ok(Step::Waiting) | Ok(Step::Contended) => None,
|
||||||
Ok(Step::Retry(reason)) => Some(reason),
|
Ok(Step::Retry(reason)) => Some(reason),
|
||||||
Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())),
|
Err(error) => Some(sanitize_worker_restore_failure_message(&error.to_string())),
|
||||||
};
|
};
|
||||||
if let Some(reason) = &reason {
|
if let Some(reason) = &reason {
|
||||||
repeated = repeated.saturating_add(1);
|
repeated = repeated.saturating_add(1);
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
//! The Runtime creation responsibility sink records only projected typed fields.
|
||||||
|
//! Source/provider strings are never safe merely because a blacklist did not match.
|
||||||
|
//! Wrappers/projectors do not write events; a failing sink cannot change the result.
|
||||||
|
use crate::{error::RuntimeError, identity::WorkerId};
|
||||||
|
|
||||||
|
pub(crate) fn write_create_failure(
|
||||||
|
worker_id: WorkerId,
|
||||||
|
workspace_id: Option<&str>,
|
||||||
|
error: &RuntimeError,
|
||||||
|
elapsed: std::time::Duration,
|
||||||
|
) {
|
||||||
|
let failure = error.create_failure();
|
||||||
|
let projected = failure
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|failure| failure.telemetry().ok());
|
||||||
|
let fields = projected.as_ref().map(error_contract::observation_fields);
|
||||||
|
let operation = failure.as_ref().map(|failure| failure.operation());
|
||||||
|
tracing::error!(
|
||||||
|
target: "yoi::worker_create",
|
||||||
|
event = "worker_create_failed",
|
||||||
|
component = "runtime",
|
||||||
|
workspace_id = workspace_id.unwrap_or(""),
|
||||||
|
worker_id = %worker_id,
|
||||||
|
domain = fields.as_ref().map_or("startup_boundary", |fields| fields.domain),
|
||||||
|
reason = fields.as_ref().map_or("observation_unavailable", |fields| fields.reason),
|
||||||
|
operation = fields.as_ref().and_then(|fields| fields.operation),
|
||||||
|
stage = fields.as_ref().and_then(|fields| fields.stage),
|
||||||
|
correlation = operation.and_then(|operation| operation.correlation.as_ref()).map(|value| value.as_str()),
|
||||||
|
acceptance = ?operation.map(|operation| operation.acceptance),
|
||||||
|
execution = ?operation.map(|operation| operation.execution),
|
||||||
|
finalize = ?operation.and_then(|operation| operation.finalize),
|
||||||
|
persistence = ?failure.as_ref().and_then(|failure| failure.startup().persistence),
|
||||||
|
resources = ?failure.as_ref().and_then(|failure| failure.startup().resources),
|
||||||
|
duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
|
||||||
|
"Worker creation returned a typed failure"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::catalog::{CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveSource};
|
||||||
|
use crate::execution::{
|
||||||
|
WorkerExecutionBackend, WorkerExecutionOperation, WorkerExecutionResult,
|
||||||
|
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||||
|
};
|
||||||
|
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveInput};
|
||||||
|
use crate::{Runtime, RuntimeOptions};
|
||||||
|
use runtime_api::startup::*;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use tracing_subscriber::prelude::*;
|
||||||
|
|
||||||
|
const CANARIES: &[&str] = &[
|
||||||
|
"aQ7xV2zN9mL5",
|
||||||
|
"/private/startup/provider",
|
||||||
|
"Bearer q3fV8aJ6",
|
||||||
|
"token=K7sM2pQ4",
|
||||||
|
];
|
||||||
|
struct FailingBackend;
|
||||||
|
impl WorkerExecutionBackend for FailingBackend {
|
||||||
|
fn backend_id(&self) -> &str {
|
||||||
|
"log-owner-fixture"
|
||||||
|
}
|
||||||
|
fn dispatch_input(
|
||||||
|
&self,
|
||||||
|
_: &crate::identity::WorkerRef,
|
||||||
|
_: crate::interaction::WorkerInput,
|
||||||
|
) -> WorkerExecutionResult {
|
||||||
|
panic!("rejected creation never dispatches input")
|
||||||
|
}
|
||||||
|
fn spawn_worker(&self, _: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||||
|
let result =
|
||||||
|
WorkerExecutionResult::errored(WorkerExecutionOperation::Spawn, CANARIES.join(" "))
|
||||||
|
.with_startup_failure(
|
||||||
|
worker::startup_failure::runtime_startup_attachment(
|
||||||
|
runtime_api::RuntimeStartupCause {
|
||||||
|
stage: runtime_api::StartupStage::Controller,
|
||||||
|
reason: runtime_api::RuntimeStartupReason::StartupFailed,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
WorkerExecutionSpawnResult::Errored(result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct Events(Arc<Mutex<Vec<BTreeMap<String, String>>>>);
|
||||||
|
struct Fields(BTreeMap<String, String>);
|
||||||
|
impl tracing::field::Visit for Fields {
|
||||||
|
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||||
|
self.0.insert(field.name().into(), value.into());
|
||||||
|
}
|
||||||
|
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||||
|
self.0.insert(field.name().into(), format!("{value:?}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Events {
|
||||||
|
fn on_event(
|
||||||
|
&self,
|
||||||
|
event: &tracing::Event<'_>,
|
||||||
|
_: tracing_subscriber::layer::Context<'_, S>,
|
||||||
|
) {
|
||||||
|
if event.metadata().target() == "yoi::worker_create" {
|
||||||
|
let mut fields = Fields(BTreeMap::new());
|
||||||
|
event.record(&mut fields);
|
||||||
|
self.0.lock().unwrap().push(fields.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn request() -> CreateWorkerRequest {
|
||||||
|
let archive = ProfileSourceArchive::build(ProfileSourceArchiveInput {
|
||||||
|
id: "log-profile".into(),
|
||||||
|
entrypoints: BTreeMap::from([("builtin:coder".into(), "profiles/coder.dcdl".into())]),
|
||||||
|
imports: BTreeMap::new(),
|
||||||
|
sources: BTreeMap::from([("profiles/coder.dcdl".into(), "{}".into())]),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
CreateWorkerRequest {
|
||||||
|
worker_id: WorkerId::now_v7(),
|
||||||
|
create_fingerprint: "log-create".into(),
|
||||||
|
profile: ProfileSelector::Builtin("builtin:coder".into()),
|
||||||
|
display_name: None,
|
||||||
|
profile_source: ProfileSourceArchiveSource::Embedded { archive },
|
||||||
|
config_bundle: None,
|
||||||
|
initial_input: None,
|
||||||
|
workdir_attachment_requests: Vec::new(),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
worker_observation_enabled: false,
|
||||||
|
worker_observation_grants: Vec::new(),
|
||||||
|
workspace_api: None,
|
||||||
|
memory_settings: None,
|
||||||
|
plugin_execution: None,
|
||||||
|
subjektiv_attached: false,
|
||||||
|
backend_job: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn real_runtime_creation_sink_omits_unlabelled_provider_canaries_and_matches_typed_result() {
|
||||||
|
let runtime =
|
||||||
|
Runtime::with_execution_backend(RuntimeOptions::default(), Arc::new(FailingBackend))
|
||||||
|
.unwrap();
|
||||||
|
runtime
|
||||||
|
.bind_runtime_identity("runtime-log-fixture")
|
||||||
|
.unwrap();
|
||||||
|
let request = request();
|
||||||
|
let worker_id = request.worker_id;
|
||||||
|
let events = Events::default();
|
||||||
|
let subscriber = tracing_subscriber::registry().with(events.clone());
|
||||||
|
let error =
|
||||||
|
tracing::subscriber::with_default(subscriber, || runtime.create_worker(request))
|
||||||
|
.unwrap_err();
|
||||||
|
let failure = error
|
||||||
|
.create_failure()
|
||||||
|
.expect("creation owner must retain a typed cause");
|
||||||
|
let wire = failure
|
||||||
|
.problem(
|
||||||
|
StartupBoundary::Runtime,
|
||||||
|
&SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let encoded = failure.encode_storage().unwrap();
|
||||||
|
let DecodeStoredWorkerCreateFailure::Known(saved) =
|
||||||
|
decode_stored_worker_create_failure(&encoded)
|
||||||
|
else {
|
||||||
|
panic!("known storage");
|
||||||
|
};
|
||||||
|
assert_eq!(saved.public_facts(), failure.public_facts());
|
||||||
|
let captured = events.0.lock().unwrap();
|
||||||
|
assert_eq!(captured.len(), 1);
|
||||||
|
let event = &captured[0];
|
||||||
|
let projection = failure.telemetry().unwrap();
|
||||||
|
let fields = error_contract::observation_fields(&projection);
|
||||||
|
assert_eq!(event["domain"], fields.domain);
|
||||||
|
assert_eq!(event["reason"], fields.reason);
|
||||||
|
assert_eq!(
|
||||||
|
event["correlation"],
|
||||||
|
StartupCorrelation::for_worker("runtime-log-fixture", &worker_id.to_string()).as_str()
|
||||||
|
);
|
||||||
|
for canary in CANARIES {
|
||||||
|
assert!(!format!("{captured:?}").contains(canary));
|
||||||
|
assert!(!String::from_utf8_lossy(wire.body()).contains(canary));
|
||||||
|
assert!(!encoded.contains(canary));
|
||||||
|
}
|
||||||
|
assert!(!event.contains_key("execution_failure_message"));
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn unsupported_failure_logs_only_fixed_fallback_not_source_text() {
|
||||||
|
let error = RuntimeError::InvalidRequest(CANARIES.join(" "));
|
||||||
|
let events = Events::default();
|
||||||
|
let subscriber = tracing_subscriber::registry().with(events.clone());
|
||||||
|
tracing::subscriber::with_default(subscriber, || {
|
||||||
|
write_create_failure(WorkerId::now_v7(), None, &error, std::time::Duration::ZERO)
|
||||||
|
});
|
||||||
|
let captured = events.0.lock().unwrap();
|
||||||
|
assert_eq!(captured.len(), 1);
|
||||||
|
assert_eq!(captured[0]["reason"], "observation_unavailable");
|
||||||
|
for canary in CANARIES {
|
||||||
|
assert!(!format!("{captured:?}").contains(canary));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -529,8 +529,8 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "http-server")]
|
#[cfg(feature = "http-server")]
|
||||||
fn generated_workspace_response<T: serde::Serialize>(
|
fn generated_workspace_response<T: serde::Serialize, E: serde::Serialize>(
|
||||||
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>,
|
response: Result<T, server_api::client_support::ClientError<E>>,
|
||||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||||
match response {
|
match response {
|
||||||
Ok(response) => Ok(WorkspaceResponse {
|
Ok(response) => Ok(WorkspaceResponse {
|
||||||
@@ -721,7 +721,19 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
|
|||||||
),
|
),
|
||||||
WorkspaceServerOperation::WorkerControlSpawn(request) => {
|
WorkspaceServerOperation::WorkerControlSpawn(request) => {
|
||||||
generated_workspace_response(
|
generated_workspace_response(
|
||||||
client.worker_control_spawn(workspace_id, request).await,
|
client.worker_control_spawn(workspace_id, request).await.map_err(|error| match error {
|
||||||
|
server_api::client_support::ClientError::Public { status, error } => {
|
||||||
|
let error = match error.failure().cloned() {
|
||||||
|
// This already authenticated Worker control caller may
|
||||||
|
// receive the enum-only public startup cause. The domain
|
||||||
|
// allowlist reprojects it, never forwards upstream text.
|
||||||
|
Some(failure) => server_api::WorkerCreateApiError::startup(failure, &protocol::session_failure::SessionFailureVisibility { can_view_cause: true }),
|
||||||
|
None => error,
|
||||||
|
};
|
||||||
|
server_api::client_support::ClientError::Public { status, error }
|
||||||
|
}
|
||||||
|
other => other,
|
||||||
|
}),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
WorkspaceServerOperation::WorkerControlInput {
|
WorkspaceServerOperation::WorkerControlInput {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ path = "src/main.rs"
|
|||||||
test-support = []
|
test-support = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
error-contract.workspace = true
|
||||||
api-macros = { workspace = true, features = ["axum"] }
|
api-macros = { workspace = true, features = ["axum"] }
|
||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
axum = { workspace = true, features = ["ws"] }
|
axum = { workspace = true, features = ["ws"] }
|
||||||
@@ -70,7 +71,6 @@ webauthn-rs = { workspace = true, features = ["conditional-ui"] }
|
|||||||
webauthn-rs-proto = "0.5.5"
|
webauthn-rs-proto = "0.5.5"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
error-contract.workspace = true
|
|
||||||
rusqlite = { workspace = true, features = ["hooks"] }
|
rusqlite = { workspace = true, features = ["hooks"] }
|
||||||
feature-storage = { workspace = true, features = ["test-support"] }
|
feature-storage = { workspace = true, features = ["test-support"] }
|
||||||
native-tls = "0.2"
|
native-tls = "0.2"
|
||||||
|
|||||||
@@ -1476,6 +1476,21 @@ fn captured_history_is_accepted_at_each_real_retained_prefix_and_cannot_be_forge
|
|||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
let conn = legacy_fixture_to_85(&dir.path().join("prefixes.sqlite"), 85);
|
let conn = legacy_fixture_to_85(&dir.path().join("prefixes.sqlite"), 85);
|
||||||
let original = pre_content_values(&conn, false);
|
let original = pre_content_values(&conn, false);
|
||||||
|
let mut expected_receipts = rows(
|
||||||
|
&conn,
|
||||||
|
"SELECT version,name FROM __yoi_schema_migrations ORDER BY version",
|
||||||
|
);
|
||||||
|
expected_receipts.extend(
|
||||||
|
store_owner::MIGRATIONS
|
||||||
|
.iter()
|
||||||
|
.filter(|migration| migration.version > 85)
|
||||||
|
.map(|migration| {
|
||||||
|
vec![
|
||||||
|
Value::Integer(migration.version),
|
||||||
|
Value::Text(migration.name.into()),
|
||||||
|
]
|
||||||
|
}),
|
||||||
|
);
|
||||||
for migration in store_owner::MIGRATIONS
|
for migration in store_owner::MIGRATIONS
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|migration| migration.version >= 86 && migration.version <= 122)
|
.filter(|migration| migration.version >= 86 && migration.version <= 122)
|
||||||
@@ -1496,10 +1511,13 @@ fn captured_history_is_accepted_at_each_real_retained_prefix_and_cannot_be_forge
|
|||||||
&conn,
|
&conn,
|
||||||
"SELECT version,name FROM __yoi_schema_migrations ORDER BY version",
|
"SELECT version,name FROM __yoi_schema_migrations ORDER BY version",
|
||||||
);
|
);
|
||||||
|
assert_eq!(receipts, expected_receipts);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
receipts.last().unwrap(),
|
store_owner::current_schema_version(&conn).unwrap(),
|
||||||
&vec![Value::Integer(123), Value::Text(MIGRATION_NAME.into())]
|
store_owner::LATEST_SCHEMA_VERSION
|
||||||
);
|
);
|
||||||
|
super::super::storage_formats::verify_history(&conn, store_owner::LATEST_SCHEMA_VERSION)
|
||||||
|
.unwrap();
|
||||||
assert_eq!(original, pre_content_values(&conn, true));
|
assert_eq!(original, pre_content_values(&conn, true));
|
||||||
assert!(conn.execute_batch("INSERT INTO backend_job_pre_content_history SELECT * FROM backend_job_pre_content_history LIMIT 1").is_err());
|
assert!(conn.execute_batch("INSERT INTO backend_job_pre_content_history SELECT * FROM backend_job_pre_content_history LIMIT 1").is_err());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -628,6 +628,10 @@ fn initial_worker_input(segments: &[Segment]) -> Option<EmbeddedWorkerInput> {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct WorkerSpawnResult {
|
pub struct WorkerSpawnResult {
|
||||||
|
/// Typed failure for internal consumers, independent of diagnostic prose.
|
||||||
|
/// HTTP publication requires a fresh audience projection, not serialization here.
|
||||||
|
#[serde(skip)]
|
||||||
|
pub failure: Option<runtime_api::startup::WorkerCreateFailure>,
|
||||||
pub state: InternalWorkerOperationState,
|
pub state: InternalWorkerOperationState,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub worker: Option<InternalWorkerSummary>,
|
pub worker: Option<InternalWorkerSummary>,
|
||||||
@@ -672,6 +676,8 @@ fn required_worker_workspace_api(
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum InternalWorkerOperationState {
|
pub enum InternalWorkerOperationState {
|
||||||
|
/// The boundary did not observe a conclusive operation result.
|
||||||
|
Unknown,
|
||||||
Accepted,
|
Accepted,
|
||||||
Unsupported,
|
Unsupported,
|
||||||
Rejected,
|
Rejected,
|
||||||
@@ -680,6 +686,7 @@ pub enum InternalWorkerOperationState {
|
|||||||
impl From<InternalWorkerOperationState> for server_api::WorkerOperationState {
|
impl From<InternalWorkerOperationState> for server_api::WorkerOperationState {
|
||||||
fn from(state: InternalWorkerOperationState) -> Self {
|
fn from(state: InternalWorkerOperationState) -> Self {
|
||||||
match state {
|
match state {
|
||||||
|
InternalWorkerOperationState::Unknown => Self::Unknown,
|
||||||
InternalWorkerOperationState::Accepted => Self::Accepted,
|
InternalWorkerOperationState::Accepted => Self::Accepted,
|
||||||
InternalWorkerOperationState::Unsupported => Self::Unsupported,
|
InternalWorkerOperationState::Unsupported => Self::Unsupported,
|
||||||
InternalWorkerOperationState::Rejected => Self::Rejected,
|
InternalWorkerOperationState::Rejected => Self::Rejected,
|
||||||
@@ -1137,6 +1144,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
request: WorkerSpawnRequest,
|
request: WorkerSpawnRequest,
|
||||||
) -> WorkerSpawnResult {
|
) -> WorkerSpawnResult {
|
||||||
WorkerSpawnResult {
|
WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Unsupported,
|
state: InternalWorkerOperationState::Unsupported,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -3210,6 +3218,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
{
|
{
|
||||||
diagnostics.push(embedded_workdir_unsupported_diagnostic());
|
diagnostics.push(embedded_workdir_unsupported_diagnostic());
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -3226,6 +3235,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
"Embedded backend Runtime is transportless; use run_accepted/create acceptance for backend-internal Workers".to_string(),
|
"Embedded backend Runtime is transportless; use run_accepted/create acceptance for backend-internal Workers".to_string(),
|
||||||
));
|
));
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -3251,6 +3261,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
error,
|
error,
|
||||||
));
|
));
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -3263,6 +3274,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
Err(diagnostic) => {
|
Err(diagnostic) => {
|
||||||
diagnostics.push(diagnostic);
|
diagnostics.push(diagnostic);
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -3285,6 +3297,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
.create_worker_scoped(&workspace_scope, create_request)
|
.create_worker_scoped(&workspace_scope, create_request)
|
||||||
{
|
{
|
||||||
Ok(detail) => WorkerSpawnResult {
|
Ok(detail) => WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Accepted,
|
state: InternalWorkerOperationState::Accepted,
|
||||||
worker: Some(self.map_worker_detail(detail)),
|
worker: Some(self.map_worker_detail(detail)),
|
||||||
acceptance_evidence: vec![
|
acceptance_evidence: vec![
|
||||||
@@ -3303,9 +3316,15 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
diagnostics,
|
diagnostics,
|
||||||
},
|
},
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
|
let failure = err.create_failure();
|
||||||
diagnostics.push(embedded_runtime_diagnostic(&err));
|
diagnostics.push(embedded_runtime_diagnostic(&err));
|
||||||
|
let state = failure.as_ref().map_or(
|
||||||
|
InternalWorkerOperationState::Rejected,
|
||||||
|
startup_failure_operation_state,
|
||||||
|
);
|
||||||
WorkerSpawnResult {
|
WorkerSpawnResult {
|
||||||
state: InternalWorkerOperationState::Rejected,
|
failure,
|
||||||
|
state,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
diagnostics,
|
diagnostics,
|
||||||
@@ -5750,6 +5769,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
WorkerSpawnAcceptanceRequirement::SocketReady
|
WorkerSpawnAcceptanceRequirement::SocketReady
|
||||||
) {
|
) {
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -5765,6 +5785,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
Ok(archive) => archive,
|
Ok(archive) => archive,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -5783,6 +5804,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
Ok(workspace_api) => workspace_api,
|
Ok(workspace_api) => workspace_api,
|
||||||
Err(diagnostic) => {
|
Err(diagnostic) => {
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -5802,6 +5824,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
Ok(create) => create,
|
Ok(create) => create,
|
||||||
Err(diagnostic) => {
|
Err(diagnostic) => {
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -5809,30 +5832,70 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
match self
|
let client = match self.runtime_api_client(
|
||||||
.run_runtime_api(
|
REMOTE_WORKER_CREATE_TIMEOUT,
|
||||||
REMOTE_WORKER_CREATE_TIMEOUT,
|
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
) {
|
||||||
move |client| async move { client.create_worker(create).await },
|
Ok(client) => client,
|
||||||
)
|
Err(diagnostic) => {
|
||||||
.and_then(|value| {
|
return WorkerSpawnResult {
|
||||||
runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>(value.worker)
|
failure: None,
|
||||||
}) {
|
state: InternalWorkerOperationState::Rejected,
|
||||||
Ok(response) => WorkerSpawnResult {
|
worker: None,
|
||||||
state: InternalWorkerOperationState::Accepted,
|
acceptance_evidence: Vec::new(),
|
||||||
worker: Some(self.map_worker_detail(response)),
|
diagnostics: vec![diagnostic],
|
||||||
acceptance_evidence: vec![WorkerSpawnAcceptanceEvidence {
|
};
|
||||||
kind: "remote_runtime_worker_created".to_string(),
|
}
|
||||||
detail: "worker-runtime REST create endpoint accepted the Worker".to_string(),
|
};
|
||||||
}],
|
let received = run_blocking_http(move || {
|
||||||
diagnostics: Vec::new(),
|
tokio::runtime::Builder::new_current_thread()
|
||||||
},
|
.enable_all()
|
||||||
Err(diagnostic) => WorkerSpawnResult {
|
.build()
|
||||||
state: InternalWorkerOperationState::Rejected,
|
.map_err(|_| {
|
||||||
worker: None,
|
runtime_api::client_support::ClientError::failure(
|
||||||
acceptance_evidence: Vec::new(),
|
runtime_api::client_support::ClientFailure::Transport,
|
||||||
diagnostics: vec![diagnostic],
|
)
|
||||||
},
|
})?
|
||||||
|
.block_on(client.create_worker(create))
|
||||||
|
});
|
||||||
|
match received {
|
||||||
|
Ok(response) => {
|
||||||
|
match runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>(
|
||||||
|
response.worker,
|
||||||
|
) {
|
||||||
|
Ok(worker) => WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
|
state: InternalWorkerOperationState::Accepted,
|
||||||
|
worker: Some(self.map_worker_detail(worker)),
|
||||||
|
acceptance_evidence: vec![WorkerSpawnAcceptanceEvidence {
|
||||||
|
kind: "remote_runtime_worker_created".into(),
|
||||||
|
detail: "Runtime create returned a Worker summary".into(),
|
||||||
|
}],
|
||||||
|
diagnostics: Vec::new(),
|
||||||
|
},
|
||||||
|
Err(diagnostic) => WorkerSpawnResult {
|
||||||
|
failure: Some(startup_observation_failure(
|
||||||
|
runtime_api::startup::StartupBoundaryReason::Decode,
|
||||||
|
)),
|
||||||
|
state: InternalWorkerOperationState::Unknown,
|
||||||
|
worker: None,
|
||||||
|
acceptance_evidence: Vec::new(),
|
||||||
|
diagnostics: vec![diagnostic],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
let failure = runtime_create_client_failure(&error);
|
||||||
|
let state = startup_failure_operation_state(&failure);
|
||||||
|
let diagnostic = runtime_api_diagnostic(&self.runtime_id, error);
|
||||||
|
WorkerSpawnResult {
|
||||||
|
failure: Some(failure),
|
||||||
|
state,
|
||||||
|
worker: None,
|
||||||
|
acceptance_evidence: Vec::new(),
|
||||||
|
diagnostics: vec![diagnostic],
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -6884,6 +6947,11 @@ fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnosti
|
|||||||
message,
|
message,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
EmbeddedRuntimeError::WorkerCreateFailed { failure, .. } => diagnostic(
|
||||||
|
"embedded_worker_create_failed",
|
||||||
|
HostDiagnosticSeverity::Warning,
|
||||||
|
worker::startup_failure::internal_startup_failure_summary(&failure.startup().primary),
|
||||||
|
),
|
||||||
EmbeddedRuntimeError::WorkerExecutionRejected {
|
EmbeddedRuntimeError::WorkerExecutionRejected {
|
||||||
operation,
|
operation,
|
||||||
outcome,
|
outcome,
|
||||||
@@ -6966,6 +7034,9 @@ fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnosti
|
|||||||
})
|
})
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
if let EmbeddedRuntimeError::WorkerCreateFailed { failure, .. } = error {
|
||||||
|
projection.startup = internal_startup_transfer(failure.startup().clone());
|
||||||
|
}
|
||||||
projection
|
projection
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7367,6 +7438,7 @@ pub fn placeholder_worker(host_id: impl Into<String>) -> InternalWorkerSummary {
|
|||||||
|
|
||||||
pub fn placeholder_spawn_response(host_id: impl Into<String>) -> WorkerSpawnResult {
|
pub fn placeholder_spawn_response(host_id: impl Into<String>) -> WorkerSpawnResult {
|
||||||
WorkerSpawnResult {
|
WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Unsupported,
|
state: InternalWorkerOperationState::Unsupported,
|
||||||
worker: Some(placeholder_worker(host_id)),
|
worker: Some(placeholder_worker(host_id)),
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -8811,7 +8883,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn embedded_runtime_spawn_execution_failure_is_rejected_and_not_input_capable() {
|
fn embedded_runtime_spawn_execution_error_preserves_unknown_acceptance_and_no_input() {
|
||||||
let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend(
|
let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend(
|
||||||
"local:test",
|
"local:test",
|
||||||
Arc::new(FailingSpawnBackend),
|
Arc::new(FailingSpawnBackend),
|
||||||
@@ -8820,14 +8892,22 @@ mod tests {
|
|||||||
let mut request = embedded_spawn_request();
|
let mut request = embedded_spawn_request();
|
||||||
request.requested_worker_name = Some("friendly-name".into());
|
request.requested_worker_name = Some("friendly-name".into());
|
||||||
let spawned = runtime.spawn_worker(test_create_binding(), request);
|
let spawned = runtime.spawn_worker(test_create_binding(), request);
|
||||||
assert_eq!(spawned.state, InternalWorkerOperationState::Rejected);
|
assert_eq!(spawned.state, InternalWorkerOperationState::Unknown);
|
||||||
|
let operation = spawned.failure.as_ref().unwrap().operation();
|
||||||
|
assert_eq!(
|
||||||
|
operation.acceptance,
|
||||||
|
runtime_api::startup::CreationAcceptance::Unknown
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
operation.execution,
|
||||||
|
runtime_api::startup::ExecutionOutcome::Errored
|
||||||
|
);
|
||||||
assert!(spawned.acceptance_evidence.is_empty());
|
assert!(spawned.acceptance_evidence.is_empty());
|
||||||
assert_eq!(spawned.diagnostics.len(), 1);
|
assert_eq!(spawned.diagnostics.len(), 1);
|
||||||
assert!(spawned.diagnostics.iter().any(|diagnostic| {
|
assert!(spawned.diagnostics.iter().any(|diagnostic| {
|
||||||
diagnostic.code == "embedded_worker_execution_rejected"
|
diagnostic.code == "embedded_worker_create_failed"
|
||||||
&& diagnostic
|
&& diagnostic.startup.is_some()
|
||||||
.message
|
&& !diagnostic.message.contains("provider setup failed")
|
||||||
.contains("provider setup failed at [redacted]")
|
|
||||||
&& !diagnostic.message.contains("/tmp/secret-provider-config")
|
&& !diagnostic.message.contains("/tmp/secret-provider-config")
|
||||||
&& !diagnostic.message.contains("secret-value")
|
&& !diagnostic.message.contains("secret-value")
|
||||||
&& !diagnostic.message.contains("session-42")
|
&& !diagnostic.message.contains("session-42")
|
||||||
@@ -9977,3 +10057,69 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A local observation failure is not a received execution failure. Its operation
|
||||||
|
/// facts remain unknown even if HTTP or compatibility diagnostics say "rejected".
|
||||||
|
fn startup_failure_operation_state(
|
||||||
|
failure: &runtime_api::startup::WorkerCreateFailure,
|
||||||
|
) -> InternalWorkerOperationState {
|
||||||
|
use runtime_api::startup::{CreationAcceptance, ExecutionOutcome};
|
||||||
|
if failure.operation().acceptance == CreationAcceptance::Unknown
|
||||||
|
|| failure.operation().execution == ExecutionOutcome::Unknown
|
||||||
|
{
|
||||||
|
InternalWorkerOperationState::Unknown
|
||||||
|
} else if failure.operation().execution == ExecutionOutcome::Unsupported {
|
||||||
|
InternalWorkerOperationState::Unsupported
|
||||||
|
} else {
|
||||||
|
InternalWorkerOperationState::Rejected
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn startup_observation_failure(
|
||||||
|
reason: runtime_api::startup::StartupBoundaryReason,
|
||||||
|
) -> runtime_api::startup::WorkerCreateFailure {
|
||||||
|
use runtime_api::startup::*;
|
||||||
|
WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause { reason }),
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Unknown,
|
||||||
|
execution: ExecutionOutcome::Unknown,
|
||||||
|
finalize: None,
|
||||||
|
correlation: None,
|
||||||
|
backend_compensation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("fixed enum-only observation fits the startup boundary")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn runtime_create_client_failure(
|
||||||
|
error: &runtime_api::client_support::ClientError<runtime_api::RuntimeApiError>,
|
||||||
|
) -> runtime_api::startup::WorkerCreateFailure {
|
||||||
|
use runtime_api::client_support::{ClientError, ClientFailure};
|
||||||
|
use runtime_api::startup::StartupBoundaryReason;
|
||||||
|
match error {
|
||||||
|
ClientError::Public { error, .. } => error
|
||||||
|
.create_failure()
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| startup_observation_failure(StartupBoundaryReason::InvalidResponse)),
|
||||||
|
ClientError::Failure(failure) => startup_observation_failure(match failure {
|
||||||
|
ClientFailure::Timeout => StartupBoundaryReason::Timeout,
|
||||||
|
ClientFailure::ResponseTooLarge { .. } => StartupBoundaryReason::TooLarge,
|
||||||
|
ClientFailure::Decode { .. } | ClientFailure::ErrorResponseDecode { .. } => {
|
||||||
|
StartupBoundaryReason::Decode
|
||||||
|
}
|
||||||
|
ClientFailure::UnsupportedErrorResponse { .. } => {
|
||||||
|
StartupBoundaryReason::UnsupportedVersion
|
||||||
|
}
|
||||||
|
ClientFailure::UnexpectedStatus { .. }
|
||||||
|
| ClientFailure::MissingResponseHeader { .. }
|
||||||
|
| ClientFailure::DuplicateResponseHeader { .. }
|
||||||
|
| ClientFailure::InvalidResponseHeader { .. }
|
||||||
|
| ClientFailure::UnexpectedBody => StartupBoundaryReason::InvalidResponse,
|
||||||
|
ClientFailure::Authorization
|
||||||
|
| ClientFailure::RequestEncoding
|
||||||
|
| ClientFailure::Connect
|
||||||
|
| ClientFailure::Transport => StartupBoundaryReason::Transport,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -695,6 +695,8 @@ CREATE TABLE worker_create_reservations (
|
|||||||
updated_at TEXT NOT NULL, request_fingerprint TEXT, memory_language TEXT,
|
updated_at TEXT NOT NULL, request_fingerprint TEXT, memory_language TEXT,
|
||||||
singleton_key TEXT,
|
singleton_key TEXT,
|
||||||
singleton_generation INTEGER CHECK (singleton_generation IS NULL OR singleton_generation > 0),
|
singleton_generation INTEGER CHECK (singleton_generation IS NULL OR singleton_generation > 0),
|
||||||
|
startup_failure_json TEXT CHECK (startup_failure_json IS NULL OR
|
||||||
|
(typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768)),
|
||||||
PRIMARY KEY (workspace_id, allocation_key),
|
PRIMARY KEY (workspace_id, allocation_key),
|
||||||
UNIQUE (workspace_id, worker_id),
|
UNIQUE (workspace_id, worker_id),
|
||||||
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
|
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ mod ticket_browse_performance_tests;
|
|||||||
mod ticket_item_checker;
|
mod ticket_item_checker;
|
||||||
pub mod workdir_create_operations;
|
pub mod workdir_create_operations;
|
||||||
mod workdir_removal;
|
mod workdir_removal;
|
||||||
|
mod worker_create_failure_store;
|
||||||
mod worker_projection;
|
mod worker_projection;
|
||||||
pub mod worker_source;
|
pub mod worker_source;
|
||||||
pub mod workspace_catalog;
|
pub mod workspace_catalog;
|
||||||
@@ -124,6 +125,15 @@ pub enum Error {
|
|||||||
InvalidRuntimeIdentifier { kind: String, value: String },
|
InvalidRuntimeIdentifier { kind: String, value: String },
|
||||||
#[error("worker name is reserved for a dedicated Workspace service: {0}")]
|
#[error("worker name is reserved for a dedicated Workspace service: {0}")]
|
||||||
ReservedWorkerName(String),
|
ReservedWorkerName(String),
|
||||||
|
/// Typed startup result; HTTP and other callers project this without parsing Display.
|
||||||
|
#[error(
|
||||||
|
"Worker creation failed; the typed result retains its cause and operation observations"
|
||||||
|
)]
|
||||||
|
WorkerCreateFailed {
|
||||||
|
failure: Box<runtime_api::startup::WorkerCreateFailure>,
|
||||||
|
#[source]
|
||||||
|
source: Option<Box<Error>>,
|
||||||
|
},
|
||||||
#[error("runtime `{runtime_id}` operation failed ({code}): {message}")]
|
#[error("runtime `{runtime_id}` operation failed ({code}): {message}")]
|
||||||
RuntimeOperationFailed {
|
RuntimeOperationFailed {
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ mod subjektiv_memory_journal;
|
|||||||
mod workdir_authorization;
|
mod workdir_authorization;
|
||||||
mod worker_operations;
|
mod worker_operations;
|
||||||
mod worker_protocol;
|
mod worker_protocol;
|
||||||
|
mod worker_startup;
|
||||||
pub(crate) use worker_operations::{
|
pub(crate) use worker_operations::{
|
||||||
WorkerOperationContext, WorkspaceWorker, WorkspaceWorkerMethodSender,
|
WorkerOperationContext, WorkspaceWorker, WorkspaceWorkerMethodSender,
|
||||||
};
|
};
|
||||||
@@ -5996,6 +5997,23 @@ impl WorkspaceApi {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read retained creation evidence for this Workspace without changing lifecycle
|
||||||
|
/// state. Trusted internal callers must enforce their current authority before
|
||||||
|
/// redisclosure; a saved record never supplies visibility or retry permission.
|
||||||
|
pub fn worker_create_failure(
|
||||||
|
&self,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
) -> Result<Option<runtime_api::startup::DecodeStoredWorkerCreateFailure>> {
|
||||||
|
let worker_id = WorkerId::parse(&worker.worker_id)
|
||||||
|
.ok_or_else(|| Error::InvalidInput("invalid Worker identity".into()))?;
|
||||||
|
self.config_store
|
||||||
|
.worker_create_failure(self.workspace_id(), &worker.runtime_id, worker_id)
|
||||||
|
.map(|encoded| {
|
||||||
|
encoded
|
||||||
|
.map(|value| runtime_api::startup::decode_stored_worker_create_failure(&value))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn spawn_workspace_worker(
|
fn spawn_workspace_worker(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
runtime_id: &str,
|
||||||
@@ -6012,7 +6030,46 @@ impl WorkspaceApi {
|
|||||||
) -> ApiResult<WorkerSpawnResult> {
|
) -> ApiResult<WorkerSpawnResult> {
|
||||||
let assignment = request.ticket_assignment.clone();
|
let assignment = request.ticket_assignment.clone();
|
||||||
let mut dispatched = false;
|
let mut dispatched = false;
|
||||||
let mut result = self.spawn_workspace_worker_inner(runtime_id, request, &mut dispatched);
|
let mut reserved_worker = None;
|
||||||
|
let mut result = self.spawn_workspace_worker_inner(
|
||||||
|
runtime_id,
|
||||||
|
request,
|
||||||
|
&mut dispatched,
|
||||||
|
&mut reserved_worker,
|
||||||
|
);
|
||||||
|
if let Some(worker) = reserved_worker.as_ref() {
|
||||||
|
result = result
|
||||||
|
.map(|mut result| {
|
||||||
|
if let Some(failure) = result.failure.take() {
|
||||||
|
result.failure = Some(worker_startup::persist(self, worker, failure));
|
||||||
|
}
|
||||||
|
result
|
||||||
|
})
|
||||||
|
.map_err(|mut error| {
|
||||||
|
error.error = match error.error {
|
||||||
|
Error::WorkerCreateFailed { failure, source } => {
|
||||||
|
Error::WorkerCreateFailed {
|
||||||
|
failure: Box::new(worker_startup::persist(self, worker, *failure)),
|
||||||
|
source,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
source if dispatched => {
|
||||||
|
// Dispatch is not proof of acceptance or execution.
|
||||||
|
let failure = worker_startup::persist(
|
||||||
|
self,
|
||||||
|
worker,
|
||||||
|
worker_startup::unknown_result(),
|
||||||
|
);
|
||||||
|
Error::WorkerCreateFailed {
|
||||||
|
failure: Box::new(failure),
|
||||||
|
source: Some(Box::new(source)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
source => source,
|
||||||
|
};
|
||||||
|
error
|
||||||
|
});
|
||||||
|
}
|
||||||
// Only a newly inserted durable receipt proves there was no earlier
|
// Only a newly inserted durable receipt proves there was no earlier
|
||||||
// dispatch. A local rejection on replay says nothing about a prior
|
// dispatch. A local rejection on replay says nothing about a prior
|
||||||
// unknown outcome; keep its fence until absence is confirmed.
|
// unknown outcome; keep its fence until absence is confirmed.
|
||||||
@@ -6040,6 +6097,7 @@ impl WorkspaceApi {
|
|||||||
runtime_id: &str,
|
runtime_id: &str,
|
||||||
mut request: WorkerSpawnRequest,
|
mut request: WorkerSpawnRequest,
|
||||||
dispatched: &mut bool,
|
dispatched: &mut bool,
|
||||||
|
reserved_identity: &mut Option<RuntimeWorkerRef>,
|
||||||
) -> ApiResult<WorkerSpawnResult> {
|
) -> ApiResult<WorkerSpawnResult> {
|
||||||
if let Some(singleton_key) = request.singleton_key.as_deref() {
|
if let Some(singleton_key) = request.singleton_key.as_deref() {
|
||||||
crate::store::validate_worker_singleton_key(singleton_key)?;
|
crate::store::validate_worker_singleton_key(singleton_key)?;
|
||||||
@@ -6192,6 +6250,7 @@ impl WorkspaceApi {
|
|||||||
.worker(&reserved_worker)
|
.worker(&reserved_worker)
|
||||||
.unwrap_or_else(|_| worker_summary_from_registry(&record));
|
.unwrap_or_else(|_| worker_summary_from_registry(&record));
|
||||||
return Ok(WorkerSpawnResult {
|
return Ok(WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Accepted,
|
state: InternalWorkerOperationState::Accepted,
|
||||||
worker: Some(summary),
|
worker: Some(summary),
|
||||||
acceptance_evidence: vec![crate::hosts::WorkerSpawnAcceptanceEvidence {
|
acceptance_evidence: vec![crate::hosts::WorkerSpawnAcceptanceEvidence {
|
||||||
@@ -6230,6 +6289,7 @@ impl WorkspaceApi {
|
|||||||
};
|
};
|
||||||
request.resolved_memory_settings = Some(reservation.memory_settings);
|
request.resolved_memory_settings = Some(reservation.memory_settings);
|
||||||
let reservation_fingerprint = create_binding.create_fingerprint.clone();
|
let reservation_fingerprint = create_binding.create_fingerprint.clone();
|
||||||
|
*reserved_identity = Some(reserved_worker.clone());
|
||||||
let lifecycle_assignment = request.ticket_assignment.clone();
|
let lifecycle_assignment = request.ticket_assignment.clone();
|
||||||
let compensation_context = WorkerSpawnCompensationContext {
|
let compensation_context = WorkerSpawnCompensationContext {
|
||||||
assignment: lifecycle_assignment.as_ref(),
|
assignment: lifecycle_assignment.as_ref(),
|
||||||
@@ -6372,19 +6432,35 @@ impl WorkspaceApi {
|
|||||||
return Err(ApiError::with_diagnostics(error.into_error(), diagnostics));
|
return Err(ApiError::with_diagnostics(error.into_error(), diagnostics));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if result.state == InternalWorkerOperationState::Accepted && result.worker.is_none() {
|
||||||
|
// A malformed success is not conclusive evidence about the reserved Worker.
|
||||||
|
result.state = InternalWorkerOperationState::Unknown;
|
||||||
|
}
|
||||||
|
if result.state != InternalWorkerOperationState::Accepted && result.failure.is_none() {
|
||||||
|
result.failure = Some(if result.state == InternalWorkerOperationState::Unknown {
|
||||||
|
worker_startup::unknown_result()
|
||||||
|
} else {
|
||||||
|
worker_startup::local_rejection()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(failure) = result.failure.take() {
|
||||||
|
result.failure = Some(worker_startup::correlate(failure, &reserved_worker));
|
||||||
|
}
|
||||||
let Some(worker) = result.worker.as_mut() else {
|
let Some(worker) = result.worker.as_mut() else {
|
||||||
result
|
let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed(
|
||||||
.diagnostics
|
self,
|
||||||
.extend(compensate_failed_workspace_worker_create(
|
runtime_id,
|
||||||
self,
|
worker_id,
|
||||||
runtime_id,
|
&reservation_fingerprint,
|
||||||
worker_id,
|
"runtime_spawn_failure",
|
||||||
&reservation_fingerprint,
|
None,
|
||||||
"runtime_spawn_rejected",
|
&compensation_context,
|
||||||
None,
|
&reserved_attachments,
|
||||||
&compensation_context,
|
);
|
||||||
&reserved_attachments,
|
if let Some(failure) = result.failure.take() {
|
||||||
));
|
result.failure = Some(worker_startup::compensated(failure, compensation));
|
||||||
|
}
|
||||||
|
result.diagnostics.extend(diagnostics);
|
||||||
return Ok(result);
|
return Ok(result);
|
||||||
};
|
};
|
||||||
// Runtime metadata is display-only. The lease returned by the same DB transaction that
|
// Runtime metadata is display-only. The lease returned by the same DB transaction that
|
||||||
@@ -6437,21 +6513,26 @@ impl WorkspaceApi {
|
|||||||
{
|
{
|
||||||
Ok(replacement) => replacement,
|
Ok(replacement) => replacement,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let diagnostics = compensate_failed_workspace_worker_create(
|
let (diagnostics, compensation) =
|
||||||
self,
|
compensate_failed_workspace_worker_create_observed(
|
||||||
runtime_id,
|
self,
|
||||||
worker_id,
|
runtime_id,
|
||||||
&reservation_fingerprint,
|
worker_id,
|
||||||
"workspace_api_transport",
|
&reservation_fingerprint,
|
||||||
Some(worker),
|
"workspace_api_transport",
|
||||||
&compensation_context,
|
Some(worker),
|
||||||
&reserved_attachments,
|
&compensation_context,
|
||||||
);
|
&reserved_attachments,
|
||||||
return Err(ApiError::with_diagnostics(error.into_error(), diagnostics));
|
);
|
||||||
|
return Err(worker_startup::finalize_error(
|
||||||
|
ApiError::with_diagnostics(error.into_error(), diagnostics),
|
||||||
|
WorkerSpawnFinalizeStage::WorkspaceApiBinding,
|
||||||
|
compensation,
|
||||||
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if replacement.state != InternalWorkerOperationState::Accepted {
|
if replacement.state != InternalWorkerOperationState::Accepted {
|
||||||
let diagnostics = compensate_failed_workspace_worker_create(
|
let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed(
|
||||||
self,
|
self,
|
||||||
runtime_id,
|
runtime_id,
|
||||||
worker_id,
|
worker_id,
|
||||||
@@ -6461,23 +6542,27 @@ impl WorkspaceApi {
|
|||||||
&compensation_context,
|
&compensation_context,
|
||||||
&reserved_attachments,
|
&reserved_attachments,
|
||||||
);
|
);
|
||||||
return Err(ApiError::with_diagnostics(
|
return Err(worker_startup::finalize_error(
|
||||||
Error::RuntimeOperationFailed {
|
ApiError::with_diagnostics(
|
||||||
runtime_id: runtime_id.to_string(),
|
Error::RuntimeOperationFailed {
|
||||||
code: "worker_workspace_api_replace_failed".to_string(),
|
runtime_id: runtime_id.to_string(),
|
||||||
message: replacement
|
code: "worker_workspace_api_replace_failed".to_string(),
|
||||||
.diagnostics
|
message: replacement
|
||||||
.first()
|
.diagnostics
|
||||||
.map(|diagnostic| diagnostic.message.clone())
|
.first()
|
||||||
.unwrap_or_else(|| {
|
.map(|diagnostic| diagnostic.message.clone())
|
||||||
"Runtime rejected Workspace API replacement after spawn".to_string()
|
.unwrap_or_else(|| {
|
||||||
}),
|
"Runtime rejected Workspace API replacement after spawn".to_string()
|
||||||
},
|
}),
|
||||||
diagnostics,
|
},
|
||||||
|
diagnostics,
|
||||||
|
),
|
||||||
|
WorkerSpawnFinalizeStage::WorkspaceApiBinding,
|
||||||
|
compensation,
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if let Err(error) = parse_runtime_worker_id_for_registry(&worker.worker.worker_id) {
|
if let Err(error) = parse_runtime_worker_id_for_registry(&worker.worker.worker_id) {
|
||||||
let diagnostics = compensate_failed_workspace_worker_create(
|
let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed(
|
||||||
self,
|
self,
|
||||||
runtime_id,
|
runtime_id,
|
||||||
worker_id,
|
worker_id,
|
||||||
@@ -6487,7 +6572,11 @@ impl WorkspaceApi {
|
|||||||
&compensation_context,
|
&compensation_context,
|
||||||
&reserved_attachments,
|
&reserved_attachments,
|
||||||
);
|
);
|
||||||
return Err(api_error_with_additional_diagnostics(error, diagnostics));
|
return Err(worker_startup::finalize_error(
|
||||||
|
api_error_with_additional_diagnostics(error, diagnostics),
|
||||||
|
WorkerSpawnFinalizeStage::WorkerIdentity,
|
||||||
|
compensation,
|
||||||
|
));
|
||||||
}
|
}
|
||||||
let registry_result = record_worker_summary(
|
let registry_result = record_worker_summary(
|
||||||
self,
|
self,
|
||||||
@@ -6556,7 +6645,7 @@ impl WorkspaceApi {
|
|||||||
.config_store
|
.config_store
|
||||||
.complete_worker_create_reservation(&self.config.workspace_id, worker_id)
|
.complete_worker_create_reservation(&self.config.workspace_id, worker_id)
|
||||||
{
|
{
|
||||||
let diagnostics = compensate_failed_workspace_worker_create(
|
let (diagnostics, compensation) = compensate_failed_workspace_worker_create_observed(
|
||||||
self,
|
self,
|
||||||
runtime_id,
|
runtime_id,
|
||||||
worker_id,
|
worker_id,
|
||||||
@@ -6566,9 +6655,10 @@ impl WorkspaceApi {
|
|||||||
&compensation_context,
|
&compensation_context,
|
||||||
&reserved_attachments,
|
&reserved_attachments,
|
||||||
);
|
);
|
||||||
return Err(ApiError::with_diagnostics(
|
return Err(worker_startup::finalize_error(
|
||||||
Error::Config(error.to_string()),
|
ApiError::with_diagnostics(Error::Config(error.to_string()), diagnostics),
|
||||||
diagnostics,
|
WorkerSpawnFinalizeStage::CreateReservation,
|
||||||
|
compensation,
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if let Err(error) = subjektiv_journal::reconcile_created_worker(self, &worker_ref) {
|
if let Err(error) = subjektiv_journal::reconcile_created_worker(self, &worker_ref) {
|
||||||
@@ -10762,8 +10852,10 @@ impl server_api::ServerApi for ServerApiContractService {
|
|||||||
&self,
|
&self,
|
||||||
context: server_api::ServerRequestContext,
|
context: server_api::ServerRequestContext,
|
||||||
request: server_api::CreateWorkspaceWorkerRequest,
|
request: server_api::CreateWorkspaceWorkerRequest,
|
||||||
) -> std::result::Result<server_api::BrowserCreateWorkerResponse, server_api::RepositoryApiError>
|
) -> std::result::Result<
|
||||||
{
|
server_api::BrowserCreateWorkerResponse,
|
||||||
|
server_api::WorkerCreateApiError,
|
||||||
|
> {
|
||||||
create_workspace_worker_authenticated(
|
create_workspace_worker_authenticated(
|
||||||
self.workspace_api()?.clone(),
|
self.workspace_api()?.clone(),
|
||||||
contract_request_headers(&context)?,
|
contract_request_headers(&context)?,
|
||||||
@@ -10773,7 +10865,7 @@ impl server_api::ServerApi for ServerApiContractService {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map(|Json(response)| response)
|
.map(|Json(response)| response)
|
||||||
.map_err(ApiError::into_repository_api_error)
|
.map_err(ApiError::into_worker_create_api_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn workspace_orchestrator_status(
|
async fn workspace_orchestrator_status(
|
||||||
@@ -10869,8 +10961,10 @@ impl server_api::ServerApi for ServerApiContractService {
|
|||||||
context: server_api::ServerRequestContext,
|
context: server_api::ServerRequestContext,
|
||||||
workspace_id: String,
|
workspace_id: String,
|
||||||
request: server_api::CreateWorkspaceWorkerRequest,
|
request: server_api::CreateWorkspaceWorkerRequest,
|
||||||
) -> std::result::Result<server_api::BrowserCreateWorkerResponse, server_api::RepositoryApiError>
|
) -> std::result::Result<
|
||||||
{
|
server_api::BrowserCreateWorkerResponse,
|
||||||
|
server_api::WorkerCreateApiError,
|
||||||
|
> {
|
||||||
spawn_known_worker_authenticated(
|
spawn_known_worker_authenticated(
|
||||||
State(self.workspace_api()?.clone()),
|
State(self.workspace_api()?.clone()),
|
||||||
AxumPath(ScopedWorkspacePath { workspace_id }),
|
AxumPath(ScopedWorkspacePath { workspace_id }),
|
||||||
@@ -10880,7 +10974,7 @@ impl server_api::ServerApi for ServerApiContractService {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map(|Json(response)| response)
|
.map(|Json(response)| response)
|
||||||
.map_err(ApiError::into_repository_api_error)
|
.map_err(ApiError::into_worker_create_api_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn worker_control_input(
|
async fn worker_control_input(
|
||||||
@@ -11048,8 +11142,10 @@ impl server_api::ServerApi for ServerApiContractService {
|
|||||||
context: server_api::ServerRequestContext,
|
context: server_api::ServerRequestContext,
|
||||||
workspace_id: String,
|
workspace_id: String,
|
||||||
request: server_api::CreateWorkspaceWorkerRequest,
|
request: server_api::CreateWorkspaceWorkerRequest,
|
||||||
) -> std::result::Result<server_api::BrowserCreateWorkerResponse, server_api::RepositoryApiError>
|
) -> std::result::Result<
|
||||||
{
|
server_api::BrowserCreateWorkerResponse,
|
||||||
|
server_api::WorkerCreateApiError,
|
||||||
|
> {
|
||||||
validate_workspace_scope(self.workspace_api()?, &workspace_id)
|
validate_workspace_scope(self.workspace_api()?, &workspace_id)
|
||||||
.map_err(|error| ApiError::from(error).into_repository_api_error())?;
|
.map_err(|error| ApiError::from(error).into_repository_api_error())?;
|
||||||
create_workspace_worker_authenticated(
|
create_workspace_worker_authenticated(
|
||||||
@@ -11061,7 +11157,7 @@ impl server_api::ServerApi for ServerApiContractService {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map(|Json(response)| response)
|
.map(|Json(response)| response)
|
||||||
.map_err(ApiError::into_repository_api_error)
|
.map_err(ApiError::into_worker_create_api_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn workspace_worker_detail(
|
async fn workspace_worker_detail(
|
||||||
@@ -15015,6 +15111,15 @@ async fn log_failed_api_response(request: Request, next: Next) -> Response {
|
|||||||
.and_then(|source| source.token_id_hash.clone());
|
.and_then(|source| source.token_id_hash.clone());
|
||||||
let response = next.run(request).await;
|
let response = next.run(request).await;
|
||||||
let status = response.status();
|
let status = response.status();
|
||||||
|
if let Some(error) = response
|
||||||
|
.extensions()
|
||||||
|
.get::<Arc<server_api::WorkerCreateApiError>>()
|
||||||
|
{
|
||||||
|
if let Some(failure) = error.failure() {
|
||||||
|
worker_startup::observe_http(failure, status);
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let is_api = uri.path().starts_with("/api/")
|
let is_api = uri.path().starts_with("/api/")
|
||||||
|| canonical_ticket_operation_workspace_id(uri.path()).is_some();
|
|| canonical_ticket_operation_workspace_id(uri.path()).is_some();
|
||||||
@@ -15024,6 +15129,12 @@ async fn log_failed_api_response(request: Request, next: Next) -> Response {
|
|||||||
.extensions()
|
.extensions()
|
||||||
.get::<Arc<server_api::RepositoryApiError>>()
|
.get::<Arc<server_api::RepositoryApiError>>()
|
||||||
.map(|error| ApiErrorLog::from_repository_error(error))
|
.map(|error| ApiErrorLog::from_repository_error(error))
|
||||||
|
.or_else(|| {
|
||||||
|
response
|
||||||
|
.extensions()
|
||||||
|
.get::<Arc<server_api::WorkerCreateApiError>>()
|
||||||
|
.map(|error| ApiErrorLog::from_repository_error(error.legacy()))
|
||||||
|
})
|
||||||
.or_else(|| {
|
.or_else(|| {
|
||||||
response
|
response
|
||||||
.extensions()
|
.extensions()
|
||||||
@@ -21673,6 +21784,7 @@ fn ticket_notification_delivery_error_category(
|
|||||||
InternalWorkerOperationState::Accepted => None,
|
InternalWorkerOperationState::Accepted => None,
|
||||||
InternalWorkerOperationState::Rejected => Some("runtime_rejected"),
|
InternalWorkerOperationState::Rejected => Some("runtime_rejected"),
|
||||||
InternalWorkerOperationState::Unsupported => Some("runtime_unsupported"),
|
InternalWorkerOperationState::Unsupported => Some("runtime_unsupported"),
|
||||||
|
InternalWorkerOperationState::Unknown => Some("runtime_outcome_unknown"),
|
||||||
},
|
},
|
||||||
Err(RuntimeRegistryError::InvalidIdentifier { .. }) => Some("invalid_identifier"),
|
Err(RuntimeRegistryError::InvalidIdentifier { .. }) => Some("invalid_identifier"),
|
||||||
Err(RuntimeRegistryError::UnknownRuntime(_)) => Some("unknown_runtime"),
|
Err(RuntimeRegistryError::UnknownRuntime(_)) => Some("unknown_runtime"),
|
||||||
@@ -34894,6 +35006,7 @@ fn record_browser_worker_spawn(
|
|||||||
if result.state != InternalWorkerOperationState::Accepted {
|
if result.state != InternalWorkerOperationState::Accepted {
|
||||||
return Err(worker_create_not_accepted_error(
|
return Err(worker_create_not_accepted_error(
|
||||||
requested_runtime_id.clone(),
|
requested_runtime_id.clone(),
|
||||||
|
result.failure,
|
||||||
result.diagnostics,
|
result.diagnostics,
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
@@ -34912,20 +35025,24 @@ fn browser_worker_response_from_summary(
|
|||||||
diagnostics: Vec<RuntimeDiagnostic>,
|
diagnostics: Vec<RuntimeDiagnostic>,
|
||||||
assignment: Option<&WorkerTicketAssignmentRequest>,
|
assignment: Option<&WorkerTicketAssignmentRequest>,
|
||||||
) -> ApiResult<BrowserCreateWorkerResponse> {
|
) -> ApiResult<BrowserCreateWorkerResponse> {
|
||||||
let worker_record = match record_worker_summary(
|
|
||||||
api,
|
|
||||||
&worker,
|
|
||||||
display_name.as_str(),
|
|
||||||
worker.profile.clone(),
|
|
||||||
WorkerRegistryDisplayNamePolicy::UseProvided,
|
|
||||||
) {
|
|
||||||
Ok(record) => record,
|
|
||||||
Err(error) => return Err(error.into()),
|
|
||||||
};
|
|
||||||
let compensation = WorkerSpawnCompensationContext {
|
let compensation = WorkerSpawnCompensationContext {
|
||||||
assignment,
|
assignment,
|
||||||
spawned_workdir_ids: &[],
|
spawned_workdir_ids: &[],
|
||||||
};
|
};
|
||||||
|
let worker_record = finalize_worker_spawn_stage(
|
||||||
|
api,
|
||||||
|
&worker,
|
||||||
|
&compensation,
|
||||||
|
WorkerSpawnFinalizeStage::WorkerRegistry,
|
||||||
|
record_worker_summary(
|
||||||
|
api,
|
||||||
|
&worker,
|
||||||
|
display_name.as_str(),
|
||||||
|
worker.profile.clone(),
|
||||||
|
WorkerRegistryDisplayNamePolicy::UseProvided,
|
||||||
|
)
|
||||||
|
.map_err(ApiError::from),
|
||||||
|
)?;
|
||||||
if let Some(assignment) = assignment {
|
if let Some(assignment) = assignment {
|
||||||
if let Err(error) = api.store.bind_ticket_assignment_operation_worker(
|
if let Err(error) = api.store.bind_ticket_assignment_operation_worker(
|
||||||
&api.config.workspace_id,
|
&api.config.workspace_id,
|
||||||
@@ -34979,12 +35096,15 @@ fn browser_worker_response_for_committed_summary(
|
|||||||
let workspace_id = api.workspace_id().to_string();
|
let workspace_id = api.workspace_id().to_string();
|
||||||
let resource_key = api
|
let resource_key = api
|
||||||
.store
|
.store
|
||||||
.worker_resource_key(&workspace_id, &worker.worker)?
|
.worker_resource_key(&workspace_id, &worker.worker)
|
||||||
.ok_or_else(|| {
|
.and_then(|key| {
|
||||||
Error::Store(format!(
|
key.ok_or_else(|| {
|
||||||
"Workspace Worker `{worker_id}` has no resource key after registration"
|
Error::Store(format!(
|
||||||
))
|
"Workspace Worker `{worker_id}` has no resource key after registration"
|
||||||
})?;
|
))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.map_err(|error| worker_startup::publication_error(api, &worker.worker, error.into()))?;
|
||||||
let worker = worker_launch_worker_summary(worker);
|
let worker = worker_launch_worker_summary(worker);
|
||||||
let console_href = browser_worker_console_href(&workspace_id, &resource_key);
|
let console_href = browser_worker_console_href(&workspace_id, &resource_key);
|
||||||
Ok(BrowserCreateWorkerResponse {
|
Ok(BrowserCreateWorkerResponse {
|
||||||
@@ -35594,6 +35714,9 @@ async fn list_runtime_workers(
|
|||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
enum WorkerSpawnFinalizeStage {
|
enum WorkerSpawnFinalizeStage {
|
||||||
|
WorkerIdentity,
|
||||||
|
WorkspaceApiBinding,
|
||||||
|
CreateReservation,
|
||||||
WorkerRegistry,
|
WorkerRegistry,
|
||||||
TicketAssignmentBind,
|
TicketAssignmentBind,
|
||||||
TicketAssignmentCurrent,
|
TicketAssignmentCurrent,
|
||||||
@@ -35602,19 +35725,6 @@ enum WorkerSpawnFinalizeStage {
|
|||||||
WorkdirAttachment,
|
WorkdirAttachment,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerSpawnFinalizeStage {
|
|
||||||
fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::WorkerRegistry => "worker_registry",
|
|
||||||
Self::TicketAssignmentBind => "ticket_assignment_bind",
|
|
||||||
Self::TicketAssignmentCurrent => "ticket_assignment_current",
|
|
||||||
Self::TicketStateAccept => "ticket_state_accept",
|
|
||||||
Self::WorkdirRegistry => "workdir_registry",
|
|
||||||
Self::WorkdirAttachment => "workdir_attachment",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
struct WorkerSpawnCompensationContext<'a> {
|
struct WorkerSpawnCompensationContext<'a> {
|
||||||
assignment: Option<&'a crate::hosts::WorkerTicketAssignmentRequest>,
|
assignment: Option<&'a crate::hosts::WorkerTicketAssignmentRequest>,
|
||||||
spawned_workdir_ids: &'a [String],
|
spawned_workdir_ids: &'a [String],
|
||||||
@@ -35630,52 +35740,19 @@ fn finalize_worker_spawn_stage<T>(
|
|||||||
let Err(source) = result else {
|
let Err(source) = result else {
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
let source_message = sanitize_backend_error(&source.error.to_string());
|
let (compensation_diagnostics, compensation) =
|
||||||
let operation_id = context
|
compensate_failed_worker_spawn_observed(api, worker, context);
|
||||||
.assignment
|
let failure = worker_startup::persist(
|
||||||
.map(|assignment| assignment.operation_id.as_str())
|
api,
|
||||||
.unwrap_or("none");
|
&worker.worker,
|
||||||
let mut diagnostics = vec![RuntimeDiagnostic {
|
worker_startup::finalize_failure(&source, stage, compensation),
|
||||||
startup: None,
|
);
|
||||||
code: format!("worker_spawn_finalize_{}_failed", stage.as_str()),
|
let mut diagnostics = source.diagnostics;
|
||||||
severity: HostDiagnosticSeverity::Error,
|
diagnostics.extend(compensation_diagnostics);
|
||||||
message: format!(
|
|
||||||
"Worker spawn finalize failed at stage `{}` for Runtime Worker {}:{} (operation `{operation_id}`): {source_message}",
|
|
||||||
stage.as_str(),
|
|
||||||
worker.worker.runtime_id,
|
|
||||||
worker.worker.worker_id
|
|
||||||
),
|
|
||||||
}];
|
|
||||||
diagnostics.extend(source.diagnostics);
|
|
||||||
let compensation_errors = compensate_failed_worker_spawn(api, worker, context);
|
|
||||||
let compensation_failed = !compensation_errors.is_empty();
|
|
||||||
diagnostics.extend(compensation_errors);
|
|
||||||
if !compensation_failed {
|
|
||||||
diagnostics.push(RuntimeDiagnostic {
|
|
||||||
startup: None,
|
|
||||||
code: "worker_spawn_compensated".to_string(),
|
|
||||||
severity: HostDiagnosticSeverity::Info,
|
|
||||||
message: format!(
|
|
||||||
"Removed Runtime Worker {}:{} and rolled back Backend spawn state",
|
|
||||||
worker.worker.runtime_id, worker.worker.worker_id
|
|
||||||
),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
let compensation = if compensation_failed {
|
|
||||||
"compensation left residual state; inspect diagnostics"
|
|
||||||
} else {
|
|
||||||
"compensation completed"
|
|
||||||
};
|
|
||||||
Err(ApiError::with_diagnostics(
|
Err(ApiError::with_diagnostics(
|
||||||
Error::RuntimeOperationFailed {
|
Error::WorkerCreateFailed {
|
||||||
runtime_id: worker.worker.runtime_id.clone(),
|
failure: Box::new(failure),
|
||||||
code: format!("worker_spawn_finalize_{}_failed", stage.as_str()),
|
source: Some(Box::new(source.error)),
|
||||||
message: format!(
|
|
||||||
"Worker spawn finalize failed at stage `{}` for Runtime Worker {}:{} (operation `{operation_id}`): {source_message}; {compensation}",
|
|
||||||
stage.as_str(),
|
|
||||||
worker.worker.runtime_id,
|
|
||||||
worker.worker.worker_id
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
diagnostics,
|
diagnostics,
|
||||||
))
|
))
|
||||||
@@ -35687,16 +35764,14 @@ fn write_workspace_worker_create_failure(
|
|||||||
phase: &str,
|
phase: &str,
|
||||||
diagnostics: &[RuntimeDiagnostic],
|
diagnostics: &[RuntimeDiagnostic],
|
||||||
) {
|
) {
|
||||||
tracing::error!(
|
tracing::warn!(
|
||||||
target: "yoi::worker_create",
|
target: "yoi::worker_create",
|
||||||
event = "worker_create_failed",
|
event = "worker_create_compensation",
|
||||||
component = "workspace_server",
|
component = "workspace_server",
|
||||||
runtime_id,
|
correlation = %worker_startup::correlation(runtime_id, &worker_id.to_string()),
|
||||||
worker_id = %worker_id,
|
|
||||||
phase,
|
phase,
|
||||||
cleanup_succeeded = diagnostics.is_empty(),
|
cleanup_completed = diagnostics.is_empty(),
|
||||||
cleanup_diagnostics = ?diagnostics,
|
"Worker creation compensation observed"
|
||||||
"Worker creation failed"
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -35771,6 +35846,32 @@ fn compensate_failed_workspace_worker_create(
|
|||||||
context: &WorkerSpawnCompensationContext<'_>,
|
context: &WorkerSpawnCompensationContext<'_>,
|
||||||
attachment_reservations: &[(String, String, String, workdir::WorkdirSessionCapabilities)],
|
attachment_reservations: &[(String, String, String, workdir::WorkdirSessionCapabilities)],
|
||||||
) -> Vec<RuntimeDiagnostic> {
|
) -> Vec<RuntimeDiagnostic> {
|
||||||
|
compensate_failed_workspace_worker_create_observed(
|
||||||
|
api,
|
||||||
|
runtime_id,
|
||||||
|
reservation_worker_id,
|
||||||
|
create_fingerprint,
|
||||||
|
failure_phase,
|
||||||
|
worker,
|
||||||
|
context,
|
||||||
|
attachment_reservations,
|
||||||
|
)
|
||||||
|
.0
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compensate_failed_workspace_worker_create_observed(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
runtime_id: &str,
|
||||||
|
reservation_worker_id: WorkerId,
|
||||||
|
create_fingerprint: &str,
|
||||||
|
failure_phase: &str,
|
||||||
|
worker: Option<&InternalWorkerSummary>,
|
||||||
|
context: &WorkerSpawnCompensationContext<'_>,
|
||||||
|
attachment_reservations: &[(String, String, String, workdir::WorkdirSessionCapabilities)],
|
||||||
|
) -> (
|
||||||
|
Vec<RuntimeDiagnostic>,
|
||||||
|
runtime_api::startup::CompensationOutcome,
|
||||||
|
) {
|
||||||
let mut diagnostics = Vec::new();
|
let mut diagnostics = Vec::new();
|
||||||
let reserved_worker_ref =
|
let reserved_worker_ref =
|
||||||
RuntimeWorkerRef::new(runtime_id.to_string(), reservation_worker_id.to_string());
|
RuntimeWorkerRef::new(runtime_id.to_string(), reservation_worker_id.to_string());
|
||||||
@@ -35840,18 +35941,31 @@ fn compensate_failed_workspace_worker_create(
|
|||||||
failure_phase,
|
failure_phase,
|
||||||
&diagnostics,
|
&diagnostics,
|
||||||
);
|
);
|
||||||
diagnostics
|
let outcome = if !reserved_worker_absent {
|
||||||
|
runtime_api::startup::CompensationOutcome::Unknown
|
||||||
|
} else if diagnostics.is_empty() {
|
||||||
|
runtime_api::startup::CompensationOutcome::Completed
|
||||||
|
} else {
|
||||||
|
runtime_api::startup::CompensationOutcome::Residual
|
||||||
|
};
|
||||||
|
(diagnostics, outcome)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn compensate_failed_worker_spawn(
|
fn compensate_failed_worker_spawn_observed(
|
||||||
api: &WorkspaceApi,
|
api: &WorkspaceApi,
|
||||||
worker: &InternalWorkerSummary,
|
worker: &InternalWorkerSummary,
|
||||||
context: &WorkerSpawnCompensationContext<'_>,
|
context: &WorkerSpawnCompensationContext<'_>,
|
||||||
) -> Vec<RuntimeDiagnostic> {
|
) -> (
|
||||||
|
Vec<RuntimeDiagnostic>,
|
||||||
|
runtime_api::startup::CompensationOutcome,
|
||||||
|
) {
|
||||||
let (runtime_deleted, mut diagnostics) =
|
let (runtime_deleted, mut diagnostics) =
|
||||||
WorkerRemovalService::new(api).delete_spawn_compensation_runtime_worker(&worker.worker);
|
WorkerRemovalService::new(api).delete_spawn_compensation_runtime_worker(&worker.worker);
|
||||||
if !runtime_deleted {
|
if !runtime_deleted {
|
||||||
return diagnostics;
|
return (
|
||||||
|
diagnostics,
|
||||||
|
runtime_api::startup::CompensationOutcome::Unknown,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
diagnostics.extend(finalize_spawn_compensation_after_worker_delete(
|
diagnostics.extend(finalize_spawn_compensation_after_worker_delete(
|
||||||
api,
|
api,
|
||||||
@@ -35872,7 +35986,12 @@ fn compensate_failed_worker_spawn(
|
|||||||
),
|
),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
diagnostics
|
let outcome = if diagnostics.is_empty() {
|
||||||
|
runtime_api::startup::CompensationOutcome::Completed
|
||||||
|
} else {
|
||||||
|
runtime_api::startup::CompensationOutcome::Residual
|
||||||
|
};
|
||||||
|
(diagnostics, outcome)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn finalize_spawn_compensation_after_worker_delete(
|
fn finalize_spawn_compensation_after_worker_delete(
|
||||||
@@ -36148,6 +36267,7 @@ async fn create_runtime_worker_authenticated_on_host(
|
|||||||
{
|
{
|
||||||
accept_queued_ticket_after_worker_spawn(&api, assignment)?;
|
accept_queued_ticket_after_worker_spawn(&api, assignment)?;
|
||||||
return Ok(Json(worker_spawn_result_to_api(WorkerSpawnResult {
|
return Ok(Json(worker_spawn_result_to_api(WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Accepted,
|
state: InternalWorkerOperationState::Accepted,
|
||||||
worker: Some(worker),
|
worker: Some(worker),
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -39159,19 +39279,15 @@ fn encode_path_segment(value: &str) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn worker_create_not_accepted_error(
|
fn worker_create_not_accepted_error(
|
||||||
runtime_id: String,
|
_runtime_id: String,
|
||||||
mut diagnostics: Vec<RuntimeDiagnostic>,
|
failure: Option<runtime_api::startup::WorkerCreateFailure>,
|
||||||
|
diagnostics: Vec<RuntimeDiagnostic>,
|
||||||
) -> ApiError {
|
) -> ApiError {
|
||||||
diagnostics.push(settings_diagnostic(
|
let failure = failure.unwrap_or_else(worker_startup::local_rejection);
|
||||||
"workspace_worker_create_not_accepted",
|
|
||||||
HostDiagnosticSeverity::Error,
|
|
||||||
"Runtime did not accept worker creation; see diagnostics for sanitized Runtime compatibility details.",
|
|
||||||
));
|
|
||||||
ApiError::with_diagnostics(
|
ApiError::with_diagnostics(
|
||||||
Error::RuntimeOperationFailed {
|
Error::WorkerCreateFailed {
|
||||||
runtime_id,
|
failure: Box::new(failure),
|
||||||
code: "workspace_worker_create_failed".to_string(),
|
source: None,
|
||||||
message: "Runtime did not accept worker creation".to_string(),
|
|
||||||
},
|
},
|
||||||
diagnostics,
|
diagnostics,
|
||||||
)
|
)
|
||||||
@@ -39666,6 +39782,18 @@ fn subjektiv_diagnostic_code(message: &str) -> Option<&'static str> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ApiError {
|
impl ApiError {
|
||||||
|
fn into_worker_create_api_error(self) -> server_api::WorkerCreateApiError {
|
||||||
|
match self.error {
|
||||||
|
Error::WorkerCreateFailed { failure, .. } => server_api::WorkerCreateApiError::startup(
|
||||||
|
*failure,
|
||||||
|
&protocol::session_failure::SessionFailureVisibility {
|
||||||
|
can_view_cause: true,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
_ => self.into_repository_api_error().into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn with_diagnostics(error: Error, diagnostics: Vec<RuntimeDiagnostic>) -> Self {
|
fn with_diagnostics(error: Error, diagnostics: Vec<RuntimeDiagnostic>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
error,
|
error,
|
||||||
@@ -39852,6 +39980,13 @@ fn api_error_response_message(error: &Error) -> String {
|
|||||||
|
|
||||||
impl IntoResponse for ApiError {
|
impl IntoResponse for ApiError {
|
||||||
fn into_response(self) -> Response {
|
fn into_response(self) -> Response {
|
||||||
|
if matches!(&self.error, Error::WorkerCreateFailed { .. }) {
|
||||||
|
let error = self.into_worker_create_api_error();
|
||||||
|
return api_macros::axum::http_error_response(
|
||||||
|
api_macros::axum::status(api_macros::HttpError::status_code(&error)),
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
}
|
||||||
let status = api_error_status(&self.error);
|
let status = api_error_status(&self.error);
|
||||||
let response_message = api_error_response_message(&self.error);
|
let response_message = api_error_response_message(&self.error);
|
||||||
let log = ApiErrorLog {
|
let log = ApiErrorLog {
|
||||||
@@ -39953,6 +40088,7 @@ mod tests {
|
|||||||
mod protocol_backpressure_tests;
|
mod protocol_backpressure_tests;
|
||||||
mod restore_workdir_tests;
|
mod restore_workdir_tests;
|
||||||
mod restored_parent_control_tests;
|
mod restored_parent_control_tests;
|
||||||
|
mod startup_result_tests;
|
||||||
mod subject_authorization_tests;
|
mod subject_authorization_tests;
|
||||||
mod subject_deletion_tests;
|
mod subject_deletion_tests;
|
||||||
mod notification_source_authorization_tests {
|
mod notification_source_authorization_tests {
|
||||||
@@ -43038,6 +43174,7 @@ mod tests {
|
|||||||
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
||||||
) {
|
) {
|
||||||
return WorkerSpawnResult {
|
return WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -43069,6 +43206,7 @@ mod tests {
|
|||||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
.push(worker.clone());
|
.push(worker.clone());
|
||||||
WorkerSpawnResult {
|
WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Accepted,
|
state: InternalWorkerOperationState::Accepted,
|
||||||
worker: Some(worker),
|
worker: Some(worker),
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -47164,7 +47302,18 @@ mod tests {
|
|||||||
guarded_spawn_payload(&fixture, "guarded-runtime-rejection"),
|
guarded_spawn_payload(&fixture, "guarded-runtime-rejection"),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||||
|
assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json");
|
||||||
|
let bytes = to_bytes(
|
||||||
|
response.into_body(),
|
||||||
|
runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let body: Value = serde_json::from_slice(&bytes).unwrap();
|
||||||
|
assert_eq!(body["yoi_failure"]["domain"], "worker_startup");
|
||||||
|
assert_eq!(body["yoi_failure"]["reason"], "failed");
|
||||||
|
assert_eq!(body["yoi_startup"]["operation"]["acceptance"], "rejected");
|
||||||
assert_eq!(fixture.runtime.spawn_requests().len(), 1);
|
assert_eq!(fixture.runtime.spawn_requests().len(), 1);
|
||||||
assert_guarded_spawn_rolled_back(&fixture);
|
assert_guarded_spawn_rolled_back(&fixture);
|
||||||
}
|
}
|
||||||
@@ -47191,7 +47340,17 @@ mod tests {
|
|||||||
guarded_spawn_payload(&fixture, "guarded-post-create-failure"),
|
guarded_spawn_payload(&fixture, "guarded-post-create-failure"),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||||
|
assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json");
|
||||||
|
let bytes = to_bytes(
|
||||||
|
response.into_body(),
|
||||||
|
runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let body: Value = serde_json::from_slice(&bytes).unwrap();
|
||||||
|
assert_eq!(body["yoi_startup"]["operation"]["acceptance"], "accepted");
|
||||||
|
assert_eq!(body["yoi_failure"]["reason"], "finalize_failed");
|
||||||
assert_eq!(fixture.runtime.spawn_requests().len(), 1);
|
assert_eq!(fixture.runtime.spawn_requests().len(), 1);
|
||||||
assert_guarded_spawn_rolled_back(&fixture);
|
assert_guarded_spawn_rolled_back(&fixture);
|
||||||
}
|
}
|
||||||
@@ -61792,7 +61951,7 @@ mod tests {
|
|||||||
sync_worker_observation(&api, &summary).unwrap();
|
sync_worker_observation(&api, &summary).unwrap();
|
||||||
let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap();
|
let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap();
|
||||||
let _ = subscriber.take_snapshot();
|
let _ = subscriber.take_snapshot();
|
||||||
let diagnostics = compensate_failed_worker_spawn(
|
let (diagnostics, _) = compensate_failed_worker_spawn_observed(
|
||||||
&api,
|
&api,
|
||||||
&summary,
|
&summary,
|
||||||
&WorkerSpawnCompensationContext {
|
&WorkerSpawnCompensationContext {
|
||||||
@@ -68042,24 +68201,32 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let response = worker_create_not_accepted_error(
|
let response = worker_create_not_accepted_error(
|
||||||
"unsupported-runtime".to_string(),
|
"unsupported-runtime".into(),
|
||||||
|
None,
|
||||||
vec![settings_diagnostic(
|
vec![settings_diagnostic(
|
||||||
"remote_runtime_unsupported",
|
"remote_runtime_unsupported",
|
||||||
HostDiagnosticSeverity::Warning,
|
HostDiagnosticSeverity::Warning,
|
||||||
"Remote Runtime provisioning is unsupported by this v0 worker launch path.",
|
"TEST_ONLY_DIAGNOSTIC_TEXT",
|
||||||
)],
|
)],
|
||||||
)
|
)
|
||||||
.into_response();
|
.into_response();
|
||||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||||
let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json");
|
||||||
|
let bytes = to_bytes(
|
||||||
|
response.into_body(),
|
||||||
|
runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
let response: Value = serde_json::from_slice(&bytes).unwrap();
|
let response: Value = serde_json::from_slice(&bytes).unwrap();
|
||||||
let diagnostics = response["diagnostics"].as_array().unwrap();
|
assert_eq!(response["yoi_failure"]["domain"], "worker_startup");
|
||||||
assert!(diagnostics.len() >= 2);
|
assert!(response.get("diagnostics").is_none());
|
||||||
assert!(
|
assert!(
|
||||||
diagnostics
|
!response
|
||||||
.iter()
|
.to_string()
|
||||||
.any(|diagnostic| { diagnostic["code"] == "workspace_worker_create_not_accepted" })
|
.contains("workspace_worker_create_not_accepted")
|
||||||
);
|
);
|
||||||
|
assert!(!response.to_string().contains("TEST_ONLY_DIAGNOSTIC_TEXT"));
|
||||||
let projected = serde_json::to_string(&response).unwrap();
|
let projected = serde_json::to_string(&response).unwrap();
|
||||||
assert!(!projected.contains("http://"));
|
assert!(!projected.contains("http://"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,14 +1,18 @@
|
|||||||
//! Focused T840 connection: real Subject policy -> signed Session POST -> real
|
//! Connected startup acceptance: real Subject policy -> signed Session POST ->
|
||||||
//! ProfileRuntimeWorkerFactory startup -> RuntimeHTTP/Host diagnostic -> saved
|
//! ProfileRuntimeWorkerFactory -> Runtime HTTP -> Backend, including the real
|
||||||
//! Worker metadata. Only the unused external model is replaced. No restore,
|
//! authenticated Browser worker-create HTTP route and saved Worker metadata.
|
||||||
//! operation replay, singleton-release, or Browser rendering contract is asserted.
|
//! Only the unused external model is replaced. Read-only reconciliation is not
|
||||||
|
//! restore, operation replay, or a Browser rendering contract.
|
||||||
use super::*;
|
use super::*;
|
||||||
use api_macros::HttpError as _;
|
use api_macros::HttpError as _;
|
||||||
use protocol::session_failure::{
|
use protocol::session_failure::{
|
||||||
SessionPrincipal, SessionRegistrationCause, session_registration_attachment,
|
SessionPrincipal, SessionRegistrationCause, session_registration_attachment,
|
||||||
};
|
};
|
||||||
use session_store::WorkerMetadataStore as _;
|
use session_store::WorkerMetadataStore as _;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use tracing::instrument::WithSubscriber as _;
|
||||||
|
use tracing_subscriber::{Layer as _, layer::SubscriberExt as _};
|
||||||
use worker_runtime::worker_backend::RuntimeWorkerFactory as _;
|
use worker_runtime::worker_backend::RuntimeWorkerFactory as _;
|
||||||
|
|
||||||
#[derive(Clone, Default)]
|
#[derive(Clone, Default)]
|
||||||
@@ -30,6 +34,57 @@ impl agen::llm_client::LlmClient for UnusedModel {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct StartupEvents(Arc<Mutex<Vec<BTreeMap<String, String>>>>);
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct StartupEventFields(BTreeMap<String, String>);
|
||||||
|
|
||||||
|
impl tracing::field::Visit for StartupEventFields {
|
||||||
|
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||||
|
self.0.insert(field.name().into(), value.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||||
|
self.0.insert(field.name().into(), format!("{value:?}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for StartupEvents {
|
||||||
|
fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) {
|
||||||
|
if event.metadata().target() == "yoi::worker_startup" {
|
||||||
|
let mut fields = StartupEventFields::default();
|
||||||
|
event.record(&mut fields);
|
||||||
|
self.0.lock().unwrap().push(fields.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A real formatter writer failure, not a replacement result/observer function.
|
||||||
|
/// The request-scoped subscriber avoids process-global test interference.
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct FailedStartupSink(Arc<AtomicUsize>);
|
||||||
|
|
||||||
|
impl std::io::Write for FailedStartupSink {
|
||||||
|
fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
|
||||||
|
self.0.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err(std::io::Error::other(
|
||||||
|
"test: startup observation sink unavailable",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush(&mut self) -> std::io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for FailedStartupSink {
|
||||||
|
type Writer = Self;
|
||||||
|
fn make_writer(&'a self) -> Self::Writer {
|
||||||
|
self.clone()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type ReleaseFence = (
|
type ReleaseFence = (
|
||||||
crate::store::WorkerSingletonLease,
|
crate::store::WorkerSingletonLease,
|
||||||
subjektiv::journal::JournalWriterBinding,
|
subjektiv::journal::JournalWriterBinding,
|
||||||
@@ -41,6 +96,8 @@ struct RegistrationFixture {
|
|||||||
callbacks: Arc<Mutex<Vec<(StatusCode, Value)>>>,
|
callbacks: Arc<Mutex<Vec<(StatusCode, Value)>>>,
|
||||||
model: UnusedModel,
|
model: UnusedModel,
|
||||||
release_fence: Arc<Mutex<Option<ReleaseFence>>>,
|
release_fence: Arc<Mutex<Option<ReleaseFence>>>,
|
||||||
|
startup_events: StartupEvents,
|
||||||
|
failed_sink: FailedStartupSink,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
@@ -49,6 +106,7 @@ enum RegistrationFault {
|
|||||||
Deny,
|
Deny,
|
||||||
DenyWithMetadataReadFailure,
|
DenyWithMetadataReadFailure,
|
||||||
ReleasePending,
|
ReleasePending,
|
||||||
|
CommittedResponseLost,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RegistrationFixture {
|
impl RegistrationFixture {
|
||||||
@@ -63,13 +121,31 @@ impl RegistrationFixture {
|
|||||||
let model = UnusedModel::default();
|
let model = UnusedModel::default();
|
||||||
let release_fence = Arc::new(Mutex::new(None));
|
let release_fence = Arc::new(Mutex::new(None));
|
||||||
let reserved = release_fence.clone();
|
let reserved = release_fence.clone();
|
||||||
|
let startup_events = StartupEvents::default();
|
||||||
|
let failed_sink = FailedStartupSink::default();
|
||||||
|
let events = startup_events.clone();
|
||||||
|
let sink = failed_sink.clone();
|
||||||
let connected = ConnectedWorkerHarness::start_with_layer(model.clone(), move |router| {
|
let connected = ConnectedWorkerHarness::start_with_layer(model.clone(), move |router| {
|
||||||
router.layer(axum::middleware::from_fn(
|
router.layer(axum::middleware::from_fn(
|
||||||
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
||||||
let hook = hook.clone();
|
let hook = hook.clone();
|
||||||
let observed = observed.clone();
|
let observed = observed.clone();
|
||||||
let reserved = reserved.clone();
|
let reserved = reserved.clone();
|
||||||
|
let events = events.clone();
|
||||||
|
let sink = sink.clone();
|
||||||
async move {
|
async move {
|
||||||
|
if request.method() == axum::http::Method::POST
|
||||||
|
&& request.uri().path() == format!("/api/w/{TEST_WORKSPACE_ID}/workers")
|
||||||
|
{
|
||||||
|
let subscriber = tracing_subscriber::registry()
|
||||||
|
.with(events)
|
||||||
|
.with(tracing_subscriber::fmt::layer()
|
||||||
|
.with_writer(sink)
|
||||||
|
.with_filter(tracing_subscriber::filter::filter_fn(|metadata| {
|
||||||
|
metadata.target() == "yoi::worker_startup"
|
||||||
|
})));
|
||||||
|
return next.run(request).with_subscriber(subscriber).await;
|
||||||
|
}
|
||||||
if request.method() != axum::http::Method::POST
|
if request.method() != axum::http::Method::POST
|
||||||
|| request.uri().path()
|
|| request.uri().path()
|
||||||
!= format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions")
|
!= format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/sessions")
|
||||||
@@ -108,7 +184,7 @@ impl RegistrationFixture {
|
|||||||
assert!(store.journal_writer_active(&writer).unwrap());
|
assert!(store.journal_writer_active(&writer).unwrap());
|
||||||
assert_eq!(api.store.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key).unwrap(), Some(lease.clone()));
|
assert_eq!(api.store.current_worker_singleton_owner(TEST_WORKSPACE_ID, &key).unwrap(), Some(lease.clone()));
|
||||||
*reserved.lock().unwrap() = Some((lease, writer));
|
*reserved.lock().unwrap() = Some((lease, writer));
|
||||||
} else {
|
} else if !matches!(fault, RegistrationFault::CommittedResponseLost) {
|
||||||
store
|
store
|
||||||
.set_subject_permissions(
|
.set_subject_permissions(
|
||||||
&subject,
|
&subject,
|
||||||
@@ -163,6 +239,20 @@ impl RegistrationFixture {
|
|||||||
std::fs::create_dir(&metadata).unwrap();
|
std::fs::create_dir(&metadata).unwrap();
|
||||||
}
|
}
|
||||||
observed.lock().unwrap().push((response.status(), input));
|
observed.lock().unwrap().push((response.status(), input));
|
||||||
|
if matches!(fault, RegistrationFault::CommittedResponseLost) {
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
// The genuine signed handler committed before its response is
|
||||||
|
// lost. Do not replace its authorization, attribution, or write.
|
||||||
|
// A broken body fails the actual Runtime HTTP client rather
|
||||||
|
// than fabricating a typed Runtime/domain failure in this layer.
|
||||||
|
let (parts, _) = response.into_parts();
|
||||||
|
let body = Body::from_stream(futures::stream::once(async {
|
||||||
|
Err::<Vec<u8>, _>(std::io::Error::other(
|
||||||
|
"test: committed registration response lost",
|
||||||
|
))
|
||||||
|
}));
|
||||||
|
return Response::from_parts(parts, body);
|
||||||
|
}
|
||||||
response
|
response
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -200,6 +290,8 @@ impl RegistrationFixture {
|
|||||||
callbacks,
|
callbacks,
|
||||||
model,
|
model,
|
||||||
release_fence,
|
release_fence,
|
||||||
|
startup_events,
|
||||||
|
failed_sink,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -222,6 +314,178 @@ impl RegistrationFixture {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn browser_failure(
|
||||||
|
&self,
|
||||||
|
expected_status: StatusCode,
|
||||||
|
) -> (
|
||||||
|
runtime_api::startup::StartupProblemDetails,
|
||||||
|
runtime_api::startup::WorkerCreateFailure,
|
||||||
|
) {
|
||||||
|
use runtime_api::startup::{
|
||||||
|
DecodeStartupProblem, PROBLEM_CONTENT_TYPE, StartupBoundary, decode_startup_problem,
|
||||||
|
};
|
||||||
|
let (status, headers, bytes) = self
|
||||||
|
.connected
|
||||||
|
.human_raw(
|
||||||
|
reqwest::Method::POST,
|
||||||
|
&format!("/api/w/{TEST_WORKSPACE_ID}/workers"),
|
||||||
|
serde_json::to_value(self.request()).unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, expected_status);
|
||||||
|
let content_type = headers[axum::http::header::CONTENT_TYPE].to_str().unwrap();
|
||||||
|
assert_eq!(content_type, PROBLEM_CONTENT_TYPE);
|
||||||
|
let DecodeStartupProblem::Known(failure) = decode_startup_problem(
|
||||||
|
&bytes,
|
||||||
|
status.as_u16(),
|
||||||
|
content_type,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
) else {
|
||||||
|
panic!("real Browser HTTP response must decode as a bounded startup Problem");
|
||||||
|
};
|
||||||
|
let problem: runtime_api::startup::StartupProblemDetails =
|
||||||
|
serde_json::from_slice(&bytes).unwrap();
|
||||||
|
assert_eq!(problem.status, status.as_u16());
|
||||||
|
// Neither raw callback diagnostics nor authority-bearing identifiers may
|
||||||
|
// be copied into a public explanation/extension by the adapters.
|
||||||
|
let body = std::str::from_utf8(&bytes).unwrap();
|
||||||
|
for secret in [
|
||||||
|
&self.connected.owner,
|
||||||
|
&self.subject,
|
||||||
|
&self.connected.backend_url,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!body.contains(secret),
|
||||||
|
"Browser Problem disclosed private fixture data"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!body.contains("test: committed registration response lost"));
|
||||||
|
assert!(!body.contains("worker-runtime-"));
|
||||||
|
// Public, bounded handler bytes, after the non-disclosure checks above.
|
||||||
|
// These provide the actual shared Browser fixture source under --nocapture;
|
||||||
|
// no metadata, callback body, credential, or private upstream text is logged.
|
||||||
|
eprintln!(
|
||||||
|
"T812_BROWSER_STARTUP_PROBLEM status={status} content_type={content_type} body={body}"
|
||||||
|
);
|
||||||
|
(problem, failure)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_backend_failure(
|
||||||
|
&self,
|
||||||
|
worker: &str,
|
||||||
|
session: &str,
|
||||||
|
problem: &runtime_api::startup::StartupProblemDetails,
|
||||||
|
browser: &runtime_api::startup::WorkerCreateFailure,
|
||||||
|
expected: SessionRegistrationCause,
|
||||||
|
) {
|
||||||
|
use protocol::startup_failure::StartupPersistenceOutcome::Persisted;
|
||||||
|
use runtime_api::startup::DecodeStoredWorkerCreateFailure;
|
||||||
|
let reference = RuntimeWorkerRef::new(CONNECTED_RUNTIME, worker);
|
||||||
|
let Some(DecodeStoredWorkerCreateFailure::Known(saved)) = self
|
||||||
|
.connected
|
||||||
|
.api
|
||||||
|
.worker_create_failure(&reference)
|
||||||
|
.unwrap()
|
||||||
|
else {
|
||||||
|
panic!("real Browser failure must be readable from the canonical Backend reservation");
|
||||||
|
};
|
||||||
|
assert_startup_semantics(saved.startup(), browser.startup(), expected);
|
||||||
|
assert_eq!(
|
||||||
|
saved.operation(),
|
||||||
|
browser.operation(),
|
||||||
|
"HTTP cannot rewrite acceptance/execution/finalization/compensation/correlation"
|
||||||
|
);
|
||||||
|
assert_eq!(saved.backend_result_persistence(), Some(Persisted));
|
||||||
|
assert_eq!(browser.backend_result_persistence(), Some(Persisted));
|
||||||
|
assert_eq!(
|
||||||
|
problem.yoi_startup.backend_result_persistence,
|
||||||
|
Some(Persisted)
|
||||||
|
);
|
||||||
|
assert_eq!(saved.public_facts(), problem.yoi_startup);
|
||||||
|
let correlation = saved
|
||||||
|
.operation()
|
||||||
|
.correlation
|
||||||
|
.as_ref()
|
||||||
|
.expect("Backend-selected lookup token");
|
||||||
|
assert_eq!(
|
||||||
|
correlation.as_str(),
|
||||||
|
worker_startup::correlation(CONNECTED_RUNTIME, worker)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
problem.instance,
|
||||||
|
format!("urn:yoi:worker-startup:{}", correlation.as_str())
|
||||||
|
);
|
||||||
|
|
||||||
|
let events = self.startup_events.0.lock().unwrap();
|
||||||
|
let result_events = events
|
||||||
|
.iter()
|
||||||
|
.filter(|event| {
|
||||||
|
event
|
||||||
|
.get("event")
|
||||||
|
.is_some_and(|value| value == "worker_create_result")
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
result_events.len(),
|
||||||
|
1,
|
||||||
|
"one HTTP responsibility boundary records the primary, not every adapter"
|
||||||
|
);
|
||||||
|
let event = result_events[0];
|
||||||
|
assert_eq!(event["domain"], problem.yoi_failure.domain);
|
||||||
|
assert_eq!(event["reason"], problem.yoi_failure.reason);
|
||||||
|
assert_eq!(event["correlation"], correlation.as_str());
|
||||||
|
assert_eq!(event["status"], problem.status.to_string());
|
||||||
|
assert_eq!(
|
||||||
|
event["acceptance"],
|
||||||
|
format!("{:?}", saved.operation().acceptance)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
event["execution"],
|
||||||
|
format!("{:?}", saved.operation().execution)
|
||||||
|
);
|
||||||
|
let logged = serde_json::to_string(&*events).unwrap();
|
||||||
|
for secret in [
|
||||||
|
self.connected.owner.as_str(),
|
||||||
|
self.subject.as_str(),
|
||||||
|
self.connected.backend_url.as_str(),
|
||||||
|
session,
|
||||||
|
worker,
|
||||||
|
"test: committed registration response lost",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!logged.contains(secret),
|
||||||
|
"startup observation leaked private fixture data"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!logged.contains(self.connected.runtime_root.to_str().unwrap()));
|
||||||
|
drop(events);
|
||||||
|
assert!(
|
||||||
|
self.failed_sink.0.load(Ordering::SeqCst) > 0,
|
||||||
|
"actual tracing formatter sink failed while HTTP/Backend/metadata results were preserved"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
self.connected
|
||||||
|
.api
|
||||||
|
.worker_create_failure(&reference)
|
||||||
|
.unwrap(),
|
||||||
|
Some(DecodeStoredWorkerCreateFailure::Known(saved)),
|
||||||
|
"lookup is read-only, never a create retry"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn attempted_worker(&self) -> String {
|
||||||
|
let workers = std::fs::read_dir(self.connected.runtime_root.join("workers"))
|
||||||
|
.unwrap()
|
||||||
|
.map(|entry| entry.unwrap().file_name().into_string().unwrap())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
workers.len(),
|
||||||
|
1,
|
||||||
|
"one real factory attempt, never a new create/replay"
|
||||||
|
);
|
||||||
|
workers[0].clone()
|
||||||
|
}
|
||||||
|
|
||||||
fn assert_callback(&self, expected: StatusCode) -> String {
|
fn assert_callback(&self, expected: StatusCode) -> String {
|
||||||
let callbacks = self.callbacks.lock().unwrap();
|
let callbacks = self.callbacks.lock().unwrap();
|
||||||
assert!(!callbacks.is_empty(), "real signed startup callback");
|
assert!(!callbacks.is_empty(), "real signed startup callback");
|
||||||
@@ -250,6 +514,42 @@ fn contains_saved_value(saved: &Value, expected: &Value) -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Runtime/Browser normalization reprojects for the receiving boundary, including
|
||||||
|
// its local explanation. Compare domain enums, not byte identity of envelopes;
|
||||||
|
// saved metadata below still checks the original Storage attachment independently.
|
||||||
|
fn session_primary(
|
||||||
|
primary: &protocol::startup_failure::StartupFailureAttachment,
|
||||||
|
) -> Option<SessionRegistrationCause> {
|
||||||
|
match primary.decode::<SessionRegistrationCause>() {
|
||||||
|
error_contract::DecodeFailure::Known(received) => Some(*received.facts().cause()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_session_primary(
|
||||||
|
primary: &protocol::startup_failure::StartupFailureAttachment,
|
||||||
|
expected: SessionRegistrationCause,
|
||||||
|
) {
|
||||||
|
assert_eq!(
|
||||||
|
session_primary(primary),
|
||||||
|
Some(expected),
|
||||||
|
"Session failure must remain a known typed domain cause"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_startup_semantics(
|
||||||
|
host: &runtime_api::WorkerStartupFailure,
|
||||||
|
wire: &runtime_api::WorkerStartupFailure,
|
||||||
|
expected: SessionRegistrationCause,
|
||||||
|
) {
|
||||||
|
assert_session_primary(&host.primary, expected);
|
||||||
|
assert_session_primary(&wire.primary, expected);
|
||||||
|
assert_eq!(host.secondary, wire.secondary);
|
||||||
|
assert_eq!(host.persistence, wire.persistence);
|
||||||
|
assert_eq!(host.resources, wire.resources);
|
||||||
|
assert_eq!(host.io_kind, wire.io_kind);
|
||||||
|
}
|
||||||
|
|
||||||
fn read_metadata(root: &std::path::Path, worker: &str) -> Value {
|
fn read_metadata(root: &std::path::Path, worker: &str) -> Value {
|
||||||
// Reopen the canonical metadata store, not the separate Runtime catalog.
|
// Reopen the canonical metadata store, not the separate Runtime catalog.
|
||||||
let name = format!("worker-runtime-{worker}");
|
let name = format!("worker-runtime-{worker}");
|
||||||
@@ -346,6 +646,168 @@ fn assert_registration_cause(body: Value, cause: SessionRegistrationCause) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
|
||||||
|
async fn browser_http_session_record_denial_retains_subject_cause_and_saved_resources() {
|
||||||
|
use protocol::startup_failure::{
|
||||||
|
StartupFailurePhase, StartupPersistenceOutcome, StartupSecondaryFailure,
|
||||||
|
};
|
||||||
|
use runtime_api::startup::{ExecutionOutcome, StartupCause};
|
||||||
|
|
||||||
|
let fixture = RegistrationFixture::new(RegistrationFault::Deny).await;
|
||||||
|
// This POST must pass the real Browser authentication/preflight and reach
|
||||||
|
// the signed callback. Only then does the fixture remove session.record.
|
||||||
|
let (problem, browser) = fixture.browser_failure(StatusCode::CONFLICT).await;
|
||||||
|
let session = fixture.assert_callback(StatusCode::FORBIDDEN);
|
||||||
|
assert_eq!(fixture.callbacks.lock().unwrap().len(), 1);
|
||||||
|
let expected = SessionRegistrationCause::denied(SessionPrincipal::Subject);
|
||||||
|
assert_eq!(problem.type_uri, "urn:yoi:problem:conflict");
|
||||||
|
assert_eq!(problem.yoi_failure.domain, "session_registration");
|
||||||
|
assert_eq!(problem.yoi_failure.reason, "permission_denied");
|
||||||
|
assert_eq!(problem.yoi_failure.details, StartupCause::Session(expected));
|
||||||
|
assert!(problem.detail.contains("Subject"), "{}", problem.detail);
|
||||||
|
assert!(
|
||||||
|
problem.detail.contains("session.record"),
|
||||||
|
"{}",
|
||||||
|
problem.detail
|
||||||
|
);
|
||||||
|
assert_eq!(browser.operation().execution, ExecutionOutcome::Errored);
|
||||||
|
assert_session_primary(&browser.startup().primary, expected);
|
||||||
|
|
||||||
|
let runtime = fixture.connected.runtime_startup_failures.lock().unwrap();
|
||||||
|
assert_eq!(runtime.len(), 1, "one actual Runtime HTTP failure");
|
||||||
|
assert_session_primary(&runtime[0].primary, expected);
|
||||||
|
assert_eq!(
|
||||||
|
browser.startup().persistence,
|
||||||
|
Some(StartupPersistenceOutcome::Persisted)
|
||||||
|
);
|
||||||
|
assert_eq!(browser.startup().persistence, runtime[0].persistence);
|
||||||
|
assert!(
|
||||||
|
runtime[0].resources.is_some(),
|
||||||
|
"resource disposition is independent of the denial"
|
||||||
|
);
|
||||||
|
assert_eq!(browser.startup().resources, runtime[0].resources);
|
||||||
|
assert!(
|
||||||
|
browser.startup().secondary.iter().any(|secondary| matches!(
|
||||||
|
secondary,
|
||||||
|
StartupSecondaryFailure::Persistence {
|
||||||
|
phase: StartupFailurePhase::RuntimeRecordDelete,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
)),
|
||||||
|
"real retention pin prevents Runtime record deletion without replacing the denial"
|
||||||
|
);
|
||||||
|
drop(runtime);
|
||||||
|
|
||||||
|
let store = open_subjektiv_store(&fixture.connected.api).unwrap();
|
||||||
|
assert!(store.session_attribution(&session).unwrap().is_none());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.subject_session_attributions(&fixture.subject)
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
let worker = fixture.attempted_worker();
|
||||||
|
fixture.assert_backend_failure(&worker, &session, &problem, &browser, expected);
|
||||||
|
let saved = read_metadata(&fixture.connected.runtime_root, &worker);
|
||||||
|
assert!(contains_saved_value(
|
||||||
|
&saved,
|
||||||
|
&serde_json::to_value(session_registration_attachment(expected).unwrap()).unwrap()
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
fixture.connected.runtime.list_workers().unwrap().len(),
|
||||||
|
1,
|
||||||
|
"retained Runtime resource is observed separately from persisted cause and HTTP409"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
read_metadata(&fixture.connected.runtime_root, &worker),
|
||||||
|
saved,
|
||||||
|
"ordinary read-only reopen, not create/restore/retry"
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.callbacks.lock().unwrap().len(), 1);
|
||||||
|
assert_eq!(fixture.model.0.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
|
||||||
|
async fn browser_http_committed_registration_response_loss_stays_unknown_without_reexecution() {
|
||||||
|
use protocol::startup_failure::StartupPersistenceOutcome;
|
||||||
|
use runtime_api::startup::StartupCause;
|
||||||
|
|
||||||
|
let fixture = RegistrationFixture::new(RegistrationFault::CommittedResponseLost).await;
|
||||||
|
let (problem, browser) = fixture.browser_failure(StatusCode::BAD_GATEWAY).await;
|
||||||
|
// This is the status of the original signed handler, before transport loss.
|
||||||
|
let session = fixture.assert_callback(StatusCode::OK);
|
||||||
|
assert_eq!(
|
||||||
|
fixture.callbacks.lock().unwrap().len(),
|
||||||
|
1,
|
||||||
|
"response loss must not automatically reexecute Session registration"
|
||||||
|
);
|
||||||
|
let expected = SessionRegistrationCause::outcome_unknown();
|
||||||
|
assert_eq!(problem.yoi_failure.domain, "session_registration");
|
||||||
|
assert_eq!(problem.yoi_failure.reason, "outcome_unknown");
|
||||||
|
assert_eq!(problem.yoi_failure.details, StartupCause::Session(expected));
|
||||||
|
assert!(
|
||||||
|
problem.detail.contains("session.record"),
|
||||||
|
"{}",
|
||||||
|
problem.detail
|
||||||
|
);
|
||||||
|
assert_session_primary(&browser.startup().primary, expected);
|
||||||
|
assert_eq!(
|
||||||
|
browser.startup().persistence,
|
||||||
|
Some(StartupPersistenceOutcome::Persisted)
|
||||||
|
);
|
||||||
|
let runtime = fixture.connected.runtime_startup_failures.lock().unwrap();
|
||||||
|
assert_eq!(runtime.len(), 1);
|
||||||
|
assert_session_primary(&runtime[0].primary, expected);
|
||||||
|
assert_eq!(browser.startup().resources, runtime[0].resources);
|
||||||
|
assert_eq!(browser.startup().persistence, runtime[0].persistence);
|
||||||
|
drop(runtime);
|
||||||
|
|
||||||
|
let store = open_subjektiv_store(&fixture.connected.api).unwrap();
|
||||||
|
let committed = store.session_attribution(&session).unwrap().unwrap();
|
||||||
|
assert_eq!(committed.subject_id, fixture.subject);
|
||||||
|
assert_eq!(committed.runtime_id, CONNECTED_RUNTIME);
|
||||||
|
let worker = fixture.attempted_worker();
|
||||||
|
fixture.assert_backend_failure(&worker, &session, &problem, &browser, expected);
|
||||||
|
assert_eq!(committed.worker_id, worker);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.subject_session_attributions(&fixture.subject)
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
1,
|
||||||
|
"HTTP failure is not proof of no Backend write"
|
||||||
|
);
|
||||||
|
let saved = read_metadata(&fixture.connected.runtime_root, &worker);
|
||||||
|
assert_eq!(
|
||||||
|
saved["subjektiv_session_attribution"]["state"], "outcome_unknown",
|
||||||
|
"saved Runtime observation must not claim confirmed registration or rejection"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
saved["subjektiv_session_attribution"]["session_id"],
|
||||||
|
session
|
||||||
|
);
|
||||||
|
assert!(contains_saved_value(
|
||||||
|
&saved,
|
||||||
|
&serde_json::to_value(session_registration_attachment(expected).unwrap()).unwrap()
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
read_metadata(&fixture.connected.runtime_root, &worker),
|
||||||
|
saved
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.session_attribution(&session).unwrap(),
|
||||||
|
Some(committed)
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.connected.runtime.list_workers().unwrap().len(), 1);
|
||||||
|
assert_eq!(fixture.attempted_worker(), worker);
|
||||||
|
assert_eq!(
|
||||||
|
fixture.callbacks.lock().unwrap().len(),
|
||||||
|
1,
|
||||||
|
"Problem decoding, attribution lookup and metadata reopen are read-only, never replay authority"
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.model.0.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
|
||||||
async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen() {
|
async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen() {
|
||||||
let fixture = RegistrationFixture::new(RegistrationFault::Deny).await;
|
let fixture = RegistrationFixture::new(RegistrationFault::Deny).await;
|
||||||
@@ -390,7 +852,10 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen()
|
|||||||
SessionPrincipal::Subject,
|
SessionPrincipal::Subject,
|
||||||
))
|
))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(wire.primary, expected_primary);
|
assert_session_primary(
|
||||||
|
&wire.primary,
|
||||||
|
SessionRegistrationCause::denied(SessionPrincipal::Subject),
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
wire.persistence,
|
wire.persistence,
|
||||||
Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted)
|
Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted)
|
||||||
@@ -399,9 +864,16 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen()
|
|||||||
.diagnostics
|
.diagnostics
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|d| d.startup.as_ref())
|
.filter_map(|d| d.startup.as_ref())
|
||||||
.find(|startup| startup.primary == wire.primary)
|
.find(|startup| {
|
||||||
|
session_primary(&startup.primary)
|
||||||
|
== Some(SessionRegistrationCause::denied(SessionPrincipal::Subject))
|
||||||
|
})
|
||||||
.expect("RemoteWorkerRuntime must preserve the typed startup primary in Host diagnostics");
|
.expect("RemoteWorkerRuntime must preserve the typed startup primary in Host diagnostics");
|
||||||
assert_eq!(host.primary, wire.primary);
|
assert_startup_semantics(
|
||||||
|
host,
|
||||||
|
wire,
|
||||||
|
SessionRegistrationCause::denied(SessionPrincipal::Subject),
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
host.secondary.iter().any(|failure| matches!(
|
host.secondary.iter().any(|failure| matches!(
|
||||||
failure,
|
failure,
|
||||||
@@ -412,7 +884,7 @@ async fn real_session_record_denial_survives_runtime_host_and_metadata_reopen()
|
|||||||
)),
|
)),
|
||||||
"a real Session retention pin rejects outer Runtime deletion without hiding the primary"
|
"a real Session retention pin rejects outer Runtime deletion without hiding the primary"
|
||||||
);
|
);
|
||||||
let expected = serde_json::to_value(&wire.primary).unwrap();
|
let expected = serde_json::to_value(&expected_primary).unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
expected.is_string(),
|
expected.is_string(),
|
||||||
"the shared attachment contains a codec envelope"
|
"the shared attachment contains a codec envelope"
|
||||||
@@ -467,12 +939,9 @@ async fn real_session_record_denial_and_metadata_save_failure_remain_separate()
|
|||||||
"one real startup attempt, not an operation replay"
|
"one real startup attempt, not an operation replay"
|
||||||
);
|
);
|
||||||
let wire = &failures[0];
|
let wire = &failures[0];
|
||||||
assert_eq!(
|
assert_session_primary(
|
||||||
wire.primary,
|
&wire.primary,
|
||||||
session_registration_attachment(SessionRegistrationCause::denied(
|
SessionRegistrationCause::denied(SessionPrincipal::Subject),
|
||||||
SessionPrincipal::Subject
|
|
||||||
),)
|
|
||||||
.unwrap()
|
|
||||||
);
|
);
|
||||||
assert_eq!(wire.persistence, Some(StartupPersistenceOutcome::NotSaved));
|
assert_eq!(wire.persistence, Some(StartupPersistenceOutcome::NotSaved));
|
||||||
assert!(wire.secondary.iter().any(|failure| matches!(
|
assert!(wire.secondary.iter().any(|failure| matches!(
|
||||||
@@ -486,11 +955,15 @@ async fn real_session_record_denial_and_metadata_save_failure_remain_separate()
|
|||||||
.diagnostics
|
.diagnostics
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|d| d.startup.as_ref())
|
.filter_map(|d| d.startup.as_ref())
|
||||||
.find(|startup| startup.primary == wire.primary)
|
.find(|startup| {
|
||||||
|
session_primary(&startup.primary)
|
||||||
|
== Some(SessionRegistrationCause::denied(SessionPrincipal::Subject))
|
||||||
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_startup_semantics(
|
||||||
host, wire,
|
host,
|
||||||
"Remote Host preserves the refusal and independent save failure"
|
wire,
|
||||||
|
SessionRegistrationCause::denied(SessionPrincipal::Subject),
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
open_subjektiv_store(&fixture.connected.api)
|
open_subjektiv_store(&fixture.connected.api)
|
||||||
@@ -623,7 +1096,7 @@ async fn release_fence_refusal_survives_signed_registration_and_metadata_reopen(
|
|||||||
let wire = &failures[0];
|
let wire = &failures[0];
|
||||||
let expected =
|
let expected =
|
||||||
session_registration_attachment(SessionRegistrationCause::binding_mismatch()).unwrap();
|
session_registration_attachment(SessionRegistrationCause::binding_mismatch()).unwrap();
|
||||||
assert_eq!(wire.primary, expected);
|
assert_session_primary(&wire.primary, SessionRegistrationCause::binding_mismatch());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
wire.persistence,
|
wire.persistence,
|
||||||
Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted)
|
Some(protocol::startup_failure::StartupPersistenceOutcome::Persisted)
|
||||||
@@ -632,9 +1105,11 @@ async fn release_fence_refusal_survives_signed_registration_and_metadata_reopen(
|
|||||||
.diagnostics
|
.diagnostics
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|d| d.startup.as_ref())
|
.filter_map(|d| d.startup.as_ref())
|
||||||
.find(|failure| failure.primary == expected)
|
.find(|failure| {
|
||||||
|
session_primary(&failure.primary) == Some(SessionRegistrationCause::binding_mismatch())
|
||||||
|
})
|
||||||
.expect("Remote Host retains known refusal");
|
.expect("Remote Host retains known refusal");
|
||||||
assert_eq!(host.primary, wire.primary);
|
assert_startup_semantics(host, wire, SessionRegistrationCause::binding_mismatch());
|
||||||
assert!(
|
assert!(
|
||||||
host.secondary.iter().any(|failure| matches!(
|
host.secondary.iter().any(|failure| matches!(
|
||||||
failure,
|
failure,
|
||||||
|
|||||||
@@ -0,0 +1,803 @@
|
|||||||
|
//! Backend result fixtures use real SQLite reservations, coordinator helpers, typed
|
||||||
|
//! reads, public ApiError responses and HTTP middleware observation. Publication
|
||||||
|
//! faults enter the production committed-summary response helper with a fixture
|
||||||
|
//! Runtime Worker; only the earlier Runtime startup is substituted. These cases do
|
||||||
|
//! not claim creation-route authorization or Session callback coverage (see
|
||||||
|
//! session_registration_connected_tests). Runs in the ordinary server lib suite;
|
||||||
|
//! each case owns its tempdir/DB and subscriber, with no external services.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
|
||||||
|
use protocol::startup_failure::{StartupFailureAttachment, StartupPersistenceOutcome};
|
||||||
|
use runtime_api::startup::{
|
||||||
|
CompensationOutcome, CreationAcceptance, DecodeStartupProblem, DecodeStoredWorkerCreateFailure,
|
||||||
|
ExecutionOutcome, FinalizeFacts, FinalizeStage, StartupBoundary, StartupCause,
|
||||||
|
StartupOperationFacts, WorkerCreateFailure, decode_startup_problem,
|
||||||
|
};
|
||||||
|
use std::io::Write;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use tracing::instrument::WithSubscriber;
|
||||||
|
|
||||||
|
const SECRETS: &[&str] = &[
|
||||||
|
"attachment-secret",
|
||||||
|
"context-secret",
|
||||||
|
"diagnostic-secret",
|
||||||
|
"source-secret",
|
||||||
|
"sqlite-secret",
|
||||||
|
"/private/startup-provider",
|
||||||
|
];
|
||||||
|
|
||||||
|
fn failure_fixture(compensation: CompensationOutcome) -> WorkerCreateFailure {
|
||||||
|
let clean = WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject)),
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Accepted,
|
||||||
|
execution: ExecutionOutcome::Errored,
|
||||||
|
finalize: Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::SessionRegistration,
|
||||||
|
compensation,
|
||||||
|
}),
|
||||||
|
backend_compensation: Some(compensation),
|
||||||
|
correlation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (mut startup, operation) = clean.into_parts();
|
||||||
|
// Real untrusted attachment bytes, not a source string which the codec never
|
||||||
|
// sees. Projection must reconstruct the cause and discard received prose/context.
|
||||||
|
let encoded: String =
|
||||||
|
serde_json::from_value(serde_json::to_value(&startup.primary).unwrap()).unwrap();
|
||||||
|
let mut envelope: Value = serde_json::from_str(&encoded).unwrap();
|
||||||
|
envelope["explanation"] = json!("attachment-secret /private/startup-provider");
|
||||||
|
envelope["context"]["operations"] = json!([{
|
||||||
|
"operation": "context-secret", "stage": "context-secret",
|
||||||
|
"operation_ref": "context-secret"
|
||||||
|
}]);
|
||||||
|
envelope["context"]["diagnostics"] = json!([{
|
||||||
|
"code": "diagnostic-secret", "explanation": "diagnostic-secret"
|
||||||
|
}]);
|
||||||
|
startup.primary = serde_json::from_value::<StartupFailureAttachment>(json!(
|
||||||
|
serde_json::to_string(&envelope).unwrap()
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
// Session metadata and Backend result persistence have independent owners.
|
||||||
|
startup.persistence = Some(StartupPersistenceOutcome::NotSaved);
|
||||||
|
let failure = WorkerCreateFailure::new(startup, operation).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
primary_cause(&failure),
|
||||||
|
StartupCause::Session(SessionRegistrationCause::denied(SessionPrincipal::Subject))
|
||||||
|
);
|
||||||
|
failure
|
||||||
|
}
|
||||||
|
|
||||||
|
fn primary_cause(failure: &WorkerCreateFailure) -> StartupCause {
|
||||||
|
match failure.startup().primary.decode::<StartupCause>() {
|
||||||
|
error_contract::DecodeFailure::Known(received) => received.facts().cause().clone(),
|
||||||
|
_ => panic!("fixture/result must retain a known typed primary"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_semantics(actual: &WorkerCreateFailure, expected: &WorkerCreateFailure) {
|
||||||
|
assert_eq!(primary_cause(actual), primary_cause(expected));
|
||||||
|
assert_eq!(actual.public_facts(), expected.public_facts());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reserve(api: &WorkspaceApi) -> RuntimeWorkerRef {
|
||||||
|
reserve_for_runtime(api, EMBEDDED_WORKER_RUNTIME_ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reserve_for_runtime(api: &WorkspaceApi, runtime_id: &str) -> RuntimeWorkerRef {
|
||||||
|
let settings = api
|
||||||
|
.config_store
|
||||||
|
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
|
||||||
|
.unwrap();
|
||||||
|
let reservation = api
|
||||||
|
.config_store
|
||||||
|
.reserve_worker_create(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
runtime_id,
|
||||||
|
"manual:startup-result-fixture",
|
||||||
|
"sha256:startup-result-fixture",
|
||||||
|
None,
|
||||||
|
&settings,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
RuntimeWorkerRef {
|
||||||
|
runtime_id: runtime_id.into(),
|
||||||
|
worker_id: reservation.worker_id.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retained(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> WorkerCreateFailure {
|
||||||
|
match api.worker_create_failure(worker).unwrap() {
|
||||||
|
Some(DecodeStoredWorkerCreateFailure::Known(failure)) => failure,
|
||||||
|
other => panic!("typed public read must return retained result: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot all reservation columns except the observation, plus allocation/lease
|
||||||
|
// rows. Saving a diagnostic must not complete/remove/reallocate the reservation.
|
||||||
|
fn lifecycle_snapshot(api: &WorkspaceApi) -> Vec<Vec<rusqlite::types::Value>> {
|
||||||
|
api.config_store.with_conn(|conn| {
|
||||||
|
let mut columns = conn.prepare("PRAGMA table_info(worker_create_reservations)")?;
|
||||||
|
let names = columns.query_map([], |row| row.get::<_, String>(1))?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
let columns = names.into_iter().filter(|name| name != "startup_failure_json")
|
||||||
|
.map(|name| format!("\"{name}\"" )).collect::<Vec<_>>().join(",");
|
||||||
|
let queries = [
|
||||||
|
format!("SELECT {columns} FROM worker_create_reservations ORDER BY workspace_id,allocation_key"),
|
||||||
|
"SELECT * FROM workspace_resource_keys ORDER BY workspace_id,resource_kind,resource_id".into(),
|
||||||
|
"SELECT * FROM worker_singleton_owners ORDER BY workspace_id,singleton_key".into(),
|
||||||
|
"SELECT * FROM worker_registry ORDER BY workspace_id,runtime_id,worker_id".into(),
|
||||||
|
"SELECT * FROM ticket_worker_assignments ORDER BY workspace_id,ticket_id,assignment_id".into(),
|
||||||
|
"SELECT * FROM ticket_assignment_operations ORDER BY workspace_id,operation_id".into(),
|
||||||
|
];
|
||||||
|
let mut snapshot = Vec::new();
|
||||||
|
for query in queries {
|
||||||
|
let mut statement = conn.prepare(&query)?;
|
||||||
|
let count = statement.column_count();
|
||||||
|
snapshot.extend(statement.query_map([], |row| {
|
||||||
|
(0..count).map(|i| row.get(i)).collect::<rusqlite::Result<Vec<_>>>()
|
||||||
|
})?.collect::<rusqlite::Result<Vec<_>>>()?);
|
||||||
|
}
|
||||||
|
Ok(snapshot)
|
||||||
|
}).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn api_error(failure: WorkerCreateFailure) -> ApiError {
|
||||||
|
ApiError::with_diagnostics(
|
||||||
|
Error::WorkerCreateFailed {
|
||||||
|
failure: Box::new(failure),
|
||||||
|
source: Some(Box::new(Error::InvalidInput(
|
||||||
|
"source-secret /private/startup-provider".into(),
|
||||||
|
))),
|
||||||
|
},
|
||||||
|
Vec::new(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn result_router(failure: WorkerCreateFailure) -> Router {
|
||||||
|
Router::new()
|
||||||
|
.route(
|
||||||
|
"/api/startup-result-fixture",
|
||||||
|
axum::routing::get(move || {
|
||||||
|
let failure = failure.clone();
|
||||||
|
async move { api_error(failure).into_response() }
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.layer(middleware::from_fn(log_failed_api_response))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn public_result(app: Router) -> (StatusCode, Value, WorkerCreateFailure) {
|
||||||
|
let response = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.uri("/api/startup-result-fixture")
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()[CONTENT_TYPE],
|
||||||
|
runtime_api::startup::PROBLEM_CONTENT_TYPE
|
||||||
|
);
|
||||||
|
let body = to_bytes(
|
||||||
|
response.into_body(),
|
||||||
|
runtime_api::startup::MAX_STARTUP_PROBLEM_BYTES,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let decoded = decode_startup_problem(
|
||||||
|
&body,
|
||||||
|
status.as_u16(),
|
||||||
|
runtime_api::startup::PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
);
|
||||||
|
let DecodeStartupProblem::Known(failure) = decoded else {
|
||||||
|
panic!("actual HTTP response must satisfy shared bounded contract: {decoded:?}");
|
||||||
|
};
|
||||||
|
let value = serde_json::from_slice(&body).unwrap();
|
||||||
|
assert_no_secrets(std::str::from_utf8(&body).unwrap());
|
||||||
|
(status, value, failure)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct Capture(Arc<std::sync::Mutex<Vec<u8>>>);
|
||||||
|
|
||||||
|
impl Write for Capture {
|
||||||
|
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||||
|
self.0.lock().unwrap().extend_from_slice(bytes);
|
||||||
|
Ok(bytes.len())
|
||||||
|
}
|
||||||
|
fn flush(&mut self) -> std::io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Capture {
|
||||||
|
fn events(&self) -> Vec<Value> {
|
||||||
|
let bytes = self.0.lock().unwrap().clone();
|
||||||
|
let text = std::str::from_utf8(&bytes).unwrap();
|
||||||
|
assert_no_secrets(text);
|
||||||
|
text.lines()
|
||||||
|
.map(|line| serde_json::from_str(line).unwrap())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn subscriber(capture: Capture) -> impl tracing::Subscriber + Send + Sync {
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(false)
|
||||||
|
.json()
|
||||||
|
.flatten_event(true)
|
||||||
|
.with_writer(move || capture.clone())
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_no_secrets(text: &str) {
|
||||||
|
for secret in SECRETS {
|
||||||
|
assert!(
|
||||||
|
!text.contains(secret),
|
||||||
|
"private value {secret} leaked: {text}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_log(event: &Value, failure: &WorkerCreateFailure, status: StatusCode) {
|
||||||
|
assert_eq!(event["target"], "yoi::worker_startup");
|
||||||
|
assert_eq!(event["event"], "worker_create_result");
|
||||||
|
assert_eq!(event["domain"], "session_registration");
|
||||||
|
assert_eq!(event["reason"], "permission_denied");
|
||||||
|
assert_eq!(
|
||||||
|
event["correlation"],
|
||||||
|
failure.operation().correlation.as_ref().unwrap().as_str()
|
||||||
|
);
|
||||||
|
assert_eq!(event["status"], status.as_u16());
|
||||||
|
assert_eq!(
|
||||||
|
event["acceptance"],
|
||||||
|
format!("{:?}", failure.operation().acceptance)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
event["execution"],
|
||||||
|
format!("{:?}", failure.operation().execution)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
event["finalize"],
|
||||||
|
format!("{:?}", failure.operation().finalize)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
event["backend_compensation"],
|
||||||
|
format!("{:?}", failure.operation().backend_compensation)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
event["backend_result_persistence"],
|
||||||
|
format!("{:?}", failure.backend_result_persistence())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
event["persistence"],
|
||||||
|
format!("{:?}", failure.startup().persistence)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn persisted_result_public_problem_and_single_http_log_have_matching_semantics() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api(dir.path()).await;
|
||||||
|
let worker = reserve(&api);
|
||||||
|
let before = lifecycle_snapshot(&api);
|
||||||
|
let original = failure_fixture(CompensationOutcome::Residual);
|
||||||
|
let returned = worker_startup::persist(&api, &worker, original.clone());
|
||||||
|
assert_eq!(
|
||||||
|
returned.backend_result_persistence(),
|
||||||
|
Some(StartupPersistenceOutcome::Persisted)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
returned.startup().persistence,
|
||||||
|
Some(StartupPersistenceOutcome::NotSaved)
|
||||||
|
);
|
||||||
|
assert_eq!(primary_cause(&returned), primary_cause(&original));
|
||||||
|
assert_eq!(lifecycle_snapshot(&api), before);
|
||||||
|
let saved = retained(&api, &worker);
|
||||||
|
assert_semantics(&saved, &returned);
|
||||||
|
// Storage projection changes envelope bytes; equality must be semantic.
|
||||||
|
assert_ne!(saved.startup().primary, original.startup().primary);
|
||||||
|
let capture = Capture::default();
|
||||||
|
let (status, problem, public) = public_result(result_router(returned.clone()))
|
||||||
|
.with_subscriber(subscriber(capture.clone()))
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::CONFLICT);
|
||||||
|
assert!(
|
||||||
|
problem["detail"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("session.record")
|
||||||
|
);
|
||||||
|
assert!(problem["detail"].as_str().unwrap().contains("Subject"));
|
||||||
|
assert!(
|
||||||
|
problem["instance"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains(returned.operation().correlation.as_ref().unwrap().as_str())
|
||||||
|
);
|
||||||
|
assert_semantics(&public, &saved);
|
||||||
|
let events = capture.events();
|
||||||
|
assert_eq!(
|
||||||
|
events.len(),
|
||||||
|
1,
|
||||||
|
"only HTTP responsibility boundary records primary: {events:?}"
|
||||||
|
);
|
||||||
|
assert_log(&events[0], &public, status);
|
||||||
|
assert_eq!(lifecycle_snapshot(&api), before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn sqlite_result_write_failure_preserves_primary_facts_prior_bytes_and_lifecycle() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api(dir.path()).await;
|
||||||
|
let worker = reserve(&api);
|
||||||
|
let prior = worker_startup::persist(
|
||||||
|
&api,
|
||||||
|
&worker,
|
||||||
|
failure_fixture(CompensationOutcome::Completed),
|
||||||
|
);
|
||||||
|
let prior_bytes = api
|
||||||
|
.config_store
|
||||||
|
.worker_create_failure(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&worker.runtime_id,
|
||||||
|
WorkerId::parse(&worker.worker_id).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let before = lifecycle_snapshot(&api);
|
||||||
|
api.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch(
|
||||||
|
"CREATE TRIGGER fail_startup_result BEFORE UPDATE OF startup_failure_json
|
||||||
|
ON worker_create_reservations BEGIN SELECT RAISE(FAIL, 'sqlite-secret'); END;",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let attempted = failure_fixture(CompensationOutcome::Unknown);
|
||||||
|
let returned = worker_startup::persist(&api, &worker, attempted.clone());
|
||||||
|
assert_eq!(
|
||||||
|
returned.backend_result_persistence(),
|
||||||
|
Some(StartupPersistenceOutcome::OutcomeUnknown)
|
||||||
|
);
|
||||||
|
assert_eq!(primary_cause(&returned), primary_cause(&attempted));
|
||||||
|
let mut expected_operation = attempted.operation().clone();
|
||||||
|
expected_operation.correlation = returned.operation().correlation.clone();
|
||||||
|
assert_eq!(returned.operation(), &expected_operation);
|
||||||
|
assert_eq!(returned.startup(), attempted.startup());
|
||||||
|
assert_eq!(lifecycle_snapshot(&api), before);
|
||||||
|
assert_eq!(
|
||||||
|
api.config_store
|
||||||
|
.worker_create_failure(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&worker.runtime_id,
|
||||||
|
WorkerId::parse(&worker.worker_id).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap(),
|
||||||
|
prior_bytes
|
||||||
|
);
|
||||||
|
assert_semantics(&retained(&api, &worker), &prior);
|
||||||
|
let capture = Capture::default();
|
||||||
|
let (status, _, public) = public_result(result_router(returned.clone()))
|
||||||
|
.with_subscriber(subscriber(capture.clone()))
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::CONFLICT);
|
||||||
|
assert_semantics(&public, &returned);
|
||||||
|
let events = capture.events();
|
||||||
|
assert_eq!(events.len(), 1);
|
||||||
|
assert_log(&events[0], &returned, status);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn finalize_compensation_does_not_replace_received_primary_or_acceptance() {
|
||||||
|
for compensation in [
|
||||||
|
CompensationOutcome::Completed,
|
||||||
|
CompensationOutcome::Residual,
|
||||||
|
CompensationOutcome::Unknown,
|
||||||
|
] {
|
||||||
|
let original = failure_fixture(CompensationOutcome::Residual);
|
||||||
|
let error = worker_startup::finalize_error(
|
||||||
|
api_error(original.clone()),
|
||||||
|
WorkerSpawnFinalizeStage::WorkdirAttachment,
|
||||||
|
compensation,
|
||||||
|
);
|
||||||
|
let Error::WorkerCreateFailed { failure, .. } = &error.error else {
|
||||||
|
panic!("finalize must keep a typed startup result");
|
||||||
|
};
|
||||||
|
assert_eq!(primary_cause(failure), primary_cause(&original));
|
||||||
|
assert_eq!(failure.startup(), original.startup());
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().acceptance,
|
||||||
|
original.operation().acceptance
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().execution,
|
||||||
|
original.operation().execution
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize,
|
||||||
|
Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::WorkdirAttachment,
|
||||||
|
compensation
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(failure.operation().backend_compensation, Some(compensation));
|
||||||
|
let expected = failure.as_ref().clone();
|
||||||
|
let (_, _, public) = public_result(result_router(expected.clone())).await;
|
||||||
|
assert_semantics(&public, &expected);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn context_and_public_projection_are_silent_until_explicit_http_observation() {
|
||||||
|
let capture = Capture::default();
|
||||||
|
let expected = async {
|
||||||
|
let worker = RuntimeWorkerRef {
|
||||||
|
runtime_id: "context-secret".into(),
|
||||||
|
worker_id: "source-secret".into(),
|
||||||
|
};
|
||||||
|
let failure =
|
||||||
|
worker_startup::correlate(failure_fixture(CompensationOutcome::Unknown), &worker);
|
||||||
|
let public = api_error(failure.clone()).into_response();
|
||||||
|
assert_eq!(public.status(), StatusCode::CONFLICT);
|
||||||
|
assert!(
|
||||||
|
capture.events().is_empty(),
|
||||||
|
"context/encoder must not log the primary"
|
||||||
|
);
|
||||||
|
worker_startup::observe_http(&failure, public.status());
|
||||||
|
failure
|
||||||
|
}
|
||||||
|
.with_subscriber(subscriber(capture.clone()))
|
||||||
|
.await;
|
||||||
|
let events = capture.events();
|
||||||
|
assert_eq!(events.len(), 1);
|
||||||
|
assert_log(&events[0], &expected, StatusCode::CONFLICT);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn assert_publication_fault_retains_committed_worker(query_failure: bool) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api(dir.path()).await;
|
||||||
|
// Freeze unrelated observation writes before deliberately damaging the DB.
|
||||||
|
api.worker_projection.shutdown_async().await;
|
||||||
|
api.store.db_executor().drain().await.unwrap();
|
||||||
|
register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await;
|
||||||
|
let stops = Arc::new(AtomicUsize::new(0));
|
||||||
|
let stop_count = stops.clone();
|
||||||
|
let runtime = WorkdirlessFixtureRuntime {
|
||||||
|
stop_callback: Some(Arc::new(move || {
|
||||||
|
stop_count.fetch_add(1, Ordering::SeqCst);
|
||||||
|
})),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
api.runtime.register_or_replace(runtime.clone());
|
||||||
|
let worker = reserve_for_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID);
|
||||||
|
let spawned = crate::hosts::WorkspaceWorkerRuntime::spawn_worker(
|
||||||
|
&runtime,
|
||||||
|
WorkerCreateBinding {
|
||||||
|
plugin_execution: None,
|
||||||
|
worker_id: WorkerId::parse(&worker.worker_id).unwrap(),
|
||||||
|
create_fingerprint: "sha256:startup-result-fixture".into(),
|
||||||
|
},
|
||||||
|
WorkerSpawnRequest {
|
||||||
|
intent: WorkerSpawnIntent::WorkspaceCompanion,
|
||||||
|
requested_worker_name: Some("Publication fixture".into()),
|
||||||
|
singleton_key: None,
|
||||||
|
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
|
||||||
|
expected_segments: 0,
|
||||||
|
},
|
||||||
|
profile: ProfileSelector::Builtin("builtin:companion".into()),
|
||||||
|
ticket_assignment: None,
|
||||||
|
initial_submit: Vec::new(),
|
||||||
|
workdir_attachment_requests: Vec::new(),
|
||||||
|
resolved_workdir_attachment_requests: Vec::new(),
|
||||||
|
resolved_workdir_attachments: Vec::new(),
|
||||||
|
resolved_config_bundle: None,
|
||||||
|
resolved_worker_observation_enabled: false,
|
||||||
|
resolved_worker_observation_grants: Vec::new(),
|
||||||
|
resolved_workspace_api: None,
|
||||||
|
resolved_memory_settings: None,
|
||||||
|
resolved_subjektiv_attached: false,
|
||||||
|
resolved_control_operation: None,
|
||||||
|
resolved_execution_grant: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert_eq!(spawned.state, InternalWorkerOperationState::Accepted);
|
||||||
|
let summary = spawned.worker.unwrap();
|
||||||
|
assert_eq!(summary.worker, worker);
|
||||||
|
let record = record_worker_summary(
|
||||||
|
&api,
|
||||||
|
&summary,
|
||||||
|
&summary.display_name,
|
||||||
|
summary.profile.clone(),
|
||||||
|
WorkerRegistryDisplayNamePolicy::UseProvided,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
api.config_store
|
||||||
|
.complete_worker_create_reservation(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
WorkerId::parse(&worker.worker_id).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let ticket = browser_ticket_backend(&api)
|
||||||
|
.unwrap()
|
||||||
|
.create(ticket::NewTicket::new("Publication retains assignment"))
|
||||||
|
.unwrap();
|
||||||
|
let assignment = TicketWorkerAssignmentRecord {
|
||||||
|
workspace_id: TEST_WORKSPACE_ID.into(),
|
||||||
|
ticket_id: ticket.id.clone(),
|
||||||
|
assignment_id: "publication-assignment".into(),
|
||||||
|
worker: worker.clone(),
|
||||||
|
assigned_by: "test-user".into(),
|
||||||
|
assigned_at: TEST_CREATED_AT.into(),
|
||||||
|
};
|
||||||
|
api.store
|
||||||
|
.set_current_ticket_worker_assignment(
|
||||||
|
&assignment,
|
||||||
|
None,
|
||||||
|
"publication-assignment-event",
|
||||||
|
"publication-assignment-operation",
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
// Establish a working publication read first; the fault changes only that
|
||||||
|
// final read, not reservation persistence or an earlier caller-identity check.
|
||||||
|
let key = api
|
||||||
|
.store
|
||||||
|
.worker_resource_key(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
api.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
if query_failure {
|
||||||
|
conn.execute_batch(
|
||||||
|
"ALTER TABLE workspace_resource_keys RENAME COLUMN resource_key TO \"sqlite-secret\";",
|
||||||
|
)?;
|
||||||
|
} else {
|
||||||
|
let deleted = conn.execute(
|
||||||
|
"DELETE FROM workspace_resource_keys WHERE workspace_id=?1 AND resource_kind='worker' AND resource_key=?2",
|
||||||
|
rusqlite::params![TEST_WORKSPACE_ID, key],
|
||||||
|
)?;
|
||||||
|
assert_eq!(deleted, 1);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let before = lifecycle_snapshot(&api);
|
||||||
|
let capture = Capture::default();
|
||||||
|
let route_api = api.clone();
|
||||||
|
let app = Router::new()
|
||||||
|
.route(
|
||||||
|
"/api/startup-result-fixture",
|
||||||
|
axum::routing::get(move || {
|
||||||
|
let api = route_api.clone();
|
||||||
|
let summary = summary.clone();
|
||||||
|
async move {
|
||||||
|
let error = browser_worker_response_for_committed_summary(
|
||||||
|
&api,
|
||||||
|
summary,
|
||||||
|
"Publication fixture".into(),
|
||||||
|
Vec::new(),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
let Error::WorkerCreateFailed { source, .. } = &error.error else {
|
||||||
|
panic!("final publication read must return a typed result: {error:?}");
|
||||||
|
};
|
||||||
|
let source = source
|
||||||
|
.as_deref()
|
||||||
|
.expect("keep the actual read error internal");
|
||||||
|
if query_failure {
|
||||||
|
assert!(matches!(source, Error::Sqlite(_)), "{source:?}");
|
||||||
|
} else {
|
||||||
|
assert!(matches!(source, Error::Store(_)), "{source:?}");
|
||||||
|
assert!(source.to_string().contains("no resource key"));
|
||||||
|
}
|
||||||
|
error.into_response()
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.layer(middleware::from_fn(log_failed_api_response));
|
||||||
|
let (status, problem, public) = public_result(app)
|
||||||
|
.with_subscriber(subscriber(capture.clone()))
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
|
||||||
|
assert_eq!(
|
||||||
|
primary_cause(&public),
|
||||||
|
StartupCause::Boundary(runtime_api::startup::StartupBoundaryCause {
|
||||||
|
reason: runtime_api::startup::StartupBoundaryReason::FinalizeFailed,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(public.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(public.operation().execution, ExecutionOutcome::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
public.operation().finalize,
|
||||||
|
Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::Publication,
|
||||||
|
compensation: CompensationOutcome::Residual,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(public.operation().backend_compensation, None);
|
||||||
|
assert_eq!(
|
||||||
|
public.startup().resources,
|
||||||
|
Some(protocol::startup_failure::StartupResourceFacts {
|
||||||
|
resources: protocol::startup_failure::StartupResources::WorkerStartup,
|
||||||
|
cleanup: protocol::startup_failure::StartupCleanup::NotNeeded,
|
||||||
|
durable_effects_possible: true,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
public.backend_result_persistence(),
|
||||||
|
Some(StartupPersistenceOutcome::Persisted)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
public.operation().correlation.as_ref().unwrap().as_str(),
|
||||||
|
worker_startup::correlation(&worker.runtime_id, &worker.worker_id)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
problem["instance"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.ends_with(public.operation().correlation.as_ref().unwrap().as_str())
|
||||||
|
);
|
||||||
|
assert_semantics(&retained(&api, &worker), &public);
|
||||||
|
let events = capture.events();
|
||||||
|
assert_eq!(events.len(), 1, "one primary HTTP event: {events:?}");
|
||||||
|
let event = &events[0];
|
||||||
|
assert_eq!(event["target"], "yoi::worker_startup");
|
||||||
|
assert_eq!(event["event"], "worker_create_result");
|
||||||
|
assert_eq!(event["domain"], "startup_boundary");
|
||||||
|
assert_eq!(event["reason"], "finalize_failed");
|
||||||
|
assert_eq!(event["acceptance"], "Accepted");
|
||||||
|
assert_eq!(event["execution"], "Accepted");
|
||||||
|
assert_eq!(
|
||||||
|
event["finalize"],
|
||||||
|
format!("{:?}", public.operation().finalize)
|
||||||
|
);
|
||||||
|
assert_eq!(event["backend_compensation"], "None");
|
||||||
|
assert_eq!(event["backend_result_persistence"], "Some(Persisted)");
|
||||||
|
assert_eq!(
|
||||||
|
event["resources"],
|
||||||
|
format!("{:?}", public.startup().resources)
|
||||||
|
);
|
||||||
|
assert_eq!(event["status"], status.as_u16());
|
||||||
|
assert_eq!(
|
||||||
|
event["correlation"],
|
||||||
|
problem["yoi_startup"]["operation"]["correlation"]
|
||||||
|
);
|
||||||
|
let exposed = format!("{problem} {events:?}");
|
||||||
|
for internal in [
|
||||||
|
worker.worker_id.as_str(),
|
||||||
|
"no such column",
|
||||||
|
"no resource key",
|
||||||
|
"workspace_resource_keys",
|
||||||
|
dir.path().to_str().unwrap(),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!exposed.contains(internal),
|
||||||
|
"internal source leaked: {exposed}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(lifecycle_snapshot(&api), before);
|
||||||
|
assert_eq!(
|
||||||
|
api.store
|
||||||
|
.get_worker_registry(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap(),
|
||||||
|
Some(record)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
api.store
|
||||||
|
.get_current_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id)
|
||||||
|
.unwrap(),
|
||||||
|
Some(assignment)
|
||||||
|
);
|
||||||
|
assert_eq!(stops.load(Ordering::SeqCst), 0);
|
||||||
|
assert_eq!(runtime.spawn_requests().len(), 1, "no reexecution");
|
||||||
|
assert_eq!(
|
||||||
|
runtime.workers.lock().unwrap().len(),
|
||||||
|
1,
|
||||||
|
"no Runtime deletion"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn publication_sqlite_read_failure_retains_committed_worker_and_typed_result() {
|
||||||
|
assert_publication_fault_retains_committed_worker(true).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn publication_missing_resource_key_retains_committed_worker_and_typed_result() {
|
||||||
|
assert_publication_fault_retains_committed_worker(false).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn publication_error_preserves_existing_primary_without_claiming_compensation() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api(dir.path()).await;
|
||||||
|
let worker = reserve(&api);
|
||||||
|
let original = failure_fixture(CompensationOutcome::Completed);
|
||||||
|
let error = worker_startup::publication_error(&api, &worker, api_error(original.clone()));
|
||||||
|
let Error::WorkerCreateFailed { failure, source } = &error.error else {
|
||||||
|
panic!("publication must retain a typed startup result");
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
source.as_deref(),
|
||||||
|
Some(Error::WorkerCreateFailed { .. })
|
||||||
|
));
|
||||||
|
assert_eq!(failure.startup().primary, original.startup().primary);
|
||||||
|
assert_eq!(failure.startup().secondary, original.startup().secondary);
|
||||||
|
assert_eq!(
|
||||||
|
failure.startup().persistence,
|
||||||
|
original.startup().persistence
|
||||||
|
);
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Accepted);
|
||||||
|
assert_eq!(failure.operation().backend_compensation, None);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize.as_ref().unwrap().stage,
|
||||||
|
FinalizeStage::Publication
|
||||||
|
);
|
||||||
|
assert_semantics(&retained(&api, &worker), failure);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct FailingWriter(Arc<AtomicUsize>);
|
||||||
|
|
||||||
|
impl Write for FailingWriter {
|
||||||
|
fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
|
||||||
|
self.0.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err(std::io::Error::other("sink unavailable"))
|
||||||
|
}
|
||||||
|
fn flush(&mut self) -> std::io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_log_sink_still_returns_original_http_problem_and_retained_result() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let api = test_api(dir.path()).await;
|
||||||
|
let worker = reserve(&api);
|
||||||
|
let saved = worker_startup::persist(
|
||||||
|
&api,
|
||||||
|
&worker,
|
||||||
|
failure_fixture(CompensationOutcome::Residual),
|
||||||
|
);
|
||||||
|
let before = lifecycle_snapshot(&api);
|
||||||
|
let capture = Capture::default();
|
||||||
|
let baseline = public_result(result_router(saved.clone()))
|
||||||
|
.with_subscriber(subscriber(capture.clone()))
|
||||||
|
.await;
|
||||||
|
assert_eq!(capture.events().len(), 1);
|
||||||
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let writer = FailingWriter(attempts.clone());
|
||||||
|
let failed_subscriber = tracing_subscriber::fmt()
|
||||||
|
.without_time()
|
||||||
|
.with_ansi(false)
|
||||||
|
.json()
|
||||||
|
.flatten_event(true)
|
||||||
|
.with_writer(move || writer.clone())
|
||||||
|
.finish();
|
||||||
|
let actual = public_result(result_router(saved.clone()))
|
||||||
|
.with_subscriber(failed_subscriber)
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
attempts.load(Ordering::SeqCst) > 0,
|
||||||
|
"must exercise actual observation sink failure"
|
||||||
|
);
|
||||||
|
assert_eq!(actual.0, baseline.0);
|
||||||
|
assert_eq!(
|
||||||
|
actual.1, baseline.1,
|
||||||
|
"sink failure must not replace the outgoing Problem"
|
||||||
|
);
|
||||||
|
assert_semantics(&actual.2, &saved);
|
||||||
|
assert_semantics(&retained(&api, &worker), &saved);
|
||||||
|
assert_eq!(lifecycle_snapshot(&api), before);
|
||||||
|
}
|
||||||
@@ -2745,24 +2745,53 @@ pub(super) struct ConnectedWorkerHarness {
|
|||||||
_dir: tempfile::TempDir,
|
_dir: tempfile::TempDir,
|
||||||
}
|
}
|
||||||
impl ConnectedWorkerHarness {
|
impl ConnectedWorkerHarness {
|
||||||
pub(super) async fn human(
|
/// Exercise the authenticated Browser TCP/HTTP boundary without discarding
|
||||||
|
/// media type or rewriting its body. Bound both elapsed time and streamed bytes;
|
||||||
|
/// redirects/retries must not turn a failed create response into another POST.
|
||||||
|
pub(super) async fn human_raw(
|
||||||
&self,
|
&self,
|
||||||
method: reqwest::Method,
|
method: reqwest::Method,
|
||||||
path: &str,
|
path: &str,
|
||||||
body: Value,
|
body: Value,
|
||||||
) -> (StatusCode, Value) {
|
) -> (StatusCode, HeaderMap, Vec<u8>) {
|
||||||
let response = reqwest::Client::builder()
|
const RESPONSE_LIMIT: usize = 2 * 1024 * 1024;
|
||||||
|
let mut response = reqwest::Client::builder()
|
||||||
.timeout(std::time::Duration::from_secs(30))
|
.timeout(std::time::Duration::from_secs(30))
|
||||||
|
.redirect(reqwest::redirect::Policy::none())
|
||||||
|
.retry(reqwest::retry::never())
|
||||||
.build()
|
.build()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.request(method.clone(), format!("{}{path}", self.backend_url))
|
.request(method, format!("{}{path}", self.backend_url))
|
||||||
.bearer_auth(&self.owner)
|
.bearer_auth(&self.owner)
|
||||||
.json(&body)
|
.json(&body)
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let status = StatusCode::from_u16(response.status().as_u16()).unwrap();
|
let status = StatusCode::from_u16(response.status().as_u16()).unwrap();
|
||||||
let bytes = response.bytes().await.unwrap();
|
let headers = response.headers().clone();
|
||||||
|
assert!(
|
||||||
|
response
|
||||||
|
.content_length()
|
||||||
|
.is_none_or(|len| len <= RESPONSE_LIMIT as u64)
|
||||||
|
);
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
while let Some(chunk) = response.chunk().await.unwrap() {
|
||||||
|
assert!(
|
||||||
|
chunk.len() <= RESPONSE_LIMIT - bytes.len(),
|
||||||
|
"Browser response exceeds test limit"
|
||||||
|
);
|
||||||
|
bytes.extend_from_slice(&chunk);
|
||||||
|
}
|
||||||
|
(status, headers, bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn human(
|
||||||
|
&self,
|
||||||
|
method: reqwest::Method,
|
||||||
|
path: &str,
|
||||||
|
body: Value,
|
||||||
|
) -> (StatusCode, Value) {
|
||||||
|
let (status, _, bytes) = self.human_raw(method.clone(), path, body).await;
|
||||||
let body = if status == StatusCode::NO_CONTENT {
|
let body = if status == StatusCode::NO_CONTENT {
|
||||||
assert!(bytes.is_empty(), "{method} {path}: HTTP204 must be empty");
|
assert!(bytes.is_empty(), "{method} {path}: HTTP204 must be empty");
|
||||||
Value::Null
|
Value::Null
|
||||||
@@ -2970,20 +2999,36 @@ impl ConnectedWorkerHarness {
|
|||||||
}
|
}
|
||||||
let (parts, body) = response.into_parts();
|
let (parts, body) = response.into_parts();
|
||||||
let bytes = to_bytes(body, 2 * 1024 * 1024).await.unwrap();
|
let bytes = to_bytes(body, 2 * 1024 * 1024).await.unwrap();
|
||||||
|
let content_type = parts
|
||||||
|
.headers
|
||||||
|
.get(axum::http::header::CONTENT_TYPE)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.unwrap_or("");
|
||||||
let body: Value = serde_json::from_slice(&bytes).unwrap();
|
let body: Value = serde_json::from_slice(&bytes).unwrap();
|
||||||
if let Some(startup) = body["error"].get("startup").filter(|v| !v.is_null())
|
let message = if content_type == runtime_api::startup::PROBLEM_CONTENT_TYPE
|
||||||
{
|
{
|
||||||
startup_failures.lock().unwrap().push(
|
use runtime_api::startup::{
|
||||||
serde_json::from_value(startup.clone())
|
DecodeStartupProblem, StartupBoundary, decode_startup_problem,
|
||||||
.expect("RuntimeHTTP structured startup failure"),
|
};
|
||||||
);
|
let DecodeStartupProblem::Known(failure) = decode_startup_problem(
|
||||||
}
|
&bytes,
|
||||||
errors.lock().unwrap().push(
|
parts.status.as_u16(),
|
||||||
|
content_type,
|
||||||
|
StartupBoundary::Runtime,
|
||||||
|
) else {
|
||||||
|
panic!("RuntimeHTTP must emit a valid typed startup Problem");
|
||||||
|
};
|
||||||
|
startup_failures
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(failure.startup().clone());
|
||||||
|
body["detail"].as_str().expect("Problem detail")
|
||||||
|
} else {
|
||||||
body["error"]["message"]
|
body["error"]["message"]
|
||||||
.as_str()
|
.as_str()
|
||||||
.unwrap_or("unknown fixture error")
|
.unwrap_or("unknown fixture error")
|
||||||
.to_owned(),
|
};
|
||||||
);
|
errors.lock().unwrap().push(message.to_owned());
|
||||||
Response::from_parts(parts, Body::from(bytes))
|
Response::from_parts(parts, Body::from(bytes))
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ use worker_runtime::execution::{
|
|||||||
// published Profile projection; leave Worker creation/control to the real Runtime.
|
// published Profile projection; leave Worker creation/control to the real Runtime.
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct LaunchExecutionBackend {
|
struct LaunchExecutionBackend {
|
||||||
|
lose_initial_ack: std::sync::atomic::AtomicBool,
|
||||||
|
fail_stop: std::sync::atomic::AtomicBool,
|
||||||
execution: DeterministicExecutionBackend,
|
execution: DeterministicExecutionBackend,
|
||||||
provider_close_gate: Mutex<Option<Arc<super::workdir_session_close_tests::ProviderCloseGate>>>,
|
provider_close_gate: Mutex<Option<Arc<super::workdir_session_close_tests::ProviderCloseGate>>>,
|
||||||
bundles: Mutex<std::collections::HashMap<String, worker_runtime::config_bundle::ConfigBundle>>,
|
bundles: Mutex<std::collections::HashMap<String, worker_runtime::config_bundle::ConfigBundle>>,
|
||||||
@@ -196,10 +198,45 @@ impl WorkerExecutionBackend for LaunchExecutionBackend {
|
|||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.push((worker.clone(), input.clone()));
|
.push((worker.clone(), input.clone()));
|
||||||
self.execution.dispatch_input(worker, input)
|
let mut result = self.execution.dispatch_input(worker, input);
|
||||||
|
if self
|
||||||
|
.lose_initial_ack
|
||||||
|
.swap(false, std::sync::atomic::Ordering::SeqCst)
|
||||||
|
{
|
||||||
|
// Lose only the response acknowledgement, after the real fixture
|
||||||
|
// execution committed its durable receipt. Never redispatch it.
|
||||||
|
result.submission = None;
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
fn termination_receipt(
|
||||||
|
&self,
|
||||||
|
worker: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> std::result::Result<Option<worker_runtime::execution::WorkerExecutionTermination>, String>
|
||||||
|
{
|
||||||
|
self.execution.termination_receipt(worker)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stop_worker_operation(
|
||||||
|
&self,
|
||||||
|
request: worker_runtime::execution::WorkerExecutionStopRequest,
|
||||||
|
) -> WorkerExecutionResult {
|
||||||
|
if self.fail_stop.load(std::sync::atomic::Ordering::SeqCst) {
|
||||||
|
return self.stop_worker(&request.worker_ref);
|
||||||
|
}
|
||||||
|
// Preserve the fixture's actual joined execution and operation-bound
|
||||||
|
// termination receipt; an accepted legacy stop alone is not that evidence.
|
||||||
|
self.execution.stop_worker_operation(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stop_worker(&self, worker: &worker_runtime::identity::WorkerRef) -> WorkerExecutionResult {
|
fn stop_worker(&self, worker: &worker_runtime::identity::WorkerRef) -> WorkerExecutionResult {
|
||||||
|
if self.fail_stop.load(std::sync::atomic::Ordering::SeqCst) {
|
||||||
|
return WorkerExecutionResult::errored(
|
||||||
|
worker_runtime::execution::WorkerExecutionOperation::Stop,
|
||||||
|
"TEST_ONLY_PRIVATE_STOP_CREDENTIAL /private/path",
|
||||||
|
);
|
||||||
|
}
|
||||||
self.execution.stop_worker(worker)
|
self.execution.stop_worker(worker)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1133,6 +1170,90 @@ async fn read_only_ticket_targets_allow_extra_external_grants_without_widening_c
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Production create owner and Backend adapters share the same executor; only
|
||||||
|
/// its response acknowledgement and cleanup boundary are fault-injected.
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn initial_input_ack_loss_preserves_creation_facts_across_embedded_and_remote() {
|
||||||
|
use runtime_api::startup::*;
|
||||||
|
for failed_cleanup in [false, true] {
|
||||||
|
let mut observed = Vec::new();
|
||||||
|
for remote in [false, true] {
|
||||||
|
let fixture = LaunchFixture::new(remote).await;
|
||||||
|
let ticket = fixture.ticket();
|
||||||
|
let payload = fixture.payload(&ticket, "lost-initial-ack", "builtin:companion", false);
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.lose_initial_ack
|
||||||
|
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.fail_stop
|
||||||
|
.store(failed_cleanup, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
let (status, body) = fixture.post(TEST_WORKSPACE_ID, payload).await;
|
||||||
|
assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}");
|
||||||
|
let DecodeStartupProblem::Known(failure) = decode_startup_problem(
|
||||||
|
&serde_json::to_vec(&body).unwrap(),
|
||||||
|
status.as_u16(),
|
||||||
|
PROBLEM_CONTENT_TYPE,
|
||||||
|
StartupBoundary::Browser,
|
||||||
|
) else {
|
||||||
|
panic!("must retain typed creation facts: {body}");
|
||||||
|
};
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(failure.operation().execution, ExecutionOutcome::Unknown);
|
||||||
|
let finalize = failure.operation().finalize.unwrap();
|
||||||
|
assert_eq!(finalize.stage, FinalizeStage::InitialInput);
|
||||||
|
assert_eq!(
|
||||||
|
finalize.compensation,
|
||||||
|
if failed_cleanup {
|
||||||
|
CompensationOutcome::Unknown
|
||||||
|
} else {
|
||||||
|
CompensationOutcome::Completed
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(body["yoi_failure"]["reason"], "invalid_response");
|
||||||
|
assert!(
|
||||||
|
!body
|
||||||
|
.to_string()
|
||||||
|
.contains("TEST_ONLY_PRIVATE_STOP_CREDENTIAL")
|
||||||
|
);
|
||||||
|
let creates = fixture.backend.creates.lock().unwrap().clone();
|
||||||
|
assert_eq!(creates.len(), 1);
|
||||||
|
let worker_id = creates[0].worker_id.to_string();
|
||||||
|
let worker = RuntimeWorkerRef::new(&fixture.runtime_id, &worker_id);
|
||||||
|
let Some(DecodeStoredWorkerCreateFailure::Known(saved)) =
|
||||||
|
fixture.api.worker_create_failure(&worker).unwrap()
|
||||||
|
else {
|
||||||
|
panic!("retain queryable create result");
|
||||||
|
};
|
||||||
|
assert_eq!(saved.public_facts(), failure.public_facts());
|
||||||
|
assert_eq!(
|
||||||
|
fixture.backend.submits_for(&worker_id).len(),
|
||||||
|
1,
|
||||||
|
"observation failure never redispatches"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.execution
|
||||||
|
.contexts
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.keys()
|
||||||
|
.any(|worker| worker.worker_id.to_string() == worker_id),
|
||||||
|
failed_cleanup
|
||||||
|
);
|
||||||
|
let mut facts = failure.public_facts();
|
||||||
|
facts.operation.correlation = None;
|
||||||
|
observed.push((status, body["yoi_failure"]["reason"].clone(), facts));
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
observed[0], observed[1],
|
||||||
|
"same owner facts through Embedded and Remote"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
async fn worker_assignment_finalize_failure_compensates_real_embedded_runtime_and_preserves_open_case()
|
async fn worker_assignment_finalize_failure_compensates_real_embedded_runtime_and_preserves_open_case()
|
||||||
{
|
{
|
||||||
@@ -1151,10 +1272,19 @@ async fn worker_assignment_finalize_failure_compensates_real_embedded_runtime_an
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
let payload = fixture.payload(&ticket_id, "compensated-worker", "builtin:companion", false);
|
let payload = fixture.payload(&ticket_id, "compensated-worker", "builtin:companion", false);
|
||||||
let (status, body) = fixture.post(TEST_WORKSPACE_ID, payload).await;
|
let (status, body) = fixture.post(TEST_WORKSPACE_ID, payload).await;
|
||||||
assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}");
|
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}");
|
||||||
|
assert_eq!(body["yoi_failure"]["domain"], "startup_boundary");
|
||||||
|
assert_eq!(body["yoi_failure"]["reason"], "finalize_failed");
|
||||||
|
let operation = &body["yoi_startup"]["operation"];
|
||||||
|
assert_eq!(operation["acceptance"], "accepted");
|
||||||
|
assert_eq!(operation["execution"], "accepted");
|
||||||
|
assert_eq!(operation["finalize"]["stage"], "ticket_assignment_bind");
|
||||||
|
assert_eq!(operation["finalize"]["compensation"], "completed");
|
||||||
|
assert!(body["detail"].as_str().unwrap().contains("finalization"));
|
||||||
assert!(
|
assert!(
|
||||||
body["message"].as_str().unwrap().contains("compensat"),
|
!body
|
||||||
"{body}"
|
.to_string()
|
||||||
|
.contains("injected Worker assignment failure")
|
||||||
);
|
);
|
||||||
let creates = fixture
|
let creates = fixture
|
||||||
.backend
|
.backend
|
||||||
@@ -1311,6 +1441,7 @@ impl crate::hosts::WorkspaceWorkerRuntime for LostSpawnResponseRuntime {
|
|||||||
assert_eq!(result.state, InternalWorkerOperationState::Accepted);
|
assert_eq!(result.state, InternalWorkerOperationState::Accepted);
|
||||||
assert!(result.worker.is_some());
|
assert!(result.worker.is_some());
|
||||||
WorkerSpawnResult {
|
WorkerSpawnResult {
|
||||||
|
failure: None,
|
||||||
state: InternalWorkerOperationState::Rejected,
|
state: InternalWorkerOperationState::Rejected,
|
||||||
worker: None,
|
worker: None,
|
||||||
acceptance_evidence: Vec::new(),
|
acceptance_evidence: Vec::new(),
|
||||||
@@ -1829,3 +1960,81 @@ async fn mr_open_retains_source_before_first_review_and_observed_result() {
|
|||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Real Runtime creation and SQLite finalization fail independently; only Stop is
|
||||||
|
// injected. Reconciliation must retain the create reservation while absence is unknown.
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn accepted_creation_finalize_and_failed_stop_retain_unknown_compensation_and_reservation() {
|
||||||
|
let fixture = LaunchFixture::new(false).await;
|
||||||
|
let ticket_id = fixture.ticket();
|
||||||
|
rusqlite::Connection::open(&fixture.api.config.database_path).unwrap().execute_batch(
|
||||||
|
"CREATE TRIGGER fail_startup_finalize BEFORE INSERT ON ticket_current_worker_assignments WHEN NEW.role='worker' BEGIN SELECT RAISE(ABORT, 'TEST_ONLY_PRIVATE_DB_CREDENTIAL'); END;"
|
||||||
|
).unwrap();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.fail_stop
|
||||||
|
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
let (status, body) = fixture
|
||||||
|
.post(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
fixture.payload(
|
||||||
|
&ticket_id,
|
||||||
|
"unknown-compensation",
|
||||||
|
"builtin:companion",
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.fail_stop
|
||||||
|
.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}");
|
||||||
|
assert_eq!(body["yoi_failure"]["reason"], "finalize_failed");
|
||||||
|
let operation = &body["yoi_startup"]["operation"];
|
||||||
|
assert_eq!(operation["acceptance"], "accepted");
|
||||||
|
assert_eq!(operation["execution"], "accepted");
|
||||||
|
assert_eq!(operation["finalize"]["compensation"], "unknown");
|
||||||
|
assert!(
|
||||||
|
operation["correlation"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.starts_with("worker-")
|
||||||
|
);
|
||||||
|
for secret in [
|
||||||
|
"TEST_ONLY_PRIVATE_STOP_CREDENTIAL",
|
||||||
|
"/private/path",
|
||||||
|
"TEST_ONLY_PRIVATE_DB_CREDENTIAL",
|
||||||
|
] {
|
||||||
|
assert!(!body.to_string().contains(secret), "{body}");
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.api
|
||||||
|
.authority
|
||||||
|
.ticket(&ticket_id)
|
||||||
|
.unwrap()
|
||||||
|
.state
|
||||||
|
.as_str(),
|
||||||
|
"open"
|
||||||
|
);
|
||||||
|
let created = fixture.backend.creates.lock().unwrap();
|
||||||
|
let created: Vec<_> = created
|
||||||
|
.iter()
|
||||||
|
.filter(|request| request.display_name.as_deref() == Some("Explicit Ticket Worker"))
|
||||||
|
.collect();
|
||||||
|
assert_eq!(created.len(), 1);
|
||||||
|
let worker_id = created[0].worker_id.to_string();
|
||||||
|
let reserved: i64 = fixture.api.config_store.with_conn(|conn| conn.query_row(
|
||||||
|
"SELECT COUNT(*) FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='reserved'",
|
||||||
|
rusqlite::params![TEST_WORKSPACE_ID, fixture.runtime_id, worker_id], |row| row.get(0),
|
||||||
|
).map_err(Error::from)).unwrap();
|
||||||
|
// A reservation may already be completed by Runtime admission, but it must
|
||||||
|
// remain present (never released as absent) until Runtime absence is confirmed.
|
||||||
|
let retained: i64 = fixture.api.config_store.with_conn(|conn| conn.query_row(
|
||||||
|
"SELECT COUNT(*) FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
|
||||||
|
rusqlite::params![TEST_WORKSPACE_ID, fixture.runtime_id, worker_id], |row| row.get(0),
|
||||||
|
).map_err(Error::from)).unwrap();
|
||||||
|
assert_eq!(retained, 1);
|
||||||
|
assert!(reserved <= retained);
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,286 @@
|
|||||||
|
//! Backend creation boundary. Cause and independent owner observations are retained
|
||||||
|
//! before HTTP projection. Correlation fingerprints are lookup aids, not operation
|
||||||
|
//! identities, idempotency keys, execution grants, or permission to retry.
|
||||||
|
//!
|
||||||
|
//! # Result investigation
|
||||||
|
//!
|
||||||
|
//! Start with the Browser Problem's `yoi_startup.operation.correlation` (also its
|
||||||
|
//! `instance` suffix). For a known Runtime/Worker identity, `correlation` derives
|
||||||
|
//! the same path-free token. Find the single `yoi::worker_startup` event named
|
||||||
|
//! `worker_create_result`; do not use the token as a request or operation key.
|
||||||
|
//! Compare its typed domain/reason with `WorkspaceApi::worker_create_failure`'s
|
||||||
|
//! retained reservation evidence. Inspect acceptance/execution first, then finalize,
|
||||||
|
//! Backend compensation and residual resources independently. Unknown/corrupt or
|
||||||
|
//! absent history remains unknown/absent, not evidence of rejection or cleanup.
|
||||||
|
//!
|
||||||
|
//! The existing Session metadata retains the Runtime startup primary and its own
|
||||||
|
//! `persistence`; `backend_result_persistence` reports only this coordinator result
|
||||||
|
//! column's atomic write. A failed sink leaves the primary and observations intact.
|
||||||
|
//! Reopening the store reads evidence, never dispatches work. Current authorization
|
||||||
|
//! and a fresh domain projection are required before redisclosing loaded evidence.
|
||||||
|
//! Unsupported bodies, credentials, source errors and filesystem paths are not log
|
||||||
|
//! fields. Context wrapping, Remote decoding and persistence do not duplicate the
|
||||||
|
//! HTTP primary event; cleanup has a separate safe coordinator event. Non-HTTP
|
||||||
|
//! consumers receive the same typed result and own their eventual observation sink.
|
||||||
|
use super::*;
|
||||||
|
use runtime_api::startup::{
|
||||||
|
CreationAcceptance, ExecutionOutcome, StartupCause, StartupOperationFacts, WorkerCreateFailure,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(super) fn local_rejection() -> WorkerCreateFailure {
|
||||||
|
WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Runtime(protocol::startup_failure::RuntimeStartupCause {
|
||||||
|
stage: protocol::startup_failure::StartupStage::Preflight,
|
||||||
|
reason: protocol::startup_failure::RuntimeStartupReason::StartupFailed,
|
||||||
|
}),
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Rejected,
|
||||||
|
execution: ExecutionOutcome::Unknown,
|
||||||
|
finalize: None,
|
||||||
|
correlation: None,
|
||||||
|
backend_compensation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("enum-only local cause fits the boundary budget")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn finalize_error(
|
||||||
|
source: ApiError,
|
||||||
|
stage: WorkerSpawnFinalizeStage,
|
||||||
|
compensation: runtime_api::startup::CompensationOutcome,
|
||||||
|
) -> ApiError {
|
||||||
|
let failure = finalize_failure(&source, stage, compensation);
|
||||||
|
ApiError::with_diagnostics(
|
||||||
|
Error::WorkerCreateFailed {
|
||||||
|
failure: Box::new(failure),
|
||||||
|
source: Some(Box::new(source.error)),
|
||||||
|
},
|
||||||
|
source.diagnostics,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn finalize_failure(
|
||||||
|
source: &ApiError,
|
||||||
|
stage: WorkerSpawnFinalizeStage,
|
||||||
|
compensation: runtime_api::startup::CompensationOutcome,
|
||||||
|
) -> WorkerCreateFailure {
|
||||||
|
use runtime_api::startup::{
|
||||||
|
FinalizeFacts, FinalizeStage, StartupBoundaryCause, StartupBoundaryReason,
|
||||||
|
};
|
||||||
|
let stage = match stage {
|
||||||
|
WorkerSpawnFinalizeStage::WorkerIdentity => FinalizeStage::WorkerIdentity,
|
||||||
|
WorkerSpawnFinalizeStage::WorkspaceApiBinding => FinalizeStage::WorkspaceApiBinding,
|
||||||
|
WorkerSpawnFinalizeStage::CreateReservation => FinalizeStage::CreateReservation,
|
||||||
|
WorkerSpawnFinalizeStage::WorkerRegistry => FinalizeStage::WorkerRegistry,
|
||||||
|
WorkerSpawnFinalizeStage::TicketAssignmentBind => FinalizeStage::TicketAssignmentBind,
|
||||||
|
WorkerSpawnFinalizeStage::TicketAssignmentCurrent => FinalizeStage::TicketAssignmentCurrent,
|
||||||
|
WorkerSpawnFinalizeStage::TicketStateAccept => FinalizeStage::TicketStateAccept,
|
||||||
|
WorkerSpawnFinalizeStage::WorkdirRegistry => FinalizeStage::WorkdirRegistry,
|
||||||
|
WorkerSpawnFinalizeStage::WorkdirAttachment => FinalizeStage::WorkdirAttachment,
|
||||||
|
};
|
||||||
|
let operation = StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Accepted,
|
||||||
|
execution: ExecutionOutcome::Accepted,
|
||||||
|
finalize: Some(FinalizeFacts {
|
||||||
|
stage,
|
||||||
|
compensation,
|
||||||
|
}),
|
||||||
|
backend_compensation: Some(compensation),
|
||||||
|
correlation: None,
|
||||||
|
};
|
||||||
|
match &source.error {
|
||||||
|
Error::WorkerCreateFailed { failure, .. } => {
|
||||||
|
let (startup, mut previous) = failure.clone().into_parts();
|
||||||
|
// Adding finalization context must not rewrite the earlier owner result.
|
||||||
|
previous.finalize = operation.finalize;
|
||||||
|
previous.backend_compensation = Some(compensation);
|
||||||
|
WorkerCreateFailure::new(startup, previous)
|
||||||
|
}
|
||||||
|
_ => match source.session_registration_cause {
|
||||||
|
Some(cause) => WorkerCreateFailure::from_cause(StartupCause::Session(cause), operation),
|
||||||
|
None => WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause {
|
||||||
|
reason: StartupBoundaryReason::FinalizeFailed,
|
||||||
|
}),
|
||||||
|
operation,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
.expect("locally constructed finalization facts fit the shared boundary budget")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Publication failure cannot undo committed creation or authorize reexecution.
|
||||||
|
/// Keep the Worker and its assignment: `Residual` describes deliberate retention,
|
||||||
|
/// not attempted rollback; Backend compensation remains unattempted (`None`).
|
||||||
|
/// The original store error stays internal while the independent reservation sink
|
||||||
|
/// retains only the typed result and a path-free correlation.
|
||||||
|
pub(super) fn publication_error(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
source: ApiError,
|
||||||
|
) -> ApiError {
|
||||||
|
use protocol::startup_failure::{StartupCleanup, StartupResourceFacts, StartupResources};
|
||||||
|
use runtime_api::startup::{
|
||||||
|
CompensationOutcome, FinalizeFacts, FinalizeStage, StartupBoundaryCause,
|
||||||
|
StartupBoundaryReason,
|
||||||
|
};
|
||||||
|
|
||||||
|
let operation = StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Accepted,
|
||||||
|
execution: ExecutionOutcome::Accepted,
|
||||||
|
finalize: Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::Publication,
|
||||||
|
compensation: CompensationOutcome::Residual,
|
||||||
|
}),
|
||||||
|
backend_compensation: None,
|
||||||
|
correlation: None,
|
||||||
|
};
|
||||||
|
let failure = match &source.error {
|
||||||
|
Error::WorkerCreateFailed { failure, .. } => failure.as_ref().clone(),
|
||||||
|
_ => WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause {
|
||||||
|
reason: StartupBoundaryReason::FinalizeFailed,
|
||||||
|
}),
|
||||||
|
operation.clone(),
|
||||||
|
)
|
||||||
|
.expect("enum-only publication cause fits the boundary budget"),
|
||||||
|
};
|
||||||
|
let (mut startup, _) = failure.into_parts();
|
||||||
|
startup.resources = Some(StartupResourceFacts {
|
||||||
|
resources: StartupResources::WorkerStartup,
|
||||||
|
cleanup: StartupCleanup::NotNeeded,
|
||||||
|
durable_effects_possible: true,
|
||||||
|
});
|
||||||
|
let failure = WorkerCreateFailure::new(startup, operation)
|
||||||
|
.expect("locally constructed publication facts fit the shared boundary budget");
|
||||||
|
let failure = persist(api, worker, failure);
|
||||||
|
ApiError::with_diagnostics(
|
||||||
|
Error::WorkerCreateFailed {
|
||||||
|
failure: Box::new(failure),
|
||||||
|
source: Some(Box::new(source.error)),
|
||||||
|
},
|
||||||
|
source.diagnostics,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn unknown_result() -> WorkerCreateFailure {
|
||||||
|
WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Boundary(runtime_api::startup::StartupBoundaryCause {
|
||||||
|
reason: runtime_api::startup::StartupBoundaryReason::InvalidResponse,
|
||||||
|
}),
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Unknown,
|
||||||
|
execution: ExecutionOutcome::Unknown,
|
||||||
|
finalize: None,
|
||||||
|
correlation: None,
|
||||||
|
backend_compensation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("enum-only observation fits the boundary budget")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn correlate(
|
||||||
|
failure: WorkerCreateFailure,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
) -> WorkerCreateFailure {
|
||||||
|
let reference = correlation(&worker.runtime_id, &worker.worker_id);
|
||||||
|
let mut operation = failure.operation().clone();
|
||||||
|
operation.correlation = Some(
|
||||||
|
runtime_api::startup::StartupCorrelation::new(reference).expect("fixed digest is safe"),
|
||||||
|
);
|
||||||
|
failure.with_operation(operation)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn compensated(
|
||||||
|
failure: WorkerCreateFailure,
|
||||||
|
outcome: runtime_api::startup::CompensationOutcome,
|
||||||
|
) -> WorkerCreateFailure {
|
||||||
|
let mut operation = failure.operation().clone();
|
||||||
|
operation.backend_compensation = Some(outcome);
|
||||||
|
failure.with_operation(operation)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Save only observations on the existing reservation, never its lifecycle state.
|
||||||
|
/// An unavailable diagnostic sink cannot replace the original primary or authorize replay.
|
||||||
|
/// The prospective persisted marker commits in the same transaction as these facts.
|
||||||
|
/// Remote-supplied Backend persistence is discarded at this responsibility boundary.
|
||||||
|
pub(super) fn persist(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
failure: WorkerCreateFailure,
|
||||||
|
) -> WorkerCreateFailure {
|
||||||
|
use protocol::startup_failure::StartupPersistenceOutcome::{
|
||||||
|
Conflict, NotSaved, OutcomeUnknown, Persisted,
|
||||||
|
};
|
||||||
|
use rusqlite::OptionalExtension;
|
||||||
|
let failure = correlate(failure, worker).with_backend_result_persistence(None);
|
||||||
|
let Some(worker_id) = WorkerId::parse(&worker.worker_id) else {
|
||||||
|
return failure.with_backend_result_persistence(Some(NotSaved));
|
||||||
|
};
|
||||||
|
let observed = api.config_store.with_conn(|conn| {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT create_fingerprint, startup_failure_json FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
|
||||||
|
rusqlite::params![api.workspace_id(), worker.runtime_id, worker.worker_id],
|
||||||
|
|row| Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?)),
|
||||||
|
).optional().map_err(Error::from)
|
||||||
|
});
|
||||||
|
let (fingerprint, previous) = match observed {
|
||||||
|
Ok(Some(observed)) => observed,
|
||||||
|
Ok(None) => return failure.with_backend_result_persistence(Some(NotSaved)),
|
||||||
|
Err(_) => return failure.with_backend_result_persistence(Some(OutcomeUnknown)),
|
||||||
|
};
|
||||||
|
let saved = failure
|
||||||
|
.clone()
|
||||||
|
.with_backend_result_persistence(Some(Persisted));
|
||||||
|
let Ok(encoded) = saved.encode_storage() else {
|
||||||
|
return failure.with_backend_result_persistence(Some(NotSaved));
|
||||||
|
};
|
||||||
|
match api.config_store.save_worker_create_failure(
|
||||||
|
api.workspace_id(),
|
||||||
|
&worker.runtime_id,
|
||||||
|
worker_id,
|
||||||
|
&fingerprint,
|
||||||
|
previous.as_deref(),
|
||||||
|
&encoded,
|
||||||
|
) {
|
||||||
|
Ok(true) => saved,
|
||||||
|
Ok(false) => failure.with_backend_result_persistence(Some(Conflict)),
|
||||||
|
Err(_) => failure.with_backend_result_persistence(Some(OutcomeUnknown)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn correlation(runtime_id: &str, worker_id: &str) -> String {
|
||||||
|
runtime_api::startup::StartupCorrelation::for_worker(runtime_id, worker_id)
|
||||||
|
.as_str()
|
||||||
|
.to_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only the HTTP result boundary records the primary failure. Context wrappers,
|
||||||
|
/// encoders, and Remote adapters do not emit duplicate failure records. The
|
||||||
|
/// independent coordinator cleanup event contains no exception/source text.
|
||||||
|
pub(super) fn observe_http(failure: &WorkerCreateFailure, status: StatusCode) {
|
||||||
|
let Ok(projected) = failure.telemetry() else {
|
||||||
|
tracing::error!(target: "yoi::worker_startup", event = "startup_projection_unavailable", status = status.as_u16());
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let fields = error_contract::observation_fields(&projected);
|
||||||
|
let operation = failure.operation();
|
||||||
|
tracing::warn!(
|
||||||
|
target: "yoi::worker_startup",
|
||||||
|
event = "worker_create_result",
|
||||||
|
domain = fields.domain,
|
||||||
|
reason = fields.reason,
|
||||||
|
operation = fields.operation,
|
||||||
|
stage = fields.stage,
|
||||||
|
correlation = operation.correlation.as_ref().map(|value| value.as_str()),
|
||||||
|
acceptance = ?operation.acceptance,
|
||||||
|
execution = ?operation.execution,
|
||||||
|
finalize = ?operation.finalize,
|
||||||
|
backend_compensation = ?operation.backend_compensation,
|
||||||
|
backend_result_persistence = ?failure.backend_result_persistence(),
|
||||||
|
persistence = ?failure.startup().persistence,
|
||||||
|
resources = ?failure.startup().resources,
|
||||||
|
status = status.as_u16(),
|
||||||
|
"Worker creation returned a typed failure"
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -76,6 +76,26 @@ pub(super) fn verify_history(conn: &Connection, current: i64) -> Result<()> {
|
|||||||
|
|
||||||
fn verify_history_records(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> {
|
fn verify_history_records(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> {
|
||||||
verify_raw_capture_shape(conn, current, rows)?;
|
verify_raw_capture_shape(conn, current, rows)?;
|
||||||
|
if current == 124 {
|
||||||
|
super::verify_worker_create_failure_schema(conn)?;
|
||||||
|
if rows == [(124, super::SCHEMA_BASELINE_NAME.to_owned())] {
|
||||||
|
verify_execution_shape(conn)?;
|
||||||
|
super::backend_job_upgrade::verify(conn)?;
|
||||||
|
verify_configuration_shape(conn)?;
|
||||||
|
require_shape(conn, Stage::Base)?;
|
||||||
|
return verify_103_features(conn, current, rows);
|
||||||
|
}
|
||||||
|
if rows.last().map(|(version, name)| (*version, name.as_str()))
|
||||||
|
!= Some((124, super::WORKER_CREATE_FAILURE_MIGRATION_NAME))
|
||||||
|
{
|
||||||
|
return Err(invalid("unrecognized Worker creation result receipt"));
|
||||||
|
}
|
||||||
|
let prefix = &rows[..rows.len() - 1];
|
||||||
|
if prefix.last().map(|row| row.0) != Some(123) {
|
||||||
|
return Err(invalid("missing Worker creation result predecessor"));
|
||||||
|
}
|
||||||
|
return verify_history_records(conn, 123, prefix);
|
||||||
|
}
|
||||||
if current == 123 {
|
if current == 123 {
|
||||||
verify_execution_shape(conn)?;
|
verify_execution_shape(conn)?;
|
||||||
super::backend_job_upgrade::verify(conn)?;
|
super::backend_job_upgrade::verify(conn)?;
|
||||||
@@ -236,10 +256,11 @@ pub(super) fn published_plan(
|
|||||||
verify_history(conn, current)?;
|
verify_history(conn, current)?;
|
||||||
Ok(Some(super::WorkspaceSchemaMigrationPlan {
|
Ok(Some(super::WorkspaceSchemaMigrationPlan {
|
||||||
current_schema_version: current,
|
current_schema_version: current,
|
||||||
target_schema_version: 123,
|
target_schema_version: 124,
|
||||||
migrations: [
|
migrations: [
|
||||||
(122, crate::config_adoption::MIGRATION_NAME),
|
(122, crate::config_adoption::MIGRATION_NAME),
|
||||||
(123, super::backend_job_upgrade::MIGRATION_NAME),
|
(123, super::backend_job_upgrade::MIGRATION_NAME),
|
||||||
|
(124, super::WORKER_CREATE_FAILURE_MIGRATION_NAME),
|
||||||
]
|
]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|(version, _)| *version > current)
|
.filter(|(version, _)| *version > current)
|
||||||
@@ -465,7 +486,7 @@ fn legacy_shape(conn: &Connection) -> Result<Shape> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn upgraded_reference(conn: &Connection, mut shape: Shape) -> Result<Shape> {
|
fn upgraded_reference(conn: &Connection, mut shape: Shape) -> Result<Shape> {
|
||||||
if super::current_schema_version(conn)? != 123 {
|
if super::current_schema_version(conn)? < 123 {
|
||||||
return Ok(shape);
|
return Ok(shape);
|
||||||
}
|
}
|
||||||
for table in ["backend_jobs", "backend_job_attempts"] {
|
for table in ["backend_jobs", "backend_job_attempts"] {
|
||||||
|
|||||||
@@ -137,7 +137,10 @@ pub(crate) mod backend_job_upgrade;
|
|||||||
mod storage_formats;
|
mod storage_formats;
|
||||||
pub use backend_job_recovery_store::BackendJobReconciliation;
|
pub use backend_job_recovery_store::BackendJobReconciliation;
|
||||||
|
|
||||||
const LATEST_SCHEMA_VERSION: i64 = 123;
|
const LATEST_SCHEMA_VERSION: i64 = 124;
|
||||||
|
const WORKER_CREATE_FAILURE_MIGRATION_NAME: &str =
|
||||||
|
"Worker create coordinator returned startup failure";
|
||||||
|
pub(crate) const WORKER_CREATE_FAILURE_MAX_BYTES: usize = 32 * 1024;
|
||||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||||
@@ -520,6 +523,11 @@ const MIGRATIONS: &[Migration] = &[
|
|||||||
name: backend_job_upgrade::MIGRATION_NAME,
|
name: backend_job_upgrade::MIGRATION_NAME,
|
||||||
apply: migrate_backend_job_execution_to_v123,
|
apply: migrate_backend_job_execution_to_v123,
|
||||||
},
|
},
|
||||||
|
Migration {
|
||||||
|
version: 124,
|
||||||
|
name: WORKER_CREATE_FAILURE_MIGRATION_NAME,
|
||||||
|
apply: migrate_worker_create_failure_to_v124,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
||||||
@@ -16388,6 +16396,50 @@ fn migrate_plugin_services_v111_to_v112(conn: &Connection) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn verify_worker_create_failure_schema(conn: &Connection) -> Result<()> {
|
||||||
|
let column = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT type, \"notnull\", dflt_value FROM pragma_table_info('worker_create_reservations') WHERE name='startup_failure_json'",
|
||||||
|
[],
|
||||||
|
|row| Ok((row.get::<_, String>(0)?, row.get::<_, bool>(1)?, row.get::<_, Option<String>>(2)?)),
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
let sql: String = conn.query_row(
|
||||||
|
"SELECT sql FROM sqlite_schema WHERE type='table' AND name='worker_create_reservations'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
let normalized: String = sql.chars().filter(|c| !c.is_whitespace()).collect();
|
||||||
|
let constraint = format!(
|
||||||
|
"CHECK(startup_failure_jsonISNULLOR(typeof(startup_failure_json)='text'ANDlength(CAST(startup_failure_jsonASBLOB))<={}))",
|
||||||
|
WORKER_CREATE_FAILURE_MAX_BYTES
|
||||||
|
);
|
||||||
|
if column != Some(("TEXT".into(), false, None)) || !normalized.contains(&constraint) {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Worker create startup failure column or byte bound is missing".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn migrate_worker_create_failure_to_v124(conn: &Connection) -> Result<()> {
|
||||||
|
if current_schema_version(conn)? != 123 {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Worker create failure migration requires schema 123".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
storage_formats::verify_history(conn, 123)?;
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
tx.execute_batch(include_str!("worker_create_failure_schema.sql"))?;
|
||||||
|
verify_worker_create_failure_schema(&tx)?;
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(124,?1)",
|
||||||
|
[WORKER_CREATE_FAILURE_MIGRATION_NAME],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn migrate_backend_job_execution_to_v123(conn: &Connection) -> Result<()> {
|
fn migrate_backend_job_execution_to_v123(conn: &Connection) -> Result<()> {
|
||||||
if current_schema_version(conn)? != 122 {
|
if current_schema_version(conn)? != 122 {
|
||||||
return Err(Error::Store(
|
return Err(Error::Store(
|
||||||
@@ -16537,6 +16589,8 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
|||||||
conn.execute_batch(content_history_store::SCHEMA)?;
|
conn.execute_batch(content_history_store::SCHEMA)?;
|
||||||
conn.execute_batch(crate::config_adoption::SCHEMA)?;
|
conn.execute_batch(crate::config_adoption::SCHEMA)?;
|
||||||
backend_job_upgrade::install_fresh(conn)?;
|
backend_job_upgrade::install_fresh(conn)?;
|
||||||
|
conn.execute_batch(include_str!("worker_create_failure_schema.sql"))?;
|
||||||
|
verify_worker_create_failure_schema(conn)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -16574,11 +16628,14 @@ fn migrate_plugin_plans_v106_to_v107(conn: &Connection) -> Result<()> {
|
|||||||
// Fresh composition uses the posts contract. Retained fixtures pass frozen host
|
// Fresh composition uses the posts contract. Retained fixtures pass frozen host
|
||||||
// and component DDL to the shared cross-domain constraint composition below.
|
// and component DDL to the shared cross-domain constraint composition below.
|
||||||
fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
|
fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
|
||||||
create_workspace_schema_structure_from_sql(
|
// The pre-124 reference composition must not already contain the column
|
||||||
conn,
|
// added by migration 124. Fresh latest creation installs it after this prefix.
|
||||||
&backend_job_upgrade::retained_checks(include_str!("latest_schema.sql")),
|
let retained_schema = backend_job_upgrade::retained_checks(include_str!("latest_schema.sql"))
|
||||||
true,
|
.replace(
|
||||||
)?;
|
" startup_failure_json TEXT CHECK (startup_failure_json IS NULL OR\n (typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768)),\n",
|
||||||
|
"",
|
||||||
|
);
|
||||||
|
create_workspace_schema_structure_from_sql(conn, &retained_schema, true)?;
|
||||||
conn.execute_batch(include_str!("activation_schema.sql"))?;
|
conn.execute_batch(include_str!("activation_schema.sql"))?;
|
||||||
conn.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
conn.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
||||||
conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?;
|
conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?;
|
||||||
@@ -17672,6 +17729,10 @@ fn table_columns(conn: &Connection, table_name: &str) -> Result<Vec<String>> {
|
|||||||
#[path = "storage_owner_tests.rs"]
|
#[path = "storage_owner_tests.rs"]
|
||||||
mod storage_owner_tests;
|
mod storage_owner_tests;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "worker_create_failure_store_tests.rs"]
|
||||||
|
mod worker_create_failure_store_tests;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "store_test_support.rs"]
|
#[path = "store_test_support.rs"]
|
||||||
mod test_support;
|
mod test_support;
|
||||||
|
|||||||
@@ -852,8 +852,6 @@ mod subject_assignment_migration_tests {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const PLAN_NAME: &str = "Immutable Worker Plugin plans and retained pins";
|
const PLAN_NAME: &str = "Immutable Worker Plugin plans and retained pins";
|
||||||
const SUBJECT_NAME: &str =
|
|
||||||
"Subject Ticket responsibility and current-body notification routing";
|
|
||||||
type Rows = Vec<Vec<rusqlite::types::Value>>;
|
type Rows = Vec<Vec<rusqlite::types::Value>>;
|
||||||
|
|
||||||
fn raw(path: &Path) -> Connection {
|
fn raw(path: &Path) -> Connection {
|
||||||
@@ -1038,8 +1036,23 @@ mod subject_assignment_migration_tests {
|
|||||||
} else {
|
} else {
|
||||||
String::new()
|
String::new()
|
||||||
};
|
};
|
||||||
|
// Keep append identities for rowid tables; current content
|
||||||
|
// history tables are WITHOUT ROWID, so compare every column
|
||||||
|
// in deterministic value order instead of inventing rowids.
|
||||||
|
let without_rowid: bool = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT wr FROM pragma_table_list WHERE schema='main' AND name=?1",
|
||||||
|
[&table],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (projection, order) = if without_rowid {
|
||||||
|
(columns.clone(), columns)
|
||||||
|
} else {
|
||||||
|
(format!("rowid,{columns}"), "rowid".into())
|
||||||
|
};
|
||||||
let query = format!(
|
let query = format!(
|
||||||
"SELECT rowid,{columns} FROM \"{table}\"{sequence_filter} ORDER BY rowid"
|
"SELECT {projection} FROM \"{table}\"{sequence_filter} ORDER BY {order}"
|
||||||
);
|
);
|
||||||
let rows = fixture_query_rows(conn, &query);
|
let rows = fixture_query_rows(conn, &query);
|
||||||
(query, rows)
|
(query, rows)
|
||||||
@@ -1120,49 +1133,15 @@ mod subject_assignment_migration_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn expected_tail(version: i64) -> Vec<(i64, String)> {
|
fn expected_tail(version: i64) -> Vec<(i64, String)> {
|
||||||
let mut tail = match version {
|
assert!(
|
||||||
96 => vec![
|
matches!(version, 96 | 106 | 107),
|
||||||
(
|
"unsupported fixture origin"
|
||||||
97,
|
);
|
||||||
"MR notifications and authenticated implementation results".into(),
|
MIGRATIONS
|
||||||
),
|
.iter()
|
||||||
(98, "Assignment-independent implementation reports".into()),
|
.filter(|migration| migration.version > version)
|
||||||
(
|
.map(|migration| (migration.version, migration.name.into()))
|
||||||
99,
|
.collect()
|
||||||
"Explicit hosting administrators and self-registration policy".into(),
|
|
||||||
),
|
|
||||||
(
|
|
||||||
100,
|
|
||||||
"Provisional users and scoped passkey activation".into(),
|
|
||||||
),
|
|
||||||
(101, "Durable explicit Subject body release".into()),
|
|
||||||
(102, "Durable failed Subject creation recovery".into()),
|
|
||||||
(103, "Passkey metadata and atomic addition results".into()),
|
|
||||||
(
|
|
||||||
104,
|
|
||||||
"Converge Passkey results and Job input retention".into(),
|
|
||||||
),
|
|
||||||
(
|
|
||||||
105,
|
|
||||||
"Durable Subject deletion snapshots and Job admission fences".into(),
|
|
||||||
),
|
|
||||||
(106, crate::plugin_catalog::MIGRATION_NAME.into()),
|
|
||||||
(107, PLAN_NAME.into()),
|
|
||||||
(108, SUBJECT_NAME.into()),
|
|
||||||
],
|
|
||||||
106 => vec![(107, PLAN_NAME.into()), (108, SUBJECT_NAME.into())],
|
|
||||||
107 => vec![(108, SUBJECT_NAME.into())],
|
|
||||||
_ => panic!("unsupported fixture origin"),
|
|
||||||
};
|
|
||||||
tail.push((109, "Durable Plugin Host invocations and HTTPS effect receipts".into()));
|
|
||||||
tail.push((110, "Worker Plugin launch selection associations".into()));
|
|
||||||
tail.push((
|
|
||||||
111,
|
|
||||||
"Typed Plugin Ticket extension data and atomic mutation receipts".into(),
|
|
||||||
));
|
|
||||||
tail.push((112, "Ticket Service outbox and fenced consumers".into()));
|
|
||||||
tail.push((113, "Ticket work-decomposition parent links".into()));
|
|
||||||
tail
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn plan_steps(version: i64) -> Vec<WorkspaceSchemaMigrationStep> {
|
fn plan_steps(version: i64) -> Vec<WorkspaceSchemaMigrationStep> {
|
||||||
|
|||||||
@@ -56,12 +56,17 @@ fn ticket_posts_schema93_preserves_initial_body_creator_comments_and_archived_au
|
|||||||
drop(conn);
|
drop(conn);
|
||||||
let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap();
|
let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap();
|
||||||
assert_eq!(plan.current_schema_version, 93);
|
assert_eq!(plan.current_schema_version, 93);
|
||||||
|
assert_eq!(plan.target_schema_version, LATEST_SCHEMA_VERSION);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
plan.migrations
|
plan.migrations,
|
||||||
|
MIGRATIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|step| step.version)
|
.filter(|migration| migration.version > 93)
|
||||||
.collect::<Vec<_>>(),
|
.map(|migration| WorkspaceSchemaMigrationStep {
|
||||||
(94..=LATEST_SCHEMA_VERSION).collect::<Vec<_>>()
|
version: migration.version,
|
||||||
|
name: migration.name.into(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
);
|
);
|
||||||
let conn = Connection::open(&path).unwrap();
|
let conn = Connection::open(&path).unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE worker_create_reservations ADD COLUMN startup_failure_json TEXT
|
||||||
|
CHECK (startup_failure_json IS NULL OR
|
||||||
|
(typeof(startup_failure_json) = 'text' AND length(CAST(startup_failure_json AS BLOB)) <= 32768));
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
//! Backend coordinator returned startup results belong to the existing create
|
||||||
|
//! reservation. Runtime primary failure remains in Session metadata. Saving or
|
||||||
|
//! reading this evidence neither transitions the reservation nor grants retry.
|
||||||
|
|
||||||
|
use runtime_api::startup::{DecodeStoredWorkerCreateFailure, decode_stored_worker_create_failure};
|
||||||
|
use rusqlite::{OptionalExtension, params};
|
||||||
|
use worker_runtime::identity::WorkerId;
|
||||||
|
|
||||||
|
use crate::store::{SqliteWorkspaceStore, WORKER_CREATE_FAILURE_MAX_BYTES};
|
||||||
|
use crate::{Error, Result};
|
||||||
|
|
||||||
|
impl SqliteWorkspaceStore {
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub(crate) fn save_worker_create_failure(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
runtime_id: &str,
|
||||||
|
worker_id: WorkerId,
|
||||||
|
expected_create_fingerprint: &str,
|
||||||
|
expected_previous: Option<&str>,
|
||||||
|
encoded: &str,
|
||||||
|
) -> Result<bool> {
|
||||||
|
// Bound the wire representation before the typed decoder allocates JSON.
|
||||||
|
if encoded.len() > WORKER_CREATE_FAILURE_MAX_BYTES
|
||||||
|
|| expected_previous.is_some_and(|value| value.len() > WORKER_CREATE_FAILURE_MAX_BYTES)
|
||||||
|
{
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"Worker create failure exceeds storage byte bound".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
decode_stored_worker_create_failure(encoded),
|
||||||
|
DecodeStoredWorkerCreateFailure::Known(_)
|
||||||
|
) {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"Worker create failure is not a known storage representation".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
self.with_immediate_transaction(|tx| {
|
||||||
|
// Identity, immutable create intent and exact prior bytes are checked
|
||||||
|
// in the same UPDATE. Removed reservations deliberately remain eligible.
|
||||||
|
let changed = tx.execute(
|
||||||
|
"UPDATE worker_create_reservations SET startup_failure_json=?6
|
||||||
|
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
||||||
|
AND create_fingerprint=?4 AND startup_failure_json IS ?5",
|
||||||
|
params![
|
||||||
|
workspace_id,
|
||||||
|
runtime_id,
|
||||||
|
worker_id.to_string(),
|
||||||
|
expected_create_fingerprint,
|
||||||
|
expected_previous,
|
||||||
|
encoded
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
Ok(changed == 1)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn worker_create_failure(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
runtime_id: &str,
|
||||||
|
worker_id: WorkerId,
|
||||||
|
) -> Result<Option<String>> {
|
||||||
|
self.with_conn(|conn| {
|
||||||
|
// Return raw bounded evidence, including unknown/corrupt versions;
|
||||||
|
// classification belongs to the typed reader, not legacy prose.
|
||||||
|
let saved = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT CASE WHEN length(CAST(startup_failure_json AS BLOB)) <= ?4
|
||||||
|
THEN startup_failure_json ELSE NULL END,
|
||||||
|
length(CAST(startup_failure_json AS BLOB))
|
||||||
|
FROM worker_create_reservations
|
||||||
|
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
|
||||||
|
params![
|
||||||
|
workspace_id,
|
||||||
|
runtime_id,
|
||||||
|
worker_id.to_string(),
|
||||||
|
WORKER_CREATE_FAILURE_MAX_BYTES as i64
|
||||||
|
],
|
||||||
|
|row| {
|
||||||
|
Ok((
|
||||||
|
row.get::<_, Option<String>>(0)?,
|
||||||
|
row.get::<_, Option<i64>>(1)?,
|
||||||
|
))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
match saved {
|
||||||
|
Some((_, Some(bytes))) if bytes > WORKER_CREATE_FAILURE_MAX_BYTES as i64 => Err(
|
||||||
|
Error::Store("Stored Worker create failure exceeds byte bound".into()),
|
||||||
|
),
|
||||||
|
Some((value, _)) => Ok(value),
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,475 @@
|
|||||||
|
use super::*;
|
||||||
|
use runtime_api::startup::{
|
||||||
|
CompensationOutcome, CreationAcceptance, DecodeStoredWorkerCreateFailure, ExecutionOutcome,
|
||||||
|
FinalizeFacts, FinalizeStage, StartupBoundaryCause, StartupBoundaryReason, StartupCause,
|
||||||
|
StartupOperationFacts, WorkerCreateFailure, decode_stored_worker_create_failure,
|
||||||
|
};
|
||||||
|
use rusqlite::types::Value;
|
||||||
|
|
||||||
|
const WORKSPACE: &str = "failure-workspace";
|
||||||
|
const RUNTIME: &str = "failure-runtime";
|
||||||
|
|
||||||
|
fn encoded_failure(compensation: CompensationOutcome) -> String {
|
||||||
|
let failure = WorkerCreateFailure::from_cause(
|
||||||
|
StartupCause::Boundary(StartupBoundaryCause {
|
||||||
|
reason: StartupBoundaryReason::FinalizeFailed,
|
||||||
|
}),
|
||||||
|
StartupOperationFacts {
|
||||||
|
acceptance: CreationAcceptance::Accepted,
|
||||||
|
execution: ExecutionOutcome::Accepted,
|
||||||
|
finalize: Some(FinalizeFacts {
|
||||||
|
stage: FinalizeStage::CreateReservation,
|
||||||
|
compensation,
|
||||||
|
}),
|
||||||
|
backend_compensation: Some(compensation),
|
||||||
|
correlation: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
failure.encode_storage().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reserve(store: &SqliteWorkspaceStore) -> WorkerCreateReservation {
|
||||||
|
store.with_conn(|conn| {
|
||||||
|
conn.execute_batch("INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at)
|
||||||
|
VALUES('failure-owner','user','failure-owner','Owner','1','1');
|
||||||
|
INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at)
|
||||||
|
VALUES('failure-workspace','failure-owner','Failure workspace','active','1','1');
|
||||||
|
INSERT INTO workspace_memory_settings(workspace_id,language,created_at,updated_at)
|
||||||
|
VALUES('failure-workspace','en','1','1');")?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
store
|
||||||
|
.reserve_worker_create(
|
||||||
|
WORKSPACE,
|
||||||
|
RUNTIME,
|
||||||
|
"manual:startup-failure",
|
||||||
|
"sha256:startup-failure",
|
||||||
|
Some("subjektiv:failed-create"),
|
||||||
|
&store.get_workspace_memory_settings(WORKSPACE).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot every non-result reservation field plus resource keys and leases, so
|
||||||
|
// a save cannot silently acquire lifecycle effects as new columns are added.
|
||||||
|
fn non_result_snapshot(store: &SqliteWorkspaceStore) -> Vec<Vec<Value>> {
|
||||||
|
store.with_conn(|conn| {
|
||||||
|
let columns = table_columns(conn, "worker_create_reservations")?
|
||||||
|
.into_iter()
|
||||||
|
.filter(|name| name != "startup_failure_json")
|
||||||
|
.map(|name| format!("\"{name}\""))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(",");
|
||||||
|
let queries = [
|
||||||
|
format!("SELECT {columns} FROM worker_create_reservations ORDER BY workspace_id,allocation_key"),
|
||||||
|
"SELECT * FROM workspace_resource_keys ORDER BY workspace_id,resource_kind,resource_id".into(),
|
||||||
|
"SELECT * FROM workspace_resource_key_counters ORDER BY workspace_id,resource_kind".into(),
|
||||||
|
"SELECT * FROM worker_singleton_owners ORDER BY workspace_id,singleton_key".into(),
|
||||||
|
];
|
||||||
|
let mut values = Vec::new();
|
||||||
|
for query in queries {
|
||||||
|
let mut statement = conn.prepare(&query)?;
|
||||||
|
let count = statement.column_count();
|
||||||
|
values.extend(statement.query_map([], |row| {
|
||||||
|
(0..count).map(|i| row.get::<_, Value>(i)).collect::<rusqlite::Result<Vec<_>>>()
|
||||||
|
})?.collect::<rusqlite::Result<Vec<_>>>()?);
|
||||||
|
}
|
||||||
|
Ok(values)
|
||||||
|
}).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save(
|
||||||
|
store: &SqliteWorkspaceStore,
|
||||||
|
reservation: &WorkerCreateReservation,
|
||||||
|
previous: Option<&str>,
|
||||||
|
encoded: &str,
|
||||||
|
) -> Result<bool> {
|
||||||
|
store.save_worker_create_failure(
|
||||||
|
WORKSPACE,
|
||||||
|
RUNTIME,
|
||||||
|
reservation.worker_id,
|
||||||
|
&reservation.create_fingerprint,
|
||||||
|
previous,
|
||||||
|
encoded,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_create_failure_survives_restart_and_compensation_without_changing_lifecycle() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let path = temp.path().join("server.db");
|
||||||
|
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
let reservation = reserve(&store);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap(),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
let original = non_result_snapshot(&store);
|
||||||
|
let encoded = encoded_failure(CompensationOutcome::Unknown);
|
||||||
|
assert!(save(&store, &reservation, None, &encoded).unwrap());
|
||||||
|
assert_eq!(non_result_snapshot(&store), original);
|
||||||
|
drop(store);
|
||||||
|
|
||||||
|
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(encoded.as_str())
|
||||||
|
);
|
||||||
|
store
|
||||||
|
.fail_worker_create_reservation(
|
||||||
|
WORKSPACE,
|
||||||
|
RUNTIME,
|
||||||
|
reservation.worker_id,
|
||||||
|
&reservation.create_fingerprint,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(encoded.as_str())
|
||||||
|
);
|
||||||
|
let removed = non_result_snapshot(&store);
|
||||||
|
let compensated = encoded_failure(CompensationOutcome::Completed);
|
||||||
|
assert!(save(&store, &reservation, Some(&encoded), &compensated).unwrap());
|
||||||
|
assert_eq!(non_result_snapshot(&store), removed);
|
||||||
|
drop(store);
|
||||||
|
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
let raw = store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(raw, compensated);
|
||||||
|
let DecodeStoredWorkerCreateFailure::Known(failure) = decode_stored_worker_create_failure(&raw)
|
||||||
|
else {
|
||||||
|
panic!("saved result must decode")
|
||||||
|
};
|
||||||
|
assert_eq!(failure.operation().acceptance, CreationAcceptance::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
failure.operation().finalize.unwrap().compensation,
|
||||||
|
CompensationOutcome::Completed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_create_failure_identity_and_exact_previous_bytes_fence_every_write() {
|
||||||
|
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
|
||||||
|
let reservation = reserve(&store);
|
||||||
|
let first = encoded_failure(CompensationOutcome::Unknown);
|
||||||
|
let next = encoded_failure(CompensationOutcome::Completed);
|
||||||
|
assert!(save(&store, &reservation, None, &first).unwrap());
|
||||||
|
let original = non_result_snapshot(&store);
|
||||||
|
for (workspace, runtime, worker, fingerprint, previous) in [
|
||||||
|
(
|
||||||
|
"other",
|
||||||
|
RUNTIME,
|
||||||
|
reservation.worker_id,
|
||||||
|
reservation.create_fingerprint.as_str(),
|
||||||
|
Some(first.as_str()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WORKSPACE,
|
||||||
|
"other",
|
||||||
|
reservation.worker_id,
|
||||||
|
reservation.create_fingerprint.as_str(),
|
||||||
|
Some(first.as_str()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WORKSPACE,
|
||||||
|
RUNTIME,
|
||||||
|
WorkerId::now_v7(),
|
||||||
|
reservation.create_fingerprint.as_str(),
|
||||||
|
Some(first.as_str()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WORKSPACE,
|
||||||
|
RUNTIME,
|
||||||
|
reservation.worker_id,
|
||||||
|
"wrong",
|
||||||
|
Some(first.as_str()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WORKSPACE,
|
||||||
|
RUNTIME,
|
||||||
|
reservation.worker_id,
|
||||||
|
reservation.create_fingerprint.as_str(),
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WORKSPACE,
|
||||||
|
RUNTIME,
|
||||||
|
reservation.worker_id,
|
||||||
|
reservation.create_fingerprint.as_str(),
|
||||||
|
Some(next.as_str()),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.save_worker_create_failure(
|
||||||
|
workspace,
|
||||||
|
runtime,
|
||||||
|
worker,
|
||||||
|
fingerprint,
|
||||||
|
previous,
|
||||||
|
&next
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(first.as_str())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let equivalent_but_different_bytes = format!("{first} ");
|
||||||
|
assert!(
|
||||||
|
!save(
|
||||||
|
&store,
|
||||||
|
&reservation,
|
||||||
|
Some(&equivalent_but_different_bytes),
|
||||||
|
&next
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
for (workspace, runtime, worker) in [
|
||||||
|
("other", RUNTIME, reservation.worker_id),
|
||||||
|
(WORKSPACE, "other", reservation.worker_id),
|
||||||
|
(WORKSPACE, RUNTIME, WorkerId::now_v7()),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(workspace, runtime, worker)
|
||||||
|
.unwrap(),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(non_result_snapshot(&store), original);
|
||||||
|
assert!(save(&store, &reservation, Some(&first), &next).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_create_failure_failed_write_rolls_back_result_and_all_other_fields() {
|
||||||
|
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
|
||||||
|
let reservation = reserve(&store);
|
||||||
|
let first = encoded_failure(CompensationOutcome::Unknown);
|
||||||
|
let next = encoded_failure(CompensationOutcome::Completed);
|
||||||
|
assert!(save(&store, &reservation, None, &first).unwrap());
|
||||||
|
let original = non_result_snapshot(&store);
|
||||||
|
store.with_conn(|conn| {
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_startup_failure AFTER UPDATE OF startup_failure_json ON worker_create_reservations
|
||||||
|
BEGIN UPDATE worker_create_reservations SET state='removed';
|
||||||
|
SELECT RAISE(ABORT,'injected startup failure write interruption'); END;")?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
assert!(save(&store, &reservation, Some(&first), &next).is_err());
|
||||||
|
assert_eq!(non_result_snapshot(&store), original);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(first.as_str())
|
||||||
|
);
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_startup_failure")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(save(&store, &reservation, Some(&first), &next).unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_create_failure_rejects_unknown_corrupt_and_oversized_writes_but_reads_raw_history() {
|
||||||
|
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
|
||||||
|
let reservation = reserve(&store);
|
||||||
|
let first = encoded_failure(CompensationOutcome::Unknown);
|
||||||
|
assert!(save(&store, &reservation, None, &first).unwrap());
|
||||||
|
let mut unknown: serde_json::Value = serde_json::from_str(&first).unwrap();
|
||||||
|
unknown["version"] = serde_json::json!(65535);
|
||||||
|
let unknown = serde_json::to_string(&unknown).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
decode_stored_worker_create_failure(&unknown),
|
||||||
|
DecodeStoredWorkerCreateFailure::Unsupported
|
||||||
|
));
|
||||||
|
for invalid in [
|
||||||
|
unknown.as_str(),
|
||||||
|
"not JSON",
|
||||||
|
&" ".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1),
|
||||||
|
] {
|
||||||
|
assert!(save(&store, &reservation, Some(&first), invalid).is_err());
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(first.as_str())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Simulate a retained newer writer/corrupt row, without interpreting prose.
|
||||||
|
for raw in [unknown.as_str(), "not JSON"] {
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE worker_create_reservations SET startup_failure_json=?1",
|
||||||
|
[raw],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(raw)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(matches!(
|
||||||
|
decode_stored_worker_create_failure("not JSON"),
|
||||||
|
DecodeStoredWorkerCreateFailure::Corrupt(_)
|
||||||
|
));
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch("PRAGMA ignore_check_constraints=ON;")?;
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE worker_create_reservations SET startup_failure_json=?1",
|
||||||
|
["x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1)],
|
||||||
|
)?;
|
||||||
|
conn.execute_batch("PRAGMA ignore_check_constraints=OFF;")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn result_column_signature(conn: &Connection) -> Vec<(String, String, bool, Option<String>)> {
|
||||||
|
conn.prepare("SELECT name,type,\"notnull\",dflt_value FROM pragma_table_info('worker_create_reservations') ORDER BY name").unwrap()
|
||||||
|
.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))).unwrap()
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_result_constraints(conn: &Connection, worker: WorkerId) {
|
||||||
|
for accepted in [
|
||||||
|
"x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES),
|
||||||
|
"é".repeat(WORKER_CREATE_FAILURE_MAX_BYTES / 2),
|
||||||
|
] {
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE worker_create_reservations SET startup_failure_json=?1 WHERE worker_id=?2",
|
||||||
|
params![accepted, worker.to_string()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
for rejected in [
|
||||||
|
"x".repeat(WORKER_CREATE_FAILURE_MAX_BYTES + 1),
|
||||||
|
"é".repeat(WORKER_CREATE_FAILURE_MAX_BYTES / 2 + 1),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE worker_create_reservations SET startup_failure_json=?1 WHERE worker_id=?2",
|
||||||
|
params![rejected, worker.to_string()]
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE worker_create_reservations SET startup_failure_json=x'00' WHERE worker_id=?1",
|
||||||
|
[worker.to_string()]
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE worker_create_reservations SET startup_failure_json=NULL WHERE worker_id=?1",
|
||||||
|
[worker.to_string()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
verify_worker_create_failure_schema(conn).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_create_failure_oldest_upgrade_preserves_legacy_null_and_matches_fresh_constraints() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let path = temp.path().join("oldest.db");
|
||||||
|
tests::prepare_schema_v50(&path, Some("workspace-a"));
|
||||||
|
let legacy_worker = WorkerId::now_v7();
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
conn.execute("INSERT INTO worker_create_reservations(workspace_id,allocation_key,worker_id,runtime_id,create_fingerprint,state,created_at,updated_at)
|
||||||
|
VALUES('workspace-a','legacy-allocation',?1,'shared','legacy-fingerprint','reserved','1','1')", [legacy_worker.to_string()]).unwrap();
|
||||||
|
drop(conn);
|
||||||
|
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
upgraded
|
||||||
|
.worker_create_failure("workspace-a", "shared", legacy_worker)
|
||||||
|
.unwrap(),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
upgraded.with_conn(|conn| {
|
||||||
|
let saved: (String, String, String) = conn.query_row("SELECT create_fingerprint,state,created_at FROM worker_create_reservations WHERE worker_id=?1", [legacy_worker.to_string()], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
|
||||||
|
assert_eq!(saved, ("legacy-fingerprint".into(), "reserved".into(), "1".into()));
|
||||||
|
assert_result_constraints(conn, legacy_worker);
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
let reservation = reserve(&upgraded);
|
||||||
|
let encoded = encoded_failure(CompensationOutcome::Residual);
|
||||||
|
assert!(save(&upgraded, &reservation, None, &encoded).unwrap());
|
||||||
|
drop(upgraded);
|
||||||
|
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
upgraded
|
||||||
|
.worker_create_failure(WORKSPACE, RUNTIME, reservation.worker_id)
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(encoded.as_str())
|
||||||
|
);
|
||||||
|
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||||
|
let reservation = reserve(&fresh);
|
||||||
|
fresh
|
||||||
|
.with_conn(|conn| {
|
||||||
|
assert_result_constraints(conn, reservation.worker_id);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
upgraded
|
||||||
|
.with_conn(|conn| Ok(result_column_signature(conn)))
|
||||||
|
.unwrap(),
|
||||||
|
fresh
|
||||||
|
.with_conn(|conn| Ok(result_column_signature(conn)))
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_create_failure_migration_failure_rolls_back_column_and_receipt() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let path = temp.path().join("upgrade.db");
|
||||||
|
tests::prepare_retained_schema(&path, 123);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
configure_sqlite(&conn).unwrap();
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_startup_failure_migration BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=124
|
||||||
|
BEGIN SELECT RAISE(ABORT,'injected startup failure migration interruption'); END;").unwrap();
|
||||||
|
assert!(migrate_worker_create_failure_to_v124(&conn).is_err());
|
||||||
|
assert_eq!(current_schema_version(&conn).unwrap(), 123);
|
||||||
|
assert!(!column_exists(&conn, "worker_create_reservations", "startup_failure_json").unwrap());
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_startup_failure_migration")
|
||||||
|
.unwrap();
|
||||||
|
migrate_worker_create_failure_to_v124(&conn).unwrap();
|
||||||
|
verify_worker_create_failure_schema(&conn).unwrap();
|
||||||
|
assert_eq!(current_schema_version(&conn).unwrap(), 124);
|
||||||
|
drop(conn);
|
||||||
|
SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user