fix: integrate command output failure diagnostics
This commit is contained in:
Generated
+2
@@ -6497,6 +6497,7 @@ name = "workdir"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
"axum",
|
||||||
"fs-operation",
|
"fs-operation",
|
||||||
"libc",
|
"libc",
|
||||||
"manifest",
|
"manifest",
|
||||||
@@ -6598,6 +6599,7 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
"tokio-tungstenite 0.29.0",
|
"tokio-tungstenite 0.29.0",
|
||||||
"toml",
|
"toml",
|
||||||
|
"tools",
|
||||||
"tower",
|
"tower",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
|||||||
@@ -132,6 +132,8 @@ pub struct WorkdirOperationApiError {
|
|||||||
pub workdir_log: Option<WorkdirOperationLogContext>,
|
pub workdir_log: Option<WorkdirOperationLogContext>,
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub denial_reason: Option<workdir::WorkdirDenialReason>,
|
pub denial_reason: Option<workdir::WorkdirDenialReason>,
|
||||||
|
#[serde(skip)]
|
||||||
|
pub failure_reason: Option<workdir::WorkdirFailureReason>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Internal operational correlation. Producers supply fixed-length ID fingerprints and
|
/// Internal operational correlation. Producers supply fixed-length ID fingerprints and
|
||||||
@@ -157,6 +159,7 @@ impl WorkdirOperationApiError {
|
|||||||
status: error.status,
|
status: error.status,
|
||||||
workdir_log: None,
|
workdir_log: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
|
failure_reason: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,6 +173,7 @@ impl WorkdirOperationApiError {
|
|||||||
status,
|
status,
|
||||||
workdir_log: None,
|
workdir_log: None,
|
||||||
denial_reason: error.denial_reason,
|
denial_reason: error.denial_reason,
|
||||||
|
failure_reason: error.failure_reason,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -15047,10 +15051,12 @@ mod workdir_error_diagnostic_tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn workdir_internal_diagnostics_never_enter_public_error_body_or_schema() {
|
fn workdir_internal_diagnostics_never_enter_public_error_body_or_schema() {
|
||||||
let mut error = WorkdirOperationApiError::provider(workdir::http::WorkdirTransportError {
|
let mut error = WorkdirOperationApiError::provider(workdir::http::WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::Denied,
|
code: workdir::http::WorkdirTransportErrorCode::Denied,
|
||||||
message: "Workdir operation was denied".to_string(),
|
message: "Workdir operation was denied".to_string(),
|
||||||
denial_reason: Some(workdir::WorkdirDenialReason::OsPermissionDenied),
|
denial_reason: Some(workdir::WorkdirDenialReason::OsPermissionDenied),
|
||||||
});
|
});
|
||||||
|
error.failure_reason = Some(workdir::WorkdirFailureReason::HttpTimeout);
|
||||||
error.workdir_log = Some(WorkdirOperationLogContext {
|
error.workdir_log = Some(WorkdirOperationLogContext {
|
||||||
operation: "write",
|
operation: "write",
|
||||||
workspace_id_hash: "internal-workspace-fingerprint".to_string(),
|
workspace_id_hash: "internal-workspace-fingerprint".to_string(),
|
||||||
@@ -15064,10 +15070,12 @@ mod workdir_error_diagnostic_tests {
|
|||||||
assert_eq!(api_macros::HttpError::status_code(&error), 403);
|
assert_eq!(api_macros::HttpError::status_code(&error), 403);
|
||||||
let decoded: WorkdirOperationApiError = serde_json::from_value(wire).unwrap();
|
let decoded: WorkdirOperationApiError = serde_json::from_value(wire).unwrap();
|
||||||
assert!(decoded.denial_reason.is_none());
|
assert!(decoded.denial_reason.is_none());
|
||||||
|
assert!(decoded.failure_reason.is_none());
|
||||||
assert!(decoded.workdir_log.is_none());
|
assert!(decoded.workdir_log.is_none());
|
||||||
let schema = serde_json::to_value(schemars::schema_for!(WorkdirOperationApiError)).unwrap();
|
let schema = serde_json::to_value(schemars::schema_for!(WorkdirOperationApiError)).unwrap();
|
||||||
assert!(schema["properties"].get("workdir_log").is_none());
|
assert!(schema["properties"].get("workdir_log").is_none());
|
||||||
assert!(schema["properties"].get("denial_reason").is_none());
|
assert!(schema["properties"].get("denial_reason").is_none());
|
||||||
|
assert!(schema["properties"].get("failure_reason").is_none());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -158,11 +158,15 @@ async fn command_page(
|
|||||||
) -> Result<workdir::CommandOutput, workdir::WorkdirError> {
|
) -> Result<workdir::CommandOutput, workdir::WorkdirError> {
|
||||||
for attempt in 0..3 {
|
for attempt in 0..3 {
|
||||||
match session.command_output(request.clone()).await {
|
match session.command_output(request.clone()).await {
|
||||||
Err(
|
Err(error)
|
||||||
workdir::WorkdirError::Transport(_)
|
if attempt < 2
|
||||||
| workdir::WorkdirError::Unavailable(_)
|
&& matches!(
|
||||||
| workdir::WorkdirError::OutcomeUnknown(_),
|
error.classification_source(),
|
||||||
) if attempt < 2 => {
|
workdir::WorkdirError::Transport(_)
|
||||||
|
| workdir::WorkdirError::Unavailable(_)
|
||||||
|
| workdir::WorkdirError::OutcomeUnknown(_)
|
||||||
|
) =>
|
||||||
|
{
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(100 << attempt)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(100 << attempt)).await;
|
||||||
}
|
}
|
||||||
result => return result,
|
result => return result,
|
||||||
@@ -363,7 +367,7 @@ impl BashTool {
|
|||||||
guard.handle = None;
|
guard.handle = None;
|
||||||
|
|
||||||
let timed_out = terminal.status == workdir::CommandStatus::TimedOut;
|
let timed_out = terminal.status == workdir::CommandStatus::TimedOut;
|
||||||
let summary =
|
let mut summary =
|
||||||
if cancellation_requested || terminal.status == workdir::CommandStatus::Cancelled {
|
if cancellation_requested || terminal.status == workdir::CommandStatus::Cancelled {
|
||||||
format!("$ {cmd_summary} (cancelled)")
|
format!("$ {cmd_summary} (cancelled)")
|
||||||
} else if timed_out {
|
} else if timed_out {
|
||||||
@@ -375,6 +379,12 @@ impl BashTool {
|
|||||||
None => format!("$ {cmd_summary} (terminated)"),
|
None => format!("$ {cmd_summary} (terminated)"),
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if matches!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::Incomplete { .. }
|
||||||
|
) {
|
||||||
|
summary.push_str(" (output incomplete)");
|
||||||
|
}
|
||||||
Ok(BashResult {
|
Ok(BashResult {
|
||||||
summary,
|
summary,
|
||||||
stdout_preview: String::from_utf8_lossy(&stdout_preview).into_owned(),
|
stdout_preview: String::from_utf8_lossy(&stdout_preview).into_owned(),
|
||||||
|
|||||||
@@ -68,8 +68,10 @@ pub struct OutputRange {
|
|||||||
pub struct OutputTerminal {
|
pub struct OutputTerminal {
|
||||||
pub status: Option<CommandStatus>,
|
pub status: Option<CommandStatus>,
|
||||||
pub exit_code: Option<i32>,
|
pub exit_code: Option<i32>,
|
||||||
/// False on a forcibly dropped collector or failed transport: received bytes are retained,
|
pub process_failure: Option<workdir::CommandFailure>,
|
||||||
/// but they are not a claim about the unobserved process tail.
|
pub output_collection: Option<workdir::CommandOutputCollection>,
|
||||||
|
/// True only for confirmed EOF collection (quota is reported separately). A terminal
|
||||||
|
/// fence or exit zero alone does not confirm the unobserved process tail.
|
||||||
pub transport_complete: bool,
|
pub transport_complete: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -405,12 +407,9 @@ impl BashOutputWriter {
|
|||||||
terminal: &workdir::CommandTerminal,
|
terminal: &workdir::CommandTerminal,
|
||||||
) -> Result<BashOutputMetadata, String> {
|
) -> Result<BashOutputMetadata, String> {
|
||||||
let mut r = self.record.lock().unwrap();
|
let mut r = self.record.lock().unwrap();
|
||||||
if terminal.stdout_delivered_bytes != r.stdout_seen
|
if !terminal.metadata_valid()
|
||||||
|
|| terminal.stdout_delivered_bytes != r.stdout_seen
|
||||||
|| terminal.stderr_delivered_bytes != r.stderr_seen
|
|| terminal.stderr_delivered_bytes != r.stderr_seen
|
||||||
|| terminal.stdout_bytes < r.stdout_seen
|
|
||||||
|| terminal.stderr_bytes < r.stderr_seen
|
|
||||||
|| ((terminal.stdout_bytes != r.stdout_seen || terminal.stderr_bytes != r.stderr_seen)
|
|
||||||
!= terminal.truncation_reason.is_some())
|
|
||||||
{
|
{
|
||||||
return Err("command terminal counts do not fence delivered bytes".into());
|
return Err("command terminal counts do not fence delivered bytes".into());
|
||||||
}
|
}
|
||||||
@@ -419,7 +418,12 @@ impl BashOutputWriter {
|
|||||||
r.terminal = Some(OutputTerminal {
|
r.terminal = Some(OutputTerminal {
|
||||||
status: Some(terminal.status),
|
status: Some(terminal.status),
|
||||||
exit_code: terminal.exit_code,
|
exit_code: terminal.exit_code,
|
||||||
transport_complete: true,
|
transport_complete: matches!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::Complete
|
||||||
|
),
|
||||||
|
process_failure: terminal.process_failure,
|
||||||
|
output_collection: Some(terminal.output_collection),
|
||||||
});
|
});
|
||||||
Ok(r.metadata())
|
Ok(r.metadata())
|
||||||
}
|
}
|
||||||
@@ -439,6 +443,8 @@ impl Drop for BashOutputWriter {
|
|||||||
status: None,
|
status: None,
|
||||||
exit_code: None,
|
exit_code: None,
|
||||||
transport_complete: false,
|
transport_complete: false,
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: None,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -456,6 +462,8 @@ mod tests {
|
|||||||
status: Some(CommandStatus::Completed),
|
status: Some(CommandStatus::Completed),
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
transport_complete: true,
|
transport_complete: true,
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: Some(workdir::CommandOutputCollection::Complete),
|
||||||
})
|
})
|
||||||
.id
|
.id
|
||||||
}
|
}
|
||||||
@@ -514,6 +522,8 @@ mod tests {
|
|||||||
status: Some(CommandStatus::Completed),
|
status: Some(CommandStatus::Completed),
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
transport_complete: true,
|
transport_complete: true,
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: Some(workdir::CommandOutputCollection::Complete),
|
||||||
});
|
});
|
||||||
assert!(root.metadata(&metadata.id).is_some());
|
assert!(root.metadata(&metadata.id).is_some());
|
||||||
let _replacement = sibling.begin().unwrap();
|
let _replacement = sibling.begin().unwrap();
|
||||||
@@ -582,7 +592,7 @@ mod tests {
|
|||||||
let root = BashOutputView::new();
|
let root = BashOutputView::new();
|
||||||
let mut writer = root.begin().unwrap();
|
let mut writer = root.begin().unwrap();
|
||||||
writer.append(CommandStream::Stdout, b"abc");
|
writer.append(CommandStream::Stdout, b"abc");
|
||||||
let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"});
|
let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"complete"},"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"});
|
||||||
assert!(
|
assert!(
|
||||||
writer
|
writer
|
||||||
.finish_transport(&serde_json::from_value(terminal.clone()).unwrap())
|
.finish_transport(&serde_json::from_value(terminal.clone()).unwrap())
|
||||||
@@ -602,4 +612,41 @@ mod tests {
|
|||||||
Some(workdir::CommandOutputTruncation::ByteBudget)
|
Some(workdir::CommandOutputTruncation::ByteBudget)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn incomplete_collection_with_quota_retains_both_causes_without_claiming_complete_output() {
|
||||||
|
let root = BashOutputView::new();
|
||||||
|
let mut writer = root.begin().unwrap();
|
||||||
|
writer.append(CommandStream::Stdout, b"abc");
|
||||||
|
let terminal: workdir::CommandTerminal = serde_json::from_value(serde_json::json!({
|
||||||
|
"status":"completed", "exit_code":0, "process_failure":null,
|
||||||
|
"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},
|
||||||
|
"stdout_bytes":9, "stderr_bytes":0, "stdout_delivered_bytes":3, "stderr_delivered_bytes":0,
|
||||||
|
"truncation_reason":"byte_budget"
|
||||||
|
})).unwrap();
|
||||||
|
let metadata = writer.finish_transport(&terminal).unwrap();
|
||||||
|
assert!(!metadata.terminal.transport_complete);
|
||||||
|
assert_eq!(metadata.terminal.exit_code, Some(0));
|
||||||
|
assert_eq!(
|
||||||
|
metadata.terminal.output_collection,
|
||||||
|
Some(terminal.output_collection)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
metadata.transport_truncation,
|
||||||
|
Some(workdir::CommandOutputTruncation::ByteBudget)
|
||||||
|
);
|
||||||
|
assert_eq!(metadata.stdout_seen, 9);
|
||||||
|
assert_eq!(metadata.stdout_saved, 3);
|
||||||
|
assert!(metadata.storage_truncated);
|
||||||
|
assert!(metadata.retention_limit.is_none());
|
||||||
|
assert_eq!(
|
||||||
|
root.read(&metadata.id, CommandStream::Stdout, 0, 100)
|
||||||
|
.unwrap(),
|
||||||
|
b"abc"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(root.metadata(&metadata.id).unwrap()).unwrap(),
|
||||||
|
serde_json::to_value(metadata).unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -147,7 +147,9 @@ impl FileTarget {
|
|||||||
pub(crate) fn checked_error(error: workdir::WorkdirError) -> ToolError {
|
pub(crate) fn checked_error(error: workdir::WorkdirError) -> ToolError {
|
||||||
use workdir::WorkdirError;
|
use workdir::WorkdirError;
|
||||||
let code = match error.classification_source() {
|
let code = match error.classification_source() {
|
||||||
WorkdirError::DenialContext { .. } => unreachable!("diagnostic context unwrapped"),
|
WorkdirError::DenialContext { .. } | WorkdirError::FailureContext { .. } => {
|
||||||
|
unreachable!("diagnostic context unwrapped")
|
||||||
|
}
|
||||||
WorkdirError::Conflict(_) | WorkdirError::NotFound(_) => "checkout_stale",
|
WorkdirError::Conflict(_) | WorkdirError::NotFound(_) => "checkout_stale",
|
||||||
WorkdirError::Denied { .. }
|
WorkdirError::Denied { .. }
|
||||||
| WorkdirError::OutOfScope { .. }
|
| WorkdirError::OutOfScope { .. }
|
||||||
|
|||||||
@@ -416,6 +416,7 @@ struct ExternalValidatingSession {
|
|||||||
truncate_first_binary_response: bool,
|
truncate_first_binary_response: bool,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize,
|
command_response_losses: std::sync::atomic::AtomicUsize,
|
||||||
lose_terminal_only: bool,
|
lose_terminal_only: bool,
|
||||||
|
collection_override: Option<workdir::CommandOutputCollection>,
|
||||||
mutate_after_first_binary_response: Option<std::path::PathBuf>,
|
mutate_after_first_binary_response: Option<std::path::PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -534,7 +535,11 @@ impl workdir::WorkdirSession for ExternalValidatingSession {
|
|||||||
&self,
|
&self,
|
||||||
request: workdir::CommandOutputRequest,
|
request: workdir::CommandOutputRequest,
|
||||||
) -> Result<workdir::CommandOutput, workdir::WorkdirError> {
|
) -> Result<workdir::CommandOutput, workdir::WorkdirError> {
|
||||||
let output = self.inner.command_output(request).await?;
|
let mut output = self.inner.command_output(request).await?;
|
||||||
|
if let (Some(collection), Some(terminal)) = (self.collection_override, &mut output.terminal)
|
||||||
|
{
|
||||||
|
terminal.output_collection = collection;
|
||||||
|
}
|
||||||
if (!self.lose_terminal_only || output.terminal.is_some())
|
if (!self.lose_terminal_only || output.terminal.is_some())
|
||||||
&& self
|
&& self
|
||||||
.command_response_losses
|
.command_response_losses
|
||||||
@@ -546,9 +551,10 @@ impl workdir::WorkdirSession for ExternalValidatingSession {
|
|||||||
.is_ok()
|
.is_ok()
|
||||||
{
|
{
|
||||||
// The production provider has committed a page, but its response is lost.
|
// The production provider has committed a page, but its response is lost.
|
||||||
return Err(workdir::WorkdirError::Transport(
|
return Err(
|
||||||
"injected lost response".into(),
|
workdir::WorkdirError::Transport("injected lost response".into())
|
||||||
));
|
.with_failure_reason(workdir::WorkdirFailureReason::HttpTimeout),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
Ok(output)
|
Ok(output)
|
||||||
}
|
}
|
||||||
@@ -574,6 +580,7 @@ async fn external_view_image_uses_a_protocol_bounded_request() {
|
|||||||
truncate_first_binary_response: false,
|
truncate_first_binary_response: false,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||||
lose_terminal_only: false,
|
lose_terminal_only: false,
|
||||||
|
collection_override: None,
|
||||||
mutate_after_first_binary_response: None,
|
mutate_after_first_binary_response: None,
|
||||||
});
|
});
|
||||||
let png = include_bytes!("fixtures/view-image/valid.png");
|
let png = include_bytes!("fixtures/view-image/valid.png");
|
||||||
@@ -599,6 +606,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() {
|
|||||||
truncate_first_binary_response: true,
|
truncate_first_binary_response: true,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||||
lose_terminal_only: false,
|
lose_terminal_only: false,
|
||||||
|
collection_override: None,
|
||||||
mutate_after_first_binary_response: None,
|
mutate_after_first_binary_response: None,
|
||||||
});
|
});
|
||||||
let (_meta, tool) = view_image_tool(truncated)();
|
let (_meta, tool) = view_image_tool(truncated)();
|
||||||
@@ -617,6 +625,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() {
|
|||||||
truncate_first_binary_response: false,
|
truncate_first_binary_response: false,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||||
lose_terminal_only: false,
|
lose_terminal_only: false,
|
||||||
|
collection_override: None,
|
||||||
mutate_after_first_binary_response: Some(image_path),
|
mutate_after_first_binary_response: Some(image_path),
|
||||||
});
|
});
|
||||||
let (_meta, tool) = view_image_tool(changing)();
|
let (_meta, tool) = view_image_tool(changing)();
|
||||||
@@ -1103,7 +1112,20 @@ async fn bash_cancellation_returns_bounded_progress_as_terminal_output() {
|
|||||||
let execution = tokio::spawn(async move {
|
let execution = tokio::spawn(async move {
|
||||||
executing.execute(r#"{"command":"printf 'before'; printf 'err-before' >&2; while :; do :; done","cwd":"."}"#, execution_context).await
|
executing.execute(r#"{"command":"printf 'before'; printf 'err-before' >&2; while :; do :; done","cwd":"."}"#, execution_context).await
|
||||||
});
|
});
|
||||||
await_command_output(&mut events).await;
|
// Cancellation must follow both writes, not merely the first stdout chunk.
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||||
|
let mut ready = [false; 2];
|
||||||
|
while !ready.iter().all(|seen| *seen) {
|
||||||
|
if let workdir::CommandEvent::Output { stream, .. } = events.recv().await.unwrap() {
|
||||||
|
ready[match stream {
|
||||||
|
workdir::CommandStream::Stdout => 0,
|
||||||
|
workdir::CommandStream::Stderr => 1,
|
||||||
|
}] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("both streams written before cancellation");
|
||||||
bash.cancel_execution(&context).await.unwrap();
|
bash.cancel_execution(&context).await.unwrap();
|
||||||
let ToolError::Cancelled(output) =
|
let ToolError::Cancelled(output) =
|
||||||
tokio::time::timeout(std::time::Duration::from_secs(5), execution)
|
tokio::time::timeout(std::time::Duration::from_secs(5), execution)
|
||||||
@@ -1280,6 +1302,7 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap
|
|||||||
mutate_after_first_binary_response: None,
|
mutate_after_first_binary_response: None,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize::new(1),
|
command_response_losses: std::sync::atomic::AtomicUsize::new(1),
|
||||||
lose_terminal_only,
|
lose_terminal_only,
|
||||||
|
collection_override: None,
|
||||||
});
|
});
|
||||||
let view = tools::bash_output::BashOutputView::new();
|
let view = tools::bash_output::BashOutputView::new();
|
||||||
let definitions = core_builtin_tools(session.clone(), Tracker::new(), view.clone());
|
let definitions = core_builtin_tools(session.clone(), Tracker::new(), view.clone());
|
||||||
@@ -1314,3 +1337,80 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap
|
|||||||
session.close().await.unwrap();
|
session.close().await.unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Provider metadata is the fixture here; actual read/drain injection and HTTP
|
||||||
|
// forwarding are tested by workdir's production-supervisor suite.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn exit_zero_with_incomplete_output_keeps_partial_bytes_and_safe_failure_in_retained_evidence()
|
||||||
|
{
|
||||||
|
use workdir::{
|
||||||
|
CommandFailure, CommandFailureStage as Stage, CommandIoErrorKind, CommandOutputCollection,
|
||||||
|
};
|
||||||
|
for stage in [
|
||||||
|
Stage::StdoutRead,
|
||||||
|
Stage::StderrRead,
|
||||||
|
Stage::StdoutDrain,
|
||||||
|
Stage::StderrDrain,
|
||||||
|
Stage::OutputDelivery,
|
||||||
|
] {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let failure = CommandFailure {
|
||||||
|
stage,
|
||||||
|
reason: CommandIoErrorKind::PermissionDenied,
|
||||||
|
os_code: None,
|
||||||
|
};
|
||||||
|
let collection = CommandOutputCollection::Incomplete { failure };
|
||||||
|
let session: WorkdirSessionHandle = Arc::new(ExternalValidatingSession {
|
||||||
|
inner: LocalWorkdirSession::new(
|
||||||
|
Scope::writable(root.path()).unwrap(),
|
||||||
|
root.path().into(),
|
||||||
|
),
|
||||||
|
truncate_first_binary_response: false,
|
||||||
|
mutate_after_first_binary_response: None,
|
||||||
|
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||||
|
lose_terminal_only: false,
|
||||||
|
collection_override: Some(collection),
|
||||||
|
});
|
||||||
|
let view = tools::bash_output::BashOutputView::new();
|
||||||
|
let (_, bash) = tools::bash_tool(session.clone(), view.clone())();
|
||||||
|
let out = call(
|
||||||
|
&bash,
|
||||||
|
json!({"cwd":".","command":"printf partial; printf err >&2"}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(out.summary.ends_with("(output incomplete)"));
|
||||||
|
let body: serde_json::Value = serde_json::from_str(&out.content.unwrap()).unwrap();
|
||||||
|
let metadata = &body["output"];
|
||||||
|
assert_eq!(body["stdout_preview"], "partial");
|
||||||
|
assert_eq!(body["stderr_preview"], "err");
|
||||||
|
assert_eq!(body["preview_omitted"], false);
|
||||||
|
assert_eq!(metadata["terminal"]["exit_code"], 0);
|
||||||
|
assert_eq!(metadata["terminal"]["status"], "completed");
|
||||||
|
assert_eq!(metadata["terminal"]["transport_complete"], false);
|
||||||
|
assert_eq!(
|
||||||
|
metadata["terminal"]["output_collection"],
|
||||||
|
serde_json::to_value(collection).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(metadata["storage_truncated"], false);
|
||||||
|
assert_eq!(metadata["transport_truncation"], serde_json::Value::Null);
|
||||||
|
assert_eq!(metadata["stdout_seen"], 7);
|
||||||
|
assert_eq!(metadata["stdout_delivered"], 7);
|
||||||
|
let id = metadata["id"].as_str().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(view.metadata(id).unwrap()).unwrap(),
|
||||||
|
*metadata
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
view.read(id, workdir::CommandStream::Stdout, 0, 100)
|
||||||
|
.unwrap(),
|
||||||
|
b"partial"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
view.read(id, workdir::CommandStream::Stderr, 0, 100)
|
||||||
|
.unwrap(),
|
||||||
|
b"err"
|
||||||
|
);
|
||||||
|
assert!(!metadata.to_string().contains(root.path().to_str().unwrap()));
|
||||||
|
session.close().await.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,5 +23,6 @@ thiserror.workspace = true
|
|||||||
tokio = { workspace = true, features = ["io-util", "process", "rt", "sync", "time"] }
|
tokio = { workspace = true, features = ["io-util", "process", "rt", "sync", "time"] }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
axum.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
tempfile.workspace = true
|
tempfile.workspace = true
|
||||||
|
|||||||
@@ -525,6 +525,8 @@ pub struct ExternalWorkdirOperationError {
|
|||||||
code: crate::http::WorkdirTransportErrorCode,
|
code: crate::http::WorkdirTransportErrorCode,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
denial_reason: Option<crate::WorkdirDenialReason>,
|
denial_reason: Option<crate::WorkdirDenialReason>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
failure_reason: Option<crate::WorkdirFailureReason>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ExternalWorkdirOperationError {
|
impl ExternalWorkdirOperationError {
|
||||||
@@ -534,11 +536,13 @@ impl ExternalWorkdirOperationError {
|
|||||||
Self {
|
Self {
|
||||||
code: error.code,
|
code: error.code,
|
||||||
denial_reason: error.denial_reason,
|
denial_reason: error.denial_reason,
|
||||||
|
failure_reason: error.failure_reason,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn into_transport_error(self) -> WorkdirTransportError {
|
pub fn into_transport_error(self) -> WorkdirTransportError {
|
||||||
WorkdirTransportError {
|
WorkdirTransportError {
|
||||||
|
failure_reason: self.failure_reason,
|
||||||
code: self.code,
|
code: self.code,
|
||||||
denial_reason: self.denial_reason,
|
denial_reason: self.denial_reason,
|
||||||
message: format!("External Workdir provider reported {}", self.code.as_str()),
|
message: format!("External Workdir provider reported {}", self.code.as_str()),
|
||||||
@@ -762,6 +766,7 @@ mod tests {
|
|||||||
] {
|
] {
|
||||||
let external =
|
let external =
|
||||||
ExternalWorkdirOperationError::from_transport_error(WorkdirTransportError {
|
ExternalWorkdirOperationError::from_transport_error(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
code: Code::Denied,
|
code: Code::Denied,
|
||||||
message: "/home/operator/private provider-secret".repeat(4096),
|
message: "/home/operator/private provider-secret".repeat(4096),
|
||||||
denial_reason: reason,
|
denial_reason: reason,
|
||||||
@@ -788,6 +793,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn provider_failures_are_closed_and_drop_provider_authored_diagnostics() {
|
fn provider_failures_are_closed_and_drop_provider_authored_diagnostics() {
|
||||||
let error = ExternalWorkdirOperationError::from_transport_error(WorkdirTransportError {
|
let error = ExternalWorkdirOperationError::from_transport_error(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
code: crate::http::WorkdirTransportErrorCode::Internal,
|
code: crate::http::WorkdirTransportErrorCode::Internal,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
message: "/home/operator/private provider-secret\ncontrol".to_string(),
|
message: "/home/operator/private provider-secret\ncontrol".to_string(),
|
||||||
@@ -918,6 +924,8 @@ mod tests {
|
|||||||
chunks: vec![],
|
chunks: vec![],
|
||||||
next_cursor: 0,
|
next_cursor: 0,
|
||||||
terminal: Some(crate::CommandTerminal {
|
terminal: Some(crate::CommandTerminal {
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: crate::CommandOutputCollection::Complete,
|
||||||
status: crate::CommandStatus::Completed,
|
status: crate::CommandStatus::Completed,
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
stdout_bytes: 0,
|
stdout_bytes: 0,
|
||||||
@@ -1076,6 +1084,8 @@ mod tests {
|
|||||||
}],
|
}],
|
||||||
next_cursor: 3,
|
next_cursor: 3,
|
||||||
terminal: Some(crate::CommandTerminal {
|
terminal: Some(crate::CommandTerminal {
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: crate::CommandOutputCollection::Complete,
|
||||||
status: crate::CommandStatus::Completed,
|
status: crate::CommandStatus::Completed,
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
stdout_bytes: 100 * 1024 * 1024,
|
stdout_bytes: 100 * 1024 * 1024,
|
||||||
|
|||||||
+110
-34
@@ -275,6 +275,8 @@ pub struct WorkdirTransportError {
|
|||||||
/// Internal provider diagnostics. Public response boundaries may omit this.
|
/// Internal provider diagnostics. Public response boundaries may omit this.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub denial_reason: Option<crate::WorkdirDenialReason>,
|
pub denial_reason: Option<crate::WorkdirDenialReason>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub failure_reason: Option<crate::WorkdirFailureReason>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkdirTransportError {
|
impl WorkdirTransportError {
|
||||||
@@ -282,6 +284,11 @@ impl WorkdirTransportError {
|
|||||||
pub fn from_workdir_error(error: &WorkdirError) -> Self {
|
pub fn from_workdir_error(error: &WorkdirError) -> Self {
|
||||||
use WorkdirTransportErrorCode as Code;
|
use WorkdirTransportErrorCode as Code;
|
||||||
let (code, message) = match error {
|
let (code, message) = match error {
|
||||||
|
WorkdirError::FailureContext { reason, source } => {
|
||||||
|
let mut transport = Self::from_workdir_error(source);
|
||||||
|
transport.failure_reason = Some(*reason);
|
||||||
|
return transport;
|
||||||
|
}
|
||||||
WorkdirError::DenialContext { reason, source } => {
|
WorkdirError::DenialContext { reason, source } => {
|
||||||
let mut transport = Self::from_workdir_error(source);
|
let mut transport = Self::from_workdir_error(source);
|
||||||
transport.denial_reason = Some(*reason);
|
transport.denial_reason = Some(*reason);
|
||||||
@@ -301,6 +308,7 @@ impl WorkdirTransportError {
|
|||||||
code: Code::Unsupported,
|
code: Code::Unsupported,
|
||||||
message: format!("Workdir capability {capability:?} is not available"),
|
message: format!("Workdir capability {capability:?} is not available"),
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
|
failure_reason: None,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
WorkdirError::UnsupportedOperation(_) => {
|
WorkdirError::UnsupportedOperation(_) => {
|
||||||
@@ -359,6 +367,7 @@ impl WorkdirTransportError {
|
|||||||
code,
|
code,
|
||||||
message: message.to_string(),
|
message: message.to_string(),
|
||||||
denial_reason: error.denial_reason(),
|
denial_reason: error.denial_reason(),
|
||||||
|
failure_reason: error.failure_reason(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -408,6 +417,11 @@ impl WorkdirTransportError {
|
|||||||
Code::Transport => WorkdirError::Transport(self.message),
|
Code::Transport => WorkdirError::Transport(self.message),
|
||||||
Code::Internal => WorkdirError::OperationFailed,
|
Code::Internal => WorkdirError::OperationFailed,
|
||||||
};
|
};
|
||||||
|
let error = if let Some(reason) = self.failure_reason {
|
||||||
|
error.with_failure_reason(reason)
|
||||||
|
} else {
|
||||||
|
error
|
||||||
|
};
|
||||||
// Preserve any supplied reason independently of the public classification.
|
// Preserve any supplied reason independently of the public classification.
|
||||||
// Without a reason, retain the existing classification's deliberate fallback.
|
// Without a reason, retain the existing classification's deliberate fallback.
|
||||||
if let Some(reason) = self.denial_reason {
|
if let Some(reason) = self.denial_reason {
|
||||||
@@ -557,9 +571,10 @@ mod client {
|
|||||||
operation: WorkdirSessionOperation,
|
operation: WorkdirSessionOperation,
|
||||||
) -> Result<WorkdirSessionOperationResult, WorkdirError> {
|
) -> Result<WorkdirSessionOperationResult, WorkdirError> {
|
||||||
if self.closed.load(Ordering::Acquire) {
|
if self.closed.load(Ordering::Acquire) {
|
||||||
return Err(WorkdirError::Unavailable(
|
return Err(
|
||||||
"Workdir session is closed".to_string(),
|
WorkdirError::Unavailable("Workdir session is closed".to_string())
|
||||||
));
|
.with_failure_reason(crate::WorkdirFailureReason::SessionClosed),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
let url = endpoint(
|
let url = endpoint(
|
||||||
&self.base_url,
|
&self.base_url,
|
||||||
@@ -767,6 +782,10 @@ mod client {
|
|||||||
let outcome = WorkdirError::OutcomeUnknown(
|
let outcome = WorkdirError::OutcomeUnknown(
|
||||||
"checkout transport failed; effects may have occurred".into(),
|
"checkout transport failed; effects may have occurred".into(),
|
||||||
);
|
);
|
||||||
|
let outcome = match error.failure_reason() {
|
||||||
|
Some(reason) => outcome.with_failure_reason(reason),
|
||||||
|
None => outcome,
|
||||||
|
};
|
||||||
Err(match error.denial_reason() {
|
Err(match error.denial_reason() {
|
||||||
Some(reason) => WorkdirError::DenialContext {
|
Some(reason) => WorkdirError::DenialContext {
|
||||||
reason,
|
reason,
|
||||||
@@ -879,34 +898,43 @@ mod client {
|
|||||||
"command byte limit is outside transport bounds".into(),
|
"command byte limit is outside transport bounds".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let wait = request.wait;
|
loop {
|
||||||
match self
|
let started = tokio::time::Instant::now();
|
||||||
.operate(WorkdirSessionOperation::CommandOutput(request.clone()))
|
match self
|
||||||
.await?
|
.operate(WorkdirSessionOperation::CommandOutput(request.clone()))
|
||||||
{
|
.await?
|
||||||
WorkdirSessionOperationResult::CommandOutput(result) => {
|
{
|
||||||
let size = result
|
WorkdirSessionOperationResult::CommandOutput(result) => {
|
||||||
.chunks
|
let size = result
|
||||||
.iter()
|
.chunks
|
||||||
.try_fold(0usize, |size, chunk| size.checked_add(chunk.bytes.len()));
|
.iter()
|
||||||
if !result.transport_bounds_valid()
|
.try_fold(0usize, |size, chunk| size.checked_add(chunk.bytes.len()));
|
||||||
|| size.is_none_or(|size| {
|
if !result.transport_bounds_valid()
|
||||||
size > request.limit
|
|| size.is_none_or(|size| {
|
||||||
|| request.cursor.checked_add(size as u64)
|
size > request.limit
|
||||||
!= Some(result.next_cursor)
|
|| request.cursor.checked_add(size as u64)
|
||||||
})
|
!= Some(result.next_cursor)
|
||||||
{
|
})
|
||||||
return Err(Self::mismatch("bounded command byte page"));
|
{
|
||||||
|
return Err(Self::mismatch("bounded command byte page"));
|
||||||
|
}
|
||||||
|
if request.wait && result.chunks.is_empty() && result.terminal.is_none() {
|
||||||
|
// A normal long-poll expiry is not a transport failure.
|
||||||
|
// Continue at the same cursor, without spending the caller's
|
||||||
|
// error retry budget or acknowledging any bytes. Pace peers
|
||||||
|
// that return empty pages immediately; a real long poll has
|
||||||
|
// already consumed this delay. The caller owns cancellation
|
||||||
|
// and the overall deadline.
|
||||||
|
tokio::time::sleep_until(
|
||||||
|
started + std::time::Duration::from_millis(100),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Ok(result);
|
||||||
}
|
}
|
||||||
if wait && result.chunks.is_empty() && result.terminal.is_none() {
|
_ => return Err(Self::mismatch("command_output")),
|
||||||
// A waiting request must block at the provider. Never spin HTTP
|
|
||||||
// requests against an immediate empty response. A caller may retry
|
|
||||||
// this exact cursor with its own bounded transport retry policy.
|
|
||||||
return Err(Self::mismatch("waiting command byte page"));
|
|
||||||
}
|
|
||||||
return Ok(result);
|
|
||||||
}
|
}
|
||||||
_ => return Err(Self::mismatch("command_output")),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -966,8 +994,10 @@ mod client {
|
|||||||
) -> Result<T, WorkdirError> {
|
) -> Result<T, WorkdirError> {
|
||||||
if response.status().is_success() {
|
if response.status().is_success() {
|
||||||
let bytes = bounded_response_bytes(response).await?;
|
let bytes = bounded_response_bytes(response).await?;
|
||||||
serde_json::from_slice(&bytes)
|
serde_json::from_slice(&bytes).map_err(|_| {
|
||||||
.map_err(|_| WorkdirError::Transport("Runtime Workdir response was invalid".into()))
|
WorkdirError::Transport("Runtime Workdir response was invalid".into())
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::HttpResponseInvalid)
|
||||||
|
})
|
||||||
} else {
|
} else {
|
||||||
Err(decode_error(response).await)
|
Err(decode_error(response).await)
|
||||||
}
|
}
|
||||||
@@ -983,14 +1013,16 @@ mod client {
|
|||||||
{
|
{
|
||||||
return Err(WorkdirError::Transport(
|
return Err(WorkdirError::Transport(
|
||||||
"Runtime Workdir response exceeded transport bound".into(),
|
"Runtime Workdir response exceeded transport bound".into(),
|
||||||
));
|
)
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::HttpResponseTooLarge));
|
||||||
}
|
}
|
||||||
let mut bytes = Vec::new();
|
let mut bytes = Vec::new();
|
||||||
while let Some(chunk) = response.chunk().await.map_err(http_unavailable)? {
|
while let Some(chunk) = response.chunk().await.map_err(http_unavailable)? {
|
||||||
if chunk.len() > MAX_RESPONSE_BYTES.saturating_sub(bytes.len()) {
|
if chunk.len() > MAX_RESPONSE_BYTES.saturating_sub(bytes.len()) {
|
||||||
return Err(WorkdirError::Transport(
|
return Err(WorkdirError::Transport(
|
||||||
"Runtime Workdir response exceeded transport bound".into(),
|
"Runtime Workdir response exceeded transport bound".into(),
|
||||||
));
|
)
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::HttpResponseTooLarge));
|
||||||
}
|
}
|
||||||
bytes.extend_from_slice(&chunk);
|
bytes.extend_from_slice(&chunk);
|
||||||
}
|
}
|
||||||
@@ -1003,13 +1035,22 @@ mod client {
|
|||||||
.map(WorkdirTransportError::into_workdir_error)
|
.map(WorkdirTransportError::into_workdir_error)
|
||||||
.unwrap_or_else(|_| {
|
.unwrap_or_else(|_| {
|
||||||
WorkdirError::Transport("Runtime Workdir error response was invalid".into())
|
WorkdirError::Transport("Runtime Workdir error response was invalid".into())
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::HttpResponseInvalid)
|
||||||
}),
|
}),
|
||||||
Err(error) => error,
|
Err(error) => error,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn http_unavailable(_error: reqwest::Error) -> WorkdirError {
|
fn http_unavailable(error: reqwest::Error) -> WorkdirError {
|
||||||
|
let reason = if error.is_timeout() {
|
||||||
|
crate::WorkdirFailureReason::HttpTimeout
|
||||||
|
} else if error.is_connect() {
|
||||||
|
crate::WorkdirFailureReason::HttpConnect
|
||||||
|
} else {
|
||||||
|
crate::WorkdirFailureReason::HttpTransfer
|
||||||
|
};
|
||||||
WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string())
|
WorkdirError::Transport("Runtime Workdir HTTP request failed".to_string())
|
||||||
|
.with_failure_reason(reason)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -1017,6 +1058,40 @@ mod client {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn http_failure_reasons_do_not_copy_urls_or_credentials() {
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
let client = Client::builder().no_proxy().build().unwrap();
|
||||||
|
// A listening socket that never replies makes a real HTTP timeout.
|
||||||
|
let error = client
|
||||||
|
.get(format!("http://{address}/private?token=secret"))
|
||||||
|
.timeout(std::time::Duration::from_millis(25))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.is_timeout());
|
||||||
|
let error = http_unavailable(error);
|
||||||
|
assert_eq!(
|
||||||
|
error.failure_reason(),
|
||||||
|
Some(crate::WorkdirFailureReason::HttpTimeout)
|
||||||
|
);
|
||||||
|
assert!(!format!("{error:?}").contains("secret"));
|
||||||
|
assert!(!format!("{error:?}").contains("/private"));
|
||||||
|
drop(listener);
|
||||||
|
let error = client
|
||||||
|
.get(format!("http://{address}/private?token=secret"))
|
||||||
|
.timeout(std::time::Duration::from_secs(2))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.is_connect());
|
||||||
|
assert_eq!(
|
||||||
|
http_unavailable(error).failure_reason(),
|
||||||
|
Some(crate::WorkdirFailureReason::HttpConnect)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
struct RejectAuthorization {
|
struct RejectAuthorization {
|
||||||
requests: Arc<Mutex<Vec<(String, String, Vec<u8>)>>>,
|
requests: Arc<Mutex<Vec<(String, String, Vec<u8>)>>>,
|
||||||
@@ -1193,6 +1268,7 @@ mod tests {
|
|||||||
(WorkdirTransportErrorCode::Internal, 500, "operation failed"),
|
(WorkdirTransportErrorCode::Internal, 500, "operation failed"),
|
||||||
] {
|
] {
|
||||||
let transport = WorkdirTransportError {
|
let transport = WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
code,
|
code,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
message: if code == WorkdirTransportErrorCode::Conflict {
|
message: if code == WorkdirTransportErrorCode::Conflict {
|
||||||
|
|||||||
@@ -401,8 +401,44 @@ impl From<&str> for WorkdirDenial {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bounded operational diagnostics, never provider text, paths or credentials.
|
||||||
|
/// Reasons describe an observation, not retry permission or proof of no effects.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum WorkdirFailureReason {
|
||||||
|
SessionClosed,
|
||||||
|
ProviderUnavailable,
|
||||||
|
CommandTerminalMissing,
|
||||||
|
HttpTimeout,
|
||||||
|
HttpConnect,
|
||||||
|
HttpTransfer,
|
||||||
|
HttpResponseInvalid,
|
||||||
|
HttpResponseTooLarge,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkdirFailureReason {
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::SessionClosed => "session_closed",
|
||||||
|
Self::ProviderUnavailable => "provider_unavailable",
|
||||||
|
Self::CommandTerminalMissing => "command_terminal_missing",
|
||||||
|
Self::HttpTimeout => "http_timeout",
|
||||||
|
Self::HttpConnect => "http_connect",
|
||||||
|
Self::HttpTransfer => "http_transfer",
|
||||||
|
Self::HttpResponseInvalid => "http_response_invalid",
|
||||||
|
Self::HttpResponseTooLarge => "http_response_too_large",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum WorkdirError {
|
pub enum WorkdirError {
|
||||||
|
#[error("{source}")]
|
||||||
|
FailureContext {
|
||||||
|
reason: WorkdirFailureReason,
|
||||||
|
#[source]
|
||||||
|
source: Box<WorkdirError>,
|
||||||
|
},
|
||||||
#[error("Workdir operation denied: {0}")]
|
#[error("Workdir operation denied: {0}")]
|
||||||
Denied(WorkdirDenial),
|
Denied(WorkdirDenial),
|
||||||
|
|
||||||
@@ -521,12 +557,29 @@ impl WorkdirError {
|
|||||||
/// and `denial_reason`: enrichment must not alter behavior or retry policy.
|
/// and `denial_reason`: enrichment must not alter behavior or retry policy.
|
||||||
pub fn classification_source(&self) -> &Self {
|
pub fn classification_source(&self) -> &Self {
|
||||||
let mut error = self;
|
let mut error = self;
|
||||||
while let Self::DenialContext { source, .. } = error {
|
while let Self::DenialContext { source, .. } | Self::FailureContext { source, .. } = error {
|
||||||
error = source;
|
error = source;
|
||||||
}
|
}
|
||||||
error
|
error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn with_failure_reason(self, reason: WorkdirFailureReason) -> Self {
|
||||||
|
Self::FailureContext {
|
||||||
|
reason,
|
||||||
|
source: Box::new(self),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn failure_reason(&self) -> Option<WorkdirFailureReason> {
|
||||||
|
match self {
|
||||||
|
Self::FailureContext { reason, .. } => Some(*reason),
|
||||||
|
Self::DenialContext { source, .. } => source.failure_reason(),
|
||||||
|
Self::SessionClosed => Some(WorkdirFailureReason::SessionClosed),
|
||||||
|
Self::Unavailable(_) => Some(WorkdirFailureReason::ProviderUnavailable),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn denied(reason: WorkdirDenialReason, message: impl Into<String>) -> Self {
|
pub fn denied(reason: WorkdirDenialReason, message: impl Into<String>) -> Self {
|
||||||
Self::Denied(WorkdirDenial {
|
Self::Denied(WorkdirDenial {
|
||||||
reason: Some(reason),
|
reason: Some(reason),
|
||||||
@@ -540,6 +593,7 @@ impl WorkdirError {
|
|||||||
match self {
|
match self {
|
||||||
Self::Denied(denial) => denial.reason,
|
Self::Denied(denial) => denial.reason,
|
||||||
Self::DenialContext { reason, .. } => Some(*reason),
|
Self::DenialContext { reason, .. } => Some(*reason),
|
||||||
|
Self::FailureContext { source, .. } => source.denial_reason(),
|
||||||
Self::OutOfScope(_) => Some(Reason::PathOutOfScope),
|
Self::OutOfScope(_) => Some(Reason::PathOutOfScope),
|
||||||
Self::SymlinkOutOfScope { .. } => Some(Reason::SymlinkTargetOutOfScope),
|
Self::SymlinkOutOfScope { .. } => Some(Reason::SymlinkTargetOutOfScope),
|
||||||
Self::ReadOnly(_) => Some(Reason::PathReadOnly),
|
Self::ReadOnly(_) => Some(Reason::PathReadOnly),
|
||||||
|
|||||||
@@ -1210,7 +1210,8 @@ impl LocalWorkdirSession {
|
|||||||
Err(WorkdirError::Unavailable(format!(
|
Err(WorkdirError::Unavailable(format!(
|
||||||
"Workdir session for {} is closed",
|
"Workdir session for {} is closed",
|
||||||
self.inner.workdir.id()
|
self.inner.workdir.id()
|
||||||
)))
|
))
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::SessionClosed))
|
||||||
} else {
|
} else {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -2546,11 +2547,18 @@ mod tests {
|
|||||||
.expect("close should wake command output waiters")
|
.expect("close should wake command output waiters")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap_err();
|
.unwrap_err();
|
||||||
assert!(matches!(waiter_error, WorkdirError::Unavailable(_)));
|
assert!(matches!(
|
||||||
|
waiter_error.classification_source(),
|
||||||
|
WorkdirError::Unavailable(_)
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
waiter_error.failure_reason(),
|
||||||
|
Some(crate::WorkdirFailureReason::SessionClosed)
|
||||||
|
);
|
||||||
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
WorkdirSession::command_status(&session, command).await,
|
WorkdirSession::command_status(&session, command).await,
|
||||||
Err(WorkdirError::Unavailable(_))
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
));
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
WorkdirSession::read(
|
WorkdirSession::read(
|
||||||
@@ -2563,7 +2571,7 @@ mod tests {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await,
|
.await,
|
||||||
Err(WorkdirError::Unavailable(_))
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
));
|
));
|
||||||
|
|
||||||
let restored = LocalWorkdirSession::materialized_bound(
|
let restored = LocalWorkdirSession::materialized_bound(
|
||||||
@@ -3531,7 +3539,7 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
Err(WorkdirError::Unavailable(message)) if message.contains("is closed")
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(message) if message.contains("is closed"))
|
||||||
));
|
));
|
||||||
assert!(
|
assert!(
|
||||||
session.command_snapshot().is_empty(),
|
session.command_snapshot().is_empty(),
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
//! A replay page survives a lost HTTP response until its next cursor is sent.
|
//! A replay page survives a lost HTTP response until its next cursor is sent.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::{CommandFailure, CommandFailureStage, CommandIoErrorKind, CommandOutputCollection};
|
||||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
@@ -113,6 +114,7 @@ impl Transport {
|
|||||||
WorkdirError::Unavailable(
|
WorkdirError::Unavailable(
|
||||||
"command transport ended before terminal fence".into(),
|
"command transport ended before terminal fence".into(),
|
||||||
)
|
)
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::CommandTerminalMissing)
|
||||||
})?;
|
})?;
|
||||||
self.terminal = Some(terminal.clone());
|
self.terminal = Some(terminal.clone());
|
||||||
page.terminal = Some(terminal);
|
page.terminal = Some(terminal);
|
||||||
@@ -145,6 +147,33 @@ pub(super) fn start(
|
|||||||
telemetry: CommandTelemetry,
|
telemetry: CommandTelemetry,
|
||||||
environment: BTreeMap<String, String>,
|
environment: BTreeMap<String, String>,
|
||||||
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||||
|
) -> LocalCommand {
|
||||||
|
let execution_id = command_id.clone();
|
||||||
|
let execution_telemetry = telemetry.clone();
|
||||||
|
start_with(
|
||||||
|
command_id,
|
||||||
|
telemetry,
|
||||||
|
resources,
|
||||||
|
move |cancel, sender| async move {
|
||||||
|
execute(
|
||||||
|
cwd,
|
||||||
|
request,
|
||||||
|
&execution_id,
|
||||||
|
&execution_telemetry,
|
||||||
|
environment,
|
||||||
|
cancel,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn start_with<F: std::future::Future<Output = crate::CommandTerminal> + Send + 'static>(
|
||||||
|
command_id: String,
|
||||||
|
telemetry: CommandTelemetry,
|
||||||
|
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||||
|
execution: impl FnOnce(watch::Receiver<bool>, mpsc::Sender<Message>) -> F + Send + 'static,
|
||||||
) -> LocalCommand {
|
) -> LocalCommand {
|
||||||
let (sender, receiver) = mpsc::channel(QUEUE_CHUNKS);
|
let (sender, receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
let (cancel, cancel_rx) = watch::channel(false);
|
let (cancel, cancel_rx) = watch::channel(false);
|
||||||
@@ -159,16 +188,7 @@ pub(super) fn start(
|
|||||||
}));
|
}));
|
||||||
let task = tokio::spawn(async move {
|
let task = tokio::spawn(async move {
|
||||||
let _resources = resources;
|
let _resources = resources;
|
||||||
let terminal = execute(
|
let terminal = execution(cancel_rx, sender.clone()).await;
|
||||||
cwd,
|
|
||||||
request,
|
|
||||||
&command_id,
|
|
||||||
&telemetry,
|
|
||||||
environment,
|
|
||||||
cancel_rx,
|
|
||||||
&sender,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
telemetry.terminal(&command_id, terminal.status, terminal.exit_code);
|
telemetry.terminal(&command_id, terminal.status, terminal.exit_code);
|
||||||
// Closing the byte queue fences this already-published terminal, without
|
// Closing the byte queue fences this already-published terminal, without
|
||||||
// requiring spare queue capacity or a receiver to release task resources.
|
// requiring spare queue capacity or a receiver to release task resources.
|
||||||
@@ -191,10 +211,18 @@ struct ByteAccounting {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ByteAccounting {
|
impl ByteAccounting {
|
||||||
fn terminal(&self, status: CommandStatus, exit_code: Option<i32>) -> crate::CommandTerminal {
|
fn terminal(
|
||||||
|
&self,
|
||||||
|
status: CommandStatus,
|
||||||
|
exit_code: Option<i32>,
|
||||||
|
process_failure: Option<CommandFailure>,
|
||||||
|
output_collection: CommandOutputCollection,
|
||||||
|
) -> crate::CommandTerminal {
|
||||||
crate::CommandTerminal {
|
crate::CommandTerminal {
|
||||||
status,
|
status,
|
||||||
exit_code,
|
exit_code,
|
||||||
|
process_failure,
|
||||||
|
output_collection,
|
||||||
stdout_bytes: self.stdout,
|
stdout_bytes: self.stdout,
|
||||||
stderr_bytes: self.stderr,
|
stderr_bytes: self.stderr,
|
||||||
stdout_delivered_bytes: self.stdout_delivered,
|
stdout_delivered_bytes: self.stdout_delivered,
|
||||||
@@ -214,7 +242,7 @@ async fn pump<R: AsyncRead + Unpin>(
|
|||||||
command_id: &str,
|
command_id: &str,
|
||||||
mut remaining: u64,
|
mut remaining: u64,
|
||||||
accounting: &mut ByteAccounting,
|
accounting: &mut ByteAccounting,
|
||||||
) -> std::io::Result<()> {
|
) -> Result<(), CommandFailure> {
|
||||||
let mut stdout_done = false;
|
let mut stdout_done = false;
|
||||||
let mut stderr_done = false;
|
let mut stderr_done = false;
|
||||||
let mut stdout_decoder = CommandOutputDecoder::default();
|
let mut stdout_decoder = CommandOutputDecoder::default();
|
||||||
@@ -227,7 +255,11 @@ async fn pump<R: AsyncRead + Unpin>(
|
|||||||
// Before quota, reserve first to retain bounded pipe backpressure.
|
// Before quota, reserve first to retain bounded pipe backpressure.
|
||||||
let permit = if remaining > 0 {
|
let permit = if remaining > 0 {
|
||||||
let Ok(permit) = sender.reserve().await else {
|
let Ok(permit) = sender.reserve().await else {
|
||||||
return Ok(());
|
return Err(CommandFailure {
|
||||||
|
stage: CommandFailureStage::OutputDelivery,
|
||||||
|
reason: CommandIoErrorKind::Unknown,
|
||||||
|
os_code: None,
|
||||||
|
});
|
||||||
};
|
};
|
||||||
Some(permit)
|
Some(permit)
|
||||||
} else {
|
} else {
|
||||||
@@ -241,7 +273,15 @@ async fn pump<R: AsyncRead + Unpin>(
|
|||||||
CommandStream::Stdout => &stdout_buffer,
|
CommandStream::Stdout => &stdout_buffer,
|
||||||
CommandStream::Stderr => &stderr_buffer,
|
CommandStream::Stderr => &stderr_buffer,
|
||||||
};
|
};
|
||||||
let count = result?;
|
let count = result.map_err(|error| {
|
||||||
|
CommandFailure::io(
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => CommandFailureStage::StdoutRead,
|
||||||
|
CommandStream::Stderr => CommandFailureStage::StderrRead,
|
||||||
|
},
|
||||||
|
&error,
|
||||||
|
)
|
||||||
|
})?;
|
||||||
let decoder = match stream {
|
let decoder = match stream {
|
||||||
CommandStream::Stdout => &mut stdout_decoder,
|
CommandStream::Stdout => &mut stdout_decoder,
|
||||||
CommandStream::Stderr => &mut stderr_decoder,
|
CommandStream::Stderr => &mut stderr_decoder,
|
||||||
@@ -287,7 +327,7 @@ async fn execute(
|
|||||||
command_id: &str,
|
command_id: &str,
|
||||||
telemetry: &CommandTelemetry,
|
telemetry: &CommandTelemetry,
|
||||||
environment: BTreeMap<String, String>,
|
environment: BTreeMap<String, String>,
|
||||||
mut cancel: watch::Receiver<bool>,
|
cancel: watch::Receiver<bool>,
|
||||||
sender: &mpsc::Sender<Message>,
|
sender: &mpsc::Sender<Message>,
|
||||||
) -> crate::CommandTerminal {
|
) -> crate::CommandTerminal {
|
||||||
telemetry.started(command_id, request.tool_call_id);
|
telemetry.started(command_id, request.tool_call_id);
|
||||||
@@ -303,74 +343,159 @@ async fn execute(
|
|||||||
.kill_on_drop(true);
|
.kill_on_drop(true);
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
command.as_std_mut().process_group(0);
|
command.as_std_mut().process_group(0);
|
||||||
let mut accounting = ByteAccounting::default();
|
let mut child = match command.spawn() {
|
||||||
let Ok(mut child) = command.spawn() else {
|
Ok(child) => child,
|
||||||
return accounting.terminal(CommandStatus::Failed, None);
|
Err(error) => {
|
||||||
|
return ByteAccounting::default().terminal(
|
||||||
|
CommandStatus::Failed,
|
||||||
|
None,
|
||||||
|
Some(CommandFailure::io(CommandFailureStage::Spawn, &error)),
|
||||||
|
CommandOutputCollection::NotStarted,
|
||||||
|
);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let mut process_group = CommandProcessGroupGuard::new(child.id());
|
|
||||||
let stdout = child.stdout.take().expect("piped stdout");
|
let stdout = child.stdout.take().expect("piped stdout");
|
||||||
let stderr = child.stderr.take().expect("piped stderr");
|
let stderr = child.stderr.take().expect("piped stderr");
|
||||||
let (status, exit_code) = {
|
supervise(
|
||||||
|
child,
|
||||||
|
stdout,
|
||||||
|
stderr,
|
||||||
|
request.timeout_secs,
|
||||||
|
request.transport_byte_budget,
|
||||||
|
command_id,
|
||||||
|
telemetry,
|
||||||
|
cancel,
|
||||||
|
sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
// Substitute only syscalls and readers: injected faults run the production supervisor.
|
||||||
|
trait CommandProcess: Send {
|
||||||
|
fn id(&self) -> Option<u32>;
|
||||||
|
fn start_kill(&mut self);
|
||||||
|
fn wait(
|
||||||
|
&mut self,
|
||||||
|
) -> impl std::future::Future<Output = std::io::Result<std::process::ExitStatus>> + Send;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CommandProcess for tokio::process::Child {
|
||||||
|
fn id(&self) -> Option<u32> {
|
||||||
|
self.id()
|
||||||
|
}
|
||||||
|
fn start_kill(&mut self) {
|
||||||
|
let _ = self.start_kill();
|
||||||
|
}
|
||||||
|
async fn wait(&mut self) -> std::io::Result<std::process::ExitStatus> {
|
||||||
|
self.wait().await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process_result(
|
||||||
|
result: std::io::Result<std::process::ExitStatus>,
|
||||||
|
) -> (CommandStatus, Option<i32>, Option<CommandFailure>) {
|
||||||
|
match result {
|
||||||
|
Ok(exit) => (
|
||||||
|
if exit.success() {
|
||||||
|
CommandStatus::Completed
|
||||||
|
} else {
|
||||||
|
CommandStatus::Failed
|
||||||
|
},
|
||||||
|
exit.code(),
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
Err(error) => (
|
||||||
|
CommandStatus::Failed,
|
||||||
|
None,
|
||||||
|
Some(CommandFailure::io(CommandFailureStage::Wait, &error)),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn supervise(
|
||||||
|
mut child: impl CommandProcess,
|
||||||
|
stdout: impl AsyncRead + Unpin,
|
||||||
|
stderr: impl AsyncRead + Unpin,
|
||||||
|
timeout_secs: u64,
|
||||||
|
byte_budget: u64,
|
||||||
|
command_id: &str,
|
||||||
|
telemetry: &CommandTelemetry,
|
||||||
|
mut cancel: watch::Receiver<bool>,
|
||||||
|
sender: &mpsc::Sender<Message>,
|
||||||
|
) -> crate::CommandTerminal {
|
||||||
|
let mut process_group = CommandProcessGroupGuard::new(child.id());
|
||||||
|
let mut accounting = ByteAccounting::default();
|
||||||
|
let (status, exit_code, process_failure, output_collection) = {
|
||||||
let output = pump(
|
let output = pump(
|
||||||
stdout,
|
stdout,
|
||||||
stderr,
|
stderr,
|
||||||
sender,
|
sender,
|
||||||
telemetry,
|
telemetry,
|
||||||
command_id,
|
command_id,
|
||||||
request.transport_byte_budget,
|
byte_budget,
|
||||||
&mut accounting,
|
&mut accounting,
|
||||||
);
|
);
|
||||||
tokio::pin!(output);
|
tokio::pin!(output);
|
||||||
let mut drained = false;
|
let mut collected = None;
|
||||||
let timeout = tokio::time::sleep(Duration::from_secs(request.timeout_secs.max(1)));
|
let timeout = tokio::time::sleep(Duration::from_secs(timeout_secs.max(1)));
|
||||||
tokio::pin!(timeout);
|
tokio::pin!(timeout);
|
||||||
let terminal = loop {
|
let terminal = loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
result = &mut output, if !drained => {
|
result = &mut output, if collected.is_none() => {
|
||||||
drained = true;
|
let failed = result.is_err();
|
||||||
if result.is_err() || sender.is_closed() {
|
collected = Some(result);
|
||||||
|
if failed {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
let _ = child.start_kill();
|
child.start_kill();
|
||||||
let _ = child.wait().await;
|
break process_result(child.wait().await);
|
||||||
break (CommandStatus::Failed, None);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
exit = child.wait() => {
|
exit = child.wait() => {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
break match exit {
|
break process_result(exit);
|
||||||
Ok(exit) => (if exit.success() { CommandStatus::Completed } else { CommandStatus::Failed }, exit.code()),
|
|
||||||
Err(_) => (CommandStatus::Failed, None),
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
_ = &mut timeout => {
|
_ = &mut timeout => {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
let _ = child.start_kill();
|
child.start_kill();
|
||||||
let code = child.wait().await.ok().and_then(|exit| exit.code());
|
let (_, code, failure) = process_result(child.wait().await);
|
||||||
break (CommandStatus::TimedOut, code);
|
break (CommandStatus::TimedOut, code, failure);
|
||||||
}
|
}
|
||||||
_ = sender.closed() => {
|
_ = sender.closed() => {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
let _ = child.start_kill();
|
child.start_kill();
|
||||||
let _ = child.wait().await;
|
let (_, code, failure) = process_result(child.wait().await);
|
||||||
break (CommandStatus::Cancelled, None);
|
break (CommandStatus::Cancelled, code, failure);
|
||||||
}
|
}
|
||||||
changed = cancel.changed() => {
|
changed = cancel.changed() => {
|
||||||
if changed.is_err() || *cancel.borrow() {
|
if changed.is_err() || *cancel.borrow() {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
let _ = child.start_kill();
|
child.start_kill();
|
||||||
let code = child.wait().await.ok().and_then(|exit| exit.code());
|
let (_, code, failure) = process_result(child.wait().await);
|
||||||
break (CommandStatus::Cancelled, code);
|
break (CommandStatus::Cancelled, code, failure);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
// Even after kill, preserve the bounded pipe tail before publishing terminal.
|
// Exit is not pipe EOF. Retain read failures first observed during drain.
|
||||||
if !drained {
|
let result = match collected {
|
||||||
let _ = output.await;
|
Some(result) => result,
|
||||||
}
|
None => output.await.map_err(|mut failure| {
|
||||||
terminal
|
failure.stage = match failure.stage {
|
||||||
|
CommandFailureStage::StdoutRead => CommandFailureStage::StdoutDrain,
|
||||||
|
CommandFailureStage::StderrRead => CommandFailureStage::StderrDrain,
|
||||||
|
stage => stage,
|
||||||
|
};
|
||||||
|
failure
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let collection = match result {
|
||||||
|
Ok(()) => CommandOutputCollection::Complete,
|
||||||
|
Err(failure) => CommandOutputCollection::Incomplete { failure },
|
||||||
|
};
|
||||||
|
(terminal.0, terminal.1, terminal.2, collection)
|
||||||
};
|
};
|
||||||
accounting.terminal(status, exit_code)
|
accounting.terminal(status, exit_code, process_failure, output_collection)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -4,6 +4,38 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn missing_terminal_is_diagnosed_without_changing_error_classification() {
|
||||||
|
let (sender, receiver) = mpsc::channel(1);
|
||||||
|
let (_status, terminal_status) = watch::channel(None);
|
||||||
|
drop(sender);
|
||||||
|
let mut transport = Transport {
|
||||||
|
receiver,
|
||||||
|
pending: None,
|
||||||
|
cursor: 0,
|
||||||
|
replay: None,
|
||||||
|
terminal: None,
|
||||||
|
terminal_status,
|
||||||
|
};
|
||||||
|
let error = transport
|
||||||
|
.read(&CommandOutputRequest {
|
||||||
|
handle: CommandHandle("diagnostic-test".into()),
|
||||||
|
cursor: 0,
|
||||||
|
limit: 16,
|
||||||
|
wait: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
error.failure_reason(),
|
||||||
|
Some(crate::WorkdirFailureReason::CommandTerminalMissing)
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
error.classification_source(),
|
||||||
|
WorkdirError::Unavailable(_)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
struct Harness {
|
struct Harness {
|
||||||
session: LocalWorkdirSession,
|
session: LocalWorkdirSession,
|
||||||
_root: TempDir,
|
_root: TempDir,
|
||||||
@@ -488,3 +520,349 @@ async fn slow_receiver_backpressures_before_quota_but_full_queue_does_not_block_
|
|||||||
assert_eq!(queued_bytes, QUEUE_CHUNKS * COMMAND_EVENT_CHUNK_BYTES);
|
assert_eq!(queued_bytes, QUEUE_CHUNKS * COMMAND_EVENT_CHUNK_BYTES);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct Gate(Arc<std::sync::Mutex<(bool, Option<std::task::Waker>)>>);
|
||||||
|
|
||||||
|
impl Gate {
|
||||||
|
fn open(&self) {
|
||||||
|
let mut state = self.0.lock().unwrap();
|
||||||
|
state.0 = true;
|
||||||
|
if let Some(waker) = state.1.take() {
|
||||||
|
waker.wake();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll(&self, cx: &mut std::task::Context<'_>) -> std::task::Poll<()> {
|
||||||
|
let mut state = self.0.lock().unwrap();
|
||||||
|
if state.0 {
|
||||||
|
std::task::Poll::Ready(())
|
||||||
|
} else {
|
||||||
|
state.1 = Some(cx.waker().clone());
|
||||||
|
std::task::Poll::Pending
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FaultProcess {
|
||||||
|
ready: Gate,
|
||||||
|
exited: Gate,
|
||||||
|
wait_error: Option<std::io::ErrorKind>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CommandProcess for FaultProcess {
|
||||||
|
fn id(&self) -> Option<u32> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
fn start_kill(&mut self) {
|
||||||
|
self.ready.open();
|
||||||
|
}
|
||||||
|
async fn wait(&mut self) -> std::io::Result<std::process::ExitStatus> {
|
||||||
|
std::future::poll_fn(|cx| self.ready.poll(cx)).await;
|
||||||
|
self.exited.open();
|
||||||
|
match self.wait_error {
|
||||||
|
Some(kind) => Err(std::io::Error::new(
|
||||||
|
kind,
|
||||||
|
"/private/path TOKEN=secret command-body",
|
||||||
|
)),
|
||||||
|
None => Ok(std::process::ExitStatus::default()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FaultReader {
|
||||||
|
prefix: &'static [u8],
|
||||||
|
prefix_read: Gate,
|
||||||
|
fail_ready: Gate,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncRead for FaultReader {
|
||||||
|
fn poll_read(
|
||||||
|
mut self: std::pin::Pin<&mut Self>,
|
||||||
|
cx: &mut std::task::Context<'_>,
|
||||||
|
buffer: &mut tokio::io::ReadBuf<'_>,
|
||||||
|
) -> std::task::Poll<std::io::Result<()>> {
|
||||||
|
if !self.prefix.is_empty() {
|
||||||
|
let count = self.prefix.len().min(buffer.remaining());
|
||||||
|
buffer.put_slice(&self.prefix[..count]);
|
||||||
|
self.prefix = &self.prefix[count..];
|
||||||
|
self.prefix_read.open();
|
||||||
|
return std::task::Poll::Ready(Ok(()));
|
||||||
|
}
|
||||||
|
std::task::ready!(self.fail_ready.poll(cx));
|
||||||
|
std::task::Poll::Ready(Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::PermissionDenied,
|
||||||
|
"/private/path TOKEN=secret command-body",
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Faults replace only OS wait/read boundaries, not collection or finalization.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_and_post_exit_drain_faults_keep_prefix_exit_and_safe_original_reason() {
|
||||||
|
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||||
|
for after_exit in [false, true] {
|
||||||
|
let ready = Gate::default();
|
||||||
|
let exited = Gate::default();
|
||||||
|
let fail_ready = if after_exit {
|
||||||
|
exited.clone()
|
||||||
|
} else {
|
||||||
|
let gate = Gate::default();
|
||||||
|
gate.open();
|
||||||
|
gate
|
||||||
|
};
|
||||||
|
let reader = FaultReader {
|
||||||
|
prefix: b"partial",
|
||||||
|
prefix_read: if after_exit {
|
||||||
|
ready.clone()
|
||||||
|
} else {
|
||||||
|
Gate::default()
|
||||||
|
},
|
||||||
|
fail_ready,
|
||||||
|
};
|
||||||
|
let process = FaultProcess {
|
||||||
|
ready,
|
||||||
|
exited,
|
||||||
|
wait_error: None,
|
||||||
|
};
|
||||||
|
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
|
let (_cancel, cancel_rx) = watch::channel(false);
|
||||||
|
let telemetry = CommandTelemetry::new();
|
||||||
|
let terminal = tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
reader,
|
||||||
|
&b""[..],
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"fault",
|
||||||
|
&telemetry,
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
CommandStream::Stderr => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
&b""[..],
|
||||||
|
reader,
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"fault",
|
||||||
|
&telemetry,
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let Message::Bytes(chunk) = receiver.recv().await.unwrap();
|
||||||
|
assert_eq!(chunk.bytes, b"partial");
|
||||||
|
assert_eq!(chunk.stream, stream);
|
||||||
|
assert_eq!(chunk.sequence, 0);
|
||||||
|
assert_eq!(chunk.offset, 0);
|
||||||
|
assert!(receiver.try_recv().is_err());
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Completed);
|
||||||
|
assert_eq!(terminal.exit_code, Some(0));
|
||||||
|
assert_eq!(terminal.process_failure, None);
|
||||||
|
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||||
|
7
|
||||||
|
);
|
||||||
|
assert_eq!(terminal.truncation_reason, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Incomplete {
|
||||||
|
failure: CommandFailure {
|
||||||
|
stage: match (stream, after_exit) {
|
||||||
|
(CommandStream::Stdout, false) => CommandFailureStage::StdoutRead,
|
||||||
|
(CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain,
|
||||||
|
(CommandStream::Stderr, false) => CommandFailureStage::StderrRead,
|
||||||
|
(CommandStream::Stderr, true) => CommandFailureStage::StderrDrain,
|
||||||
|
},
|
||||||
|
reason: CommandIoErrorKind::PermissionDenied,
|
||||||
|
os_code: None
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let wire = serde_json::to_string(&terminal).unwrap();
|
||||||
|
for secret in ["/private", "TOKEN", "secret", "command-body"] {
|
||||||
|
assert!(!wire.contains(secret));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn wait_fault_keeps_collected_bytes_and_does_not_invent_exit_information() {
|
||||||
|
let ready = Gate::default();
|
||||||
|
ready.open();
|
||||||
|
let process = FaultProcess {
|
||||||
|
ready,
|
||||||
|
exited: Gate::default(),
|
||||||
|
wait_error: Some(std::io::ErrorKind::InvalidData),
|
||||||
|
};
|
||||||
|
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
|
let (_cancel, cancel_rx) = watch::channel(false);
|
||||||
|
let terminal = supervise(
|
||||||
|
process,
|
||||||
|
&b"out"[..],
|
||||||
|
&b"err"[..],
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"wait-fault",
|
||||||
|
&CommandTelemetry::new(),
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.exit_code, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.process_failure,
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Wait,
|
||||||
|
reason: CommandIoErrorKind::InvalidData,
|
||||||
|
os_code: None
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Complete
|
||||||
|
);
|
||||||
|
assert_eq!(terminal.stdout_bytes, 3);
|
||||||
|
assert_eq!(terminal.stderr_bytes, 3);
|
||||||
|
let mut chunks = Vec::new();
|
||||||
|
while let Ok(Message::Bytes(chunk)) = receiver.try_recv() {
|
||||||
|
chunks.push(chunk);
|
||||||
|
}
|
||||||
|
assert_eq!(stream_bytes(&chunks, CommandStream::Stdout), b"out");
|
||||||
|
assert_eq!(stream_bytes(&chunks, CommandStream::Stderr), b"err");
|
||||||
|
assert!(!serde_json::to_string(&terminal).unwrap().contains("secret"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn spawn_fault_retains_safe_reason_without_claiming_output_collection() {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let (sender, _receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
|
let (_cancel, cancel_rx) = watch::channel(false);
|
||||||
|
let terminal = execute(
|
||||||
|
root.path().join("missing-secret-path"),
|
||||||
|
Harness::request("secret-command", 10, 100),
|
||||||
|
"spawn-fault",
|
||||||
|
&CommandTelemetry::new(),
|
||||||
|
BTreeMap::new(),
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
terminal.process_failure,
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Spawn,
|
||||||
|
reason: CommandIoErrorKind::NotFound,
|
||||||
|
os_code: Some(libc::ENOENT)
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::NotStarted
|
||||||
|
);
|
||||||
|
assert_eq!(terminal.exit_code, None);
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 0);
|
||||||
|
assert!(!serde_json::to_string(&terminal).unwrap().contains("secret"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-client")]
|
||||||
|
mod http;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn drain_failure_and_byte_budget_keep_independent_loss_diagnostics() {
|
||||||
|
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||||
|
let ready = Gate::default();
|
||||||
|
let exited = Gate::default();
|
||||||
|
let process = FaultProcess {
|
||||||
|
ready: ready.clone(),
|
||||||
|
exited: exited.clone(),
|
||||||
|
wait_error: None,
|
||||||
|
};
|
||||||
|
let reader = FaultReader {
|
||||||
|
prefix: b"partial",
|
||||||
|
prefix_read: ready,
|
||||||
|
fail_ready: exited,
|
||||||
|
};
|
||||||
|
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
|
let (_cancel, cancel_rx) = watch::channel(false);
|
||||||
|
let telemetry = CommandTelemetry::new();
|
||||||
|
let terminal = tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
reader,
|
||||||
|
&b""[..],
|
||||||
|
10,
|
||||||
|
3,
|
||||||
|
"quota-fault",
|
||||||
|
&telemetry,
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
CommandStream::Stderr => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
&b""[..],
|
||||||
|
reader,
|
||||||
|
10,
|
||||||
|
3,
|
||||||
|
"quota-fault",
|
||||||
|
&telemetry,
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let Message::Bytes(chunk) = receiver.recv().await.unwrap();
|
||||||
|
assert_eq!(chunk.stream, stream);
|
||||||
|
assert_eq!(chunk.bytes, b"par");
|
||||||
|
assert!(receiver.try_recv().is_err());
|
||||||
|
assert_eq!(terminal.exit_code, Some(0));
|
||||||
|
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||||
|
3
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.truncation_reason,
|
||||||
|
Some(crate::CommandOutputTruncation::ByteBudget)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Incomplete {
|
||||||
|
failure: CommandFailure {
|
||||||
|
stage: match stream {
|
||||||
|
CommandStream::Stdout => CommandFailureStage::StdoutDrain,
|
||||||
|
CommandStream::Stderr => CommandFailureStage::StderrDrain,
|
||||||
|
},
|
||||||
|
reason: CommandIoErrorKind::PermissionDenied,
|
||||||
|
os_code: None,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert!(terminal.metadata_valid());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,238 @@
|
|||||||
|
//! Loopback integration of the real supervisor, byte queue/replay, local session,
|
||||||
|
//! HTTP dispatcher/client. OS wait/read are
|
||||||
|
//! deterministic faults; session open and start bind that injected process fixture.
|
||||||
|
//! Authorization is not asserted here (Runtime's signed HTTP suite owns it).
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::http::*;
|
||||||
|
use axum::{Json, Router, extract::State, routing::post};
|
||||||
|
|
||||||
|
struct Server(tokio::task::JoinHandle<()>);
|
||||||
|
impl Drop for Server {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn operation(
|
||||||
|
State(session): State<Arc<LocalWorkdirSession>>,
|
||||||
|
Json(request): Json<WorkdirSessionOperationRequest>,
|
||||||
|
) -> Json<WorkdirSessionOperationResult> {
|
||||||
|
if matches!(request.operation, WorkdirSessionOperation::CommandStart(_)) {
|
||||||
|
return Json(WorkdirSessionOperationResult::CommandStart(CommandHandle(
|
||||||
|
"injected".into(),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Json(
|
||||||
|
dispatch_workdir_session_operation(session.as_ref(), request.operation)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pipe_and_wait_faults_cross_real_http_with_prefix_and_terminal_replay() {
|
||||||
|
tokio::time::timeout(Duration::from_secs(20), async {
|
||||||
|
for (stream, after_exit, wait_fault) in [
|
||||||
|
(CommandStream::Stdout, false, false),
|
||||||
|
(CommandStream::Stderr, false, false),
|
||||||
|
(CommandStream::Stdout, true, false),
|
||||||
|
(CommandStream::Stderr, true, false),
|
||||||
|
(CommandStream::Stdout, false, true),
|
||||||
|
] {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let session = Arc::new(LocalWorkdirSession::new(
|
||||||
|
Scope::writable(root.path()).unwrap(),
|
||||||
|
root.path().into(),
|
||||||
|
));
|
||||||
|
let ready = Gate::default();
|
||||||
|
let exited = Gate::default();
|
||||||
|
if wait_fault {
|
||||||
|
ready.open();
|
||||||
|
}
|
||||||
|
let process = FaultProcess {
|
||||||
|
ready: ready.clone(),
|
||||||
|
exited: exited.clone(),
|
||||||
|
wait_error: wait_fault.then_some(std::io::ErrorKind::InvalidData),
|
||||||
|
};
|
||||||
|
let fail_ready = if after_exit {
|
||||||
|
exited
|
||||||
|
} else {
|
||||||
|
let gate = Gate::default();
|
||||||
|
gate.open();
|
||||||
|
gate
|
||||||
|
};
|
||||||
|
let reader = FaultReader {
|
||||||
|
prefix: b"partial",
|
||||||
|
prefix_read: if after_exit { ready } else { Gate::default() },
|
||||||
|
fail_ready,
|
||||||
|
};
|
||||||
|
let command = start_with(
|
||||||
|
"injected".into(),
|
||||||
|
CommandTelemetry::new(),
|
||||||
|
vec![],
|
||||||
|
move |cancel, sender| async move {
|
||||||
|
let telemetry = CommandTelemetry::new();
|
||||||
|
if wait_fault {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
&b"partial"[..],
|
||||||
|
&b""[..],
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"injected",
|
||||||
|
&telemetry,
|
||||||
|
cancel,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
reader,
|
||||||
|
&b""[..],
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"injected",
|
||||||
|
&telemetry,
|
||||||
|
cancel,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
CommandStream::Stderr => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
&b""[..],
|
||||||
|
reader,
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"injected",
|
||||||
|
&telemetry,
|
||||||
|
cancel,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
session
|
||||||
|
.inner
|
||||||
|
.commands
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.insert("injected".into(), command);
|
||||||
|
let app = Router::new()
|
||||||
|
.route(
|
||||||
|
"/v1/working-directories/{id}/sessions",
|
||||||
|
post(|| async {
|
||||||
|
Json(OpenWorkdirSessionResponse {
|
||||||
|
session_id: WorkdirSessionId::new("fault-session").unwrap(),
|
||||||
|
workdir_id: crate::Workdir::new("fault-workdir").id().clone(),
|
||||||
|
capabilities: crate::WorkdirSessionCapabilities::ALL,
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route("/v1/workdir-sessions/{id}/operations", post(operation))
|
||||||
|
.with_state(session.clone());
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
let _server = Server(tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
}));
|
||||||
|
let remote: crate::WorkdirSessionHandle = Arc::new(
|
||||||
|
RemoteWorkdirSession::open(
|
||||||
|
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||||
|
base.parse().unwrap(),
|
||||||
|
"fixture-token",
|
||||||
|
crate::Workdir::new("fault-workdir").id().clone(),
|
||||||
|
OpenWorkdirSessionRequest::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let handle = CommandHandle("injected".into());
|
||||||
|
let mut request = CommandOutputRequest {
|
||||||
|
handle,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 1,
|
||||||
|
wait: true,
|
||||||
|
};
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
let terminal = loop {
|
||||||
|
let page = remote.command_output(request.clone()).await.unwrap();
|
||||||
|
assert_eq!(remote.command_output(request.clone()).await.unwrap(), page);
|
||||||
|
for chunk in &page.chunks {
|
||||||
|
assert_eq!(chunk.sequence, request.cursor);
|
||||||
|
assert_eq!(chunk.offset, request.cursor);
|
||||||
|
assert_eq!(chunk.stream, stream);
|
||||||
|
bytes.extend_from_slice(&chunk.bytes);
|
||||||
|
request.cursor += chunk.bytes.len() as u64;
|
||||||
|
}
|
||||||
|
assert_eq!(page.next_cursor, request.cursor);
|
||||||
|
if let Some(terminal) = page.terminal {
|
||||||
|
break terminal;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
assert_eq!(bytes, b"partial");
|
||||||
|
assert_eq!(terminal.truncation_reason, None);
|
||||||
|
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||||
|
7
|
||||||
|
);
|
||||||
|
if wait_fault {
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.exit_code, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.process_failure,
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Wait,
|
||||||
|
reason: CommandIoErrorKind::InvalidData,
|
||||||
|
os_code: None
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Complete
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Completed);
|
||||||
|
assert_eq!(terminal.exit_code, Some(0));
|
||||||
|
let stage = match (stream, after_exit) {
|
||||||
|
(CommandStream::Stdout, false) => CommandFailureStage::StdoutRead,
|
||||||
|
(CommandStream::Stderr, false) => CommandFailureStage::StderrRead,
|
||||||
|
(CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain,
|
||||||
|
(CommandStream::Stderr, true) => CommandFailureStage::StderrDrain,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Incomplete {
|
||||||
|
failure: CommandFailure {
|
||||||
|
stage,
|
||||||
|
reason: CommandIoErrorKind::PermissionDenied,
|
||||||
|
os_code: None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let wire = serde_json::to_string(&terminal).unwrap();
|
||||||
|
for secret in [
|
||||||
|
"/private",
|
||||||
|
"TOKEN",
|
||||||
|
"secret",
|
||||||
|
"command-body",
|
||||||
|
"fixture-token",
|
||||||
|
] {
|
||||||
|
assert!(!wire.contains(secret));
|
||||||
|
}
|
||||||
|
session.close().await.unwrap();
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded loopback fault projection");
|
||||||
|
}
|
||||||
+257
-10
@@ -132,12 +132,151 @@ pub enum CommandOutputTruncation {
|
|||||||
ByteBudget,
|
ByteBudget,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Raw totals include discarded pipe bytes. Delivered totals count only prefixes
|
/// Closed diagnostic vocabulary: never serialize OS messages, paths, or errno text.
|
||||||
/// admitted to the transport queue, not page reads or retries by a consumer.
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CommandIoErrorKind {
|
||||||
|
NotFound,
|
||||||
|
PermissionDenied,
|
||||||
|
ConnectionRefused,
|
||||||
|
ConnectionReset,
|
||||||
|
ConnectionAborted,
|
||||||
|
NotConnected,
|
||||||
|
BrokenPipe,
|
||||||
|
AlreadyExists,
|
||||||
|
WouldBlock,
|
||||||
|
InvalidInput,
|
||||||
|
InvalidData,
|
||||||
|
TimedOut,
|
||||||
|
WriteZero,
|
||||||
|
Interrupted,
|
||||||
|
UnexpectedEof,
|
||||||
|
OutOfMemory,
|
||||||
|
HostUnreachable,
|
||||||
|
NetworkUnreachable,
|
||||||
|
AddrInUse,
|
||||||
|
AddrNotAvailable,
|
||||||
|
NetworkDown,
|
||||||
|
NotADirectory,
|
||||||
|
IsADirectory,
|
||||||
|
DirectoryNotEmpty,
|
||||||
|
ReadOnlyFilesystem,
|
||||||
|
StaleNetworkFileHandle,
|
||||||
|
StorageFull,
|
||||||
|
NotSeekable,
|
||||||
|
QuotaExceeded,
|
||||||
|
FileTooLarge,
|
||||||
|
ResourceBusy,
|
||||||
|
ExecutableFileBusy,
|
||||||
|
Deadlock,
|
||||||
|
CrossesDevices,
|
||||||
|
TooManyLinks,
|
||||||
|
InvalidFilename,
|
||||||
|
ArgumentListTooLong,
|
||||||
|
Unsupported,
|
||||||
|
Other,
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<std::io::ErrorKind> for CommandIoErrorKind {
|
||||||
|
fn from(kind: std::io::ErrorKind) -> Self {
|
||||||
|
use std::io::ErrorKind as Io;
|
||||||
|
match kind {
|
||||||
|
Io::NotFound => Self::NotFound,
|
||||||
|
Io::PermissionDenied => Self::PermissionDenied,
|
||||||
|
Io::ConnectionRefused => Self::ConnectionRefused,
|
||||||
|
Io::ConnectionReset => Self::ConnectionReset,
|
||||||
|
Io::ConnectionAborted => Self::ConnectionAborted,
|
||||||
|
Io::NotConnected => Self::NotConnected,
|
||||||
|
Io::BrokenPipe => Self::BrokenPipe,
|
||||||
|
Io::AlreadyExists => Self::AlreadyExists,
|
||||||
|
Io::WouldBlock => Self::WouldBlock,
|
||||||
|
Io::InvalidInput => Self::InvalidInput,
|
||||||
|
Io::InvalidData => Self::InvalidData,
|
||||||
|
Io::TimedOut => Self::TimedOut,
|
||||||
|
Io::WriteZero => Self::WriteZero,
|
||||||
|
Io::Interrupted => Self::Interrupted,
|
||||||
|
Io::UnexpectedEof => Self::UnexpectedEof,
|
||||||
|
Io::OutOfMemory => Self::OutOfMemory,
|
||||||
|
Io::HostUnreachable => Self::HostUnreachable,
|
||||||
|
Io::NetworkUnreachable => Self::NetworkUnreachable,
|
||||||
|
Io::AddrInUse => Self::AddrInUse,
|
||||||
|
Io::AddrNotAvailable => Self::AddrNotAvailable,
|
||||||
|
Io::NetworkDown => Self::NetworkDown,
|
||||||
|
Io::NotADirectory => Self::NotADirectory,
|
||||||
|
Io::IsADirectory => Self::IsADirectory,
|
||||||
|
Io::DirectoryNotEmpty => Self::DirectoryNotEmpty,
|
||||||
|
Io::ReadOnlyFilesystem => Self::ReadOnlyFilesystem,
|
||||||
|
Io::StaleNetworkFileHandle => Self::StaleNetworkFileHandle,
|
||||||
|
Io::StorageFull => Self::StorageFull,
|
||||||
|
Io::NotSeekable => Self::NotSeekable,
|
||||||
|
Io::QuotaExceeded => Self::QuotaExceeded,
|
||||||
|
Io::FileTooLarge => Self::FileTooLarge,
|
||||||
|
Io::ResourceBusy => Self::ResourceBusy,
|
||||||
|
Io::ExecutableFileBusy => Self::ExecutableFileBusy,
|
||||||
|
Io::Deadlock => Self::Deadlock,
|
||||||
|
Io::CrossesDevices => Self::CrossesDevices,
|
||||||
|
Io::TooManyLinks => Self::TooManyLinks,
|
||||||
|
Io::InvalidFilename => Self::InvalidFilename,
|
||||||
|
Io::ArgumentListTooLong => Self::ArgumentListTooLong,
|
||||||
|
Io::Unsupported => Self::Unsupported,
|
||||||
|
Io::Other => Self::Other,
|
||||||
|
_ => Self::Unknown,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CommandFailureStage {
|
||||||
|
Spawn,
|
||||||
|
Wait,
|
||||||
|
StdoutRead,
|
||||||
|
StderrRead,
|
||||||
|
StdoutDrain,
|
||||||
|
StderrDrain,
|
||||||
|
OutputDelivery,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct CommandFailure {
|
||||||
|
pub stage: CommandFailureStage,
|
||||||
|
pub reason: CommandIoErrorKind,
|
||||||
|
/// Observed numeric OS code, if supplied by the source. Never inferred from text.
|
||||||
|
pub os_code: Option<i32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CommandFailure {
|
||||||
|
pub(crate) fn io(stage: CommandFailureStage, error: &std::io::Error) -> Self {
|
||||||
|
Self {
|
||||||
|
stage,
|
||||||
|
reason: error.kind().into(),
|
||||||
|
os_code: error.raw_os_error(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// EOF collection is independent of exit status and the explicit byte budget.
|
||||||
|
/// Incomplete counts are lower bounds on bytes actually produced by the process.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)]
|
||||||
|
pub enum CommandOutputCollection {
|
||||||
|
NotStarted,
|
||||||
|
Complete,
|
||||||
|
Incomplete { failure: CommandFailure },
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Raw totals include discarded *observed* pipe bytes, not an unknown failed tail.
|
||||||
|
/// Delivered totals count only prefixes admitted to the transport queue, not page
|
||||||
|
/// reads or retries. Neither terminal arrival nor exit zero implies EOF collection.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct CommandTerminal {
|
pub struct CommandTerminal {
|
||||||
pub status: CommandStatus,
|
pub status: CommandStatus,
|
||||||
pub exit_code: Option<i32>,
|
pub exit_code: Option<i32>,
|
||||||
|
pub process_failure: Option<CommandFailure>,
|
||||||
|
pub output_collection: CommandOutputCollection,
|
||||||
pub stdout_bytes: u64,
|
pub stdout_bytes: u64,
|
||||||
pub stderr_bytes: u64,
|
pub stderr_bytes: u64,
|
||||||
pub stdout_delivered_bytes: u64,
|
pub stdout_delivered_bytes: u64,
|
||||||
@@ -145,6 +284,54 @@ pub struct CommandTerminal {
|
|||||||
pub truncation_reason: Option<CommandOutputTruncation>,
|
pub truncation_reason: Option<CommandOutputTruncation>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl CommandTerminal {
|
||||||
|
/// Validate provider-authored metadata before publishing it as collected evidence.
|
||||||
|
pub fn metadata_valid(&self) -> bool {
|
||||||
|
let truncated = self.stdout_bytes > self.stdout_delivered_bytes
|
||||||
|
|| self.stderr_bytes > self.stderr_delivered_bytes;
|
||||||
|
if self.status == CommandStatus::Running
|
||||||
|
|| self.stdout_delivered_bytes > self.stdout_bytes
|
||||||
|
|| self.stderr_delivered_bytes > self.stderr_bytes
|
||||||
|
|| self.truncation_reason != truncated.then_some(CommandOutputTruncation::ByteBudget)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
match self.process_failure {
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Spawn,
|
||||||
|
..
|
||||||
|
}) => {
|
||||||
|
self.status == CommandStatus::Failed
|
||||||
|
&& self.exit_code.is_none()
|
||||||
|
&& self.output_collection == CommandOutputCollection::NotStarted
|
||||||
|
&& self.stdout_bytes == 0
|
||||||
|
&& self.stderr_bytes == 0
|
||||||
|
}
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Wait,
|
||||||
|
..
|
||||||
|
}) => {
|
||||||
|
self.status != CommandStatus::Completed
|
||||||
|
&& self.exit_code.is_none()
|
||||||
|
&& self.collection_started()
|
||||||
|
}
|
||||||
|
Some(_) => false,
|
||||||
|
None => self.collection_started(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collection_started(&self) -> bool {
|
||||||
|
match self.output_collection {
|
||||||
|
CommandOutputCollection::NotStarted => false,
|
||||||
|
CommandOutputCollection::Complete => true,
|
||||||
|
CommandOutputCollection::Incomplete { failure } => !matches!(
|
||||||
|
failure.stage,
|
||||||
|
CommandFailureStage::Spawn | CommandFailureStage::Wait
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Sending next_cursor acknowledges this page. Only the current page may be
|
/// Sending next_cursor acknowledges this page. Only the current page may be
|
||||||
/// retried; acknowledged bytes expire. Terminal fences all captured bytes,
|
/// retried; acknowledged bytes expire. Terminal fences all captured bytes,
|
||||||
/// including on timeout/cancel. Consumers must pull while the process runs.
|
/// including on timeout/cancel. Consumers must pull while the process runs.
|
||||||
@@ -186,19 +373,13 @@ impl CommandOutput {
|
|||||||
self.next_cursor <= MAX_COMMAND_TRANSPORT_BYTES
|
self.next_cursor <= MAX_COMMAND_TRANSPORT_BYTES
|
||||||
&& end.is_none_or(|end| end == self.next_cursor)
|
&& end.is_none_or(|end| end == self.next_cursor)
|
||||||
&& self.terminal.as_ref().is_none_or(|terminal| {
|
&& self.terminal.as_ref().is_none_or(|terminal| {
|
||||||
let truncated = terminal.stdout_bytes > terminal.stdout_delivered_bytes
|
terminal.metadata_valid()
|
||||||
|| terminal.stderr_bytes > terminal.stderr_delivered_bytes;
|
&& stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes)
|
||||||
stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes)
|
|
||||||
&& stream_ends[1].is_none_or(|end| end == terminal.stderr_delivered_bytes)
|
&& stream_ends[1].is_none_or(|end| end == terminal.stderr_delivered_bytes)
|
||||||
&& terminal.status != CommandStatus::Running
|
|
||||||
&& terminal.stdout_delivered_bytes <= terminal.stdout_bytes
|
|
||||||
&& terminal.stderr_delivered_bytes <= terminal.stderr_bytes
|
|
||||||
&& terminal
|
&& terminal
|
||||||
.stdout_delivered_bytes
|
.stdout_delivered_bytes
|
||||||
.checked_add(terminal.stderr_delivered_bytes)
|
.checked_add(terminal.stderr_delivered_bytes)
|
||||||
== Some(self.next_cursor)
|
== Some(self.next_cursor)
|
||||||
&& terminal.truncation_reason
|
|
||||||
== truncated.then_some(CommandOutputTruncation::ByteBudget)
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -262,4 +443,70 @@ mod tests {
|
|||||||
.contains(&serde_json::json!("transport_byte_budget"))
|
.contains(&serde_json::json!("transport_byte_budget"))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failure_wire_is_closed_and_does_not_infer_os_codes_from_custom_messages() {
|
||||||
|
let error = std::io::Error::new(
|
||||||
|
std::io::ErrorKind::Other,
|
||||||
|
"/host/private TOKEN=secret EACCES command",
|
||||||
|
);
|
||||||
|
let failure = CommandFailure::io(CommandFailureStage::Wait, &error);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(failure).unwrap(),
|
||||||
|
serde_json::json!({"stage":"wait","reason":"other","os_code":null})
|
||||||
|
);
|
||||||
|
let os_error = std::io::Error::from_raw_os_error(libc::EIO);
|
||||||
|
let actual = CommandFailure::io(CommandFailureStage::StdoutDrain, &os_error);
|
||||||
|
assert_eq!(actual.os_code, Some(libc::EIO));
|
||||||
|
for invalid in [
|
||||||
|
serde_json::json!({"stage":"wait","reason":"/host/private TOKEN=secret"}),
|
||||||
|
serde_json::json!({"stage":"/host/private","reason":"other"}),
|
||||||
|
serde_json::json!({"stage":"wait","reason":"other","message":"secret"}),
|
||||||
|
] {
|
||||||
|
assert!(serde_json::from_value::<CommandFailure>(invalid).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_terminal_rejects_inconsistent_failure_domains_and_unstarted_byte_claims() {
|
||||||
|
let wire = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},"stdout_bytes":3,"stderr_bytes":0,"stdout_delivered_bytes":3,"stderr_delivered_bytes":0,"truncation_reason":null});
|
||||||
|
let terminal: CommandTerminal = serde_json::from_value(wire.clone()).unwrap();
|
||||||
|
assert!(terminal.metadata_valid());
|
||||||
|
for (field, value) in [
|
||||||
|
(
|
||||||
|
"output_collection",
|
||||||
|
serde_json::json!({"state":"not_started"}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"output_collection",
|
||||||
|
serde_json::json!({"state":"incomplete","failure":{"stage":"wait","reason":"other"}}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"process_failure",
|
||||||
|
serde_json::json!({"stage":"stdout_read","reason":"other"}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"process_failure",
|
||||||
|
serde_json::json!({"stage":"spawn","reason":"not_found"}),
|
||||||
|
),
|
||||||
|
("truncation_reason", serde_json::json!("byte_budget")),
|
||||||
|
] {
|
||||||
|
let mut invalid = wire.clone();
|
||||||
|
invalid[field] = value;
|
||||||
|
assert!(
|
||||||
|
!serde_json::from_value::<CommandTerminal>(invalid)
|
||||||
|
.unwrap()
|
||||||
|
.metadata_valid()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut unclassified = wire.clone();
|
||||||
|
unclassified
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.remove("output_collection");
|
||||||
|
assert!(serde_json::from_value::<CommandTerminal>(unclassified).is_err());
|
||||||
|
let mut raw = wire;
|
||||||
|
raw["message"] = serde_json::json!("/private TOKEN=secret");
|
||||||
|
assert!(serde_json::from_value::<CommandTerminal>(raw).is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -842,7 +842,7 @@ mod tests {
|
|||||||
assert!(router.is_empty());
|
assert!(router.is_empty());
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
underlying.command_status(handle).await,
|
underlying.command_status(handle).await,
|
||||||
Err(WorkdirError::Unavailable(_))
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
use workdir::external::ExternalWorkdirOperationError;
|
||||||
|
use workdir::http::{WorkdirTransportError, WorkdirTransportErrorCode as Code};
|
||||||
|
use workdir::{WorkdirError, WorkdirFailureReason as Reason};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failure_reasons_survive_internal_transports_without_provider_text() {
|
||||||
|
for reason in [
|
||||||
|
Reason::SessionClosed,
|
||||||
|
Reason::ProviderUnavailable,
|
||||||
|
Reason::CommandTerminalMissing,
|
||||||
|
Reason::HttpTimeout,
|
||||||
|
Reason::HttpConnect,
|
||||||
|
Reason::HttpTransfer,
|
||||||
|
Reason::HttpResponseInvalid,
|
||||||
|
Reason::HttpResponseTooLarge,
|
||||||
|
] {
|
||||||
|
let source = WorkdirError::Unavailable("secret-token /private/path".into())
|
||||||
|
.with_failure_reason(reason);
|
||||||
|
let transport = WorkdirTransportError::from_workdir_error(&source);
|
||||||
|
let wire = serde_json::to_string(&transport).unwrap();
|
||||||
|
assert!(!wire.contains("secret-token"));
|
||||||
|
assert!(!wire.contains("/private/path"));
|
||||||
|
assert_eq!(transport.code, Code::Unavailable);
|
||||||
|
assert_eq!(transport.code.http_status(), 503);
|
||||||
|
assert_eq!(transport.message, "Workdir session is unavailable");
|
||||||
|
assert_eq!(serde_json::to_value(reason).unwrap(), reason.as_str());
|
||||||
|
let decoded: WorkdirTransportError = serde_json::from_str(&wire).unwrap();
|
||||||
|
let external = ExternalWorkdirOperationError::from_transport_error(decoded);
|
||||||
|
let external_wire = serde_json::to_string(&external).unwrap();
|
||||||
|
let external: ExternalWorkdirOperationError = serde_json::from_str(&external_wire).unwrap();
|
||||||
|
let error = external.into_transport_error().into_workdir_error();
|
||||||
|
assert_eq!(error.failure_reason(), Some(reason));
|
||||||
|
assert!(matches!(
|
||||||
|
error.classification_source(),
|
||||||
|
WorkdirError::Unavailable(_)
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
WorkdirTransportError::from_workdir_error(&error).failure_reason,
|
||||||
|
Some(reason)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_errors_and_nested_context_preserve_classification() {
|
||||||
|
let wire = r#"{"code":"unavailable","message":"Workdir session is unavailable"}"#;
|
||||||
|
let old: WorkdirTransportError = serde_json::from_str(wire).unwrap();
|
||||||
|
assert!(old.failure_reason.is_none());
|
||||||
|
assert!(matches!(
|
||||||
|
old.into_workdir_error(),
|
||||||
|
WorkdirError::Unavailable(_)
|
||||||
|
));
|
||||||
|
let closed = WorkdirTransportError::from_workdir_error(&WorkdirError::SessionClosed);
|
||||||
|
assert_eq!(closed.failure_reason, Some(Reason::SessionClosed));
|
||||||
|
// Reason-looking text never becomes a typed diagnosis.
|
||||||
|
let generic = WorkdirError::Unavailable("http_timeout".into());
|
||||||
|
assert_eq!(generic.failure_reason(), Some(Reason::ProviderUnavailable));
|
||||||
|
let error = WorkdirError::DenialContext {
|
||||||
|
reason: workdir::WorkdirDenialReason::OsPermissionDenied,
|
||||||
|
source: Box::new(
|
||||||
|
WorkdirError::OutcomeUnknown("do not retry".into())
|
||||||
|
.with_failure_reason(Reason::HttpTransfer),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
let error = WorkdirTransportError::from_workdir_error(&error).into_workdir_error();
|
||||||
|
assert!(matches!(
|
||||||
|
error.classification_source(),
|
||||||
|
WorkdirError::OutcomeUnknown(_)
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
error.denial_reason(),
|
||||||
|
Some(workdir::WorkdirDenialReason::OsPermissionDenied)
|
||||||
|
);
|
||||||
|
assert_eq!(error.failure_reason(), Some(Reason::HttpTransfer));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_cannot_inject_free_text_failure_reason() {
|
||||||
|
assert!(
|
||||||
|
serde_json::from_str::<WorkdirTransportError>(
|
||||||
|
r#"{"code":"unavailable","message":"x","failure_reason":"secret /path"}"#
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -56,6 +56,7 @@ server-api = { path = "../server-api" }
|
|||||||
workdir.workspace = true
|
workdir.workspace = true
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
tools.workspace = true
|
||||||
workdir = { workspace = true, features = ["http-client"] }
|
workdir = { workspace = true, features = ["http-client"] }
|
||||||
futures.workspace = true
|
futures.workspace = true
|
||||||
agen.workspace = true
|
agen.workspace = true
|
||||||
|
|||||||
@@ -2146,6 +2146,7 @@ impl RuntimeHttpWorkdirError {
|
|||||||
Self {
|
Self {
|
||||||
status,
|
status,
|
||||||
payload: WorkdirTransportError {
|
payload: WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code,
|
code,
|
||||||
message: message.into(),
|
message: message.into(),
|
||||||
@@ -2214,6 +2215,16 @@ impl From<workdir::WorkdirError> for RuntimeHttpWorkdirError {
|
|||||||
|
|
||||||
impl IntoResponse for RuntimeHttpWorkdirError {
|
impl IntoResponse for RuntimeHttpWorkdirError {
|
||||||
fn into_response(self) -> Response {
|
fn into_response(self) -> Response {
|
||||||
|
if let Some(reason) = self.payload.failure_reason {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "worker_runtime::workdir",
|
||||||
|
event = "workdir_provider_failure",
|
||||||
|
status = self.status.as_u16(),
|
||||||
|
code = self.payload.code.as_str(),
|
||||||
|
failure_reason = reason.as_str(),
|
||||||
|
"Workdir provider operation failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
(self.status, Json(self.payload)).into_response()
|
(self.status, Json(self.payload)).into_response()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3195,9 +3206,13 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect_err("close must be terminal");
|
.expect_err("close must be terminal");
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
closed_error,
|
closed_error.classification_source(),
|
||||||
workdir::WorkdirError::Unavailable(_)
|
workdir::WorkdirError::Unavailable(_)
|
||||||
));
|
));
|
||||||
|
assert_eq!(
|
||||||
|
closed_error.failure_reason(),
|
||||||
|
Some(workdir::WorkdirFailureReason::SessionClosed)
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
state
|
state
|
||||||
.workdir_sessions
|
.workdir_sessions
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
//! pre-seeded session is a fixture; no Runtime Worker/materializer is claimed here.
|
//! pre-seeded session is a fixture; no Runtime Worker/materializer is claimed here.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use workdir::http::{
|
use workdir::http::{
|
||||||
RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
|
RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
|
||||||
@@ -70,14 +71,23 @@ impl Harness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn new() -> Self {
|
async fn new() -> Self {
|
||||||
let root = tempfile::tempdir().unwrap();
|
Self::with_environment(tempfile::tempdir().unwrap(), BTreeMap::new()).await
|
||||||
let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound(
|
}
|
||||||
Workdir::new("wd-transport"),
|
|
||||||
root.path().to_path_buf(),
|
async fn with_environment(
|
||||||
root.path().to_path_buf(),
|
root: tempfile::TempDir,
|
||||||
SharedScope::new(Scope::writable(root.path()).unwrap()),
|
command_environment: BTreeMap<String, String>,
|
||||||
WorkdirSessionCapabilities::ALL,
|
) -> Self {
|
||||||
));
|
let session: WorkdirSessionHandle =
|
||||||
|
Arc::new(LocalWorkdirSession::materialized_bound_with_environment(
|
||||||
|
Workdir::new("wd-transport"),
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
SharedScope::new(Scope::writable(root.path()).unwrap()),
|
||||||
|
WorkdirSessionCapabilities::ALL,
|
||||||
|
command_environment,
|
||||||
|
Vec::new(),
|
||||||
|
));
|
||||||
let identity = RuntimeIdentityMaterial::generate("transport-key").unwrap();
|
let identity = RuntimeIdentityMaterial::generate("transport-key").unwrap();
|
||||||
let trust = workspace_trust(&identity);
|
let trust = workspace_trust(&identity);
|
||||||
let state = RuntimeHttpState {
|
let state = RuntimeHttpState {
|
||||||
@@ -156,6 +166,336 @@ impl Harness {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn real_http_waiting_command_survives_long_poll_expiry() {
|
||||||
|
tokio::time::timeout(Duration::from_secs(45), async {
|
||||||
|
let harness = Harness::new().await;
|
||||||
|
let handle = harness
|
||||||
|
.remote
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
transport_byte_budget: 1024,
|
||||||
|
command: "printf ready; while [ ! -e release ]; do sleep 0.05; done; printf after; printf err >&2; exit 7".into(),
|
||||||
|
timeout_secs: 40,
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
tool_call_id: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut request = CommandOutputRequest {
|
||||||
|
handle,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 1024,
|
||||||
|
wait: true,
|
||||||
|
};
|
||||||
|
let ready = harness.remote.command_output(request.clone()).await.unwrap();
|
||||||
|
assert_eq!(ready.chunks.iter().flat_map(|c| c.bytes.iter().copied()).collect::<Vec<_>>(), b"ready");
|
||||||
|
assert!(ready.terminal.is_none());
|
||||||
|
request.cursor = ready.next_cursor;
|
||||||
|
|
||||||
|
// Keep the command silent until the production HTTP handler has expired
|
||||||
|
// its 20-second long poll. The client must keep waiting, not fail or
|
||||||
|
// expose an empty page as completed output.
|
||||||
|
let waiting = harness.remote.command_output(request.clone());
|
||||||
|
tokio::pin!(waiting);
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut waiting => panic!("waiting read ended before release: {result:?}"),
|
||||||
|
_ = tokio::time::sleep(Duration::from_secs(21)) => {}
|
||||||
|
}
|
||||||
|
std::fs::write(harness._root.path().join("release"), b"").unwrap();
|
||||||
|
let mut page = waiting.await.unwrap();
|
||||||
|
let mut stdout = Vec::new();
|
||||||
|
let mut stderr = Vec::new();
|
||||||
|
let terminal = loop {
|
||||||
|
// The retry cursor still replays the exact nonempty/terminal page.
|
||||||
|
assert_eq!(harness.remote.command_output(request.clone()).await.unwrap(), page);
|
||||||
|
for chunk in &page.chunks {
|
||||||
|
assert_eq!(chunk.sequence, request.cursor);
|
||||||
|
request.cursor += chunk.bytes.len() as u64;
|
||||||
|
match chunk.stream {
|
||||||
|
CommandStream::Stdout => stdout.extend_from_slice(&chunk.bytes),
|
||||||
|
CommandStream::Stderr => stderr.extend_from_slice(&chunk.bytes),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(page.next_cursor, request.cursor);
|
||||||
|
if let Some(terminal) = page.terminal {
|
||||||
|
break terminal;
|
||||||
|
}
|
||||||
|
page = harness.remote.command_output(request.clone()).await.unwrap();
|
||||||
|
};
|
||||||
|
assert_eq!(stdout, b"after");
|
||||||
|
assert_eq!(stderr, b"err");
|
||||||
|
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.exit_code, Some(7));
|
||||||
|
harness.remote.close().await.unwrap();
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded long-poll continuation over HTTP");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn real_http_spawn_failure_preserves_safe_cause_counts_and_terminal_replay() {
|
||||||
|
tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let missing_path = root.path().join("private-spawn-path-must-not-exist");
|
||||||
|
assert!(!missing_path.exists());
|
||||||
|
let private_path = missing_path.to_str().unwrap().to_owned();
|
||||||
|
let environment_secret = "spawn-environment-secret-must-not-leak";
|
||||||
|
let command_secret = "spawn-command-secret-must-not-leak";
|
||||||
|
let tool_call_secret = "spawn-tool-call-secret-must-not-leak";
|
||||||
|
// This is an OS spawn failure in the production local provider, not a
|
||||||
|
// fabricated terminal: PATH contains no bash. The test process environment
|
||||||
|
// is untouched, so parallel command tests retain their normal PATH.
|
||||||
|
let harness = Harness::with_environment(
|
||||||
|
root,
|
||||||
|
BTreeMap::from([
|
||||||
|
("PATH".into(), private_path.clone()),
|
||||||
|
("PRIVATE_AUTH_TOKEN".into(), environment_secret.into()),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let command = format!("printf '{command_secret}'; printf ran > must-not-run");
|
||||||
|
let handle = harness
|
||||||
|
.remote
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
transport_byte_budget: 1024,
|
||||||
|
command: command.clone(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
tool_call_id: Some(tool_call_secret.into()),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let request = CommandOutputRequest {
|
||||||
|
handle,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 16,
|
||||||
|
wait: true,
|
||||||
|
};
|
||||||
|
let page = harness.remote.command_output(request.clone()).await.unwrap();
|
||||||
|
assert!(page.chunks.is_empty());
|
||||||
|
assert_eq!(page.next_cursor, 0);
|
||||||
|
let terminal = page.terminal.as_ref().expect("spawn failure is terminal");
|
||||||
|
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.exit_code, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.process_failure,
|
||||||
|
Some(workdir::CommandFailure {
|
||||||
|
stage: workdir::CommandFailureStage::Spawn,
|
||||||
|
reason: workdir::CommandIoErrorKind::NotFound,
|
||||||
|
os_code: Some(libc::ENOENT),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::NotStarted
|
||||||
|
);
|
||||||
|
assert_eq!(terminal.stdout_bytes, 0);
|
||||||
|
assert_eq!(terminal.stderr_bytes, 0);
|
||||||
|
assert_eq!(terminal.stdout_delivered_bytes, 0);
|
||||||
|
assert_eq!(terminal.stderr_delivered_bytes, 0);
|
||||||
|
assert_eq!(terminal.truncation_reason, None);
|
||||||
|
|
||||||
|
// A zero-byte terminal page remains replayable at its unchanged cursor;
|
||||||
|
// neither a retry nor a smaller nonwaiting read changes its accounting.
|
||||||
|
assert_eq!(
|
||||||
|
harness.remote.command_output(request.clone()).await.unwrap(),
|
||||||
|
page
|
||||||
|
);
|
||||||
|
let replay_request = CommandOutputRequest {
|
||||||
|
wait: false,
|
||||||
|
limit: 1,
|
||||||
|
..request
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
harness
|
||||||
|
.remote
|
||||||
|
.command_output(replay_request.clone())
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
page
|
||||||
|
);
|
||||||
|
let response = harness
|
||||||
|
.raw_operation(
|
||||||
|
serde_json::to_value(WorkdirSessionOperation::CommandOutput(
|
||||||
|
replay_request.clone(),
|
||||||
|
))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
for secret in [
|
||||||
|
private_path.as_str(),
|
||||||
|
harness._root.path().to_str().unwrap(),
|
||||||
|
"private-spawn-path-must-not-exist",
|
||||||
|
"PRIVATE_AUTH_TOKEN",
|
||||||
|
environment_secret,
|
||||||
|
command_secret,
|
||||||
|
command.as_str(),
|
||||||
|
tool_call_secret,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!body.contains(secret),
|
||||||
|
"private input leaked into HTTP terminal"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let wire: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
wire,
|
||||||
|
serde_json::json!({
|
||||||
|
"operation": "command_output",
|
||||||
|
"result": {
|
||||||
|
"chunks": [],
|
||||||
|
"next_cursor": 0,
|
||||||
|
"terminal": {
|
||||||
|
"status": "failed",
|
||||||
|
"exit_code": null,
|
||||||
|
"process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT},
|
||||||
|
"output_collection": {"state": "not_started"},
|
||||||
|
"stdout_bytes": 0,
|
||||||
|
"stderr_bytes": 0,
|
||||||
|
"stdout_delivered_bytes": 0,
|
||||||
|
"stderr_delivered_bytes": 0,
|
||||||
|
"truncation_reason": null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
harness.remote.command_output(replay_request).await.unwrap(),
|
||||||
|
page
|
||||||
|
);
|
||||||
|
// Exercise the same real spawn failure through Bash, not a terminal
|
||||||
|
// fixture. Opening another remote client reuses the pre-seeded provider
|
||||||
|
// session and still passes through the signed HTTP middleware.
|
||||||
|
let bash_remote: WorkdirSessionHandle = Arc::new(
|
||||||
|
RemoteWorkdirSession::open_with_authorization(
|
||||||
|
harness.client.clone(),
|
||||||
|
harness.operation_url.join("/").unwrap(),
|
||||||
|
harness.authorization.clone(),
|
||||||
|
Workdir::new("wd-transport").id().clone(),
|
||||||
|
OpenWorkdirSessionRequest::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let view = tools::bash_output::BashOutputView::new();
|
||||||
|
let (_, bash) = tools::bash_tool(bash_remote, view.clone())();
|
||||||
|
let input = serde_json::json!({
|
||||||
|
"command": command,
|
||||||
|
"cwd": ".",
|
||||||
|
"timeout": 5
|
||||||
|
})
|
||||||
|
.to_string();
|
||||||
|
let output = bash
|
||||||
|
.execute(
|
||||||
|
&input,
|
||||||
|
agen::tool::ToolExecutionContext::new(tool_call_secret, "http-spawn-attempt", 0),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("Bash returns a structured failed-command result");
|
||||||
|
let content = output.content.as_deref().expect("Bash ToolOutput body");
|
||||||
|
let tool_body: serde_json::Value = serde_json::from_str(content).unwrap();
|
||||||
|
assert_eq!(tool_body["stdout_preview"], "");
|
||||||
|
assert_eq!(tool_body["stderr_preview"], "");
|
||||||
|
assert_eq!(tool_body["preview_omitted"], false);
|
||||||
|
assert_eq!(
|
||||||
|
tool_body["output"]["terminal"],
|
||||||
|
serde_json::json!({
|
||||||
|
"status": "failed",
|
||||||
|
"exit_code": null,
|
||||||
|
"process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT},
|
||||||
|
"output_collection": {"state": "not_started"},
|
||||||
|
"transport_complete": false
|
||||||
|
})
|
||||||
|
);
|
||||||
|
let output_id = tool_body["output"]["id"].as_str().unwrap();
|
||||||
|
let retained = view.metadata(output_id).expect("retained spawn failure");
|
||||||
|
assert_eq!(retained.terminal.status, Some(workdir::CommandStatus::Failed));
|
||||||
|
assert_eq!(retained.terminal.exit_code, None);
|
||||||
|
assert_eq!(retained.terminal.process_failure, terminal.process_failure);
|
||||||
|
assert_eq!(
|
||||||
|
retained.terminal.output_collection,
|
||||||
|
Some(workdir::CommandOutputCollection::NotStarted)
|
||||||
|
);
|
||||||
|
assert!(!retained.terminal.transport_complete);
|
||||||
|
assert_eq!(retained.stdout_saved, 0);
|
||||||
|
assert_eq!(retained.stderr_saved, 0);
|
||||||
|
assert_eq!(retained.stdout_seen, 0);
|
||||||
|
assert_eq!(retained.stderr_seen, 0);
|
||||||
|
assert_eq!(retained.stdout_delivered, 0);
|
||||||
|
assert_eq!(retained.stderr_delivered, 0);
|
||||||
|
assert!(!retained.storage_truncated);
|
||||||
|
assert!(retained.retention_limit.is_none());
|
||||||
|
assert_eq!(retained.transport_truncation, None);
|
||||||
|
let retained_wire = serde_json::to_value(&retained).unwrap();
|
||||||
|
assert_eq!(retained_wire, tool_body["output"]);
|
||||||
|
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||||
|
assert_eq!(view.read(output_id, stream, 0, 1), Some(Vec::new()));
|
||||||
|
}
|
||||||
|
for secret in [
|
||||||
|
private_path.as_str(),
|
||||||
|
harness._root.path().to_str().unwrap(),
|
||||||
|
"private-spawn-path-must-not-exist",
|
||||||
|
"PRIVATE_AUTH_TOKEN",
|
||||||
|
environment_secret,
|
||||||
|
command_secret,
|
||||||
|
command.as_str(),
|
||||||
|
tool_call_secret,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!content.contains(secret),
|
||||||
|
"private input leaked into Bash ToolOutput body"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!harness._root.path().join("must-not-run").exists());
|
||||||
|
harness.remote.close().await.unwrap();
|
||||||
|
assert!(harness.session.command_snapshot().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(view.metadata(output_id).unwrap()).unwrap(),
|
||||||
|
retained_wire
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded real spawn failure over signed HTTP");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn real_http_closed_session_retains_failure_reason() {
|
||||||
|
let harness = Harness::new().await;
|
||||||
|
let handle = harness
|
||||||
|
.remote
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
transport_byte_budget: 1024,
|
||||||
|
command: "printf done".into(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
tool_call_id: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
// Keep the HTTP session identity registered, but close the underlying provider.
|
||||||
|
harness.session.close().await.unwrap();
|
||||||
|
let error = harness
|
||||||
|
.remote
|
||||||
|
.command_output(CommandOutputRequest {
|
||||||
|
handle,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 16,
|
||||||
|
wait: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
error.failure_reason(),
|
||||||
|
Some(workdir::WorkdirFailureReason::SessionClosed)
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
error.classification_source(),
|
||||||
|
workdir::WorkdirError::Unavailable(_)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn real_http_command_pages_preserve_binary_streams_replay_and_nonzero_exit() {
|
async fn real_http_command_pages_preserve_binary_streams_replay_and_nonzero_exit() {
|
||||||
tokio::time::timeout(Duration::from_secs(20), async {
|
tokio::time::timeout(Duration::from_secs(20), async {
|
||||||
@@ -189,6 +529,11 @@ async fn real_http_command_pages_preserve_binary_streams_replay_and_nonzero_exit
|
|||||||
assert_eq!(stderr, b"stderr\xe6\x97\xa5");
|
assert_eq!(stderr, b"stderr\xe6\x97\xa5");
|
||||||
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||||
assert_eq!(terminal.exit_code, Some(9));
|
assert_eq!(terminal.exit_code, Some(9));
|
||||||
|
assert_eq!(terminal.process_failure, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::Complete
|
||||||
|
);
|
||||||
assert!(harness.remote.command_output(CommandOutputRequest { handle, cursor: 0, limit: 1, wait: false }).await.is_err());
|
assert!(harness.remote.command_output(CommandOutputRequest { handle, cursor: 0, limit: 1, wait: false }).await.is_err());
|
||||||
harness.remote.close().await.unwrap();
|
harness.remote.close().await.unwrap();
|
||||||
assert!(harness.session.command_snapshot().is_empty());
|
assert!(harness.session.command_snapshot().is_empty());
|
||||||
@@ -333,6 +678,11 @@ async fn real_http_command_budget_bounds_unique_payload_and_reports_larger_seen_
|
|||||||
);
|
);
|
||||||
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||||
assert_eq!(terminal.exit_code, Some(9));
|
assert_eq!(terminal.exit_code, Some(9));
|
||||||
|
assert_eq!(terminal.process_failure, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::Complete
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
harness
|
harness
|
||||||
.remote
|
.remote
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ impl ProviderState {
|
|||||||
return Err((
|
return Err((
|
||||||
StatusCode::UNAUTHORIZED,
|
StatusCode::UNAUTHORIZED,
|
||||||
AxumJson(WorkdirTransportError {
|
AxumJson(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: WorkdirTransportErrorCode::Denied,
|
code: WorkdirTransportErrorCode::Denied,
|
||||||
message: "provider authorization denied".into(),
|
message: "provider authorization denied".into(),
|
||||||
@@ -885,21 +886,32 @@ async fn remote_native_bash_retains_streams_quota_and_replays_lost_terminal_thro
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn waiting_remote_command_rejects_empty_response_without_http_busy_loop() {
|
async fn waiting_remote_command_paces_empty_responses_and_keeps_cursor_until_output() {
|
||||||
watchdog(async {
|
watchdog(async {
|
||||||
let f = HttpFixture::new(WorkdirSessionCapabilities::ALL).await;
|
let f = HttpFixture::new(WorkdirSessionCapabilities::ALL).await;
|
||||||
let remote = f.open(TOKEN).await.unwrap();
|
let remote = f.open(TOKEN).await.unwrap();
|
||||||
*f.state.command_fault.lock().unwrap() = Some(CommandResponseFault::EmptyWaiting);
|
let handle = remote
|
||||||
let error = remote
|
.start_command(workdir::CommandRequest {
|
||||||
.command_output(workdir::CommandOutputRequest {
|
command: "printf done; exit 7".into(),
|
||||||
handle: workdir::CommandHandle("unused".into()),
|
cwd: WorkdirPath::root(),
|
||||||
cursor: 0,
|
timeout_secs: 5,
|
||||||
limit: 1024,
|
transport_byte_budget: 1024,
|
||||||
wait: true,
|
tool_call_id: None,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.unwrap_err();
|
.unwrap();
|
||||||
assert!(matches!(error, WorkdirError::Unavailable(_)));
|
*f.state.command_fault.lock().unwrap() = Some(CommandResponseFault::EmptyWaiting);
|
||||||
|
let mut request = workdir::CommandOutputRequest {
|
||||||
|
handle,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 1024,
|
||||||
|
wait: false,
|
||||||
|
};
|
||||||
|
// Nonwaiting polls still return their empty page after one HTTP request.
|
||||||
|
let page = remote.command_output(request.clone()).await.unwrap();
|
||||||
|
assert!(page.chunks.is_empty());
|
||||||
|
assert!(page.terminal.is_none());
|
||||||
|
assert_eq!(page.next_cursor, 0);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
f.state
|
f.state
|
||||||
.operations
|
.operations
|
||||||
@@ -910,6 +922,57 @@ async fn waiting_remote_command_rejects_empty_response_without_http_busy_loop()
|
|||||||
.count(),
|
.count(),
|
||||||
1
|
1
|
||||||
);
|
);
|
||||||
|
f.state.operations.lock().unwrap().clear();
|
||||||
|
|
||||||
|
request.wait = true;
|
||||||
|
// Repeated immediate empty responses remain pending, are paced rather
|
||||||
|
// than spun, and can still be stopped by the caller's deadline.
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(
|
||||||
|
Duration::from_millis(350),
|
||||||
|
remote.command_output(request.clone()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let operations = f.state.operations.lock().unwrap().clone();
|
||||||
|
assert!(
|
||||||
|
(1..=4).contains(&operations.len()),
|
||||||
|
"empty polls were not paced: {}",
|
||||||
|
operations.len()
|
||||||
|
);
|
||||||
|
for operation in &operations {
|
||||||
|
let WorkdirSessionOperation::CommandOutput(output) = operation else {
|
||||||
|
panic!("unexpected operation: {operation:?}");
|
||||||
|
};
|
||||||
|
assert_eq!(output, &request);
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||||
|
assert_eq!(
|
||||||
|
f.state.operations.lock().unwrap().len(),
|
||||||
|
operations.len(),
|
||||||
|
"dropped waiting read kept polling"
|
||||||
|
);
|
||||||
|
|
||||||
|
// After the empty replies stop, the exact same cursor still retrieves
|
||||||
|
// the bytes and terminal from the original command, without a restart.
|
||||||
|
*f.state.command_fault.lock().unwrap() = None;
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
loop {
|
||||||
|
let page = remote.command_output(request.clone()).await.unwrap();
|
||||||
|
assert_eq!(remote.command_output(request.clone()).await.unwrap(), page);
|
||||||
|
for chunk in page.chunks {
|
||||||
|
assert_eq!(chunk.sequence, request.cursor);
|
||||||
|
request.cursor += chunk.bytes.len() as u64;
|
||||||
|
bytes.extend(chunk.bytes);
|
||||||
|
}
|
||||||
|
assert_eq!(page.next_cursor, request.cursor);
|
||||||
|
if let Some(terminal) = page.terminal {
|
||||||
|
assert_eq!(terminal.exit_code, Some(7));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(bytes, b"done");
|
||||||
remote.close().await.unwrap();
|
remote.close().await.unwrap();
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
|
|||||||
@@ -409,10 +409,20 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession {
|
|||||||
"mismatched checkout mutation response".into(),
|
"mismatched checkout mutation response".into(),
|
||||||
)),
|
)),
|
||||||
Ok(_) => Err(Self::mismatch("checkout_execute")),
|
Ok(_) => Err(Self::mismatch("checkout_execute")),
|
||||||
Err(WorkdirError::Transport(_) | WorkdirError::Unavailable(_)) if mutation => {
|
Err(error)
|
||||||
Err(WorkdirError::OutcomeUnknown(
|
if mutation
|
||||||
|
&& matches!(
|
||||||
|
error.classification_source(),
|
||||||
|
WorkdirError::Transport(_) | WorkdirError::Unavailable(_)
|
||||||
|
) =>
|
||||||
|
{
|
||||||
|
let outcome = WorkdirError::OutcomeUnknown(
|
||||||
"checkout mutation response lost; inspect effects before retry".into(),
|
"checkout mutation response lost; inspect effects before retry".into(),
|
||||||
))
|
);
|
||||||
|
Err(match error.failure_reason() {
|
||||||
|
Some(reason) => outcome.with_failure_reason(reason),
|
||||||
|
None => outcome,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
Err(error) => Err(error),
|
Err(error) => Err(error),
|
||||||
}
|
}
|
||||||
@@ -1230,6 +1240,7 @@ mod tests {
|
|||||||
WorkspaceResponse {
|
WorkspaceResponse {
|
||||||
status,
|
status,
|
||||||
body: serde_json::to_string(&workdir::http::WorkdirTransportError {
|
body: serde_json::to_string(&workdir::http::WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code,
|
code,
|
||||||
message: message.to_string(),
|
message: message.to_string(),
|
||||||
|
|||||||
@@ -480,7 +480,7 @@ async fn shutdown_closes_bound_workdir_session() {
|
|||||||
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
session.command_status(command).await,
|
session.command_status(command).await,
|
||||||
Err(WorkdirError::Unavailable(_))
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -758,7 +758,7 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
|||||||
tool_call_id: None,
|
tool_call_id: None,
|
||||||
})
|
})
|
||||||
.await,
|
.await,
|
||||||
Err(WorkdirError::Unavailable(_))
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8866,7 +8866,14 @@ mod tests {
|
|||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect_err("closed session must reject local operation without another request");
|
.expect_err("closed session must reject local operation without another request");
|
||||||
assert!(matches!(error, WorkdirError::Unavailable(_)));
|
assert!(matches!(
|
||||||
|
error.classification_source(),
|
||||||
|
WorkdirError::Unavailable(_)
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
error.failure_reason(),
|
||||||
|
Some(workdir::WorkdirFailureReason::SessionClosed)
|
||||||
|
);
|
||||||
server.join().expect("mock remote server finished");
|
server.join().expect("mock remote server finished");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1411,6 +1411,10 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession {
|
|||||||
let outcome = workdir::WorkdirError::OutcomeUnknown(
|
let outcome = workdir::WorkdirError::OutcomeUnknown(
|
||||||
"checkout provider response lost; inspect effects before retry".into(),
|
"checkout provider response lost; inspect effects before retry".into(),
|
||||||
);
|
);
|
||||||
|
let outcome = match error.failure_reason() {
|
||||||
|
Some(reason) => outcome.with_failure_reason(reason),
|
||||||
|
None => outcome,
|
||||||
|
};
|
||||||
Err(match error.denial_reason() {
|
Err(match error.denial_reason() {
|
||||||
Some(reason) => workdir::WorkdirError::DenialContext {
|
Some(reason) => workdir::WorkdirError::DenialContext {
|
||||||
reason,
|
reason,
|
||||||
@@ -13501,6 +13505,7 @@ async fn log_failed_api_response(request: Request, next: Next) -> Response {
|
|||||||
attachment_alias_hash = context.map(|ctx| ctx.attachment_alias_hash.as_str()),
|
attachment_alias_hash = context.map(|ctx| ctx.attachment_alias_hash.as_str()),
|
||||||
request_token_id_hash = context.and(request_token_id_hash.as_deref()),
|
request_token_id_hash = context.and(request_token_id_hash.as_deref()),
|
||||||
denial_reason = error.and_then(|error| error.denial_reason).map(|reason| reason.as_str()),
|
denial_reason = error.and_then(|error| error.denial_reason).map(|reason| reason.as_str()),
|
||||||
|
failure_reason = error.and_then(|error| error.failure_reason).map(|reason| reason.as_str()),
|
||||||
status = event.status,
|
status = event.status,
|
||||||
kind = event.kind.unwrap_or("unknown"),
|
kind = event.kind.unwrap_or("unknown"),
|
||||||
message = event.message.unwrap_or(""),
|
message = event.message.unwrap_or(""),
|
||||||
@@ -20989,9 +20994,14 @@ impl InternalWorkdirOperationError {
|
|||||||
Error::WorkdirSessionOpenFailed { transport, .. } => transport.denial_reason,
|
Error::WorkdirSessionOpenFailed { transport, .. } => transport.denial_reason,
|
||||||
_ => None,
|
_ => None,
|
||||||
};
|
};
|
||||||
|
let failure_reason = match &error.error {
|
||||||
|
Error::WorkdirSessionOpenFailed { transport, .. } => transport.failure_reason,
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
let mut error =
|
let mut error =
|
||||||
server_api::WorkdirOperationApiError::api(error.into_repository_api_error());
|
server_api::WorkdirOperationApiError::api(error.into_repository_api_error());
|
||||||
error.denial_reason = reason;
|
error.denial_reason = reason;
|
||||||
|
error.failure_reason = failure_reason;
|
||||||
error
|
error
|
||||||
}
|
}
|
||||||
Self::Provider(error) => server_api::WorkdirOperationApiError::provider(error),
|
Self::Provider(error) => server_api::WorkdirOperationApiError::provider(error),
|
||||||
@@ -27162,6 +27172,7 @@ async fn serve_external_workdir_provider(
|
|||||||
Some(ExternalProviderCommand::Operation { operation_id, operation, response }) => {
|
Some(ExternalProviderCommand::Operation { operation_id, operation, response }) => {
|
||||||
if revoking {
|
if revoking {
|
||||||
let _ = response.send(Err(WorkdirTransportError {
|
let _ = response.send(Err(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::Unavailable,
|
code: workdir::http::WorkdirTransportErrorCode::Unavailable,
|
||||||
message: "External Workdir provider shutdown is in progress".to_string(),
|
message: "External Workdir provider shutdown is in progress".to_string(),
|
||||||
@@ -27175,6 +27186,7 @@ async fn serve_external_workdir_provider(
|
|||||||
};
|
};
|
||||||
if pending.len() >= pending_limit {
|
if pending.len() >= pending_limit {
|
||||||
let _ = response.send(Err(WorkdirTransportError {
|
let _ = response.send(Err(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::Unavailable,
|
code: workdir::http::WorkdirTransportErrorCode::Unavailable,
|
||||||
message: "External Workdir provider is at its in-flight operation limit".to_string(),
|
message: "External Workdir provider is at its in-flight operation limit".to_string(),
|
||||||
@@ -27186,6 +27198,7 @@ async fn serve_external_workdir_provider(
|
|||||||
Ok(operation) => operation,
|
Ok(operation) => operation,
|
||||||
Err(message) => {
|
Err(message) => {
|
||||||
let _ = response.send(Err(WorkdirTransportError {
|
let _ = response.send(Err(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::InvalidRequest,
|
code: workdir::http::WorkdirTransportErrorCode::InvalidRequest,
|
||||||
message,
|
message,
|
||||||
@@ -27243,6 +27256,7 @@ async fn serve_external_workdir_provider(
|
|||||||
Err(error.into_transport_error())
|
Err(error.into_transport_error())
|
||||||
}
|
}
|
||||||
ExternalWorkdirOperationOutcome::Cancelled => Err(WorkdirTransportError {
|
ExternalWorkdirOperationOutcome::Cancelled => Err(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::Unavailable,
|
code: workdir::http::WorkdirTransportErrorCode::Unavailable,
|
||||||
message: "External Workdir operation was cancelled".to_string(),
|
message: "External Workdir operation was cancelled".to_string(),
|
||||||
@@ -36896,6 +36910,7 @@ struct ApiErrorLog {
|
|||||||
diagnostics: Vec<RuntimeDiagnostic>,
|
diagnostics: Vec<RuntimeDiagnostic>,
|
||||||
workdir_log: Option<server_api::WorkdirOperationLogContext>,
|
workdir_log: Option<server_api::WorkdirOperationLogContext>,
|
||||||
denial_reason: Option<workdir::WorkdirDenialReason>,
|
denial_reason: Option<workdir::WorkdirDenialReason>,
|
||||||
|
failure_reason: Option<workdir::WorkdirFailureReason>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ApiErrorLog {
|
impl ApiErrorLog {
|
||||||
@@ -36910,6 +36925,7 @@ impl ApiErrorLog {
|
|||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
workdir_log: error.workdir_log.clone(),
|
workdir_log: error.workdir_log.clone(),
|
||||||
denial_reason: error.denial_reason,
|
denial_reason: error.denial_reason,
|
||||||
|
failure_reason: error.failure_reason,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
let mut log = Self::from_repository_error(&server_api::RepositoryApiError::new(
|
let mut log = Self::from_repository_error(&server_api::RepositoryApiError::new(
|
||||||
@@ -36924,6 +36940,7 @@ impl ApiErrorLog {
|
|||||||
));
|
));
|
||||||
log.workdir_log = error.workdir_log.clone();
|
log.workdir_log = error.workdir_log.clone();
|
||||||
log.denial_reason = error.denial_reason;
|
log.denial_reason = error.denial_reason;
|
||||||
|
log.failure_reason = error.failure_reason;
|
||||||
if log.workdir_log.is_some() {
|
if log.workdir_log.is_some() {
|
||||||
// Existing public API details are unchanged; do not copy request-derived
|
// Existing public API details are unchanged; do not copy request-derived
|
||||||
// alias/handle text or session-open errors into operational diagnostics.
|
// alias/handle text or session-open errors into operational diagnostics.
|
||||||
@@ -36940,6 +36957,7 @@ impl ApiErrorLog {
|
|||||||
Self {
|
Self {
|
||||||
workdir_log: None,
|
workdir_log: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
|
failure_reason: None,
|
||||||
kind: error
|
kind: error
|
||||||
.diagnostics
|
.diagnostics
|
||||||
.first()
|
.first()
|
||||||
@@ -37254,6 +37272,7 @@ impl IntoResponse for ApiError {
|
|||||||
let log = ApiErrorLog {
|
let log = ApiErrorLog {
|
||||||
workdir_log: None,
|
workdir_log: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
|
failure_reason: None,
|
||||||
kind: self
|
kind: self
|
||||||
.diagnostics
|
.diagnostics
|
||||||
.first()
|
.first()
|
||||||
@@ -37601,6 +37620,7 @@ mod tests {
|
|||||||
assert!(matches!(operation, WorkdirSessionOperation::Read(_)));
|
assert!(matches!(operation, WorkdirSessionOperation::Read(_)));
|
||||||
response
|
response
|
||||||
.send(Err(WorkdirTransportError {
|
.send(Err(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::NotFound,
|
code: workdir::http::WorkdirTransportErrorCode::NotFound,
|
||||||
message: "logical file was not found".to_string(),
|
message: "logical file was not found".to_string(),
|
||||||
@@ -37771,10 +37791,15 @@ mod tests {
|
|||||||
)))
|
)))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
let error = write.await.unwrap().unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
write.await.unwrap(),
|
error.classification_source(),
|
||||||
Err(workdir::WorkdirError::OutcomeUnknown(_))
|
workdir::WorkdirError::OutcomeUnknown(_)
|
||||||
));
|
));
|
||||||
|
assert_eq!(
|
||||||
|
error.failure_reason(),
|
||||||
|
lose_response.then_some(workdir::WorkdirFailureReason::ProviderUnavailable)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -38025,6 +38050,8 @@ mod tests {
|
|||||||
chunks: Vec::new(),
|
chunks: Vec::new(),
|
||||||
next_cursor: 0,
|
next_cursor: 0,
|
||||||
terminal: Some(workdir::CommandTerminal {
|
terminal: Some(workdir::CommandTerminal {
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: workdir::CommandOutputCollection::Complete,
|
||||||
status: workdir::CommandStatus::Cancelled,
|
status: workdir::CommandStatus::Cancelled,
|
||||||
exit_code: None,
|
exit_code: None,
|
||||||
|
|
||||||
@@ -38127,6 +38154,8 @@ mod tests {
|
|||||||
chunks: Vec::new(),
|
chunks: Vec::new(),
|
||||||
next_cursor: 0,
|
next_cursor: 0,
|
||||||
terminal: Some(workdir::CommandTerminal {
|
terminal: Some(workdir::CommandTerminal {
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: workdir::CommandOutputCollection::Complete,
|
||||||
status: workdir::CommandStatus::Cancelled,
|
status: workdir::CommandStatus::Cancelled,
|
||||||
exit_code: None,
|
exit_code: None,
|
||||||
|
|
||||||
@@ -39219,6 +39248,8 @@ mod tests {
|
|||||||
chunks: Vec::new(),
|
chunks: Vec::new(),
|
||||||
next_cursor: 0,
|
next_cursor: 0,
|
||||||
terminal: Some(workdir::CommandTerminal {
|
terminal: Some(workdir::CommandTerminal {
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: workdir::CommandOutputCollection::Complete,
|
||||||
status: workdir::CommandStatus::Cancelled,
|
status: workdir::CommandStatus::Cancelled,
|
||||||
exit_code: None,
|
exit_code: None,
|
||||||
|
|
||||||
@@ -41788,6 +41819,7 @@ mod tests {
|
|||||||
let error = ApiErrorLog {
|
let error = ApiErrorLog {
|
||||||
workdir_log: None,
|
workdir_log: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
|
failure_reason: None,
|
||||||
kind: "ticket_backend_error".to_string(),
|
kind: "ticket_backend_error".to_string(),
|
||||||
message: "sqlite error: FOREIGN KEY constraint failed".to_string(),
|
message: "sqlite error: FOREIGN KEY constraint failed".to_string(),
|
||||||
diagnostics: vec![RuntimeDiagnostic {
|
diagnostics: vec![RuntimeDiagnostic {
|
||||||
@@ -41929,6 +41961,7 @@ mod tests {
|
|||||||
),
|
),
|
||||||
] {
|
] {
|
||||||
let response = InternalWorkdirOperationError::Provider(WorkdirTransportError {
|
let response = InternalWorkdirOperationError::Provider(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code,
|
code,
|
||||||
message: "safe provider message".to_string(),
|
message: "safe provider message".to_string(),
|
||||||
@@ -41953,6 +41986,7 @@ mod tests {
|
|||||||
workdir_operation_transport_error(workdir::WorkdirError::InvalidGlob("[".to_string()));
|
workdir_operation_transport_error(workdir::WorkdirError::InvalidGlob("[".to_string()));
|
||||||
let remote = workdir_operation_transport_error(
|
let remote = workdir_operation_transport_error(
|
||||||
WorkdirTransportError {
|
WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::InvalidRequest,
|
code: workdir::http::WorkdirTransportErrorCode::InvalidRequest,
|
||||||
message: "Workdir operation request is invalid".to_string(),
|
message: "Workdir operation request is invalid".to_string(),
|
||||||
|
|||||||
@@ -563,6 +563,7 @@ async fn generated_workdir_provider_errors_preserve_response_and_log_details() {
|
|||||||
));
|
));
|
||||||
response
|
response
|
||||||
.send(Err(WorkdirTransportError {
|
.send(Err(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: reason,
|
denial_reason: reason,
|
||||||
code,
|
code,
|
||||||
message: "body-secret /private/provider/path".to_string(),
|
message: "body-secret /private/provider/path".to_string(),
|
||||||
@@ -600,6 +601,64 @@ async fn generated_workdir_provider_errors_preserve_response_and_log_details() {
|
|||||||
64
|
64
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
for reason in [
|
||||||
|
workdir::WorkdirFailureReason::SessionClosed,
|
||||||
|
workdir::WorkdirFailureReason::CommandTerminalMissing,
|
||||||
|
workdir::WorkdirFailureReason::HttpTimeout,
|
||||||
|
] {
|
||||||
|
let request = runtime_source_request(
|
||||||
|
&identity,
|
||||||
|
Some(&fixture.worker.worker_id),
|
||||||
|
"POST",
|
||||||
|
&path,
|
||||||
|
serde_json::to_vec(&server_api::CurrentWorkerWorkdirOperationRequest {
|
||||||
|
target_workdir: "checkout".into(),
|
||||||
|
operation: WorkdirSessionOperation::CommandOutput(workdir::CommandOutputRequest {
|
||||||
|
handle: handle.clone(),
|
||||||
|
cursor: 0,
|
||||||
|
limit: 16,
|
||||||
|
wait: true,
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let provider = async {
|
||||||
|
let Some(ExternalProviderCommand::Operation {
|
||||||
|
operation,
|
||||||
|
response,
|
||||||
|
..
|
||||||
|
}) = commands.recv().await
|
||||||
|
else {
|
||||||
|
panic!("expected output request")
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
operation,
|
||||||
|
WorkdirSessionOperation::CommandOutput(_)
|
||||||
|
));
|
||||||
|
response
|
||||||
|
.send(Err(WorkdirTransportError {
|
||||||
|
code: Code::Unavailable,
|
||||||
|
message: "body-secret /private/provider/path".into(),
|
||||||
|
denial_reason: None,
|
||||||
|
failure_reason: Some(reason),
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
};
|
||||||
|
let ((status, body, events), ()) =
|
||||||
|
tokio::join!(request_with_logs(app.clone(), request), provider);
|
||||||
|
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
assert_eq!(
|
||||||
|
body,
|
||||||
|
json!({"code":"unavailable", "message":"Workdir session is unavailable"})
|
||||||
|
);
|
||||||
|
assert_eq!(events.len(), 1, "{events:?}");
|
||||||
|
assert_eq!(events[0]["failure_reason"], reason.as_str());
|
||||||
|
assert_eq!(events[0]["operation"], "command_output");
|
||||||
|
assert_eq!(events[0]["workdir_stage"], "provider_dispatch");
|
||||||
|
let log = serde_json::to_string(&events).unwrap();
|
||||||
|
assert!(!log.contains("body-secret"));
|
||||||
|
assert!(!log.contains("/private/provider/path"));
|
||||||
|
}
|
||||||
let provider = async {
|
let provider = async {
|
||||||
let Some(ExternalProviderCommand::Operation { response, .. }) = commands.recv().await
|
let Some(ExternalProviderCommand::Operation { response, .. }) = commands.recv().await
|
||||||
else {
|
else {
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ fn page(cursor: u64, bytes: &[u8], terminal: bool) -> workdir::CommandOutput {
|
|||||||
},
|
},
|
||||||
next_cursor: cursor + bytes.len() as u64,
|
next_cursor: cursor + bytes.len() as u64,
|
||||||
terminal: terminal.then_some(workdir::CommandTerminal {
|
terminal: terminal.then_some(workdir::CommandTerminal {
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: workdir::CommandOutputCollection::Complete,
|
||||||
status: workdir::CommandStatus::Cancelled,
|
status: workdir::CommandStatus::Cancelled,
|
||||||
exit_code: None,
|
exit_code: None,
|
||||||
stdout_bytes: cursor + bytes.len() as u64,
|
stdout_bytes: cursor + bytes.len() as u64,
|
||||||
|
|||||||
@@ -836,6 +836,7 @@ where
|
|||||||
.unwrap_or_else(|_| ExternalWorkdirOperationOutcome::Failed {
|
.unwrap_or_else(|_| ExternalWorkdirOperationOutcome::Failed {
|
||||||
error: ExternalWorkdirOperationError::from_transport_error(
|
error: ExternalWorkdirOperationError::from_transport_error(
|
||||||
WorkdirTransportError {
|
WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::Internal,
|
code: workdir::http::WorkdirTransportErrorCode::Internal,
|
||||||
message: "External Workdir operation executor failed".to_string(),
|
message: "External Workdir operation executor failed".to_string(),
|
||||||
@@ -954,6 +955,7 @@ async fn execute_operation(
|
|||||||
Ok(result) => ExternalWorkdirOperationOutcome::Completed { result },
|
Ok(result) => ExternalWorkdirOperationOutcome::Completed { result },
|
||||||
Err(message) => ExternalWorkdirOperationOutcome::Failed {
|
Err(message) => ExternalWorkdirOperationOutcome::Failed {
|
||||||
error: ExternalWorkdirOperationError::from_transport_error(WorkdirTransportError {
|
error: ExternalWorkdirOperationError::from_transport_error(WorkdirTransportError {
|
||||||
|
failure_reason: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
code: workdir::http::WorkdirTransportErrorCode::InvalidRequest,
|
code: workdir::http::WorkdirTransportErrorCode::InvalidRequest,
|
||||||
message,
|
message,
|
||||||
|
|||||||
Reference in New Issue
Block a user