fix: log sanitized worker spawn failures

This commit is contained in:
2026-09-17 19:37:07 +09:00
parent f4b34a289e
commit e722d21d98
+101 -4
View File
@@ -3970,6 +3970,68 @@ fn runtime_worker_create_failure_fields(
} }
} }
const WORKER_CREATE_FAILURE_MESSAGE_LIMIT: usize = 512;
fn sanitize_worker_create_failure_message(message: &str) -> String {
let mut redact_next = false;
let detail = message
.split_whitespace()
.map(|part| {
if redact_next {
redact_next = false;
return "[redacted]";
}
let lowercase = part.to_ascii_lowercase();
let label =
lowercase.trim_matches(|character: char| !character.is_ascii_alphanumeric());
let redact_following = matches!(
label,
"bearer" | "credential" | "key" | "password" | "secret" | "session" | "token"
);
if redact_following {
redact_next = true;
}
if part.contains('/')
|| part.contains('\\')
|| (!redact_following
&& ["credential", "password", "secret", "session", "token"]
.iter()
.any(|marker| label.contains(marker)))
|| lowercase.contains("credential=")
|| lowercase.contains("key=")
|| lowercase.contains("password=")
|| lowercase.contains("secret=")
|| lowercase.contains("session=")
|| lowercase.contains("session_id=")
|| lowercase.contains("token=")
{
"[redacted]"
} else {
part
}
})
.collect::<Vec<_>>()
.join(" ");
let truncated = detail.chars().count() > WORKER_CREATE_FAILURE_MESSAGE_LIMIT;
let mut detail = detail
.chars()
.take(WORKER_CREATE_FAILURE_MESSAGE_LIMIT)
.collect::<String>();
if truncated {
detail.push('…');
}
detail
}
fn runtime_worker_create_failure_message(error: &RuntimeError) -> Option<String> {
match error {
RuntimeError::WorkerExecutionRejected { message, .. } => {
Some(sanitize_worker_create_failure_message(message))
}
_ => None,
}
}
fn write_runtime_worker_create_failure( fn write_runtime_worker_create_failure(
worker_id: WorkerId, worker_id: WorkerId,
workspace_id: Option<&str>, workspace_id: Option<&str>,
@@ -3983,6 +4045,7 @@ fn write_runtime_worker_create_failure(
} }
_ => "", _ => "",
}; };
let execution_failure_message = runtime_worker_create_failure_message(error);
tracing::error!( tracing::error!(
target: "yoi::worker_create", target: "yoi::worker_create",
event = "worker_create_failed", event = "worker_create_failed",
@@ -3993,6 +4056,7 @@ fn write_runtime_worker_create_failure(
operation = operation.as_deref().unwrap_or(""), operation = operation.as_deref().unwrap_or(""),
outcome = outcome.as_deref().unwrap_or(""), outcome = outcome.as_deref().unwrap_or(""),
execution_failure_code, execution_failure_code,
execution_failure_message = execution_failure_message.as_deref().unwrap_or(""),
duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX), duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
"Worker creation failed" "Worker creation failed"
); );
@@ -4123,13 +4187,46 @@ mod tests {
use std::sync::{Arc, Barrier, Condvar, Mutex}; use std::sync::{Arc, Barrier, Condvar, Mutex};
#[test] #[test]
fn worker_create_failure_fields_exclude_raw_error_messages() { fn worker_create_failure_fields_only_include_sanitized_execution_detail() {
let (error_kind, operation, outcome) = runtime_worker_create_failure_fields( let invalid_request = RuntimeError::InvalidRequest("private-token".to_string());
&RuntimeError::InvalidRequest("private-token".to_string()), let (error_kind, operation, outcome) =
); runtime_worker_create_failure_fields(&invalid_request);
assert_eq!(error_kind, "invalid_request"); assert_eq!(error_kind, "invalid_request");
assert_eq!(operation, None); assert_eq!(operation, None);
assert_eq!(outcome, None); assert_eq!(outcome, None);
assert_eq!(
runtime_worker_create_failure_message(&invalid_request),
None
);
let result = WorkerExecutionResult::errored(
WorkerExecutionOperation::Spawn,
"failed to spawn controller in /private/path: File exists; token super-secret",
);
let error = RuntimeError::WorkerExecutionRejected {
worker_id: WorkerId::now_v7(),
operation: result.operation,
outcome: result.outcome,
message: result.message_or_default(),
result,
};
assert_eq!(
runtime_worker_create_failure_message(&error).as_deref(),
Some("failed to spawn controller in [redacted] File exists; token [redacted]")
);
assert_eq!(
sanitize_worker_create_failure_message("backend private-token failed"),
"backend [redacted] failed"
);
let truncated = sanitize_worker_create_failure_message(&"x".repeat(513));
assert_eq!(truncated.chars().count(), 513);
assert!(truncated.ends_with('…'));
}
#[test]
fn worker_create_failure_code_classifies_durable_acceptance_timeout() {
let timeout = WorkerExecutionResult::errored( let timeout = WorkerExecutionResult::errored(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
"timed out waiting for durable Worker Submit acceptance; private-token", "timed out waiting for durable Worker Submit acceptance; private-token",