fix: allow slow initial input commits
This commit is contained in:
@@ -786,9 +786,15 @@ impl Runtime {
|
||||
) -> Result<WorkerDetail, RuntimeError> {
|
||||
let worker_id = request.worker_id;
|
||||
let workspace_id = scope.map(|scope| scope.workspace_id.as_str());
|
||||
let started_at = std::time::Instant::now();
|
||||
let result = self.create_worker_with_workspace_inner(request, scope);
|
||||
if let Err(error) = &result {
|
||||
write_runtime_worker_create_failure(worker_id, workspace_id, error);
|
||||
write_runtime_worker_create_failure(
|
||||
worker_id,
|
||||
workspace_id,
|
||||
error,
|
||||
started_at.elapsed(),
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -3470,6 +3476,7 @@ fn write_runtime_worker_create_failure(
|
||||
worker_id: WorkerId,
|
||||
workspace_id: Option<&str>,
|
||||
error: &RuntimeError,
|
||||
elapsed: std::time::Duration,
|
||||
) {
|
||||
let (error_kind, operation, outcome) = runtime_worker_create_failure_fields(error);
|
||||
let execution_failure_code = match error {
|
||||
@@ -3488,6 +3495,7 @@ fn write_runtime_worker_create_failure(
|
||||
operation = operation.as_deref().unwrap_or(""),
|
||||
outcome = outcome.as_deref().unwrap_or(""),
|
||||
execution_failure_code,
|
||||
duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
|
||||
"Worker creation failed"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -90,11 +90,11 @@ use worker::{
|
||||
const DEFAULT_BACKEND_ID: &str = "worker-crate";
|
||||
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const SPAWN_RESTORE_TASK_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const USER_INPUT_TASK_TIMEOUT: Duration = Duration::from_secs(35);
|
||||
const USER_INPUT_TASK_TIMEOUT: Duration = Duration::from_secs(125);
|
||||
const WORKSPACE_CONFIG_HTTP_TIMEOUT: Duration = Duration::from_secs(8);
|
||||
const MAX_WORKSPACE_CONFIG_RESPONSE_BYTES: usize = 72 * 1024 * 1024;
|
||||
// Leave adapter cancellation margin after the durable submission deadline.
|
||||
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
|
||||
pub struct RuntimeWorkerController {
|
||||
pub handle: WorkerHandle,
|
||||
@@ -2541,6 +2541,15 @@ mod tests {
|
||||
WorkerExecutionContext::new(worker_ref)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_commit_budget_leaves_adapter_cancellation_margin() {
|
||||
assert!(USER_INPUT_TASK_TIMEOUT > USER_INPUT_COMMIT_TIMEOUT);
|
||||
assert_eq!(
|
||||
USER_INPUT_TASK_TIMEOUT - USER_INPUT_COMMIT_TIMEOUT,
|
||||
Duration::from_secs(5)
|
||||
);
|
||||
}
|
||||
|
||||
struct DelayedFactory {
|
||||
completed: Arc<AtomicBool>,
|
||||
delay: Duration,
|
||||
|
||||
@@ -69,9 +69,9 @@ const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host";
|
||||
const REMOTE_HOST_KIND: &str = "remote-worker-runtime-host";
|
||||
const MAX_DIAGNOSTICS: usize = 16;
|
||||
const MAX_RUNTIME_PING_RESPONSE_BYTES: usize = 8 * 1024;
|
||||
// Runtime creation can spend up to 60s bootstrapping, 35s waiting for the
|
||||
// Runtime creation can spend up to 60s bootstrapping, 125s waiting for the
|
||||
// durable initial-input acknowledgement, and 5s confirming shutdown.
|
||||
const REMOTE_WORKER_CREATE_TIMEOUT: Duration = Duration::from_secs(105);
|
||||
const REMOTE_WORKER_CREATE_TIMEOUT: Duration = Duration::from_secs(195);
|
||||
const MAX_HOST_SCAN: usize = 256;
|
||||
const MAX_IDENTIFIER_LEN: usize = 120;
|
||||
const ID_DIGEST_HEX_LEN: usize = 16;
|
||||
@@ -4734,6 +4734,11 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
|
||||
#[test]
|
||||
fn remote_worker_create_timeout_covers_runtime_phase_budgets() {
|
||||
assert!(REMOTE_WORKER_CREATE_TIMEOUT > Duration::from_secs(60 + 125 + 5));
|
||||
}
|
||||
|
||||
fn test_create_binding() -> WorkerCreateBinding {
|
||||
WorkerCreateBinding {
|
||||
worker_id: EmbeddedWorkerId::now_v7(),
|
||||
|
||||
Reference in New Issue
Block a user