fix: allow slow initial input commits
This commit is contained in:
@@ -786,9 +786,15 @@ impl Runtime {
|
|||||||
) -> Result<WorkerDetail, RuntimeError> {
|
) -> Result<WorkerDetail, RuntimeError> {
|
||||||
let worker_id = request.worker_id;
|
let worker_id = request.worker_id;
|
||||||
let workspace_id = scope.map(|scope| scope.workspace_id.as_str());
|
let workspace_id = scope.map(|scope| scope.workspace_id.as_str());
|
||||||
|
let started_at = std::time::Instant::now();
|
||||||
let result = self.create_worker_with_workspace_inner(request, scope);
|
let result = self.create_worker_with_workspace_inner(request, scope);
|
||||||
if let Err(error) = &result {
|
if let Err(error) = &result {
|
||||||
write_runtime_worker_create_failure(worker_id, workspace_id, error);
|
write_runtime_worker_create_failure(
|
||||||
|
worker_id,
|
||||||
|
workspace_id,
|
||||||
|
error,
|
||||||
|
started_at.elapsed(),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
@@ -3470,6 +3476,7 @@ fn write_runtime_worker_create_failure(
|
|||||||
worker_id: WorkerId,
|
worker_id: WorkerId,
|
||||||
workspace_id: Option<&str>,
|
workspace_id: Option<&str>,
|
||||||
error: &RuntimeError,
|
error: &RuntimeError,
|
||||||
|
elapsed: std::time::Duration,
|
||||||
) {
|
) {
|
||||||
let (error_kind, operation, outcome) = runtime_worker_create_failure_fields(error);
|
let (error_kind, operation, outcome) = runtime_worker_create_failure_fields(error);
|
||||||
let execution_failure_code = match error {
|
let execution_failure_code = match error {
|
||||||
@@ -3488,6 +3495,7 @@ fn write_runtime_worker_create_failure(
|
|||||||
operation = operation.as_deref().unwrap_or(""),
|
operation = operation.as_deref().unwrap_or(""),
|
||||||
outcome = outcome.as_deref().unwrap_or(""),
|
outcome = outcome.as_deref().unwrap_or(""),
|
||||||
execution_failure_code,
|
execution_failure_code,
|
||||||
|
duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
|
||||||
"Worker creation failed"
|
"Worker creation failed"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -90,11 +90,11 @@ use worker::{
|
|||||||
const DEFAULT_BACKEND_ID: &str = "worker-crate";
|
const DEFAULT_BACKEND_ID: &str = "worker-crate";
|
||||||
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
|
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
const SPAWN_RESTORE_TASK_TIMEOUT: Duration = Duration::from_secs(60);
|
const SPAWN_RESTORE_TASK_TIMEOUT: Duration = Duration::from_secs(60);
|
||||||
const USER_INPUT_TASK_TIMEOUT: Duration = Duration::from_secs(35);
|
const USER_INPUT_TASK_TIMEOUT: Duration = Duration::from_secs(125);
|
||||||
const WORKSPACE_CONFIG_HTTP_TIMEOUT: Duration = Duration::from_secs(8);
|
const WORKSPACE_CONFIG_HTTP_TIMEOUT: Duration = Duration::from_secs(8);
|
||||||
const MAX_WORKSPACE_CONFIG_RESPONSE_BYTES: usize = 72 * 1024 * 1024;
|
const MAX_WORKSPACE_CONFIG_RESPONSE_BYTES: usize = 72 * 1024 * 1024;
|
||||||
// Leave adapter cancellation margin after the durable submission deadline.
|
// Leave adapter cancellation margin after the durable submission deadline.
|
||||||
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(30);
|
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(120);
|
||||||
|
|
||||||
pub struct RuntimeWorkerController {
|
pub struct RuntimeWorkerController {
|
||||||
pub handle: WorkerHandle,
|
pub handle: WorkerHandle,
|
||||||
@@ -2541,6 +2541,15 @@ mod tests {
|
|||||||
WorkerExecutionContext::new(worker_ref)
|
WorkerExecutionContext::new(worker_ref)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn input_commit_budget_leaves_adapter_cancellation_margin() {
|
||||||
|
assert!(USER_INPUT_TASK_TIMEOUT > USER_INPUT_COMMIT_TIMEOUT);
|
||||||
|
assert_eq!(
|
||||||
|
USER_INPUT_TASK_TIMEOUT - USER_INPUT_COMMIT_TIMEOUT,
|
||||||
|
Duration::from_secs(5)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
struct DelayedFactory {
|
struct DelayedFactory {
|
||||||
completed: Arc<AtomicBool>,
|
completed: Arc<AtomicBool>,
|
||||||
delay: Duration,
|
delay: Duration,
|
||||||
|
|||||||
@@ -69,9 +69,9 @@ const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host";
|
|||||||
const REMOTE_HOST_KIND: &str = "remote-worker-runtime-host";
|
const REMOTE_HOST_KIND: &str = "remote-worker-runtime-host";
|
||||||
const MAX_DIAGNOSTICS: usize = 16;
|
const MAX_DIAGNOSTICS: usize = 16;
|
||||||
const MAX_RUNTIME_PING_RESPONSE_BYTES: usize = 8 * 1024;
|
const MAX_RUNTIME_PING_RESPONSE_BYTES: usize = 8 * 1024;
|
||||||
// Runtime creation can spend up to 60s bootstrapping, 35s waiting for the
|
// Runtime creation can spend up to 60s bootstrapping, 125s waiting for the
|
||||||
// durable initial-input acknowledgement, and 5s confirming shutdown.
|
// durable initial-input acknowledgement, and 5s confirming shutdown.
|
||||||
const REMOTE_WORKER_CREATE_TIMEOUT: Duration = Duration::from_secs(105);
|
const REMOTE_WORKER_CREATE_TIMEOUT: Duration = Duration::from_secs(195);
|
||||||
const MAX_HOST_SCAN: usize = 256;
|
const MAX_HOST_SCAN: usize = 256;
|
||||||
const MAX_IDENTIFIER_LEN: usize = 120;
|
const MAX_IDENTIFIER_LEN: usize = 120;
|
||||||
const ID_DIGEST_HEX_LEN: usize = 16;
|
const ID_DIGEST_HEX_LEN: usize = 16;
|
||||||
@@ -4734,6 +4734,11 @@ mod tests {
|
|||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn remote_worker_create_timeout_covers_runtime_phase_budgets() {
|
||||||
|
assert!(REMOTE_WORKER_CREATE_TIMEOUT > Duration::from_secs(60 + 125 + 5));
|
||||||
|
}
|
||||||
|
|
||||||
fn test_create_binding() -> WorkerCreateBinding {
|
fn test_create_binding() -> WorkerCreateBinding {
|
||||||
WorkerCreateBinding {
|
WorkerCreateBinding {
|
||||||
worker_id: EmbeddedWorkerId::now_v7(),
|
worker_id: EmbeddedWorkerId::now_v7(),
|
||||||
|
|||||||
Reference in New Issue
Block a user