Merge remote-tracking branch 'origin/work/T-674-backend-job-runner' into develop
This commit is contained in:
@@ -94,6 +94,13 @@ pub const BUILTIN_PROFILE_RESOURCES: &[BuiltinProfileResource] = &[
|
||||
description: "Workspace orchestration and Worker control.",
|
||||
imports: BASE_IMPORT,
|
||||
},
|
||||
BuiltinProfileResource {
|
||||
selector: Some("builtin:backend-job"),
|
||||
path: "profiles/backend-job.dcdl",
|
||||
source: include_str!("../../../resources/profiles/backend-job.dcdl"),
|
||||
description: "Backend-owned bounded Job Worker.",
|
||||
imports: BASE_IMPORT,
|
||||
},
|
||||
BuiltinProfileResource {
|
||||
selector: Some("builtin:memory-consolidation"),
|
||||
path: "profiles/memory-consolidation.dcdl",
|
||||
|
||||
@@ -1005,6 +1005,18 @@ pub enum Event {
|
||||
submission_request_id: String,
|
||||
message: String,
|
||||
},
|
||||
/// Durable notification acceptance. The request identity is persisted with
|
||||
/// the notification receipt before this event is emitted. Repeating the
|
||||
/// same request id and exact payload returns the same acknowledgement
|
||||
/// without appending the notification twice.
|
||||
NotificationAccepted {
|
||||
notification_request_id: String,
|
||||
},
|
||||
/// Correlated notification rejection before durable acceptance.
|
||||
NotificationRejected {
|
||||
notification_request_id: String,
|
||||
message: String,
|
||||
},
|
||||
/// Revisioned FIFO replacement following enqueue, activation, cancel, or clear.
|
||||
PendingSubmissionsChanged {
|
||||
pending: PendingSubmissionsSnapshot,
|
||||
@@ -2809,6 +2821,32 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_receipt_events_roundtrip_with_request_identity() {
|
||||
for event in [
|
||||
Event::NotificationAccepted {
|
||||
notification_request_id: "notification-1".into(),
|
||||
},
|
||||
Event::NotificationRejected {
|
||||
notification_request_id: "notification-1".into(),
|
||||
message: "invalid notification".into(),
|
||||
},
|
||||
] {
|
||||
let json = serde_json::to_string(&event).unwrap();
|
||||
let decoded: Event = serde_json::from_str(&json).unwrap();
|
||||
match decoded {
|
||||
Event::NotificationAccepted {
|
||||
notification_request_id,
|
||||
}
|
||||
| Event::NotificationRejected {
|
||||
notification_request_id,
|
||||
..
|
||||
} => assert_eq!(notification_request_id, "notification-1"),
|
||||
other => panic!("expected notification receipt, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_error_format() {
|
||||
let event = Event::Error {
|
||||
|
||||
@@ -631,6 +631,24 @@ impl SubscriptionWorkerWorkdirAttachment {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SubscriptionWorkerJob {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub purpose: String,
|
||||
}
|
||||
|
||||
impl SubscriptionWorkerJob {
|
||||
pub fn validate(&self) -> Result<(), SubscriptionValidationError> {
|
||||
validate_identifier("job_id", &self.job_id, MAX_RESOURCE_ID_BYTES)?;
|
||||
validate_identifier("attempt_id", &self.attempt_id, MAX_RESOURCE_ID_BYTES)?;
|
||||
validate_identifier("job_purpose", &self.purpose, MAX_RESOURCE_ID_BYTES)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
@@ -664,6 +682,10 @@ pub struct SubscriptionWorker {
|
||||
pub display_name: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub profile: Option<String>,
|
||||
/// Backend-owned Job binding. Runtime producers leave this unset; the
|
||||
/// Workspace projection enriches both snapshots and live updates.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub job: Option<SubscriptionWorkerJob>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub workdir_attachments: Vec<SubscriptionWorkerWorkdirAttachment>,
|
||||
}
|
||||
@@ -677,6 +699,9 @@ impl SubscriptionWorker {
|
||||
if let Some(resource_key) = &self.resource_key {
|
||||
validate_identifier("resource_key", resource_key, MAX_RESOURCE_ID_BYTES)?;
|
||||
}
|
||||
if let Some(job) = &self.job {
|
||||
job.validate()?;
|
||||
}
|
||||
let mut aliases = HashSet::new();
|
||||
let mut workdir_ids = HashSet::new();
|
||||
for attachment in &self.workdir_attachments {
|
||||
@@ -952,6 +977,7 @@ mod tests {
|
||||
workspace_id: Some("workspace-1".to_string()),
|
||||
display_name: Some(format!("Worker {value}")),
|
||||
profile: Some("builtin:coder".to_string()),
|
||||
job: None,
|
||||
workdir_attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::{
|
||||
SubscriptionRequest, SubscriptionRequestId, SubscriptionResponse, SubscriptionSnapshot,
|
||||
SubscriptionTerminationCode, SubscriptionWorkdirId, SubscriptionWorker,
|
||||
SubscriptionWorkerAvailability, SubscriptionWorkerId, SubscriptionWorkerIds,
|
||||
SubscriptionWorkerProtocolMethod, SubscriptionWorkerState,
|
||||
SubscriptionWorkerJob, SubscriptionWorkerProtocolMethod, SubscriptionWorkerState,
|
||||
SubscriptionWorkerWorkdirAttachment, WorkspaceSubscriptionWorkdir,
|
||||
},
|
||||
};
|
||||
@@ -132,6 +132,7 @@ pub fn generated_protocol_types() -> String {
|
||||
push_decl::<SubscriptionWorkerAvailability>(&cfg, &mut output);
|
||||
push_decl::<EventSubscriptionSelector>(&cfg, &mut output);
|
||||
push_decl::<SubscriptionWorkerWorkdirAttachment>(&cfg, &mut output);
|
||||
push_decl::<SubscriptionWorkerJob>(&cfg, &mut output);
|
||||
push_decl::<SubscriptionWorker>(&cfg, &mut output);
|
||||
push_decl::<WorkspaceSubscriptionWorkdir>(&cfg, &mut output);
|
||||
push_decl::<SubscriptionSnapshot>(&cfg, &mut output);
|
||||
@@ -232,6 +233,7 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec<SubscriptionFrame> {
|
||||
workspace_id: Some("fixture-workspace".to_string()),
|
||||
display_name: None,
|
||||
profile: None,
|
||||
job: None,
|
||||
workdir_attachments: vec![SubscriptionWorkerWorkdirAttachment {
|
||||
alias: "external".to_string(),
|
||||
repository_id: None,
|
||||
|
||||
@@ -1143,8 +1143,9 @@ impl App {
|
||||
}
|
||||
|
||||
match event {
|
||||
Event::SubmissionAccepted { .. } => {}
|
||||
Event::SubmissionRejected { message, .. } => self.push_error(message),
|
||||
Event::SubmissionAccepted { .. } | Event::NotificationAccepted { .. } => {}
|
||||
Event::SubmissionRejected { message, .. }
|
||||
| Event::NotificationRejected { message, .. } => self.push_error(message),
|
||||
Event::PendingSubmissionsChanged { pending } => {
|
||||
self.pending_submissions = pending;
|
||||
}
|
||||
|
||||
@@ -66,6 +66,12 @@ pub struct WorkerSubmissionAck {
|
||||
pub disposition: protocol::SubmissionDisposition,
|
||||
}
|
||||
|
||||
/// Evidence that a notification request reached the durable Worker session boundary.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WorkerNotificationAck {
|
||||
pub notification_request_id: String,
|
||||
}
|
||||
|
||||
/// Typed execution result class. Results are transient operation outcomes and
|
||||
/// are not persisted as Worker lifecycle authority.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -78,6 +84,8 @@ pub struct WorkerExecutionResult {
|
||||
pub message: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub submission: Option<WorkerSubmissionAck>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub notification: Option<WorkerNotificationAck>,
|
||||
}
|
||||
|
||||
/// Backend result class for a Worker execution operation.
|
||||
@@ -99,6 +107,7 @@ impl WorkerExecutionResult {
|
||||
worker_state: None,
|
||||
message: None,
|
||||
submission: None,
|
||||
notification: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,6 +132,23 @@ impl WorkerExecutionResult {
|
||||
submission_id: submission_id.into(),
|
||||
disposition,
|
||||
}),
|
||||
notification: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn accepted_notification(
|
||||
operation: WorkerExecutionOperation,
|
||||
notification_request_id: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
operation,
|
||||
outcome: WorkerExecutionOutcome::Accepted,
|
||||
worker_state: None,
|
||||
message: None,
|
||||
submission: None,
|
||||
notification: Some(WorkerNotificationAck {
|
||||
notification_request_id: notification_request_id.into(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,6 +159,7 @@ impl WorkerExecutionResult {
|
||||
worker_state: None,
|
||||
message: Some(message.into()),
|
||||
submission: None,
|
||||
notification: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,6 +170,7 @@ impl WorkerExecutionResult {
|
||||
worker_state: None,
|
||||
message: Some(message.into()),
|
||||
submission: None,
|
||||
notification: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,6 +181,7 @@ impl WorkerExecutionResult {
|
||||
worker_state: None,
|
||||
message: Some(message.into()),
|
||||
submission: None,
|
||||
notification: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,6 +192,7 @@ impl WorkerExecutionResult {
|
||||
worker_state: None,
|
||||
message: Some(message.into()),
|
||||
submission: None,
|
||||
notification: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -94,6 +94,9 @@ pub struct WorkerInteractionAck {
|
||||
pub worker_ref: WorkerRef,
|
||||
pub status: WorkerStatus,
|
||||
/// Present for User Submit and absent for non-Submit interactions.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub submission: Option<crate::execution::WorkerSubmissionAck>,
|
||||
/// Present only after a Notify request reaches the Worker's durable receipt boundary.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub notification: Option<crate::execution::WorkerNotificationAck>,
|
||||
}
|
||||
|
||||
@@ -2104,6 +2104,12 @@ impl Runtime {
|
||||
.clone()
|
||||
.expect("User input request id must be assigned before dispatch")
|
||||
});
|
||||
let expected_notification_id = (input.kind == WorkerInputKind::Notify).then(|| {
|
||||
input
|
||||
.submission_request_id
|
||||
.clone()
|
||||
.expect("Notify request id must be assigned before dispatch")
|
||||
});
|
||||
self.ensure_worker_execution(worker_ref)?;
|
||||
let (backend, handle) = {
|
||||
let state = self.lock()?;
|
||||
@@ -2160,7 +2166,28 @@ impl Runtime {
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(expected_notification_id) = expected_notification_id
|
||||
&& dispatch_result
|
||||
.notification
|
||||
.as_ref()
|
||||
.is_none_or(|ack| ack.notification_request_id != expected_notification_id)
|
||||
{
|
||||
let result = WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::Input,
|
||||
"execution backend did not acknowledge the durable Runtime notification request id",
|
||||
);
|
||||
self.record_execution_result(worker_ref, result.clone())?;
|
||||
return Err(RuntimeError::WorkerExecutionRejected {
|
||||
worker_id: worker_ref.worker_id.clone(),
|
||||
operation: result.operation,
|
||||
outcome: result.outcome,
|
||||
message: result.message_or_default(),
|
||||
result,
|
||||
});
|
||||
}
|
||||
|
||||
let submission = dispatch_result.submission.clone();
|
||||
let notification = dispatch_result.notification.clone();
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
let worker = state.worker_mut(worker_ref)?;
|
||||
@@ -2180,6 +2207,7 @@ impl Runtime {
|
||||
worker_ref: worker_ref.clone(),
|
||||
status,
|
||||
submission,
|
||||
notification,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4180,6 +4208,7 @@ impl RuntimeState {
|
||||
workspace_id: worker.workspace_id.clone(),
|
||||
display_name: worker.display_name.clone(),
|
||||
profile,
|
||||
job: None,
|
||||
workdir_attachments,
|
||||
})
|
||||
}
|
||||
@@ -6450,7 +6479,8 @@ mod tests {
|
||||
_handle: &WorkerExecutionHandle,
|
||||
input: WorkerInput,
|
||||
) -> WorkerExecutionResult {
|
||||
let submission_id = input.submission_request_id.clone();
|
||||
let request_id = input.submission_request_id.clone();
|
||||
let input_kind = input.kind.clone();
|
||||
self.dispatched_inputs.lock().unwrap().push(input);
|
||||
let mut result = self
|
||||
.dispatch_result
|
||||
@@ -6458,20 +6488,28 @@ mod tests {
|
||||
.unwrap()
|
||||
.clone()
|
||||
.unwrap_or_else(|| {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
"request-test",
|
||||
"test-submission",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
if input_kind == WorkerInputKind::Notify {
|
||||
WorkerExecutionResult::accepted_notification(
|
||||
WorkerExecutionOperation::Input,
|
||||
request_id
|
||||
.clone()
|
||||
.expect("Runtime assigns Notify request identity before dispatch"),
|
||||
)
|
||||
} else {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
"request-test",
|
||||
"test-submission",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
}
|
||||
});
|
||||
if !self
|
||||
.preserve_submission_acknowledgement_id
|
||||
.load(Ordering::SeqCst)
|
||||
&& let (Some(ack), Some(submission_id)) =
|
||||
(result.submission.as_mut(), submission_id)
|
||||
&& let (Some(ack), Some(request_id)) = (result.submission.as_mut(), request_id)
|
||||
{
|
||||
ack.submission_request_id = submission_id;
|
||||
ack.submission_request_id = request_id;
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -7910,6 +7948,29 @@ mod tests {
|
||||
assert!(runtime.list_workers().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notify_rejects_backend_enqueue_without_durable_worker_receipt() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Input,
|
||||
));
|
||||
let detail = runtime
|
||||
.create_worker(task_request("notify receipt"))
|
||||
.unwrap();
|
||||
let mut input = WorkerInput::notify("do not acknowledge enqueue alone");
|
||||
input.submission_request_id = Some("notification-request".to_string());
|
||||
|
||||
let error = runtime.send_input(&detail.worker_ref, input).unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
RuntimeError::WorkerExecutionRejected {
|
||||
outcome: crate::execution::WorkerExecutionOutcome::Rejected,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connected_backend_busy_dispatch_is_typed_and_not_transcribed() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
|
||||
@@ -45,6 +45,11 @@ use protocol::{Event, Method, Segment, WorkerCommandEnvelope};
|
||||
|
||||
static NEXT_INTERNAL_COMMAND_ID: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
enum NotificationAcceptance {
|
||||
Accepted,
|
||||
Rejected(String),
|
||||
}
|
||||
|
||||
fn rollback_materialized_spawn_workdirs(
|
||||
materializer: &dyn WorkingDirectoryMaterializer,
|
||||
bindings: &BTreeMap<WorkdirAttachmentAlias, WorkingDirectoryBinding>,
|
||||
@@ -1883,6 +1888,75 @@ where
|
||||
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
|
||||
}
|
||||
|
||||
fn send_notification_and_wait_for_acceptance(
|
||||
&self,
|
||||
operation: WorkerExecutionOperation,
|
||||
worker: WorkerHandle,
|
||||
method: Method,
|
||||
notification_request_id: String,
|
||||
) -> WorkerExecutionResult {
|
||||
let request_id = notification_request_id.clone();
|
||||
let result = self.run_cancellable_on_adapter_runtime(USER_INPUT_TASK_TIMEOUT, async move {
|
||||
// Subscribe before enqueueing so a fast durable acceptance cannot
|
||||
// race the Runtime acknowledgement.
|
||||
let mut events = worker.subscribe();
|
||||
worker
|
||||
.send(method)
|
||||
.await
|
||||
.map_err(|err| format!("failed to send Worker method: {err}"))?;
|
||||
|
||||
tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move {
|
||||
loop {
|
||||
match events.recv().await {
|
||||
Ok(Event::NotificationAccepted {
|
||||
notification_request_id,
|
||||
}) if notification_request_id == request_id => {
|
||||
return Ok(NotificationAcceptance::Accepted);
|
||||
}
|
||||
Ok(Event::NotificationRejected {
|
||||
notification_request_id,
|
||||
message,
|
||||
}) if notification_request_id == request_id => {
|
||||
return Ok(NotificationAcceptance::Rejected(message));
|
||||
}
|
||||
Ok(Event::Shutdown) => {
|
||||
return Err(
|
||||
"worker shut down before notification was durably accepted"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
|
||||
return Err(format!(
|
||||
"worker notification acknowledgement lagged by {skipped} protocol event(s)"
|
||||
));
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
return Err(
|
||||
"worker event stream closed before notification was durably accepted"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
"timed out waiting for durable Worker notification acceptance".to_string()
|
||||
})?
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(NotificationAcceptance::Accepted) => {
|
||||
WorkerExecutionResult::accepted_notification(operation, notification_request_id)
|
||||
}
|
||||
Ok(NotificationAcceptance::Rejected(message)) => {
|
||||
WorkerExecutionResult::rejected(operation, message)
|
||||
}
|
||||
Err(message) => WorkerExecutionResult::errored(operation, message),
|
||||
}
|
||||
}
|
||||
|
||||
fn retain_uncertain_unconnected_controller(
|
||||
&self,
|
||||
worker_ref: &crate::identity::WorkerRef,
|
||||
@@ -2879,16 +2953,17 @@ where
|
||||
let notification_request_id = input
|
||||
.submission_request_id
|
||||
.unwrap_or_else(protocol::new_submission_request_id);
|
||||
return self.send_method(
|
||||
return self.send_notification_and_wait_for_acceptance(
|
||||
WorkerExecutionOperation::Input,
|
||||
worker,
|
||||
Method::NotifyTracked {
|
||||
notification_request_id: notification_request_id.clone(),
|
||||
message: input.content,
|
||||
source: protocol::AuthenticatedInputSource::Backend {
|
||||
operation_id: notification_request_id,
|
||||
operation_id: notification_request_id.clone(),
|
||||
},
|
||||
},
|
||||
notification_request_id,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -5108,7 +5183,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn running_worker_accepts_notify_without_a_submit_receipt() {
|
||||
fn running_worker_waits_for_durable_notify_receipt() {
|
||||
let client = MockClient::sequential(vec![MockResponse::Hang(vec![])]);
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
let cwd = tempfile::tempdir().unwrap();
|
||||
@@ -5143,6 +5218,13 @@ mod tests {
|
||||
.expect("Running Worker must accept Notify without a Submit receipt");
|
||||
|
||||
assert!(acknowledgement.submission.is_none());
|
||||
assert_eq!(
|
||||
acknowledgement
|
||||
.notification
|
||||
.as_ref()
|
||||
.map(|ack| ack.notification_request_id.as_str()),
|
||||
Some("notification-request")
|
||||
);
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running);
|
||||
}
|
||||
|
||||
|
||||
+156
-113
@@ -1178,39 +1178,42 @@ where
|
||||
),
|
||||
None => durable_parent_notifications,
|
||||
};
|
||||
let backend_job_profile = worker.manifest().engine.instruction == "internal.backend_job_system";
|
||||
let prompts = worker.prompts().clone();
|
||||
let paste_store = worker.store().clone();
|
||||
let paste_session_id = worker.session_id();
|
||||
worker
|
||||
.engine_mut()
|
||||
.register_tool(crate::paste_artifact_tool::search_input_artifact_tool(
|
||||
paste_store.clone(),
|
||||
paste_session_id,
|
||||
));
|
||||
worker
|
||||
.engine_mut()
|
||||
.register_tool(crate::paste_artifact_tool::read_input_artifact_tool(
|
||||
paste_store,
|
||||
paste_session_id,
|
||||
));
|
||||
// Keep the alias-routed schemas installed for the Worker lifetime. The
|
||||
// attachment set may be empty or mutate after registration, so target
|
||||
// resolution and provider capability checks happen per invocation.
|
||||
let tracker = tools::Tracker::new();
|
||||
let workdir_tools = if restrict_workdir_tools_to_current_capabilities {
|
||||
tools::routed_builtin_tools_for_current_capabilities(
|
||||
workdir_sessions.clone(),
|
||||
tracker.clone(),
|
||||
bash_output_dir.clone(),
|
||||
)
|
||||
} else {
|
||||
tools::routed_builtin_tools(
|
||||
workdir_sessions.clone(),
|
||||
tracker.clone(),
|
||||
bash_output_dir.clone(),
|
||||
)
|
||||
};
|
||||
worker.engine_mut().register_tools(workdir_tools);
|
||||
if !backend_job_profile {
|
||||
let paste_store = worker.store().clone();
|
||||
let paste_session_id = worker.session_id();
|
||||
worker
|
||||
.engine_mut()
|
||||
.register_tool(crate::paste_artifact_tool::search_input_artifact_tool(
|
||||
paste_store.clone(),
|
||||
paste_session_id,
|
||||
));
|
||||
worker
|
||||
.engine_mut()
|
||||
.register_tool(crate::paste_artifact_tool::read_input_artifact_tool(
|
||||
paste_store,
|
||||
paste_session_id,
|
||||
));
|
||||
// Keep the alias-routed schemas installed for the Worker lifetime. The
|
||||
// attachment set may be empty or mutate after registration, so target
|
||||
// resolution and provider capability checks happen per invocation.
|
||||
let workdir_tools = if restrict_workdir_tools_to_current_capabilities {
|
||||
tools::routed_builtin_tools_for_current_capabilities(
|
||||
workdir_sessions.clone(),
|
||||
tracker.clone(),
|
||||
bash_output_dir.clone(),
|
||||
)
|
||||
} else {
|
||||
tools::routed_builtin_tools(
|
||||
workdir_sessions.clone(),
|
||||
tracker.clone(),
|
||||
bash_output_dir.clone(),
|
||||
)
|
||||
};
|
||||
worker.engine_mut().register_tools(workdir_tools);
|
||||
}
|
||||
if feature_config.image.enabled && model_supports_image_attachments(&spawner_manifest.model) {
|
||||
worker
|
||||
.engine_mut()
|
||||
@@ -1225,6 +1228,20 @@ where
|
||||
let worker_enabled = feature_config.worker.enabled;
|
||||
let sub_worker_enabled = feature_config.sub_worker.enabled;
|
||||
let mut feature_registry = FeatureRegistryBuilder::new();
|
||||
if backend_job_profile {
|
||||
let workspace_client = worker.workspace_client_handle();
|
||||
if !workspace_client.is_available() || workspace_client.workspace_id().is_none() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"Backend Job result capability requires Backend Workspace API authority",
|
||||
));
|
||||
}
|
||||
feature_registry.add_module(
|
||||
crate::feature::builtin::backend_job_result::BackendJobResultFeature::new(
|
||||
workspace_client,
|
||||
),
|
||||
);
|
||||
}
|
||||
let memory_install_plan = crate::feature::builtin::memory::MemoryFeatureInstallPlan::prepare(
|
||||
worker.manifest(),
|
||||
worker.workspace_client_handle(),
|
||||
@@ -1472,6 +1489,40 @@ where
|
||||
Ok(workdir_for_view)
|
||||
}
|
||||
|
||||
fn durably_accept_notification<St>(
|
||||
notification_request_id: String,
|
||||
message: String,
|
||||
source_namespace: String,
|
||||
provenance: session_store::LoggedSessionHistoryOrigin,
|
||||
pending_submissions: &PendingSubmissionHandle<St>,
|
||||
working_event_tx: &broadcast::Sender<Event>,
|
||||
) -> bool
|
||||
where
|
||||
St: session_store::Store + Clone,
|
||||
{
|
||||
let request_id = notification_request_id.clone();
|
||||
match pending_submissions.accept_notification_from_source(
|
||||
notification_request_id,
|
||||
message,
|
||||
source_namespace,
|
||||
provenance,
|
||||
) {
|
||||
Ok(_) => {
|
||||
let _ = working_event_tx.send(Event::NotificationAccepted {
|
||||
notification_request_id: request_id,
|
||||
});
|
||||
true
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = working_event_tx.send(Event::NotificationRejected {
|
||||
notification_request_id: request_id,
|
||||
message: error.to_string(),
|
||||
});
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn durably_accept_method_while_busy<St>(
|
||||
method: Method,
|
||||
pending_submissions: &PendingSubmissionHandle<St>,
|
||||
@@ -1544,17 +1595,14 @@ where
|
||||
notification_request_id,
|
||||
message,
|
||||
} => {
|
||||
if let Err(error) = pending_submissions.accept_notification_from_source(
|
||||
durably_accept_notification(
|
||||
notification_request_id,
|
||||
message,
|
||||
pending_submissions.direct_client_namespace(),
|
||||
session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
) {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
pending_submissions,
|
||||
working_event_tx,
|
||||
);
|
||||
None
|
||||
}
|
||||
Method::NotifyTracked {
|
||||
@@ -1564,17 +1612,14 @@ where
|
||||
} => {
|
||||
let (source_namespace, provenance) =
|
||||
resolved_input_source(pending_submissions, &source);
|
||||
if let Err(error) = pending_submissions.accept_notification_from_source(
|
||||
durably_accept_notification(
|
||||
notification_request_id,
|
||||
message,
|
||||
source_namespace,
|
||||
provenance,
|
||||
) {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
pending_submissions,
|
||||
working_event_tx,
|
||||
);
|
||||
None
|
||||
}
|
||||
method => Some(method),
|
||||
@@ -1602,9 +1647,16 @@ fn reject_method_while_attention_locked(
|
||||
});
|
||||
true
|
||||
}
|
||||
Method::Notify { .. } | Method::NotifyTracked { .. } => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
Method::Notify {
|
||||
notification_request_id,
|
||||
..
|
||||
}
|
||||
| Method::NotifyTracked {
|
||||
notification_request_id,
|
||||
..
|
||||
} => {
|
||||
let _ = working_event_tx.send(Event::NotificationRejected {
|
||||
notification_request_id: notification_request_id.clone(),
|
||||
message: message.to_owned(),
|
||||
});
|
||||
true
|
||||
@@ -2014,21 +2066,14 @@ async fn controller_loop<C, St>(
|
||||
notification_request_id,
|
||||
message,
|
||||
} => {
|
||||
let source_namespace = pending_submissions.direct_client_namespace();
|
||||
match pending_submissions.accept_notification_from_source(
|
||||
durably_accept_notification(
|
||||
notification_request_id,
|
||||
message,
|
||||
source_namespace.clone(),
|
||||
pending_submissions.direct_client_namespace(),
|
||||
session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
) {
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
&pending_submissions,
|
||||
&working_event_tx,
|
||||
);
|
||||
}
|
||||
|
||||
Method::NotifyTracked {
|
||||
@@ -2038,20 +2083,14 @@ async fn controller_loop<C, St>(
|
||||
} => {
|
||||
let (source_namespace, provenance) =
|
||||
resolved_input_source(&pending_submissions, &source);
|
||||
match pending_submissions.accept_notification_from_source(
|
||||
durably_accept_notification(
|
||||
notification_request_id,
|
||||
message,
|
||||
source_namespace.clone(),
|
||||
source_namespace,
|
||||
provenance,
|
||||
) {
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
&pending_submissions,
|
||||
&working_event_tx,
|
||||
);
|
||||
}
|
||||
|
||||
Method::ListPendingSubmissions => {
|
||||
@@ -3096,22 +3135,15 @@ where
|
||||
notification_request_id,
|
||||
message,
|
||||
}) => {
|
||||
let source_namespace = pending_submissions.direct_client_namespace();
|
||||
match pending_submissions.accept_notification_from_source(
|
||||
if durably_accept_notification(
|
||||
notification_request_id,
|
||||
message,
|
||||
source_namespace.clone(),
|
||||
pending_submissions.direct_client_namespace(),
|
||||
session_store::LoggedSessionHistoryOrigin::LegacyUnknown,
|
||||
pending_submissions,
|
||||
working_event_tx,
|
||||
) {
|
||||
Ok(_) => {
|
||||
stage_pending_notifications(&pending_submissions, notify_buffer);
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
stage_pending_notifications(&pending_submissions, notify_buffer);
|
||||
}
|
||||
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
|
||||
pending: pending_submissions.snapshot(),
|
||||
@@ -3124,21 +3156,15 @@ where
|
||||
}) => {
|
||||
let (source_namespace, provenance) =
|
||||
resolved_input_source(pending_submissions, &source);
|
||||
match pending_submissions.accept_notification_from_source(
|
||||
if durably_accept_notification(
|
||||
notification_request_id,
|
||||
message,
|
||||
source_namespace.clone(),
|
||||
source_namespace,
|
||||
provenance,
|
||||
pending_submissions,
|
||||
working_event_tx,
|
||||
) {
|
||||
Ok(_) => {
|
||||
stage_pending_notifications(&pending_submissions, notify_buffer);
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
stage_pending_notifications(&pending_submissions, notify_buffer);
|
||||
}
|
||||
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
|
||||
pending: pending_submissions.snapshot(),
|
||||
@@ -3605,29 +3631,46 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn busy_notification_is_durable_before_shutdown_can_observe_it() {
|
||||
fn busy_notification_receipt_follows_durable_idempotent_acceptance() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let pending = PendingSubmissionHandle::for_test(dir.path());
|
||||
let (event_tx, _event_rx) = broadcast::channel(4);
|
||||
let (event_tx, mut event_rx) = broadcast::channel(4);
|
||||
let method = Method::NotifyTracked {
|
||||
notification_request_id: "completion-1".into(),
|
||||
message: "subworker completed".into(),
|
||||
source: protocol::AuthenticatedInputSource::UntrustedWire,
|
||||
};
|
||||
|
||||
assert!(
|
||||
durably_accept_method_while_busy(
|
||||
Method::NotifyTracked {
|
||||
notification_request_id: "completion-1".into(),
|
||||
message: "subworker completed".into(),
|
||||
source: protocol::AuthenticatedInputSource::UntrustedWire,
|
||||
},
|
||||
&pending,
|
||||
&event_tx,
|
||||
)
|
||||
.is_none()
|
||||
for _ in 0..2 {
|
||||
assert!(
|
||||
durably_accept_method_while_busy(method.clone(), &pending, &event_tx).is_none()
|
||||
);
|
||||
let entries = pending.persisted_entries_for_test();
|
||||
assert!(entries.iter().any(|entry| matches!(
|
||||
entry,
|
||||
LogEntry::Extension { domain, .. }
|
||||
if domain == "worker.pending_activations.v1"
|
||||
)));
|
||||
assert!(matches!(
|
||||
event_rx.try_recv().unwrap(),
|
||||
Event::NotificationAccepted {
|
||||
notification_request_id,
|
||||
} if notification_request_id == "completion-1"
|
||||
));
|
||||
}
|
||||
assert_eq!(pending.snapshot().notification_count, 1);
|
||||
assert_eq!(
|
||||
pending
|
||||
.persisted_entries_for_test()
|
||||
.iter()
|
||||
.filter(|entry| matches!(
|
||||
entry,
|
||||
LogEntry::Extension { domain, .. }
|
||||
if domain == "worker.pending_activations.v1"
|
||||
))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
let entries = pending.persisted_entries_for_test();
|
||||
assert!(entries.iter().any(|entry| matches!(
|
||||
entry,
|
||||
LogEntry::Extension { domain, .. }
|
||||
if domain == "worker.pending_activations.v1"
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
//! same descriptor-approved registry path used by feature modules. They are not
|
||||
//! an external plugin-loading surface.
|
||||
|
||||
pub mod backend_job_result;
|
||||
pub mod flow_transition;
|
||||
pub mod manage_workdir;
|
||||
pub mod manage_worker;
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
//! Structured completion capability for Backend-owned Job Workers.
|
||||
//!
|
||||
//! The tool intentionally accepts only the result value. Workspace and Worker
|
||||
//! identity come from the authenticated Workspace client, while Job, attempt,
|
||||
//! and input revision are resolved from the Backend-owned binding.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
|
||||
use async_trait::async_trait;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::feature::{
|
||||
FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureModule, ToolContribution,
|
||||
ToolDeclaration, ToolDefinition,
|
||||
};
|
||||
use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod};
|
||||
|
||||
const TOOL_NAME: &str = "SubmitBackendJobResult";
|
||||
const TOOL_DESCRIPTION: &str = "Submit the bounded structured result for this Backend-owned Job attempt. Workspace, Job, attempt, input revision, and source Worker identity are resolved and fenced by Backend authority; normal final prose is not success.";
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SubmitResultInput {
|
||||
result: Value,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SubmitResultTool {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for SubmitResultTool {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
_context: ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let input: SubmitResultInput = serde_json::from_str(input_json)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
let workspace_id = self.client.workspace_id().ok_or_else(|| {
|
||||
ToolError::ExecutionFailed(
|
||||
"Backend Job result submission requires Workspace identity".to_string(),
|
||||
)
|
||||
})?;
|
||||
let response = self
|
||||
.client
|
||||
.execute(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
format!("/api/w/{workspace_id}/workers/self/backend-job-result"),
|
||||
serde_json::to_string(&serde_json::json!({ "result": input.result }))
|
||||
.map_err(|error| ToolError::Internal(error.to_string()))?,
|
||||
))
|
||||
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||
if !response.is_success() {
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Backend Job result submission was rejected with HTTP {}: {}",
|
||||
response.status, response.body
|
||||
)));
|
||||
}
|
||||
Ok(ToolOutput {
|
||||
summary: "Backend Job result accepted".to_string(),
|
||||
content: Some(response.body),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn definition(client: Arc<dyn WorkspaceClient>) -> ToolDefinition {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(SubmitResultInput))
|
||||
.expect("Backend Job result schema must serialize");
|
||||
Arc::new(move || {
|
||||
(
|
||||
ToolMeta::new(TOOL_NAME)
|
||||
.description(TOOL_DESCRIPTION)
|
||||
.input_schema(schema.clone()),
|
||||
Arc::new(SubmitResultTool {
|
||||
client: client.clone(),
|
||||
}) as Arc<dyn Tool>,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct BackendJobResultFeature {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
}
|
||||
|
||||
impl BackendJobResultFeature {
|
||||
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
}
|
||||
|
||||
impl FeatureModule for BackendJobResultFeature {
|
||||
fn descriptor(&self) -> FeatureDescriptor {
|
||||
FeatureDescriptor::builtin("backend-job-result", "Backend Job result")
|
||||
.with_description("Single-purpose structured result capability for Backend-owned Jobs.")
|
||||
.with_tool(ToolDeclaration::new(TOOL_NAME, TOOL_DESCRIPTION))
|
||||
}
|
||||
|
||||
fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> {
|
||||
context.tools().register(ToolContribution::new(
|
||||
TOOL_NAME,
|
||||
definition(self.client.clone()),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::feature::FeatureRegistryBuilder;
|
||||
use crate::hook::HookRegistryBuilder;
|
||||
use crate::worker::{WorkspaceClientError, WorkspaceResponse};
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct RecordingClient {
|
||||
requests: Mutex<Vec<WorkspaceRequest>>,
|
||||
}
|
||||
|
||||
impl WorkspaceClient for RecordingClient {
|
||||
fn workspace_id(&self) -> Option<&str> {
|
||||
Some("workspace-test")
|
||||
}
|
||||
fn kind(&self) -> &str {
|
||||
"test"
|
||||
}
|
||||
fn is_available(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn execute(
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
self.requests.lock().unwrap().push(request);
|
||||
Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: "{\"replayed\":false}".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feature_exposes_only_identity_free_structured_result_input() {
|
||||
let client = Arc::new(RecordingClient::default());
|
||||
let mut pending = Vec::new();
|
||||
let mut hooks = HookRegistryBuilder::default();
|
||||
let report = FeatureRegistryBuilder::new()
|
||||
.with_module(BackendJobResultFeature::new(client.clone()))
|
||||
.install_into_pending(&mut pending, &mut hooks);
|
||||
assert_eq!(report.installed_tool_names(), [TOOL_NAME]);
|
||||
let (meta, tool) = pending.remove(0)();
|
||||
assert_eq!(meta.input_schema["required"], serde_json::json!(["result"]));
|
||||
assert!(meta.input_schema["properties"].get("job_id").is_none());
|
||||
tool.execute(
|
||||
r#"{"result":{"valid":true}}"#,
|
||||
ToolExecutionContext::direct(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert_eq!(requests[0].method, WorkspaceRequestMethod::Post);
|
||||
assert!(
|
||||
requests[0]
|
||||
.path
|
||||
.ends_with("/workers/self/backend-job-result")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -95,6 +95,7 @@ pub enum WorkerPrompt {
|
||||
MemoryExtractSystem,
|
||||
MemoryConsolidationSystem,
|
||||
FlowVerifierSystem,
|
||||
BackendJobSystem,
|
||||
NotifyWrapper,
|
||||
InterruptToolResultSummary,
|
||||
InterruptSystemNote,
|
||||
@@ -112,6 +113,7 @@ impl WorkerPrompt {
|
||||
Self::MemoryExtractSystem => "internal.memory_extract_system",
|
||||
Self::MemoryConsolidationSystem => "internal.memory_consolidation_system",
|
||||
Self::FlowVerifierSystem => "internal.flow_verifier_system",
|
||||
Self::BackendJobSystem => "internal.backend_job_system",
|
||||
Self::NotifyWrapper => "internal.notify_wrapper",
|
||||
Self::InterruptToolResultSummary => "internal.interrupt_tool_result_summary",
|
||||
Self::InterruptSystemNote => "internal.interrupt_system_note",
|
||||
@@ -130,6 +132,7 @@ impl WorkerPrompt {
|
||||
WorkerPrompt::MemoryExtractSystem,
|
||||
WorkerPrompt::MemoryConsolidationSystem,
|
||||
WorkerPrompt::FlowVerifierSystem,
|
||||
WorkerPrompt::BackendJobSystem,
|
||||
WorkerPrompt::NotifyWrapper,
|
||||
WorkerPrompt::InterruptToolResultSummary,
|
||||
WorkerPrompt::InterruptSystemNote,
|
||||
|
||||
@@ -837,6 +837,39 @@ async fn feature_flags_default_to_core_tool_surface_only() {
|
||||
assert!(!names.iter().any(|name| name == "SubWorkerSpawn"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_profile_exposes_only_structured_result_capability() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let resolved = ProfileResolver::new()
|
||||
.with_workspace_base(workspace.path())
|
||||
.resolve(
|
||||
&ProfileSelector::source_named(ProfileRegistrySource::Builtin, "backend-job"),
|
||||
ProfileResolveOptions::with_worker_name("backend-job-worker"),
|
||||
)
|
||||
.unwrap();
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let client_for_assert = client.clone();
|
||||
let (worker, _pwd) = make_worker_with_manifest_and_workspace_context(
|
||||
client,
|
||||
resolved.manifest,
|
||||
WorkerWorkspaceContext::with_client(None, Arc::new(AvailableWorkspaceClient)),
|
||||
)
|
||||
.await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Run the bounded Backend Job.",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
assert_eq!(request_tool_names(&request), vec!["SubmitBackendJobResult"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enabled_task_and_web_features_register_their_tools() {
|
||||
let manifest = r#"
|
||||
|
||||
@@ -0,0 +1,538 @@
|
||||
//! Durable Backend-owned jobs executed by ordinary Runtime-managed Workers.
|
||||
//!
|
||||
//! A job is durable intent. An attempt binds that intent to one dedicated
|
||||
//! Worker, and a result is accepted only through the fenced structured-result
|
||||
//! API below. Runtime lifecycle and final prose are deliberately not success
|
||||
//! authority.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
use worker_runtime::identity::RuntimeWorkerRef;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
pub const BACKEND_JOB_RUNTIME_ID: &str = crate::hosts::EMBEDDED_RUNTIME_ID;
|
||||
pub const DEFAULT_MAX_CONCURRENT_JOBS: u16 = 8;
|
||||
pub const ABSOLUTE_MAX_CONCURRENT_JOBS: u16 = 64;
|
||||
pub const DEFAULT_JOB_TIMEOUT_SECONDS: u32 = 120;
|
||||
pub const MAX_JOB_TIMEOUT_SECONDS: u32 = 600;
|
||||
pub const DEFAULT_MAX_RESULT_BYTES: u32 = 16 * 1024;
|
||||
pub const ABSOLUTE_MAX_RESULT_BYTES: u32 = 64 * 1024;
|
||||
pub const DEFAULT_MAX_ATTEMPTS: u8 = 2;
|
||||
pub const ABSOLUTE_MAX_ATTEMPTS: u8 = 3;
|
||||
pub const MAX_JOB_INPUT_BYTES: usize = 128 * 1024;
|
||||
pub const MAX_JOB_INSTRUCTION_BYTES: usize = 16 * 1024;
|
||||
pub const MAX_JOB_PURPOSE_BYTES: usize = 120;
|
||||
pub const MAX_JOB_REFERENCE_BYTES: usize = 512;
|
||||
pub const MAX_JOB_FAILURE_BYTES: usize = 1024;
|
||||
pub const MAX_JOB_DELIVERY_BYTES: usize = 16 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BackendJobState {
|
||||
Pending,
|
||||
Completed,
|
||||
Failed,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl BackendJobState {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Pending => "pending",
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse(value: &str) -> Result<Self> {
|
||||
match value {
|
||||
"pending" => Ok(Self::Pending),
|
||||
"completed" => Ok(Self::Completed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
"unknown" => Ok(Self::Unknown),
|
||||
other => Err(Error::Store(format!("unknown Backend Job state `{other}`"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BackendJobAttemptState {
|
||||
Reserved,
|
||||
Dispatched,
|
||||
Completed,
|
||||
Failed,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl BackendJobAttemptState {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Reserved => "reserved",
|
||||
Self::Dispatched => "dispatched",
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse(value: &str) -> Result<Self> {
|
||||
match value {
|
||||
"reserved" => Ok(Self::Reserved),
|
||||
"dispatched" => Ok(Self::Dispatched),
|
||||
"completed" => Ok(Self::Completed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
"unknown" => Ok(Self::Unknown),
|
||||
other => Err(Error::Store(format!(
|
||||
"unknown Backend Job attempt state `{other}`"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn terminal(self) -> bool {
|
||||
matches!(self, Self::Completed | Self::Failed | Self::Unknown)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BackendJobDeliveryState {
|
||||
Pending,
|
||||
Sending,
|
||||
Completed,
|
||||
Failed,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl BackendJobDeliveryState {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Pending => "pending",
|
||||
Self::Sending => "sending",
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse(value: &str) -> Result<Self> {
|
||||
match value {
|
||||
"pending" => Ok(Self::Pending),
|
||||
"sending" => Ok(Self::Sending),
|
||||
"completed" => Ok(Self::Completed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
"unknown" => Ok(Self::Unknown),
|
||||
other => Err(Error::Store(format!(
|
||||
"unknown Backend Job delivery state `{other}`"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobLimits {
|
||||
#[serde(default = "default_max_concurrent_jobs")]
|
||||
pub max_concurrent_jobs: u16,
|
||||
#[serde(default = "default_job_timeout_seconds")]
|
||||
pub timeout_seconds: u32,
|
||||
#[serde(default = "default_max_result_bytes")]
|
||||
pub max_result_bytes: u32,
|
||||
#[serde(default = "default_max_attempts")]
|
||||
pub max_attempts: u8,
|
||||
}
|
||||
|
||||
impl Default for BackendJobLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_concurrent_jobs: DEFAULT_MAX_CONCURRENT_JOBS,
|
||||
timeout_seconds: DEFAULT_JOB_TIMEOUT_SECONDS,
|
||||
max_result_bytes: DEFAULT_MAX_RESULT_BYTES,
|
||||
max_attempts: DEFAULT_MAX_ATTEMPTS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendJobLimits {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.max_concurrent_jobs == 0 || self.max_concurrent_jobs > ABSOLUTE_MAX_CONCURRENT_JOBS
|
||||
{
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_concurrent_jobs must be in 1..={ABSOLUTE_MAX_CONCURRENT_JOBS}"
|
||||
)));
|
||||
}
|
||||
if self.timeout_seconds == 0 || self.timeout_seconds > MAX_JOB_TIMEOUT_SECONDS {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job timeout_seconds must be in 1..={MAX_JOB_TIMEOUT_SECONDS}"
|
||||
)));
|
||||
}
|
||||
if self.max_result_bytes == 0 || self.max_result_bytes > ABSOLUTE_MAX_RESULT_BYTES {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_result_bytes must be in 1..={ABSOLUTE_MAX_RESULT_BYTES}"
|
||||
)));
|
||||
}
|
||||
if self.max_attempts == 0 || self.max_attempts > ABSOLUTE_MAX_ATTEMPTS {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_attempts must be in 1..={ABSOLUTE_MAX_ATTEMPTS}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const fn default_max_concurrent_jobs() -> u16 {
|
||||
DEFAULT_MAX_CONCURRENT_JOBS
|
||||
}
|
||||
const fn default_job_timeout_seconds() -> u32 {
|
||||
DEFAULT_JOB_TIMEOUT_SECONDS
|
||||
}
|
||||
const fn default_max_result_bytes() -> u32 {
|
||||
DEFAULT_MAX_RESULT_BYTES
|
||||
}
|
||||
const fn default_max_attempts() -> u8 {
|
||||
DEFAULT_MAX_ATTEMPTS
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobRequest {
|
||||
/// Stable caller-selected idempotency key within one Workspace.
|
||||
pub job_id: String,
|
||||
/// Stable machine-readable use-site name, for example `ticket_item_check`.
|
||||
pub purpose: String,
|
||||
/// Revision fenced by the structured result.
|
||||
pub input_revision: String,
|
||||
/// Durable domain reference used to reload or explain the input.
|
||||
pub input_ref: String,
|
||||
/// Bounded immutable input snapshot. This must not contain credentials.
|
||||
pub input: Value,
|
||||
/// Bounded task instruction; the profile supplies the system instruction.
|
||||
pub instruction: String,
|
||||
/// Existing configured profile selector. Job dispatch never accepts raw profile source.
|
||||
pub profile: String,
|
||||
/// Writer provenance and optional advisory destination, not parent ownership.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub source_worker: Option<RuntimeWorkerRef>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub notification_target: Option<RuntimeWorkerRef>,
|
||||
#[serde(default)]
|
||||
pub limits: BackendJobLimits,
|
||||
}
|
||||
|
||||
impl BackendJobRequest {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
validate_identifier("job_id", &self.job_id, 256)?;
|
||||
validate_identifier("purpose", &self.purpose, MAX_JOB_PURPOSE_BYTES)?;
|
||||
validate_identifier("input_revision", &self.input_revision, 256)?;
|
||||
validate_bounded_text("input_ref", &self.input_ref, MAX_JOB_REFERENCE_BYTES)?;
|
||||
validate_bounded_text("instruction", &self.instruction, MAX_JOB_INSTRUCTION_BYTES)?;
|
||||
validate_identifier("profile", &self.profile, 256)?;
|
||||
if self.profile != "builtin:backend-job" {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Jobs must use the dedicated builtin:backend-job profile".to_string(),
|
||||
));
|
||||
}
|
||||
let input = serde_json::to_vec(&self.input).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job input: {error}"))
|
||||
})?;
|
||||
if input.len() > MAX_JOB_INPUT_BYTES {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job input exceeds {MAX_JOB_INPUT_BYTES} bytes"
|
||||
)));
|
||||
}
|
||||
self.limits.validate()
|
||||
}
|
||||
|
||||
pub fn fingerprint(&self) -> Result<String> {
|
||||
self.validate()?;
|
||||
let encoded = serde_json::to_vec(self).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job intent: {error}"))
|
||||
})?;
|
||||
Ok(sha256(&encoded))
|
||||
}
|
||||
|
||||
pub fn worker_input(&self, attempt_id: &str) -> Result<String> {
|
||||
let input = serde_json::to_string(&self.input).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job input: {error}"))
|
||||
})?;
|
||||
Ok(format!(
|
||||
"{}\n\nBackend Job envelope (immutable):\njob_id: {}\nattempt_id: {}\ninput_revision: {}\ninput_ref: {}\ninput_json: {}\n\nReturn success only through the structured Backend Job result capability. Final prose and Worker Idle/Stopped state are not result authority.",
|
||||
self.instruction, self.job_id, attempt_id, self.input_revision, self.input_ref, input
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct BackendJobRecord {
|
||||
pub workspace_id: String,
|
||||
pub request: BackendJobRequest,
|
||||
pub intent_fingerprint: String,
|
||||
pub state: BackendJobState,
|
||||
pub current_attempt: u8,
|
||||
pub result: Option<Value>,
|
||||
pub result_digest: Option<String>,
|
||||
pub failure_category: Option<String>,
|
||||
pub failure_detail: Option<String>,
|
||||
pub created_at: String,
|
||||
pub updated_at: String,
|
||||
pub completed_at: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct BackendJobAttemptRecord {
|
||||
pub workspace_id: String,
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub attempt: u8,
|
||||
pub input_revision: String,
|
||||
pub state: BackendJobAttemptState,
|
||||
pub worker: Option<RuntimeWorkerRef>,
|
||||
pub runtime_run_id: Option<String>,
|
||||
pub dispatched_at: Option<String>,
|
||||
pub deadline_at: String,
|
||||
pub result: Option<Value>,
|
||||
pub result_digest: Option<String>,
|
||||
pub failure_category: Option<String>,
|
||||
pub failure_detail: Option<String>,
|
||||
pub created_at: String,
|
||||
pub updated_at: String,
|
||||
pub completed_at: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct BackendJobReservation {
|
||||
pub job: BackendJobRecord,
|
||||
pub attempt: BackendJobAttemptRecord,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobResultSubmission {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub input_revision: String,
|
||||
pub result: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct BackendJobResultAcceptance {
|
||||
pub job: BackendJobRecord,
|
||||
pub attempt: BackendJobAttemptRecord,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BackendJobWorkerBinding {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub purpose: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BackendJobDeliveryRecord {
|
||||
pub workspace_id: String,
|
||||
pub delivery_id: String,
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub target: RuntimeWorkerRef,
|
||||
pub state: BackendJobDeliveryState,
|
||||
pub failure_category: Option<String>,
|
||||
pub failure_detail: Option<String>,
|
||||
pub created_at: String,
|
||||
pub updated_at: String,
|
||||
pub delivered_at: Option<String>,
|
||||
}
|
||||
|
||||
pub fn attempt_id(job_id: &str, attempt: u8) -> String {
|
||||
format!("{job_id}:attempt:{attempt}")
|
||||
}
|
||||
|
||||
pub fn allocation_key(attempt_id: &str) -> String {
|
||||
format!("backend-job:{attempt_id}")
|
||||
}
|
||||
|
||||
/// Fixed-size Runtime request identity for the exact Job attempt. Caller-selected
|
||||
/// Job ids may be longer than the Worker protocol request-id bound, so never
|
||||
/// embed them directly in tracked Runtime identities.
|
||||
pub fn runtime_run_id(attempt_id: &str) -> String {
|
||||
tracked_request_id("run", attempt_id.as_bytes())
|
||||
}
|
||||
|
||||
/// Fixed-size durable delivery identity, also reused as the Worker's tracked
|
||||
/// notification request id so restart replay converges on one receipt.
|
||||
pub fn delivery_id(job_id: &str, attempt_id: &str) -> String {
|
||||
let mut identity = Vec::with_capacity(job_id.len() + attempt_id.len() + 1);
|
||||
identity.extend_from_slice(job_id.as_bytes());
|
||||
identity.push(0);
|
||||
identity.extend_from_slice(attempt_id.as_bytes());
|
||||
tracked_request_id("notification", &identity)
|
||||
}
|
||||
|
||||
fn tracked_request_id(kind: &str, identity: &[u8]) -> String {
|
||||
format!("backend-job-{kind}:{}", sha256(identity))
|
||||
}
|
||||
|
||||
pub fn result_digest(result: &Value, max_bytes: u32) -> Result<(String, String)> {
|
||||
let encoded = serde_json::to_string(result)
|
||||
.map_err(|error| Error::InvalidInput(format!("serialize Backend Job result: {error}")))?;
|
||||
if encoded.len() > max_bytes as usize {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job result exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok((sha256(encoded.as_bytes()), encoded))
|
||||
}
|
||||
|
||||
pub(crate) fn bounded_failure_detail(detail: &str) -> String {
|
||||
if detail.len() <= MAX_JOB_FAILURE_BYTES {
|
||||
return detail.to_string();
|
||||
}
|
||||
let mut end = MAX_JOB_FAILURE_BYTES;
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
detail[..end].to_string()
|
||||
}
|
||||
|
||||
fn validate_identifier(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
validate_bounded_text(field, value, max_bytes)?;
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} must not contain control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bounded_text(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
if value.is_empty() || value.trim() != value {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} must be non-empty and trimmed"
|
||||
)));
|
||||
}
|
||||
if value.len() > max_bytes {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sha256(bytes: &[u8]) -> String {
|
||||
let digest = Sha256::digest(bytes);
|
||||
let mut output = String::with_capacity(7 + digest.len() * 2);
|
||||
output.push_str("sha256:");
|
||||
for byte in digest {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(output, "{byte:02x}");
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn request() -> BackendJobRequest {
|
||||
BackendJobRequest {
|
||||
job_id: "ticket-check:T-1:r7".to_string(),
|
||||
purpose: "ticket_item_check".to_string(),
|
||||
input_revision: "7".to_string(),
|
||||
input_ref: "ticket://T-1/revisions/7".to_string(),
|
||||
input: serde_json::json!({"title": "Check me"}),
|
||||
instruction: "Check this immutable Ticket snapshot.".to_string(),
|
||||
profile: "builtin:backend-job".to_string(),
|
||||
source_worker: Some(RuntimeWorkerRef::new("runtime-a", "worker-a")),
|
||||
notification_target: Some(RuntimeWorkerRef::new("runtime-a", "worker-a")),
|
||||
limits: BackendJobLimits::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_covers_revision_profile_limits_and_notification_provenance() {
|
||||
let request = request();
|
||||
let first = request.fingerprint().unwrap();
|
||||
assert_eq!(first, request.fingerprint().unwrap());
|
||||
for mutate in [
|
||||
|request: &mut BackendJobRequest| request.input_revision = "8".to_string(),
|
||||
|request: &mut BackendJobRequest| request.purpose = "alternate_check".to_string(),
|
||||
|request: &mut BackendJobRequest| request.limits.max_attempts = 3,
|
||||
|request: &mut BackendJobRequest| request.notification_target = None,
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
assert_ne!(first, changed.fingerprint().unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_runtime_ids_are_fixed_size_for_maximum_job_ids() {
|
||||
let job_id = "j".repeat(256);
|
||||
let attempt_id = attempt_id(&job_id, ABSOLUTE_MAX_ATTEMPTS);
|
||||
let run_id = runtime_run_id(&attempt_id);
|
||||
let notification_id = delivery_id(&job_id, &attempt_id);
|
||||
|
||||
assert!(run_id.len() <= 128, "run id was {} bytes", run_id.len());
|
||||
assert!(
|
||||
notification_id.len() <= 128,
|
||||
"notification id was {} bytes",
|
||||
notification_id.len()
|
||||
);
|
||||
assert_eq!(run_id, runtime_run_id(&attempt_id));
|
||||
assert_eq!(notification_id, delivery_id(&job_id, &attempt_id));
|
||||
assert_ne!(run_id, runtime_run_id(&format!("{attempt_id}-other")));
|
||||
assert_ne!(
|
||||
notification_id,
|
||||
delivery_id(&job_id, &format!("{attempt_id}-other"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn result_is_bounded_before_digesting() {
|
||||
let value = serde_json::json!({"ok": true});
|
||||
let (digest, encoded) = result_digest(&value, 128).unwrap();
|
||||
assert!(digest.starts_with("sha256:"));
|
||||
assert_eq!(encoded, r#"{"ok":true}"#);
|
||||
assert!(result_digest(&serde_json::json!({"body": "too long"}), 4).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_detail_is_bounded_by_utf8_bytes_without_splitting_code_points() {
|
||||
let detail = "界".repeat(MAX_JOB_FAILURE_BYTES);
|
||||
let bounded = bounded_failure_detail(&detail);
|
||||
assert!(bounded.len() <= MAX_JOB_FAILURE_BYTES);
|
||||
assert!(detail.starts_with(&bounded));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_limits_reject_unbounded_execution_and_retry() {
|
||||
for limits in [
|
||||
BackendJobLimits {
|
||||
timeout_seconds: 0,
|
||||
..Default::default()
|
||||
},
|
||||
BackendJobLimits {
|
||||
timeout_seconds: MAX_JOB_TIMEOUT_SECONDS + 1,
|
||||
..Default::default()
|
||||
},
|
||||
BackendJobLimits {
|
||||
max_attempts: 0,
|
||||
..Default::default()
|
||||
},
|
||||
BackendJobLimits {
|
||||
max_attempts: ABSOLUTE_MAX_ATTEMPTS + 1,
|
||||
..Default::default()
|
||||
},
|
||||
BackendJobLimits {
|
||||
max_concurrent_jobs: 0,
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(limits.validate().is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -544,6 +544,11 @@ pub enum WorkerSpawnIntent {
|
||||
WorkspaceCompanion,
|
||||
WorkspaceOrchestrator,
|
||||
WorkspaceCoding,
|
||||
BackendJob {
|
||||
job_id: String,
|
||||
attempt_id: String,
|
||||
purpose: String,
|
||||
},
|
||||
TicketRole {
|
||||
ticket_id: String,
|
||||
role: TicketWorkerRole,
|
||||
@@ -705,6 +710,8 @@ pub struct WorkerInputRequest {
|
||||
pub kind: WorkerInputKind,
|
||||
pub content: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub submission_request_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub segments: Option<Vec<protocol::Segment>>,
|
||||
}
|
||||
|
||||
@@ -725,11 +732,22 @@ pub struct WorkerCompletionsResult {
|
||||
pub diagnostics: Vec<RuntimeDiagnostic>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerInputDisposition {
|
||||
Accepted,
|
||||
Rejected,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct WorkerInputResult {
|
||||
pub state: InternalWorkerOperationState,
|
||||
pub disposition: WorkerInputDisposition,
|
||||
#[serde(flatten)]
|
||||
pub worker: RuntimeWorkerRef,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub runtime_run_id: Option<String>,
|
||||
pub diagnostics: Vec<RuntimeDiagnostic>,
|
||||
}
|
||||
|
||||
@@ -1200,7 +1218,9 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
fn send_input(&self, worker_id: &str, _request: WorkerInputRequest) -> WorkerInputResult {
|
||||
WorkerInputResult {
|
||||
state: InternalWorkerOperationState::Unsupported,
|
||||
disposition: WorkerInputDisposition::Rejected,
|
||||
worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()),
|
||||
runtime_run_id: None,
|
||||
diagnostics: vec![diagnostic(
|
||||
"worker_input_pending",
|
||||
HostDiagnosticSeverity::Info,
|
||||
@@ -2985,6 +3005,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
return embedded_input_rejected(
|
||||
&self.runtime_id,
|
||||
worker_id,
|
||||
WorkerInputDisposition::Rejected,
|
||||
diagnostic(
|
||||
"embedded_worker_execution_unavailable",
|
||||
HostDiagnosticSeverity::Info,
|
||||
@@ -2998,6 +3019,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
return embedded_input_rejected(
|
||||
&self.runtime_id,
|
||||
worker_id,
|
||||
WorkerInputDisposition::Rejected,
|
||||
diagnostic(
|
||||
"embedded_worker_id_invalid",
|
||||
HostDiagnosticSeverity::Warning,
|
||||
@@ -3014,20 +3036,34 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
||||
},
|
||||
content: request.content,
|
||||
submission_request_id: None,
|
||||
submission_request_id: request.submission_request_id,
|
||||
segments: request.segments,
|
||||
};
|
||||
match self.runtime.send_input(&worker_ref, input) {
|
||||
Ok(_) => WorkerInputResult {
|
||||
Ok(ack) => WorkerInputResult {
|
||||
state: InternalWorkerOperationState::Accepted,
|
||||
disposition: WorkerInputDisposition::Accepted,
|
||||
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||
runtime_run_id: ack
|
||||
.submission
|
||||
.map(|submission| submission.submission_request_id),
|
||||
diagnostics: Vec::new(),
|
||||
},
|
||||
Err(error) => embedded_input_rejected(
|
||||
&self.runtime_id,
|
||||
worker_id,
|
||||
embedded_runtime_diagnostic(&error),
|
||||
),
|
||||
Err(error) => {
|
||||
let disposition = match &error {
|
||||
EmbeddedRuntimeError::WorkerExecutionRejected {
|
||||
outcome: worker_runtime::execution::WorkerExecutionOutcome::Errored,
|
||||
..
|
||||
} => WorkerInputDisposition::Unknown,
|
||||
_ => WorkerInputDisposition::Rejected,
|
||||
};
|
||||
embedded_input_rejected(
|
||||
&self.runtime_id,
|
||||
worker_id,
|
||||
disposition,
|
||||
embedded_runtime_diagnostic(&error),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4991,7 +5027,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
||||
},
|
||||
content: request.content,
|
||||
submission_request_id: None,
|
||||
submission_request_id: request.submission_request_id,
|
||||
segments: request.segments,
|
||||
};
|
||||
let input = match runtime_contract_convert(input) {
|
||||
@@ -4999,7 +5035,9 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
Err(diagnostic) => {
|
||||
return WorkerInputResult {
|
||||
state: InternalWorkerOperationState::Rejected,
|
||||
disposition: WorkerInputDisposition::Rejected,
|
||||
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||
runtime_run_id: None,
|
||||
diagnostics: vec![diagnostic],
|
||||
};
|
||||
}
|
||||
@@ -5010,9 +5048,14 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||
move |client| async move { client.send_worker_input(worker_id_owned, input).await },
|
||||
) {
|
||||
Ok(_) => WorkerInputResult {
|
||||
Ok(response) => WorkerInputResult {
|
||||
state: InternalWorkerOperationState::Accepted,
|
||||
disposition: WorkerInputDisposition::Accepted,
|
||||
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||
runtime_run_id: response
|
||||
.ack
|
||||
.submission
|
||||
.map(|submission| submission.submission_request_id),
|
||||
diagnostics: Vec::new(),
|
||||
},
|
||||
Err(diagnostic) => remote_input_rejected(&self.runtime_id, worker_id, diagnostic),
|
||||
@@ -5396,11 +5439,14 @@ fn profile_display_name(profile_label: &str) -> String {
|
||||
fn embedded_input_rejected(
|
||||
runtime_id: &str,
|
||||
worker_id: &str,
|
||||
disposition: WorkerInputDisposition,
|
||||
diagnostic: RuntimeDiagnostic,
|
||||
) -> WorkerInputResult {
|
||||
WorkerInputResult {
|
||||
state: InternalWorkerOperationState::Rejected,
|
||||
disposition,
|
||||
worker: RuntimeWorkerRef::new(runtime_id.to_string(), worker_id.to_string()),
|
||||
runtime_run_id: None,
|
||||
diagnostics: vec![diagnostic],
|
||||
}
|
||||
}
|
||||
@@ -5412,7 +5458,9 @@ fn remote_input_rejected(
|
||||
) -> WorkerInputResult {
|
||||
WorkerInputResult {
|
||||
state: InternalWorkerOperationState::Rejected,
|
||||
disposition: WorkerInputDisposition::Unknown,
|
||||
worker: RuntimeWorkerRef::new(runtime_id.to_string(), worker_id.to_string()),
|
||||
runtime_run_id: None,
|
||||
diagnostics: vec![diagnostic],
|
||||
}
|
||||
}
|
||||
@@ -5890,6 +5938,7 @@ fn worker_spawn_intent_label(intent: &WorkerSpawnIntent) -> &'static str {
|
||||
WorkerSpawnIntent::WorkspaceCompanion => "workspace_companion",
|
||||
WorkerSpawnIntent::WorkspaceOrchestrator => "workspace_orchestrator",
|
||||
WorkerSpawnIntent::WorkspaceCoding => "workspace_coding",
|
||||
WorkerSpawnIntent::BackendJob { .. } => "backend_job",
|
||||
WorkerSpawnIntent::TicketRole { role, .. } => match role {
|
||||
TicketWorkerRole::Intake => "ticket_intake",
|
||||
TicketWorkerRole::Orchestrator => "ticket_orchestrator",
|
||||
@@ -7088,10 +7137,13 @@ mod tests {
|
||||
WorkerInputRequest {
|
||||
kind: WorkerInputKind::User,
|
||||
content: "hello".to_string(),
|
||||
submission_request_id: Some("tracked-input-1".to_string()),
|
||||
segments: None,
|
||||
},
|
||||
);
|
||||
assert_eq!(input.state, InternalWorkerOperationState::Accepted);
|
||||
assert_eq!(input.disposition, WorkerInputDisposition::Accepted);
|
||||
assert_eq!(input.runtime_run_id.as_deref(), Some("tracked-input-1"));
|
||||
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
@@ -7183,6 +7235,7 @@ mod tests {
|
||||
WorkerInputRequest {
|
||||
kind: WorkerInputKind::User,
|
||||
content: "hello embedded runtime".to_string(),
|
||||
submission_request_id: None,
|
||||
segments: None,
|
||||
},
|
||||
)
|
||||
@@ -7467,6 +7520,7 @@ mod tests {
|
||||
WorkerInputRequest {
|
||||
kind: WorkerInputKind::User,
|
||||
content: "hello remote".to_string(),
|
||||
submission_request_id: None,
|
||||
segments: None,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -806,6 +806,79 @@ CREATE TABLE "worker_registry" (
|
||||
PRIMARY KEY (workspace_id, worker_id),
|
||||
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TABLE backend_jobs (
|
||||
workspace_id TEXT NOT NULL,
|
||||
job_id TEXT NOT NULL,
|
||||
purpose TEXT NOT NULL,
|
||||
input_revision TEXT NOT NULL,
|
||||
input_ref TEXT NOT NULL,
|
||||
request_json TEXT NOT NULL,
|
||||
intent_fingerprint TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown')),
|
||||
current_attempt INTEGER NOT NULL CHECK (current_attempt > 0 AND current_attempt <= 3),
|
||||
result_json TEXT,
|
||||
result_digest TEXT,
|
||||
failure_category TEXT,
|
||||
failure_detail TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
completed_at TEXT,
|
||||
PRIMARY KEY (workspace_id, job_id),
|
||||
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
|
||||
CHECK ((result_json IS NULL) = (result_digest IS NULL)),
|
||||
CHECK ((state = 'completed') = (result_json IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX backend_jobs_active
|
||||
ON backend_jobs(workspace_id, state, updated_at);
|
||||
CREATE TABLE backend_job_attempts (
|
||||
workspace_id TEXT NOT NULL,
|
||||
job_id TEXT NOT NULL,
|
||||
attempt_id TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL CHECK (attempt > 0 AND attempt <= 3),
|
||||
input_revision TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatched', 'completed', 'failed', 'unknown')),
|
||||
runtime_id TEXT,
|
||||
worker_id TEXT,
|
||||
runtime_run_id TEXT,
|
||||
dispatched_at TEXT,
|
||||
deadline_at TEXT NOT NULL,
|
||||
result_json TEXT,
|
||||
result_digest TEXT,
|
||||
failure_category TEXT,
|
||||
failure_detail TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
completed_at TEXT,
|
||||
PRIMARY KEY (workspace_id, job_id, attempt_id),
|
||||
UNIQUE (workspace_id, job_id, attempt),
|
||||
UNIQUE (workspace_id, runtime_id, worker_id),
|
||||
FOREIGN KEY (workspace_id, job_id)
|
||||
REFERENCES backend_jobs(workspace_id, job_id) ON DELETE CASCADE,
|
||||
CHECK ((runtime_id IS NULL) = (worker_id IS NULL)),
|
||||
CHECK ((result_json IS NULL) = (result_digest IS NULL)),
|
||||
CHECK ((state = 'completed') = (result_json IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX backend_job_attempts_recovery
|
||||
ON backend_job_attempts(workspace_id, state, deadline_at);
|
||||
CREATE TABLE backend_job_deliveries (
|
||||
workspace_id TEXT NOT NULL,
|
||||
delivery_id TEXT NOT NULL,
|
||||
job_id TEXT NOT NULL,
|
||||
attempt_id TEXT NOT NULL,
|
||||
target_runtime_id TEXT NOT NULL,
|
||||
target_worker_id TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK (state IN ('pending', 'sending', 'completed', 'failed', 'unknown')),
|
||||
failure_category TEXT,
|
||||
failure_detail TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
delivered_at TEXT,
|
||||
PRIMARY KEY (workspace_id, delivery_id),
|
||||
FOREIGN KEY (workspace_id, job_id, attempt_id)
|
||||
REFERENCES backend_job_attempts(workspace_id, job_id, attempt_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX backend_job_deliveries_pending
|
||||
ON backend_job_deliveries(workspace_id, state, updated_at);
|
||||
CREATE TABLE worker_registry_observations (
|
||||
workspace_id TEXT NOT NULL,
|
||||
runtime_id TEXT NOT NULL,
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
pub mod auth;
|
||||
pub mod authority;
|
||||
pub mod backend_job;
|
||||
pub mod companion;
|
||||
pub mod config;
|
||||
pub mod config_source;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -18,6 +18,9 @@ use crate::{Error, Result};
|
||||
const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
||||
"artifacts",
|
||||
"audit_events",
|
||||
"backend_job_deliveries",
|
||||
"backend_job_attempts",
|
||||
"backend_jobs",
|
||||
"external_workdir_grants",
|
||||
"flow_source_revisions",
|
||||
"flow_sources",
|
||||
|
||||
@@ -449,6 +449,7 @@ fn project_registry_worker(
|
||||
workspace_id: Some(record.registry.workspace_id.clone()),
|
||||
display_name: Some(record.registry.display_name.clone()),
|
||||
profile: record.registry.profile.clone(),
|
||||
job: None,
|
||||
workdir_attachments: Vec::new(),
|
||||
}
|
||||
};
|
||||
@@ -457,6 +458,13 @@ fn project_registry_worker(
|
||||
worker.workspace_id = Some(record.registry.workspace_id);
|
||||
worker.display_name = Some(record.registry.display_name);
|
||||
worker.profile = record.registry.profile;
|
||||
worker.job = record
|
||||
.job
|
||||
.map(|job| protocol::subscription::SubscriptionWorkerJob {
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
purpose: job.purpose,
|
||||
});
|
||||
if !project_working_directory(api, &record.registry.worker, &mut worker) {
|
||||
return None;
|
||||
}
|
||||
@@ -616,6 +624,7 @@ mod tests {
|
||||
workspace_id: Some("workspace-1".to_string()),
|
||||
display_name: Some("Companion".to_string()),
|
||||
profile: Some("builtin:companion".to_string()),
|
||||
job: None,
|
||||
workdir_attachments: vec![
|
||||
protocol::subscription::SubscriptionWorkerWorkdirAttachment {
|
||||
alias: "external".to_string(),
|
||||
|
||||
Reference in New Issue
Block a user