feat: execute durable standalone Jobs through Host-owned Internal Workers (T-708)
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "job"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
manifest.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
@@ -0,0 +1,336 @@
|
||||
//! Transport-neutral immutable Job intent and structured-result contract.
|
||||
//!
|
||||
//! Hosts own execution, authority, persistence and lifecycle state. This crate
|
||||
//! deliberately contains no runner, state machine, Runtime or domain grants.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub const DEFAULT_MAX_CONCURRENT_JOBS: u16 = 8;
|
||||
pub const ABSOLUTE_MAX_CONCURRENT_JOBS: u16 = 64;
|
||||
pub const DEFAULT_JOB_TIMEOUT_SECONDS: u32 = 120;
|
||||
pub const MAX_JOB_TIMEOUT_SECONDS: u32 = 600;
|
||||
pub const DEFAULT_MAX_RESULT_BYTES: u32 = 16 * 1024;
|
||||
pub const ABSOLUTE_MAX_RESULT_BYTES: u32 = 64 * 1024;
|
||||
pub const DEFAULT_MAX_ATTEMPTS: u8 = 2;
|
||||
pub const ABSOLUTE_MAX_ATTEMPTS: u8 = 3;
|
||||
pub const MAX_JOB_INPUT_BYTES: usize = 128 * 1024;
|
||||
pub const MAX_JOB_INSTRUCTION_BYTES: usize = 16 * 1024;
|
||||
pub const MAX_JOB_PURPOSE_BYTES: usize = 120;
|
||||
pub const MAX_JOB_REFERENCE_BYTES: usize = 512;
|
||||
pub const MAX_JOB_FAILURE_BYTES: usize = 1024;
|
||||
pub const MAX_JOB_DELIVERY_BYTES: usize = 16 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum JobError {
|
||||
#[error("{0}")]
|
||||
InvalidInput(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, JobError>;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobLimits {
|
||||
#[serde(default = "default_max_concurrent_jobs")]
|
||||
pub max_concurrent_jobs: u16,
|
||||
#[serde(default = "default_job_timeout_seconds")]
|
||||
pub timeout_seconds: u32,
|
||||
#[serde(default = "default_max_result_bytes")]
|
||||
pub max_result_bytes: u32,
|
||||
#[serde(default = "default_max_attempts")]
|
||||
pub max_attempts: u8,
|
||||
}
|
||||
|
||||
impl Default for JobLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_concurrent_jobs: DEFAULT_MAX_CONCURRENT_JOBS,
|
||||
timeout_seconds: DEFAULT_JOB_TIMEOUT_SECONDS,
|
||||
max_result_bytes: DEFAULT_MAX_RESULT_BYTES,
|
||||
max_attempts: DEFAULT_MAX_ATTEMPTS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JobLimits {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.max_concurrent_jobs == 0 || self.max_concurrent_jobs > ABSOLUTE_MAX_CONCURRENT_JOBS
|
||||
{
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job max_concurrent_jobs must be in 1..={ABSOLUTE_MAX_CONCURRENT_JOBS}"
|
||||
)));
|
||||
}
|
||||
if self.timeout_seconds == 0 || self.timeout_seconds > MAX_JOB_TIMEOUT_SECONDS {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job timeout_seconds must be in 1..={MAX_JOB_TIMEOUT_SECONDS}"
|
||||
)));
|
||||
}
|
||||
if self.max_result_bytes == 0 || self.max_result_bytes > ABSOLUTE_MAX_RESULT_BYTES {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job max_result_bytes must be in 1..={ABSOLUTE_MAX_RESULT_BYTES}"
|
||||
)));
|
||||
}
|
||||
if self.max_attempts == 0 || self.max_attempts > ABSOLUTE_MAX_ATTEMPTS {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job max_attempts must be in 1..={ABSOLUTE_MAX_ATTEMPTS}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const fn default_max_concurrent_jobs() -> u16 {
|
||||
DEFAULT_MAX_CONCURRENT_JOBS
|
||||
}
|
||||
const fn default_job_timeout_seconds() -> u32 {
|
||||
DEFAULT_JOB_TIMEOUT_SECONDS
|
||||
}
|
||||
const fn default_max_result_bytes() -> u32 {
|
||||
DEFAULT_MAX_RESULT_BYTES
|
||||
}
|
||||
const fn default_max_attempts() -> u8 {
|
||||
DEFAULT_MAX_ATTEMPTS
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobRequest {
|
||||
/// Stable caller-selected idempotency key within the host's namespace.
|
||||
pub job_id: String,
|
||||
pub purpose: String,
|
||||
/// Revision fenced by the structured result.
|
||||
pub input_revision: String,
|
||||
pub input_ref: String,
|
||||
/// Bounded immutable snapshot; must not contain credentials.
|
||||
pub input: Value,
|
||||
pub instruction: String,
|
||||
/// Manifest registry selector, never raw source or a filesystem path.
|
||||
pub profile: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub serialization_key: Option<String>,
|
||||
#[serde(default)]
|
||||
pub limits: JobLimits,
|
||||
}
|
||||
|
||||
impl JobRequest {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
self.validate_fields()?;
|
||||
self.validate_input()?;
|
||||
self.limits.validate()
|
||||
}
|
||||
|
||||
/// Request metadata validation, separable so adapters can retain their
|
||||
/// domain-validation ordering before validating input and limits.
|
||||
pub fn validate_fields(&self) -> Result<()> {
|
||||
validate_identifier("job_id", &self.job_id, 256)?;
|
||||
validate_identifier("purpose", &self.purpose, MAX_JOB_PURPOSE_BYTES)?;
|
||||
validate_identifier("input_revision", &self.input_revision, 256)?;
|
||||
validate_bounded_text("input_ref", &self.input_ref, MAX_JOB_REFERENCE_BYTES)?;
|
||||
validate_bounded_text("instruction", &self.instruction, MAX_JOB_INSTRUCTION_BYTES)?;
|
||||
self.profile_selector()?;
|
||||
if let Some(key) = &self.serialization_key {
|
||||
validate_identifier("serialization_key", key, MAX_JOB_REFERENCE_BYTES)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_input(&self) -> Result<()> {
|
||||
let input = serde_json::to_vec(&self.input)
|
||||
.map_err(|error| JobError::InvalidInput(format!("serialize Job input: {error}")))?;
|
||||
if input.len() > MAX_JOB_INPUT_BYTES {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job input exceeds {MAX_JOB_INPUT_BYTES} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Syntax only; registry existence and resolution are host authority.
|
||||
pub fn profile_selector(&self) -> Result<manifest::ProfileSelector> {
|
||||
profile_selector(&self.profile)
|
||||
}
|
||||
|
||||
pub fn fingerprint(&self) -> Result<String> {
|
||||
self.validate()?;
|
||||
fingerprint(self)
|
||||
}
|
||||
|
||||
pub fn worker_input(&self, attempt_id: &str) -> Result<String> {
|
||||
worker_input(self, attempt_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobResultSubmission {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub input_revision: String,
|
||||
pub result: Value,
|
||||
}
|
||||
|
||||
/// Client-facing lifecycle vocabulary, not a runner or transition policy.
|
||||
/// A variant's presence does not imply every host supports that operation.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobState {
|
||||
Pending,
|
||||
Completed,
|
||||
Failed,
|
||||
Unknown,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobAttemptState {
|
||||
Reserved,
|
||||
Dispatching,
|
||||
Dispatched,
|
||||
Completed,
|
||||
Failed,
|
||||
Unknown,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// Transport-neutral read snapshot. Job status/result and the corresponding
|
||||
/// attempt identity/status remain distinct; host identities, delivery and
|
||||
/// resource cleanup are deliberately not part of the model outcome.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobOutcome {
|
||||
pub job_id: String,
|
||||
pub input_revision: String,
|
||||
pub attempt_id: String,
|
||||
pub attempt: u8,
|
||||
pub state: JobState,
|
||||
pub attempt_state: JobAttemptState,
|
||||
pub result: Option<Value>,
|
||||
pub failure_category: Option<String>,
|
||||
pub failure_detail: Option<String>,
|
||||
}
|
||||
|
||||
pub fn profile_selector(profile: &str) -> Result<manifest::ProfileSelector> {
|
||||
use manifest::ProfileSelector;
|
||||
validate_identifier("profile", profile, 256)?;
|
||||
let selector = ProfileSelector::parse_cli(profile);
|
||||
let valid = match &selector {
|
||||
ProfileSelector::Default => true,
|
||||
ProfileSelector::Path { .. } => false,
|
||||
ProfileSelector::Named { source, name } => {
|
||||
!name.is_empty()
|
||||
&& !name.starts_with('.')
|
||||
&& !name.contains(['/', '\\', ':'])
|
||||
&& name
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.'))
|
||||
&& ![".toml", ".json", ".dcdl", ".nix"]
|
||||
.iter()
|
||||
.any(|suffix| name.ends_with(suffix))
|
||||
&& (source.is_some() || name != "inherit")
|
||||
}
|
||||
};
|
||||
if !valid {
|
||||
return Err(JobError::InvalidInput(
|
||||
"Job profile must be a manifest registry selector, not raw source or a path".into(),
|
||||
));
|
||||
}
|
||||
Ok(selector)
|
||||
}
|
||||
|
||||
/// Hash the full serialized host intent, not just its neutral projection.
|
||||
/// Callers must validate the complete intent first. JSON field ordering and
|
||||
/// omission rules are intentionally preserved for durable replay compatibility.
|
||||
pub fn fingerprint<T: Serialize + ?Sized>(intent: &T) -> Result<String> {
|
||||
let encoded = serde_json::to_vec(intent)
|
||||
.map_err(|error| JobError::InvalidInput(format!("serialize Job intent: {error}")))?;
|
||||
Ok(sha256(&encoded))
|
||||
}
|
||||
|
||||
pub fn worker_input(request: &JobRequest, attempt_id: &str) -> Result<String> {
|
||||
worker_input_with_label(request, attempt_id, "Job")
|
||||
}
|
||||
|
||||
/// Adapter-specific capability label; only the host, never the model, selects it.
|
||||
/// Keeps an existing host's prompt envelope byte-for-byte compatible.
|
||||
pub fn worker_input_with_label(
|
||||
request: &JobRequest,
|
||||
attempt_id: &str,
|
||||
label: &str,
|
||||
) -> Result<String> {
|
||||
let input = serde_json::to_string(&request.input)
|
||||
.map_err(|error| JobError::InvalidInput(format!("serialize Job input: {error}")))?;
|
||||
Ok(format!(
|
||||
"{}\n\n{label} envelope (immutable):\njob_id: {}\nattempt_id: {}\ninput_revision: {}\ninput_ref: {}\ninput_json: {}\n\nReturn success only through the structured {label} result capability. Final prose and Worker Idle/Stopped state are not result authority.",
|
||||
request.instruction,
|
||||
request.job_id,
|
||||
attempt_id,
|
||||
request.input_revision,
|
||||
request.input_ref,
|
||||
input
|
||||
))
|
||||
}
|
||||
|
||||
pub fn attempt_id(job_id: &str, attempt: u8) -> String {
|
||||
format!("{job_id}:attempt:{attempt}")
|
||||
}
|
||||
|
||||
/// Bound the encoded JSON before hashing. Returns (digest, exact JSON).
|
||||
pub fn result_digest(result: &Value, max_bytes: u32) -> Result<(String, String)> {
|
||||
let encoded = serde_json::to_string(result)
|
||||
.map_err(|error| JobError::InvalidInput(format!("serialize Job result: {error}")))?;
|
||||
if encoded.len() > max_bytes as usize {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job result exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok((sha256(encoded.as_bytes()), encoded))
|
||||
}
|
||||
|
||||
pub fn bounded_failure_detail(detail: &str) -> String {
|
||||
if detail.len() <= MAX_JOB_FAILURE_BYTES {
|
||||
return detail.to_string();
|
||||
}
|
||||
let mut end = MAX_JOB_FAILURE_BYTES;
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
detail[..end].to_string()
|
||||
}
|
||||
|
||||
pub fn validate_identifier(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
validate_bounded_text(field, value, max_bytes)?;
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job {field} must not contain control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_bounded_text(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
if value.is_empty() || value.trim() != value {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job {field} must be non-empty and trimmed"
|
||||
)));
|
||||
}
|
||||
if value.len() > max_bytes {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job {field} exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sha256(bytes: &[u8]) -> String {
|
||||
let digest = Sha256::digest(bytes);
|
||||
let mut output = String::with_capacity(7 + digest.len() * 2);
|
||||
output.push_str("sha256:");
|
||||
for byte in digest {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(output, "{byte:02x}");
|
||||
}
|
||||
output
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
use job::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn request() -> JobRequest {
|
||||
JobRequest {
|
||||
job_id: "check:r7".into(),
|
||||
purpose: "snapshot_check".into(),
|
||||
input_revision: "7".into(),
|
||||
input_ref: "snapshot://7".into(),
|
||||
input: json!({"title": "Check me"}),
|
||||
instruction: "Check this immutable snapshot.".into(),
|
||||
profile: "project:custom-check".into(),
|
||||
serialization_key: None,
|
||||
limits: JobLimits::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_lifecycle_enums_round_trip_all_host_states() {
|
||||
for (state, name) in [
|
||||
(JobState::Pending, "pending"),
|
||||
(JobState::Completed, "completed"),
|
||||
(JobState::Failed, "failed"),
|
||||
(JobState::Unknown, "unknown"),
|
||||
(JobState::Cancelled, "cancelled"),
|
||||
] {
|
||||
assert_eq!(serde_json::to_value(state).unwrap(), json!(name));
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobState>(json!(name)).unwrap(),
|
||||
state
|
||||
);
|
||||
}
|
||||
for (state, name) in [
|
||||
(JobAttemptState::Reserved, "reserved"),
|
||||
(JobAttemptState::Dispatching, "dispatching"),
|
||||
(JobAttemptState::Dispatched, "dispatched"),
|
||||
(JobAttemptState::Completed, "completed"),
|
||||
(JobAttemptState::Failed, "failed"),
|
||||
(JobAttemptState::Unknown, "unknown"),
|
||||
(JobAttemptState::Cancelled, "cancelled"),
|
||||
] {
|
||||
assert_eq!(serde_json::to_value(state).unwrap(), json!(name));
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobAttemptState>(json!(name)).unwrap(),
|
||||
state
|
||||
);
|
||||
}
|
||||
assert!(serde_json::from_value::<JobState>(json!("running")).is_err());
|
||||
assert!(serde_json::from_value::<JobAttemptState>(json!("idle")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_is_a_neutral_read_snapshot_with_optional_result_and_failure() {
|
||||
let mut outcome = JobOutcome {
|
||||
job_id: "job-1".into(),
|
||||
input_revision: "7".into(),
|
||||
attempt_id: "job-1:attempt:2".into(),
|
||||
attempt: 2,
|
||||
state: JobState::Pending,
|
||||
attempt_state: JobAttemptState::Dispatching,
|
||||
result: None,
|
||||
failure_category: None,
|
||||
failure_detail: None,
|
||||
};
|
||||
let encoded = serde_json::to_value(&outcome).unwrap();
|
||||
assert_eq!(
|
||||
encoded,
|
||||
json!({
|
||||
"job_id": "job-1", "input_revision": "7", "attempt_id": "job-1:attempt:2",
|
||||
"attempt": 2, "state": "pending", "attempt_state": "dispatching",
|
||||
"result": null, "failure_category": null, "failure_detail": null
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobOutcome>(encoded.clone()).unwrap(),
|
||||
outcome
|
||||
);
|
||||
let mut with_host_identity = encoded;
|
||||
with_host_identity["workspace_id"] = json!("workspace-a");
|
||||
assert!(serde_json::from_value::<JobOutcome>(with_host_identity).is_err());
|
||||
for (state, attempt_state, result, category, detail) in [
|
||||
(
|
||||
JobState::Completed,
|
||||
JobAttemptState::Completed,
|
||||
Some(json!({"ok": true})),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
(
|
||||
JobState::Failed,
|
||||
JobAttemptState::Failed,
|
||||
None,
|
||||
Some("execution"),
|
||||
Some("known failure"),
|
||||
),
|
||||
(
|
||||
JobState::Unknown,
|
||||
JobAttemptState::Unknown,
|
||||
None,
|
||||
Some("interrupted"),
|
||||
Some("requires reconciliation"),
|
||||
),
|
||||
(
|
||||
JobState::Cancelled,
|
||||
JobAttemptState::Cancelled,
|
||||
None,
|
||||
Some("cancelled"),
|
||||
Some("caller cancelled"),
|
||||
),
|
||||
] {
|
||||
outcome.state = state;
|
||||
outcome.attempt_state = attempt_state;
|
||||
outcome.result = result;
|
||||
outcome.failure_category = category.map(str::to_owned);
|
||||
outcome.failure_detail = detail.map(str::to_owned);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<JobOutcome>(&serde_json::to_string(&outcome).unwrap()).unwrap(),
|
||||
outcome
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn neutral_request_and_result_have_strict_round_trip_contracts() {
|
||||
let request = request();
|
||||
let encoded = serde_json::to_value(&request).unwrap();
|
||||
assert_eq!(encoded.as_object().unwrap().len(), 8);
|
||||
assert!(encoded.get("serialization_key").is_none());
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobRequest>(encoded.clone()).unwrap(),
|
||||
request
|
||||
);
|
||||
let mut with_grant = encoded.clone();
|
||||
with_grant["grants"] = json!({});
|
||||
assert!(serde_json::from_value::<JobRequest>(with_grant).is_err());
|
||||
let mut with_provenance = encoded;
|
||||
with_provenance["source_worker"] = json!({});
|
||||
assert!(serde_json::from_value::<JobRequest>(with_provenance).is_err());
|
||||
|
||||
let submission = JobResultSubmission {
|
||||
job_id: request.job_id,
|
||||
attempt_id: "check:r7:attempt:1".into(),
|
||||
input_revision: request.input_revision,
|
||||
result: json!({"ok": true}),
|
||||
};
|
||||
let mut encoded = serde_json::to_value(&submission).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobResultSubmission>(encoded.clone()).unwrap(),
|
||||
submission
|
||||
);
|
||||
encoded["authority"] = json!("profile");
|
||||
assert!(serde_json::from_value::<JobResultSubmission>(encoded).is_err());
|
||||
assert_eq!(
|
||||
serde_json::from_str::<JobLimits>("{}").unwrap(),
|
||||
JobLimits::default()
|
||||
);
|
||||
assert!(serde_json::from_str::<JobLimits>(r#"{"turns":999}"#).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_covers_every_immutable_field_and_preserves_replay() {
|
||||
let request = request();
|
||||
let digest = request.fingerprint().unwrap();
|
||||
let decoded: JobRequest =
|
||||
serde_json::from_str(&serde_json::to_string(&request).unwrap()).unwrap();
|
||||
assert_eq!(digest, decoded.fingerprint().unwrap());
|
||||
assert_eq!(digest, fingerprint(&request).unwrap());
|
||||
for mutate in [
|
||||
|r: &mut JobRequest| r.job_id.push('x'),
|
||||
|r: &mut JobRequest| r.purpose.push('x'),
|
||||
|r: &mut JobRequest| r.input_revision.push('x'),
|
||||
|r: &mut JobRequest| r.input_ref.push('x'),
|
||||
|r: &mut JobRequest| r.input = json!({"title": "changed"}),
|
||||
|r: &mut JobRequest| r.instruction.push('x'),
|
||||
|r: &mut JobRequest| r.profile = "builtin:backend-job".into(),
|
||||
|r: &mut JobRequest| r.serialization_key = Some("resource:7".into()),
|
||||
|r: &mut JobRequest| r.limits.max_attempts = 3,
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
assert_ne!(digest, changed.fingerprint().unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_profiles_are_not_authority_or_a_fixed_allowlist() {
|
||||
for profile in [
|
||||
"default",
|
||||
"custom",
|
||||
"project:custom",
|
||||
"user:custom",
|
||||
"builtin:future-job",
|
||||
] {
|
||||
profile_selector(profile).unwrap();
|
||||
}
|
||||
for profile in [
|
||||
"",
|
||||
" inherit",
|
||||
"inherit",
|
||||
"path:custom",
|
||||
"./a.toml",
|
||||
"/tmp/a",
|
||||
"project:../a",
|
||||
"project:a.toml",
|
||||
"builtin:",
|
||||
"unknown:a",
|
||||
"project:a:b",
|
||||
"C:\\custom",
|
||||
"{\"worker\":{}}",
|
||||
] {
|
||||
assert!(profile_selector(profile).is_err(), "accepted {profile}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validation_enforces_utf8_bytes_and_metadata_bounds() {
|
||||
let mut r = request();
|
||||
r.job_id = "x".repeat(256);
|
||||
r.input_revision = "x".repeat(256);
|
||||
r.purpose = "x".repeat(MAX_JOB_PURPOSE_BYTES);
|
||||
r.input_ref = "x".repeat(MAX_JOB_REFERENCE_BYTES);
|
||||
r.instruction = "x".repeat(MAX_JOB_INSTRUCTION_BYTES);
|
||||
r.serialization_key = Some("x".repeat(MAX_JOB_REFERENCE_BYTES));
|
||||
r.input = json!("x".repeat(MAX_JOB_INPUT_BYTES - 2));
|
||||
r.validate().unwrap();
|
||||
for mutate in [
|
||||
|r: &mut JobRequest| r.job_id.push('x'),
|
||||
|r: &mut JobRequest| r.input_revision.push('x'),
|
||||
|r: &mut JobRequest| r.purpose.push('x'),
|
||||
|r: &mut JobRequest| r.input_ref.push('x'),
|
||||
|r: &mut JobRequest| r.instruction.push('x'),
|
||||
|r: &mut JobRequest| r.serialization_key.as_mut().unwrap().push('x'),
|
||||
|r: &mut JobRequest| r.input = json!("x".repeat(MAX_JOB_INPUT_BYTES - 1)),
|
||||
] {
|
||||
let mut changed = r.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(changed.validate().is_err());
|
||||
assert!(changed.fingerprint().is_err());
|
||||
}
|
||||
assert!(validate_identifier("id", "a\nb", 256).is_err());
|
||||
assert!(validate_bounded_text("text", " untrimmed", 256).is_err());
|
||||
assert!(validate_bounded_text("text", "", 256).is_err());
|
||||
assert!(validate_bounded_text("text", "界", 2).is_err());
|
||||
validate_bounded_text("text", "a\nb", 256).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_limit_dimensions_reject_zero_and_above_absolute_bounds() {
|
||||
let maximum = JobLimits {
|
||||
max_concurrent_jobs: ABSOLUTE_MAX_CONCURRENT_JOBS,
|
||||
timeout_seconds: MAX_JOB_TIMEOUT_SECONDS,
|
||||
max_result_bytes: ABSOLUTE_MAX_RESULT_BYTES,
|
||||
max_attempts: ABSOLUTE_MAX_ATTEMPTS,
|
||||
};
|
||||
maximum.validate().unwrap();
|
||||
for mutate in [
|
||||
|l: &mut JobLimits| l.max_concurrent_jobs = 0,
|
||||
|l: &mut JobLimits| l.max_concurrent_jobs += 1,
|
||||
|l: &mut JobLimits| l.timeout_seconds = 0,
|
||||
|l: &mut JobLimits| l.timeout_seconds += 1,
|
||||
|l: &mut JobLimits| l.max_result_bytes = 0,
|
||||
|l: &mut JobLimits| l.max_result_bytes += 1,
|
||||
|l: &mut JobLimits| l.max_attempts = 0,
|
||||
|l: &mut JobLimits| l.max_attempts += 1,
|
||||
] {
|
||||
let mut changed = maximum.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(changed.validate().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn result_digest_and_failure_detail_bound_exact_utf8_encoding() {
|
||||
let result = json!({"ok": true});
|
||||
let (digest, encoded) = result_digest(&result, 11).unwrap();
|
||||
assert_eq!(encoded, r#"{"ok":true}"#);
|
||||
assert_eq!(digest, sha256(encoded.as_bytes()));
|
||||
assert!(result_digest(&result, 10).is_err());
|
||||
let unicode = json!("界");
|
||||
assert!(result_digest(&unicode, 4).is_err());
|
||||
result_digest(&unicode, 5).unwrap();
|
||||
let detail = "界".repeat(MAX_JOB_FAILURE_BYTES);
|
||||
let bounded = bounded_failure_detail(&detail);
|
||||
assert_eq!(bounded.len(), MAX_JOB_FAILURE_BYTES - 1);
|
||||
assert!(detail.starts_with(&bounded));
|
||||
assert_eq!(bounded_failure_detail("known failure"), "known failure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_input_is_revision_and_attempt_bound_not_final_prose_authority() {
|
||||
let request = request();
|
||||
let attempt = attempt_id(&request.job_id, 2);
|
||||
assert_eq!(attempt, "check:r7:attempt:2");
|
||||
let input = request.worker_input(&attempt).unwrap();
|
||||
assert!(input.starts_with(&request.instruction));
|
||||
for field in [
|
||||
"Job envelope (immutable):",
|
||||
"job_id: check:r7",
|
||||
"attempt_id: check:r7:attempt:2",
|
||||
"input_revision: 7",
|
||||
"input_ref: snapshot://7",
|
||||
r#"input_json: {"title":"Check me"}"#,
|
||||
"structured Job result capability",
|
||||
"Final prose and Worker Idle/Stopped state are not result authority.",
|
||||
] {
|
||||
assert!(input.contains(field), "missing {field}");
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,13 @@ pub const BUILTIN_PROFILE_RESOURCES: &[BuiltinProfileResource] = &[
|
||||
description: "Workspace orchestration and Worker control.",
|
||||
imports: BASE_IMPORT,
|
||||
},
|
||||
BuiltinProfileResource {
|
||||
selector: Some("builtin:job"),
|
||||
path: "profiles/job.dcdl",
|
||||
source: include_str!("../../../resources/profiles/job.dcdl"),
|
||||
description: "Result-only caller-selected Job policy.",
|
||||
imports: NO_IMPORTS,
|
||||
},
|
||||
BuiltinProfileResource {
|
||||
selector: Some("builtin:backend-job"),
|
||||
path: "profiles/backend-job.dcdl",
|
||||
|
||||
@@ -1257,6 +1257,13 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(resolved.manifest.worker.name, "role-worker");
|
||||
if entry.name == "job" {
|
||||
assert!(resolved.manifest.compaction.is_none());
|
||||
assert!(resolved.manifest.scope.allow.is_empty());
|
||||
assert!(resolved.manifest.delegation_scope.allow.is_empty());
|
||||
assert_eq!(resolved.manifest.engine.instruction, "internal.job_system");
|
||||
continue;
|
||||
}
|
||||
let compaction = resolved.manifest.compaction.as_ref().unwrap();
|
||||
assert_eq!(compaction.worker_max_turns, Some(100), "{}", entry.name);
|
||||
assert_eq!(
|
||||
|
||||
@@ -10,6 +10,8 @@ agen.workspace = true
|
||||
client.workspace = true
|
||||
fs4.workspace = true
|
||||
manifest.workspace = true
|
||||
job.workspace = true
|
||||
rusqlite.workspace = true
|
||||
protocol.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -43,6 +43,7 @@ pub struct StandaloneHost {
|
||||
worker_store: FsWorkerStore,
|
||||
record: StandaloneWorkerRecord,
|
||||
lease: Option<StandaloneWorkerLease>,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
@@ -77,6 +78,8 @@ pub enum StandaloneStartupError {
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
pub enum StandaloneShutdownError {
|
||||
#[error("standalone Job execution cleanup or terminal persistence could not be confirmed")]
|
||||
JobCleanup,
|
||||
#[error("the standalone Worker did not stop before the shutdown deadline")]
|
||||
DeadlineExceeded,
|
||||
#[error("the standalone Worker shutdown confirmation was lost")]
|
||||
@@ -110,6 +113,16 @@ impl StandaloneHost {
|
||||
.allocate(&launch.cwd, StaleLeasePolicy::Reject)
|
||||
.map_err(classify_store_startup_error)?;
|
||||
let worker_id = allocation.worker_id();
|
||||
let jobs = match crate::jobs::StandaloneJobs::open(
|
||||
&store.jobs_path(worker_id),
|
||||
launch.cwd.clone(),
|
||||
) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(_) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
};
|
||||
|
||||
// WorkerId is the stable identity. The current Worker store remains
|
||||
// name-keyed, so keep its derived storage key separate from the
|
||||
@@ -181,6 +194,7 @@ impl StandaloneHost {
|
||||
worker_store,
|
||||
record,
|
||||
allocation.into_lease(),
|
||||
jobs,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -216,6 +230,11 @@ impl StandaloneHost {
|
||||
let lease = store
|
||||
.acquire_lease(worker_id, StaleLeasePolicy::Recover)
|
||||
.map_err(classify_store_startup_error)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&store.jobs_path(worker_id),
|
||||
record.cwd.canonical_path.clone(),
|
||||
)
|
||||
.map_err(|_| StandaloneStartupError::StateStore)?;
|
||||
let (backing_store, worker_store) = backing_store(&store, worker_id)?;
|
||||
let storage_key = record.storage_key.clone();
|
||||
let mut manifest = record.manifest.clone();
|
||||
@@ -270,6 +289,7 @@ impl StandaloneHost {
|
||||
worker_store,
|
||||
record,
|
||||
lease,
|
||||
jobs,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -279,6 +299,7 @@ impl StandaloneHost {
|
||||
worker_store: FsWorkerStore,
|
||||
record: StandaloneWorkerRecord,
|
||||
lease: StandaloneWorkerLease,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
) -> Self {
|
||||
Self {
|
||||
handle: started.handle,
|
||||
@@ -288,9 +309,16 @@ impl StandaloneHost {
|
||||
worker_store,
|
||||
record,
|
||||
lease: Some(lease),
|
||||
jobs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit Host execution capability for local Features, separate from the
|
||||
/// interactive Worker protocol and its model-created SubWorkers.
|
||||
pub fn jobs(&self) -> crate::jobs::StandaloneJobs {
|
||||
self.jobs.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn worker_id(&self) -> WorkerId {
|
||||
self.record.worker_id
|
||||
@@ -318,6 +346,7 @@ impl StandaloneHost {
|
||||
}
|
||||
|
||||
pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> {
|
||||
let jobs_result = self.jobs.shutdown().await;
|
||||
let command = protocol::WorkerCommandEnvelope::new(u64::MAX);
|
||||
let _ = self.handle.send(Method::Shutdown { command }).await;
|
||||
let Some(shutdown) = self.shutdown.take() else {
|
||||
@@ -335,6 +364,10 @@ impl StandaloneHost {
|
||||
return Err(StandaloneShutdownError::DeadlineExceeded);
|
||||
}
|
||||
}
|
||||
if jobs_result.is_err() {
|
||||
self.retain_lease();
|
||||
return Err(StandaloneShutdownError::JobCleanup);
|
||||
}
|
||||
let active = match active_pointer(&self.worker_store, &self.record.storage_key) {
|
||||
Ok(active) => active,
|
||||
Err(_) => {
|
||||
@@ -370,6 +403,16 @@ impl StandaloneHost {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StandaloneHost {
|
||||
fn drop(&mut self) {
|
||||
self.jobs.close();
|
||||
// A dropped Host cannot synchronously confirm either controller or Job
|
||||
// cleanup. Keep its lease until explicit shutdown or process recovery,
|
||||
// preventing another Host from reopening a still-live execution store.
|
||||
self.retain_lease();
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_protocol_session(
|
||||
handle: worker::WorkerHandle,
|
||||
streams: WorkerProtocolSessionStreams,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,393 @@
|
||||
//! Host-provided local Jobs. No Runtime registry, Workspace client or dialog-session state.
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use agen::llm_client::LlmClient;
|
||||
use job::{JobRequest, JobResultSubmission};
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::job_store::JobStore;
|
||||
pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum JobServiceError {
|
||||
#[error(transparent)]
|
||||
Store(#[from] JobStoreError),
|
||||
#[error("standalone Job execution requirements are unavailable: {0}")]
|
||||
Configuration(String),
|
||||
#[error("the standalone Job Host is shutting down")]
|
||||
Closed,
|
||||
#[error("standalone Job execution capacity is exhausted")]
|
||||
Capacity,
|
||||
#[error("standalone Job cleanup failed: {0}")]
|
||||
Cleanup(String),
|
||||
}
|
||||
|
||||
struct Execution {
|
||||
cancel: watch::Sender<bool>,
|
||||
complete: watch::Receiver<Option<Result<(), String>>>,
|
||||
attempt_id: String,
|
||||
resource_key: Option<String>,
|
||||
max_concurrent: u16,
|
||||
task: tokio::task::JoinHandle<Result<(), JobServiceError>>,
|
||||
}
|
||||
struct Executions {
|
||||
closed: bool,
|
||||
jobs: HashMap<String, Execution>,
|
||||
}
|
||||
|
||||
/// Cloneable Feature-facing access to a Host-owned service. Cloning this handle
|
||||
/// neither extends the Host's authority nor permits opening/recovering its store.
|
||||
#[derive(Clone)]
|
||||
pub struct StandaloneJobs {
|
||||
store: Arc<Mutex<Option<JobStore>>>,
|
||||
executions: Arc<Mutex<Executions>>,
|
||||
cwd: PathBuf,
|
||||
capacity: u16,
|
||||
}
|
||||
|
||||
impl StandaloneJobs {
|
||||
pub(crate) fn open(path: &Path, cwd: PathBuf) -> Result<Self, JobServiceError> {
|
||||
Ok(Self {
|
||||
store: Arc::new(Mutex::new(Some(JobStore::open(path)?))),
|
||||
executions: Arc::new(Mutex::new(Executions {
|
||||
closed: false,
|
||||
jobs: HashMap::new(),
|
||||
})),
|
||||
cwd,
|
||||
capacity: job::DEFAULT_MAX_CONCURRENT_JOBS,
|
||||
})
|
||||
}
|
||||
|
||||
fn with_store<T>(
|
||||
&self,
|
||||
operation: impl FnOnce(&mut JobStore) -> Result<T, JobStoreError>,
|
||||
) -> Result<T, JobServiceError> {
|
||||
let mut slot = self.store.lock().expect("Job store poisoned");
|
||||
Ok(operation(slot.as_mut().ok_or(JobServiceError::Closed)?)?)
|
||||
}
|
||||
|
||||
/// Persist intent without starting a model. Exact resend returns the same
|
||||
/// current attempt, including after acknowledgement or cancellation.
|
||||
pub fn request(&self, request: JobRequest) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
Ok(self.with_store(|store| store.reserve(request))?)
|
||||
}
|
||||
|
||||
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
Ok(self.with_store(|store| store.get(job_id))?)
|
||||
}
|
||||
|
||||
/// Pending attempts survive shutdown. The consumer chooses when to resume
|
||||
/// them; opening a Host never implicitly replays unknown side effects.
|
||||
pub fn pending(&self) -> Result<Vec<JobSnapshot>, JobServiceError> {
|
||||
Ok(self.with_store(|store| store.pending())?)
|
||||
}
|
||||
|
||||
/// Explicit new attempt, bounded by immutable intent limits. Unknown
|
||||
/// outcomes cannot be retried without domain reconciliation/new intent.
|
||||
pub fn retry(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
if executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| !run.task.is_finished())
|
||||
{
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
Ok(self.with_store(|store| store.retry(job_id))?)
|
||||
}
|
||||
|
||||
/// Successful model processing and consumer postprocessing have independent
|
||||
/// lifetimes. A failed delivery simply leaves this acknowledgement pending.
|
||||
pub fn acknowledge(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
Ok(self.with_store(|store| store.ack(job_id))?)
|
||||
}
|
||||
|
||||
pub fn cancel(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
let snapshot = self.with_store(|store| store.cancel(job_id))?;
|
||||
if let Some(execution) = executions.jobs.get(job_id) {
|
||||
execution.cancel.send_replace(true);
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
pub async fn start(&self, job_id: &str) -> Result<(), JobServiceError> {
|
||||
self.start_with_optional_model_client(job_id, None).await
|
||||
}
|
||||
|
||||
/// Injectable model transport for embedders and scripted execution tests.
|
||||
/// The requested registry Profile still resolves/validates normally; the
|
||||
/// transport grants no filesystem, Feature or result-submission authority.
|
||||
pub async fn start_with_model_client<C>(
|
||||
&self,
|
||||
job_id: &str,
|
||||
client: C,
|
||||
) -> Result<(), JobServiceError>
|
||||
where
|
||||
C: LlmClient + 'static,
|
||||
{
|
||||
self.start_with_optional_model_client(job_id, Some(Box::new(client)))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn start_with_optional_model_client(
|
||||
&self,
|
||||
job_id: &str,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
) -> Result<(), JobServiceError> {
|
||||
let snapshot = self.get(job_id)?;
|
||||
let prepared = worker::job::prepare_job(&snapshot.request, &self.cwd, client)
|
||||
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
executions.jobs.retain(|_, run| {
|
||||
!(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
|
||||
});
|
||||
// Count live resources, not only result states: acceptance can precede
|
||||
// the Internal Worker's cleanup and must not free a concurrency slot.
|
||||
let live = || {
|
||||
executions
|
||||
.jobs
|
||||
.values()
|
||||
.filter(|run| !run.task.is_finished())
|
||||
};
|
||||
let count = live().count();
|
||||
let active_limit = live()
|
||||
.map(|run| run.max_concurrent)
|
||||
.min()
|
||||
.unwrap_or(self.capacity);
|
||||
if count >= usize::from(self.capacity.min(active_limit))
|
||||
|| count >= usize::from(snapshot.request.limits.max_concurrent_jobs)
|
||||
|| executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| !run.task.is_finished())
|
||||
{
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
let resource_key = snapshot.request.serialization_key.clone();
|
||||
if let Some(key) = &resource_key {
|
||||
if executions
|
||||
.jobs
|
||||
.values()
|
||||
.any(|run| !run.task.is_finished() && run.resource_key.as_ref() == Some(key))
|
||||
{
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
}
|
||||
// Retain errors from finished tasks until a caller observes them rather
|
||||
// than silently swallowing store/cleanup failures.
|
||||
if let Some(previous) = executions.jobs.get(job_id) {
|
||||
if !previous.task.is_finished() {
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
}
|
||||
let snapshot = self.with_store(|store| {
|
||||
store.dispatch(job_id, &snapshot.attempt.attempt_id, self.capacity)
|
||||
})?;
|
||||
let store = self.store.clone();
|
||||
let id = job_id.to_string();
|
||||
let attempt_id = snapshot.attempt.attempt_id.clone();
|
||||
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
|
||||
let sink = Arc::new(BoundResultSink {
|
||||
store: store.clone(),
|
||||
job_id: id.clone(),
|
||||
attempt_id: attempt_id.clone(),
|
||||
input_revision: snapshot.request.input_revision.clone(),
|
||||
deadline: tokio::time::Instant::now() + timeout,
|
||||
});
|
||||
let (cancel, mut cancellation) = watch::channel(false);
|
||||
let (complete_tx, complete) = watch::channel(None);
|
||||
let execution_attempt = attempt_id.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let outcome: Result<(), JobServiceError> = async {
|
||||
let (sender_tx, mut sender_rx) = tokio::sync::oneshot::channel();
|
||||
let run = prepared.run(attempt_id.clone(), sink, move |sender| { let _ = sender_tx.send(sender); });
|
||||
tokio::pin!(run);
|
||||
let reason = tokio::select! {
|
||||
result = &mut run => {
|
||||
let (state, detail) = match result {
|
||||
Ok(_) => (JobAttemptState::Failed, "Worker finished without an accepted structured result".to_string()),
|
||||
Err(error @ worker::job::JobExecutionError::ResultUnconfirmed(_)) => (JobAttemptState::Unknown, error.to_string()),
|
||||
Err(error) => (JobAttemptState::Failed, error.to_string()),
|
||||
};
|
||||
finish_if_pending(&store, &id, &attempt_id, state, &detail)?;
|
||||
return Ok(());
|
||||
}
|
||||
_ = tokio::time::sleep(timeout) => "timeout",
|
||||
_ = cancellation.wait_for(|cancelled| *cancelled) => "Host cancellation/shutdown",
|
||||
};
|
||||
// Fence timeout/shutdown before cancelling so a late tool call can
|
||||
// never turn an interrupted unaccepted execution into success.
|
||||
let fence_result = finish_if_pending(&store, &id, &attempt_id, JobAttemptState::Unknown, reason);
|
||||
// Cancellation is routed through Engine; keep the future alive to
|
||||
// allow normal Feature cleanup. Hard stop is a fenced unknown
|
||||
// outcome, never an automatic retry. Result acceptance wins races.
|
||||
if let Ok(sender) = sender_rx.try_recv() {
|
||||
let _ = sender.try_send(());
|
||||
} else if let Ok(Ok(sender)) = tokio::time::timeout(Duration::from_secs(1), &mut sender_rx).await {
|
||||
let _ = sender.try_send(());
|
||||
}
|
||||
let cleanup = tokio::time::timeout(Duration::from_secs(2), &mut run).await;
|
||||
fence_result?;
|
||||
if cleanup.is_err() { return Err(JobServiceError::Cleanup("Internal Worker cancellation deadline exceeded; outcome fenced unknown".into())); }
|
||||
Ok(())
|
||||
}.await;
|
||||
complete_tx.send_replace(Some(
|
||||
outcome.as_ref().map(|_| ()).map_err(ToString::to_string),
|
||||
));
|
||||
outcome
|
||||
});
|
||||
executions.jobs.insert(
|
||||
job_id.to_string(),
|
||||
Execution {
|
||||
cancel,
|
||||
complete,
|
||||
task,
|
||||
attempt_id: execution_attempt,
|
||||
resource_key,
|
||||
max_concurrent: snapshot.request.limits.max_concurrent_jobs,
|
||||
},
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Observe execution/cleanup, not success: read the durable result separately.
|
||||
pub async fn wait(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
let execution = self
|
||||
.executions
|
||||
.lock()
|
||||
.expect("Job executions poisoned")
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.map(|run| (run.attempt_id.clone(), run.complete.clone()));
|
||||
if let Some((attempt, mut completion)) = execution {
|
||||
let result = completion
|
||||
.wait_for(Option::is_some)
|
||||
.await
|
||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))?
|
||||
.clone()
|
||||
.expect("completion");
|
||||
result.map_err(JobServiceError::Cleanup)?;
|
||||
let task = {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| run.attempt_id == attempt)
|
||||
{
|
||||
executions.jobs.remove(job_id)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(execution) = task {
|
||||
execution
|
||||
.task
|
||||
.await
|
||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))??;
|
||||
}
|
||||
}
|
||||
self.get(job_id)
|
||||
}
|
||||
|
||||
pub(crate) fn close(&self) {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
executions.closed = true;
|
||||
for execution in executions.jobs.values() {
|
||||
execution.cancel.send_replace(true);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
|
||||
self.close();
|
||||
let tasks = {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
std::mem::take(&mut executions.jobs)
|
||||
};
|
||||
let mut error = None;
|
||||
for execution in tasks.into_values() {
|
||||
match execution.task.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(failure)) => {
|
||||
error = Some(failure);
|
||||
}
|
||||
Err(failure) => {
|
||||
error = Some(JobServiceError::Cleanup(failure.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Close SQLite before the Host releases its lease. Cloned handles may
|
||||
// outlive the Host but cannot mutate or retain the live connection.
|
||||
self.store.lock().expect("Job store poisoned").take();
|
||||
error.map_or(Ok(()), Err)
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_if_pending(
|
||||
store: &Mutex<Option<JobStore>>,
|
||||
id: &str,
|
||||
attempt: &str,
|
||||
state: JobAttemptState,
|
||||
detail: &str,
|
||||
) -> Result<(), JobServiceError> {
|
||||
let mut slot = store.lock().expect("Job store poisoned");
|
||||
let store = slot.as_mut().ok_or(JobServiceError::Closed)?;
|
||||
let snapshot = store.get(id)?;
|
||||
if snapshot.state == JobState::Pending {
|
||||
store.finish(id, attempt, state, Some(detail))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct BoundResultSink {
|
||||
store: Arc<Mutex<Option<JobStore>>>,
|
||||
job_id: String,
|
||||
attempt_id: String,
|
||||
input_revision: String,
|
||||
deadline: tokio::time::Instant,
|
||||
}
|
||||
impl worker::job::JobResultSink for BoundResultSink {
|
||||
fn submit(&self, submission: JobResultSubmission) -> Result<(), String> {
|
||||
if submission.job_id != self.job_id
|
||||
|| submission.attempt_id != self.attempt_id
|
||||
|| submission.input_revision != self.input_revision
|
||||
{
|
||||
return Err("Job result capability binding mismatch".into());
|
||||
}
|
||||
let mut slot = self
|
||||
.store
|
||||
.lock()
|
||||
.map_err(|_| "Job store poisoned".to_string())?;
|
||||
let store = slot.as_mut().ok_or_else(|| "Job Host closed".to_string())?;
|
||||
let snapshot = store.get(&self.job_id).map_err(|error| error.to_string())?;
|
||||
if tokio::time::Instant::now() >= self.deadline && snapshot.state != JobState::Completed {
|
||||
return Err("Job result deadline expired".into());
|
||||
}
|
||||
store
|
||||
.accept(&submission)
|
||||
.map(|_| ())
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
use super::*;
|
||||
use worker::job::JobResultSink;
|
||||
|
||||
fn request(id: &str) -> JobRequest {
|
||||
JobRequest {
|
||||
job_id: id.into(),
|
||||
purpose: "test".into(),
|
||||
input_revision: "r1".into(),
|
||||
input_ref: "fixture:input".into(),
|
||||
input: serde_json::json!({"data":42}),
|
||||
instruction: "Return a structured result".into(),
|
||||
profile: "builtin:job".into(),
|
||||
serialization_key: None,
|
||||
limits: job::JobLimits::default(),
|
||||
}
|
||||
}
|
||||
fn sink(jobs: &StandaloneJobs, id: &str, attempt: &str) -> BoundResultSink {
|
||||
BoundResultSink {
|
||||
store: jobs.store.clone(),
|
||||
job_id: id.into(),
|
||||
attempt_id: attempt.into(),
|
||||
input_revision: "r1".into(),
|
||||
deadline: tokio::time::Instant::now() + Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
fn result(id: &str, attempt: &str) -> JobResultSubmission {
|
||||
JobResultSubmission {
|
||||
job_id: id.into(),
|
||||
attempt_id: attempt.into(),
|
||||
input_revision: "r1".into(),
|
||||
result: serde_json::json!({"valid":true}),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn host_result_capability_cannot_cross_job_attempt_revision_or_deadline() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let jobs =
|
||||
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
|
||||
let first = jobs.request(request("first")).unwrap();
|
||||
let second = jobs.request(request("second")).unwrap();
|
||||
for snapshot in [&first, &second] {
|
||||
jobs.with_store(|store| {
|
||||
store.dispatch(&snapshot.request.job_id, &snapshot.attempt.attempt_id, 8)
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
let bound = sink(&jobs, "first", &first.attempt.attempt_id);
|
||||
assert!(
|
||||
bound
|
||||
.submit(result("second", &second.attempt.attempt_id))
|
||||
.is_err()
|
||||
);
|
||||
assert!(bound.submit(result("first", "first:attempt:2")).is_err());
|
||||
let mut stale = result("first", &first.attempt.attempt_id);
|
||||
stale.input_revision = "r0".into();
|
||||
assert!(bound.submit(stale).is_err());
|
||||
let mut expired = sink(&jobs, "first", &first.attempt.attempt_id);
|
||||
expired.deadline = tokio::time::Instant::now() - Duration::from_secs(1);
|
||||
assert!(
|
||||
expired
|
||||
.submit(result("first", &first.attempt.attempt_id))
|
||||
.is_err()
|
||||
);
|
||||
assert!(jobs.get("first").unwrap().result.is_none());
|
||||
bound
|
||||
.submit(result("first", &first.attempt.attempt_id))
|
||||
.unwrap();
|
||||
// Accepted exact replay is still allowed after the deadline, never a new result.
|
||||
expired
|
||||
.submit(result("first", &first.attempt.attempt_id))
|
||||
.unwrap();
|
||||
let mut changed = result("first", &first.attempt.attempt_id);
|
||||
changed.result = serde_json::json!({"valid":false});
|
||||
assert!(expired.submit(changed).is_err());
|
||||
assert_eq!(jobs.get("second").unwrap().state, JobState::Pending);
|
||||
jobs.shutdown().await.unwrap();
|
||||
assert!(
|
||||
bound
|
||||
.submit(result("first", &first.attempt.attempt_id))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_handles_are_closed() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let jobs =
|
||||
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
|
||||
let cancelled = jobs.request(request("cancelled")).unwrap();
|
||||
jobs.cancel("cancelled").unwrap();
|
||||
finish_if_pending(
|
||||
&jobs.store,
|
||||
"cancelled",
|
||||
&cancelled.attempt.attempt_id,
|
||||
JobAttemptState::Unknown,
|
||||
"cleanup",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
|
||||
let accepted = jobs.request(request("accepted")).unwrap();
|
||||
jobs.with_store(|store| store.dispatch("accepted", &accepted.attempt.attempt_id, 8))
|
||||
.unwrap();
|
||||
sink(&jobs, "accepted", &accepted.attempt.attempt_id)
|
||||
.submit(result("accepted", &accepted.attempt.attempt_id))
|
||||
.unwrap();
|
||||
finish_if_pending(
|
||||
&jobs.store,
|
||||
"accepted",
|
||||
&accepted.attempt.attempt_id,
|
||||
JobAttemptState::Failed,
|
||||
"model failed later",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.get("accepted").unwrap().state, JobState::Completed);
|
||||
jobs.shutdown().await.unwrap();
|
||||
assert!(jobs.store.lock().unwrap().is_none());
|
||||
for operation in [
|
||||
jobs.cancel("cancelled"),
|
||||
jobs.acknowledge("accepted"),
|
||||
jobs.retry("cancelled"),
|
||||
jobs.request(request("new")),
|
||||
] {
|
||||
assert!(matches!(operation, Err(JobServiceError::Closed)));
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
//! Server, HTTP, WebSocket, subprocess Worker, or alternative execution path.
|
||||
|
||||
pub mod host;
|
||||
mod job_store;
|
||||
pub mod jobs;
|
||||
pub mod launch;
|
||||
pub mod store;
|
||||
|
||||
|
||||
@@ -352,6 +352,10 @@ impl StandaloneWorkerStore {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn jobs_path(&self, id: WorkerId) -> PathBuf {
|
||||
self.worker_dir(id).join("jobs.sqlite3")
|
||||
}
|
||||
|
||||
pub(crate) fn runtime_dir(&self, id: WorkerId) -> PathBuf {
|
||||
self.worker_dir(id).join("runtime")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use agen::llm_client::{
|
||||
LlmClient,
|
||||
error::ClientError,
|
||||
event::{Event as LlmEvent, StopReason},
|
||||
types::Request,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use futures::{Stream, stream};
|
||||
use job::{JobLimits, JobRequest};
|
||||
use standalone::jobs::{JobServiceError, JobState};
|
||||
use standalone::{StandaloneHost, StandaloneLaunchConfig};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ScriptedClient {
|
||||
responses: Arc<Mutex<VecDeque<Vec<LlmEvent>>>>,
|
||||
requests: Arc<Mutex<Vec<Request>>>,
|
||||
started: Arc<tokio::sync::Notify>,
|
||||
stall: bool,
|
||||
fail: bool,
|
||||
}
|
||||
impl ScriptedClient {
|
||||
fn new(responses: Vec<Vec<LlmEvent>>) -> Self {
|
||||
Self {
|
||||
responses: Arc::new(Mutex::new(responses.into())),
|
||||
requests: Arc::new(Mutex::new(Vec::new())),
|
||||
started: Arc::new(tokio::sync::Notify::new()),
|
||||
stall: false,
|
||||
fail: false,
|
||||
}
|
||||
}
|
||||
fn stall() -> Self {
|
||||
Self {
|
||||
stall: true,
|
||||
..Self::new(Vec::new())
|
||||
}
|
||||
}
|
||||
fn requests(&self) -> Vec<Request> {
|
||||
self.requests.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl LlmClient for ScriptedClient {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
self.requests.lock().unwrap().push(request);
|
||||
self.started.notify_one();
|
||||
if self.fail {
|
||||
return Err(ClientError::Config("scripted model failure".into()));
|
||||
}
|
||||
if self.stall {
|
||||
return Ok(Box::pin(stream::pending()));
|
||||
}
|
||||
let events = self
|
||||
.responses
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.expect("scripted response");
|
||||
Ok(Box::pin(stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
fn prose() -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
LlmEvent::text_delta(0, "done"),
|
||||
LlmEvent::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||
]
|
||||
}
|
||||
fn success() -> ScriptedClient {
|
||||
ScriptedClient::new(vec![
|
||||
vec![
|
||||
LlmEvent::tool_use_start(0, "result-1", "SubmitJobResult"),
|
||||
LlmEvent::tool_input_delta(0, r#"{"result":{"valid":true}}"#),
|
||||
LlmEvent::tool_use_stop(0),
|
||||
],
|
||||
prose(),
|
||||
])
|
||||
}
|
||||
fn request(id: &str) -> JobRequest {
|
||||
JobRequest {
|
||||
job_id: id.into(),
|
||||
purpose: "generic_test".into(),
|
||||
input_revision: "r1".into(),
|
||||
input_ref: "test:immutable".into(),
|
||||
input: serde_json::json!({"test_input":42}),
|
||||
instruction: "Return a structured validity result".into(),
|
||||
profile: "builtin:job".into(),
|
||||
serialization_key: None,
|
||||
limits: JobLimits::default(),
|
||||
}
|
||||
}
|
||||
async fn host(temp: &tempfile::TempDir) -> (StandaloneHost, ScriptedClient) {
|
||||
let launch = StandaloneLaunchConfig::new(
|
||||
temp.path(),
|
||||
temp.path().join("state"),
|
||||
manifest::ProfileSelector::Default,
|
||||
"job-host",
|
||||
)
|
||||
.resolve()
|
||||
.unwrap();
|
||||
let client = ScriptedClient::new(Vec::new());
|
||||
let host = StandaloneHost::start_with_model_client(launch, client.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
(host, client)
|
||||
}
|
||||
async fn wait_started(client: &ScriptedClient) {
|
||||
tokio::time::timeout(Duration::from_secs(10), client.started.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn host_job_runs_without_dialog_and_result_survives_restore_and_ack() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, dialog) = host(&temp).await;
|
||||
let id = host.worker_id();
|
||||
let jobs = host.jobs();
|
||||
let intent = request("success");
|
||||
let reserved = jobs.request(intent.clone()).unwrap();
|
||||
assert_eq!(jobs.request(intent.clone()).unwrap(), reserved);
|
||||
let model = success();
|
||||
jobs.start_with_model_client("success", model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = jobs.wait("success").await.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
assert_eq!(completed.result, Some(serde_json::json!({"valid":true})));
|
||||
assert!(!completed.acknowledged);
|
||||
let neutral = completed.outcome();
|
||||
assert_eq!(neutral.state, job::JobState::Completed);
|
||||
assert_eq!(neutral.result, completed.result);
|
||||
assert_eq!(neutral.attempt_state, job::JobAttemptState::Completed);
|
||||
assert!(
|
||||
dialog.requests().is_empty(),
|
||||
"Job never runs on the interactive model/session"
|
||||
);
|
||||
assert_eq!(
|
||||
model.requests()[0]
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| tool.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["SubmitJobResult"]
|
||||
);
|
||||
assert_eq!(jobs.request(intent.clone()).unwrap(), completed);
|
||||
assert!(
|
||||
jobs.start_with_model_client("success", success())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(jobs.retry("success").is_err());
|
||||
let db = temp
|
||||
.path()
|
||||
.join("state")
|
||||
.join(id.to_string())
|
||||
.join("jobs.sqlite3");
|
||||
assert!(db.exists());
|
||||
host.shutdown().await.unwrap();
|
||||
assert!(matches!(
|
||||
jobs.request(request("after-shutdown")),
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
assert!(matches!(
|
||||
jobs.cancel("success"),
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
assert!(matches!(
|
||||
jobs.acknowledge("success"),
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
assert!(matches!(
|
||||
jobs.start_with_model_client("success", success()).await,
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
let restored = StandaloneHost::restore_with_model_client(
|
||||
temp.path().join("state"),
|
||||
id,
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let restored_jobs = restored.jobs();
|
||||
assert_eq!(restored_jobs.get("success").unwrap(), completed);
|
||||
assert!(restored_jobs.acknowledge("success").unwrap().acknowledged);
|
||||
restored.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_resource_limits_cancellation_and_wait_are_host_owned() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
let mut first = request("first");
|
||||
first.limits.max_concurrent_jobs = 1;
|
||||
jobs.request(first).unwrap();
|
||||
jobs.request(request("second")).unwrap();
|
||||
let stalled = ScriptedClient::stall();
|
||||
jobs.start_with_model_client("first", stalled.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_started(&stalled).await;
|
||||
// Waiting must not remove the Host-owned task/cancellation capability.
|
||||
let waiting_jobs = jobs.clone();
|
||||
let waiter = tokio::spawn(async move { waiting_jobs.wait("first").await });
|
||||
assert!(matches!(
|
||||
jobs.start_with_model_client("second", success()).await,
|
||||
Err(JobServiceError::Capacity)
|
||||
));
|
||||
jobs.cancel("first").unwrap();
|
||||
let cancelled = tokio::time::timeout(Duration::from_secs(10), waiter)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(cancelled.state, JobState::Cancelled);
|
||||
assert!(jobs.retry("first").is_err());
|
||||
jobs.start_with_model_client("second", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("second").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prose_failure_requires_explicit_bounded_new_attempt_and_exact_replay_does_not_retry() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
let intent = request("prose");
|
||||
jobs.request(intent.clone()).unwrap();
|
||||
jobs.start_with_model_client("prose", ScriptedClient::new(vec![prose()]))
|
||||
.await
|
||||
.unwrap();
|
||||
let failed = jobs.wait("prose").await.unwrap();
|
||||
assert_eq!(failed.state, JobState::Failed);
|
||||
assert!(failed.result.is_none());
|
||||
assert_eq!(jobs.request(intent).unwrap(), failed);
|
||||
let retry = jobs.retry("prose").unwrap();
|
||||
assert_eq!(retry.attempt.number, 2);
|
||||
assert_ne!(retry.attempt.attempt_id, failed.attempt.attempt_id);
|
||||
jobs.start_with_model_client("prose", ScriptedClient::new(vec![prose()]))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(jobs.wait("prose").await.unwrap().state, JobState::Failed);
|
||||
assert!(jobs.retry("prose").is_err());
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timeout_is_unknown_not_prose_success_or_automatic_retry() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
let mut intent = request("timeout");
|
||||
intent.limits.timeout_seconds = 1;
|
||||
jobs.request(intent).unwrap();
|
||||
let stalled = ScriptedClient::stall();
|
||||
jobs.start_with_model_client("timeout", stalled.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_started(&stalled).await;
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(10), jobs.wait("timeout"))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(outcome.state, JobState::Unknown);
|
||||
assert!(outcome.result.is_none());
|
||||
assert!(jobs.retry("timeout").is_err());
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_keeps_reserved_results_and_fences_running_attempts_on_reopen() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let id = host.worker_id();
|
||||
let jobs = host.jobs();
|
||||
jobs.request(request("reserved")).unwrap();
|
||||
jobs.request(request("cancelled")).unwrap();
|
||||
jobs.cancel("cancelled").unwrap();
|
||||
jobs.request(request("running")).unwrap();
|
||||
let stalled = ScriptedClient::stall();
|
||||
jobs.start_with_model_client("running", stalled.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_started(&stalled).await;
|
||||
host.shutdown().await.unwrap();
|
||||
assert!(matches!(jobs.get("running"), Err(JobServiceError::Closed)));
|
||||
let restored = StandaloneHost::restore_with_model_client(
|
||||
temp.path().join("state"),
|
||||
id,
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let jobs = restored.jobs();
|
||||
assert_eq!(
|
||||
jobs.pending()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|snapshot| snapshot.request.job_id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["reserved"]
|
||||
);
|
||||
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
|
||||
assert_eq!(jobs.get("running").unwrap().state, JobState::Unknown);
|
||||
assert!(jobs.retry("running").is_err());
|
||||
jobs.start_with_model_client("reserved", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("reserved").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
restored.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_requirements_are_rejected_without_model_execution_or_implicit_fallback() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
for profile in [
|
||||
"builtin:coder",
|
||||
"builtin:backend-job",
|
||||
"builtin:does-not-exist",
|
||||
] {
|
||||
let mut intent = request(profile);
|
||||
intent.profile = profile.into();
|
||||
jobs.request(intent).unwrap();
|
||||
let model = success();
|
||||
assert!(matches!(
|
||||
jobs.start_with_model_client(profile, model.clone()).await,
|
||||
Err(JobServiceError::Configuration(_))
|
||||
));
|
||||
assert!(model.requests().is_empty());
|
||||
assert_eq!(jobs.get(profile).unwrap().state, JobState::Pending);
|
||||
}
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
// Only this test harness starts a subprocess: production Jobs remain in-process.
|
||||
// The child exits deliberately without destructors to prove actual process-loss
|
||||
// recovery, rather than treating a hand-edited SQLite fixture as that evidence.
|
||||
#[tokio::test]
|
||||
#[ignore = "entry point for process_loss_recovers_durable_jobs_without_replaying_unknown"]
|
||||
async fn process_job_fixture() {
|
||||
let root = std::path::PathBuf::from(
|
||||
std::env::var_os("YOI_JOB_PROCESS_FIXTURE_DIR").expect("fixture root"),
|
||||
);
|
||||
let launch = StandaloneLaunchConfig::new(
|
||||
&root,
|
||||
root.join("state"),
|
||||
manifest::ProfileSelector::Default,
|
||||
"job-process-fixture",
|
||||
)
|
||||
.resolve()
|
||||
.unwrap();
|
||||
let host = StandaloneHost::start_with_model_client(launch, ScriptedClient::new(Vec::new()))
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::write(root.join("worker-id"), host.worker_id().to_string()).unwrap();
|
||||
let jobs = host.jobs();
|
||||
jobs.request(request("reserved")).unwrap();
|
||||
jobs.request(request("cancelled")).unwrap();
|
||||
jobs.cancel("cancelled").unwrap();
|
||||
jobs.request(request("accepted")).unwrap();
|
||||
jobs.start_with_model_client("accepted", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("accepted").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
jobs.request(request("interrupted")).unwrap();
|
||||
let stalled = ScriptedClient::stall();
|
||||
jobs.start_with_model_client("interrupted", stalled.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_started(&stalled).await;
|
||||
// Test-owned process only, never the operating Yoi Worker or dogfood daemon.
|
||||
std::process::exit(23);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn process_loss_recovers_durable_jobs_without_replaying_unknown() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let output = std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "process_job_fixture", "--ignored", "--nocapture"])
|
||||
.env("YOI_JOB_PROCESS_FIXTURE_DIR", temp.path())
|
||||
.output()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
output.status.code(),
|
||||
Some(23),
|
||||
"fixture failed: {}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let worker_id = std::fs::read_to_string(temp.path().join("worker-id"))
|
||||
.unwrap()
|
||||
.parse()
|
||||
.unwrap();
|
||||
let host = StandaloneHost::restore_with_model_client(
|
||||
temp.path().join("state"),
|
||||
worker_id,
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let jobs = host.jobs();
|
||||
assert_eq!(jobs.get("interrupted").unwrap().state, JobState::Unknown);
|
||||
assert!(jobs.retry("interrupted").is_err());
|
||||
let completed = jobs.get("accepted").unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
assert_eq!(completed.result, Some(serde_json::json!({"valid":true})));
|
||||
assert!(!completed.acknowledged);
|
||||
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
|
||||
assert_eq!(jobs.pending().unwrap().len(), 1);
|
||||
jobs.start_with_model_client("reserved", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("reserved").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_failure_is_durable_known_failure_with_explicit_successful_retry() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
jobs.request(request("model-failure")).unwrap();
|
||||
let client = ScriptedClient {
|
||||
fail: true,
|
||||
..ScriptedClient::new(Vec::new())
|
||||
};
|
||||
jobs.start_with_model_client("model-failure", client)
|
||||
.await
|
||||
.unwrap();
|
||||
let failure = jobs.wait("model-failure").await.unwrap();
|
||||
assert_eq!(failure.state, JobState::Failed);
|
||||
assert!(failure.result.is_none());
|
||||
assert!(
|
||||
failure
|
||||
.attempt
|
||||
.failure
|
||||
.unwrap()
|
||||
.contains("scripted model failure")
|
||||
);
|
||||
let next = jobs.retry("model-failure").unwrap();
|
||||
assert_eq!(next.attempt.number, 2);
|
||||
jobs.start_with_model_client("model-failure", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("model-failure").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
@@ -16,6 +16,7 @@ agen = { workspace = true, features = ["codex"] }
|
||||
session-store = { workspace = true }
|
||||
secrets = { workspace = true }
|
||||
manifest = { workspace = true }
|
||||
job = { workspace = true }
|
||||
mcp = { workspace = true }
|
||||
protocol = { workspace = true, features = ["json-schema"] }
|
||||
client = { workspace = true }
|
||||
|
||||
@@ -205,6 +205,9 @@ where
|
||||
.copied()
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
worker
|
||||
.stop_feature_runtime("internal Worker feature installation failed")
|
||||
.await;
|
||||
return Err(InternalWorkerError {
|
||||
source: WorkerError::FeatureInstall(format!(
|
||||
"internal Worker feature installation failed: {diagnostics}; missing tools: {missing}"
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
//! Host-facing, result-only Jobs executed by the private Internal Worker substrate.
|
||||
//!
|
||||
//! The host owns persistence, concurrency, timeout, retries and result acceptance. Normal
|
||||
//! timeout/shutdown routes through the cancellation sender and awaits `run`, including Worker
|
||||
//! resource cleanup. An emergency drop/abort does not confirm asynchronous cleanup; the Host
|
||||
//! must retain its lease and reconcile any unknown outcome before allowing another execution.
|
||||
//! The result-only Feature owns no background tasks, but an in-flight synchronous Host sink
|
||||
//! cannot be preempted by dropping its tool future and must itself be bounded.
|
||||
|
||||
mod result;
|
||||
pub use result::{JobResultFeature, JobResultSink, SUBMIT_JOB_RESULT_TOOL};
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use agen::llm_client::LlmClient;
|
||||
use manifest::{ProfileDiscovery, ProfileResolveOptions, ProfileResolver, Scope, WorkerManifest};
|
||||
|
||||
use crate::feature::FeatureRegistryBuilder;
|
||||
use crate::internal_worker::{
|
||||
InternalWorkerAuthority, InternalWorkerIdentity, InternalWorkerSpec,
|
||||
run_internal_worker_with_cancel_sender,
|
||||
};
|
||||
use crate::{
|
||||
PromptCatalog, PromptCatalogSource, SystemPromptContext, SystemPromptTemplate,
|
||||
WorkerFilesystemAuthority, WorkerPrompt, WorkerRunResult, WorkerWorkspaceContext,
|
||||
};
|
||||
|
||||
/// Host adapter ceilings, applied even if the selected Profile leaves budgets unbounded.
|
||||
pub const MAX_JOB_TURNS: u32 = 32;
|
||||
pub const MAX_JOB_TOKENS: u32 = 8192;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum JobExecutionError {
|
||||
#[error(transparent)]
|
||||
Request(#[from] ::job::JobError),
|
||||
#[error("Job Profile resolution failed: {0}")]
|
||||
Profile(#[from] manifest::ProfileError),
|
||||
#[error("result-only Job cannot satisfy Profile requirements: {0}")]
|
||||
UnsupportedProfile(String),
|
||||
#[error("Job instruction resolution failed: {0}")]
|
||||
Instruction(#[from] crate::SystemPromptError),
|
||||
#[error("Job Prompt catalog resolution failed: {0}")]
|
||||
Catalog(#[from] crate::CatalogError),
|
||||
#[error("Job model configuration failed: {0}")]
|
||||
Model(#[from] crate::ProviderError),
|
||||
#[error("Job execution failed: {0}")]
|
||||
Worker(String),
|
||||
/// A sink error can follow durable acceptance. Hosts must reconcile their store and must
|
||||
/// not classify this as a known retryable failure unless they can prove non-acceptance.
|
||||
#[error(
|
||||
"Host Job result acceptance is unconfirmed; reconcile the Host store before retrying: {0}"
|
||||
)]
|
||||
ResultUnconfirmed(String),
|
||||
#[error("Job execution was cancelled without an accepted result")]
|
||||
Cancelled,
|
||||
#[error("Job ended without an accepted SubmitJobResult; final prose is not success")]
|
||||
MissingResult,
|
||||
}
|
||||
|
||||
/// Confirms only a synchronous, successful Host acceptance. Lifecycle/prose cannot create this.
|
||||
#[derive(Debug)]
|
||||
pub struct InternalJobResult {
|
||||
pub submission: ::job::JobResultSubmission,
|
||||
pub usage: Option<agen::timeline::event::UsageEvent>,
|
||||
}
|
||||
|
||||
/// Resolved policy/model/prompt, with no running resources and no inherited authority.
|
||||
pub struct PreparedInternalJob {
|
||||
request: ::job::JobRequest,
|
||||
manifest: WorkerManifest,
|
||||
client: Box<dyn LlmClient>,
|
||||
system_prompt: String,
|
||||
}
|
||||
|
||||
/// Resolve exactly the requested registry Profile using the existing user settings registry.
|
||||
/// `cwd` is a resolution base only; it never becomes filesystem or Workspace authority.
|
||||
/// An injected client is useful for scripted executions and does not bypass policy validation.
|
||||
pub fn prepare_job(
|
||||
request: &::job::JobRequest,
|
||||
cwd: &Path,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
) -> Result<PreparedInternalJob, JobExecutionError> {
|
||||
let registry = ProfileDiscovery::user_settings().discover()?;
|
||||
prepare_job_from_registry(request, cwd, client, ®istry)
|
||||
}
|
||||
|
||||
fn prepare_job_from_registry(
|
||||
request: &::job::JobRequest,
|
||||
cwd: &Path,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
registry: &manifest::ProfileRegistry,
|
||||
) -> Result<PreparedInternalJob, JobExecutionError> {
|
||||
request.validate()?;
|
||||
let resolved = ProfileResolver::new()
|
||||
.with_workspace_base(cwd)
|
||||
.resolve_from_registry(
|
||||
&request.profile_selector()?,
|
||||
registry,
|
||||
ProfileResolveOptions::with_worker_name("internal-job"),
|
||||
)?;
|
||||
let mut manifest = resolved.manifest;
|
||||
validate_result_only_profile(&manifest)?;
|
||||
let scope = Scope::empty();
|
||||
let prompts = PromptCatalog::builtins_only()?;
|
||||
let template = SystemPromptTemplate::parse(
|
||||
&manifest.engine.instruction,
|
||||
PromptCatalogSource::builtins_only(),
|
||||
)?;
|
||||
let mut system_prompt = template.render(&SystemPromptContext {
|
||||
now: chrono::Utc::now(),
|
||||
cwd: Cow::Borrowed("(no filesystem)"),
|
||||
language: &manifest.engine.language,
|
||||
scope: &scope,
|
||||
tool_names: vec![SUBMIT_JOB_RESULT_TOOL.into()],
|
||||
feature_instructions: &[],
|
||||
agents_md: None,
|
||||
resident_summary: None,
|
||||
prompts: &prompts,
|
||||
})?;
|
||||
system_prompt.push_str("\n\n");
|
||||
system_prompt.push_str(&prompts.render(WorkerPrompt::JobSystem, minijinja::context! {})?);
|
||||
// Host budgets attenuate Profile output/turn policy, never grant another capability.
|
||||
manifest.engine.max_turns = std::num::NonZeroU32::new(
|
||||
manifest
|
||||
.engine
|
||||
.max_turns
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(MAX_JOB_TURNS)
|
||||
.min(MAX_JOB_TURNS),
|
||||
);
|
||||
manifest.engine.max_tokens = Some(
|
||||
manifest
|
||||
.engine
|
||||
.max_tokens
|
||||
.unwrap_or(MAX_JOB_TOKENS)
|
||||
.min(MAX_JOB_TOKENS),
|
||||
);
|
||||
let model_config = manifest::model_catalog::resolve_model_manifest(&manifest.model)
|
||||
.map_err(crate::ProviderError::from)?;
|
||||
let client = match client {
|
||||
Some(client) => client,
|
||||
None => crate::model_client::build_client_from_config(&model_config)?,
|
||||
};
|
||||
Ok(PreparedInternalJob {
|
||||
request: request.clone(),
|
||||
manifest,
|
||||
client,
|
||||
system_prompt,
|
||||
})
|
||||
}
|
||||
|
||||
impl PreparedInternalJob {
|
||||
/// Run one immutable attempt. The sink must perform fenced, durable acceptance before
|
||||
/// returning Ok. Timeout is Host-owned: cancel and then await this future for cleanup.
|
||||
/// A result already accepted by the sink remains success even if the model subsequently
|
||||
/// fails, reaches its turn limit or is cancelled; callers never rerun an accepted result.
|
||||
pub async fn run<F>(
|
||||
self,
|
||||
attempt_id: String,
|
||||
submit: Arc<dyn JobResultSink>,
|
||||
on_cancel_sender: F,
|
||||
) -> Result<InternalJobResult, JobExecutionError>
|
||||
where
|
||||
F: FnOnce(tokio::sync::mpsc::Sender<()>),
|
||||
{
|
||||
let feature = JobResultFeature::new(&self.request, attempt_id.clone(), submit)?;
|
||||
let accepted = feature.accepted();
|
||||
let engine_policy = self.manifest.engine.clone();
|
||||
let spec = InternalWorkerSpec {
|
||||
identity: InternalWorkerIdentity {
|
||||
kind: "job",
|
||||
run_id: uuid::Uuid::now_v7(),
|
||||
},
|
||||
max_turns: engine_policy.max_turns.map(|n| n.get()),
|
||||
engine_configurator: Some(Box::new(move |engine| {
|
||||
crate::apply_worker_manifest(engine, &engine_policy);
|
||||
})),
|
||||
manifest: self.manifest,
|
||||
client: self.client,
|
||||
system_prompt: self.system_prompt,
|
||||
input: self.request.worker_input(&attempt_id)?,
|
||||
cache_key: None,
|
||||
features: FeatureRegistryBuilder::new().with_module(feature.clone()),
|
||||
required_tools: &[SUBMIT_JOB_RESULT_TOOL],
|
||||
authority: InternalWorkerAuthority {
|
||||
workspace: WorkerWorkspaceContext::no_workspace(),
|
||||
filesystem: WorkerFilesystemAuthority::None,
|
||||
scope: Scope::empty(),
|
||||
workdir_session: None,
|
||||
},
|
||||
};
|
||||
let outcome = run_internal_worker_with_cancel_sender(spec, on_cancel_sender).await;
|
||||
let usage = match &outcome {
|
||||
Ok(run) => run.usage.clone(),
|
||||
Err(error) => error.usage.clone(),
|
||||
};
|
||||
if let Some(submission) = accepted.lock().expect("Job result state poisoned").clone() {
|
||||
return Ok(InternalJobResult { submission, usage });
|
||||
}
|
||||
if let Some(error) = feature.unconfirmed() {
|
||||
return Err(JobExecutionError::ResultUnconfirmed(error));
|
||||
}
|
||||
match outcome {
|
||||
Err(error) => Err(JobExecutionError::Worker(error.source.to_string())),
|
||||
Ok(run)
|
||||
if matches!(
|
||||
run.lifecycle,
|
||||
WorkerRunResult::Cancelled | WorkerRunResult::RolledBack
|
||||
) =>
|
||||
{
|
||||
Err(JobExecutionError::Cancelled)
|
||||
}
|
||||
Ok(_) => Err(JobExecutionError::MissingResult),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate requirements, not Profile names. Nothing configured as active is silently stripped.
|
||||
fn validate_result_only_profile(manifest: &WorkerManifest) -> Result<(), JobExecutionError> {
|
||||
let manifest::FeatureConfig {
|
||||
task,
|
||||
memory,
|
||||
subjektiv,
|
||||
web,
|
||||
image,
|
||||
sub_worker,
|
||||
flow,
|
||||
worker,
|
||||
workspace_worker_discovery,
|
||||
objective,
|
||||
manage_workdir,
|
||||
workdir_catalog,
|
||||
workspace_config,
|
||||
ticket,
|
||||
merge_request,
|
||||
orchestration,
|
||||
} = &manifest.feature;
|
||||
let mut unavailable = Vec::new();
|
||||
for (name, enabled) in [
|
||||
("feature.task", task.enabled),
|
||||
(
|
||||
"feature.memory",
|
||||
memory.profile.enabled
|
||||
|| memory.profile.staging_tools
|
||||
|| memory.workspace_settings.is_some(),
|
||||
),
|
||||
(
|
||||
"feature.subjektiv",
|
||||
subjektiv.profile.enabled
|
||||
|| subjektiv.profile.consolidation_tools
|
||||
|| subjektiv.workspace_settings.is_some(),
|
||||
),
|
||||
("feature.web", web.enabled),
|
||||
("feature.image", image.enabled),
|
||||
("feature.sub_worker", sub_worker.enabled),
|
||||
("feature.flow", flow.enabled),
|
||||
("feature.worker", worker.enabled),
|
||||
(
|
||||
"feature.workspace_worker_discovery",
|
||||
workspace_worker_discovery.enabled,
|
||||
),
|
||||
("feature.objective", objective.enabled),
|
||||
("feature.manage_workdir", manage_workdir.enabled),
|
||||
("feature.workspace_config", workspace_config.enabled),
|
||||
(
|
||||
"feature.ticket",
|
||||
ticket.enabled || ticket.authoring || ticket.thread || ticket.intake || ticket.workflow,
|
||||
),
|
||||
(
|
||||
"feature.merge_request",
|
||||
merge_request.show
|
||||
|| merge_request.open
|
||||
|| merge_request.review
|
||||
|| merge_request.readiness_check
|
||||
|| merge_request.complete,
|
||||
),
|
||||
("feature.orchestration", orchestration.enabled),
|
||||
// The default WIP-only catalog is inert in Tools mode, not a tool requirement.
|
||||
(
|
||||
"feature.workdir_catalog",
|
||||
workdir_catalog.enabled && manifest.worker.mode == manifest::WorkerMode::Wip,
|
||||
),
|
||||
(
|
||||
"worker.mode=wip",
|
||||
manifest.worker.mode == manifest::WorkerMode::Wip,
|
||||
),
|
||||
("scope", !manifest.scope.allow.is_empty()),
|
||||
(
|
||||
"delegation_scope",
|
||||
!manifest.delegation_scope.allow.is_empty(),
|
||||
),
|
||||
("mcp", !manifest.mcp.stdio_servers.is_empty()),
|
||||
("skills", manifest.skills.is_some()),
|
||||
("compaction", manifest.compaction.is_some()),
|
||||
(
|
||||
"web",
|
||||
manifest
|
||||
.web
|
||||
.as_ref()
|
||||
.is_some_and(|config| config.enabled == Some(true)),
|
||||
),
|
||||
(
|
||||
"session.record_event_trace",
|
||||
manifest.session.record_event_trace,
|
||||
),
|
||||
] {
|
||||
if enabled {
|
||||
unavailable.push(name);
|
||||
}
|
||||
}
|
||||
if unavailable.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(JobExecutionError::UnsupportedProfile(
|
||||
unavailable.join(", "),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,179 @@
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use agen::tool::{Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
|
||||
use async_trait::async_trait;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::JobExecutionError;
|
||||
use crate::feature::{
|
||||
FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureModule, ToolContribution,
|
||||
ToolDeclaration,
|
||||
};
|
||||
|
||||
pub const SUBMIT_JOB_RESULT_TOOL: &str = "SubmitJobResult";
|
||||
const DESCRIPTION: &str = "Submit this Host Job's bounded structured result. Job, attempt and input revision are immutable Host bindings. Only result is accepted as input; final prose is not success.";
|
||||
|
||||
/// Host acceptance authority. Implementations must validate the immutable fences and commit
|
||||
/// acceptance durably before returning Ok. Err may mean an unknown outcome; identical retries
|
||||
/// must be idempotent. This synchronous callback must be bounded and must not wait for async work.
|
||||
pub trait JobResultSink: Send + Sync {
|
||||
fn submit(&self, submission: ::job::JobResultSubmission) -> Result<(), String>;
|
||||
}
|
||||
|
||||
#[derive(Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SubmitJobResultInput {
|
||||
result: Value,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SubmissionState {
|
||||
sealed: Option<Value>,
|
||||
unconfirmed: Option<String>,
|
||||
}
|
||||
|
||||
/// Single-purpose FeatureModule; Profile names never install it or confer result authority.
|
||||
#[derive(Clone)]
|
||||
pub struct JobResultFeature {
|
||||
job_id: String,
|
||||
attempt_id: String,
|
||||
input_revision: String,
|
||||
max_result_bytes: u32,
|
||||
sink: Arc<dyn JobResultSink>,
|
||||
state: Arc<Mutex<SubmissionState>>,
|
||||
accepted: Arc<Mutex<Option<::job::JobResultSubmission>>>,
|
||||
cancelled: Arc<Mutex<HashSet<String>>>,
|
||||
}
|
||||
|
||||
impl JobResultFeature {
|
||||
pub fn new(
|
||||
request: &::job::JobRequest,
|
||||
attempt_id: String,
|
||||
sink: Arc<dyn JobResultSink>,
|
||||
) -> Result<Self, JobExecutionError> {
|
||||
request.validate()?;
|
||||
::job::validate_identifier("attempt_id", &attempt_id, 512)?;
|
||||
Ok(Self {
|
||||
job_id: request.job_id.clone(),
|
||||
attempt_id,
|
||||
input_revision: request.input_revision.clone(),
|
||||
max_result_bytes: request.limits.max_result_bytes,
|
||||
sink,
|
||||
state: Arc::default(),
|
||||
accepted: Arc::default(),
|
||||
cancelled: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn accepted(&self) -> Arc<Mutex<Option<::job::JobResultSubmission>>> {
|
||||
self.accepted.clone()
|
||||
}
|
||||
|
||||
pub(super) fn unconfirmed(&self) -> Option<String> {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("Job submission state poisoned")
|
||||
.unconfirmed
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn definition(&self) -> ToolDefinition {
|
||||
let feature = self.clone();
|
||||
Arc::new(move || {
|
||||
(
|
||||
ToolMeta::new(SUBMIT_JOB_RESULT_TOOL)
|
||||
.description(DESCRIPTION)
|
||||
.input_schema(
|
||||
serde_json::to_value(schemars::schema_for!(SubmitJobResultInput))
|
||||
.expect("Job result schema serializes"),
|
||||
),
|
||||
Arc::new(feature.clone()) as Arc<dyn Tool>,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for JobResultFeature {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
context: ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let input: SubmitJobResultInput = serde_json::from_str(input_json)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
::job::result_digest(&input.result, self.max_result_bytes)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
let mut state = self.state.lock().expect("Job submission state poisoned");
|
||||
if self
|
||||
.cancelled
|
||||
.lock()
|
||||
.expect("Job cancellation state poisoned")
|
||||
.contains(&context.execution_id())
|
||||
{
|
||||
return Err(ToolError::Cancelled(ToolOutput {
|
||||
summary: "Job result submission cancelled".into(),
|
||||
content: None,
|
||||
attachments: vec![],
|
||||
}));
|
||||
}
|
||||
if let Some(sealed) = &state.sealed {
|
||||
if sealed != &input.result {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"Job result is sealed; retry only the identical result".into(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
// Seal before dispatch: a sink error can follow durable Host acceptance.
|
||||
state.sealed = Some(input.result.clone());
|
||||
}
|
||||
let submission = ::job::JobResultSubmission {
|
||||
job_id: self.job_id.clone(),
|
||||
attempt_id: self.attempt_id.clone(),
|
||||
input_revision: self.input_revision.clone(),
|
||||
result: input.result,
|
||||
};
|
||||
let mut accepted = self.accepted.lock().expect("Job result state poisoned");
|
||||
if accepted.is_none() {
|
||||
if let Err(error) = self.sink.submit(submission.clone()) {
|
||||
let error = ::job::bounded_failure_detail(&error);
|
||||
state.unconfirmed = Some(error.clone());
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Host Job result acceptance was not confirmed; retry only the identical result: {error}"
|
||||
)));
|
||||
}
|
||||
state.unconfirmed = None;
|
||||
*accepted = Some(submission);
|
||||
}
|
||||
Ok(ToolOutput {
|
||||
summary: "Host Job result accepted".into(),
|
||||
content: None,
|
||||
attachments: vec![],
|
||||
})
|
||||
}
|
||||
|
||||
async fn cancel_execution(&self, context: &ToolExecutionContext) -> Result<(), ToolError> {
|
||||
self.cancelled
|
||||
.lock()
|
||||
.expect("Job cancellation state poisoned")
|
||||
.insert(context.execution_id());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FeatureModule for JobResultFeature {
|
||||
fn descriptor(&self) -> FeatureDescriptor {
|
||||
FeatureDescriptor::builtin("host-job-result", "Host Job result")
|
||||
.with_description("Result-only immutable Host Job submission capability.")
|
||||
.with_tool(ToolDeclaration::new(SUBMIT_JOB_RESULT_TOOL, DESCRIPTION))
|
||||
}
|
||||
fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> {
|
||||
context.tools().register(ToolContribution::new(
|
||||
SUBMIT_JOB_RESULT_TOOL,
|
||||
self.definition(),
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,666 @@
|
||||
//! Scripted in-process model executions, not provider/process tests.
|
||||
use super::*;
|
||||
use agen::llm_client::event::{Event, ResponseStatus, StatusEvent};
|
||||
use agen::llm_client::{ClientError, Request};
|
||||
use agen::tool::{Tool, ToolExecutionContext};
|
||||
use async_trait::async_trait;
|
||||
use futures::Stream;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ScriptedClient {
|
||||
scripts: Arc<Mutex<VecDeque<Vec<Event>>>>,
|
||||
requests: Arc<Mutex<Vec<Request>>>,
|
||||
pending: bool,
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
fail: bool,
|
||||
}
|
||||
impl ScriptedClient {
|
||||
fn new(scripts: Vec<Vec<Event>>) -> Self {
|
||||
Self {
|
||||
scripts: Arc::new(Mutex::new(scripts.into())),
|
||||
requests: Arc::default(),
|
||||
pending: false,
|
||||
entered: Arc::default(),
|
||||
fail: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl LlmClient for ScriptedClient {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<Event, ClientError>> + Send>>, ClientError> {
|
||||
self.requests.lock().unwrap().push(request);
|
||||
if self.fail && self.scripts.lock().unwrap().is_empty() {
|
||||
return Err(ClientError::Config("scripted failure".into()));
|
||||
}
|
||||
if self.pending && self.scripts.lock().unwrap().is_empty() {
|
||||
self.entered.notify_one();
|
||||
return Ok(Box::pin(futures::stream::pending()));
|
||||
}
|
||||
let events = self
|
||||
.scripts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.expect("unexpected extra model turn");
|
||||
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct RecordingSink(Mutex<Vec<::job::JobResultSubmission>>);
|
||||
impl JobResultSink for RecordingSink {
|
||||
fn submit(&self, result: ::job::JobResultSubmission) -> Result<(), String> {
|
||||
self.0.lock().unwrap().push(result);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn prose() -> Vec<Event> {
|
||||
vec![
|
||||
Event::text_block_start(0),
|
||||
Event::text_delta(0, "success"),
|
||||
Event::text_block_stop(0, None),
|
||||
Event::Status(StatusEvent {
|
||||
status: ResponseStatus::Completed,
|
||||
}),
|
||||
]
|
||||
}
|
||||
fn call(name: &str, input: Value) -> Vec<Event> {
|
||||
vec![
|
||||
Event::tool_use_start(0, "call", name),
|
||||
Event::tool_input_delta(0, input.to_string()),
|
||||
Event::tool_use_stop(0),
|
||||
Event::Status(StatusEvent {
|
||||
status: ResponseStatus::Completed,
|
||||
}),
|
||||
]
|
||||
}
|
||||
fn request(profile: &str) -> ::job::JobRequest {
|
||||
::job::JobRequest {
|
||||
job_id: "job-a".into(),
|
||||
purpose: "scripted".into(),
|
||||
input_revision: "rev-1".into(),
|
||||
input_ref: "snapshot:one".into(),
|
||||
input: json!({"text":"input"}),
|
||||
instruction: "Summarize the input".into(),
|
||||
profile: profile.into(),
|
||||
serialization_key: None,
|
||||
limits: Default::default(),
|
||||
}
|
||||
}
|
||||
fn artifact(instruction: &str, tokens: u32, turns: u32) -> Value {
|
||||
json!({"model":{"scheme":"anthropic", "model_id":"scripted-model", "auth":{"kind":"none"}},
|
||||
"engine":{"instruction":instruction, "language":"English", "max_tokens":tokens, "max_turns":turns}})
|
||||
}
|
||||
fn registry(root: &Path, a: Value, b: Value) -> manifest::ProfileRegistry {
|
||||
std::fs::write(root.join("a.json"), a.to_string()).unwrap();
|
||||
std::fs::write(root.join("b.json"), b.to_string()).unwrap();
|
||||
let settings = root.join("profiles.toml");
|
||||
std::fs::write(
|
||||
&settings,
|
||||
"default = 'user:a'\n[profile]\na = 'a.json'\nb = 'b.json'\n",
|
||||
)
|
||||
.unwrap();
|
||||
ProfileDiscovery::with_sources(Some(settings), None)
|
||||
.discover()
|
||||
.unwrap()
|
||||
}
|
||||
fn prepared(
|
||||
root: &Path,
|
||||
request: &::job::JobRequest,
|
||||
client: ScriptedClient,
|
||||
) -> PreparedInternalJob {
|
||||
let registry = registry(
|
||||
root,
|
||||
artifact("default", 512, 3),
|
||||
artifact("internal.flow_verifier_system", 100000, 1000),
|
||||
);
|
||||
prepare_job_from_registry(request, root, Some(Box::new(client)), ®istry).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn executes_selected_profiles_with_only_result_authority_and_bounded_policy() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = registry(
|
||||
root.path(),
|
||||
artifact("default", 512, 3),
|
||||
artifact("internal.flow_verifier_system", 100000, 1000),
|
||||
);
|
||||
let contract = PromptCatalog::builtins_only()
|
||||
.unwrap()
|
||||
.render(WorkerPrompt::JobSystem, minijinja::context! {})
|
||||
.unwrap();
|
||||
let mut prompts = vec![];
|
||||
for (profile, tokens) in [
|
||||
("user:a", 512),
|
||||
("user:b", MAX_JOB_TOKENS),
|
||||
("default", 512),
|
||||
] {
|
||||
let client = ScriptedClient::new(vec![
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{"ok":true}})),
|
||||
prose(),
|
||||
]);
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let job = prepare_job_from_registry(
|
||||
&request(profile),
|
||||
root.path(),
|
||||
Some(Box::new(client.clone())),
|
||||
®istry,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
job.manifest.engine.max_turns.unwrap().get(),
|
||||
if profile == "user:b" {
|
||||
MAX_JOB_TURNS
|
||||
} else {
|
||||
3
|
||||
}
|
||||
);
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let result = job
|
||||
.run("attempt-7".into(), sink.clone(), move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
rx.await.unwrap().is_closed(),
|
||||
"Worker must be released before returning acceptance"
|
||||
);
|
||||
assert_eq!(
|
||||
result.submission,
|
||||
::job::JobResultSubmission {
|
||||
job_id: "job-a".into(),
|
||||
attempt_id: "attempt-7".into(),
|
||||
input_revision: "rev-1".into(),
|
||||
result: json!({"ok":true})
|
||||
}
|
||||
);
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 1);
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
for request in requests.iter() {
|
||||
assert_eq!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| tool.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[SUBMIT_JOB_RESULT_TOOL]
|
||||
);
|
||||
assert_eq!(request.config.max_tokens, Some(tokens));
|
||||
assert_eq!(
|
||||
request.tools[0].input_schema["properties"]
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.keys()
|
||||
.collect::<Vec<_>>(),
|
||||
["result"]
|
||||
);
|
||||
assert!(request.system_prompt.as_ref().unwrap().ends_with(&contract));
|
||||
}
|
||||
prompts.push(requests[0].system_prompt.clone());
|
||||
}
|
||||
assert_ne!(prompts[0], prompts[1]);
|
||||
assert_eq!(prompts[0], prompts[2]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn selected_builtin_job_executes_with_catalog_contract_and_only_result_tool() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = ProfileDiscovery::with_sources(None, None)
|
||||
.discover()
|
||||
.unwrap();
|
||||
let request = request("builtin:job");
|
||||
let client = ScriptedClient::new(vec![
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{"ok":true}})),
|
||||
prose(),
|
||||
]);
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let job = prepare_job_from_registry(
|
||||
&request,
|
||||
root.path(),
|
||||
Some(Box::new(client.clone())),
|
||||
®istry,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
job.manifest.engine.instruction,
|
||||
WorkerPrompt::JobSystem.key()
|
||||
);
|
||||
assert_eq!(
|
||||
job.manifest.model.ref_.as_deref(),
|
||||
Some("codex-oauth/gpt-5.6-luna")
|
||||
);
|
||||
assert_eq!(job.manifest.engine.max_turns.unwrap().get(), MAX_JOB_TURNS);
|
||||
assert!(job.manifest.scope.allow.is_empty());
|
||||
assert!(job.manifest.delegation_scope.allow.is_empty());
|
||||
assert!(WorkerPrompt::ALL.contains(&WorkerPrompt::JobSystem));
|
||||
let contract = PromptCatalog::builtins_only()
|
||||
.unwrap()
|
||||
.render(WorkerPrompt::JobSystem, minijinja::context! {})
|
||||
.unwrap();
|
||||
assert!(job.system_prompt.ends_with(&contract));
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let result = job
|
||||
.run("builtin-attempt".into(), sink.clone(), move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(rx.await.unwrap().is_closed());
|
||||
assert_eq!(result.submission.job_id, request.job_id);
|
||||
assert_eq!(result.submission.attempt_id, "builtin-attempt");
|
||||
assert_eq!(result.submission.input_revision, request.input_revision);
|
||||
assert_eq!(result.submission.result, json!({"ok":true}));
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 1);
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
for request in requests.iter() {
|
||||
assert_eq!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| tool.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[SUBMIT_JOB_RESULT_TOOL]
|
||||
);
|
||||
assert_eq!(request.config.max_tokens, Some(MAX_JOB_TOKENS));
|
||||
assert!(request.system_prompt.as_ref().unwrap().ends_with(&contract));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn final_prose_and_ungranted_tool_calls_are_not_success() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for scripts in [
|
||||
vec![prose()],
|
||||
vec![
|
||||
call("Bash", json!({"command":"touch should-not-exist"})),
|
||||
prose(),
|
||||
],
|
||||
vec![
|
||||
call("SubmitBackendJobResult", json!({"result":{"ok":true}})),
|
||||
prose(),
|
||||
],
|
||||
vec![
|
||||
call(
|
||||
SUBMIT_JOB_RESULT_TOOL,
|
||||
json!({"job_id":"forged","result":{"ok":true}}),
|
||||
),
|
||||
prose(),
|
||||
],
|
||||
] {
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let outcome = prepared(
|
||||
root.path(),
|
||||
&request("user:a"),
|
||||
ScriptedClient::new(scripts),
|
||||
)
|
||||
.run("attempt".into(), sink.clone(), |_| {})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(outcome, Err(JobExecutionError::MissingResult)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert!(sink.0.lock().unwrap().is_empty());
|
||||
assert!(!root.path().join("should-not-exist").exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn result_size_and_identity_are_fenced_before_host_submission() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut request = request("user:a");
|
||||
request.limits.max_result_bytes = 5;
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let job = prepared(
|
||||
root.path(),
|
||||
&request,
|
||||
ScriptedClient::new(vec![
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":"too big"})),
|
||||
prose(),
|
||||
]),
|
||||
);
|
||||
assert!(matches!(
|
||||
job.run("attempt".into(), sink.clone(), |_| {}).await,
|
||||
Err(JobExecutionError::MissingResult)
|
||||
));
|
||||
assert!(sink.0.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_and_errors_return_without_acceptance() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut client = ScriptedClient::new(vec![]);
|
||||
client.pending = true;
|
||||
let entered = client.entered.clone();
|
||||
let job = prepared(root.path(), &request("user:a"), client);
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let run = tokio::spawn(async move {
|
||||
job.run("attempt".into(), sink, move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
})
|
||||
.await
|
||||
});
|
||||
let sender = rx.await.unwrap();
|
||||
entered.notified().await;
|
||||
sender.send(()).await.unwrap();
|
||||
assert!(matches!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), run)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Err(JobExecutionError::Cancelled)
|
||||
));
|
||||
assert!(
|
||||
sender.is_closed(),
|
||||
"normal Worker release must close cancellation receiver"
|
||||
);
|
||||
let mut client = ScriptedClient::new(vec![]);
|
||||
client.fail = true;
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let outcome = prepared(root.path(), &request("user:a"), client)
|
||||
.run(
|
||||
"attempt".into(),
|
||||
Arc::new(RecordingSink::default()),
|
||||
move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, Err(JobExecutionError::Worker(_))));
|
||||
assert!(
|
||||
rx.await.unwrap().is_closed(),
|
||||
"error must release the Worker"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_result_survives_later_provider_error_and_cancellation() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for cancel in [false, true] {
|
||||
let mut client =
|
||||
ScriptedClient::new(vec![call(SUBMIT_JOB_RESULT_TOOL, json!({"result":true}))]);
|
||||
client.pending = cancel;
|
||||
client.fail = !cancel;
|
||||
let entered = client.entered.clone();
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let job = prepared(root.path(), &request("user:a"), client);
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let recording = sink.clone();
|
||||
let run = tokio::spawn(async move {
|
||||
job.run("attempt".into(), recording, move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
})
|
||||
.await
|
||||
});
|
||||
let sender = rx.await.unwrap();
|
||||
if cancel {
|
||||
entered.notified().await;
|
||||
assert_eq!(
|
||||
sink.0.lock().unwrap().len(),
|
||||
1,
|
||||
"acceptance must precede cancellation"
|
||||
);
|
||||
sender.send(()).await.unwrap();
|
||||
}
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(2), run)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(result.submission.result, json!(true));
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 1);
|
||||
assert!(sender.is_closed());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_limit_is_bounded_and_accepted_result_survives_later_failure() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = registry(
|
||||
root.path(),
|
||||
artifact("default", 512, 1),
|
||||
artifact("default", 512, 1),
|
||||
);
|
||||
for (tool, expected) in [("UnknownTool", false), (SUBMIT_JOB_RESULT_TOOL, true)] {
|
||||
let client = ScriptedClient::new(vec![call(tool, json!({"result":true}))]);
|
||||
let prepared = prepare_job_from_registry(
|
||||
&request("user:a"),
|
||||
root.path(),
|
||||
Some(Box::new(client.clone())),
|
||||
®istry,
|
||||
)
|
||||
.unwrap();
|
||||
let outcome = prepared
|
||||
.run("attempt".into(), Arc::new(RecordingSink::default()), |_| {})
|
||||
.await;
|
||||
assert_eq!(outcome.is_ok(), expected);
|
||||
assert_eq!(client.requests.lock().unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profiles_with_unavailable_requirements_are_rejected_without_name_allowlist_or_fallback() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let base = artifact("default", 512, 3);
|
||||
for (key, value, needle) in [
|
||||
("feature", json!({"task":{"enabled":true}}), "feature.task"),
|
||||
(
|
||||
"feature",
|
||||
json!({"workspace_config":{"enabled":true}}),
|
||||
"feature.workspace_config",
|
||||
),
|
||||
(
|
||||
"feature",
|
||||
json!({"subjektiv":{"consolidation_tools":true}}),
|
||||
"feature.subjektiv",
|
||||
),
|
||||
(
|
||||
"feature",
|
||||
json!({"memory":{"staging_tools":true}}),
|
||||
"feature.memory",
|
||||
),
|
||||
("scope", json!("workspace_read"), "scope"),
|
||||
(
|
||||
"delegation_scope",
|
||||
json!("workspace_write"),
|
||||
"delegation_scope",
|
||||
),
|
||||
(
|
||||
"session",
|
||||
json!({"record_event_trace":true}),
|
||||
"session.record_event_trace",
|
||||
),
|
||||
("web", json!({"enabled":true}), "web"),
|
||||
(
|
||||
"compaction",
|
||||
json!({"kind":"tokens","threshold":10000}),
|
||||
"compaction",
|
||||
),
|
||||
("worker", json!({"mode":"wip"}), "worker.mode=wip"),
|
||||
("skills", json!({"directories":["skills"]}), "skills"),
|
||||
(
|
||||
"mcp",
|
||||
json!({"stdio_server":[{"name":"test","command":"fake"}]}),
|
||||
"mcp",
|
||||
),
|
||||
] {
|
||||
let mut profile = base.clone();
|
||||
profile[key] = value;
|
||||
let registry = registry(root.path(), profile, base.clone());
|
||||
let error = prepare_job_from_registry(
|
||||
&request("user:a"),
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry,
|
||||
)
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(error, JobExecutionError::UnsupportedProfile(_)),
|
||||
"{error}"
|
||||
);
|
||||
assert!(error.to_string().contains(needle), "{error}");
|
||||
}
|
||||
for name in [
|
||||
"web",
|
||||
"image",
|
||||
"sub_worker",
|
||||
"flow",
|
||||
"worker",
|
||||
"workspace_worker_discovery",
|
||||
"objective",
|
||||
"manage_workdir",
|
||||
"ticket",
|
||||
"orchestration",
|
||||
] {
|
||||
let mut profile = base.clone();
|
||||
profile["feature"] = json!({name:{"enabled":true}});
|
||||
let registry = registry(root.path(), profile, base.clone());
|
||||
assert!(matches!(
|
||||
prepare_job_from_registry(
|
||||
&request("user:a"),
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry
|
||||
),
|
||||
Err(JobExecutionError::UnsupportedProfile(_))
|
||||
));
|
||||
}
|
||||
let registry = registry(root.path(), base.clone(), base);
|
||||
for selector in ["user:missing", "project:a", "./a.json", "inherit"] {
|
||||
assert!(
|
||||
prepare_job_from_registry(
|
||||
&request(selector),
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let builtin = request("builtin:backend-job");
|
||||
assert!(matches!(
|
||||
prepare_job_from_registry(
|
||||
&builtin,
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry
|
||||
),
|
||||
Err(JobExecutionError::UnsupportedProfile(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_prompt_and_model_are_not_substituted_even_with_injected_client() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut unknown_model = artifact("default", 512, 3);
|
||||
unknown_model["model"] = json!({"ref":"not-a-provider/not-a-model"});
|
||||
let registry = registry(
|
||||
root.path(),
|
||||
artifact("internal.missing", 512, 3),
|
||||
unknown_model,
|
||||
);
|
||||
for (profile, prompt_error) in [("user:a", true), ("user:b", false)] {
|
||||
let error = prepare_job_from_registry(
|
||||
&request(profile),
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry,
|
||||
)
|
||||
.err()
|
||||
.unwrap();
|
||||
if prompt_error {
|
||||
assert!(
|
||||
matches!(error, JobExecutionError::Instruction(_)),
|
||||
"{error}"
|
||||
);
|
||||
} else {
|
||||
assert!(matches!(error, JobExecutionError::Model(_)), "{error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct UnknownSink(Mutex<Vec<::job::JobResultSubmission>>);
|
||||
impl JobResultSink for UnknownSink {
|
||||
fn submit(&self, value: ::job::JobResultSubmission) -> Result<(), String> {
|
||||
self.0.lock().unwrap().push(value);
|
||||
Err("unknown".into())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unconfirmed_sink_acceptance_is_not_classified_as_missing_result_or_known_failure() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let sink = Arc::new(UnknownSink(Mutex::new(vec![])));
|
||||
let job = prepared(
|
||||
root.path(),
|
||||
&request("user:a"),
|
||||
ScriptedClient::new(vec![
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":true})),
|
||||
prose(),
|
||||
]),
|
||||
);
|
||||
assert!(matches!(
|
||||
job.run("attempt".into(), sink.clone(), |_| {}).await,
|
||||
Err(JobExecutionError::ResultUnconfirmed(_))
|
||||
));
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feature_registration_rejects_forged_fences_seals_unknown_outcomes_and_honors_cancel() {
|
||||
let sink = Arc::new(UnknownSink(Mutex::new(vec![])));
|
||||
let feature =
|
||||
JobResultFeature::new(&request("user:a"), "attempt".into(), sink.clone()).unwrap();
|
||||
let mut pending = vec![];
|
||||
let mut hooks = crate::HookRegistryBuilder::new();
|
||||
let report = FeatureRegistryBuilder::new()
|
||||
.with_module(feature.clone())
|
||||
.install_into_pending(&mut pending, &mut hooks);
|
||||
assert_eq!(report.installed_tool_names(), [SUBMIT_JOB_RESULT_TOOL]);
|
||||
let (_, tool) = pending.remove(0)();
|
||||
assert!(
|
||||
tool.execute(
|
||||
r#"{"attempt_id":"forged","result":true}"#,
|
||||
ToolExecutionContext::direct()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(sink.0.lock().unwrap().is_empty());
|
||||
assert!(
|
||||
tool.execute(r#"{"result":true}"#, ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
tool.execute(r#"{"result":false}"#, ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
tool.execute(r#"{"result":true}"#, ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 2);
|
||||
let context = ToolExecutionContext::new("cancelled", "batch", 0);
|
||||
feature.cancel_execution(&context).await.unwrap();
|
||||
assert!(matches!(
|
||||
tool.execute(r#"{"result":true}"#, context).await,
|
||||
Err(agen::tool::ToolError::Cancelled(_))
|
||||
));
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 2);
|
||||
}
|
||||
@@ -12,6 +12,7 @@ pub mod fs_view;
|
||||
pub mod hook;
|
||||
pub(crate) mod in_flight;
|
||||
pub mod ipc;
|
||||
pub mod job;
|
||||
pub mod model_client;
|
||||
mod paste_artifact_tool;
|
||||
pub mod prompt;
|
||||
|
||||
@@ -97,6 +97,7 @@ pub enum WorkerPrompt {
|
||||
SubjektivMemoryConsolidationSystem,
|
||||
SubjektivMemorySurfaceSystem,
|
||||
FlowVerifierSystem,
|
||||
JobSystem,
|
||||
BackendJobSystem,
|
||||
NotifyWrapper,
|
||||
InterruptToolResultSummary,
|
||||
@@ -121,6 +122,7 @@ impl WorkerPrompt {
|
||||
}
|
||||
Self::SubjektivMemorySurfaceSystem => "internal.subjektiv_memory_surface_system",
|
||||
Self::FlowVerifierSystem => "internal.flow_verifier_system",
|
||||
Self::JobSystem => "internal.job_system",
|
||||
Self::BackendJobSystem => "internal.backend_job_system",
|
||||
Self::NotifyWrapper => "internal.notify_wrapper",
|
||||
Self::InterruptToolResultSummary => "internal.interrupt_tool_result_summary",
|
||||
@@ -144,6 +146,7 @@ impl WorkerPrompt {
|
||||
WorkerPrompt::SubjektivMemoryConsolidationSystem,
|
||||
WorkerPrompt::SubjektivMemorySurfaceSystem,
|
||||
WorkerPrompt::FlowVerifierSystem,
|
||||
WorkerPrompt::JobSystem,
|
||||
WorkerPrompt::BackendJobSystem,
|
||||
WorkerPrompt::NotifyWrapper,
|
||||
WorkerPrompt::InterruptToolResultSummary,
|
||||
|
||||
@@ -18,6 +18,7 @@ futures.workspace = true
|
||||
config-source.workspace = true
|
||||
flow = { path = "../flow" }
|
||||
manifest.workspace = true
|
||||
job.workspace = true
|
||||
protocol = { workspace = true }
|
||||
runtime-api.workspace = true
|
||||
server-api.workspace = true
|
||||
|
||||
@@ -7,26 +7,28 @@
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
use worker_runtime::identity::RuntimeWorkerRef;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
pub const BACKEND_JOB_RUNTIME_ID: &str = crate::hosts::EMBEDDED_RUNTIME_ID;
|
||||
pub const DEFAULT_MAX_CONCURRENT_JOBS: u16 = 8;
|
||||
pub const ABSOLUTE_MAX_CONCURRENT_JOBS: u16 = 64;
|
||||
pub const DEFAULT_JOB_TIMEOUT_SECONDS: u32 = 120;
|
||||
pub const MAX_JOB_TIMEOUT_SECONDS: u32 = 600;
|
||||
pub const DEFAULT_MAX_RESULT_BYTES: u32 = 16 * 1024;
|
||||
pub const ABSOLUTE_MAX_RESULT_BYTES: u32 = 64 * 1024;
|
||||
pub const DEFAULT_MAX_ATTEMPTS: u8 = 2;
|
||||
pub const ABSOLUTE_MAX_ATTEMPTS: u8 = 3;
|
||||
pub const MAX_JOB_INPUT_BYTES: usize = 128 * 1024;
|
||||
pub const MAX_JOB_INSTRUCTION_BYTES: usize = 16 * 1024;
|
||||
pub const MAX_JOB_PURPOSE_BYTES: usize = 120;
|
||||
pub const MAX_JOB_REFERENCE_BYTES: usize = 512;
|
||||
pub const MAX_JOB_FAILURE_BYTES: usize = 1024;
|
||||
pub const MAX_JOB_DELIVERY_BYTES: usize = 16 * 1024;
|
||||
pub use job::{
|
||||
ABSOLUTE_MAX_ATTEMPTS, ABSOLUTE_MAX_CONCURRENT_JOBS, ABSOLUTE_MAX_RESULT_BYTES,
|
||||
DEFAULT_JOB_TIMEOUT_SECONDS, DEFAULT_MAX_ATTEMPTS, DEFAULT_MAX_CONCURRENT_JOBS,
|
||||
DEFAULT_MAX_RESULT_BYTES, JobResultSubmission as BackendJobResultSubmission,
|
||||
MAX_JOB_DELIVERY_BYTES, MAX_JOB_FAILURE_BYTES, MAX_JOB_INPUT_BYTES, MAX_JOB_INSTRUCTION_BYTES,
|
||||
MAX_JOB_PURPOSE_BYTES, MAX_JOB_REFERENCE_BYTES, MAX_JOB_TIMEOUT_SECONDS, attempt_id,
|
||||
bounded_failure_detail,
|
||||
};
|
||||
|
||||
// Preserve the Backend InvalidInput surface and its exact historical messages.
|
||||
fn backend_error(error: job::JobError) -> Error {
|
||||
match error {
|
||||
job::JobError::InvalidInput(detail) => {
|
||||
Error::InvalidInput(detail.replacen("Job ", "Backend Job ", 1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -37,6 +39,17 @@ pub enum BackendJobState {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl From<BackendJobState> for job::JobState {
|
||||
fn from(state: BackendJobState) -> Self {
|
||||
match state {
|
||||
BackendJobState::Pending => Self::Pending,
|
||||
BackendJobState::Completed => Self::Completed,
|
||||
BackendJobState::Failed => Self::Failed,
|
||||
BackendJobState::Unknown => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendJobState {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
@@ -69,6 +82,19 @@ pub enum BackendJobAttemptState {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl From<BackendJobAttemptState> for job::JobAttemptState {
|
||||
fn from(state: BackendJobAttemptState) -> Self {
|
||||
match state {
|
||||
BackendJobAttemptState::Reserved => Self::Reserved,
|
||||
BackendJobAttemptState::Dispatching => Self::Dispatching,
|
||||
BackendJobAttemptState::Dispatched => Self::Dispatched,
|
||||
BackendJobAttemptState::Completed => Self::Completed,
|
||||
BackendJobAttemptState::Failed => Self::Failed,
|
||||
BackendJobAttemptState::Unknown => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendJobAttemptState {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
@@ -161,28 +187,29 @@ impl Default for BackendJobLimits {
|
||||
|
||||
impl BackendJobLimits {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.max_concurrent_jobs == 0 || self.max_concurrent_jobs > ABSOLUTE_MAX_CONCURRENT_JOBS
|
||||
{
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_concurrent_jobs must be in 1..={ABSOLUTE_MAX_CONCURRENT_JOBS}"
|
||||
)));
|
||||
job::JobLimits::from(self).validate().map_err(backend_error)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&BackendJobLimits> for job::JobLimits {
|
||||
fn from(limits: &BackendJobLimits) -> Self {
|
||||
Self {
|
||||
max_concurrent_jobs: limits.max_concurrent_jobs,
|
||||
timeout_seconds: limits.timeout_seconds,
|
||||
max_result_bytes: limits.max_result_bytes,
|
||||
max_attempts: limits.max_attempts,
|
||||
}
|
||||
if self.timeout_seconds == 0 || self.timeout_seconds > MAX_JOB_TIMEOUT_SECONDS {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job timeout_seconds must be in 1..={MAX_JOB_TIMEOUT_SECONDS}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
impl From<job::JobLimits> for BackendJobLimits {
|
||||
fn from(limits: job::JobLimits) -> Self {
|
||||
Self {
|
||||
max_concurrent_jobs: limits.max_concurrent_jobs,
|
||||
timeout_seconds: limits.timeout_seconds,
|
||||
max_result_bytes: limits.max_result_bytes,
|
||||
max_attempts: limits.max_attempts,
|
||||
}
|
||||
if self.max_result_bytes == 0 || self.max_result_bytes > ABSOLUTE_MAX_RESULT_BYTES {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_result_bytes must be in 1..={ABSOLUTE_MAX_RESULT_BYTES}"
|
||||
)));
|
||||
}
|
||||
if self.max_attempts == 0 || self.max_attempts > ABSOLUTE_MAX_ATTEMPTS {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_attempts must be in 1..={ABSOLUTE_MAX_ATTEMPTS}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -261,17 +288,29 @@ pub struct BackendJobRequest {
|
||||
}
|
||||
|
||||
impl BackendJobRequest {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
validate_identifier("job_id", &self.job_id, 256)?;
|
||||
validate_identifier("purpose", &self.purpose, MAX_JOB_PURPOSE_BYTES)?;
|
||||
validate_identifier("input_revision", &self.input_revision, 256)?;
|
||||
validate_bounded_text("input_ref", &self.input_ref, MAX_JOB_REFERENCE_BYTES)?;
|
||||
validate_bounded_text("instruction", &self.instruction, MAX_JOB_INSTRUCTION_BYTES)?;
|
||||
validate_identifier("profile", &self.profile, 256)?;
|
||||
self.profile_selector()?;
|
||||
if let Some(key) = &self.serialization_key {
|
||||
validate_identifier("serialization_key", key, MAX_JOB_REFERENCE_BYTES)?;
|
||||
/// Adapt neutral intent without granting domain access or writer/notification
|
||||
/// authority. Validation remains the Backend submission boundary's job;
|
||||
/// the caller-selected profile is retained exactly, with no fallback.
|
||||
pub fn from_job_request(request: job::JobRequest) -> Self {
|
||||
Self {
|
||||
job_id: request.job_id,
|
||||
purpose: request.purpose,
|
||||
input_revision: request.input_revision,
|
||||
input_ref: request.input_ref,
|
||||
input: request.input,
|
||||
instruction: request.instruction,
|
||||
profile: request.profile,
|
||||
grants: BackendJobGrants::default(),
|
||||
serialization_key: request.serialization_key,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: request.limits.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
let request = self.to_job_request();
|
||||
request.validate_fields().map_err(backend_error)?;
|
||||
if let Some(grant) = &self.grants.subjektiv_consolidation {
|
||||
validate_identifier("grant subject_id", &grant.subject_id, 256)?;
|
||||
if grant.candidate_ids.len() > 256 {
|
||||
@@ -297,44 +336,30 @@ impl BackendJobRequest {
|
||||
));
|
||||
}
|
||||
}
|
||||
let input = serde_json::to_vec(&self.input).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job input: {error}"))
|
||||
})?;
|
||||
if input.len() > MAX_JOB_INPUT_BYTES {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job input exceeds {MAX_JOB_INPUT_BYTES} bytes"
|
||||
)));
|
||||
}
|
||||
request.validate_input().map_err(backend_error)?;
|
||||
self.limits.validate()
|
||||
}
|
||||
|
||||
/// Transport-neutral projection only. Domain grants, writer authority and
|
||||
/// derived resource locks remain Backend-owned; never fingerprint this
|
||||
/// projection as a substitute for the full Backend intent.
|
||||
pub fn to_job_request(&self) -> job::JobRequest {
|
||||
job::JobRequest {
|
||||
job_id: self.job_id.clone(),
|
||||
purpose: self.purpose.clone(),
|
||||
input_revision: self.input_revision.clone(),
|
||||
input_ref: self.input_ref.clone(),
|
||||
input: self.input.clone(),
|
||||
instruction: self.instruction.clone(),
|
||||
profile: self.profile.clone(),
|
||||
serialization_key: self.serialization_key.clone(),
|
||||
limits: (&self.limits).into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate selector syntax only. Registry existence and resolution are dispatch authority.
|
||||
pub fn profile_selector(&self) -> Result<manifest::ProfileSelector> {
|
||||
use manifest::ProfileSelector;
|
||||
validate_identifier("profile", &self.profile, 256)?;
|
||||
let selector = ProfileSelector::parse_cli(&self.profile);
|
||||
let valid = match &selector {
|
||||
ProfileSelector::Default => true,
|
||||
ProfileSelector::Path { .. } => false,
|
||||
ProfileSelector::Named { source, name } => {
|
||||
!name.is_empty()
|
||||
&& !name.starts_with('.')
|
||||
&& !name.contains(['/', '\\', ':'])
|
||||
&& name
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.'))
|
||||
&& ![".toml", ".json", ".dcdl", ".nix"]
|
||||
.iter()
|
||||
.any(|suffix| name.ends_with(suffix))
|
||||
&& (source.is_some() || name != "inherit")
|
||||
}
|
||||
};
|
||||
if !valid {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Job profile must be a manifest registry selector, not raw source or a path".into(),
|
||||
));
|
||||
}
|
||||
Ok(selector)
|
||||
job::profile_selector(&self.profile).map_err(backend_error)
|
||||
}
|
||||
|
||||
/// Effective single-flight key. Typed grants cannot bypass their subject fence.
|
||||
@@ -348,20 +373,12 @@ impl BackendJobRequest {
|
||||
|
||||
pub fn fingerprint(&self) -> Result<String> {
|
||||
self.validate()?;
|
||||
let encoded = serde_json::to_vec(self).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job intent: {error}"))
|
||||
})?;
|
||||
Ok(sha256(&encoded))
|
||||
job::fingerprint(self).map_err(backend_error)
|
||||
}
|
||||
|
||||
pub fn worker_input(&self, attempt_id: &str) -> Result<String> {
|
||||
let input = serde_json::to_string(&self.input).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job input: {error}"))
|
||||
})?;
|
||||
Ok(format!(
|
||||
"{}\n\nBackend Job envelope (immutable):\njob_id: {}\nattempt_id: {}\ninput_revision: {}\ninput_ref: {}\ninput_json: {}\n\nReturn success only through the structured Backend Job result capability. Final prose and Worker Idle/Stopped state are not result authority.",
|
||||
self.instruction, self.job_id, attempt_id, self.input_revision, self.input_ref, input
|
||||
))
|
||||
job::worker_input_with_label(&self.to_job_request(), attempt_id, "Backend Job")
|
||||
.map_err(backend_error)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -451,13 +468,10 @@ pub struct BackendJobReservation {
|
||||
pub resource_reused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobResultSubmission {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub input_revision: String,
|
||||
pub result: Value,
|
||||
impl BackendJobReservation {
|
||||
pub fn outcome(&self) -> job::JobOutcome {
|
||||
job_outcome(&self.job, &self.attempt)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
@@ -467,6 +481,28 @@ pub struct BackendJobResultAcceptance {
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
impl BackendJobResultAcceptance {
|
||||
pub fn outcome(&self) -> job::JobOutcome {
|
||||
job_outcome(&self.job, &self.attempt)
|
||||
}
|
||||
}
|
||||
|
||||
// Project the durable Job outcome, not Worker lifecycle, cleanup or delivery.
|
||||
// The paired attempt supplies its own identity, number and execution state.
|
||||
fn job_outcome(job: &BackendJobRecord, attempt: &BackendJobAttemptRecord) -> job::JobOutcome {
|
||||
job::JobOutcome {
|
||||
job_id: job.request.job_id.clone(),
|
||||
input_revision: job.request.input_revision.clone(),
|
||||
attempt_id: attempt.attempt_id.clone(),
|
||||
attempt: attempt.attempt,
|
||||
state: job.state.into(),
|
||||
attempt_state: attempt.state.into(),
|
||||
result: job.result.clone(),
|
||||
failure_category: job.failure_category.clone(),
|
||||
failure_detail: job.failure_detail.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BackendJobWorkerBinding {
|
||||
pub job_id: String,
|
||||
@@ -489,10 +525,6 @@ pub struct BackendJobDeliveryRecord {
|
||||
pub delivered_at: Option<String>,
|
||||
}
|
||||
|
||||
pub fn attempt_id(job_id: &str, attempt: u8) -> String {
|
||||
format!("{job_id}:attempt:{attempt}")
|
||||
}
|
||||
|
||||
pub fn allocation_key(attempt_id: &str) -> String {
|
||||
format!("backend-job:{attempt_id}")
|
||||
}
|
||||
@@ -515,64 +547,15 @@ pub fn delivery_id(job_id: &str, attempt_id: &str) -> String {
|
||||
}
|
||||
|
||||
fn tracked_request_id(kind: &str, identity: &[u8]) -> String {
|
||||
format!("backend-job-{kind}:{}", sha256(identity))
|
||||
format!("backend-job-{kind}:{}", job::sha256(identity))
|
||||
}
|
||||
|
||||
pub fn result_digest(result: &Value, max_bytes: u32) -> Result<(String, String)> {
|
||||
let encoded = serde_json::to_string(result)
|
||||
.map_err(|error| Error::InvalidInput(format!("serialize Backend Job result: {error}")))?;
|
||||
if encoded.len() > max_bytes as usize {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job result exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok((sha256(encoded.as_bytes()), encoded))
|
||||
}
|
||||
|
||||
pub(crate) fn bounded_failure_detail(detail: &str) -> String {
|
||||
if detail.len() <= MAX_JOB_FAILURE_BYTES {
|
||||
return detail.to_string();
|
||||
}
|
||||
let mut end = MAX_JOB_FAILURE_BYTES;
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
detail[..end].to_string()
|
||||
job::result_digest(result, max_bytes).map_err(backend_error)
|
||||
}
|
||||
|
||||
fn validate_identifier(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
validate_bounded_text(field, value, max_bytes)?;
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} must not contain control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bounded_text(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
if value.is_empty() || value.trim() != value {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} must be non-empty and trimmed"
|
||||
)));
|
||||
}
|
||||
if value.len() > max_bytes {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sha256(bytes: &[u8]) -> String {
|
||||
let digest = Sha256::digest(bytes);
|
||||
let mut output = String::with_capacity(7 + digest.len() * 2);
|
||||
output.push_str("sha256:");
|
||||
for byte in digest {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(output, "{byte:02x}");
|
||||
}
|
||||
output
|
||||
job::validate_identifier(field, value, max_bytes).map_err(backend_error)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -596,6 +579,258 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn reservation() -> BackendJobReservation {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"job": {
|
||||
"workspace_id": "workspace-a",
|
||||
"request": request(),
|
||||
"intent_fingerprint": request().fingerprint().unwrap(),
|
||||
"state": "pending",
|
||||
"current_attempt": 2,
|
||||
"created_at": "created",
|
||||
"updated_at": "updated"
|
||||
},
|
||||
"attempt": {
|
||||
"workspace_id": "workspace-a",
|
||||
"job_id": "ticket-check:T-1:r7",
|
||||
"attempt_id": "ticket-check:T-1:r7:attempt:2",
|
||||
"attempt": 2,
|
||||
"input_revision": "7",
|
||||
"state": "reserved",
|
||||
"worker": {"runtime_id": "runtime-a", "worker_id": "worker-a"},
|
||||
"runtime_run_id": "run-a",
|
||||
"deadline_at": "deadline",
|
||||
"created_at": "created",
|
||||
"updated_at": "updated"
|
||||
},
|
||||
"replayed": false,
|
||||
"resource_reused": false
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backend_state_projections_preserve_existing_wire_vocabulary() {
|
||||
for state in [
|
||||
BackendJobState::Pending,
|
||||
BackendJobState::Completed,
|
||||
BackendJobState::Failed,
|
||||
BackendJobState::Unknown,
|
||||
] {
|
||||
let shared = job::JobState::from(state);
|
||||
assert_eq!(
|
||||
serde_json::to_value(state).unwrap(),
|
||||
serde_json::to_value(shared).unwrap()
|
||||
);
|
||||
}
|
||||
for state in [
|
||||
BackendJobAttemptState::Reserved,
|
||||
BackendJobAttemptState::Dispatching,
|
||||
BackendJobAttemptState::Dispatched,
|
||||
BackendJobAttemptState::Completed,
|
||||
BackendJobAttemptState::Failed,
|
||||
BackendJobAttemptState::Unknown,
|
||||
] {
|
||||
let shared = job::JobAttemptState::from(state);
|
||||
assert_eq!(
|
||||
serde_json::to_value(state).unwrap(),
|
||||
serde_json::to_value(shared).unwrap()
|
||||
);
|
||||
}
|
||||
// The shared read vocabulary does not add Backend cancellation support.
|
||||
assert!(BackendJobState::parse("cancelled").is_err());
|
||||
assert!(BackendJobAttemptState::parse("cancelled").is_err());
|
||||
assert!(serde_json::from_str::<BackendJobState>("\"cancelled\"").is_err());
|
||||
assert!(serde_json::from_str::<BackendJobAttemptState>("\"cancelled\"").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn neutral_request_round_trip_does_not_infer_authority_or_replace_profile() {
|
||||
for profile in ["builtin:backend-job", "builtin:job", "project:custom-job"] {
|
||||
for key in [None, Some("resource:7".to_string())] {
|
||||
let mut neutral = request().to_job_request();
|
||||
neutral.profile = profile.into();
|
||||
neutral.serialization_key = key;
|
||||
neutral.limits = job::JobLimits {
|
||||
max_concurrent_jobs: 3,
|
||||
timeout_seconds: 17,
|
||||
max_result_bytes: 777,
|
||||
max_attempts: 1,
|
||||
};
|
||||
let backend = BackendJobRequest::from_job_request(neutral.clone());
|
||||
backend.validate().unwrap();
|
||||
assert_eq!(backend.to_job_request(), neutral);
|
||||
assert!(backend.grants.is_empty());
|
||||
assert!(backend.source_worker.is_none());
|
||||
assert!(backend.notification_target.is_none());
|
||||
let encoded = serde_json::to_value(&backend).unwrap();
|
||||
for field in ["grants", "source_worker", "notification_target"] {
|
||||
assert!(encoded.get(field).is_none());
|
||||
}
|
||||
assert_eq!(encoded["profile"], profile);
|
||||
}
|
||||
}
|
||||
// Conversion is not validation and must not normalize an invalid request.
|
||||
let mut neutral = request().to_job_request();
|
||||
neutral.limits.timeout_seconds = 0;
|
||||
neutral.profile = "inherit".into();
|
||||
let backend = BackendJobRequest::from_job_request(neutral.clone());
|
||||
assert_eq!(backend.to_job_request(), neutral);
|
||||
assert!(backend.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reservation_and_acceptance_outcomes_expose_durable_job_and_attempt_not_host_lifecycle() {
|
||||
let mut reservation = reservation();
|
||||
let encoded_before = serde_json::to_value(&reservation).unwrap();
|
||||
assert_eq!(
|
||||
reservation.outcome(),
|
||||
job::JobOutcome {
|
||||
job_id: reservation.job.request.job_id.clone(),
|
||||
input_revision: "7".into(),
|
||||
attempt_id: "ticket-check:T-1:r7:attempt:2".into(),
|
||||
attempt: 2,
|
||||
state: job::JobState::Pending,
|
||||
attempt_state: job::JobAttemptState::Reserved,
|
||||
result: None,
|
||||
failure_category: None,
|
||||
failure_detail: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(serde_json::to_value(&reservation).unwrap(), encoded_before);
|
||||
for (state, attempt_state) in [
|
||||
(
|
||||
BackendJobState::Pending,
|
||||
BackendJobAttemptState::Dispatching,
|
||||
),
|
||||
(BackendJobState::Pending, BackendJobAttemptState::Dispatched),
|
||||
(BackendJobState::Failed, BackendJobAttemptState::Failed),
|
||||
(BackendJobState::Unknown, BackendJobAttemptState::Unknown),
|
||||
] {
|
||||
reservation.job.state = state;
|
||||
reservation.attempt.state = attempt_state;
|
||||
reservation.job.failure_category = Some("job-category".into());
|
||||
reservation.job.failure_detail = Some("job-detail".into());
|
||||
reservation.attempt.failure_category = Some("attempt-category".into());
|
||||
let outcome = reservation.outcome();
|
||||
assert_eq!(outcome.state, state.into());
|
||||
assert_eq!(outcome.attempt_state, attempt_state.into());
|
||||
assert_eq!(outcome.failure_category.as_deref(), Some("job-category"));
|
||||
assert_eq!(outcome.failure_detail.as_deref(), Some("job-detail"));
|
||||
assert!(outcome.result.is_none());
|
||||
let accepted = BackendJobResultAcceptance {
|
||||
job: reservation.job.clone(),
|
||||
attempt: reservation.attempt.clone(),
|
||||
replayed: true,
|
||||
};
|
||||
assert_eq!(accepted.outcome(), outcome);
|
||||
}
|
||||
reservation.job.state = BackendJobState::Completed;
|
||||
reservation.attempt.state = BackendJobAttemptState::Completed;
|
||||
reservation.job.result = Some(serde_json::json!({"ok": true}));
|
||||
reservation.job.failure_category = None;
|
||||
reservation.job.failure_detail = None;
|
||||
reservation.attempt.worker_cleanup_state = Some(BackendJobWorkerCleanupState::Failed);
|
||||
reservation.attempt.worker_cleanup_failure_detail =
|
||||
Some("cleanup is not model failure".into());
|
||||
reservation.replayed = true;
|
||||
reservation.resource_reused = true;
|
||||
let acceptance = BackendJobResultAcceptance {
|
||||
job: reservation.job.clone(),
|
||||
attempt: reservation.attempt.clone(),
|
||||
replayed: false,
|
||||
};
|
||||
let outcome = acceptance.outcome();
|
||||
assert_eq!(outcome, reservation.outcome());
|
||||
assert_eq!(outcome.state, job::JobState::Completed);
|
||||
assert_eq!(outcome.result, Some(serde_json::json!({"ok": true})));
|
||||
assert!(outcome.failure_category.is_none());
|
||||
assert!(outcome.failure_detail.is_none());
|
||||
let encoded = serde_json::to_value(outcome).unwrap();
|
||||
assert_eq!(encoded.as_object().unwrap().len(), 9);
|
||||
for field in [
|
||||
"workspace_id",
|
||||
"worker",
|
||||
"runtime_run_id",
|
||||
"request",
|
||||
"replayed",
|
||||
"resource_reused",
|
||||
"worker_cleanup_state",
|
||||
] {
|
||||
assert!(encoded.get(field).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_preserves_backend_wire_fingerprint_and_worker_envelope() {
|
||||
let request = request();
|
||||
let encoded = serde_json::to_string(&request).unwrap();
|
||||
assert_eq!(
|
||||
encoded,
|
||||
r#"{"job_id":"ticket-check:T-1:r7","purpose":"ticket_item_check","input_revision":"7","input_ref":"ticket://T-1/revisions/7","input":{"title":"Check me"},"instruction":"Check this immutable Ticket snapshot.","profile":"builtin:backend-job","source_worker":{"runtime_id":"runtime-a","worker_id":"worker-a"},"notification_target":{"runtime_id":"runtime-a","worker_id":"worker-a"},"limits":{"max_concurrent_jobs":8,"timeout_seconds":120,"max_result_bytes":16384,"max_attempts":2}}"#
|
||||
);
|
||||
assert_eq!(
|
||||
request.fingerprint().unwrap(),
|
||||
"sha256:920187fa9efa111a97c4d6869d134943e45b52edb2861cad19a0a173233c8f60"
|
||||
);
|
||||
let neutral = request.to_job_request();
|
||||
neutral.validate().unwrap();
|
||||
assert_ne!(
|
||||
request.fingerprint().unwrap(),
|
||||
neutral.fingerprint().unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&request.limits).unwrap(),
|
||||
serde_json::to_value(&neutral.limits).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
request.worker_input("attempt-1").unwrap(),
|
||||
"Check this immutable Ticket snapshot.\n\nBackend Job envelope (immutable):\njob_id: ticket-check:T-1:r7\nattempt_id: attempt-1\ninput_revision: 7\ninput_ref: ticket://T-1/revisions/7\ninput_json: {\"title\":\"Check me\"}\n\nReturn success only through the structured Backend Job result capability. Final prose and Worker Idle/Stopped state are not result authority."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_result_contract_preserves_backend_wire_and_errors() {
|
||||
let submission = BackendJobResultSubmission {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "job-1:attempt:1".into(),
|
||||
input_revision: "7".into(),
|
||||
result: serde_json::json!({"ok": true}),
|
||||
};
|
||||
let encoded = serde_json::to_string(&submission).unwrap();
|
||||
assert_eq!(
|
||||
encoded,
|
||||
r#"{"job_id":"job-1","attempt_id":"job-1:attempt:1","input_revision":"7","result":{"ok":true}}"#
|
||||
);
|
||||
let neutral: job::JobResultSubmission = serde_json::from_str(&encoded).unwrap();
|
||||
assert_eq!(neutral, submission);
|
||||
assert_eq!(
|
||||
result_digest(&submission.result, 128).unwrap(),
|
||||
job::result_digest(&submission.result, 128).unwrap()
|
||||
);
|
||||
assert!(
|
||||
matches!(result_digest(&submission.result, 4), Err(Error::InvalidInput(detail)) if detail == "Backend Job result exceeds 4 bytes")
|
||||
);
|
||||
let mut request = request();
|
||||
request.job_id = "".into();
|
||||
assert!(
|
||||
matches!(request.validate(), Err(Error::InvalidInput(detail)) if detail == "Backend Job job_id must be non-empty and trimmed")
|
||||
);
|
||||
request.job_id = "job-1".into();
|
||||
request.limits.timeout_seconds = 0;
|
||||
assert!(
|
||||
matches!(request.validate(), Err(Error::InvalidInput(detail)) if detail == "Backend Job timeout_seconds must be in 1..=600")
|
||||
);
|
||||
// Preserve domain-before-input/limits error ordering as well.
|
||||
request.grants.subjektiv_consolidation = Some(SubjektivConsolidationGrant {
|
||||
subject_id: "".into(),
|
||||
candidate_ids: vec![],
|
||||
});
|
||||
assert!(
|
||||
matches!(request.validate(), Err(Error::InvalidInput(detail)) if detail == "Backend Job grant subject_id must be non-empty and trimmed")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_covers_revision_profile_limits_and_notification_provenance() {
|
||||
let request = request();
|
||||
|
||||
Reference in New Issue
Block a user