feat: execute durable standalone Jobs through Host-owned Internal Workers (T-708)
This commit is contained in:
Generated
+15
@@ -2339,6 +2339,17 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "job"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"manifest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.34"
|
||||
@@ -4625,8 +4636,10 @@ dependencies = [
|
||||
"client",
|
||||
"fs4",
|
||||
"futures",
|
||||
"job",
|
||||
"manifest",
|
||||
"protocol",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"session-store",
|
||||
@@ -6432,6 +6445,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"http",
|
||||
"include_dir",
|
||||
"job",
|
||||
"jsonschema",
|
||||
"libc",
|
||||
"manifest",
|
||||
@@ -6615,6 +6629,7 @@ dependencies = [
|
||||
"config-source",
|
||||
"flow",
|
||||
"futures",
|
||||
"job",
|
||||
"manifest",
|
||||
"memory",
|
||||
"merge-request",
|
||||
|
||||
@@ -8,6 +8,7 @@ members = [
|
||||
"crates/api-macros-impl",
|
||||
"crates/session-store",
|
||||
"crates/standalone",
|
||||
"crates/job",
|
||||
"crates/secrets",
|
||||
"crates/manifest",
|
||||
"crates/mcp",
|
||||
@@ -43,6 +44,7 @@ default-members = [
|
||||
"crates/api-macros-impl",
|
||||
"crates/session-store",
|
||||
"crates/standalone",
|
||||
"crates/job",
|
||||
"crates/secrets",
|
||||
"crates/manifest",
|
||||
"crates/mcp",
|
||||
@@ -103,6 +105,7 @@ session-metrics = { path = "crates/session-metrics" }
|
||||
session-analytics = { path = "crates/session-analytics" }
|
||||
session-store = { path = "crates/session-store" }
|
||||
standalone = { path = "crates/standalone" }
|
||||
job = { path = "crates/job" }
|
||||
runtime-api = { path = "crates/runtime-api" }
|
||||
server-api = { path = "crates/server-api" }
|
||||
secrets = { path = "crates/secrets" }
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "job"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
manifest.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
@@ -0,0 +1,336 @@
|
||||
//! Transport-neutral immutable Job intent and structured-result contract.
|
||||
//!
|
||||
//! Hosts own execution, authority, persistence and lifecycle state. This crate
|
||||
//! deliberately contains no runner, state machine, Runtime or domain grants.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub const DEFAULT_MAX_CONCURRENT_JOBS: u16 = 8;
|
||||
pub const ABSOLUTE_MAX_CONCURRENT_JOBS: u16 = 64;
|
||||
pub const DEFAULT_JOB_TIMEOUT_SECONDS: u32 = 120;
|
||||
pub const MAX_JOB_TIMEOUT_SECONDS: u32 = 600;
|
||||
pub const DEFAULT_MAX_RESULT_BYTES: u32 = 16 * 1024;
|
||||
pub const ABSOLUTE_MAX_RESULT_BYTES: u32 = 64 * 1024;
|
||||
pub const DEFAULT_MAX_ATTEMPTS: u8 = 2;
|
||||
pub const ABSOLUTE_MAX_ATTEMPTS: u8 = 3;
|
||||
pub const MAX_JOB_INPUT_BYTES: usize = 128 * 1024;
|
||||
pub const MAX_JOB_INSTRUCTION_BYTES: usize = 16 * 1024;
|
||||
pub const MAX_JOB_PURPOSE_BYTES: usize = 120;
|
||||
pub const MAX_JOB_REFERENCE_BYTES: usize = 512;
|
||||
pub const MAX_JOB_FAILURE_BYTES: usize = 1024;
|
||||
pub const MAX_JOB_DELIVERY_BYTES: usize = 16 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum JobError {
|
||||
#[error("{0}")]
|
||||
InvalidInput(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, JobError>;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobLimits {
|
||||
#[serde(default = "default_max_concurrent_jobs")]
|
||||
pub max_concurrent_jobs: u16,
|
||||
#[serde(default = "default_job_timeout_seconds")]
|
||||
pub timeout_seconds: u32,
|
||||
#[serde(default = "default_max_result_bytes")]
|
||||
pub max_result_bytes: u32,
|
||||
#[serde(default = "default_max_attempts")]
|
||||
pub max_attempts: u8,
|
||||
}
|
||||
|
||||
impl Default for JobLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_concurrent_jobs: DEFAULT_MAX_CONCURRENT_JOBS,
|
||||
timeout_seconds: DEFAULT_JOB_TIMEOUT_SECONDS,
|
||||
max_result_bytes: DEFAULT_MAX_RESULT_BYTES,
|
||||
max_attempts: DEFAULT_MAX_ATTEMPTS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JobLimits {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.max_concurrent_jobs == 0 || self.max_concurrent_jobs > ABSOLUTE_MAX_CONCURRENT_JOBS
|
||||
{
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job max_concurrent_jobs must be in 1..={ABSOLUTE_MAX_CONCURRENT_JOBS}"
|
||||
)));
|
||||
}
|
||||
if self.timeout_seconds == 0 || self.timeout_seconds > MAX_JOB_TIMEOUT_SECONDS {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job timeout_seconds must be in 1..={MAX_JOB_TIMEOUT_SECONDS}"
|
||||
)));
|
||||
}
|
||||
if self.max_result_bytes == 0 || self.max_result_bytes > ABSOLUTE_MAX_RESULT_BYTES {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job max_result_bytes must be in 1..={ABSOLUTE_MAX_RESULT_BYTES}"
|
||||
)));
|
||||
}
|
||||
if self.max_attempts == 0 || self.max_attempts > ABSOLUTE_MAX_ATTEMPTS {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job max_attempts must be in 1..={ABSOLUTE_MAX_ATTEMPTS}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const fn default_max_concurrent_jobs() -> u16 {
|
||||
DEFAULT_MAX_CONCURRENT_JOBS
|
||||
}
|
||||
const fn default_job_timeout_seconds() -> u32 {
|
||||
DEFAULT_JOB_TIMEOUT_SECONDS
|
||||
}
|
||||
const fn default_max_result_bytes() -> u32 {
|
||||
DEFAULT_MAX_RESULT_BYTES
|
||||
}
|
||||
const fn default_max_attempts() -> u8 {
|
||||
DEFAULT_MAX_ATTEMPTS
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobRequest {
|
||||
/// Stable caller-selected idempotency key within the host's namespace.
|
||||
pub job_id: String,
|
||||
pub purpose: String,
|
||||
/// Revision fenced by the structured result.
|
||||
pub input_revision: String,
|
||||
pub input_ref: String,
|
||||
/// Bounded immutable snapshot; must not contain credentials.
|
||||
pub input: Value,
|
||||
pub instruction: String,
|
||||
/// Manifest registry selector, never raw source or a filesystem path.
|
||||
pub profile: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub serialization_key: Option<String>,
|
||||
#[serde(default)]
|
||||
pub limits: JobLimits,
|
||||
}
|
||||
|
||||
impl JobRequest {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
self.validate_fields()?;
|
||||
self.validate_input()?;
|
||||
self.limits.validate()
|
||||
}
|
||||
|
||||
/// Request metadata validation, separable so adapters can retain their
|
||||
/// domain-validation ordering before validating input and limits.
|
||||
pub fn validate_fields(&self) -> Result<()> {
|
||||
validate_identifier("job_id", &self.job_id, 256)?;
|
||||
validate_identifier("purpose", &self.purpose, MAX_JOB_PURPOSE_BYTES)?;
|
||||
validate_identifier("input_revision", &self.input_revision, 256)?;
|
||||
validate_bounded_text("input_ref", &self.input_ref, MAX_JOB_REFERENCE_BYTES)?;
|
||||
validate_bounded_text("instruction", &self.instruction, MAX_JOB_INSTRUCTION_BYTES)?;
|
||||
self.profile_selector()?;
|
||||
if let Some(key) = &self.serialization_key {
|
||||
validate_identifier("serialization_key", key, MAX_JOB_REFERENCE_BYTES)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_input(&self) -> Result<()> {
|
||||
let input = serde_json::to_vec(&self.input)
|
||||
.map_err(|error| JobError::InvalidInput(format!("serialize Job input: {error}")))?;
|
||||
if input.len() > MAX_JOB_INPUT_BYTES {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job input exceeds {MAX_JOB_INPUT_BYTES} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Syntax only; registry existence and resolution are host authority.
|
||||
pub fn profile_selector(&self) -> Result<manifest::ProfileSelector> {
|
||||
profile_selector(&self.profile)
|
||||
}
|
||||
|
||||
pub fn fingerprint(&self) -> Result<String> {
|
||||
self.validate()?;
|
||||
fingerprint(self)
|
||||
}
|
||||
|
||||
pub fn worker_input(&self, attempt_id: &str) -> Result<String> {
|
||||
worker_input(self, attempt_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobResultSubmission {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub input_revision: String,
|
||||
pub result: Value,
|
||||
}
|
||||
|
||||
/// Client-facing lifecycle vocabulary, not a runner or transition policy.
|
||||
/// A variant's presence does not imply every host supports that operation.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobState {
|
||||
Pending,
|
||||
Completed,
|
||||
Failed,
|
||||
Unknown,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobAttemptState {
|
||||
Reserved,
|
||||
Dispatching,
|
||||
Dispatched,
|
||||
Completed,
|
||||
Failed,
|
||||
Unknown,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// Transport-neutral read snapshot. Job status/result and the corresponding
|
||||
/// attempt identity/status remain distinct; host identities, delivery and
|
||||
/// resource cleanup are deliberately not part of the model outcome.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobOutcome {
|
||||
pub job_id: String,
|
||||
pub input_revision: String,
|
||||
pub attempt_id: String,
|
||||
pub attempt: u8,
|
||||
pub state: JobState,
|
||||
pub attempt_state: JobAttemptState,
|
||||
pub result: Option<Value>,
|
||||
pub failure_category: Option<String>,
|
||||
pub failure_detail: Option<String>,
|
||||
}
|
||||
|
||||
pub fn profile_selector(profile: &str) -> Result<manifest::ProfileSelector> {
|
||||
use manifest::ProfileSelector;
|
||||
validate_identifier("profile", profile, 256)?;
|
||||
let selector = ProfileSelector::parse_cli(profile);
|
||||
let valid = match &selector {
|
||||
ProfileSelector::Default => true,
|
||||
ProfileSelector::Path { .. } => false,
|
||||
ProfileSelector::Named { source, name } => {
|
||||
!name.is_empty()
|
||||
&& !name.starts_with('.')
|
||||
&& !name.contains(['/', '\\', ':'])
|
||||
&& name
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.'))
|
||||
&& ![".toml", ".json", ".dcdl", ".nix"]
|
||||
.iter()
|
||||
.any(|suffix| name.ends_with(suffix))
|
||||
&& (source.is_some() || name != "inherit")
|
||||
}
|
||||
};
|
||||
if !valid {
|
||||
return Err(JobError::InvalidInput(
|
||||
"Job profile must be a manifest registry selector, not raw source or a path".into(),
|
||||
));
|
||||
}
|
||||
Ok(selector)
|
||||
}
|
||||
|
||||
/// Hash the full serialized host intent, not just its neutral projection.
|
||||
/// Callers must validate the complete intent first. JSON field ordering and
|
||||
/// omission rules are intentionally preserved for durable replay compatibility.
|
||||
pub fn fingerprint<T: Serialize + ?Sized>(intent: &T) -> Result<String> {
|
||||
let encoded = serde_json::to_vec(intent)
|
||||
.map_err(|error| JobError::InvalidInput(format!("serialize Job intent: {error}")))?;
|
||||
Ok(sha256(&encoded))
|
||||
}
|
||||
|
||||
pub fn worker_input(request: &JobRequest, attempt_id: &str) -> Result<String> {
|
||||
worker_input_with_label(request, attempt_id, "Job")
|
||||
}
|
||||
|
||||
/// Adapter-specific capability label; only the host, never the model, selects it.
|
||||
/// Keeps an existing host's prompt envelope byte-for-byte compatible.
|
||||
pub fn worker_input_with_label(
|
||||
request: &JobRequest,
|
||||
attempt_id: &str,
|
||||
label: &str,
|
||||
) -> Result<String> {
|
||||
let input = serde_json::to_string(&request.input)
|
||||
.map_err(|error| JobError::InvalidInput(format!("serialize Job input: {error}")))?;
|
||||
Ok(format!(
|
||||
"{}\n\n{label} envelope (immutable):\njob_id: {}\nattempt_id: {}\ninput_revision: {}\ninput_ref: {}\ninput_json: {}\n\nReturn success only through the structured {label} result capability. Final prose and Worker Idle/Stopped state are not result authority.",
|
||||
request.instruction,
|
||||
request.job_id,
|
||||
attempt_id,
|
||||
request.input_revision,
|
||||
request.input_ref,
|
||||
input
|
||||
))
|
||||
}
|
||||
|
||||
pub fn attempt_id(job_id: &str, attempt: u8) -> String {
|
||||
format!("{job_id}:attempt:{attempt}")
|
||||
}
|
||||
|
||||
/// Bound the encoded JSON before hashing. Returns (digest, exact JSON).
|
||||
pub fn result_digest(result: &Value, max_bytes: u32) -> Result<(String, String)> {
|
||||
let encoded = serde_json::to_string(result)
|
||||
.map_err(|error| JobError::InvalidInput(format!("serialize Job result: {error}")))?;
|
||||
if encoded.len() > max_bytes as usize {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job result exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok((sha256(encoded.as_bytes()), encoded))
|
||||
}
|
||||
|
||||
pub fn bounded_failure_detail(detail: &str) -> String {
|
||||
if detail.len() <= MAX_JOB_FAILURE_BYTES {
|
||||
return detail.to_string();
|
||||
}
|
||||
let mut end = MAX_JOB_FAILURE_BYTES;
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
detail[..end].to_string()
|
||||
}
|
||||
|
||||
pub fn validate_identifier(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
validate_bounded_text(field, value, max_bytes)?;
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job {field} must not contain control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_bounded_text(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
if value.is_empty() || value.trim() != value {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job {field} must be non-empty and trimmed"
|
||||
)));
|
||||
}
|
||||
if value.len() > max_bytes {
|
||||
return Err(JobError::InvalidInput(format!(
|
||||
"Job {field} exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sha256(bytes: &[u8]) -> String {
|
||||
let digest = Sha256::digest(bytes);
|
||||
let mut output = String::with_capacity(7 + digest.len() * 2);
|
||||
output.push_str("sha256:");
|
||||
for byte in digest {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(output, "{byte:02x}");
|
||||
}
|
||||
output
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
use job::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn request() -> JobRequest {
|
||||
JobRequest {
|
||||
job_id: "check:r7".into(),
|
||||
purpose: "snapshot_check".into(),
|
||||
input_revision: "7".into(),
|
||||
input_ref: "snapshot://7".into(),
|
||||
input: json!({"title": "Check me"}),
|
||||
instruction: "Check this immutable snapshot.".into(),
|
||||
profile: "project:custom-check".into(),
|
||||
serialization_key: None,
|
||||
limits: JobLimits::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_lifecycle_enums_round_trip_all_host_states() {
|
||||
for (state, name) in [
|
||||
(JobState::Pending, "pending"),
|
||||
(JobState::Completed, "completed"),
|
||||
(JobState::Failed, "failed"),
|
||||
(JobState::Unknown, "unknown"),
|
||||
(JobState::Cancelled, "cancelled"),
|
||||
] {
|
||||
assert_eq!(serde_json::to_value(state).unwrap(), json!(name));
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobState>(json!(name)).unwrap(),
|
||||
state
|
||||
);
|
||||
}
|
||||
for (state, name) in [
|
||||
(JobAttemptState::Reserved, "reserved"),
|
||||
(JobAttemptState::Dispatching, "dispatching"),
|
||||
(JobAttemptState::Dispatched, "dispatched"),
|
||||
(JobAttemptState::Completed, "completed"),
|
||||
(JobAttemptState::Failed, "failed"),
|
||||
(JobAttemptState::Unknown, "unknown"),
|
||||
(JobAttemptState::Cancelled, "cancelled"),
|
||||
] {
|
||||
assert_eq!(serde_json::to_value(state).unwrap(), json!(name));
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobAttemptState>(json!(name)).unwrap(),
|
||||
state
|
||||
);
|
||||
}
|
||||
assert!(serde_json::from_value::<JobState>(json!("running")).is_err());
|
||||
assert!(serde_json::from_value::<JobAttemptState>(json!("idle")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_is_a_neutral_read_snapshot_with_optional_result_and_failure() {
|
||||
let mut outcome = JobOutcome {
|
||||
job_id: "job-1".into(),
|
||||
input_revision: "7".into(),
|
||||
attempt_id: "job-1:attempt:2".into(),
|
||||
attempt: 2,
|
||||
state: JobState::Pending,
|
||||
attempt_state: JobAttemptState::Dispatching,
|
||||
result: None,
|
||||
failure_category: None,
|
||||
failure_detail: None,
|
||||
};
|
||||
let encoded = serde_json::to_value(&outcome).unwrap();
|
||||
assert_eq!(
|
||||
encoded,
|
||||
json!({
|
||||
"job_id": "job-1", "input_revision": "7", "attempt_id": "job-1:attempt:2",
|
||||
"attempt": 2, "state": "pending", "attempt_state": "dispatching",
|
||||
"result": null, "failure_category": null, "failure_detail": null
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobOutcome>(encoded.clone()).unwrap(),
|
||||
outcome
|
||||
);
|
||||
let mut with_host_identity = encoded;
|
||||
with_host_identity["workspace_id"] = json!("workspace-a");
|
||||
assert!(serde_json::from_value::<JobOutcome>(with_host_identity).is_err());
|
||||
for (state, attempt_state, result, category, detail) in [
|
||||
(
|
||||
JobState::Completed,
|
||||
JobAttemptState::Completed,
|
||||
Some(json!({"ok": true})),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
(
|
||||
JobState::Failed,
|
||||
JobAttemptState::Failed,
|
||||
None,
|
||||
Some("execution"),
|
||||
Some("known failure"),
|
||||
),
|
||||
(
|
||||
JobState::Unknown,
|
||||
JobAttemptState::Unknown,
|
||||
None,
|
||||
Some("interrupted"),
|
||||
Some("requires reconciliation"),
|
||||
),
|
||||
(
|
||||
JobState::Cancelled,
|
||||
JobAttemptState::Cancelled,
|
||||
None,
|
||||
Some("cancelled"),
|
||||
Some("caller cancelled"),
|
||||
),
|
||||
] {
|
||||
outcome.state = state;
|
||||
outcome.attempt_state = attempt_state;
|
||||
outcome.result = result;
|
||||
outcome.failure_category = category.map(str::to_owned);
|
||||
outcome.failure_detail = detail.map(str::to_owned);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<JobOutcome>(&serde_json::to_string(&outcome).unwrap()).unwrap(),
|
||||
outcome
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn neutral_request_and_result_have_strict_round_trip_contracts() {
|
||||
let request = request();
|
||||
let encoded = serde_json::to_value(&request).unwrap();
|
||||
assert_eq!(encoded.as_object().unwrap().len(), 8);
|
||||
assert!(encoded.get("serialization_key").is_none());
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobRequest>(encoded.clone()).unwrap(),
|
||||
request
|
||||
);
|
||||
let mut with_grant = encoded.clone();
|
||||
with_grant["grants"] = json!({});
|
||||
assert!(serde_json::from_value::<JobRequest>(with_grant).is_err());
|
||||
let mut with_provenance = encoded;
|
||||
with_provenance["source_worker"] = json!({});
|
||||
assert!(serde_json::from_value::<JobRequest>(with_provenance).is_err());
|
||||
|
||||
let submission = JobResultSubmission {
|
||||
job_id: request.job_id,
|
||||
attempt_id: "check:r7:attempt:1".into(),
|
||||
input_revision: request.input_revision,
|
||||
result: json!({"ok": true}),
|
||||
};
|
||||
let mut encoded = serde_json::to_value(&submission).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::from_value::<JobResultSubmission>(encoded.clone()).unwrap(),
|
||||
submission
|
||||
);
|
||||
encoded["authority"] = json!("profile");
|
||||
assert!(serde_json::from_value::<JobResultSubmission>(encoded).is_err());
|
||||
assert_eq!(
|
||||
serde_json::from_str::<JobLimits>("{}").unwrap(),
|
||||
JobLimits::default()
|
||||
);
|
||||
assert!(serde_json::from_str::<JobLimits>(r#"{"turns":999}"#).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_covers_every_immutable_field_and_preserves_replay() {
|
||||
let request = request();
|
||||
let digest = request.fingerprint().unwrap();
|
||||
let decoded: JobRequest =
|
||||
serde_json::from_str(&serde_json::to_string(&request).unwrap()).unwrap();
|
||||
assert_eq!(digest, decoded.fingerprint().unwrap());
|
||||
assert_eq!(digest, fingerprint(&request).unwrap());
|
||||
for mutate in [
|
||||
|r: &mut JobRequest| r.job_id.push('x'),
|
||||
|r: &mut JobRequest| r.purpose.push('x'),
|
||||
|r: &mut JobRequest| r.input_revision.push('x'),
|
||||
|r: &mut JobRequest| r.input_ref.push('x'),
|
||||
|r: &mut JobRequest| r.input = json!({"title": "changed"}),
|
||||
|r: &mut JobRequest| r.instruction.push('x'),
|
||||
|r: &mut JobRequest| r.profile = "builtin:backend-job".into(),
|
||||
|r: &mut JobRequest| r.serialization_key = Some("resource:7".into()),
|
||||
|r: &mut JobRequest| r.limits.max_attempts = 3,
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
assert_ne!(digest, changed.fingerprint().unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_profiles_are_not_authority_or_a_fixed_allowlist() {
|
||||
for profile in [
|
||||
"default",
|
||||
"custom",
|
||||
"project:custom",
|
||||
"user:custom",
|
||||
"builtin:future-job",
|
||||
] {
|
||||
profile_selector(profile).unwrap();
|
||||
}
|
||||
for profile in [
|
||||
"",
|
||||
" inherit",
|
||||
"inherit",
|
||||
"path:custom",
|
||||
"./a.toml",
|
||||
"/tmp/a",
|
||||
"project:../a",
|
||||
"project:a.toml",
|
||||
"builtin:",
|
||||
"unknown:a",
|
||||
"project:a:b",
|
||||
"C:\\custom",
|
||||
"{\"worker\":{}}",
|
||||
] {
|
||||
assert!(profile_selector(profile).is_err(), "accepted {profile}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validation_enforces_utf8_bytes_and_metadata_bounds() {
|
||||
let mut r = request();
|
||||
r.job_id = "x".repeat(256);
|
||||
r.input_revision = "x".repeat(256);
|
||||
r.purpose = "x".repeat(MAX_JOB_PURPOSE_BYTES);
|
||||
r.input_ref = "x".repeat(MAX_JOB_REFERENCE_BYTES);
|
||||
r.instruction = "x".repeat(MAX_JOB_INSTRUCTION_BYTES);
|
||||
r.serialization_key = Some("x".repeat(MAX_JOB_REFERENCE_BYTES));
|
||||
r.input = json!("x".repeat(MAX_JOB_INPUT_BYTES - 2));
|
||||
r.validate().unwrap();
|
||||
for mutate in [
|
||||
|r: &mut JobRequest| r.job_id.push('x'),
|
||||
|r: &mut JobRequest| r.input_revision.push('x'),
|
||||
|r: &mut JobRequest| r.purpose.push('x'),
|
||||
|r: &mut JobRequest| r.input_ref.push('x'),
|
||||
|r: &mut JobRequest| r.instruction.push('x'),
|
||||
|r: &mut JobRequest| r.serialization_key.as_mut().unwrap().push('x'),
|
||||
|r: &mut JobRequest| r.input = json!("x".repeat(MAX_JOB_INPUT_BYTES - 1)),
|
||||
] {
|
||||
let mut changed = r.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(changed.validate().is_err());
|
||||
assert!(changed.fingerprint().is_err());
|
||||
}
|
||||
assert!(validate_identifier("id", "a\nb", 256).is_err());
|
||||
assert!(validate_bounded_text("text", " untrimmed", 256).is_err());
|
||||
assert!(validate_bounded_text("text", "", 256).is_err());
|
||||
assert!(validate_bounded_text("text", "界", 2).is_err());
|
||||
validate_bounded_text("text", "a\nb", 256).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_limit_dimensions_reject_zero_and_above_absolute_bounds() {
|
||||
let maximum = JobLimits {
|
||||
max_concurrent_jobs: ABSOLUTE_MAX_CONCURRENT_JOBS,
|
||||
timeout_seconds: MAX_JOB_TIMEOUT_SECONDS,
|
||||
max_result_bytes: ABSOLUTE_MAX_RESULT_BYTES,
|
||||
max_attempts: ABSOLUTE_MAX_ATTEMPTS,
|
||||
};
|
||||
maximum.validate().unwrap();
|
||||
for mutate in [
|
||||
|l: &mut JobLimits| l.max_concurrent_jobs = 0,
|
||||
|l: &mut JobLimits| l.max_concurrent_jobs += 1,
|
||||
|l: &mut JobLimits| l.timeout_seconds = 0,
|
||||
|l: &mut JobLimits| l.timeout_seconds += 1,
|
||||
|l: &mut JobLimits| l.max_result_bytes = 0,
|
||||
|l: &mut JobLimits| l.max_result_bytes += 1,
|
||||
|l: &mut JobLimits| l.max_attempts = 0,
|
||||
|l: &mut JobLimits| l.max_attempts += 1,
|
||||
] {
|
||||
let mut changed = maximum.clone();
|
||||
mutate(&mut changed);
|
||||
assert!(changed.validate().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn result_digest_and_failure_detail_bound_exact_utf8_encoding() {
|
||||
let result = json!({"ok": true});
|
||||
let (digest, encoded) = result_digest(&result, 11).unwrap();
|
||||
assert_eq!(encoded, r#"{"ok":true}"#);
|
||||
assert_eq!(digest, sha256(encoded.as_bytes()));
|
||||
assert!(result_digest(&result, 10).is_err());
|
||||
let unicode = json!("界");
|
||||
assert!(result_digest(&unicode, 4).is_err());
|
||||
result_digest(&unicode, 5).unwrap();
|
||||
let detail = "界".repeat(MAX_JOB_FAILURE_BYTES);
|
||||
let bounded = bounded_failure_detail(&detail);
|
||||
assert_eq!(bounded.len(), MAX_JOB_FAILURE_BYTES - 1);
|
||||
assert!(detail.starts_with(&bounded));
|
||||
assert_eq!(bounded_failure_detail("known failure"), "known failure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_input_is_revision_and_attempt_bound_not_final_prose_authority() {
|
||||
let request = request();
|
||||
let attempt = attempt_id(&request.job_id, 2);
|
||||
assert_eq!(attempt, "check:r7:attempt:2");
|
||||
let input = request.worker_input(&attempt).unwrap();
|
||||
assert!(input.starts_with(&request.instruction));
|
||||
for field in [
|
||||
"Job envelope (immutable):",
|
||||
"job_id: check:r7",
|
||||
"attempt_id: check:r7:attempt:2",
|
||||
"input_revision: 7",
|
||||
"input_ref: snapshot://7",
|
||||
r#"input_json: {"title":"Check me"}"#,
|
||||
"structured Job result capability",
|
||||
"Final prose and Worker Idle/Stopped state are not result authority.",
|
||||
] {
|
||||
assert!(input.contains(field), "missing {field}");
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,13 @@ pub const BUILTIN_PROFILE_RESOURCES: &[BuiltinProfileResource] = &[
|
||||
description: "Workspace orchestration and Worker control.",
|
||||
imports: BASE_IMPORT,
|
||||
},
|
||||
BuiltinProfileResource {
|
||||
selector: Some("builtin:job"),
|
||||
path: "profiles/job.dcdl",
|
||||
source: include_str!("../../../resources/profiles/job.dcdl"),
|
||||
description: "Result-only caller-selected Job policy.",
|
||||
imports: NO_IMPORTS,
|
||||
},
|
||||
BuiltinProfileResource {
|
||||
selector: Some("builtin:backend-job"),
|
||||
path: "profiles/backend-job.dcdl",
|
||||
|
||||
@@ -1257,6 +1257,13 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(resolved.manifest.worker.name, "role-worker");
|
||||
if entry.name == "job" {
|
||||
assert!(resolved.manifest.compaction.is_none());
|
||||
assert!(resolved.manifest.scope.allow.is_empty());
|
||||
assert!(resolved.manifest.delegation_scope.allow.is_empty());
|
||||
assert_eq!(resolved.manifest.engine.instruction, "internal.job_system");
|
||||
continue;
|
||||
}
|
||||
let compaction = resolved.manifest.compaction.as_ref().unwrap();
|
||||
assert_eq!(compaction.worker_max_turns, Some(100), "{}", entry.name);
|
||||
assert_eq!(
|
||||
|
||||
@@ -10,6 +10,8 @@ agen.workspace = true
|
||||
client.workspace = true
|
||||
fs4.workspace = true
|
||||
manifest.workspace = true
|
||||
job.workspace = true
|
||||
rusqlite.workspace = true
|
||||
protocol.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -43,6 +43,7 @@ pub struct StandaloneHost {
|
||||
worker_store: FsWorkerStore,
|
||||
record: StandaloneWorkerRecord,
|
||||
lease: Option<StandaloneWorkerLease>,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
@@ -77,6 +78,8 @@ pub enum StandaloneStartupError {
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
pub enum StandaloneShutdownError {
|
||||
#[error("standalone Job execution cleanup or terminal persistence could not be confirmed")]
|
||||
JobCleanup,
|
||||
#[error("the standalone Worker did not stop before the shutdown deadline")]
|
||||
DeadlineExceeded,
|
||||
#[error("the standalone Worker shutdown confirmation was lost")]
|
||||
@@ -110,6 +113,16 @@ impl StandaloneHost {
|
||||
.allocate(&launch.cwd, StaleLeasePolicy::Reject)
|
||||
.map_err(classify_store_startup_error)?;
|
||||
let worker_id = allocation.worker_id();
|
||||
let jobs = match crate::jobs::StandaloneJobs::open(
|
||||
&store.jobs_path(worker_id),
|
||||
launch.cwd.clone(),
|
||||
) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(_) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
return Err(StandaloneStartupError::StateStore);
|
||||
}
|
||||
};
|
||||
|
||||
// WorkerId is the stable identity. The current Worker store remains
|
||||
// name-keyed, so keep its derived storage key separate from the
|
||||
@@ -181,6 +194,7 @@ impl StandaloneHost {
|
||||
worker_store,
|
||||
record,
|
||||
allocation.into_lease(),
|
||||
jobs,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -216,6 +230,11 @@ impl StandaloneHost {
|
||||
let lease = store
|
||||
.acquire_lease(worker_id, StaleLeasePolicy::Recover)
|
||||
.map_err(classify_store_startup_error)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&store.jobs_path(worker_id),
|
||||
record.cwd.canonical_path.clone(),
|
||||
)
|
||||
.map_err(|_| StandaloneStartupError::StateStore)?;
|
||||
let (backing_store, worker_store) = backing_store(&store, worker_id)?;
|
||||
let storage_key = record.storage_key.clone();
|
||||
let mut manifest = record.manifest.clone();
|
||||
@@ -270,6 +289,7 @@ impl StandaloneHost {
|
||||
worker_store,
|
||||
record,
|
||||
lease,
|
||||
jobs,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -279,6 +299,7 @@ impl StandaloneHost {
|
||||
worker_store: FsWorkerStore,
|
||||
record: StandaloneWorkerRecord,
|
||||
lease: StandaloneWorkerLease,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
) -> Self {
|
||||
Self {
|
||||
handle: started.handle,
|
||||
@@ -288,9 +309,16 @@ impl StandaloneHost {
|
||||
worker_store,
|
||||
record,
|
||||
lease: Some(lease),
|
||||
jobs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit Host execution capability for local Features, separate from the
|
||||
/// interactive Worker protocol and its model-created SubWorkers.
|
||||
pub fn jobs(&self) -> crate::jobs::StandaloneJobs {
|
||||
self.jobs.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn worker_id(&self) -> WorkerId {
|
||||
self.record.worker_id
|
||||
@@ -318,6 +346,7 @@ impl StandaloneHost {
|
||||
}
|
||||
|
||||
pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> {
|
||||
let jobs_result = self.jobs.shutdown().await;
|
||||
let command = protocol::WorkerCommandEnvelope::new(u64::MAX);
|
||||
let _ = self.handle.send(Method::Shutdown { command }).await;
|
||||
let Some(shutdown) = self.shutdown.take() else {
|
||||
@@ -335,6 +364,10 @@ impl StandaloneHost {
|
||||
return Err(StandaloneShutdownError::DeadlineExceeded);
|
||||
}
|
||||
}
|
||||
if jobs_result.is_err() {
|
||||
self.retain_lease();
|
||||
return Err(StandaloneShutdownError::JobCleanup);
|
||||
}
|
||||
let active = match active_pointer(&self.worker_store, &self.record.storage_key) {
|
||||
Ok(active) => active,
|
||||
Err(_) => {
|
||||
@@ -370,6 +403,16 @@ impl StandaloneHost {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StandaloneHost {
|
||||
fn drop(&mut self) {
|
||||
self.jobs.close();
|
||||
// A dropped Host cannot synchronously confirm either controller or Job
|
||||
// cleanup. Keep its lease until explicit shutdown or process recovery,
|
||||
// preventing another Host from reopening a still-live execution store.
|
||||
self.retain_lease();
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_protocol_session(
|
||||
handle: worker::WorkerHandle,
|
||||
streams: WorkerProtocolSessionStreams,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,393 @@
|
||||
//! Host-provided local Jobs. No Runtime registry, Workspace client or dialog-session state.
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use agen::llm_client::LlmClient;
|
||||
use job::{JobRequest, JobResultSubmission};
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::job_store::JobStore;
|
||||
pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum JobServiceError {
|
||||
#[error(transparent)]
|
||||
Store(#[from] JobStoreError),
|
||||
#[error("standalone Job execution requirements are unavailable: {0}")]
|
||||
Configuration(String),
|
||||
#[error("the standalone Job Host is shutting down")]
|
||||
Closed,
|
||||
#[error("standalone Job execution capacity is exhausted")]
|
||||
Capacity,
|
||||
#[error("standalone Job cleanup failed: {0}")]
|
||||
Cleanup(String),
|
||||
}
|
||||
|
||||
struct Execution {
|
||||
cancel: watch::Sender<bool>,
|
||||
complete: watch::Receiver<Option<Result<(), String>>>,
|
||||
attempt_id: String,
|
||||
resource_key: Option<String>,
|
||||
max_concurrent: u16,
|
||||
task: tokio::task::JoinHandle<Result<(), JobServiceError>>,
|
||||
}
|
||||
struct Executions {
|
||||
closed: bool,
|
||||
jobs: HashMap<String, Execution>,
|
||||
}
|
||||
|
||||
/// Cloneable Feature-facing access to a Host-owned service. Cloning this handle
|
||||
/// neither extends the Host's authority nor permits opening/recovering its store.
|
||||
#[derive(Clone)]
|
||||
pub struct StandaloneJobs {
|
||||
store: Arc<Mutex<Option<JobStore>>>,
|
||||
executions: Arc<Mutex<Executions>>,
|
||||
cwd: PathBuf,
|
||||
capacity: u16,
|
||||
}
|
||||
|
||||
impl StandaloneJobs {
|
||||
pub(crate) fn open(path: &Path, cwd: PathBuf) -> Result<Self, JobServiceError> {
|
||||
Ok(Self {
|
||||
store: Arc::new(Mutex::new(Some(JobStore::open(path)?))),
|
||||
executions: Arc::new(Mutex::new(Executions {
|
||||
closed: false,
|
||||
jobs: HashMap::new(),
|
||||
})),
|
||||
cwd,
|
||||
capacity: job::DEFAULT_MAX_CONCURRENT_JOBS,
|
||||
})
|
||||
}
|
||||
|
||||
fn with_store<T>(
|
||||
&self,
|
||||
operation: impl FnOnce(&mut JobStore) -> Result<T, JobStoreError>,
|
||||
) -> Result<T, JobServiceError> {
|
||||
let mut slot = self.store.lock().expect("Job store poisoned");
|
||||
Ok(operation(slot.as_mut().ok_or(JobServiceError::Closed)?)?)
|
||||
}
|
||||
|
||||
/// Persist intent without starting a model. Exact resend returns the same
|
||||
/// current attempt, including after acknowledgement or cancellation.
|
||||
pub fn request(&self, request: JobRequest) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
Ok(self.with_store(|store| store.reserve(request))?)
|
||||
}
|
||||
|
||||
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
Ok(self.with_store(|store| store.get(job_id))?)
|
||||
}
|
||||
|
||||
/// Pending attempts survive shutdown. The consumer chooses when to resume
|
||||
/// them; opening a Host never implicitly replays unknown side effects.
|
||||
pub fn pending(&self) -> Result<Vec<JobSnapshot>, JobServiceError> {
|
||||
Ok(self.with_store(|store| store.pending())?)
|
||||
}
|
||||
|
||||
/// Explicit new attempt, bounded by immutable intent limits. Unknown
|
||||
/// outcomes cannot be retried without domain reconciliation/new intent.
|
||||
pub fn retry(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
if executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| !run.task.is_finished())
|
||||
{
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
Ok(self.with_store(|store| store.retry(job_id))?)
|
||||
}
|
||||
|
||||
/// Successful model processing and consumer postprocessing have independent
|
||||
/// lifetimes. A failed delivery simply leaves this acknowledgement pending.
|
||||
pub fn acknowledge(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
Ok(self.with_store(|store| store.ack(job_id))?)
|
||||
}
|
||||
|
||||
pub fn cancel(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
let snapshot = self.with_store(|store| store.cancel(job_id))?;
|
||||
if let Some(execution) = executions.jobs.get(job_id) {
|
||||
execution.cancel.send_replace(true);
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
pub async fn start(&self, job_id: &str) -> Result<(), JobServiceError> {
|
||||
self.start_with_optional_model_client(job_id, None).await
|
||||
}
|
||||
|
||||
/// Injectable model transport for embedders and scripted execution tests.
|
||||
/// The requested registry Profile still resolves/validates normally; the
|
||||
/// transport grants no filesystem, Feature or result-submission authority.
|
||||
pub async fn start_with_model_client<C>(
|
||||
&self,
|
||||
job_id: &str,
|
||||
client: C,
|
||||
) -> Result<(), JobServiceError>
|
||||
where
|
||||
C: LlmClient + 'static,
|
||||
{
|
||||
self.start_with_optional_model_client(job_id, Some(Box::new(client)))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn start_with_optional_model_client(
|
||||
&self,
|
||||
job_id: &str,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
) -> Result<(), JobServiceError> {
|
||||
let snapshot = self.get(job_id)?;
|
||||
let prepared = worker::job::prepare_job(&snapshot.request, &self.cwd, client)
|
||||
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
executions.jobs.retain(|_, run| {
|
||||
!(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
|
||||
});
|
||||
// Count live resources, not only result states: acceptance can precede
|
||||
// the Internal Worker's cleanup and must not free a concurrency slot.
|
||||
let live = || {
|
||||
executions
|
||||
.jobs
|
||||
.values()
|
||||
.filter(|run| !run.task.is_finished())
|
||||
};
|
||||
let count = live().count();
|
||||
let active_limit = live()
|
||||
.map(|run| run.max_concurrent)
|
||||
.min()
|
||||
.unwrap_or(self.capacity);
|
||||
if count >= usize::from(self.capacity.min(active_limit))
|
||||
|| count >= usize::from(snapshot.request.limits.max_concurrent_jobs)
|
||||
|| executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| !run.task.is_finished())
|
||||
{
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
let resource_key = snapshot.request.serialization_key.clone();
|
||||
if let Some(key) = &resource_key {
|
||||
if executions
|
||||
.jobs
|
||||
.values()
|
||||
.any(|run| !run.task.is_finished() && run.resource_key.as_ref() == Some(key))
|
||||
{
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
}
|
||||
// Retain errors from finished tasks until a caller observes them rather
|
||||
// than silently swallowing store/cleanup failures.
|
||||
if let Some(previous) = executions.jobs.get(job_id) {
|
||||
if !previous.task.is_finished() {
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
}
|
||||
let snapshot = self.with_store(|store| {
|
||||
store.dispatch(job_id, &snapshot.attempt.attempt_id, self.capacity)
|
||||
})?;
|
||||
let store = self.store.clone();
|
||||
let id = job_id.to_string();
|
||||
let attempt_id = snapshot.attempt.attempt_id.clone();
|
||||
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
|
||||
let sink = Arc::new(BoundResultSink {
|
||||
store: store.clone(),
|
||||
job_id: id.clone(),
|
||||
attempt_id: attempt_id.clone(),
|
||||
input_revision: snapshot.request.input_revision.clone(),
|
||||
deadline: tokio::time::Instant::now() + timeout,
|
||||
});
|
||||
let (cancel, mut cancellation) = watch::channel(false);
|
||||
let (complete_tx, complete) = watch::channel(None);
|
||||
let execution_attempt = attempt_id.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let outcome: Result<(), JobServiceError> = async {
|
||||
let (sender_tx, mut sender_rx) = tokio::sync::oneshot::channel();
|
||||
let run = prepared.run(attempt_id.clone(), sink, move |sender| { let _ = sender_tx.send(sender); });
|
||||
tokio::pin!(run);
|
||||
let reason = tokio::select! {
|
||||
result = &mut run => {
|
||||
let (state, detail) = match result {
|
||||
Ok(_) => (JobAttemptState::Failed, "Worker finished without an accepted structured result".to_string()),
|
||||
Err(error @ worker::job::JobExecutionError::ResultUnconfirmed(_)) => (JobAttemptState::Unknown, error.to_string()),
|
||||
Err(error) => (JobAttemptState::Failed, error.to_string()),
|
||||
};
|
||||
finish_if_pending(&store, &id, &attempt_id, state, &detail)?;
|
||||
return Ok(());
|
||||
}
|
||||
_ = tokio::time::sleep(timeout) => "timeout",
|
||||
_ = cancellation.wait_for(|cancelled| *cancelled) => "Host cancellation/shutdown",
|
||||
};
|
||||
// Fence timeout/shutdown before cancelling so a late tool call can
|
||||
// never turn an interrupted unaccepted execution into success.
|
||||
let fence_result = finish_if_pending(&store, &id, &attempt_id, JobAttemptState::Unknown, reason);
|
||||
// Cancellation is routed through Engine; keep the future alive to
|
||||
// allow normal Feature cleanup. Hard stop is a fenced unknown
|
||||
// outcome, never an automatic retry. Result acceptance wins races.
|
||||
if let Ok(sender) = sender_rx.try_recv() {
|
||||
let _ = sender.try_send(());
|
||||
} else if let Ok(Ok(sender)) = tokio::time::timeout(Duration::from_secs(1), &mut sender_rx).await {
|
||||
let _ = sender.try_send(());
|
||||
}
|
||||
let cleanup = tokio::time::timeout(Duration::from_secs(2), &mut run).await;
|
||||
fence_result?;
|
||||
if cleanup.is_err() { return Err(JobServiceError::Cleanup("Internal Worker cancellation deadline exceeded; outcome fenced unknown".into())); }
|
||||
Ok(())
|
||||
}.await;
|
||||
complete_tx.send_replace(Some(
|
||||
outcome.as_ref().map(|_| ()).map_err(ToString::to_string),
|
||||
));
|
||||
outcome
|
||||
});
|
||||
executions.jobs.insert(
|
||||
job_id.to_string(),
|
||||
Execution {
|
||||
cancel,
|
||||
complete,
|
||||
task,
|
||||
attempt_id: execution_attempt,
|
||||
resource_key,
|
||||
max_concurrent: snapshot.request.limits.max_concurrent_jobs,
|
||||
},
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Observe execution/cleanup, not success: read the durable result separately.
|
||||
pub async fn wait(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
let execution = self
|
||||
.executions
|
||||
.lock()
|
||||
.expect("Job executions poisoned")
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.map(|run| (run.attempt_id.clone(), run.complete.clone()));
|
||||
if let Some((attempt, mut completion)) = execution {
|
||||
let result = completion
|
||||
.wait_for(Option::is_some)
|
||||
.await
|
||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))?
|
||||
.clone()
|
||||
.expect("completion");
|
||||
result.map_err(JobServiceError::Cleanup)?;
|
||||
let task = {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| run.attempt_id == attempt)
|
||||
{
|
||||
executions.jobs.remove(job_id)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(execution) = task {
|
||||
execution
|
||||
.task
|
||||
.await
|
||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))??;
|
||||
}
|
||||
}
|
||||
self.get(job_id)
|
||||
}
|
||||
|
||||
pub(crate) fn close(&self) {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
executions.closed = true;
|
||||
for execution in executions.jobs.values() {
|
||||
execution.cancel.send_replace(true);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
|
||||
self.close();
|
||||
let tasks = {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
std::mem::take(&mut executions.jobs)
|
||||
};
|
||||
let mut error = None;
|
||||
for execution in tasks.into_values() {
|
||||
match execution.task.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(failure)) => {
|
||||
error = Some(failure);
|
||||
}
|
||||
Err(failure) => {
|
||||
error = Some(JobServiceError::Cleanup(failure.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Close SQLite before the Host releases its lease. Cloned handles may
|
||||
// outlive the Host but cannot mutate or retain the live connection.
|
||||
self.store.lock().expect("Job store poisoned").take();
|
||||
error.map_or(Ok(()), Err)
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_if_pending(
|
||||
store: &Mutex<Option<JobStore>>,
|
||||
id: &str,
|
||||
attempt: &str,
|
||||
state: JobAttemptState,
|
||||
detail: &str,
|
||||
) -> Result<(), JobServiceError> {
|
||||
let mut slot = store.lock().expect("Job store poisoned");
|
||||
let store = slot.as_mut().ok_or(JobServiceError::Closed)?;
|
||||
let snapshot = store.get(id)?;
|
||||
if snapshot.state == JobState::Pending {
|
||||
store.finish(id, attempt, state, Some(detail))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct BoundResultSink {
|
||||
store: Arc<Mutex<Option<JobStore>>>,
|
||||
job_id: String,
|
||||
attempt_id: String,
|
||||
input_revision: String,
|
||||
deadline: tokio::time::Instant,
|
||||
}
|
||||
impl worker::job::JobResultSink for BoundResultSink {
|
||||
fn submit(&self, submission: JobResultSubmission) -> Result<(), String> {
|
||||
if submission.job_id != self.job_id
|
||||
|| submission.attempt_id != self.attempt_id
|
||||
|| submission.input_revision != self.input_revision
|
||||
{
|
||||
return Err("Job result capability binding mismatch".into());
|
||||
}
|
||||
let mut slot = self
|
||||
.store
|
||||
.lock()
|
||||
.map_err(|_| "Job store poisoned".to_string())?;
|
||||
let store = slot.as_mut().ok_or_else(|| "Job Host closed".to_string())?;
|
||||
let snapshot = store.get(&self.job_id).map_err(|error| error.to_string())?;
|
||||
if tokio::time::Instant::now() >= self.deadline && snapshot.state != JobState::Completed {
|
||||
return Err("Job result deadline expired".into());
|
||||
}
|
||||
store
|
||||
.accept(&submission)
|
||||
.map(|_| ())
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
use super::*;
|
||||
use worker::job::JobResultSink;
|
||||
|
||||
fn request(id: &str) -> JobRequest {
|
||||
JobRequest {
|
||||
job_id: id.into(),
|
||||
purpose: "test".into(),
|
||||
input_revision: "r1".into(),
|
||||
input_ref: "fixture:input".into(),
|
||||
input: serde_json::json!({"data":42}),
|
||||
instruction: "Return a structured result".into(),
|
||||
profile: "builtin:job".into(),
|
||||
serialization_key: None,
|
||||
limits: job::JobLimits::default(),
|
||||
}
|
||||
}
|
||||
fn sink(jobs: &StandaloneJobs, id: &str, attempt: &str) -> BoundResultSink {
|
||||
BoundResultSink {
|
||||
store: jobs.store.clone(),
|
||||
job_id: id.into(),
|
||||
attempt_id: attempt.into(),
|
||||
input_revision: "r1".into(),
|
||||
deadline: tokio::time::Instant::now() + Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
fn result(id: &str, attempt: &str) -> JobResultSubmission {
|
||||
JobResultSubmission {
|
||||
job_id: id.into(),
|
||||
attempt_id: attempt.into(),
|
||||
input_revision: "r1".into(),
|
||||
result: serde_json::json!({"valid":true}),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn host_result_capability_cannot_cross_job_attempt_revision_or_deadline() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let jobs =
|
||||
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
|
||||
let first = jobs.request(request("first")).unwrap();
|
||||
let second = jobs.request(request("second")).unwrap();
|
||||
for snapshot in [&first, &second] {
|
||||
jobs.with_store(|store| {
|
||||
store.dispatch(&snapshot.request.job_id, &snapshot.attempt.attempt_id, 8)
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
let bound = sink(&jobs, "first", &first.attempt.attempt_id);
|
||||
assert!(
|
||||
bound
|
||||
.submit(result("second", &second.attempt.attempt_id))
|
||||
.is_err()
|
||||
);
|
||||
assert!(bound.submit(result("first", "first:attempt:2")).is_err());
|
||||
let mut stale = result("first", &first.attempt.attempt_id);
|
||||
stale.input_revision = "r0".into();
|
||||
assert!(bound.submit(stale).is_err());
|
||||
let mut expired = sink(&jobs, "first", &first.attempt.attempt_id);
|
||||
expired.deadline = tokio::time::Instant::now() - Duration::from_secs(1);
|
||||
assert!(
|
||||
expired
|
||||
.submit(result("first", &first.attempt.attempt_id))
|
||||
.is_err()
|
||||
);
|
||||
assert!(jobs.get("first").unwrap().result.is_none());
|
||||
bound
|
||||
.submit(result("first", &first.attempt.attempt_id))
|
||||
.unwrap();
|
||||
// Accepted exact replay is still allowed after the deadline, never a new result.
|
||||
expired
|
||||
.submit(result("first", &first.attempt.attempt_id))
|
||||
.unwrap();
|
||||
let mut changed = result("first", &first.attempt.attempt_id);
|
||||
changed.result = serde_json::json!({"valid":false});
|
||||
assert!(expired.submit(changed).is_err());
|
||||
assert_eq!(jobs.get("second").unwrap().state, JobState::Pending);
|
||||
jobs.shutdown().await.unwrap();
|
||||
assert!(
|
||||
bound
|
||||
.submit(result("first", &first.attempt.attempt_id))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_handles_are_closed() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let jobs =
|
||||
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
|
||||
let cancelled = jobs.request(request("cancelled")).unwrap();
|
||||
jobs.cancel("cancelled").unwrap();
|
||||
finish_if_pending(
|
||||
&jobs.store,
|
||||
"cancelled",
|
||||
&cancelled.attempt.attempt_id,
|
||||
JobAttemptState::Unknown,
|
||||
"cleanup",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
|
||||
let accepted = jobs.request(request("accepted")).unwrap();
|
||||
jobs.with_store(|store| store.dispatch("accepted", &accepted.attempt.attempt_id, 8))
|
||||
.unwrap();
|
||||
sink(&jobs, "accepted", &accepted.attempt.attempt_id)
|
||||
.submit(result("accepted", &accepted.attempt.attempt_id))
|
||||
.unwrap();
|
||||
finish_if_pending(
|
||||
&jobs.store,
|
||||
"accepted",
|
||||
&accepted.attempt.attempt_id,
|
||||
JobAttemptState::Failed,
|
||||
"model failed later",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.get("accepted").unwrap().state, JobState::Completed);
|
||||
jobs.shutdown().await.unwrap();
|
||||
assert!(jobs.store.lock().unwrap().is_none());
|
||||
for operation in [
|
||||
jobs.cancel("cancelled"),
|
||||
jobs.acknowledge("accepted"),
|
||||
jobs.retry("cancelled"),
|
||||
jobs.request(request("new")),
|
||||
] {
|
||||
assert!(matches!(operation, Err(JobServiceError::Closed)));
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
//! Server, HTTP, WebSocket, subprocess Worker, or alternative execution path.
|
||||
|
||||
pub mod host;
|
||||
mod job_store;
|
||||
pub mod jobs;
|
||||
pub mod launch;
|
||||
pub mod store;
|
||||
|
||||
|
||||
@@ -352,6 +352,10 @@ impl StandaloneWorkerStore {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn jobs_path(&self, id: WorkerId) -> PathBuf {
|
||||
self.worker_dir(id).join("jobs.sqlite3")
|
||||
}
|
||||
|
||||
pub(crate) fn runtime_dir(&self, id: WorkerId) -> PathBuf {
|
||||
self.worker_dir(id).join("runtime")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use agen::llm_client::{
|
||||
LlmClient,
|
||||
error::ClientError,
|
||||
event::{Event as LlmEvent, StopReason},
|
||||
types::Request,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use futures::{Stream, stream};
|
||||
use job::{JobLimits, JobRequest};
|
||||
use standalone::jobs::{JobServiceError, JobState};
|
||||
use standalone::{StandaloneHost, StandaloneLaunchConfig};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ScriptedClient {
|
||||
responses: Arc<Mutex<VecDeque<Vec<LlmEvent>>>>,
|
||||
requests: Arc<Mutex<Vec<Request>>>,
|
||||
started: Arc<tokio::sync::Notify>,
|
||||
stall: bool,
|
||||
fail: bool,
|
||||
}
|
||||
impl ScriptedClient {
|
||||
fn new(responses: Vec<Vec<LlmEvent>>) -> Self {
|
||||
Self {
|
||||
responses: Arc::new(Mutex::new(responses.into())),
|
||||
requests: Arc::new(Mutex::new(Vec::new())),
|
||||
started: Arc::new(tokio::sync::Notify::new()),
|
||||
stall: false,
|
||||
fail: false,
|
||||
}
|
||||
}
|
||||
fn stall() -> Self {
|
||||
Self {
|
||||
stall: true,
|
||||
..Self::new(Vec::new())
|
||||
}
|
||||
}
|
||||
fn requests(&self) -> Vec<Request> {
|
||||
self.requests.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl LlmClient for ScriptedClient {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
self.requests.lock().unwrap().push(request);
|
||||
self.started.notify_one();
|
||||
if self.fail {
|
||||
return Err(ClientError::Config("scripted model failure".into()));
|
||||
}
|
||||
if self.stall {
|
||||
return Ok(Box::pin(stream::pending()));
|
||||
}
|
||||
let events = self
|
||||
.responses
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.expect("scripted response");
|
||||
Ok(Box::pin(stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
fn prose() -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
LlmEvent::text_delta(0, "done"),
|
||||
LlmEvent::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||
]
|
||||
}
|
||||
fn success() -> ScriptedClient {
|
||||
ScriptedClient::new(vec![
|
||||
vec![
|
||||
LlmEvent::tool_use_start(0, "result-1", "SubmitJobResult"),
|
||||
LlmEvent::tool_input_delta(0, r#"{"result":{"valid":true}}"#),
|
||||
LlmEvent::tool_use_stop(0),
|
||||
],
|
||||
prose(),
|
||||
])
|
||||
}
|
||||
fn request(id: &str) -> JobRequest {
|
||||
JobRequest {
|
||||
job_id: id.into(),
|
||||
purpose: "generic_test".into(),
|
||||
input_revision: "r1".into(),
|
||||
input_ref: "test:immutable".into(),
|
||||
input: serde_json::json!({"test_input":42}),
|
||||
instruction: "Return a structured validity result".into(),
|
||||
profile: "builtin:job".into(),
|
||||
serialization_key: None,
|
||||
limits: JobLimits::default(),
|
||||
}
|
||||
}
|
||||
async fn host(temp: &tempfile::TempDir) -> (StandaloneHost, ScriptedClient) {
|
||||
let launch = StandaloneLaunchConfig::new(
|
||||
temp.path(),
|
||||
temp.path().join("state"),
|
||||
manifest::ProfileSelector::Default,
|
||||
"job-host",
|
||||
)
|
||||
.resolve()
|
||||
.unwrap();
|
||||
let client = ScriptedClient::new(Vec::new());
|
||||
let host = StandaloneHost::start_with_model_client(launch, client.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
(host, client)
|
||||
}
|
||||
async fn wait_started(client: &ScriptedClient) {
|
||||
tokio::time::timeout(Duration::from_secs(10), client.started.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn host_job_runs_without_dialog_and_result_survives_restore_and_ack() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, dialog) = host(&temp).await;
|
||||
let id = host.worker_id();
|
||||
let jobs = host.jobs();
|
||||
let intent = request("success");
|
||||
let reserved = jobs.request(intent.clone()).unwrap();
|
||||
assert_eq!(jobs.request(intent.clone()).unwrap(), reserved);
|
||||
let model = success();
|
||||
jobs.start_with_model_client("success", model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = jobs.wait("success").await.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
assert_eq!(completed.result, Some(serde_json::json!({"valid":true})));
|
||||
assert!(!completed.acknowledged);
|
||||
let neutral = completed.outcome();
|
||||
assert_eq!(neutral.state, job::JobState::Completed);
|
||||
assert_eq!(neutral.result, completed.result);
|
||||
assert_eq!(neutral.attempt_state, job::JobAttemptState::Completed);
|
||||
assert!(
|
||||
dialog.requests().is_empty(),
|
||||
"Job never runs on the interactive model/session"
|
||||
);
|
||||
assert_eq!(
|
||||
model.requests()[0]
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| tool.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["SubmitJobResult"]
|
||||
);
|
||||
assert_eq!(jobs.request(intent.clone()).unwrap(), completed);
|
||||
assert!(
|
||||
jobs.start_with_model_client("success", success())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(jobs.retry("success").is_err());
|
||||
let db = temp
|
||||
.path()
|
||||
.join("state")
|
||||
.join(id.to_string())
|
||||
.join("jobs.sqlite3");
|
||||
assert!(db.exists());
|
||||
host.shutdown().await.unwrap();
|
||||
assert!(matches!(
|
||||
jobs.request(request("after-shutdown")),
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
assert!(matches!(
|
||||
jobs.cancel("success"),
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
assert!(matches!(
|
||||
jobs.acknowledge("success"),
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
assert!(matches!(
|
||||
jobs.start_with_model_client("success", success()).await,
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
let restored = StandaloneHost::restore_with_model_client(
|
||||
temp.path().join("state"),
|
||||
id,
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let restored_jobs = restored.jobs();
|
||||
assert_eq!(restored_jobs.get("success").unwrap(), completed);
|
||||
assert!(restored_jobs.acknowledge("success").unwrap().acknowledged);
|
||||
restored.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_resource_limits_cancellation_and_wait_are_host_owned() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
let mut first = request("first");
|
||||
first.limits.max_concurrent_jobs = 1;
|
||||
jobs.request(first).unwrap();
|
||||
jobs.request(request("second")).unwrap();
|
||||
let stalled = ScriptedClient::stall();
|
||||
jobs.start_with_model_client("first", stalled.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_started(&stalled).await;
|
||||
// Waiting must not remove the Host-owned task/cancellation capability.
|
||||
let waiting_jobs = jobs.clone();
|
||||
let waiter = tokio::spawn(async move { waiting_jobs.wait("first").await });
|
||||
assert!(matches!(
|
||||
jobs.start_with_model_client("second", success()).await,
|
||||
Err(JobServiceError::Capacity)
|
||||
));
|
||||
jobs.cancel("first").unwrap();
|
||||
let cancelled = tokio::time::timeout(Duration::from_secs(10), waiter)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(cancelled.state, JobState::Cancelled);
|
||||
assert!(jobs.retry("first").is_err());
|
||||
jobs.start_with_model_client("second", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("second").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prose_failure_requires_explicit_bounded_new_attempt_and_exact_replay_does_not_retry() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
let intent = request("prose");
|
||||
jobs.request(intent.clone()).unwrap();
|
||||
jobs.start_with_model_client("prose", ScriptedClient::new(vec![prose()]))
|
||||
.await
|
||||
.unwrap();
|
||||
let failed = jobs.wait("prose").await.unwrap();
|
||||
assert_eq!(failed.state, JobState::Failed);
|
||||
assert!(failed.result.is_none());
|
||||
assert_eq!(jobs.request(intent).unwrap(), failed);
|
||||
let retry = jobs.retry("prose").unwrap();
|
||||
assert_eq!(retry.attempt.number, 2);
|
||||
assert_ne!(retry.attempt.attempt_id, failed.attempt.attempt_id);
|
||||
jobs.start_with_model_client("prose", ScriptedClient::new(vec![prose()]))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(jobs.wait("prose").await.unwrap().state, JobState::Failed);
|
||||
assert!(jobs.retry("prose").is_err());
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timeout_is_unknown_not_prose_success_or_automatic_retry() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
let mut intent = request("timeout");
|
||||
intent.limits.timeout_seconds = 1;
|
||||
jobs.request(intent).unwrap();
|
||||
let stalled = ScriptedClient::stall();
|
||||
jobs.start_with_model_client("timeout", stalled.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_started(&stalled).await;
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(10), jobs.wait("timeout"))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(outcome.state, JobState::Unknown);
|
||||
assert!(outcome.result.is_none());
|
||||
assert!(jobs.retry("timeout").is_err());
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_keeps_reserved_results_and_fences_running_attempts_on_reopen() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let id = host.worker_id();
|
||||
let jobs = host.jobs();
|
||||
jobs.request(request("reserved")).unwrap();
|
||||
jobs.request(request("cancelled")).unwrap();
|
||||
jobs.cancel("cancelled").unwrap();
|
||||
jobs.request(request("running")).unwrap();
|
||||
let stalled = ScriptedClient::stall();
|
||||
jobs.start_with_model_client("running", stalled.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_started(&stalled).await;
|
||||
host.shutdown().await.unwrap();
|
||||
assert!(matches!(jobs.get("running"), Err(JobServiceError::Closed)));
|
||||
let restored = StandaloneHost::restore_with_model_client(
|
||||
temp.path().join("state"),
|
||||
id,
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let jobs = restored.jobs();
|
||||
assert_eq!(
|
||||
jobs.pending()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|snapshot| snapshot.request.job_id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["reserved"]
|
||||
);
|
||||
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
|
||||
assert_eq!(jobs.get("running").unwrap().state, JobState::Unknown);
|
||||
assert!(jobs.retry("running").is_err());
|
||||
jobs.start_with_model_client("reserved", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("reserved").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
restored.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_requirements_are_rejected_without_model_execution_or_implicit_fallback() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
for profile in [
|
||||
"builtin:coder",
|
||||
"builtin:backend-job",
|
||||
"builtin:does-not-exist",
|
||||
] {
|
||||
let mut intent = request(profile);
|
||||
intent.profile = profile.into();
|
||||
jobs.request(intent).unwrap();
|
||||
let model = success();
|
||||
assert!(matches!(
|
||||
jobs.start_with_model_client(profile, model.clone()).await,
|
||||
Err(JobServiceError::Configuration(_))
|
||||
));
|
||||
assert!(model.requests().is_empty());
|
||||
assert_eq!(jobs.get(profile).unwrap().state, JobState::Pending);
|
||||
}
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
// Only this test harness starts a subprocess: production Jobs remain in-process.
|
||||
// The child exits deliberately without destructors to prove actual process-loss
|
||||
// recovery, rather than treating a hand-edited SQLite fixture as that evidence.
|
||||
#[tokio::test]
|
||||
#[ignore = "entry point for process_loss_recovers_durable_jobs_without_replaying_unknown"]
|
||||
async fn process_job_fixture() {
|
||||
let root = std::path::PathBuf::from(
|
||||
std::env::var_os("YOI_JOB_PROCESS_FIXTURE_DIR").expect("fixture root"),
|
||||
);
|
||||
let launch = StandaloneLaunchConfig::new(
|
||||
&root,
|
||||
root.join("state"),
|
||||
manifest::ProfileSelector::Default,
|
||||
"job-process-fixture",
|
||||
)
|
||||
.resolve()
|
||||
.unwrap();
|
||||
let host = StandaloneHost::start_with_model_client(launch, ScriptedClient::new(Vec::new()))
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::write(root.join("worker-id"), host.worker_id().to_string()).unwrap();
|
||||
let jobs = host.jobs();
|
||||
jobs.request(request("reserved")).unwrap();
|
||||
jobs.request(request("cancelled")).unwrap();
|
||||
jobs.cancel("cancelled").unwrap();
|
||||
jobs.request(request("accepted")).unwrap();
|
||||
jobs.start_with_model_client("accepted", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("accepted").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
jobs.request(request("interrupted")).unwrap();
|
||||
let stalled = ScriptedClient::stall();
|
||||
jobs.start_with_model_client("interrupted", stalled.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_started(&stalled).await;
|
||||
// Test-owned process only, never the operating Yoi Worker or dogfood daemon.
|
||||
std::process::exit(23);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn process_loss_recovers_durable_jobs_without_replaying_unknown() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let output = std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "process_job_fixture", "--ignored", "--nocapture"])
|
||||
.env("YOI_JOB_PROCESS_FIXTURE_DIR", temp.path())
|
||||
.output()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
output.status.code(),
|
||||
Some(23),
|
||||
"fixture failed: {}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let worker_id = std::fs::read_to_string(temp.path().join("worker-id"))
|
||||
.unwrap()
|
||||
.parse()
|
||||
.unwrap();
|
||||
let host = StandaloneHost::restore_with_model_client(
|
||||
temp.path().join("state"),
|
||||
worker_id,
|
||||
ScriptedClient::new(Vec::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let jobs = host.jobs();
|
||||
assert_eq!(jobs.get("interrupted").unwrap().state, JobState::Unknown);
|
||||
assert!(jobs.retry("interrupted").is_err());
|
||||
let completed = jobs.get("accepted").unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
assert_eq!(completed.result, Some(serde_json::json!({"valid":true})));
|
||||
assert!(!completed.acknowledged);
|
||||
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
|
||||
assert_eq!(jobs.pending().unwrap().len(), 1);
|
||||
jobs.start_with_model_client("reserved", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("reserved").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_failure_is_durable_known_failure_with_explicit_successful_retry() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let (host, _) = host(&temp).await;
|
||||
let jobs = host.jobs();
|
||||
jobs.request(request("model-failure")).unwrap();
|
||||
let client = ScriptedClient {
|
||||
fail: true,
|
||||
..ScriptedClient::new(Vec::new())
|
||||
};
|
||||
jobs.start_with_model_client("model-failure", client)
|
||||
.await
|
||||
.unwrap();
|
||||
let failure = jobs.wait("model-failure").await.unwrap();
|
||||
assert_eq!(failure.state, JobState::Failed);
|
||||
assert!(failure.result.is_none());
|
||||
assert!(
|
||||
failure
|
||||
.attempt
|
||||
.failure
|
||||
.unwrap()
|
||||
.contains("scripted model failure")
|
||||
);
|
||||
let next = jobs.retry("model-failure").unwrap();
|
||||
assert_eq!(next.attempt.number, 2);
|
||||
jobs.start_with_model_client("model-failure", success())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
jobs.wait("model-failure").await.unwrap().state,
|
||||
JobState::Completed
|
||||
);
|
||||
host.shutdown().await.unwrap();
|
||||
}
|
||||
@@ -16,6 +16,7 @@ agen = { workspace = true, features = ["codex"] }
|
||||
session-store = { workspace = true }
|
||||
secrets = { workspace = true }
|
||||
manifest = { workspace = true }
|
||||
job = { workspace = true }
|
||||
mcp = { workspace = true }
|
||||
protocol = { workspace = true, features = ["json-schema"] }
|
||||
client = { workspace = true }
|
||||
|
||||
@@ -205,6 +205,9 @@ where
|
||||
.copied()
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
worker
|
||||
.stop_feature_runtime("internal Worker feature installation failed")
|
||||
.await;
|
||||
return Err(InternalWorkerError {
|
||||
source: WorkerError::FeatureInstall(format!(
|
||||
"internal Worker feature installation failed: {diagnostics}; missing tools: {missing}"
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
//! Host-facing, result-only Jobs executed by the private Internal Worker substrate.
|
||||
//!
|
||||
//! The host owns persistence, concurrency, timeout, retries and result acceptance. Normal
|
||||
//! timeout/shutdown routes through the cancellation sender and awaits `run`, including Worker
|
||||
//! resource cleanup. An emergency drop/abort does not confirm asynchronous cleanup; the Host
|
||||
//! must retain its lease and reconcile any unknown outcome before allowing another execution.
|
||||
//! The result-only Feature owns no background tasks, but an in-flight synchronous Host sink
|
||||
//! cannot be preempted by dropping its tool future and must itself be bounded.
|
||||
|
||||
mod result;
|
||||
pub use result::{JobResultFeature, JobResultSink, SUBMIT_JOB_RESULT_TOOL};
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use agen::llm_client::LlmClient;
|
||||
use manifest::{ProfileDiscovery, ProfileResolveOptions, ProfileResolver, Scope, WorkerManifest};
|
||||
|
||||
use crate::feature::FeatureRegistryBuilder;
|
||||
use crate::internal_worker::{
|
||||
InternalWorkerAuthority, InternalWorkerIdentity, InternalWorkerSpec,
|
||||
run_internal_worker_with_cancel_sender,
|
||||
};
|
||||
use crate::{
|
||||
PromptCatalog, PromptCatalogSource, SystemPromptContext, SystemPromptTemplate,
|
||||
WorkerFilesystemAuthority, WorkerPrompt, WorkerRunResult, WorkerWorkspaceContext,
|
||||
};
|
||||
|
||||
/// Host adapter ceilings, applied even if the selected Profile leaves budgets unbounded.
|
||||
pub const MAX_JOB_TURNS: u32 = 32;
|
||||
pub const MAX_JOB_TOKENS: u32 = 8192;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum JobExecutionError {
|
||||
#[error(transparent)]
|
||||
Request(#[from] ::job::JobError),
|
||||
#[error("Job Profile resolution failed: {0}")]
|
||||
Profile(#[from] manifest::ProfileError),
|
||||
#[error("result-only Job cannot satisfy Profile requirements: {0}")]
|
||||
UnsupportedProfile(String),
|
||||
#[error("Job instruction resolution failed: {0}")]
|
||||
Instruction(#[from] crate::SystemPromptError),
|
||||
#[error("Job Prompt catalog resolution failed: {0}")]
|
||||
Catalog(#[from] crate::CatalogError),
|
||||
#[error("Job model configuration failed: {0}")]
|
||||
Model(#[from] crate::ProviderError),
|
||||
#[error("Job execution failed: {0}")]
|
||||
Worker(String),
|
||||
/// A sink error can follow durable acceptance. Hosts must reconcile their store and must
|
||||
/// not classify this as a known retryable failure unless they can prove non-acceptance.
|
||||
#[error(
|
||||
"Host Job result acceptance is unconfirmed; reconcile the Host store before retrying: {0}"
|
||||
)]
|
||||
ResultUnconfirmed(String),
|
||||
#[error("Job execution was cancelled without an accepted result")]
|
||||
Cancelled,
|
||||
#[error("Job ended without an accepted SubmitJobResult; final prose is not success")]
|
||||
MissingResult,
|
||||
}
|
||||
|
||||
/// Confirms only a synchronous, successful Host acceptance. Lifecycle/prose cannot create this.
|
||||
#[derive(Debug)]
|
||||
pub struct InternalJobResult {
|
||||
pub submission: ::job::JobResultSubmission,
|
||||
pub usage: Option<agen::timeline::event::UsageEvent>,
|
||||
}
|
||||
|
||||
/// Resolved policy/model/prompt, with no running resources and no inherited authority.
|
||||
pub struct PreparedInternalJob {
|
||||
request: ::job::JobRequest,
|
||||
manifest: WorkerManifest,
|
||||
client: Box<dyn LlmClient>,
|
||||
system_prompt: String,
|
||||
}
|
||||
|
||||
/// Resolve exactly the requested registry Profile using the existing user settings registry.
|
||||
/// `cwd` is a resolution base only; it never becomes filesystem or Workspace authority.
|
||||
/// An injected client is useful for scripted executions and does not bypass policy validation.
|
||||
pub fn prepare_job(
|
||||
request: &::job::JobRequest,
|
||||
cwd: &Path,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
) -> Result<PreparedInternalJob, JobExecutionError> {
|
||||
let registry = ProfileDiscovery::user_settings().discover()?;
|
||||
prepare_job_from_registry(request, cwd, client, ®istry)
|
||||
}
|
||||
|
||||
fn prepare_job_from_registry(
|
||||
request: &::job::JobRequest,
|
||||
cwd: &Path,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
registry: &manifest::ProfileRegistry,
|
||||
) -> Result<PreparedInternalJob, JobExecutionError> {
|
||||
request.validate()?;
|
||||
let resolved = ProfileResolver::new()
|
||||
.with_workspace_base(cwd)
|
||||
.resolve_from_registry(
|
||||
&request.profile_selector()?,
|
||||
registry,
|
||||
ProfileResolveOptions::with_worker_name("internal-job"),
|
||||
)?;
|
||||
let mut manifest = resolved.manifest;
|
||||
validate_result_only_profile(&manifest)?;
|
||||
let scope = Scope::empty();
|
||||
let prompts = PromptCatalog::builtins_only()?;
|
||||
let template = SystemPromptTemplate::parse(
|
||||
&manifest.engine.instruction,
|
||||
PromptCatalogSource::builtins_only(),
|
||||
)?;
|
||||
let mut system_prompt = template.render(&SystemPromptContext {
|
||||
now: chrono::Utc::now(),
|
||||
cwd: Cow::Borrowed("(no filesystem)"),
|
||||
language: &manifest.engine.language,
|
||||
scope: &scope,
|
||||
tool_names: vec![SUBMIT_JOB_RESULT_TOOL.into()],
|
||||
feature_instructions: &[],
|
||||
agents_md: None,
|
||||
resident_summary: None,
|
||||
prompts: &prompts,
|
||||
})?;
|
||||
system_prompt.push_str("\n\n");
|
||||
system_prompt.push_str(&prompts.render(WorkerPrompt::JobSystem, minijinja::context! {})?);
|
||||
// Host budgets attenuate Profile output/turn policy, never grant another capability.
|
||||
manifest.engine.max_turns = std::num::NonZeroU32::new(
|
||||
manifest
|
||||
.engine
|
||||
.max_turns
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(MAX_JOB_TURNS)
|
||||
.min(MAX_JOB_TURNS),
|
||||
);
|
||||
manifest.engine.max_tokens = Some(
|
||||
manifest
|
||||
.engine
|
||||
.max_tokens
|
||||
.unwrap_or(MAX_JOB_TOKENS)
|
||||
.min(MAX_JOB_TOKENS),
|
||||
);
|
||||
let model_config = manifest::model_catalog::resolve_model_manifest(&manifest.model)
|
||||
.map_err(crate::ProviderError::from)?;
|
||||
let client = match client {
|
||||
Some(client) => client,
|
||||
None => crate::model_client::build_client_from_config(&model_config)?,
|
||||
};
|
||||
Ok(PreparedInternalJob {
|
||||
request: request.clone(),
|
||||
manifest,
|
||||
client,
|
||||
system_prompt,
|
||||
})
|
||||
}
|
||||
|
||||
impl PreparedInternalJob {
|
||||
/// Run one immutable attempt. The sink must perform fenced, durable acceptance before
|
||||
/// returning Ok. Timeout is Host-owned: cancel and then await this future for cleanup.
|
||||
/// A result already accepted by the sink remains success even if the model subsequently
|
||||
/// fails, reaches its turn limit or is cancelled; callers never rerun an accepted result.
|
||||
pub async fn run<F>(
|
||||
self,
|
||||
attempt_id: String,
|
||||
submit: Arc<dyn JobResultSink>,
|
||||
on_cancel_sender: F,
|
||||
) -> Result<InternalJobResult, JobExecutionError>
|
||||
where
|
||||
F: FnOnce(tokio::sync::mpsc::Sender<()>),
|
||||
{
|
||||
let feature = JobResultFeature::new(&self.request, attempt_id.clone(), submit)?;
|
||||
let accepted = feature.accepted();
|
||||
let engine_policy = self.manifest.engine.clone();
|
||||
let spec = InternalWorkerSpec {
|
||||
identity: InternalWorkerIdentity {
|
||||
kind: "job",
|
||||
run_id: uuid::Uuid::now_v7(),
|
||||
},
|
||||
max_turns: engine_policy.max_turns.map(|n| n.get()),
|
||||
engine_configurator: Some(Box::new(move |engine| {
|
||||
crate::apply_worker_manifest(engine, &engine_policy);
|
||||
})),
|
||||
manifest: self.manifest,
|
||||
client: self.client,
|
||||
system_prompt: self.system_prompt,
|
||||
input: self.request.worker_input(&attempt_id)?,
|
||||
cache_key: None,
|
||||
features: FeatureRegistryBuilder::new().with_module(feature.clone()),
|
||||
required_tools: &[SUBMIT_JOB_RESULT_TOOL],
|
||||
authority: InternalWorkerAuthority {
|
||||
workspace: WorkerWorkspaceContext::no_workspace(),
|
||||
filesystem: WorkerFilesystemAuthority::None,
|
||||
scope: Scope::empty(),
|
||||
workdir_session: None,
|
||||
},
|
||||
};
|
||||
let outcome = run_internal_worker_with_cancel_sender(spec, on_cancel_sender).await;
|
||||
let usage = match &outcome {
|
||||
Ok(run) => run.usage.clone(),
|
||||
Err(error) => error.usage.clone(),
|
||||
};
|
||||
if let Some(submission) = accepted.lock().expect("Job result state poisoned").clone() {
|
||||
return Ok(InternalJobResult { submission, usage });
|
||||
}
|
||||
if let Some(error) = feature.unconfirmed() {
|
||||
return Err(JobExecutionError::ResultUnconfirmed(error));
|
||||
}
|
||||
match outcome {
|
||||
Err(error) => Err(JobExecutionError::Worker(error.source.to_string())),
|
||||
Ok(run)
|
||||
if matches!(
|
||||
run.lifecycle,
|
||||
WorkerRunResult::Cancelled | WorkerRunResult::RolledBack
|
||||
) =>
|
||||
{
|
||||
Err(JobExecutionError::Cancelled)
|
||||
}
|
||||
Ok(_) => Err(JobExecutionError::MissingResult),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate requirements, not Profile names. Nothing configured as active is silently stripped.
|
||||
fn validate_result_only_profile(manifest: &WorkerManifest) -> Result<(), JobExecutionError> {
|
||||
let manifest::FeatureConfig {
|
||||
task,
|
||||
memory,
|
||||
subjektiv,
|
||||
web,
|
||||
image,
|
||||
sub_worker,
|
||||
flow,
|
||||
worker,
|
||||
workspace_worker_discovery,
|
||||
objective,
|
||||
manage_workdir,
|
||||
workdir_catalog,
|
||||
workspace_config,
|
||||
ticket,
|
||||
merge_request,
|
||||
orchestration,
|
||||
} = &manifest.feature;
|
||||
let mut unavailable = Vec::new();
|
||||
for (name, enabled) in [
|
||||
("feature.task", task.enabled),
|
||||
(
|
||||
"feature.memory",
|
||||
memory.profile.enabled
|
||||
|| memory.profile.staging_tools
|
||||
|| memory.workspace_settings.is_some(),
|
||||
),
|
||||
(
|
||||
"feature.subjektiv",
|
||||
subjektiv.profile.enabled
|
||||
|| subjektiv.profile.consolidation_tools
|
||||
|| subjektiv.workspace_settings.is_some(),
|
||||
),
|
||||
("feature.web", web.enabled),
|
||||
("feature.image", image.enabled),
|
||||
("feature.sub_worker", sub_worker.enabled),
|
||||
("feature.flow", flow.enabled),
|
||||
("feature.worker", worker.enabled),
|
||||
(
|
||||
"feature.workspace_worker_discovery",
|
||||
workspace_worker_discovery.enabled,
|
||||
),
|
||||
("feature.objective", objective.enabled),
|
||||
("feature.manage_workdir", manage_workdir.enabled),
|
||||
("feature.workspace_config", workspace_config.enabled),
|
||||
(
|
||||
"feature.ticket",
|
||||
ticket.enabled || ticket.authoring || ticket.thread || ticket.intake || ticket.workflow,
|
||||
),
|
||||
(
|
||||
"feature.merge_request",
|
||||
merge_request.show
|
||||
|| merge_request.open
|
||||
|| merge_request.review
|
||||
|| merge_request.readiness_check
|
||||
|| merge_request.complete,
|
||||
),
|
||||
("feature.orchestration", orchestration.enabled),
|
||||
// The default WIP-only catalog is inert in Tools mode, not a tool requirement.
|
||||
(
|
||||
"feature.workdir_catalog",
|
||||
workdir_catalog.enabled && manifest.worker.mode == manifest::WorkerMode::Wip,
|
||||
),
|
||||
(
|
||||
"worker.mode=wip",
|
||||
manifest.worker.mode == manifest::WorkerMode::Wip,
|
||||
),
|
||||
("scope", !manifest.scope.allow.is_empty()),
|
||||
(
|
||||
"delegation_scope",
|
||||
!manifest.delegation_scope.allow.is_empty(),
|
||||
),
|
||||
("mcp", !manifest.mcp.stdio_servers.is_empty()),
|
||||
("skills", manifest.skills.is_some()),
|
||||
("compaction", manifest.compaction.is_some()),
|
||||
(
|
||||
"web",
|
||||
manifest
|
||||
.web
|
||||
.as_ref()
|
||||
.is_some_and(|config| config.enabled == Some(true)),
|
||||
),
|
||||
(
|
||||
"session.record_event_trace",
|
||||
manifest.session.record_event_trace,
|
||||
),
|
||||
] {
|
||||
if enabled {
|
||||
unavailable.push(name);
|
||||
}
|
||||
}
|
||||
if unavailable.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(JobExecutionError::UnsupportedProfile(
|
||||
unavailable.join(", "),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,179 @@
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use agen::tool::{Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
|
||||
use async_trait::async_trait;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::JobExecutionError;
|
||||
use crate::feature::{
|
||||
FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureModule, ToolContribution,
|
||||
ToolDeclaration,
|
||||
};
|
||||
|
||||
pub const SUBMIT_JOB_RESULT_TOOL: &str = "SubmitJobResult";
|
||||
const DESCRIPTION: &str = "Submit this Host Job's bounded structured result. Job, attempt and input revision are immutable Host bindings. Only result is accepted as input; final prose is not success.";
|
||||
|
||||
/// Host acceptance authority. Implementations must validate the immutable fences and commit
|
||||
/// acceptance durably before returning Ok. Err may mean an unknown outcome; identical retries
|
||||
/// must be idempotent. This synchronous callback must be bounded and must not wait for async work.
|
||||
pub trait JobResultSink: Send + Sync {
|
||||
fn submit(&self, submission: ::job::JobResultSubmission) -> Result<(), String>;
|
||||
}
|
||||
|
||||
#[derive(Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SubmitJobResultInput {
|
||||
result: Value,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SubmissionState {
|
||||
sealed: Option<Value>,
|
||||
unconfirmed: Option<String>,
|
||||
}
|
||||
|
||||
/// Single-purpose FeatureModule; Profile names never install it or confer result authority.
|
||||
#[derive(Clone)]
|
||||
pub struct JobResultFeature {
|
||||
job_id: String,
|
||||
attempt_id: String,
|
||||
input_revision: String,
|
||||
max_result_bytes: u32,
|
||||
sink: Arc<dyn JobResultSink>,
|
||||
state: Arc<Mutex<SubmissionState>>,
|
||||
accepted: Arc<Mutex<Option<::job::JobResultSubmission>>>,
|
||||
cancelled: Arc<Mutex<HashSet<String>>>,
|
||||
}
|
||||
|
||||
impl JobResultFeature {
|
||||
pub fn new(
|
||||
request: &::job::JobRequest,
|
||||
attempt_id: String,
|
||||
sink: Arc<dyn JobResultSink>,
|
||||
) -> Result<Self, JobExecutionError> {
|
||||
request.validate()?;
|
||||
::job::validate_identifier("attempt_id", &attempt_id, 512)?;
|
||||
Ok(Self {
|
||||
job_id: request.job_id.clone(),
|
||||
attempt_id,
|
||||
input_revision: request.input_revision.clone(),
|
||||
max_result_bytes: request.limits.max_result_bytes,
|
||||
sink,
|
||||
state: Arc::default(),
|
||||
accepted: Arc::default(),
|
||||
cancelled: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn accepted(&self) -> Arc<Mutex<Option<::job::JobResultSubmission>>> {
|
||||
self.accepted.clone()
|
||||
}
|
||||
|
||||
pub(super) fn unconfirmed(&self) -> Option<String> {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("Job submission state poisoned")
|
||||
.unconfirmed
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn definition(&self) -> ToolDefinition {
|
||||
let feature = self.clone();
|
||||
Arc::new(move || {
|
||||
(
|
||||
ToolMeta::new(SUBMIT_JOB_RESULT_TOOL)
|
||||
.description(DESCRIPTION)
|
||||
.input_schema(
|
||||
serde_json::to_value(schemars::schema_for!(SubmitJobResultInput))
|
||||
.expect("Job result schema serializes"),
|
||||
),
|
||||
Arc::new(feature.clone()) as Arc<dyn Tool>,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for JobResultFeature {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
context: ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let input: SubmitJobResultInput = serde_json::from_str(input_json)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
::job::result_digest(&input.result, self.max_result_bytes)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
let mut state = self.state.lock().expect("Job submission state poisoned");
|
||||
if self
|
||||
.cancelled
|
||||
.lock()
|
||||
.expect("Job cancellation state poisoned")
|
||||
.contains(&context.execution_id())
|
||||
{
|
||||
return Err(ToolError::Cancelled(ToolOutput {
|
||||
summary: "Job result submission cancelled".into(),
|
||||
content: None,
|
||||
attachments: vec![],
|
||||
}));
|
||||
}
|
||||
if let Some(sealed) = &state.sealed {
|
||||
if sealed != &input.result {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"Job result is sealed; retry only the identical result".into(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
// Seal before dispatch: a sink error can follow durable Host acceptance.
|
||||
state.sealed = Some(input.result.clone());
|
||||
}
|
||||
let submission = ::job::JobResultSubmission {
|
||||
job_id: self.job_id.clone(),
|
||||
attempt_id: self.attempt_id.clone(),
|
||||
input_revision: self.input_revision.clone(),
|
||||
result: input.result,
|
||||
};
|
||||
let mut accepted = self.accepted.lock().expect("Job result state poisoned");
|
||||
if accepted.is_none() {
|
||||
if let Err(error) = self.sink.submit(submission.clone()) {
|
||||
let error = ::job::bounded_failure_detail(&error);
|
||||
state.unconfirmed = Some(error.clone());
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Host Job result acceptance was not confirmed; retry only the identical result: {error}"
|
||||
)));
|
||||
}
|
||||
state.unconfirmed = None;
|
||||
*accepted = Some(submission);
|
||||
}
|
||||
Ok(ToolOutput {
|
||||
summary: "Host Job result accepted".into(),
|
||||
content: None,
|
||||
attachments: vec![],
|
||||
})
|
||||
}
|
||||
|
||||
async fn cancel_execution(&self, context: &ToolExecutionContext) -> Result<(), ToolError> {
|
||||
self.cancelled
|
||||
.lock()
|
||||
.expect("Job cancellation state poisoned")
|
||||
.insert(context.execution_id());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FeatureModule for JobResultFeature {
|
||||
fn descriptor(&self) -> FeatureDescriptor {
|
||||
FeatureDescriptor::builtin("host-job-result", "Host Job result")
|
||||
.with_description("Result-only immutable Host Job submission capability.")
|
||||
.with_tool(ToolDeclaration::new(SUBMIT_JOB_RESULT_TOOL, DESCRIPTION))
|
||||
}
|
||||
fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> {
|
||||
context.tools().register(ToolContribution::new(
|
||||
SUBMIT_JOB_RESULT_TOOL,
|
||||
self.definition(),
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,666 @@
|
||||
//! Scripted in-process model executions, not provider/process tests.
|
||||
use super::*;
|
||||
use agen::llm_client::event::{Event, ResponseStatus, StatusEvent};
|
||||
use agen::llm_client::{ClientError, Request};
|
||||
use agen::tool::{Tool, ToolExecutionContext};
|
||||
use async_trait::async_trait;
|
||||
use futures::Stream;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ScriptedClient {
|
||||
scripts: Arc<Mutex<VecDeque<Vec<Event>>>>,
|
||||
requests: Arc<Mutex<Vec<Request>>>,
|
||||
pending: bool,
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
fail: bool,
|
||||
}
|
||||
impl ScriptedClient {
|
||||
fn new(scripts: Vec<Vec<Event>>) -> Self {
|
||||
Self {
|
||||
scripts: Arc::new(Mutex::new(scripts.into())),
|
||||
requests: Arc::default(),
|
||||
pending: false,
|
||||
entered: Arc::default(),
|
||||
fail: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl LlmClient for ScriptedClient {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<Event, ClientError>> + Send>>, ClientError> {
|
||||
self.requests.lock().unwrap().push(request);
|
||||
if self.fail && self.scripts.lock().unwrap().is_empty() {
|
||||
return Err(ClientError::Config("scripted failure".into()));
|
||||
}
|
||||
if self.pending && self.scripts.lock().unwrap().is_empty() {
|
||||
self.entered.notify_one();
|
||||
return Ok(Box::pin(futures::stream::pending()));
|
||||
}
|
||||
let events = self
|
||||
.scripts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.expect("unexpected extra model turn");
|
||||
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
}
|
||||
#[derive(Default)]
|
||||
struct RecordingSink(Mutex<Vec<::job::JobResultSubmission>>);
|
||||
impl JobResultSink for RecordingSink {
|
||||
fn submit(&self, result: ::job::JobResultSubmission) -> Result<(), String> {
|
||||
self.0.lock().unwrap().push(result);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn prose() -> Vec<Event> {
|
||||
vec![
|
||||
Event::text_block_start(0),
|
||||
Event::text_delta(0, "success"),
|
||||
Event::text_block_stop(0, None),
|
||||
Event::Status(StatusEvent {
|
||||
status: ResponseStatus::Completed,
|
||||
}),
|
||||
]
|
||||
}
|
||||
fn call(name: &str, input: Value) -> Vec<Event> {
|
||||
vec![
|
||||
Event::tool_use_start(0, "call", name),
|
||||
Event::tool_input_delta(0, input.to_string()),
|
||||
Event::tool_use_stop(0),
|
||||
Event::Status(StatusEvent {
|
||||
status: ResponseStatus::Completed,
|
||||
}),
|
||||
]
|
||||
}
|
||||
fn request(profile: &str) -> ::job::JobRequest {
|
||||
::job::JobRequest {
|
||||
job_id: "job-a".into(),
|
||||
purpose: "scripted".into(),
|
||||
input_revision: "rev-1".into(),
|
||||
input_ref: "snapshot:one".into(),
|
||||
input: json!({"text":"input"}),
|
||||
instruction: "Summarize the input".into(),
|
||||
profile: profile.into(),
|
||||
serialization_key: None,
|
||||
limits: Default::default(),
|
||||
}
|
||||
}
|
||||
fn artifact(instruction: &str, tokens: u32, turns: u32) -> Value {
|
||||
json!({"model":{"scheme":"anthropic", "model_id":"scripted-model", "auth":{"kind":"none"}},
|
||||
"engine":{"instruction":instruction, "language":"English", "max_tokens":tokens, "max_turns":turns}})
|
||||
}
|
||||
fn registry(root: &Path, a: Value, b: Value) -> manifest::ProfileRegistry {
|
||||
std::fs::write(root.join("a.json"), a.to_string()).unwrap();
|
||||
std::fs::write(root.join("b.json"), b.to_string()).unwrap();
|
||||
let settings = root.join("profiles.toml");
|
||||
std::fs::write(
|
||||
&settings,
|
||||
"default = 'user:a'\n[profile]\na = 'a.json'\nb = 'b.json'\n",
|
||||
)
|
||||
.unwrap();
|
||||
ProfileDiscovery::with_sources(Some(settings), None)
|
||||
.discover()
|
||||
.unwrap()
|
||||
}
|
||||
fn prepared(
|
||||
root: &Path,
|
||||
request: &::job::JobRequest,
|
||||
client: ScriptedClient,
|
||||
) -> PreparedInternalJob {
|
||||
let registry = registry(
|
||||
root,
|
||||
artifact("default", 512, 3),
|
||||
artifact("internal.flow_verifier_system", 100000, 1000),
|
||||
);
|
||||
prepare_job_from_registry(request, root, Some(Box::new(client)), ®istry).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn executes_selected_profiles_with_only_result_authority_and_bounded_policy() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = registry(
|
||||
root.path(),
|
||||
artifact("default", 512, 3),
|
||||
artifact("internal.flow_verifier_system", 100000, 1000),
|
||||
);
|
||||
let contract = PromptCatalog::builtins_only()
|
||||
.unwrap()
|
||||
.render(WorkerPrompt::JobSystem, minijinja::context! {})
|
||||
.unwrap();
|
||||
let mut prompts = vec![];
|
||||
for (profile, tokens) in [
|
||||
("user:a", 512),
|
||||
("user:b", MAX_JOB_TOKENS),
|
||||
("default", 512),
|
||||
] {
|
||||
let client = ScriptedClient::new(vec![
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{"ok":true}})),
|
||||
prose(),
|
||||
]);
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let job = prepare_job_from_registry(
|
||||
&request(profile),
|
||||
root.path(),
|
||||
Some(Box::new(client.clone())),
|
||||
®istry,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
job.manifest.engine.max_turns.unwrap().get(),
|
||||
if profile == "user:b" {
|
||||
MAX_JOB_TURNS
|
||||
} else {
|
||||
3
|
||||
}
|
||||
);
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let result = job
|
||||
.run("attempt-7".into(), sink.clone(), move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
rx.await.unwrap().is_closed(),
|
||||
"Worker must be released before returning acceptance"
|
||||
);
|
||||
assert_eq!(
|
||||
result.submission,
|
||||
::job::JobResultSubmission {
|
||||
job_id: "job-a".into(),
|
||||
attempt_id: "attempt-7".into(),
|
||||
input_revision: "rev-1".into(),
|
||||
result: json!({"ok":true})
|
||||
}
|
||||
);
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 1);
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
for request in requests.iter() {
|
||||
assert_eq!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| tool.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[SUBMIT_JOB_RESULT_TOOL]
|
||||
);
|
||||
assert_eq!(request.config.max_tokens, Some(tokens));
|
||||
assert_eq!(
|
||||
request.tools[0].input_schema["properties"]
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.keys()
|
||||
.collect::<Vec<_>>(),
|
||||
["result"]
|
||||
);
|
||||
assert!(request.system_prompt.as_ref().unwrap().ends_with(&contract));
|
||||
}
|
||||
prompts.push(requests[0].system_prompt.clone());
|
||||
}
|
||||
assert_ne!(prompts[0], prompts[1]);
|
||||
assert_eq!(prompts[0], prompts[2]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn selected_builtin_job_executes_with_catalog_contract_and_only_result_tool() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = ProfileDiscovery::with_sources(None, None)
|
||||
.discover()
|
||||
.unwrap();
|
||||
let request = request("builtin:job");
|
||||
let client = ScriptedClient::new(vec![
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{"ok":true}})),
|
||||
prose(),
|
||||
]);
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let job = prepare_job_from_registry(
|
||||
&request,
|
||||
root.path(),
|
||||
Some(Box::new(client.clone())),
|
||||
®istry,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
job.manifest.engine.instruction,
|
||||
WorkerPrompt::JobSystem.key()
|
||||
);
|
||||
assert_eq!(
|
||||
job.manifest.model.ref_.as_deref(),
|
||||
Some("codex-oauth/gpt-5.6-luna")
|
||||
);
|
||||
assert_eq!(job.manifest.engine.max_turns.unwrap().get(), MAX_JOB_TURNS);
|
||||
assert!(job.manifest.scope.allow.is_empty());
|
||||
assert!(job.manifest.delegation_scope.allow.is_empty());
|
||||
assert!(WorkerPrompt::ALL.contains(&WorkerPrompt::JobSystem));
|
||||
let contract = PromptCatalog::builtins_only()
|
||||
.unwrap()
|
||||
.render(WorkerPrompt::JobSystem, minijinja::context! {})
|
||||
.unwrap();
|
||||
assert!(job.system_prompt.ends_with(&contract));
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let result = job
|
||||
.run("builtin-attempt".into(), sink.clone(), move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(rx.await.unwrap().is_closed());
|
||||
assert_eq!(result.submission.job_id, request.job_id);
|
||||
assert_eq!(result.submission.attempt_id, "builtin-attempt");
|
||||
assert_eq!(result.submission.input_revision, request.input_revision);
|
||||
assert_eq!(result.submission.result, json!({"ok":true}));
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 1);
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
for request in requests.iter() {
|
||||
assert_eq!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.map(|tool| tool.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[SUBMIT_JOB_RESULT_TOOL]
|
||||
);
|
||||
assert_eq!(request.config.max_tokens, Some(MAX_JOB_TOKENS));
|
||||
assert!(request.system_prompt.as_ref().unwrap().ends_with(&contract));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn final_prose_and_ungranted_tool_calls_are_not_success() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for scripts in [
|
||||
vec![prose()],
|
||||
vec![
|
||||
call("Bash", json!({"command":"touch should-not-exist"})),
|
||||
prose(),
|
||||
],
|
||||
vec![
|
||||
call("SubmitBackendJobResult", json!({"result":{"ok":true}})),
|
||||
prose(),
|
||||
],
|
||||
vec![
|
||||
call(
|
||||
SUBMIT_JOB_RESULT_TOOL,
|
||||
json!({"job_id":"forged","result":{"ok":true}}),
|
||||
),
|
||||
prose(),
|
||||
],
|
||||
] {
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let outcome = prepared(
|
||||
root.path(),
|
||||
&request("user:a"),
|
||||
ScriptedClient::new(scripts),
|
||||
)
|
||||
.run("attempt".into(), sink.clone(), |_| {})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(outcome, Err(JobExecutionError::MissingResult)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert!(sink.0.lock().unwrap().is_empty());
|
||||
assert!(!root.path().join("should-not-exist").exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn result_size_and_identity_are_fenced_before_host_submission() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut request = request("user:a");
|
||||
request.limits.max_result_bytes = 5;
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let job = prepared(
|
||||
root.path(),
|
||||
&request,
|
||||
ScriptedClient::new(vec![
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":"too big"})),
|
||||
prose(),
|
||||
]),
|
||||
);
|
||||
assert!(matches!(
|
||||
job.run("attempt".into(), sink.clone(), |_| {}).await,
|
||||
Err(JobExecutionError::MissingResult)
|
||||
));
|
||||
assert!(sink.0.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_and_errors_return_without_acceptance() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut client = ScriptedClient::new(vec![]);
|
||||
client.pending = true;
|
||||
let entered = client.entered.clone();
|
||||
let job = prepared(root.path(), &request("user:a"), client);
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let run = tokio::spawn(async move {
|
||||
job.run("attempt".into(), sink, move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
})
|
||||
.await
|
||||
});
|
||||
let sender = rx.await.unwrap();
|
||||
entered.notified().await;
|
||||
sender.send(()).await.unwrap();
|
||||
assert!(matches!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), run)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Err(JobExecutionError::Cancelled)
|
||||
));
|
||||
assert!(
|
||||
sender.is_closed(),
|
||||
"normal Worker release must close cancellation receiver"
|
||||
);
|
||||
let mut client = ScriptedClient::new(vec![]);
|
||||
client.fail = true;
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let outcome = prepared(root.path(), &request("user:a"), client)
|
||||
.run(
|
||||
"attempt".into(),
|
||||
Arc::new(RecordingSink::default()),
|
||||
move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, Err(JobExecutionError::Worker(_))));
|
||||
assert!(
|
||||
rx.await.unwrap().is_closed(),
|
||||
"error must release the Worker"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_result_survives_later_provider_error_and_cancellation() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for cancel in [false, true] {
|
||||
let mut client =
|
||||
ScriptedClient::new(vec![call(SUBMIT_JOB_RESULT_TOOL, json!({"result":true}))]);
|
||||
client.pending = cancel;
|
||||
client.fail = !cancel;
|
||||
let entered = client.entered.clone();
|
||||
let sink = Arc::new(RecordingSink::default());
|
||||
let job = prepared(root.path(), &request("user:a"), client);
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
let recording = sink.clone();
|
||||
let run = tokio::spawn(async move {
|
||||
job.run("attempt".into(), recording, move |sender| {
|
||||
let _ = tx.send(sender);
|
||||
})
|
||||
.await
|
||||
});
|
||||
let sender = rx.await.unwrap();
|
||||
if cancel {
|
||||
entered.notified().await;
|
||||
assert_eq!(
|
||||
sink.0.lock().unwrap().len(),
|
||||
1,
|
||||
"acceptance must precede cancellation"
|
||||
);
|
||||
sender.send(()).await.unwrap();
|
||||
}
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(2), run)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(result.submission.result, json!(true));
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 1);
|
||||
assert!(sender.is_closed());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_limit_is_bounded_and_accepted_result_survives_later_failure() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = registry(
|
||||
root.path(),
|
||||
artifact("default", 512, 1),
|
||||
artifact("default", 512, 1),
|
||||
);
|
||||
for (tool, expected) in [("UnknownTool", false), (SUBMIT_JOB_RESULT_TOOL, true)] {
|
||||
let client = ScriptedClient::new(vec![call(tool, json!({"result":true}))]);
|
||||
let prepared = prepare_job_from_registry(
|
||||
&request("user:a"),
|
||||
root.path(),
|
||||
Some(Box::new(client.clone())),
|
||||
®istry,
|
||||
)
|
||||
.unwrap();
|
||||
let outcome = prepared
|
||||
.run("attempt".into(), Arc::new(RecordingSink::default()), |_| {})
|
||||
.await;
|
||||
assert_eq!(outcome.is_ok(), expected);
|
||||
assert_eq!(client.requests.lock().unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profiles_with_unavailable_requirements_are_rejected_without_name_allowlist_or_fallback() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let base = artifact("default", 512, 3);
|
||||
for (key, value, needle) in [
|
||||
("feature", json!({"task":{"enabled":true}}), "feature.task"),
|
||||
(
|
||||
"feature",
|
||||
json!({"workspace_config":{"enabled":true}}),
|
||||
"feature.workspace_config",
|
||||
),
|
||||
(
|
||||
"feature",
|
||||
json!({"subjektiv":{"consolidation_tools":true}}),
|
||||
"feature.subjektiv",
|
||||
),
|
||||
(
|
||||
"feature",
|
||||
json!({"memory":{"staging_tools":true}}),
|
||||
"feature.memory",
|
||||
),
|
||||
("scope", json!("workspace_read"), "scope"),
|
||||
(
|
||||
"delegation_scope",
|
||||
json!("workspace_write"),
|
||||
"delegation_scope",
|
||||
),
|
||||
(
|
||||
"session",
|
||||
json!({"record_event_trace":true}),
|
||||
"session.record_event_trace",
|
||||
),
|
||||
("web", json!({"enabled":true}), "web"),
|
||||
(
|
||||
"compaction",
|
||||
json!({"kind":"tokens","threshold":10000}),
|
||||
"compaction",
|
||||
),
|
||||
("worker", json!({"mode":"wip"}), "worker.mode=wip"),
|
||||
("skills", json!({"directories":["skills"]}), "skills"),
|
||||
(
|
||||
"mcp",
|
||||
json!({"stdio_server":[{"name":"test","command":"fake"}]}),
|
||||
"mcp",
|
||||
),
|
||||
] {
|
||||
let mut profile = base.clone();
|
||||
profile[key] = value;
|
||||
let registry = registry(root.path(), profile, base.clone());
|
||||
let error = prepare_job_from_registry(
|
||||
&request("user:a"),
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry,
|
||||
)
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(error, JobExecutionError::UnsupportedProfile(_)),
|
||||
"{error}"
|
||||
);
|
||||
assert!(error.to_string().contains(needle), "{error}");
|
||||
}
|
||||
for name in [
|
||||
"web",
|
||||
"image",
|
||||
"sub_worker",
|
||||
"flow",
|
||||
"worker",
|
||||
"workspace_worker_discovery",
|
||||
"objective",
|
||||
"manage_workdir",
|
||||
"ticket",
|
||||
"orchestration",
|
||||
] {
|
||||
let mut profile = base.clone();
|
||||
profile["feature"] = json!({name:{"enabled":true}});
|
||||
let registry = registry(root.path(), profile, base.clone());
|
||||
assert!(matches!(
|
||||
prepare_job_from_registry(
|
||||
&request("user:a"),
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry
|
||||
),
|
||||
Err(JobExecutionError::UnsupportedProfile(_))
|
||||
));
|
||||
}
|
||||
let registry = registry(root.path(), base.clone(), base);
|
||||
for selector in ["user:missing", "project:a", "./a.json", "inherit"] {
|
||||
assert!(
|
||||
prepare_job_from_registry(
|
||||
&request(selector),
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let builtin = request("builtin:backend-job");
|
||||
assert!(matches!(
|
||||
prepare_job_from_registry(
|
||||
&builtin,
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry
|
||||
),
|
||||
Err(JobExecutionError::UnsupportedProfile(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_prompt_and_model_are_not_substituted_even_with_injected_client() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut unknown_model = artifact("default", 512, 3);
|
||||
unknown_model["model"] = json!({"ref":"not-a-provider/not-a-model"});
|
||||
let registry = registry(
|
||||
root.path(),
|
||||
artifact("internal.missing", 512, 3),
|
||||
unknown_model,
|
||||
);
|
||||
for (profile, prompt_error) in [("user:a", true), ("user:b", false)] {
|
||||
let error = prepare_job_from_registry(
|
||||
&request(profile),
|
||||
root.path(),
|
||||
Some(Box::new(ScriptedClient::new(vec![]))),
|
||||
®istry,
|
||||
)
|
||||
.err()
|
||||
.unwrap();
|
||||
if prompt_error {
|
||||
assert!(
|
||||
matches!(error, JobExecutionError::Instruction(_)),
|
||||
"{error}"
|
||||
);
|
||||
} else {
|
||||
assert!(matches!(error, JobExecutionError::Model(_)), "{error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct UnknownSink(Mutex<Vec<::job::JobResultSubmission>>);
|
||||
impl JobResultSink for UnknownSink {
|
||||
fn submit(&self, value: ::job::JobResultSubmission) -> Result<(), String> {
|
||||
self.0.lock().unwrap().push(value);
|
||||
Err("unknown".into())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unconfirmed_sink_acceptance_is_not_classified_as_missing_result_or_known_failure() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let sink = Arc::new(UnknownSink(Mutex::new(vec![])));
|
||||
let job = prepared(
|
||||
root.path(),
|
||||
&request("user:a"),
|
||||
ScriptedClient::new(vec![
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":true})),
|
||||
prose(),
|
||||
]),
|
||||
);
|
||||
assert!(matches!(
|
||||
job.run("attempt".into(), sink.clone(), |_| {}).await,
|
||||
Err(JobExecutionError::ResultUnconfirmed(_))
|
||||
));
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feature_registration_rejects_forged_fences_seals_unknown_outcomes_and_honors_cancel() {
|
||||
let sink = Arc::new(UnknownSink(Mutex::new(vec![])));
|
||||
let feature =
|
||||
JobResultFeature::new(&request("user:a"), "attempt".into(), sink.clone()).unwrap();
|
||||
let mut pending = vec![];
|
||||
let mut hooks = crate::HookRegistryBuilder::new();
|
||||
let report = FeatureRegistryBuilder::new()
|
||||
.with_module(feature.clone())
|
||||
.install_into_pending(&mut pending, &mut hooks);
|
||||
assert_eq!(report.installed_tool_names(), [SUBMIT_JOB_RESULT_TOOL]);
|
||||
let (_, tool) = pending.remove(0)();
|
||||
assert!(
|
||||
tool.execute(
|
||||
r#"{"attempt_id":"forged","result":true}"#,
|
||||
ToolExecutionContext::direct()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(sink.0.lock().unwrap().is_empty());
|
||||
assert!(
|
||||
tool.execute(r#"{"result":true}"#, ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
tool.execute(r#"{"result":false}"#, ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
tool.execute(r#"{"result":true}"#, ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 2);
|
||||
let context = ToolExecutionContext::new("cancelled", "batch", 0);
|
||||
feature.cancel_execution(&context).await.unwrap();
|
||||
assert!(matches!(
|
||||
tool.execute(r#"{"result":true}"#, context).await,
|
||||
Err(agen::tool::ToolError::Cancelled(_))
|
||||
));
|
||||
assert_eq!(sink.0.lock().unwrap().len(), 2);
|
||||
}
|
||||
@@ -12,6 +12,7 @@ pub mod fs_view;
|
||||
pub mod hook;
|
||||
pub(crate) mod in_flight;
|
||||
pub mod ipc;
|
||||
pub mod job;
|
||||
pub mod model_client;
|
||||
mod paste_artifact_tool;
|
||||
pub mod prompt;
|
||||
|
||||
@@ -97,6 +97,7 @@ pub enum WorkerPrompt {
|
||||
SubjektivMemoryConsolidationSystem,
|
||||
SubjektivMemorySurfaceSystem,
|
||||
FlowVerifierSystem,
|
||||
JobSystem,
|
||||
BackendJobSystem,
|
||||
NotifyWrapper,
|
||||
InterruptToolResultSummary,
|
||||
@@ -121,6 +122,7 @@ impl WorkerPrompt {
|
||||
}
|
||||
Self::SubjektivMemorySurfaceSystem => "internal.subjektiv_memory_surface_system",
|
||||
Self::FlowVerifierSystem => "internal.flow_verifier_system",
|
||||
Self::JobSystem => "internal.job_system",
|
||||
Self::BackendJobSystem => "internal.backend_job_system",
|
||||
Self::NotifyWrapper => "internal.notify_wrapper",
|
||||
Self::InterruptToolResultSummary => "internal.interrupt_tool_result_summary",
|
||||
@@ -144,6 +146,7 @@ impl WorkerPrompt {
|
||||
WorkerPrompt::SubjektivMemoryConsolidationSystem,
|
||||
WorkerPrompt::SubjektivMemorySurfaceSystem,
|
||||
WorkerPrompt::FlowVerifierSystem,
|
||||
WorkerPrompt::JobSystem,
|
||||
WorkerPrompt::BackendJobSystem,
|
||||
WorkerPrompt::NotifyWrapper,
|
||||
WorkerPrompt::InterruptToolResultSummary,
|
||||
|
||||
@@ -18,6 +18,7 @@ futures.workspace = true
|
||||
config-source.workspace = true
|
||||
flow = { path = "../flow" }
|
||||
manifest.workspace = true
|
||||
job.workspace = true
|
||||
protocol = { workspace = true }
|
||||
runtime-api.workspace = true
|
||||
server-api.workspace = true
|
||||
|
||||
@@ -7,26 +7,28 @@
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
use worker_runtime::identity::RuntimeWorkerRef;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
pub const BACKEND_JOB_RUNTIME_ID: &str = crate::hosts::EMBEDDED_RUNTIME_ID;
|
||||
pub const DEFAULT_MAX_CONCURRENT_JOBS: u16 = 8;
|
||||
pub const ABSOLUTE_MAX_CONCURRENT_JOBS: u16 = 64;
|
||||
pub const DEFAULT_JOB_TIMEOUT_SECONDS: u32 = 120;
|
||||
pub const MAX_JOB_TIMEOUT_SECONDS: u32 = 600;
|
||||
pub const DEFAULT_MAX_RESULT_BYTES: u32 = 16 * 1024;
|
||||
pub const ABSOLUTE_MAX_RESULT_BYTES: u32 = 64 * 1024;
|
||||
pub const DEFAULT_MAX_ATTEMPTS: u8 = 2;
|
||||
pub const ABSOLUTE_MAX_ATTEMPTS: u8 = 3;
|
||||
pub const MAX_JOB_INPUT_BYTES: usize = 128 * 1024;
|
||||
pub const MAX_JOB_INSTRUCTION_BYTES: usize = 16 * 1024;
|
||||
pub const MAX_JOB_PURPOSE_BYTES: usize = 120;
|
||||
pub const MAX_JOB_REFERENCE_BYTES: usize = 512;
|
||||
pub const MAX_JOB_FAILURE_BYTES: usize = 1024;
|
||||
pub const MAX_JOB_DELIVERY_BYTES: usize = 16 * 1024;
|
||||
pub use job::{
|
||||
ABSOLUTE_MAX_ATTEMPTS, ABSOLUTE_MAX_CONCURRENT_JOBS, ABSOLUTE_MAX_RESULT_BYTES,
|
||||
DEFAULT_JOB_TIMEOUT_SECONDS, DEFAULT_MAX_ATTEMPTS, DEFAULT_MAX_CONCURRENT_JOBS,
|
||||
DEFAULT_MAX_RESULT_BYTES, JobResultSubmission as BackendJobResultSubmission,
|
||||
MAX_JOB_DELIVERY_BYTES, MAX_JOB_FAILURE_BYTES, MAX_JOB_INPUT_BYTES, MAX_JOB_INSTRUCTION_BYTES,
|
||||
MAX_JOB_PURPOSE_BYTES, MAX_JOB_REFERENCE_BYTES, MAX_JOB_TIMEOUT_SECONDS, attempt_id,
|
||||
bounded_failure_detail,
|
||||
};
|
||||
|
||||
// Preserve the Backend InvalidInput surface and its exact historical messages.
|
||||
fn backend_error(error: job::JobError) -> Error {
|
||||
match error {
|
||||
job::JobError::InvalidInput(detail) => {
|
||||
Error::InvalidInput(detail.replacen("Job ", "Backend Job ", 1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -37,6 +39,17 @@ pub enum BackendJobState {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl From<BackendJobState> for job::JobState {
|
||||
fn from(state: BackendJobState) -> Self {
|
||||
match state {
|
||||
BackendJobState::Pending => Self::Pending,
|
||||
BackendJobState::Completed => Self::Completed,
|
||||
BackendJobState::Failed => Self::Failed,
|
||||
BackendJobState::Unknown => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendJobState {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
@@ -69,6 +82,19 @@ pub enum BackendJobAttemptState {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl From<BackendJobAttemptState> for job::JobAttemptState {
|
||||
fn from(state: BackendJobAttemptState) -> Self {
|
||||
match state {
|
||||
BackendJobAttemptState::Reserved => Self::Reserved,
|
||||
BackendJobAttemptState::Dispatching => Self::Dispatching,
|
||||
BackendJobAttemptState::Dispatched => Self::Dispatched,
|
||||
BackendJobAttemptState::Completed => Self::Completed,
|
||||
BackendJobAttemptState::Failed => Self::Failed,
|
||||
BackendJobAttemptState::Unknown => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendJobAttemptState {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
@@ -161,28 +187,29 @@ impl Default for BackendJobLimits {
|
||||
|
||||
impl BackendJobLimits {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.max_concurrent_jobs == 0 || self.max_concurrent_jobs > ABSOLUTE_MAX_CONCURRENT_JOBS
|
||||
{
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_concurrent_jobs must be in 1..={ABSOLUTE_MAX_CONCURRENT_JOBS}"
|
||||
)));
|
||||
job::JobLimits::from(self).validate().map_err(backend_error)
|
||||
}
|
||||
if self.timeout_seconds == 0 || self.timeout_seconds > MAX_JOB_TIMEOUT_SECONDS {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job timeout_seconds must be in 1..={MAX_JOB_TIMEOUT_SECONDS}"
|
||||
)));
|
||||
}
|
||||
if self.max_result_bytes == 0 || self.max_result_bytes > ABSOLUTE_MAX_RESULT_BYTES {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_result_bytes must be in 1..={ABSOLUTE_MAX_RESULT_BYTES}"
|
||||
)));
|
||||
|
||||
impl From<&BackendJobLimits> for job::JobLimits {
|
||||
fn from(limits: &BackendJobLimits) -> Self {
|
||||
Self {
|
||||
max_concurrent_jobs: limits.max_concurrent_jobs,
|
||||
timeout_seconds: limits.timeout_seconds,
|
||||
max_result_bytes: limits.max_result_bytes,
|
||||
max_attempts: limits.max_attempts,
|
||||
}
|
||||
if self.max_attempts == 0 || self.max_attempts > ABSOLUTE_MAX_ATTEMPTS {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job max_attempts must be in 1..={ABSOLUTE_MAX_ATTEMPTS}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl From<job::JobLimits> for BackendJobLimits {
|
||||
fn from(limits: job::JobLimits) -> Self {
|
||||
Self {
|
||||
max_concurrent_jobs: limits.max_concurrent_jobs,
|
||||
timeout_seconds: limits.timeout_seconds,
|
||||
max_result_bytes: limits.max_result_bytes,
|
||||
max_attempts: limits.max_attempts,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -261,17 +288,29 @@ pub struct BackendJobRequest {
|
||||
}
|
||||
|
||||
impl BackendJobRequest {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
validate_identifier("job_id", &self.job_id, 256)?;
|
||||
validate_identifier("purpose", &self.purpose, MAX_JOB_PURPOSE_BYTES)?;
|
||||
validate_identifier("input_revision", &self.input_revision, 256)?;
|
||||
validate_bounded_text("input_ref", &self.input_ref, MAX_JOB_REFERENCE_BYTES)?;
|
||||
validate_bounded_text("instruction", &self.instruction, MAX_JOB_INSTRUCTION_BYTES)?;
|
||||
validate_identifier("profile", &self.profile, 256)?;
|
||||
self.profile_selector()?;
|
||||
if let Some(key) = &self.serialization_key {
|
||||
validate_identifier("serialization_key", key, MAX_JOB_REFERENCE_BYTES)?;
|
||||
/// Adapt neutral intent without granting domain access or writer/notification
|
||||
/// authority. Validation remains the Backend submission boundary's job;
|
||||
/// the caller-selected profile is retained exactly, with no fallback.
|
||||
pub fn from_job_request(request: job::JobRequest) -> Self {
|
||||
Self {
|
||||
job_id: request.job_id,
|
||||
purpose: request.purpose,
|
||||
input_revision: request.input_revision,
|
||||
input_ref: request.input_ref,
|
||||
input: request.input,
|
||||
instruction: request.instruction,
|
||||
profile: request.profile,
|
||||
grants: BackendJobGrants::default(),
|
||||
serialization_key: request.serialization_key,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: request.limits.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
let request = self.to_job_request();
|
||||
request.validate_fields().map_err(backend_error)?;
|
||||
if let Some(grant) = &self.grants.subjektiv_consolidation {
|
||||
validate_identifier("grant subject_id", &grant.subject_id, 256)?;
|
||||
if grant.candidate_ids.len() > 256 {
|
||||
@@ -297,44 +336,30 @@ impl BackendJobRequest {
|
||||
));
|
||||
}
|
||||
}
|
||||
let input = serde_json::to_vec(&self.input).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job input: {error}"))
|
||||
})?;
|
||||
if input.len() > MAX_JOB_INPUT_BYTES {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job input exceeds {MAX_JOB_INPUT_BYTES} bytes"
|
||||
)));
|
||||
}
|
||||
request.validate_input().map_err(backend_error)?;
|
||||
self.limits.validate()
|
||||
}
|
||||
|
||||
/// Transport-neutral projection only. Domain grants, writer authority and
|
||||
/// derived resource locks remain Backend-owned; never fingerprint this
|
||||
/// projection as a substitute for the full Backend intent.
|
||||
pub fn to_job_request(&self) -> job::JobRequest {
|
||||
job::JobRequest {
|
||||
job_id: self.job_id.clone(),
|
||||
purpose: self.purpose.clone(),
|
||||
input_revision: self.input_revision.clone(),
|
||||
input_ref: self.input_ref.clone(),
|
||||
input: self.input.clone(),
|
||||
instruction: self.instruction.clone(),
|
||||
profile: self.profile.clone(),
|
||||
serialization_key: self.serialization_key.clone(),
|
||||
limits: (&self.limits).into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate selector syntax only. Registry existence and resolution are dispatch authority.
|
||||
pub fn profile_selector(&self) -> Result<manifest::ProfileSelector> {
|
||||
use manifest::ProfileSelector;
|
||||
validate_identifier("profile", &self.profile, 256)?;
|
||||
let selector = ProfileSelector::parse_cli(&self.profile);
|
||||
let valid = match &selector {
|
||||
ProfileSelector::Default => true,
|
||||
ProfileSelector::Path { .. } => false,
|
||||
ProfileSelector::Named { source, name } => {
|
||||
!name.is_empty()
|
||||
&& !name.starts_with('.')
|
||||
&& !name.contains(['/', '\\', ':'])
|
||||
&& name
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.'))
|
||||
&& ![".toml", ".json", ".dcdl", ".nix"]
|
||||
.iter()
|
||||
.any(|suffix| name.ends_with(suffix))
|
||||
&& (source.is_some() || name != "inherit")
|
||||
}
|
||||
};
|
||||
if !valid {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Job profile must be a manifest registry selector, not raw source or a path".into(),
|
||||
));
|
||||
}
|
||||
Ok(selector)
|
||||
job::profile_selector(&self.profile).map_err(backend_error)
|
||||
}
|
||||
|
||||
/// Effective single-flight key. Typed grants cannot bypass their subject fence.
|
||||
@@ -348,20 +373,12 @@ impl BackendJobRequest {
|
||||
|
||||
pub fn fingerprint(&self) -> Result<String> {
|
||||
self.validate()?;
|
||||
let encoded = serde_json::to_vec(self).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job intent: {error}"))
|
||||
})?;
|
||||
Ok(sha256(&encoded))
|
||||
job::fingerprint(self).map_err(backend_error)
|
||||
}
|
||||
|
||||
pub fn worker_input(&self, attempt_id: &str) -> Result<String> {
|
||||
let input = serde_json::to_string(&self.input).map_err(|error| {
|
||||
Error::InvalidInput(format!("serialize Backend Job input: {error}"))
|
||||
})?;
|
||||
Ok(format!(
|
||||
"{}\n\nBackend Job envelope (immutable):\njob_id: {}\nattempt_id: {}\ninput_revision: {}\ninput_ref: {}\ninput_json: {}\n\nReturn success only through the structured Backend Job result capability. Final prose and Worker Idle/Stopped state are not result authority.",
|
||||
self.instruction, self.job_id, attempt_id, self.input_revision, self.input_ref, input
|
||||
))
|
||||
job::worker_input_with_label(&self.to_job_request(), attempt_id, "Backend Job")
|
||||
.map_err(backend_error)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -451,13 +468,10 @@ pub struct BackendJobReservation {
|
||||
pub resource_reused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BackendJobResultSubmission {
|
||||
pub job_id: String,
|
||||
pub attempt_id: String,
|
||||
pub input_revision: String,
|
||||
pub result: Value,
|
||||
impl BackendJobReservation {
|
||||
pub fn outcome(&self) -> job::JobOutcome {
|
||||
job_outcome(&self.job, &self.attempt)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
@@ -467,6 +481,28 @@ pub struct BackendJobResultAcceptance {
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
impl BackendJobResultAcceptance {
|
||||
pub fn outcome(&self) -> job::JobOutcome {
|
||||
job_outcome(&self.job, &self.attempt)
|
||||
}
|
||||
}
|
||||
|
||||
// Project the durable Job outcome, not Worker lifecycle, cleanup or delivery.
|
||||
// The paired attempt supplies its own identity, number and execution state.
|
||||
fn job_outcome(job: &BackendJobRecord, attempt: &BackendJobAttemptRecord) -> job::JobOutcome {
|
||||
job::JobOutcome {
|
||||
job_id: job.request.job_id.clone(),
|
||||
input_revision: job.request.input_revision.clone(),
|
||||
attempt_id: attempt.attempt_id.clone(),
|
||||
attempt: attempt.attempt,
|
||||
state: job.state.into(),
|
||||
attempt_state: attempt.state.into(),
|
||||
result: job.result.clone(),
|
||||
failure_category: job.failure_category.clone(),
|
||||
failure_detail: job.failure_detail.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BackendJobWorkerBinding {
|
||||
pub job_id: String,
|
||||
@@ -489,10 +525,6 @@ pub struct BackendJobDeliveryRecord {
|
||||
pub delivered_at: Option<String>,
|
||||
}
|
||||
|
||||
pub fn attempt_id(job_id: &str, attempt: u8) -> String {
|
||||
format!("{job_id}:attempt:{attempt}")
|
||||
}
|
||||
|
||||
pub fn allocation_key(attempt_id: &str) -> String {
|
||||
format!("backend-job:{attempt_id}")
|
||||
}
|
||||
@@ -515,64 +547,15 @@ pub fn delivery_id(job_id: &str, attempt_id: &str) -> String {
|
||||
}
|
||||
|
||||
fn tracked_request_id(kind: &str, identity: &[u8]) -> String {
|
||||
format!("backend-job-{kind}:{}", sha256(identity))
|
||||
format!("backend-job-{kind}:{}", job::sha256(identity))
|
||||
}
|
||||
|
||||
pub fn result_digest(result: &Value, max_bytes: u32) -> Result<(String, String)> {
|
||||
let encoded = serde_json::to_string(result)
|
||||
.map_err(|error| Error::InvalidInput(format!("serialize Backend Job result: {error}")))?;
|
||||
if encoded.len() > max_bytes as usize {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job result exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok((sha256(encoded.as_bytes()), encoded))
|
||||
}
|
||||
|
||||
pub(crate) fn bounded_failure_detail(detail: &str) -> String {
|
||||
if detail.len() <= MAX_JOB_FAILURE_BYTES {
|
||||
return detail.to_string();
|
||||
}
|
||||
let mut end = MAX_JOB_FAILURE_BYTES;
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
detail[..end].to_string()
|
||||
job::result_digest(result, max_bytes).map_err(backend_error)
|
||||
}
|
||||
|
||||
fn validate_identifier(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
validate_bounded_text(field, value, max_bytes)?;
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} must not contain control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bounded_text(field: &str, value: &str, max_bytes: usize) -> Result<()> {
|
||||
if value.is_empty() || value.trim() != value {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} must be non-empty and trimmed"
|
||||
)));
|
||||
}
|
||||
if value.len() > max_bytes {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job {field} exceeds {max_bytes} bytes"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sha256(bytes: &[u8]) -> String {
|
||||
let digest = Sha256::digest(bytes);
|
||||
let mut output = String::with_capacity(7 + digest.len() * 2);
|
||||
output.push_str("sha256:");
|
||||
for byte in digest {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(output, "{byte:02x}");
|
||||
}
|
||||
output
|
||||
job::validate_identifier(field, value, max_bytes).map_err(backend_error)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -596,6 +579,258 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn reservation() -> BackendJobReservation {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"job": {
|
||||
"workspace_id": "workspace-a",
|
||||
"request": request(),
|
||||
"intent_fingerprint": request().fingerprint().unwrap(),
|
||||
"state": "pending",
|
||||
"current_attempt": 2,
|
||||
"created_at": "created",
|
||||
"updated_at": "updated"
|
||||
},
|
||||
"attempt": {
|
||||
"workspace_id": "workspace-a",
|
||||
"job_id": "ticket-check:T-1:r7",
|
||||
"attempt_id": "ticket-check:T-1:r7:attempt:2",
|
||||
"attempt": 2,
|
||||
"input_revision": "7",
|
||||
"state": "reserved",
|
||||
"worker": {"runtime_id": "runtime-a", "worker_id": "worker-a"},
|
||||
"runtime_run_id": "run-a",
|
||||
"deadline_at": "deadline",
|
||||
"created_at": "created",
|
||||
"updated_at": "updated"
|
||||
},
|
||||
"replayed": false,
|
||||
"resource_reused": false
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backend_state_projections_preserve_existing_wire_vocabulary() {
|
||||
for state in [
|
||||
BackendJobState::Pending,
|
||||
BackendJobState::Completed,
|
||||
BackendJobState::Failed,
|
||||
BackendJobState::Unknown,
|
||||
] {
|
||||
let shared = job::JobState::from(state);
|
||||
assert_eq!(
|
||||
serde_json::to_value(state).unwrap(),
|
||||
serde_json::to_value(shared).unwrap()
|
||||
);
|
||||
}
|
||||
for state in [
|
||||
BackendJobAttemptState::Reserved,
|
||||
BackendJobAttemptState::Dispatching,
|
||||
BackendJobAttemptState::Dispatched,
|
||||
BackendJobAttemptState::Completed,
|
||||
BackendJobAttemptState::Failed,
|
||||
BackendJobAttemptState::Unknown,
|
||||
] {
|
||||
let shared = job::JobAttemptState::from(state);
|
||||
assert_eq!(
|
||||
serde_json::to_value(state).unwrap(),
|
||||
serde_json::to_value(shared).unwrap()
|
||||
);
|
||||
}
|
||||
// The shared read vocabulary does not add Backend cancellation support.
|
||||
assert!(BackendJobState::parse("cancelled").is_err());
|
||||
assert!(BackendJobAttemptState::parse("cancelled").is_err());
|
||||
assert!(serde_json::from_str::<BackendJobState>("\"cancelled\"").is_err());
|
||||
assert!(serde_json::from_str::<BackendJobAttemptState>("\"cancelled\"").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn neutral_request_round_trip_does_not_infer_authority_or_replace_profile() {
|
||||
for profile in ["builtin:backend-job", "builtin:job", "project:custom-job"] {
|
||||
for key in [None, Some("resource:7".to_string())] {
|
||||
let mut neutral = request().to_job_request();
|
||||
neutral.profile = profile.into();
|
||||
neutral.serialization_key = key;
|
||||
neutral.limits = job::JobLimits {
|
||||
max_concurrent_jobs: 3,
|
||||
timeout_seconds: 17,
|
||||
max_result_bytes: 777,
|
||||
max_attempts: 1,
|
||||
};
|
||||
let backend = BackendJobRequest::from_job_request(neutral.clone());
|
||||
backend.validate().unwrap();
|
||||
assert_eq!(backend.to_job_request(), neutral);
|
||||
assert!(backend.grants.is_empty());
|
||||
assert!(backend.source_worker.is_none());
|
||||
assert!(backend.notification_target.is_none());
|
||||
let encoded = serde_json::to_value(&backend).unwrap();
|
||||
for field in ["grants", "source_worker", "notification_target"] {
|
||||
assert!(encoded.get(field).is_none());
|
||||
}
|
||||
assert_eq!(encoded["profile"], profile);
|
||||
}
|
||||
}
|
||||
// Conversion is not validation and must not normalize an invalid request.
|
||||
let mut neutral = request().to_job_request();
|
||||
neutral.limits.timeout_seconds = 0;
|
||||
neutral.profile = "inherit".into();
|
||||
let backend = BackendJobRequest::from_job_request(neutral.clone());
|
||||
assert_eq!(backend.to_job_request(), neutral);
|
||||
assert!(backend.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reservation_and_acceptance_outcomes_expose_durable_job_and_attempt_not_host_lifecycle() {
|
||||
let mut reservation = reservation();
|
||||
let encoded_before = serde_json::to_value(&reservation).unwrap();
|
||||
assert_eq!(
|
||||
reservation.outcome(),
|
||||
job::JobOutcome {
|
||||
job_id: reservation.job.request.job_id.clone(),
|
||||
input_revision: "7".into(),
|
||||
attempt_id: "ticket-check:T-1:r7:attempt:2".into(),
|
||||
attempt: 2,
|
||||
state: job::JobState::Pending,
|
||||
attempt_state: job::JobAttemptState::Reserved,
|
||||
result: None,
|
||||
failure_category: None,
|
||||
failure_detail: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(serde_json::to_value(&reservation).unwrap(), encoded_before);
|
||||
for (state, attempt_state) in [
|
||||
(
|
||||
BackendJobState::Pending,
|
||||
BackendJobAttemptState::Dispatching,
|
||||
),
|
||||
(BackendJobState::Pending, BackendJobAttemptState::Dispatched),
|
||||
(BackendJobState::Failed, BackendJobAttemptState::Failed),
|
||||
(BackendJobState::Unknown, BackendJobAttemptState::Unknown),
|
||||
] {
|
||||
reservation.job.state = state;
|
||||
reservation.attempt.state = attempt_state;
|
||||
reservation.job.failure_category = Some("job-category".into());
|
||||
reservation.job.failure_detail = Some("job-detail".into());
|
||||
reservation.attempt.failure_category = Some("attempt-category".into());
|
||||
let outcome = reservation.outcome();
|
||||
assert_eq!(outcome.state, state.into());
|
||||
assert_eq!(outcome.attempt_state, attempt_state.into());
|
||||
assert_eq!(outcome.failure_category.as_deref(), Some("job-category"));
|
||||
assert_eq!(outcome.failure_detail.as_deref(), Some("job-detail"));
|
||||
assert!(outcome.result.is_none());
|
||||
let accepted = BackendJobResultAcceptance {
|
||||
job: reservation.job.clone(),
|
||||
attempt: reservation.attempt.clone(),
|
||||
replayed: true,
|
||||
};
|
||||
assert_eq!(accepted.outcome(), outcome);
|
||||
}
|
||||
reservation.job.state = BackendJobState::Completed;
|
||||
reservation.attempt.state = BackendJobAttemptState::Completed;
|
||||
reservation.job.result = Some(serde_json::json!({"ok": true}));
|
||||
reservation.job.failure_category = None;
|
||||
reservation.job.failure_detail = None;
|
||||
reservation.attempt.worker_cleanup_state = Some(BackendJobWorkerCleanupState::Failed);
|
||||
reservation.attempt.worker_cleanup_failure_detail =
|
||||
Some("cleanup is not model failure".into());
|
||||
reservation.replayed = true;
|
||||
reservation.resource_reused = true;
|
||||
let acceptance = BackendJobResultAcceptance {
|
||||
job: reservation.job.clone(),
|
||||
attempt: reservation.attempt.clone(),
|
||||
replayed: false,
|
||||
};
|
||||
let outcome = acceptance.outcome();
|
||||
assert_eq!(outcome, reservation.outcome());
|
||||
assert_eq!(outcome.state, job::JobState::Completed);
|
||||
assert_eq!(outcome.result, Some(serde_json::json!({"ok": true})));
|
||||
assert!(outcome.failure_category.is_none());
|
||||
assert!(outcome.failure_detail.is_none());
|
||||
let encoded = serde_json::to_value(outcome).unwrap();
|
||||
assert_eq!(encoded.as_object().unwrap().len(), 9);
|
||||
for field in [
|
||||
"workspace_id",
|
||||
"worker",
|
||||
"runtime_run_id",
|
||||
"request",
|
||||
"replayed",
|
||||
"resource_reused",
|
||||
"worker_cleanup_state",
|
||||
] {
|
||||
assert!(encoded.get(field).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_preserves_backend_wire_fingerprint_and_worker_envelope() {
|
||||
let request = request();
|
||||
let encoded = serde_json::to_string(&request).unwrap();
|
||||
assert_eq!(
|
||||
encoded,
|
||||
r#"{"job_id":"ticket-check:T-1:r7","purpose":"ticket_item_check","input_revision":"7","input_ref":"ticket://T-1/revisions/7","input":{"title":"Check me"},"instruction":"Check this immutable Ticket snapshot.","profile":"builtin:backend-job","source_worker":{"runtime_id":"runtime-a","worker_id":"worker-a"},"notification_target":{"runtime_id":"runtime-a","worker_id":"worker-a"},"limits":{"max_concurrent_jobs":8,"timeout_seconds":120,"max_result_bytes":16384,"max_attempts":2}}"#
|
||||
);
|
||||
assert_eq!(
|
||||
request.fingerprint().unwrap(),
|
||||
"sha256:920187fa9efa111a97c4d6869d134943e45b52edb2861cad19a0a173233c8f60"
|
||||
);
|
||||
let neutral = request.to_job_request();
|
||||
neutral.validate().unwrap();
|
||||
assert_ne!(
|
||||
request.fingerprint().unwrap(),
|
||||
neutral.fingerprint().unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&request.limits).unwrap(),
|
||||
serde_json::to_value(&neutral.limits).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
request.worker_input("attempt-1").unwrap(),
|
||||
"Check this immutable Ticket snapshot.\n\nBackend Job envelope (immutable):\njob_id: ticket-check:T-1:r7\nattempt_id: attempt-1\ninput_revision: 7\ninput_ref: ticket://T-1/revisions/7\ninput_json: {\"title\":\"Check me\"}\n\nReturn success only through the structured Backend Job result capability. Final prose and Worker Idle/Stopped state are not result authority."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_result_contract_preserves_backend_wire_and_errors() {
|
||||
let submission = BackendJobResultSubmission {
|
||||
job_id: "job-1".into(),
|
||||
attempt_id: "job-1:attempt:1".into(),
|
||||
input_revision: "7".into(),
|
||||
result: serde_json::json!({"ok": true}),
|
||||
};
|
||||
let encoded = serde_json::to_string(&submission).unwrap();
|
||||
assert_eq!(
|
||||
encoded,
|
||||
r#"{"job_id":"job-1","attempt_id":"job-1:attempt:1","input_revision":"7","result":{"ok":true}}"#
|
||||
);
|
||||
let neutral: job::JobResultSubmission = serde_json::from_str(&encoded).unwrap();
|
||||
assert_eq!(neutral, submission);
|
||||
assert_eq!(
|
||||
result_digest(&submission.result, 128).unwrap(),
|
||||
job::result_digest(&submission.result, 128).unwrap()
|
||||
);
|
||||
assert!(
|
||||
matches!(result_digest(&submission.result, 4), Err(Error::InvalidInput(detail)) if detail == "Backend Job result exceeds 4 bytes")
|
||||
);
|
||||
let mut request = request();
|
||||
request.job_id = "".into();
|
||||
assert!(
|
||||
matches!(request.validate(), Err(Error::InvalidInput(detail)) if detail == "Backend Job job_id must be non-empty and trimmed")
|
||||
);
|
||||
request.job_id = "job-1".into();
|
||||
request.limits.timeout_seconds = 0;
|
||||
assert!(
|
||||
matches!(request.validate(), Err(Error::InvalidInput(detail)) if detail == "Backend Job timeout_seconds must be in 1..=600")
|
||||
);
|
||||
// Preserve domain-before-input/limits error ordering as well.
|
||||
request.grants.subjektiv_consolidation = Some(SubjektivConsolidationGrant {
|
||||
subject_id: "".into(),
|
||||
candidate_ids: vec![],
|
||||
});
|
||||
assert!(
|
||||
matches!(request.validate(), Err(Error::InvalidInput(detail)) if detail == "Backend Job grant subject_id must be non-empty and trimmed")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_covers_revision_profile_limits_and_notification_provenance() {
|
||||
let request = request();
|
||||
|
||||
@@ -26,7 +26,8 @@ worker
|
||||
- Controller transport は `InProcess` に固定する。通常起動で Worker subprocess、HTTP/WS server、Unix socket を作らない。
|
||||
- model provider は通常の resolved Manifest から構築する。埋め込み host と deterministic test は `start_with_model_client` で同じ bootstrap に process-owned client を注入できる。
|
||||
- feature plan/install は既存 `WorkerController` が行う。Task や optional direct SubWorker を standalone 側で再実装しない。
|
||||
- `shutdown()` は既存 `Method::Shutdown` を送り、controller が active run、SubWorker registry、Workdir session、MachineScope allocation を順に片付けた後の confirmation を待つ。
|
||||
- `shutdown()` は Host-owned Job service の scheduling を閉じ、Internal Worker の取消・cleanup とローカル SQLite connection の close を確認してから、既存 `Method::Shutdown` を送る。controller が active run、SubWorker registry、Workdir session、MachineScope allocation を順に片付けた後の confirmation を待つ。cleanup を確認できない drop/error では lease を保持する。
|
||||
- `jobs()` は Feature へ明示的に渡す汎用 Job 実行 capability。対話 Session・モデル作成 SubWorker とは別で、Profile/result authority と永続化の契約は [Standalone Jobs](standalone-jobs.md) に記録する。
|
||||
- startup error は category のみを公開し、credential、prompt 本文、session metadata、内部 path を error text に含めない。
|
||||
|
||||
## CLI / TUI routing
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
# Host-owned standalone Jobs (T-708)
|
||||
|
||||
## Implementation design recorded before code changes
|
||||
|
||||
Base: `3c424fd17d1a5e8bb5542de2d0ad88f55338e0ea` (current origin/develop; T-704 integrated). T-709's subjektiv consumer is not part of this change.
|
||||
|
||||
The reusable `job` crate owns the transport-neutral immutable request (registry Profile selector, input snapshot/reference/revision, caller Job ID, instruction, serialization key and bounded limits), result envelope/digest and validation. Backend retains its existing SQLite state machine, Runtime binding, domain grants, authorization, unknown-outcome and cleanup implementation; its adapter uses the shared request validation and result contract, not a second runner.
|
||||
|
||||
Standalone owns a separate adapter, not a WorkspaceClient impersonation. `StandaloneHost` owns a Job service and its connection lifecycle. The concrete local placement is `<standalone state directory>/<WorkerId>/jobs.sqlite3` (using the existing Worker directory layout; alongside, not inside, interactive sessions). The `standalone` crate owns the versioned local schema: durable intents, immutable attempts, execution state, accepted result/digest, cancellation and consumer acknowledgement. A single Host connection is protected by the existing Worker lease. SQLite transactions serialize reserve, dispatch, result acceptance and terminal transitions. No subjektiv tables or fake Workspace/Runtime registry are created.
|
||||
|
||||
Local adapter states follow the existing intent/attempt/result distinctions. Exact immutable request replay returns the same Job, never starts another attempt. Explicit retry creates a bounded new attempt only for known failure; unknown outcomes require caller reconciliation and never auto-replay. Unstarted pending attempts remain pending on shutdown/reopen. Interrupted dispatch/running attempts without an accepted result become unknown; accepted results survive restart and remain available until consumer acknowledgement. Cancellation is durable and never auto-resumed. Consumer acknowledgement/delivery failure is separate from model success and never reruns the model.
|
||||
|
||||
Execution uses the existing private Internal Worker substrate with a new Host-facing bounded Job entry point. It gets a distinct ephemeral session, explicitly resolved selected Profile/system prompt/model, no ambient parent tools, filesystem, Workspace or Feature connections, and only a Job/attempt/revision-bound structured result submission capability. Profiles requiring unavailable tools/features are explicitly rejected rather than silently stripped. Result authority is independent of Profile name; final prose, Idle and Worker termination are never success evidence. This first generic adapter grants no domain operations; future consumers must provide explicit domain capabilities rather than gaining them through a profile tag.
|
||||
|
||||
The Host limits concurrent active resources independently of caller limits, enforces timeout/turn/output limits, routes cancellation into the Internal Worker lifecycle and waits for resource cleanup. Accepted results are committed before resource release; required domain postprocessing must finish before submission (or be independently durable in its consumer). No daemon, pool, subprocess, HTTP/WS listener or Server dependency is introduced.
|
||||
|
||||
Validation will distinguish scripted in-process Internal Worker executions and SQLite reopen/recovery fixtures from any real-provider/process tests. It will cover alternative Profiles, immutable retry, invalid/stale results, resource limits, failure/cancellation/timeout, shutdown and restart. Backend focused regression tests must still prove its existing fences and cleanup.
|
||||
|
||||
## API and ownership
|
||||
|
||||
`job::{JobRequest, JobLimits, JobResultSubmission, JobOutcome}` are the common
|
||||
Feature-facing data contract. `JobOutcome` distinguishes intent state from attempt
|
||||
state and omits Workspace/Runtime identities. These enums are read vocabulary,
|
||||
not another transition engine. Backend converts neutral requests with
|
||||
`BackendJobRequest::from_job_request` and projects reservation/acceptance with
|
||||
`.outcome()`; Backend domain grants, provenance, delivery and cleanup remain on
|
||||
its own adapter and retain their existing durable/wire format and fingerprints.
|
||||
|
||||
Standalone consumers receive `StandaloneHost::jobs()` explicitly, not a fake
|
||||
Workspace client and not a tool on the interactive Worker. They can persist
|
||||
`request`, read `get`/`pending`, explicitly `start` or `retry`, `cancel`, and
|
||||
`acknowledge`. `.wait()` observes execution cleanup; `.outcome()` on the returned
|
||||
snapshot is the common read view. `Completed` with its accepted structured result
|
||||
is success authority; merely returning from `wait` is not. The injected-client
|
||||
entry point resolves/validates the same selected Profile and model configuration;
|
||||
it replaces only the model transport for embedders/tests.
|
||||
|
||||
A typical generic consumer supplies its own stable Job ID/revision/reference,
|
||||
immutable JSON input, instruction and registry Profile:
|
||||
|
||||
```rust,ignore
|
||||
let jobs = host.jobs();
|
||||
let snapshot = jobs.request(request)?;
|
||||
if snapshot.state == standalone::jobs::JobState::Pending {
|
||||
jobs.start(&snapshot.request.job_id).await?;
|
||||
}
|
||||
let outcome = jobs.wait(&snapshot.request.job_id).await?.outcome();
|
||||
// Validate/apply the consumer's domain result here. A failed notification or
|
||||
// delivery leaves the successful result unacknowledged; do not retry the model.
|
||||
if outcome.state == job::JobState::Completed {
|
||||
deliver_idempotently(&outcome)?;
|
||||
jobs.acknowledge(&outcome.job_id)?;
|
||||
}
|
||||
```
|
||||
|
||||
The Host retains tasks/cancellation even while a consumer waits; dropping a wait
|
||||
future does not detach cancellation ownership or release a concurrency slot.
|
||||
Active resources are bounded to eight by the Host and by the minimum of all live
|
||||
requests' ceilings, including after result acceptance and before cleanup.
|
||||
Serialization keys likewise fence live resources, not just pending result state.
|
||||
Busy `start` returns a typed capacity error and leaves the reserved attempt
|
||||
unchanged; callers decide when to start/resume pending work. There is no pool,
|
||||
background daemon, implicit retry or independent Feature-owned Internal Worker.
|
||||
|
||||
## Profile and capability policy
|
||||
|
||||
The caller must select an existing built-in or user registry Profile. Path/raw
|
||||
selectors and `inherit` are not accepted. Registry existence, selected instruction
|
||||
and model configuration are checked before dispatch. The ordinary user settings
|
||||
registry and Prompt catalog are used; repository-local recipes are not discovered
|
||||
by the standalone Host, and default never silently maps to a different Profile.
|
||||
|
||||
`builtin:job` is an **optional**, compatible result-only policy. It is not a
|
||||
runner-selected default or authority token. User registry Profiles with different
|
||||
instructions/models/bounds execute the same way, as demonstrated by the scripted
|
||||
Worker tests. This adapter deliberately rejects active unsupported Feature,
|
||||
filesystem/delegation scope, MCP, skills, compaction and durable event-trace
|
||||
requirements. Existing `builtin:backend-job` has such requirements and is rejected
|
||||
by this standalone adapter, while its Backend behavior is unchanged. A future
|
||||
consumer requiring domain operations must arrange explicit Host capabilities;
|
||||
choosing `builtin:subjektiv-memory-consolidation` alone grants nothing.
|
||||
|
||||
Every run uses the existing private Internal Worker substrate, a distinct identity
|
||||
and ephemeral Session. Only `SubmitJobResult` is installed by a FeatureModule.
|
||||
Its model input is `{result: ...}`; Job/attempt/input-revision bindings come from
|
||||
the Host capability, not model arguments. The sink checks those bindings, the
|
||||
current persisted attempt, deadline, byte limit and result digest. Exact result
|
||||
replay is idempotent; changed/foreign/stale results are rejected. Submission is
|
||||
sealed before acceptance, so uncertain responses permit only identical retries.
|
||||
An ambiguous sink failure is reconciled against SQLite; absent accepted evidence
|
||||
it is `Unknown`, not a known retryable model failure. No domain logic is embedded
|
||||
in this sink. Required consumer/domain postprocessing must complete before
|
||||
submission or have its own durable ownership outside the ephemeral Worker.
|
||||
|
||||
Selected instructions are retained. The Host execution contract is rendered from
|
||||
`resources/prompts/internal/job_system.md` before the run's initial history is
|
||||
created, not injected into a previous/transient turn. Turn/output policy is
|
||||
attenuated to at most 32 turns and 8192 output tokens per model request, preserving
|
||||
tighter selected Profile bounds. Structured results retain the shared 16 KiB
|
||||
default / 64 KiB absolute byte limits, not a larger standalone exception.
|
||||
|
||||
## Persistence and termination
|
||||
|
||||
Local schema version **1** is owned only by `standalone::job_store`: `job_intents`
|
||||
contains immutable serialized intent/fingerprint, serialization key, current
|
||||
attempt and acknowledgement; `job_attempts` contains immutable identity/revision,
|
||||
execution state, failure and accepted result/digest. Foreign keys, immutable
|
||||
identity/result triggers, transactions, `WAL` and `synchronous=FULL` protect the
|
||||
boundary. No Server schema or subjektiv/Memory schema is copied or opened.
|
||||
The Host opens and recovers the store only under its existing top-level Worker
|
||||
lease; clones cannot open or recover it themselves.
|
||||
|
||||
- **Exact request replay:** same immutable intent returns its current snapshot.
|
||||
Changing any intent field under the same ID is a conflict, not a new attempt.
|
||||
- **Known failure:** only explicit `retry` creates a new immutable attempt, up to
|
||||
the intent's bounded maximum. Prior attempts/results remain in SQLite.
|
||||
- **Timeout/shutdown before acceptance:** fence `Unknown` before requesting Engine
|
||||
cancellation; late result submissions cannot create success. Await normal
|
||||
cleanup; retain the Host lease and report an error if confirmation is lost.
|
||||
- **Normal Host shutdown:** seal scheduling/mutations, cancel active runs, await
|
||||
resource cleanup, close SQLite, then stop the interactive controller and release
|
||||
its lease. Reserved/unstarted requests remain resumable. Old cloned handles
|
||||
return `Closed`, rather than mutating a store reopened by another Host.
|
||||
- **Unexpected Host drop:** request cancellation and retain the lease because
|
||||
synchronous drop cannot attest cleanup. No daemon continues after process exit.
|
||||
- **Process loss/reopen:** reserved attempts stay pending; unaccepted dispatched
|
||||
attempts become unknown; accepted results/acknowledgements remain intact.
|
||||
SQLite result evidence, never final prose or an ephemeral Session, decides success.
|
||||
- **Cancellation:** durable and never automatically resumed. Cancelling already
|
||||
unknown intent does not erase its attempt's uncertain execution evidence or
|
||||
release its serialization fence. `retry` still refuses it. There is no generic
|
||||
operation that declares uncertain domain side effects unexecuted; callers must
|
||||
reconcile domain evidence rather than bypassing the fence with an implicit replay.
|
||||
- **After acceptance:** model error, cancellation or consumer delivery failure
|
||||
cannot erase success or rerun the model. Acknowledgement is independently durable
|
||||
and idempotent. Execution resources are reclaimed independently of result lifetime.
|
||||
|
||||
Emergency cancellation has a bounded grace (one second for cancellation setup,
|
||||
two seconds for normal cleanup). Expiry is **not** cleanup confirmation: it returns
|
||||
an error, keeps unknown evidence/accepted results, and the Host retains its lease.
|
||||
The result-only Feature owns no async/background tasks or domain connections;
|
||||
the dropped engine stream/tool pumps request abort. Synchronous SQLite submission
|
||||
is bounded by its five-second busy timeout and cannot be preempted mid-transaction.
|
||||
No resource or side effect is claimed rolled back merely because a future dropped.
|
||||
|
||||
## Validation evidence
|
||||
|
||||
See [T-708 validation ledger](../report/2026-10-06-t708-validation.md). It separates
|
||||
scripted in-process Worker/Host runs, direct SQLite/reopen/rollback fixtures and
|
||||
an actual test-owned child-process loss/Host restore test from live-provider or
|
||||
live dogfood validation. T-709's subjektiv consumer is intentionally not migrated
|
||||
here; Backend T-704 Profile selection and T-690 cleanup remain regression-tested.
|
||||
@@ -0,0 +1,134 @@
|
||||
# T-708 implementation and validation ledger
|
||||
|
||||
Base: clean detached assigned checkout `3c424fd17d1a5e8bb5542de2d0ad88f55338e0ea`,
|
||||
also the inspected `origin/develop`. T-704 was authoritative `done` with that
|
||||
integrated source. Created `work/T-708-standalone-jobs` without overwriting an
|
||||
existing branch or discarding changes. T-709's subjektiv consumer is separate.
|
||||
The concrete SQLite placement/schema ownership was recorded in
|
||||
[the design](../design/standalone-jobs.md) before implementation; it uses the
|
||||
existing `<state_dir>/<WorkerId>` directory layout, not a new `workers` subdirectory.
|
||||
|
||||
Three scoped implementation children supplied the common contract/Backend
|
||||
projection, Worker execution adapter, and local SQLite store. Parent inspected
|
||||
committed sessions, exact source changes and concrete test results, then integrated
|
||||
Host/service ownership, live-resource ceilings, deadline fencing, independent
|
||||
acknowledgement, shared outcome projection, process-loss tests and documentation.
|
||||
Independent published-MR review is separate from these implementation children.
|
||||
|
||||
## Implemented boundaries
|
||||
|
||||
- `job`: neutral immutable request/limits/result envelope, fingerprint, strict
|
||||
registry selectors, bounded digests and read-only intent/attempt/outcome DTOs.
|
||||
No HTTP, Workspace, Runtime identity, domain grants, runner or workflow DSL.
|
||||
- Backend reuses validation/result helpers and neutral projections while retaining
|
||||
its exact request wire/fingerprint/envelope, domain grants, persistent states,
|
||||
Runtime authorization, unknown fences, resource serialization and cleanup.
|
||||
- `worker::job`: selected registry Profile/instruction/model, explicit requirement
|
||||
rejection, bounded policy, distinct ephemeral Internal Worker, result-only
|
||||
FeatureModule and Host-fenced structured acceptance. No parent filesystem,
|
||||
Workspace, Feature or domain connections; no Profile-name authorization.
|
||||
Ambiguous sink acceptance is Unknown unless the Host's durable result proves success.
|
||||
- Standalone Host owns schema-1 SQLite intent/attempt/result/ack state and a bounded
|
||||
execution service. It opens and recovers under its Worker lease; shutdown cancels
|
||||
and awaits executions, closes SQLite, then releases the lease after controller
|
||||
confirmation. Old cloned handles are closed. Drop cannot attest cleanup and
|
||||
retains the lease. Busy/unknown/cancelled/terminal work is not blindly replayed.
|
||||
- Exact immutable request resend differs from explicit bounded new-attempt retry.
|
||||
Results commit before resource release, survive model/notification failure, and
|
||||
are independently acknowledged by the consumer. Waiting does not relinquish
|
||||
Host ownership of cancellation/concurrency. Cancelling unknown intent retains
|
||||
its uncertain attempt evidence and serialization fence.
|
||||
- Optional `builtin:job` is a caller-selected compatible policy, not a fixed runner
|
||||
default. Different user registry Profiles are demonstrated in Worker fixtures.
|
||||
Existing `builtin:backend-job` retains Backend behavior; standalone rejects its
|
||||
unavailable requirements rather than silently stripping or replacing them.
|
||||
- No production Server/Runtime process, HTTP/WS listener, Worker registry,
|
||||
subprocess Worker, daemon/pool or subjektiv-specific runner is added.
|
||||
|
||||
## Validation results
|
||||
|
||||
Development began with narrow contract/module targets before full changed-crate
|
||||
validation. Final root check covers the normal default-members TUI/Server closure.
|
||||
|
||||
| Command | Concrete result |
|
||||
| --- | --- |
|
||||
| `cargo test -p job` | 9 contract tests passed; no unit/doc tests |
|
||||
| `cargo test -p standalone --lib job_store` | 19 fixtures passed: transactional replay/conflict, all limits, result fences, retry/old-attempt preservation, cancellation, concurrency races, SQL rollback, schema rejection and reopen recovery |
|
||||
| `cargo test -p standalone --lib jobs::tests` | 2 passed: Host capability binding/deadline/replay and success/cancellation/closed-handle lifetime |
|
||||
| `cargo test -p standalone --test jobs process_loss_recovers_durable_jobs_without_replaying_unknown` | Actual test-owned process exit and Host stale-lease/SQLite recovery passed |
|
||||
| Final `cargo test -p standalone` | 25 library + 10 existing Host integration + 8 Job integration passed; 1 child-process fixture entry point marked ignored in parent harness, explicitly invoked with `--ignored` by the passing process-loss test |
|
||||
| `cargo test -p manifest` | All 185 tests passed, including new optional builtin Profile catalogue resolution; no resolver/runtime/schema implementation changed |
|
||||
| `cargo test -p worker` | All 826 library + 110 integration tests passed, including 11 new Job tests and 8 existing Internal Worker substrate tests |
|
||||
| `cargo test -p yoi-workspace-server --lib backend_job` | All 38 passed: neutral adapter/wire/fingerprint, T-704 Profiles/no fallback, structured fences, unknown/recovery, resource serialization and T-690 cleanup |
|
||||
| `cargo test -p yoi-workspace-server --bins` | All 11 binary tests passed |
|
||||
| `cargo test -p worker-runtime --lib backend_job` | All 3 Job binding/dispatch regressions passed |
|
||||
| `cargo test -p server-api --features typescript` | All 77 passed, including checked-in generated/OpenAPI/TypeScript freshness; existing ts-rs serde-attribute warnings |
|
||||
| Root `cargo check` | Passed after integration and again after final local adapter/tests changes |
|
||||
| `cargo fmt --all -- --check` / `git diff --check HEAD` | Passed |
|
||||
| `cargo test -p yoi-workspace-server` | **Failed: 643 passed, 1 failed** in library; see retained anomaly below. Binary tests were therefore run separately. This is not a green full Server suite. |
|
||||
| `cargo test -p yoi-workspace-server -- --test-threads=4` | **Failed: 643 passed, 1 failed**, same unchanged checker assertion; not claimed passed |
|
||||
|
||||
Profile fixtures use two different existing user registry entries with distinct
|
||||
instructions and budgets, plus optional `builtin:job`. They prove only
|
||||
`SubmitJobResult` is exposed, selected instructions/policy are retained, scopes and
|
||||
Backend/domain tools are absent, malformed/oversized/forged results are rejected,
|
||||
uncertain sink submission permits only identical replay, final prose is not
|
||||
success, provider failure/cancellation/turn limits do not manufacture success,
|
||||
and accepted results survive later provider failure.
|
||||
|
||||
Host Job integration additionally proves explicit known-model-failure retry,
|
||||
request replay, live concurrency while a consumer waits, cancellation, timeout,
|
||||
shutdown, pending resume, consumer acknowledgement and unsupported Profile
|
||||
rejection without model calls. Direct-store fixtures prove persisted facts and
|
||||
rollback/reopen behavior, not live model or process behavior.
|
||||
|
||||
### Actual process verification versus fixtures
|
||||
|
||||
`process_loss_recovers_durable_jobs_without_replaying_unknown` starts **only a
|
||||
test-owned copy of the integration test executable**, using scripted model clients.
|
||||
The child owns a real StandaloneHost/SQLite connection/Worker lease, persists a
|
||||
reserved request, a cancelled request and an accepted result, then exits with code
|
||||
23 while a separate Internal Worker is executing. It deliberately skips destructors
|
||||
and graceful shutdown. The parent verifies that exit and restores the Host through
|
||||
the real stale-process lease path: accepted result retained/unacknowledged,
|
||||
cancelled work never resumed, reserved work explicitly resumable and interrupted
|
||||
work Unknown/no automatic retry. The helper's ignored annotation is an entry-point
|
||||
selector, not skipped semantic coverage.
|
||||
|
||||
These are real Rust test/compiler processes plus scripted in-process model
|
||||
executions and SQLite fixtures. They are **not** live-provider LLM calls, a live
|
||||
standalone CLI/TUI session, a Backend/Runtime deployment or dogfood environment
|
||||
update. No operating Yoi Worker or live dogfood process was stopped or updated.
|
||||
Production Jobs do not create the subprocess used solely by that crash test.
|
||||
|
||||
## Retained failures and evidence limits
|
||||
|
||||
Initial integrated compilation exposed `get` snapshot/Option disagreement between
|
||||
concurrently authored store/service; signatures were aligned and the subsequent
|
||||
store/Host/full standalone runs passed. A private tool-type import during child
|
||||
integration was corrected before focused/full Worker validation. No passing
|
||||
result is attributed to those initial failing builds.
|
||||
|
||||
Both full Server runs and one isolated run of
|
||||
`server::tests::ticket_checker_suppresses_stale_and_clean_results_and_diagnoses_invalid_output`
|
||||
failed the unchanged post-stale-result `assert!(execution.take_inputs().is_empty())`
|
||||
at `server.rs:44748`. All 38 Job-filter tests pass, including unchanged Backend
|
||||
envelope/fingerprint compatibility and late-result/cleanup fences. The same
|
||||
checker assertion/race was independently reproduced on the earlier original base
|
||||
in the [T-702 ledger](2026-10-06-t702-stale-checker-validation.md), and explicitly
|
||||
retained as unfixed in the [T-704 ledger](2026-10-06-t704-validation.md).
|
||||
No new isolated build of this Ticket's exact base was run, and its root cause is
|
||||
not claimed proven or fixed here. The checker implementation/test was neither
|
||||
changed nor skipped to manufacture a green suite. This validation anomaly remains
|
||||
visible for Reviewer/Orchestrator attention.
|
||||
|
||||
No whole-workspace tests, E2E, Nix/Docker builds or live-provider validation were
|
||||
claimed. Backend generated contracts were regression checked despite no wire
|
||||
change. Result-only Jobs reject unsupported Feature/authority requirements;
|
||||
subjektiv consumer capabilities/postprocessing are T-709 work, not inferred grants.
|
||||
|
||||
Publication must use only the clean committed Ticket work branch by normal
|
||||
non-force push, verified with exact provider source ref. No target/tags/unrelated
|
||||
refs, replacement MR, integration branch, merge or Ticket completion is performed
|
||||
by this Coder. Final immutable source/MR/review identities belong in the MR thread
|
||||
and approved Ticket handoff rather than being guessed in this ledger.
|
||||
@@ -0,0 +1,27 @@
|
||||
slug = "job";
|
||||
description = "Result-only Job policy; the Host grants attempt/result authority independently.";
|
||||
model = { ref = "codex-oauth/gpt-5.6-luna"; };
|
||||
engine = {
|
||||
instruction = "internal.job_system";
|
||||
reasoning = "medium";
|
||||
max_tokens = 8192;
|
||||
max_turns = 32;
|
||||
};
|
||||
session = { record_event_trace = false; };
|
||||
feature = {
|
||||
task = { enabled = false; };
|
||||
memory = { enabled = false; staging_tools = false; };
|
||||
subjektiv = { enabled = false; consolidation_tools = false; extraction = { enabled = false; }; };
|
||||
web = { enabled = false; };
|
||||
image = { enabled = false; };
|
||||
sub_worker = { enabled = false; };
|
||||
flow = { enabled = false; };
|
||||
worker = { enabled = false; direct_spawn = false; };
|
||||
workspace_worker_discovery = { enabled = false; };
|
||||
objective = { enabled = false; };
|
||||
manage_workdir = { enabled = false; };
|
||||
workspace_config = { enabled = false; };
|
||||
ticket = { enabled = false; authoring = false; thread = false; intake = false; workflow = false; };
|
||||
merge_request = { show = false; open = false; review = false; readiness_check = false; complete = false; };
|
||||
orchestration = { enabled = false; };
|
||||
};
|
||||
@@ -21,6 +21,7 @@ roleReviewer = import "./role/reviewer.md";
|
||||
internalCompactSystem = import "./internal/compact_system.md";
|
||||
internalFlowVerifierSystem = import "./internal/flow_verifier_system.md";
|
||||
internalBackendJobSystem = import "./internal/backend_job_system.md";
|
||||
internalJobSystem = import "./internal/job_system.md";
|
||||
internalTicketItemChecker = import "./internal/ticket_item_checker.md";
|
||||
internalMemoryConsolidationSystem = import "./internal/memory_consolidation_system.md";
|
||||
internalSubjektivMemoryConsolidationSystem = import "./internal/subjektiv_memory_consolidation_system.md";
|
||||
@@ -62,6 +63,7 @@ in
|
||||
compact_system = internalCompactSystem.content;
|
||||
flow_verifier_system = internalFlowVerifierSystem.content;
|
||||
backend_job_system = internalBackendJobSystem.content;
|
||||
job_system = internalJobSystem.content;
|
||||
ticket_item_checker = internalTicketItemChecker.content;
|
||||
memory_consolidation_system = internalMemoryConsolidationSystem.content;
|
||||
subjektiv_memory_consolidation_system = internalSubjektivMemoryConsolidationSystem.content;
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
You execute one bounded Host-owned Job against its immutable input envelope.
|
||||
|
||||
The caller selects the Profile. The Host, not the Profile name or task prose, binds Job/attempt/input identity and grants tools. Use only the capabilities provided for this execution. Do not infer Workspace, filesystem, parent Worker, or domain permissions from the input.
|
||||
|
||||
Submit a structured result through the Host-provided Job result tool. Complete any required postprocessing before submitting, unless its durable ownership is explicitly established independently of this Worker. Final prose, Idle, and Worker termination are not success authority. Do not claim a submission succeeded if the tool rejects it.
|
||||
|
||||
On a definitive validation rejection, correct the result. If acceptance is uncertain, retry only the exact submitted result; never change a result already accepted. Never ask to run another attempt or access another Job's capabilities.
|
||||
Reference in New Issue
Block a user