feat(subjektiv): share domain and add explicit standalone Subject hosts
This commit is contained in:
@@ -9,6 +9,12 @@ license.workspace = true
|
||||
agen.workspace = true
|
||||
client.workspace = true
|
||||
fs4.workspace = true
|
||||
feature-storage.workspace = true
|
||||
subjektiv.workspace = true
|
||||
memory.workspace = true
|
||||
server-api.workspace = true
|
||||
libc.workspace = true
|
||||
async-trait.workspace = true
|
||||
manifest.workspace = true
|
||||
job.workspace = true
|
||||
rusqlite.workspace = true
|
||||
|
||||
+219
-10
@@ -44,12 +44,15 @@ pub struct StandaloneHost {
|
||||
record: StandaloneWorkerRecord,
|
||||
lease: Option<StandaloneWorkerLease>,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
subject: Option<crate::subjektiv::SubjectConnection>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum StandaloneStartupError {
|
||||
#[error("the standalone state store could not be opened or validated")]
|
||||
StateStore,
|
||||
#[error("local Subject connection rejected: {0}")]
|
||||
Subject(String),
|
||||
#[error("the standalone Worker is already active")]
|
||||
WorkerActive,
|
||||
#[error("the standalone Worker lease cannot be observed safely; recovery is rejected")]
|
||||
@@ -107,16 +110,53 @@ impl StandaloneHost {
|
||||
launch: ResolvedStandaloneLaunch,
|
||||
model_client: Option<Box<dyn LlmClient>>,
|
||||
) -> Result<Self, StandaloneStartupError> {
|
||||
if launch.subject_id.is_some()
|
||||
&& (!launch.profile.manifest.feature.subjektiv.profile.enabled
|
||||
|| launch
|
||||
.profile
|
||||
.manifest
|
||||
.feature
|
||||
.subjektiv
|
||||
.profile
|
||||
.consolidation_tools
|
||||
|| launch
|
||||
.profile
|
||||
.manifest
|
||||
.feature
|
||||
.subjektiv
|
||||
.workspace_settings
|
||||
.is_some())
|
||||
{
|
||||
return Err(StandaloneStartupError::Subject(
|
||||
"incompatible local Subject policy".into(),
|
||||
));
|
||||
}
|
||||
let store =
|
||||
StandaloneWorkerStore::open(&launch.state_dir).map_err(classify_store_startup_error)?;
|
||||
let allocation = store
|
||||
.allocate(&launch.cwd, StaleLeasePolicy::Reject)
|
||||
.map_err(classify_store_startup_error)?;
|
||||
let worker_id = allocation.worker_id();
|
||||
let jobs = match crate::jobs::StandaloneJobs::open(
|
||||
&store.jobs_path(worker_id),
|
||||
launch.cwd.clone(),
|
||||
) {
|
||||
let mut subject = match launch
|
||||
.subject_id
|
||||
.as_deref()
|
||||
.map(|id| {
|
||||
crate::subjektiv::StandaloneSubjects::open_existing(&launch.state_dir)?
|
||||
.connect(id, worker_id)
|
||||
})
|
||||
.transpose()
|
||||
{
|
||||
Ok(subject) => subject,
|
||||
Err(error) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
return Err(StandaloneStartupError::Subject(error.to_string()));
|
||||
}
|
||||
};
|
||||
let jobs_path = subject
|
||||
.as_ref()
|
||||
.map(|s| s.jobs_path.clone())
|
||||
.unwrap_or_else(|| store.jobs_path(worker_id));
|
||||
let jobs = match crate::jobs::StandaloneJobs::open(&jobs_path, launch.cwd.clone()) {
|
||||
Ok(jobs) => jobs,
|
||||
Err(_) => {
|
||||
let _ = store.abandon_allocation(allocation);
|
||||
@@ -124,6 +164,12 @@ impl StandaloneHost {
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(client) = &model_client {
|
||||
jobs.bind_model_client(client.clone_boxed());
|
||||
}
|
||||
if let Some(subject) = &subject {
|
||||
subject.host.bind_jobs(jobs.clone())?;
|
||||
}
|
||||
// WorkerId is the stable identity. The current Worker store remains
|
||||
// name-keyed, so keep its derived storage key separate from the
|
||||
// user-facing profile name.
|
||||
@@ -156,9 +202,15 @@ impl StandaloneHost {
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
);
|
||||
if let Some(subject) = &subject {
|
||||
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
|
||||
}
|
||||
if let Some(model_client) = model_client {
|
||||
bootstrap = bootstrap.with_model_client(model_client);
|
||||
}
|
||||
if let Some(subject) = &mut subject {
|
||||
subject.arm();
|
||||
}
|
||||
let started = match bootstrap.start().await {
|
||||
Ok(started) => started,
|
||||
Err(error) => {
|
||||
@@ -174,12 +226,13 @@ impl StandaloneHost {
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let record = match store.commit_created(
|
||||
let record = match store.commit_created_connected(
|
||||
&allocation,
|
||||
manifest,
|
||||
storage_key,
|
||||
active.session_id,
|
||||
active.segment_id,
|
||||
subject.as_ref().map(|s| s.binding.clone()),
|
||||
) {
|
||||
Ok(record) => record,
|
||||
Err(_) => {
|
||||
@@ -195,6 +248,7 @@ impl StandaloneHost {
|
||||
record,
|
||||
allocation.into_lease(),
|
||||
jobs,
|
||||
subject,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -230,11 +284,50 @@ impl StandaloneHost {
|
||||
let lease = store
|
||||
.acquire_lease(worker_id, StaleLeasePolicy::Recover)
|
||||
.map_err(classify_store_startup_error)?;
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&store.jobs_path(worker_id),
|
||||
record.cwd.canonical_path.clone(),
|
||||
)
|
||||
.map_err(|_| StandaloneStartupError::StateStore)?;
|
||||
let mut subject = record
|
||||
.subject
|
||||
.as_ref()
|
||||
.map(|binding| {
|
||||
if !record.manifest.feature.subjektiv.profile.enabled
|
||||
|| record
|
||||
.manifest
|
||||
.feature
|
||||
.subjektiv
|
||||
.profile
|
||||
.consolidation_tools
|
||||
|| record
|
||||
.manifest
|
||||
.feature
|
||||
.subjektiv
|
||||
.workspace_settings
|
||||
.is_some()
|
||||
{
|
||||
return Err(crate::subjektiv::SubjectError::InvalidScope(
|
||||
"persisted Subject policy changed".into(),
|
||||
));
|
||||
}
|
||||
let catalog = crate::subjektiv::StandaloneSubjects::open_existing(store.root())?;
|
||||
if catalog.scope_id() != binding.scope_id {
|
||||
return Err(crate::subjektiv::SubjectError::InvalidScope(
|
||||
"persisted local storage scope changed".into(),
|
||||
));
|
||||
}
|
||||
catalog.connect(&binding.subject_id, worker_id)
|
||||
})
|
||||
.transpose()
|
||||
.map_err(|e| StandaloneStartupError::Subject(e.to_string()))?;
|
||||
let jobs_path = subject
|
||||
.as_ref()
|
||||
.map(|s| s.jobs_path.clone())
|
||||
.unwrap_or_else(|| store.jobs_path(worker_id));
|
||||
let jobs = crate::jobs::StandaloneJobs::open(&jobs_path, record.cwd.canonical_path.clone())
|
||||
.map_err(|_| StandaloneStartupError::StateStore)?;
|
||||
if let Some(client) = &model_client {
|
||||
jobs.bind_model_client(client.clone_boxed());
|
||||
}
|
||||
if let Some(subject) = &subject {
|
||||
subject.host.bind_jobs(jobs.clone())?;
|
||||
}
|
||||
let (backing_store, worker_store) = backing_store(&store, worker_id)?;
|
||||
let storage_key = record.storage_key.clone();
|
||||
let mut manifest = record.manifest.clone();
|
||||
@@ -259,6 +352,9 @@ impl StandaloneHost {
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
);
|
||||
if let Some(subject) = &subject {
|
||||
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
|
||||
}
|
||||
if let Some(model_client) = model_client {
|
||||
bootstrap = bootstrap.with_model_client(model_client);
|
||||
}
|
||||
@@ -266,6 +362,9 @@ impl StandaloneHost {
|
||||
.prepare_restored(&storage_key)
|
||||
.await
|
||||
.map_err(classify_startup_error)?;
|
||||
if let Some(subject) = &mut subject {
|
||||
subject.arm();
|
||||
}
|
||||
let started = prepared.start().await.map_err(classify_startup_error)?;
|
||||
let active = match active_pointer(&worker_store, &storage_key) {
|
||||
Ok(active) => active,
|
||||
@@ -290,6 +389,7 @@ impl StandaloneHost {
|
||||
record,
|
||||
lease,
|
||||
jobs,
|
||||
subject,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -300,7 +400,11 @@ impl StandaloneHost {
|
||||
record: StandaloneWorkerRecord,
|
||||
lease: StandaloneWorkerLease,
|
||||
jobs: crate::jobs::StandaloneJobs,
|
||||
subject: Option<crate::subjektiv::SubjectConnection>,
|
||||
) -> Self {
|
||||
if let Some(connection) = &subject {
|
||||
connection.host.recover_jobs();
|
||||
}
|
||||
Self {
|
||||
handle: started.handle,
|
||||
shutdown: Some(started.shutdown),
|
||||
@@ -310,6 +414,7 @@ impl StandaloneHost {
|
||||
record,
|
||||
lease: Some(lease),
|
||||
jobs,
|
||||
subject,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -388,6 +493,12 @@ impl StandaloneHost {
|
||||
self.retain_lease();
|
||||
return Err(StandaloneShutdownError::StateStore);
|
||||
}
|
||||
if let Some(subject) = &mut self.subject {
|
||||
if subject.close().is_err() {
|
||||
self.retain_lease();
|
||||
return Err(StandaloneShutdownError::StateStore);
|
||||
}
|
||||
}
|
||||
if let Some(lease) = self.lease.take() {
|
||||
lease
|
||||
.release()
|
||||
@@ -397,6 +508,9 @@ impl StandaloneHost {
|
||||
}
|
||||
|
||||
fn retain_lease(&mut self) {
|
||||
if let Some(subject) = &mut self.subject {
|
||||
subject.retain();
|
||||
}
|
||||
if let Some(lease) = self.lease.take() {
|
||||
lease.retain();
|
||||
}
|
||||
@@ -413,6 +527,51 @@ impl Drop for StandaloneHost {
|
||||
}
|
||||
}
|
||||
|
||||
// The in-process channel is owned by the local operator's Host, not an
|
||||
// arbitrary socket or Backend account. decode_method discards wire provenance;
|
||||
// only this boundary can stamp accepted local human input.
|
||||
fn authorize_local_input(method: protocol::Method) -> protocol::Method {
|
||||
use protocol::{AuthenticatedInputSource, Method};
|
||||
match method {
|
||||
Method::Submit {
|
||||
submission_request_id,
|
||||
input,
|
||||
}
|
||||
| Method::SubmitTracked {
|
||||
submission_request_id,
|
||||
input,
|
||||
..
|
||||
} => Method::SubmitTracked {
|
||||
submission_request_id,
|
||||
input,
|
||||
source: AuthenticatedInputSource::LocalOperator,
|
||||
},
|
||||
Method::SubmitIfIdle {
|
||||
submission_request_id,
|
||||
input,
|
||||
..
|
||||
} => Method::SubmitIfIdle {
|
||||
submission_request_id,
|
||||
input,
|
||||
source: AuthenticatedInputSource::LocalOperator,
|
||||
},
|
||||
Method::Notify {
|
||||
notification_request_id,
|
||||
message,
|
||||
}
|
||||
| Method::NotifyTracked {
|
||||
notification_request_id,
|
||||
message,
|
||||
..
|
||||
} => Method::NotifyTracked {
|
||||
notification_request_id,
|
||||
message,
|
||||
source: AuthenticatedInputSource::LocalOperator,
|
||||
},
|
||||
method => method,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_protocol_session(
|
||||
handle: worker::WorkerHandle,
|
||||
streams: WorkerProtocolSessionStreams,
|
||||
@@ -438,6 +597,7 @@ async fn run_protocol_session(
|
||||
let Ok(method) = decode_method(&message) else {
|
||||
return;
|
||||
};
|
||||
let method = authorize_local_input(method);
|
||||
if let Some(event) = dispatch_worker_protocol_method(&handle, method).await
|
||||
&& !send_protocol_event(&peer, event).await
|
||||
{
|
||||
@@ -578,6 +738,9 @@ fn classify_startup_error(error: WorkerBootstrapError) -> StandaloneStartupError
|
||||
WorkerBootstrapError::Worker(WorkerError::Provider(_)) => {
|
||||
StandaloneStartupError::ModelProvider
|
||||
}
|
||||
WorkerBootstrapError::Worker(WorkerError::SubjektivSessionAttribution {
|
||||
message, ..
|
||||
}) => StandaloneStartupError::Subject(message),
|
||||
WorkerBootstrapError::Worker(_) => StandaloneStartupError::WorkerConfiguration,
|
||||
WorkerBootstrapError::Controller { source, .. }
|
||||
if source.kind() == std::io::ErrorKind::Other =>
|
||||
@@ -587,3 +750,49 @@ fn classify_startup_error(error: WorkerBootstrapError) -> StandaloneStartupError
|
||||
WorkerBootstrapError::Controller { .. } => StandaloneStartupError::Controller,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod local_input_tests {
|
||||
use super::*;
|
||||
use protocol::{AuthenticatedInputSource, Method};
|
||||
|
||||
#[test]
|
||||
fn local_transport_stamps_operator_without_backend_account() {
|
||||
let source = AuthenticatedInputSource::Account {
|
||||
account_id: "forged-account".into(),
|
||||
};
|
||||
for method in [
|
||||
Method::submit_text("plain", "hello"),
|
||||
Method::SubmitTracked {
|
||||
submission_request_id: "tracked".into(),
|
||||
input: vec![],
|
||||
source: source.clone(),
|
||||
},
|
||||
Method::SubmitIfIdle {
|
||||
submission_request_id: "idle".into(),
|
||||
input: vec![],
|
||||
source: source.clone(),
|
||||
},
|
||||
Method::Notify {
|
||||
notification_request_id: "plain-notify".into(),
|
||||
message: "hello".into(),
|
||||
},
|
||||
Method::NotifyTracked {
|
||||
notification_request_id: "tracked-notify".into(),
|
||||
message: "hello".into(),
|
||||
source,
|
||||
},
|
||||
] {
|
||||
let decoded: Method =
|
||||
serde_json::from_str(&serde_json::to_string(&method).unwrap()).unwrap();
|
||||
match authorize_local_input(decoded) {
|
||||
Method::SubmitTracked { source, .. }
|
||||
| Method::SubmitIfIdle { source, .. }
|
||||
| Method::NotifyTracked { source, .. } => {
|
||||
assert_eq!(source, AuthenticatedInputSource::LocalOperator)
|
||||
}
|
||||
other => panic!("unexpected local input {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
//! authority. The service owns the connection mutex and live-resource cleanup.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use job::{JobRequest, JobResultSubmission};
|
||||
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
|
||||
@@ -14,7 +13,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
|
||||
const SCHEMA_VERSION: i64 = 1;
|
||||
const SCHEMA_VERSION: i64 = 2;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -57,6 +56,8 @@ impl JobAttemptState {
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct JobSnapshot {
|
||||
pub request: JobRequest,
|
||||
/// Durable Host-issued domain delegation, not model input or Profile identity.
|
||||
pub domain_grant: Option<Value>,
|
||||
pub state: JobState,
|
||||
pub attempt: JobAttempt,
|
||||
pub result: Option<Value>,
|
||||
@@ -112,6 +113,8 @@ pub struct JobAttempt {
|
||||
pub enum JobStoreError {
|
||||
#[error("Job database: {0}")]
|
||||
Sqlite(#[from] rusqlite::Error),
|
||||
#[error(transparent)]
|
||||
Storage(#[from] feature_storage::FeatureStorageError),
|
||||
#[error("Job JSON: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error(transparent)]
|
||||
@@ -151,16 +154,30 @@ impl JobStore {
|
||||
if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.mode(0o600)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(path)?;
|
||||
}
|
||||
let mut conn = Connection::open(path)?;
|
||||
conn.busy_timeout(Duration::from_secs(5))?;
|
||||
conn.pragma_update(None, "foreign_keys", "ON")?;
|
||||
conn.pragma_update(None, "journal_mode", "WAL")?;
|
||||
conn.pragma_update(None, "synchronous", "FULL")?;
|
||||
feature_storage::configure_connection(&conn)?;
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let version: i64 = tx.pragma_query_value(None, "user_version", |row| row.get(0))?;
|
||||
match version {
|
||||
0 => {
|
||||
tx.execute_batch(SCHEMA)?;
|
||||
tx.execute_batch(GRANT_SCHEMA)?;
|
||||
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
||||
}
|
||||
1 => {
|
||||
tx.execute_batch(GRANT_SCHEMA)?;
|
||||
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
||||
}
|
||||
SCHEMA_VERSION => {}
|
||||
@@ -186,7 +203,17 @@ impl JobStore {
|
||||
}
|
||||
|
||||
pub(crate) fn reserve(&mut self, request: JobRequest) -> Result<JobSnapshot, JobStoreError> {
|
||||
self.reserve_granted(request, None)
|
||||
}
|
||||
pub(crate) fn reserve_granted(
|
||||
&mut self,
|
||||
request: JobRequest,
|
||||
grant: Option<Value>,
|
||||
) -> Result<JobSnapshot, JobStoreError> {
|
||||
request.validate()?;
|
||||
if let Some(value) = &grant {
|
||||
job::result_digest(value, job::ABSOLUTE_MAX_RESULT_BYTES)?;
|
||||
}
|
||||
let fingerprint = request.fingerprint()?;
|
||||
let request_json = serde_json::to_string(&request)?;
|
||||
let tx = self
|
||||
@@ -198,7 +225,10 @@ impl JobStore {
|
||||
[&request.job_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if stored != fingerprint || snapshot.request != request {
|
||||
if stored != fingerprint
|
||||
|| snapshot.request != request
|
||||
|| snapshot.domain_grant != grant
|
||||
{
|
||||
return Err(JobStoreError::IntentConflict(request.job_id));
|
||||
}
|
||||
tx.commit()?;
|
||||
@@ -219,6 +249,12 @@ impl JobStore {
|
||||
],
|
||||
)?;
|
||||
insert_attempt(&tx, &request, 1)?;
|
||||
if let Some(grant) = grant {
|
||||
tx.execute(
|
||||
"INSERT INTO job_domain_grants(job_id, grant_json) VALUES (?1, ?2)",
|
||||
params![request.job_id, serde_json::to_string(&grant)?],
|
||||
)?;
|
||||
}
|
||||
let snapshot = require_snapshot(&tx, &request.job_id)?;
|
||||
tx.commit()?;
|
||||
Ok(snapshot)
|
||||
@@ -255,6 +291,16 @@ impl JobStore {
|
||||
require_snapshot(&self.conn, job_id)
|
||||
}
|
||||
|
||||
pub(crate) fn unacknowledged(&self) -> Result<Vec<JobSnapshot>, JobStoreError> {
|
||||
let mut statement = self.conn.prepare("SELECT job_id FROM job_intents WHERE state = 'completed' AND acknowledged = 0 ORDER BY rowid")?;
|
||||
let ids = statement
|
||||
.query_map([], |row| row.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ids.into_iter()
|
||||
.map(|id| require_snapshot(&self.conn, &id))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Only unstarted current attempts are eligible for service startup/resume.
|
||||
pub(crate) fn pending(&self) -> Result<Vec<JobSnapshot>, JobStoreError> {
|
||||
let mut statement = self.conn.prepare(
|
||||
@@ -570,6 +616,15 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
};
|
||||
Ok(JobSnapshot {
|
||||
request: serde_json::from_str(&request)?,
|
||||
domain_grant: conn
|
||||
.query_row(
|
||||
"SELECT grant_json FROM job_domain_grants WHERE job_id = ?1",
|
||||
[job_id],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.optional()?
|
||||
.map(|s| serde_json::from_str(&s))
|
||||
.transpose()?,
|
||||
state,
|
||||
acknowledged,
|
||||
attempt: JobAttempt {
|
||||
@@ -585,6 +640,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
.transpose()
|
||||
}
|
||||
|
||||
const GRANT_SCHEMA: &str = "
|
||||
CREATE TABLE job_domain_grants (
|
||||
job_id TEXT PRIMARY KEY NOT NULL REFERENCES job_intents(job_id),
|
||||
grant_json TEXT NOT NULL
|
||||
);
|
||||
CREATE TRIGGER job_grant_immutable BEFORE UPDATE ON job_domain_grants
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job domain grant'); END;
|
||||
CREATE TRIGGER job_grant_retained BEFORE DELETE ON job_domain_grants
|
||||
BEGIN SELECT RAISE(ABORT, 'retained Job domain grant'); END;
|
||||
";
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE job_intents (
|
||||
job_id TEXT PRIMARY KEY NOT NULL,
|
||||
|
||||
@@ -27,6 +27,17 @@ pub enum JobServiceError {
|
||||
Cleanup(String),
|
||||
}
|
||||
|
||||
pub(crate) trait JobAttemptFence: Send + Sync {
|
||||
fn ensure_live(&self) -> Result<(), String>;
|
||||
}
|
||||
pub(crate) trait JobDomainProvider: Send + Sync {
|
||||
fn grant(
|
||||
&self,
|
||||
snapshot: &JobSnapshot,
|
||||
fence: Arc<dyn JobAttemptFence>,
|
||||
) -> Result<worker::job::JobFeatureGrant, String>;
|
||||
}
|
||||
|
||||
struct Execution {
|
||||
cancel: watch::Sender<bool>,
|
||||
complete: watch::Receiver<Option<Result<(), String>>>,
|
||||
@@ -48,6 +59,8 @@ pub struct StandaloneJobs {
|
||||
executions: Arc<Mutex<Executions>>,
|
||||
cwd: PathBuf,
|
||||
capacity: u16,
|
||||
domain: Arc<Mutex<Option<Arc<dyn JobDomainProvider>>>>,
|
||||
model_client: Arc<Mutex<Option<Box<dyn LlmClient>>>>,
|
||||
}
|
||||
|
||||
impl StandaloneJobs {
|
||||
@@ -60,9 +73,43 @@ impl StandaloneJobs {
|
||||
})),
|
||||
cwd,
|
||||
capacity: job::DEFAULT_MAX_CONCURRENT_JOBS,
|
||||
domain: Arc::default(),
|
||||
model_client: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn bind_model_client(&self, client: Box<dyn LlmClient>) {
|
||||
*self
|
||||
.model_client
|
||||
.lock()
|
||||
.expect("Job model transport poisoned") = Some(client);
|
||||
}
|
||||
pub(crate) fn bind_domain(
|
||||
&self,
|
||||
provider: Arc<dyn JobDomainProvider>,
|
||||
) -> Result<(), JobServiceError> {
|
||||
let mut domain = self.domain.lock().expect("Job domain poisoned");
|
||||
if domain.is_some() {
|
||||
return Err(JobServiceError::Configuration(
|
||||
"Job domain already bound".into(),
|
||||
));
|
||||
}
|
||||
*domain = Some(provider);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn request_granted(
|
||||
&self,
|
||||
request: JobRequest,
|
||||
grant: serde_json::Value,
|
||||
) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
self.with_store(|store| store.reserve_granted(request, Some(grant)))
|
||||
}
|
||||
|
||||
fn with_store<T>(
|
||||
&self,
|
||||
operation: impl FnOnce(&mut JobStore) -> Result<T, JobStoreError>,
|
||||
@@ -91,6 +138,10 @@ impl StandaloneJobs {
|
||||
Ok(self.with_store(|store| store.pending())?)
|
||||
}
|
||||
|
||||
pub fn unacknowledged_results(&self) -> Result<Vec<JobSnapshot>, JobServiceError> {
|
||||
self.with_store(|store| store.unacknowledged())
|
||||
}
|
||||
|
||||
/// Explicit new attempt, bounded by immutable intent limits. Unknown
|
||||
/// outcomes cannot be retried without domain reconciliation/new intent.
|
||||
pub fn retry(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
@@ -154,9 +205,40 @@ impl StandaloneJobs {
|
||||
job_id: &str,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
) -> Result<(), JobServiceError> {
|
||||
let client = client.or_else(|| {
|
||||
self.model_client
|
||||
.lock()
|
||||
.expect("Job model transport poisoned")
|
||||
.as_ref()
|
||||
.map(|c| c.clone_boxed())
|
||||
});
|
||||
let snapshot = self.get(job_id)?;
|
||||
let prepared = worker::job::prepare_job(&snapshot.request, &self.cwd, client)
|
||||
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
|
||||
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
|
||||
store: self.store.clone(),
|
||||
job_id: job_id.into(),
|
||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||
input_revision: snapshot.request.input_revision.clone(),
|
||||
deadline,
|
||||
});
|
||||
let grant = match (
|
||||
&snapshot.domain_grant,
|
||||
self.domain.lock().expect("Job domain poisoned").as_ref(),
|
||||
) {
|
||||
(Some(_), Some(provider)) => provider
|
||||
.grant(&snapshot, fence)
|
||||
.map_err(JobServiceError::Configuration)?,
|
||||
(Some(_), None) => {
|
||||
return Err(JobServiceError::Configuration(
|
||||
"persisted Job requires an unavailable explicit domain capability".into(),
|
||||
));
|
||||
}
|
||||
(None, _) => worker::job::JobFeatureGrant::default(),
|
||||
};
|
||||
let prepared =
|
||||
worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant)
|
||||
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
@@ -209,13 +291,12 @@ impl StandaloneJobs {
|
||||
let store = self.store.clone();
|
||||
let id = job_id.to_string();
|
||||
let attempt_id = snapshot.attempt.attempt_id.clone();
|
||||
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
|
||||
let sink = Arc::new(BoundResultSink {
|
||||
store: store.clone(),
|
||||
job_id: id.clone(),
|
||||
attempt_id: attempt_id.clone(),
|
||||
input_revision: snapshot.request.input_revision.clone(),
|
||||
deadline: tokio::time::Instant::now() + timeout,
|
||||
deadline,
|
||||
});
|
||||
let (cancel, mut cancellation) = watch::channel(false);
|
||||
let (complete_tx, complete) = watch::channel(None);
|
||||
@@ -341,6 +422,11 @@ impl StandaloneJobs {
|
||||
// Close SQLite before the Host releases its lease. Cloned handles may
|
||||
// outlive the Host but cannot mutate or retain the live connection.
|
||||
self.store.lock().expect("Job store poisoned").take();
|
||||
self.domain.lock().expect("Job domain poisoned").take();
|
||||
self.model_client
|
||||
.lock()
|
||||
.expect("Job model transport poisoned")
|
||||
.take();
|
||||
error.map_or(Ok(()), Err)
|
||||
}
|
||||
}
|
||||
@@ -391,3 +477,32 @@ impl worker::job::JobResultSink for BoundResultSink {
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
struct BoundAttemptFence {
|
||||
store: Arc<Mutex<Option<JobStore>>>,
|
||||
job_id: String,
|
||||
attempt_id: String,
|
||||
input_revision: String,
|
||||
deadline: tokio::time::Instant,
|
||||
}
|
||||
impl JobAttemptFence for BoundAttemptFence {
|
||||
fn ensure_live(&self) -> Result<(), String> {
|
||||
if tokio::time::Instant::now() >= self.deadline {
|
||||
return Err("Job domain deadline expired".into());
|
||||
}
|
||||
let slot = self.store.lock().map_err(|_| "Job store poisoned")?;
|
||||
let snapshot = slot
|
||||
.as_ref()
|
||||
.ok_or("Job Host closed")?
|
||||
.get(&self.job_id)
|
||||
.map_err(|e| e.to_string())?;
|
||||
if snapshot.state != JobState::Pending
|
||||
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
||||
|| snapshot.attempt.attempt_id != self.attempt_id
|
||||
|| snapshot.request.input_revision != self.input_revision
|
||||
{
|
||||
return Err("Job domain grant is not bound to the current live attempt".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
use super::*;
|
||||
use agen::llm_client::{ClientError, Request, event::Event};
|
||||
use async_trait::async_trait;
|
||||
use futures::Stream;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use worker::job::JobResultSink;
|
||||
|
||||
fn request(id: &str) -> JobRequest {
|
||||
@@ -32,6 +38,345 @@ fn result(id: &str, attempt: &str) -> JobResultSubmission {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct DomainTestClient {
|
||||
scripts: Arc<Mutex<VecDeque<Vec<Event>>>>,
|
||||
requests: Arc<Mutex<Vec<Request>>>,
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
stall: bool,
|
||||
failure_release: Option<Arc<tokio::sync::Notify>>,
|
||||
}
|
||||
impl DomainTestClient {
|
||||
fn success() -> Self {
|
||||
Self {
|
||||
scripts: Arc::new(Mutex::new(VecDeque::from([
|
||||
vec![
|
||||
Event::tool_use_start(0, "result-1", "SubmitJobResult"),
|
||||
Event::tool_input_delta(0, r#"{"result":{"valid":true}}"#),
|
||||
Event::tool_use_stop(0),
|
||||
],
|
||||
vec![
|
||||
Event::text_block_start(0),
|
||||
Event::text_delta(0, "done"),
|
||||
Event::text_block_stop(0, None),
|
||||
],
|
||||
]))),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
#[async_trait]
|
||||
impl LlmClient for DomainTestClient {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<Event, ClientError>> + Send>>, ClientError> {
|
||||
self.requests.lock().unwrap().push(request);
|
||||
self.entered.notify_one();
|
||||
if let Some(release) = &self.failure_release {
|
||||
release.notified().await;
|
||||
return Err(ClientError::Config(
|
||||
"known scripted provider failure".into(),
|
||||
));
|
||||
}
|
||||
if self.stall {
|
||||
return Ok(Box::pin(futures::stream::pending()));
|
||||
}
|
||||
let events = self
|
||||
.scripts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.expect("unexpected model turn");
|
||||
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
}
|
||||
|
||||
type CapturedDomainAttempt = (JobSnapshot, Arc<dyn JobAttemptFence>);
|
||||
#[derive(Default)]
|
||||
struct RecordingDomainProvider(Mutex<Vec<CapturedDomainAttempt>>);
|
||||
fn domain_grant(id: &str) -> Value {
|
||||
json!({"consumer":"test-domain", "job_id":id, "input_revision":"r1"})
|
||||
}
|
||||
fn domain_request(id: &str) -> JobRequest {
|
||||
let mut intent = request(id);
|
||||
intent.profile = "builtin:standalone-subjektiv-consolidation".into();
|
||||
intent
|
||||
}
|
||||
impl JobDomainProvider for RecordingDomainProvider {
|
||||
fn grant(
|
||||
&self,
|
||||
snapshot: &JobSnapshot,
|
||||
fence: Arc<dyn JobAttemptFence>,
|
||||
) -> Result<worker::job::JobFeatureGrant, String> {
|
||||
if snapshot.domain_grant != Some(domain_grant(&snapshot.request.job_id)) {
|
||||
return Err("foreign or mismatched immutable domain grant".into());
|
||||
}
|
||||
self.0.lock().unwrap().push((snapshot.clone(), fence));
|
||||
Ok(worker::job::JobFeatureGrant {
|
||||
satisfied_requirements: vec!["feature.subjektiv"],
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
async fn await_model(client: &DomainTestClient) {
|
||||
tokio::time::timeout(Duration::from_secs(2), client.entered.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
async fn await_job(jobs: &StandaloneJobs, id: &str) -> JobSnapshot {
|
||||
tokio::time::timeout(Duration::from_secs(3), jobs.wait(id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pending_domain_grants_survive_service_reopen_and_remain_immutable() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("jobs.sqlite3");
|
||||
let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
|
||||
let intent = domain_request("granted");
|
||||
let grant = domain_grant("granted");
|
||||
let reserved = jobs.request_granted(intent.clone(), grant.clone()).unwrap();
|
||||
let root = jobs.request(request("root")).unwrap();
|
||||
assert_eq!(reserved.domain_grant, Some(grant.clone()));
|
||||
assert!(root.domain_grant.is_none());
|
||||
assert_eq!(
|
||||
jobs.request_granted(intent.clone(), grant.clone()).unwrap(),
|
||||
reserved
|
||||
);
|
||||
assert!(matches!(
|
||||
jobs.request(intent.clone()),
|
||||
Err(JobServiceError::Store(JobStoreError::IntentConflict(_)))
|
||||
));
|
||||
assert!(matches!(
|
||||
jobs.request_granted(intent.clone(), domain_grant("foreign")),
|
||||
Err(JobServiceError::Store(JobStoreError::IntentConflict(_)))
|
||||
));
|
||||
assert!(matches!(
|
||||
jobs.request_granted(request("root"), grant.clone()),
|
||||
Err(JobServiceError::Store(JobStoreError::IntentConflict(_)))
|
||||
));
|
||||
assert_eq!(jobs.pending().unwrap().len(), 2);
|
||||
jobs.shutdown().await.unwrap();
|
||||
|
||||
let reopened = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
|
||||
let pending = reopened.pending().unwrap();
|
||||
assert_eq!(pending.len(), 2);
|
||||
assert!(pending.contains(&reserved));
|
||||
assert!(pending.contains(&root));
|
||||
assert_eq!(reopened.get("granted").unwrap(), reserved);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.request_granted(intent.clone(), grant.clone())
|
||||
.unwrap(),
|
||||
reserved
|
||||
);
|
||||
assert!(matches!(
|
||||
reopened.request_granted(intent, json!({"different":true})),
|
||||
Err(JobServiceError::Store(JobStoreError::IntentConflict(_)))
|
||||
));
|
||||
let provider = Arc::new(RecordingDomainProvider::default());
|
||||
reopened.bind_domain(provider.clone()).unwrap();
|
||||
reopened
|
||||
.start_with_model_client("granted", DomainTestClient::success())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = await_job(&reopened, "granted").await;
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
assert_eq!(completed.domain_grant, Some(grant));
|
||||
assert_eq!(completed.result, Some(json!({"valid":true})));
|
||||
let captured = provider.0.lock().unwrap()[0].0.clone();
|
||||
assert_eq!(
|
||||
captured, reserved,
|
||||
"provider receives the persisted immutable intent and grant"
|
||||
);
|
||||
reopened.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_grant_requires_provider_and_missing_grant_cannot_be_derived_from_profile() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let jobs =
|
||||
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
|
||||
let granted = jobs
|
||||
.request_granted(domain_request("granted"), domain_grant("granted"))
|
||||
.unwrap();
|
||||
let client = DomainTestClient::success();
|
||||
let error = jobs
|
||||
.start_with_model_client("granted", client.clone())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, JobServiceError::Configuration(_)));
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("unavailable explicit domain capability"),
|
||||
"{error}"
|
||||
);
|
||||
assert_eq!(jobs.get("granted").unwrap(), granted);
|
||||
assert!(client.requests.lock().unwrap().is_empty());
|
||||
|
||||
let provider = Arc::new(RecordingDomainProvider::default());
|
||||
jobs.bind_domain(provider.clone()).unwrap();
|
||||
assert!(matches!(
|
||||
jobs.bind_domain(provider.clone()),
|
||||
Err(JobServiceError::Configuration(_))
|
||||
));
|
||||
let ungranted = jobs.request(domain_request("ungranted")).unwrap();
|
||||
let error = jobs
|
||||
.start_with_model_client("ungranted", client.clone())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, JobServiceError::Configuration(_)));
|
||||
assert!(error.to_string().contains("feature.subjektiv"), "{error}");
|
||||
assert_eq!(jobs.get("ungranted").unwrap(), ungranted);
|
||||
assert!(
|
||||
provider.0.lock().unwrap().is_empty(),
|
||||
"bound provider is not a profile-derived grant"
|
||||
);
|
||||
assert!(client.requests.lock().unwrap().is_empty());
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreign_job_revision_and_consumer_grants_are_refused_before_model_dispatch() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let jobs =
|
||||
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
|
||||
let provider = Arc::new(RecordingDomainProvider::default());
|
||||
jobs.bind_domain(provider.clone()).unwrap();
|
||||
for (id, grant) in [
|
||||
("foreign-job", domain_grant("another-job")),
|
||||
(
|
||||
"foreign-revision",
|
||||
json!({"consumer":"test-domain","job_id":"foreign-revision","input_revision":"r0"}),
|
||||
),
|
||||
(
|
||||
"foreign-consumer",
|
||||
json!({"consumer":"another-domain","job_id":"foreign-consumer","input_revision":"r1"}),
|
||||
),
|
||||
] {
|
||||
let reserved = jobs.request_granted(domain_request(id), grant).unwrap();
|
||||
let client = DomainTestClient::success();
|
||||
let error = jobs
|
||||
.start_with_model_client(id, client.clone())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, JobServiceError::Configuration(_)));
|
||||
assert!(
|
||||
error.to_string().contains("foreign or mismatched"),
|
||||
"{error}"
|
||||
);
|
||||
assert_eq!(jobs.get(id).unwrap(), reserved);
|
||||
assert!(client.requests.lock().unwrap().is_empty());
|
||||
}
|
||||
assert!(provider.0.lock().unwrap().is_empty());
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn domain_fence_is_live_only_for_current_dispatched_attempt_and_revoked_on_cancel() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let jobs =
|
||||
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
|
||||
let provider = Arc::new(RecordingDomainProvider::default());
|
||||
jobs.bind_domain(provider.clone()).unwrap();
|
||||
let reserved = jobs
|
||||
.request_granted(domain_request("fenced"), domain_grant("fenced"))
|
||||
.unwrap();
|
||||
let failure_release = Arc::new(tokio::sync::Notify::new());
|
||||
let first = DomainTestClient {
|
||||
failure_release: Some(failure_release.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
jobs.start_with_model_client("fenced", first.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
await_model(&first).await;
|
||||
let old_fence = provider.0.lock().unwrap()[0].1.clone();
|
||||
old_fence.ensure_live().unwrap();
|
||||
failure_release.notify_one();
|
||||
let failed = await_job(&jobs, "fenced").await;
|
||||
assert_eq!(failed.state, JobState::Failed);
|
||||
assert!(failed.result.is_none());
|
||||
assert!(old_fence.ensure_live().is_err());
|
||||
// Only known failure permits a retry; cancellation remains terminal.
|
||||
let retried = jobs.retry("fenced").unwrap();
|
||||
assert_ne!(retried.attempt.attempt_id, reserved.attempt.attempt_id);
|
||||
assert_eq!(retried.domain_grant, reserved.domain_grant);
|
||||
assert!(old_fence.ensure_live().is_err());
|
||||
let second = DomainTestClient {
|
||||
stall: true,
|
||||
..Default::default()
|
||||
};
|
||||
jobs.start_with_model_client("fenced", second.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
await_model(&second).await;
|
||||
let current_fence = provider.0.lock().unwrap()[1].1.clone();
|
||||
current_fence.ensure_live().unwrap();
|
||||
assert!(
|
||||
old_fence.ensure_live().is_err(),
|
||||
"a new attempt must not revive a stale capability"
|
||||
);
|
||||
let cancelled = jobs.cancel("fenced").unwrap();
|
||||
assert_eq!(cancelled.state, JobState::Cancelled);
|
||||
assert!(
|
||||
current_fence.ensure_live().is_err(),
|
||||
"durable cancellation revokes current domain operations immediately"
|
||||
);
|
||||
assert!(old_fence.ensure_live().is_err());
|
||||
let cleaned = await_job(&jobs, "fenced").await;
|
||||
assert_eq!(cleaned.state, JobState::Cancelled);
|
||||
assert!(cleaned.result.is_none());
|
||||
assert!(
|
||||
jobs.retry("fenced").is_err(),
|
||||
"cancelled intent is not a known retryable failure"
|
||||
);
|
||||
jobs.shutdown().await.unwrap();
|
||||
assert!(old_fence.ensure_live().is_err());
|
||||
assert!(current_fence.ensure_live().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bound_domain_does_not_change_root_generic_result_only_execution() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let jobs =
|
||||
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
|
||||
let provider = Arc::new(RecordingDomainProvider::default());
|
||||
jobs.bind_domain(provider.clone()).unwrap();
|
||||
let root = jobs.request(request("root")).unwrap();
|
||||
assert!(root.domain_grant.is_none());
|
||||
let client = DomainTestClient::success();
|
||||
jobs.start_with_model_client("root", client.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = await_job(&jobs, "root").await;
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
assert_eq!(completed.result, Some(json!({"valid":true})));
|
||||
assert!(completed.domain_grant.is_none());
|
||||
assert!(provider.0.lock().unwrap().is_empty());
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
for sent in requests.iter() {
|
||||
assert_eq!(
|
||||
sent.tools
|
||||
.iter()
|
||||
.map(|tool| tool.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["SubmitJobResult"]
|
||||
);
|
||||
assert_eq!(sent.config.max_tokens, Some(worker::job::MAX_JOB_TOKENS));
|
||||
}
|
||||
drop(requests);
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn host_result_capability_cannot_cross_job_attempt_revision_or_deadline() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -15,6 +15,7 @@ pub struct StandaloneLaunchConfig {
|
||||
pub state_dir: PathBuf,
|
||||
pub profile: ProfileSelector,
|
||||
pub worker_name: String,
|
||||
pub subject_id: Option<String>,
|
||||
}
|
||||
|
||||
pub struct ResolvedStandaloneLaunch {
|
||||
@@ -22,6 +23,7 @@ pub struct ResolvedStandaloneLaunch {
|
||||
pub state_dir: PathBuf,
|
||||
pub profile: ResolvedProfile,
|
||||
pub prompt_catalog: PromptCatalogSource,
|
||||
pub subject_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
@@ -32,6 +34,10 @@ pub enum StandaloneLaunchError {
|
||||
PathProfileUnsupported,
|
||||
#[error("the standalone profile could not be resolved")]
|
||||
ProfileResolutionFailed,
|
||||
#[error(
|
||||
"local Subject selection requires an enabled, non-consolidator subjektiv Profile and canonical Subject id"
|
||||
)]
|
||||
InvalidSubjectPolicy,
|
||||
}
|
||||
|
||||
impl StandaloneLaunchConfig {
|
||||
@@ -46,9 +52,15 @@ impl StandaloneLaunchConfig {
|
||||
state_dir: state_dir.into(),
|
||||
profile,
|
||||
worker_name: worker_name.into(),
|
||||
subject_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_subject(mut self, subject_id: String) -> Self {
|
||||
self.subject_id = Some(subject_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// Resolve only built-in/XDG profile authority and bind standalone scope
|
||||
/// to the canonical process cwd. Repository-local profile discovery is
|
||||
/// deliberately not part of this path.
|
||||
@@ -74,12 +86,32 @@ impl StandaloneLaunchConfig {
|
||||
.map_err(|_| StandaloneLaunchError::ProfileResolutionFailed)?;
|
||||
validate_profile_execution_target(&profile.manifest, ProfileExecutionTarget::Standalone)
|
||||
.map_err(|_| StandaloneLaunchError::ProfileResolutionFailed)?;
|
||||
if let Some(id) = &self.subject_id {
|
||||
if !crate::subjektiv::valid_subject_id(id)
|
||||
|| !profile.manifest.feature.subjektiv.profile.enabled
|
||||
|| profile
|
||||
.manifest
|
||||
.feature
|
||||
.subjektiv
|
||||
.profile
|
||||
.consolidation_tools
|
||||
|| profile
|
||||
.manifest
|
||||
.feature
|
||||
.subjektiv
|
||||
.workspace_settings
|
||||
.is_some()
|
||||
{
|
||||
return Err(StandaloneLaunchError::InvalidSubjectPolicy);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ResolvedStandaloneLaunch {
|
||||
cwd,
|
||||
state_dir: self.state_dir,
|
||||
profile,
|
||||
prompt_catalog: PromptCatalogSource::builtins_only(),
|
||||
subject_id: self.subject_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ mod job_store;
|
||||
pub mod jobs;
|
||||
pub mod launch;
|
||||
pub mod store;
|
||||
pub mod subjektiv;
|
||||
mod subjektiv_sessions;
|
||||
|
||||
pub use host::{StandaloneHost, StandaloneShutdownError, StandaloneStartupError};
|
||||
pub use launch::{ResolvedStandaloneLaunch, StandaloneLaunchConfig, StandaloneLaunchError};
|
||||
|
||||
@@ -87,6 +87,9 @@ pub struct StandaloneWorkerRecord {
|
||||
pub storage_key: String,
|
||||
pub cwd: StandaloneCwdIdentity,
|
||||
pub manifest: WorkerManifest,
|
||||
/// Original explicit local Subject/storage binding; never inferred from policy.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub subject: Option<crate::subjektiv::StandaloneSubjectBinding>,
|
||||
pub active_session_id: SessionId,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub active_segment_id: Option<SegmentId>,
|
||||
@@ -127,6 +130,16 @@ impl StandaloneWorkerStore {
|
||||
Ok(Self { root })
|
||||
}
|
||||
|
||||
/// Observational constructor: never creates a missing root or repairs state.
|
||||
pub fn open_existing(root: impl Into<PathBuf>) -> Result<Self, StandaloneStoreError> {
|
||||
let root = root.into();
|
||||
let metadata = fs::symlink_metadata(&root).map_err(StandaloneStoreError::Io)?;
|
||||
if !metadata.is_dir() || metadata.file_type().is_symlink() {
|
||||
return Err(StandaloneStoreError::NotDirectory);
|
||||
}
|
||||
Ok(Self { root })
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn root(&self) -> &Path {
|
||||
&self.root
|
||||
@@ -158,6 +171,25 @@ impl StandaloneWorkerStore {
|
||||
storage_key: String,
|
||||
active_session_id: SessionId,
|
||||
active_segment_id: Option<SegmentId>,
|
||||
) -> Result<StandaloneWorkerRecord, StandaloneStoreError> {
|
||||
self.commit_created_connected(
|
||||
allocation,
|
||||
manifest,
|
||||
storage_key,
|
||||
active_session_id,
|
||||
active_segment_id,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn commit_created_connected(
|
||||
&self,
|
||||
allocation: &StandaloneWorkerAllocation,
|
||||
manifest: WorkerManifest,
|
||||
storage_key: String,
|
||||
active_session_id: SessionId,
|
||||
active_segment_id: Option<SegmentId>,
|
||||
subject: Option<crate::subjektiv::StandaloneSubjectBinding>,
|
||||
) -> Result<StandaloneWorkerRecord, StandaloneStoreError> {
|
||||
let now = now_unix_ms()?;
|
||||
let record = StandaloneWorkerRecord {
|
||||
@@ -168,6 +200,7 @@ impl StandaloneWorkerStore {
|
||||
storage_key,
|
||||
cwd: allocation.cwd.clone(),
|
||||
manifest,
|
||||
subject,
|
||||
active_session_id,
|
||||
active_segment_id,
|
||||
status: StandaloneWorkerStatus::Active,
|
||||
@@ -793,6 +826,7 @@ permission = "write"
|
||||
inode: None,
|
||||
},
|
||||
manifest,
|
||||
subject: None,
|
||||
active_session_id: "01a05782-d5dd-78f1-b9cd-ce37535bdb9e".parse().unwrap(),
|
||||
active_segment_id: None,
|
||||
status: StandaloneWorkerStatus::Stopped,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,130 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn catalog_reopen_preserves_random_scope_subject_and_behavior() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("state");
|
||||
let catalog = StandaloneSubjects::open(&root).unwrap();
|
||||
let subject = catalog
|
||||
.create(
|
||||
"continuing-owner",
|
||||
Some("Keep user behavior separate from Memory."),
|
||||
)
|
||||
.unwrap();
|
||||
let scope = catalog.scope_id().to_owned();
|
||||
assert!(uuid::Uuid::parse_str(&scope).is_ok());
|
||||
assert!(uuid::Uuid::parse_str(subject.id.strip_prefix("subject-").unwrap()).is_ok());
|
||||
assert_ne!(scope, subject.id);
|
||||
drop(catalog);
|
||||
let reopened = StandaloneSubjects::open_existing(&root).unwrap();
|
||||
assert_eq!(reopened.scope_id(), scope);
|
||||
assert_eq!(reopened.list().unwrap(), vec![subject]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn implicit_open_does_not_create_subject_storage() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("missing");
|
||||
assert!(StandaloneSubjects::open_existing(&root).is_err());
|
||||
assert!(!root.exists());
|
||||
assert!(StandaloneSubjects::open_existing(temp.path()).is_err());
|
||||
assert!(!temp.path().join("subjektiv").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
|
||||
let a = catalog.create("a", None).unwrap();
|
||||
let b = catalog.create("b", None).unwrap();
|
||||
let mut first = catalog
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&a.id, protocol::WorkerId::now_v7()),
|
||||
Err(SubjectError::Active(_))
|
||||
));
|
||||
let mut second = StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&b.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
first.close().unwrap();
|
||||
let mut replacement = StandaloneSubjects::open_existing(&temp.path().join("state"))
|
||||
.unwrap()
|
||||
.connect(&a.id, protocol::WorkerId::now_v7())
|
||||
.unwrap();
|
||||
replacement.close().unwrap();
|
||||
second.close().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closed_capabilities_and_foreign_worker_context_are_rejected() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
|
||||
let subject = catalog.create("owner", None).unwrap();
|
||||
let id = protocol::WorkerId::now_v7();
|
||||
let mut connection = catalog.connect(&subject.id, id).unwrap();
|
||||
let port = connection.host.clone();
|
||||
let foreign = SubjektivHostContext {
|
||||
worker_id: protocol::WorkerId::now_v7().to_string(),
|
||||
session_id: session_store::SessionId::now_v7().to_string(),
|
||||
};
|
||||
let read = SubjektivMemoryBackendOperation::ResidentContext(
|
||||
memory::backend::MemoryResidentSummaryOperation {},
|
||||
);
|
||||
assert!(port.memory(&foreign, read.clone()).is_err());
|
||||
let owner = SubjektivHostContext {
|
||||
worker_id: format!("standalone-{id}"),
|
||||
session_id: session_store::SessionId::now_v7().to_string(),
|
||||
};
|
||||
assert!(port.memory(&owner, read.clone()).is_ok());
|
||||
assert!(
|
||||
port.record_session(&owner, false).is_err(),
|
||||
"unattributed Session is not read authority"
|
||||
);
|
||||
connection.close().unwrap();
|
||||
assert!(port.memory(&owner, read).is_err());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn scope_rejects_symlinks_and_non_private_roots() {
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let alias = temp.path().join("alias");
|
||||
let real = temp.path().join("real");
|
||||
fs::create_dir(&real).unwrap();
|
||||
fs::set_permissions(&real, fs::Permissions::from_mode(0o700)).unwrap();
|
||||
symlink(&real, &alias).unwrap();
|
||||
assert!(StandaloneSubjects::open(&alias).is_err());
|
||||
let public = temp.path().join("public");
|
||||
fs::create_dir(&public).unwrap();
|
||||
fs::set_permissions(&public, fs::Permissions::from_mode(0o755)).unwrap();
|
||||
assert!(StandaloneSubjects::open(&public).is_err());
|
||||
let catalog = StandaloneSubjects::open(&real).unwrap();
|
||||
drop(catalog);
|
||||
let scope = real.join("subjektiv/scope.json");
|
||||
let original = fs::read(&scope).unwrap();
|
||||
fs::remove_file(&scope).unwrap();
|
||||
let foreign = temp.path().join("foreign.json");
|
||||
fs::write(&foreign, &original).unwrap();
|
||||
symlink(&foreign, &scope).unwrap();
|
||||
assert!(StandaloneSubjects::open_existing(&real).is_err());
|
||||
assert_eq!(fs::read(&foreign).unwrap(), original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_local_roots_do_not_authorize_shared_subject_ids() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let first = StandaloneSubjects::open(temp.path().join("first")).unwrap();
|
||||
let subject = first.create("owner", None).unwrap();
|
||||
let other = StandaloneSubjects::open(temp.path().join("other")).unwrap();
|
||||
assert_ne!(first.scope_id(), other.scope_id());
|
||||
assert!(
|
||||
other
|
||||
.connect(&subject.id, protocol::WorkerId::now_v7())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user