feat(subjektiv): share domain and add explicit standalone Subject hosts

This commit is contained in:
2026-10-07 01:52:06 +09:00
parent 13055d7ee1
commit 24ae6288a5
65 changed files with 18273 additions and 9785 deletions
+6
View File
@@ -9,6 +9,12 @@ license.workspace = true
agen.workspace = true
client.workspace = true
fs4.workspace = true
feature-storage.workspace = true
subjektiv.workspace = true
memory.workspace = true
server-api.workspace = true
libc.workspace = true
async-trait.workspace = true
manifest.workspace = true
job.workspace = true
rusqlite.workspace = true
+219 -10
View File
@@ -44,12 +44,15 @@ pub struct StandaloneHost {
record: StandaloneWorkerRecord,
lease: Option<StandaloneWorkerLease>,
jobs: crate::jobs::StandaloneJobs,
subject: Option<crate::subjektiv::SubjectConnection>,
}
#[derive(Debug, Clone, PartialEq, Eq, Error)]
pub enum StandaloneStartupError {
#[error("the standalone state store could not be opened or validated")]
StateStore,
#[error("local Subject connection rejected: {0}")]
Subject(String),
#[error("the standalone Worker is already active")]
WorkerActive,
#[error("the standalone Worker lease cannot be observed safely; recovery is rejected")]
@@ -107,16 +110,53 @@ impl StandaloneHost {
launch: ResolvedStandaloneLaunch,
model_client: Option<Box<dyn LlmClient>>,
) -> Result<Self, StandaloneStartupError> {
if launch.subject_id.is_some()
&& (!launch.profile.manifest.feature.subjektiv.profile.enabled
|| launch
.profile
.manifest
.feature
.subjektiv
.profile
.consolidation_tools
|| launch
.profile
.manifest
.feature
.subjektiv
.workspace_settings
.is_some())
{
return Err(StandaloneStartupError::Subject(
"incompatible local Subject policy".into(),
));
}
let store =
StandaloneWorkerStore::open(&launch.state_dir).map_err(classify_store_startup_error)?;
let allocation = store
.allocate(&launch.cwd, StaleLeasePolicy::Reject)
.map_err(classify_store_startup_error)?;
let worker_id = allocation.worker_id();
let jobs = match crate::jobs::StandaloneJobs::open(
&store.jobs_path(worker_id),
launch.cwd.clone(),
) {
let mut subject = match launch
.subject_id
.as_deref()
.map(|id| {
crate::subjektiv::StandaloneSubjects::open_existing(&launch.state_dir)?
.connect(id, worker_id)
})
.transpose()
{
Ok(subject) => subject,
Err(error) => {
let _ = store.abandon_allocation(allocation);
return Err(StandaloneStartupError::Subject(error.to_string()));
}
};
let jobs_path = subject
.as_ref()
.map(|s| s.jobs_path.clone())
.unwrap_or_else(|| store.jobs_path(worker_id));
let jobs = match crate::jobs::StandaloneJobs::open(&jobs_path, launch.cwd.clone()) {
Ok(jobs) => jobs,
Err(_) => {
let _ = store.abandon_allocation(allocation);
@@ -124,6 +164,12 @@ impl StandaloneHost {
}
};
if let Some(client) = &model_client {
jobs.bind_model_client(client.clone_boxed());
}
if let Some(subject) = &subject {
subject.host.bind_jobs(jobs.clone())?;
}
// WorkerId is the stable identity. The current Worker store remains
// name-keyed, so keep its derived storage key separate from the
// user-facing profile name.
@@ -156,9 +202,15 @@ impl StandaloneHost {
},
WorkerControllerTransport::InProcess,
);
if let Some(subject) = &subject {
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
}
if let Some(model_client) = model_client {
bootstrap = bootstrap.with_model_client(model_client);
}
if let Some(subject) = &mut subject {
subject.arm();
}
let started = match bootstrap.start().await {
Ok(started) => started,
Err(error) => {
@@ -174,12 +226,13 @@ impl StandaloneHost {
return Err(error);
}
};
let record = match store.commit_created(
let record = match store.commit_created_connected(
&allocation,
manifest,
storage_key,
active.session_id,
active.segment_id,
subject.as_ref().map(|s| s.binding.clone()),
) {
Ok(record) => record,
Err(_) => {
@@ -195,6 +248,7 @@ impl StandaloneHost {
record,
allocation.into_lease(),
jobs,
subject,
))
}
@@ -230,11 +284,50 @@ impl StandaloneHost {
let lease = store
.acquire_lease(worker_id, StaleLeasePolicy::Recover)
.map_err(classify_store_startup_error)?;
let jobs = crate::jobs::StandaloneJobs::open(
&store.jobs_path(worker_id),
record.cwd.canonical_path.clone(),
)
.map_err(|_| StandaloneStartupError::StateStore)?;
let mut subject = record
.subject
.as_ref()
.map(|binding| {
if !record.manifest.feature.subjektiv.profile.enabled
|| record
.manifest
.feature
.subjektiv
.profile
.consolidation_tools
|| record
.manifest
.feature
.subjektiv
.workspace_settings
.is_some()
{
return Err(crate::subjektiv::SubjectError::InvalidScope(
"persisted Subject policy changed".into(),
));
}
let catalog = crate::subjektiv::StandaloneSubjects::open_existing(store.root())?;
if catalog.scope_id() != binding.scope_id {
return Err(crate::subjektiv::SubjectError::InvalidScope(
"persisted local storage scope changed".into(),
));
}
catalog.connect(&binding.subject_id, worker_id)
})
.transpose()
.map_err(|e| StandaloneStartupError::Subject(e.to_string()))?;
let jobs_path = subject
.as_ref()
.map(|s| s.jobs_path.clone())
.unwrap_or_else(|| store.jobs_path(worker_id));
let jobs = crate::jobs::StandaloneJobs::open(&jobs_path, record.cwd.canonical_path.clone())
.map_err(|_| StandaloneStartupError::StateStore)?;
if let Some(client) = &model_client {
jobs.bind_model_client(client.clone_boxed());
}
if let Some(subject) = &subject {
subject.host.bind_jobs(jobs.clone())?;
}
let (backing_store, worker_store) = backing_store(&store, worker_id)?;
let storage_key = record.storage_key.clone();
let mut manifest = record.manifest.clone();
@@ -259,6 +352,9 @@ impl StandaloneHost {
},
WorkerControllerTransport::InProcess,
);
if let Some(subject) = &subject {
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
}
if let Some(model_client) = model_client {
bootstrap = bootstrap.with_model_client(model_client);
}
@@ -266,6 +362,9 @@ impl StandaloneHost {
.prepare_restored(&storage_key)
.await
.map_err(classify_startup_error)?;
if let Some(subject) = &mut subject {
subject.arm();
}
let started = prepared.start().await.map_err(classify_startup_error)?;
let active = match active_pointer(&worker_store, &storage_key) {
Ok(active) => active,
@@ -290,6 +389,7 @@ impl StandaloneHost {
record,
lease,
jobs,
subject,
))
}
@@ -300,7 +400,11 @@ impl StandaloneHost {
record: StandaloneWorkerRecord,
lease: StandaloneWorkerLease,
jobs: crate::jobs::StandaloneJobs,
subject: Option<crate::subjektiv::SubjectConnection>,
) -> Self {
if let Some(connection) = &subject {
connection.host.recover_jobs();
}
Self {
handle: started.handle,
shutdown: Some(started.shutdown),
@@ -310,6 +414,7 @@ impl StandaloneHost {
record,
lease: Some(lease),
jobs,
subject,
}
}
@@ -388,6 +493,12 @@ impl StandaloneHost {
self.retain_lease();
return Err(StandaloneShutdownError::StateStore);
}
if let Some(subject) = &mut self.subject {
if subject.close().is_err() {
self.retain_lease();
return Err(StandaloneShutdownError::StateStore);
}
}
if let Some(lease) = self.lease.take() {
lease
.release()
@@ -397,6 +508,9 @@ impl StandaloneHost {
}
fn retain_lease(&mut self) {
if let Some(subject) = &mut self.subject {
subject.retain();
}
if let Some(lease) = self.lease.take() {
lease.retain();
}
@@ -413,6 +527,51 @@ impl Drop for StandaloneHost {
}
}
// The in-process channel is owned by the local operator's Host, not an
// arbitrary socket or Backend account. decode_method discards wire provenance;
// only this boundary can stamp accepted local human input.
fn authorize_local_input(method: protocol::Method) -> protocol::Method {
use protocol::{AuthenticatedInputSource, Method};
match method {
Method::Submit {
submission_request_id,
input,
}
| Method::SubmitTracked {
submission_request_id,
input,
..
} => Method::SubmitTracked {
submission_request_id,
input,
source: AuthenticatedInputSource::LocalOperator,
},
Method::SubmitIfIdle {
submission_request_id,
input,
..
} => Method::SubmitIfIdle {
submission_request_id,
input,
source: AuthenticatedInputSource::LocalOperator,
},
Method::Notify {
notification_request_id,
message,
}
| Method::NotifyTracked {
notification_request_id,
message,
..
} => Method::NotifyTracked {
notification_request_id,
message,
source: AuthenticatedInputSource::LocalOperator,
},
method => method,
}
}
async fn run_protocol_session(
handle: worker::WorkerHandle,
streams: WorkerProtocolSessionStreams,
@@ -438,6 +597,7 @@ async fn run_protocol_session(
let Ok(method) = decode_method(&message) else {
return;
};
let method = authorize_local_input(method);
if let Some(event) = dispatch_worker_protocol_method(&handle, method).await
&& !send_protocol_event(&peer, event).await
{
@@ -578,6 +738,9 @@ fn classify_startup_error(error: WorkerBootstrapError) -> StandaloneStartupError
WorkerBootstrapError::Worker(WorkerError::Provider(_)) => {
StandaloneStartupError::ModelProvider
}
WorkerBootstrapError::Worker(WorkerError::SubjektivSessionAttribution {
message, ..
}) => StandaloneStartupError::Subject(message),
WorkerBootstrapError::Worker(_) => StandaloneStartupError::WorkerConfiguration,
WorkerBootstrapError::Controller { source, .. }
if source.kind() == std::io::ErrorKind::Other =>
@@ -587,3 +750,49 @@ fn classify_startup_error(error: WorkerBootstrapError) -> StandaloneStartupError
WorkerBootstrapError::Controller { .. } => StandaloneStartupError::Controller,
}
}
#[cfg(test)]
mod local_input_tests {
use super::*;
use protocol::{AuthenticatedInputSource, Method};
#[test]
fn local_transport_stamps_operator_without_backend_account() {
let source = AuthenticatedInputSource::Account {
account_id: "forged-account".into(),
};
for method in [
Method::submit_text("plain", "hello"),
Method::SubmitTracked {
submission_request_id: "tracked".into(),
input: vec![],
source: source.clone(),
},
Method::SubmitIfIdle {
submission_request_id: "idle".into(),
input: vec![],
source: source.clone(),
},
Method::Notify {
notification_request_id: "plain-notify".into(),
message: "hello".into(),
},
Method::NotifyTracked {
notification_request_id: "tracked-notify".into(),
message: "hello".into(),
source,
},
] {
let decoded: Method =
serde_json::from_str(&serde_json::to_string(&method).unwrap()).unwrap();
match authorize_local_input(decoded) {
Method::SubmitTracked { source, .. }
| Method::SubmitIfIdle { source, .. }
| Method::NotifyTracked { source, .. } => {
assert_eq!(source, AuthenticatedInputSource::LocalOperator)
}
other => panic!("unexpected local input {other:?}"),
}
}
}
}
+73 -7
View File
@@ -6,7 +6,6 @@
//! authority. The service owns the connection mutex and live-resource cleanup.
use std::path::Path;
use std::time::Duration;
use job::{JobRequest, JobResultSubmission};
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
@@ -14,7 +13,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use thiserror::Error;
const SCHEMA_VERSION: i64 = 1;
const SCHEMA_VERSION: i64 = 2;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -57,6 +56,8 @@ impl JobAttemptState {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct JobSnapshot {
pub request: JobRequest,
/// Durable Host-issued domain delegation, not model input or Profile identity.
pub domain_grant: Option<Value>,
pub state: JobState,
pub attempt: JobAttempt,
pub result: Option<Value>,
@@ -112,6 +113,8 @@ pub struct JobAttempt {
pub enum JobStoreError {
#[error("Job database: {0}")]
Sqlite(#[from] rusqlite::Error),
#[error(transparent)]
Storage(#[from] feature_storage::FeatureStorageError),
#[error("Job JSON: {0}")]
Json(#[from] serde_json::Error),
#[error(transparent)]
@@ -151,16 +154,30 @@ impl JobStore {
if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
std::fs::create_dir_all(parent)?;
}
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW)
.open(path)?;
}
let mut conn = Connection::open(path)?;
conn.busy_timeout(Duration::from_secs(5))?;
conn.pragma_update(None, "foreign_keys", "ON")?;
conn.pragma_update(None, "journal_mode", "WAL")?;
conn.pragma_update(None, "synchronous", "FULL")?;
feature_storage::configure_connection(&conn)?;
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let version: i64 = tx.pragma_query_value(None, "user_version", |row| row.get(0))?;
match version {
0 => {
tx.execute_batch(SCHEMA)?;
tx.execute_batch(GRANT_SCHEMA)?;
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
1 => {
tx.execute_batch(GRANT_SCHEMA)?;
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
SCHEMA_VERSION => {}
@@ -186,7 +203,17 @@ impl JobStore {
}
pub(crate) fn reserve(&mut self, request: JobRequest) -> Result<JobSnapshot, JobStoreError> {
self.reserve_granted(request, None)
}
pub(crate) fn reserve_granted(
&mut self,
request: JobRequest,
grant: Option<Value>,
) -> Result<JobSnapshot, JobStoreError> {
request.validate()?;
if let Some(value) = &grant {
job::result_digest(value, job::ABSOLUTE_MAX_RESULT_BYTES)?;
}
let fingerprint = request.fingerprint()?;
let request_json = serde_json::to_string(&request)?;
let tx = self
@@ -198,7 +225,10 @@ impl JobStore {
[&request.job_id],
|row| row.get(0),
)?;
if stored != fingerprint || snapshot.request != request {
if stored != fingerprint
|| snapshot.request != request
|| snapshot.domain_grant != grant
{
return Err(JobStoreError::IntentConflict(request.job_id));
}
tx.commit()?;
@@ -219,6 +249,12 @@ impl JobStore {
],
)?;
insert_attempt(&tx, &request, 1)?;
if let Some(grant) = grant {
tx.execute(
"INSERT INTO job_domain_grants(job_id, grant_json) VALUES (?1, ?2)",
params![request.job_id, serde_json::to_string(&grant)?],
)?;
}
let snapshot = require_snapshot(&tx, &request.job_id)?;
tx.commit()?;
Ok(snapshot)
@@ -255,6 +291,16 @@ impl JobStore {
require_snapshot(&self.conn, job_id)
}
pub(crate) fn unacknowledged(&self) -> Result<Vec<JobSnapshot>, JobStoreError> {
let mut statement = self.conn.prepare("SELECT job_id FROM job_intents WHERE state = 'completed' AND acknowledged = 0 ORDER BY rowid")?;
let ids = statement
.query_map([], |row| row.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
ids.into_iter()
.map(|id| require_snapshot(&self.conn, &id))
.collect()
}
/// Only unstarted current attempts are eligible for service startup/resume.
pub(crate) fn pending(&self) -> Result<Vec<JobSnapshot>, JobStoreError> {
let mut statement = self.conn.prepare(
@@ -570,6 +616,15 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
};
Ok(JobSnapshot {
request: serde_json::from_str(&request)?,
domain_grant: conn
.query_row(
"SELECT grant_json FROM job_domain_grants WHERE job_id = ?1",
[job_id],
|row| row.get::<_, String>(0),
)
.optional()?
.map(|s| serde_json::from_str(&s))
.transpose()?,
state,
acknowledged,
attempt: JobAttempt {
@@ -585,6 +640,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
.transpose()
}
const GRANT_SCHEMA: &str = "
CREATE TABLE job_domain_grants (
job_id TEXT PRIMARY KEY NOT NULL REFERENCES job_intents(job_id),
grant_json TEXT NOT NULL
);
CREATE TRIGGER job_grant_immutable BEFORE UPDATE ON job_domain_grants
BEGIN SELECT RAISE(ABORT, 'immutable Job domain grant'); END;
CREATE TRIGGER job_grant_retained BEFORE DELETE ON job_domain_grants
BEGIN SELECT RAISE(ABORT, 'retained Job domain grant'); END;
";
const SCHEMA: &str = "
CREATE TABLE job_intents (
job_id TEXT PRIMARY KEY NOT NULL,
+119 -4
View File
@@ -27,6 +27,17 @@ pub enum JobServiceError {
Cleanup(String),
}
pub(crate) trait JobAttemptFence: Send + Sync {
fn ensure_live(&self) -> Result<(), String>;
}
pub(crate) trait JobDomainProvider: Send + Sync {
fn grant(
&self,
snapshot: &JobSnapshot,
fence: Arc<dyn JobAttemptFence>,
) -> Result<worker::job::JobFeatureGrant, String>;
}
struct Execution {
cancel: watch::Sender<bool>,
complete: watch::Receiver<Option<Result<(), String>>>,
@@ -48,6 +59,8 @@ pub struct StandaloneJobs {
executions: Arc<Mutex<Executions>>,
cwd: PathBuf,
capacity: u16,
domain: Arc<Mutex<Option<Arc<dyn JobDomainProvider>>>>,
model_client: Arc<Mutex<Option<Box<dyn LlmClient>>>>,
}
impl StandaloneJobs {
@@ -60,9 +73,43 @@ impl StandaloneJobs {
})),
cwd,
capacity: job::DEFAULT_MAX_CONCURRENT_JOBS,
domain: Arc::default(),
model_client: Arc::default(),
})
}
pub(crate) fn bind_model_client(&self, client: Box<dyn LlmClient>) {
*self
.model_client
.lock()
.expect("Job model transport poisoned") = Some(client);
}
pub(crate) fn bind_domain(
&self,
provider: Arc<dyn JobDomainProvider>,
) -> Result<(), JobServiceError> {
let mut domain = self.domain.lock().expect("Job domain poisoned");
if domain.is_some() {
return Err(JobServiceError::Configuration(
"Job domain already bound".into(),
));
}
*domain = Some(provider);
Ok(())
}
pub(crate) fn request_granted(
&self,
request: JobRequest,
grant: serde_json::Value,
) -> Result<JobSnapshot, JobServiceError> {
let executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
self.with_store(|store| store.reserve_granted(request, Some(grant)))
}
fn with_store<T>(
&self,
operation: impl FnOnce(&mut JobStore) -> Result<T, JobStoreError>,
@@ -91,6 +138,10 @@ impl StandaloneJobs {
Ok(self.with_store(|store| store.pending())?)
}
pub fn unacknowledged_results(&self) -> Result<Vec<JobSnapshot>, JobServiceError> {
self.with_store(|store| store.unacknowledged())
}
/// Explicit new attempt, bounded by immutable intent limits. Unknown
/// outcomes cannot be retried without domain reconciliation/new intent.
pub fn retry(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
@@ -154,9 +205,40 @@ impl StandaloneJobs {
job_id: &str,
client: Option<Box<dyn LlmClient>>,
) -> Result<(), JobServiceError> {
let client = client.or_else(|| {
self.model_client
.lock()
.expect("Job model transport poisoned")
.as_ref()
.map(|c| c.clone_boxed())
});
let snapshot = self.get(job_id)?;
let prepared = worker::job::prepare_job(&snapshot.request, &self.cwd, client)
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
let deadline = tokio::time::Instant::now() + timeout;
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
store: self.store.clone(),
job_id: job_id.into(),
attempt_id: snapshot.attempt.attempt_id.clone(),
input_revision: snapshot.request.input_revision.clone(),
deadline,
});
let grant = match (
&snapshot.domain_grant,
self.domain.lock().expect("Job domain poisoned").as_ref(),
) {
(Some(_), Some(provider)) => provider
.grant(&snapshot, fence)
.map_err(JobServiceError::Configuration)?,
(Some(_), None) => {
return Err(JobServiceError::Configuration(
"persisted Job requires an unavailable explicit domain capability".into(),
));
}
(None, _) => worker::job::JobFeatureGrant::default(),
};
let prepared =
worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant)
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
@@ -209,13 +291,12 @@ impl StandaloneJobs {
let store = self.store.clone();
let id = job_id.to_string();
let attempt_id = snapshot.attempt.attempt_id.clone();
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
let sink = Arc::new(BoundResultSink {
store: store.clone(),
job_id: id.clone(),
attempt_id: attempt_id.clone(),
input_revision: snapshot.request.input_revision.clone(),
deadline: tokio::time::Instant::now() + timeout,
deadline,
});
let (cancel, mut cancellation) = watch::channel(false);
let (complete_tx, complete) = watch::channel(None);
@@ -341,6 +422,11 @@ impl StandaloneJobs {
// Close SQLite before the Host releases its lease. Cloned handles may
// outlive the Host but cannot mutate or retain the live connection.
self.store.lock().expect("Job store poisoned").take();
self.domain.lock().expect("Job domain poisoned").take();
self.model_client
.lock()
.expect("Job model transport poisoned")
.take();
error.map_or(Ok(()), Err)
}
}
@@ -391,3 +477,32 @@ impl worker::job::JobResultSink for BoundResultSink {
.map_err(|error| error.to_string())
}
}
struct BoundAttemptFence {
store: Arc<Mutex<Option<JobStore>>>,
job_id: String,
attempt_id: String,
input_revision: String,
deadline: tokio::time::Instant,
}
impl JobAttemptFence for BoundAttemptFence {
fn ensure_live(&self) -> Result<(), String> {
if tokio::time::Instant::now() >= self.deadline {
return Err("Job domain deadline expired".into());
}
let slot = self.store.lock().map_err(|_| "Job store poisoned")?;
let snapshot = slot
.as_ref()
.ok_or("Job Host closed")?
.get(&self.job_id)
.map_err(|e| e.to_string())?;
if snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Dispatched
|| snapshot.attempt.attempt_id != self.attempt_id
|| snapshot.request.input_revision != self.input_revision
{
return Err("Job domain grant is not bound to the current live attempt".into());
}
Ok(())
}
}
+345
View File
@@ -1,4 +1,10 @@
use super::*;
use agen::llm_client::{ClientError, Request, event::Event};
use async_trait::async_trait;
use futures::Stream;
use serde_json::{Value, json};
use std::collections::VecDeque;
use std::pin::Pin;
use worker::job::JobResultSink;
fn request(id: &str) -> JobRequest {
@@ -32,6 +38,345 @@ fn result(id: &str, attempt: &str) -> JobResultSubmission {
}
}
#[derive(Clone, Default)]
struct DomainTestClient {
scripts: Arc<Mutex<VecDeque<Vec<Event>>>>,
requests: Arc<Mutex<Vec<Request>>>,
entered: Arc<tokio::sync::Notify>,
stall: bool,
failure_release: Option<Arc<tokio::sync::Notify>>,
}
impl DomainTestClient {
fn success() -> Self {
Self {
scripts: Arc::new(Mutex::new(VecDeque::from([
vec![
Event::tool_use_start(0, "result-1", "SubmitJobResult"),
Event::tool_input_delta(0, r#"{"result":{"valid":true}}"#),
Event::tool_use_stop(0),
],
vec![
Event::text_block_start(0),
Event::text_delta(0, "done"),
Event::text_block_stop(0, None),
],
]))),
..Default::default()
}
}
}
#[async_trait]
impl LlmClient for DomainTestClient {
fn clone_boxed(&self) -> Box<dyn LlmClient> {
Box::new(self.clone())
}
async fn stream(
&self,
request: Request,
) -> Result<Pin<Box<dyn Stream<Item = Result<Event, ClientError>> + Send>>, ClientError> {
self.requests.lock().unwrap().push(request);
self.entered.notify_one();
if let Some(release) = &self.failure_release {
release.notified().await;
return Err(ClientError::Config(
"known scripted provider failure".into(),
));
}
if self.stall {
return Ok(Box::pin(futures::stream::pending()));
}
let events = self
.scripts
.lock()
.unwrap()
.pop_front()
.expect("unexpected model turn");
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
}
}
type CapturedDomainAttempt = (JobSnapshot, Arc<dyn JobAttemptFence>);
#[derive(Default)]
struct RecordingDomainProvider(Mutex<Vec<CapturedDomainAttempt>>);
fn domain_grant(id: &str) -> Value {
json!({"consumer":"test-domain", "job_id":id, "input_revision":"r1"})
}
fn domain_request(id: &str) -> JobRequest {
let mut intent = request(id);
intent.profile = "builtin:standalone-subjektiv-consolidation".into();
intent
}
impl JobDomainProvider for RecordingDomainProvider {
fn grant(
&self,
snapshot: &JobSnapshot,
fence: Arc<dyn JobAttemptFence>,
) -> Result<worker::job::JobFeatureGrant, String> {
if snapshot.domain_grant != Some(domain_grant(&snapshot.request.job_id)) {
return Err("foreign or mismatched immutable domain grant".into());
}
self.0.lock().unwrap().push((snapshot.clone(), fence));
Ok(worker::job::JobFeatureGrant {
satisfied_requirements: vec!["feature.subjektiv"],
..Default::default()
})
}
}
async fn await_model(client: &DomainTestClient) {
tokio::time::timeout(Duration::from_secs(2), client.entered.notified())
.await
.unwrap();
}
async fn await_job(jobs: &StandaloneJobs, id: &str) -> JobSnapshot {
tokio::time::timeout(Duration::from_secs(3), jobs.wait(id))
.await
.unwrap()
.unwrap()
}
#[tokio::test]
async fn pending_domain_grants_survive_service_reopen_and_remain_immutable() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("jobs.sqlite3");
let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
let intent = domain_request("granted");
let grant = domain_grant("granted");
let reserved = jobs.request_granted(intent.clone(), grant.clone()).unwrap();
let root = jobs.request(request("root")).unwrap();
assert_eq!(reserved.domain_grant, Some(grant.clone()));
assert!(root.domain_grant.is_none());
assert_eq!(
jobs.request_granted(intent.clone(), grant.clone()).unwrap(),
reserved
);
assert!(matches!(
jobs.request(intent.clone()),
Err(JobServiceError::Store(JobStoreError::IntentConflict(_)))
));
assert!(matches!(
jobs.request_granted(intent.clone(), domain_grant("foreign")),
Err(JobServiceError::Store(JobStoreError::IntentConflict(_)))
));
assert!(matches!(
jobs.request_granted(request("root"), grant.clone()),
Err(JobServiceError::Store(JobStoreError::IntentConflict(_)))
));
assert_eq!(jobs.pending().unwrap().len(), 2);
jobs.shutdown().await.unwrap();
let reopened = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
let pending = reopened.pending().unwrap();
assert_eq!(pending.len(), 2);
assert!(pending.contains(&reserved));
assert!(pending.contains(&root));
assert_eq!(reopened.get("granted").unwrap(), reserved);
assert_eq!(
reopened
.request_granted(intent.clone(), grant.clone())
.unwrap(),
reserved
);
assert!(matches!(
reopened.request_granted(intent, json!({"different":true})),
Err(JobServiceError::Store(JobStoreError::IntentConflict(_)))
));
let provider = Arc::new(RecordingDomainProvider::default());
reopened.bind_domain(provider.clone()).unwrap();
reopened
.start_with_model_client("granted", DomainTestClient::success())
.await
.unwrap();
let completed = await_job(&reopened, "granted").await;
assert_eq!(completed.state, JobState::Completed);
assert_eq!(completed.domain_grant, Some(grant));
assert_eq!(completed.result, Some(json!({"valid":true})));
let captured = provider.0.lock().unwrap()[0].0.clone();
assert_eq!(
captured, reserved,
"provider receives the persisted immutable intent and grant"
);
reopened.shutdown().await.unwrap();
}
#[tokio::test]
async fn persisted_grant_requires_provider_and_missing_grant_cannot_be_derived_from_profile() {
let temp = tempfile::tempdir().unwrap();
let jobs =
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
let granted = jobs
.request_granted(domain_request("granted"), domain_grant("granted"))
.unwrap();
let client = DomainTestClient::success();
let error = jobs
.start_with_model_client("granted", client.clone())
.await
.unwrap_err();
assert!(matches!(error, JobServiceError::Configuration(_)));
assert!(
error
.to_string()
.contains("unavailable explicit domain capability"),
"{error}"
);
assert_eq!(jobs.get("granted").unwrap(), granted);
assert!(client.requests.lock().unwrap().is_empty());
let provider = Arc::new(RecordingDomainProvider::default());
jobs.bind_domain(provider.clone()).unwrap();
assert!(matches!(
jobs.bind_domain(provider.clone()),
Err(JobServiceError::Configuration(_))
));
let ungranted = jobs.request(domain_request("ungranted")).unwrap();
let error = jobs
.start_with_model_client("ungranted", client.clone())
.await
.unwrap_err();
assert!(matches!(error, JobServiceError::Configuration(_)));
assert!(error.to_string().contains("feature.subjektiv"), "{error}");
assert_eq!(jobs.get("ungranted").unwrap(), ungranted);
assert!(
provider.0.lock().unwrap().is_empty(),
"bound provider is not a profile-derived grant"
);
assert!(client.requests.lock().unwrap().is_empty());
jobs.shutdown().await.unwrap();
}
#[tokio::test]
async fn foreign_job_revision_and_consumer_grants_are_refused_before_model_dispatch() {
let temp = tempfile::tempdir().unwrap();
let jobs =
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
let provider = Arc::new(RecordingDomainProvider::default());
jobs.bind_domain(provider.clone()).unwrap();
for (id, grant) in [
("foreign-job", domain_grant("another-job")),
(
"foreign-revision",
json!({"consumer":"test-domain","job_id":"foreign-revision","input_revision":"r0"}),
),
(
"foreign-consumer",
json!({"consumer":"another-domain","job_id":"foreign-consumer","input_revision":"r1"}),
),
] {
let reserved = jobs.request_granted(domain_request(id), grant).unwrap();
let client = DomainTestClient::success();
let error = jobs
.start_with_model_client(id, client.clone())
.await
.unwrap_err();
assert!(matches!(error, JobServiceError::Configuration(_)));
assert!(
error.to_string().contains("foreign or mismatched"),
"{error}"
);
assert_eq!(jobs.get(id).unwrap(), reserved);
assert!(client.requests.lock().unwrap().is_empty());
}
assert!(provider.0.lock().unwrap().is_empty());
jobs.shutdown().await.unwrap();
}
#[tokio::test]
async fn domain_fence_is_live_only_for_current_dispatched_attempt_and_revoked_on_cancel() {
let temp = tempfile::tempdir().unwrap();
let jobs =
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
let provider = Arc::new(RecordingDomainProvider::default());
jobs.bind_domain(provider.clone()).unwrap();
let reserved = jobs
.request_granted(domain_request("fenced"), domain_grant("fenced"))
.unwrap();
let failure_release = Arc::new(tokio::sync::Notify::new());
let first = DomainTestClient {
failure_release: Some(failure_release.clone()),
..Default::default()
};
jobs.start_with_model_client("fenced", first.clone())
.await
.unwrap();
await_model(&first).await;
let old_fence = provider.0.lock().unwrap()[0].1.clone();
old_fence.ensure_live().unwrap();
failure_release.notify_one();
let failed = await_job(&jobs, "fenced").await;
assert_eq!(failed.state, JobState::Failed);
assert!(failed.result.is_none());
assert!(old_fence.ensure_live().is_err());
// Only known failure permits a retry; cancellation remains terminal.
let retried = jobs.retry("fenced").unwrap();
assert_ne!(retried.attempt.attempt_id, reserved.attempt.attempt_id);
assert_eq!(retried.domain_grant, reserved.domain_grant);
assert!(old_fence.ensure_live().is_err());
let second = DomainTestClient {
stall: true,
..Default::default()
};
jobs.start_with_model_client("fenced", second.clone())
.await
.unwrap();
await_model(&second).await;
let current_fence = provider.0.lock().unwrap()[1].1.clone();
current_fence.ensure_live().unwrap();
assert!(
old_fence.ensure_live().is_err(),
"a new attempt must not revive a stale capability"
);
let cancelled = jobs.cancel("fenced").unwrap();
assert_eq!(cancelled.state, JobState::Cancelled);
assert!(
current_fence.ensure_live().is_err(),
"durable cancellation revokes current domain operations immediately"
);
assert!(old_fence.ensure_live().is_err());
let cleaned = await_job(&jobs, "fenced").await;
assert_eq!(cleaned.state, JobState::Cancelled);
assert!(cleaned.result.is_none());
assert!(
jobs.retry("fenced").is_err(),
"cancelled intent is not a known retryable failure"
);
jobs.shutdown().await.unwrap();
assert!(old_fence.ensure_live().is_err());
assert!(current_fence.ensure_live().is_err());
}
#[tokio::test]
async fn bound_domain_does_not_change_root_generic_result_only_execution() {
let temp = tempfile::tempdir().unwrap();
let jobs =
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
let provider = Arc::new(RecordingDomainProvider::default());
jobs.bind_domain(provider.clone()).unwrap();
let root = jobs.request(request("root")).unwrap();
assert!(root.domain_grant.is_none());
let client = DomainTestClient::success();
jobs.start_with_model_client("root", client.clone())
.await
.unwrap();
let completed = await_job(&jobs, "root").await;
assert_eq!(completed.state, JobState::Completed);
assert_eq!(completed.result, Some(json!({"valid":true})));
assert!(completed.domain_grant.is_none());
assert!(provider.0.lock().unwrap().is_empty());
let requests = client.requests.lock().unwrap();
assert_eq!(requests.len(), 2);
for sent in requests.iter() {
assert_eq!(
sent.tools
.iter()
.map(|tool| tool.name.as_str())
.collect::<Vec<_>>(),
["SubmitJobResult"]
);
assert_eq!(sent.config.max_tokens, Some(worker::job::MAX_JOB_TOKENS));
}
drop(requests);
jobs.shutdown().await.unwrap();
}
#[tokio::test]
async fn host_result_capability_cannot_cross_job_attempt_revision_or_deadline() {
let temp = tempfile::tempdir().unwrap();
+32
View File
@@ -15,6 +15,7 @@ pub struct StandaloneLaunchConfig {
pub state_dir: PathBuf,
pub profile: ProfileSelector,
pub worker_name: String,
pub subject_id: Option<String>,
}
pub struct ResolvedStandaloneLaunch {
@@ -22,6 +23,7 @@ pub struct ResolvedStandaloneLaunch {
pub state_dir: PathBuf,
pub profile: ResolvedProfile,
pub prompt_catalog: PromptCatalogSource,
pub subject_id: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
@@ -32,6 +34,10 @@ pub enum StandaloneLaunchError {
PathProfileUnsupported,
#[error("the standalone profile could not be resolved")]
ProfileResolutionFailed,
#[error(
"local Subject selection requires an enabled, non-consolidator subjektiv Profile and canonical Subject id"
)]
InvalidSubjectPolicy,
}
impl StandaloneLaunchConfig {
@@ -46,9 +52,15 @@ impl StandaloneLaunchConfig {
state_dir: state_dir.into(),
profile,
worker_name: worker_name.into(),
subject_id: None,
}
}
pub fn with_subject(mut self, subject_id: String) -> Self {
self.subject_id = Some(subject_id);
self
}
/// Resolve only built-in/XDG profile authority and bind standalone scope
/// to the canonical process cwd. Repository-local profile discovery is
/// deliberately not part of this path.
@@ -74,12 +86,32 @@ impl StandaloneLaunchConfig {
.map_err(|_| StandaloneLaunchError::ProfileResolutionFailed)?;
validate_profile_execution_target(&profile.manifest, ProfileExecutionTarget::Standalone)
.map_err(|_| StandaloneLaunchError::ProfileResolutionFailed)?;
if let Some(id) = &self.subject_id {
if !crate::subjektiv::valid_subject_id(id)
|| !profile.manifest.feature.subjektiv.profile.enabled
|| profile
.manifest
.feature
.subjektiv
.profile
.consolidation_tools
|| profile
.manifest
.feature
.subjektiv
.workspace_settings
.is_some()
{
return Err(StandaloneLaunchError::InvalidSubjectPolicy);
}
}
Ok(ResolvedStandaloneLaunch {
cwd,
state_dir: self.state_dir,
profile,
prompt_catalog: PromptCatalogSource::builtins_only(),
subject_id: self.subject_id,
})
}
}
+2
View File
@@ -9,6 +9,8 @@ mod job_store;
pub mod jobs;
pub mod launch;
pub mod store;
pub mod subjektiv;
mod subjektiv_sessions;
pub use host::{StandaloneHost, StandaloneShutdownError, StandaloneStartupError};
pub use launch::{ResolvedStandaloneLaunch, StandaloneLaunchConfig, StandaloneLaunchError};
+34
View File
@@ -87,6 +87,9 @@ pub struct StandaloneWorkerRecord {
pub storage_key: String,
pub cwd: StandaloneCwdIdentity,
pub manifest: WorkerManifest,
/// Original explicit local Subject/storage binding; never inferred from policy.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub subject: Option<crate::subjektiv::StandaloneSubjectBinding>,
pub active_session_id: SessionId,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub active_segment_id: Option<SegmentId>,
@@ -127,6 +130,16 @@ impl StandaloneWorkerStore {
Ok(Self { root })
}
/// Observational constructor: never creates a missing root or repairs state.
pub fn open_existing(root: impl Into<PathBuf>) -> Result<Self, StandaloneStoreError> {
let root = root.into();
let metadata = fs::symlink_metadata(&root).map_err(StandaloneStoreError::Io)?;
if !metadata.is_dir() || metadata.file_type().is_symlink() {
return Err(StandaloneStoreError::NotDirectory);
}
Ok(Self { root })
}
#[must_use]
pub fn root(&self) -> &Path {
&self.root
@@ -158,6 +171,25 @@ impl StandaloneWorkerStore {
storage_key: String,
active_session_id: SessionId,
active_segment_id: Option<SegmentId>,
) -> Result<StandaloneWorkerRecord, StandaloneStoreError> {
self.commit_created_connected(
allocation,
manifest,
storage_key,
active_session_id,
active_segment_id,
None,
)
}
pub(crate) fn commit_created_connected(
&self,
allocation: &StandaloneWorkerAllocation,
manifest: WorkerManifest,
storage_key: String,
active_session_id: SessionId,
active_segment_id: Option<SegmentId>,
subject: Option<crate::subjektiv::StandaloneSubjectBinding>,
) -> Result<StandaloneWorkerRecord, StandaloneStoreError> {
let now = now_unix_ms()?;
let record = StandaloneWorkerRecord {
@@ -168,6 +200,7 @@ impl StandaloneWorkerStore {
storage_key,
cwd: allocation.cwd.clone(),
manifest,
subject,
active_session_id,
active_segment_id,
status: StandaloneWorkerStatus::Active,
@@ -793,6 +826,7 @@ permission = "write"
inode: None,
},
manifest,
subject: None,
active_session_id: "01a05782-d5dd-78f1-b9cd-ce37535bdb9e".parse().unwrap(),
active_segment_id: None,
status: StandaloneWorkerStatus::Stopped,
File diff suppressed because it is too large Load Diff
+130
View File
@@ -0,0 +1,130 @@
use super::*;
#[test]
fn catalog_reopen_preserves_random_scope_subject_and_behavior() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("state");
let catalog = StandaloneSubjects::open(&root).unwrap();
let subject = catalog
.create(
"continuing-owner",
Some("Keep user behavior separate from Memory."),
)
.unwrap();
let scope = catalog.scope_id().to_owned();
assert!(uuid::Uuid::parse_str(&scope).is_ok());
assert!(uuid::Uuid::parse_str(subject.id.strip_prefix("subject-").unwrap()).is_ok());
assert_ne!(scope, subject.id);
drop(catalog);
let reopened = StandaloneSubjects::open_existing(&root).unwrap();
assert_eq!(reopened.scope_id(), scope);
assert_eq!(reopened.list().unwrap(), vec![subject]);
}
#[test]
fn implicit_open_does_not_create_subject_storage() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("missing");
assert!(StandaloneSubjects::open_existing(&root).is_err());
assert!(!root.exists());
assert!(StandaloneSubjects::open_existing(temp.path()).is_err());
assert!(!temp.path().join("subjektiv").exists());
}
#[test]
fn subject_kernel_lease_is_not_worker_or_profile_scoped() {
let temp = tempfile::tempdir().unwrap();
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
let a = catalog.create("a", None).unwrap();
let b = catalog.create("b", None).unwrap();
let mut first = catalog
.connect(&a.id, protocol::WorkerId::now_v7())
.unwrap();
assert!(matches!(
StandaloneSubjects::open_existing(&temp.path().join("state"))
.unwrap()
.connect(&a.id, protocol::WorkerId::now_v7()),
Err(SubjectError::Active(_))
));
let mut second = StandaloneSubjects::open_existing(&temp.path().join("state"))
.unwrap()
.connect(&b.id, protocol::WorkerId::now_v7())
.unwrap();
first.close().unwrap();
let mut replacement = StandaloneSubjects::open_existing(&temp.path().join("state"))
.unwrap()
.connect(&a.id, protocol::WorkerId::now_v7())
.unwrap();
replacement.close().unwrap();
second.close().unwrap();
}
#[test]
fn closed_capabilities_and_foreign_worker_context_are_rejected() {
let temp = tempfile::tempdir().unwrap();
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
let subject = catalog.create("owner", None).unwrap();
let id = protocol::WorkerId::now_v7();
let mut connection = catalog.connect(&subject.id, id).unwrap();
let port = connection.host.clone();
let foreign = SubjektivHostContext {
worker_id: protocol::WorkerId::now_v7().to_string(),
session_id: session_store::SessionId::now_v7().to_string(),
};
let read = SubjektivMemoryBackendOperation::ResidentContext(
memory::backend::MemoryResidentSummaryOperation {},
);
assert!(port.memory(&foreign, read.clone()).is_err());
let owner = SubjektivHostContext {
worker_id: format!("standalone-{id}"),
session_id: session_store::SessionId::now_v7().to_string(),
};
assert!(port.memory(&owner, read.clone()).is_ok());
assert!(
port.record_session(&owner, false).is_err(),
"unattributed Session is not read authority"
);
connection.close().unwrap();
assert!(port.memory(&owner, read).is_err());
}
#[cfg(unix)]
#[test]
fn scope_rejects_symlinks_and_non_private_roots() {
use std::os::unix::fs::{PermissionsExt, symlink};
let temp = tempfile::tempdir().unwrap();
let alias = temp.path().join("alias");
let real = temp.path().join("real");
fs::create_dir(&real).unwrap();
fs::set_permissions(&real, fs::Permissions::from_mode(0o700)).unwrap();
symlink(&real, &alias).unwrap();
assert!(StandaloneSubjects::open(&alias).is_err());
let public = temp.path().join("public");
fs::create_dir(&public).unwrap();
fs::set_permissions(&public, fs::Permissions::from_mode(0o755)).unwrap();
assert!(StandaloneSubjects::open(&public).is_err());
let catalog = StandaloneSubjects::open(&real).unwrap();
drop(catalog);
let scope = real.join("subjektiv/scope.json");
let original = fs::read(&scope).unwrap();
fs::remove_file(&scope).unwrap();
let foreign = temp.path().join("foreign.json");
fs::write(&foreign, &original).unwrap();
symlink(&foreign, &scope).unwrap();
assert!(StandaloneSubjects::open_existing(&real).is_err());
assert_eq!(fs::read(&foreign).unwrap(), original);
}
#[test]
fn different_local_roots_do_not_authorize_shared_subject_ids() {
let temp = tempfile::tempdir().unwrap();
let first = StandaloneSubjects::open(temp.path().join("first")).unwrap();
let subject = first.create("owner", None).unwrap();
let other = StandaloneSubjects::open(temp.path().join("other")).unwrap();
assert_ne!(first.scope_id(), other.scope_id());
assert!(
other
.connect(&subject.id, protocol::WorkerId::now_v7())
.is_err()
);
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff