Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

Merge remote-tracking branch 'origin/develop' into work/T-689-web-subject-create

Browse Source
This commit is contained in:
Hare
2026-10-04 22:35:43 +09:00
parent 1c5966c535 728f03952e
commit 3a9def7b80
57 changed files with 3768 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/Cargo.toml
+1 -1
View File
@@ -40,6 +40,7 @@ tokio-tungstenite = { workspace = true, features = ["rustls-tls-webpki-roots"] }
worker.workspace = true
workdir = { workspace = true, features = ["http-client"] }
worker-runtime.workspace = true
session-store.workspace = true
toml.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
@@ -51,7 +52,6 @@ webauthn-rs = { workspace = true }
[dev-dependencies]
agen.workspace = true
session-store.workspace = true
tempfile.workspace = true
tools.workspace = true
yoi = { path = "../yoi" }
crates/workspace-server/src/backend_job.rs
+37
View File
@@ -284,6 +284,38 @@ pub struct BackendJobRecord {
pub completed_at: Option<String>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum BackendJobWorkerCleanupState {
Pending,
Executing,
Completed,
Failed,
}
impl BackendJobWorkerCleanupState {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Pending => "pending",
Self::Executing => "executing",
Self::Completed => "completed",
Self::Failed => "failed",
}
}
pub(crate) fn parse(value: &str) -> Result<Self> {
match value {
"pending" => Ok(Self::Pending),
"executing" => Ok(Self::Executing),
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
other => Err(Error::Store(format!(
"unknown Backend Job Worker cleanup state `{other}`"
))),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct BackendJobAttemptRecord {
pub workspace_id: String,
@@ -300,6 +332,11 @@ pub struct BackendJobAttemptRecord {
pub result_digest: Option<String>,
pub failure_category: Option<String>,
pub failure_detail: Option<String>,
pub worker_cleanup_state: Option<BackendJobWorkerCleanupState>,
pub worker_cleanup_failure_category: Option<String>,
pub worker_cleanup_failure_detail: Option<String>,
pub worker_cleanup_updated_at: Option<String>,
pub worker_cleanup_completed_at: Option<String>,
pub created_at: String,
pub updated_at: String,
pub completed_at: Option<String>,
crates/workspace-server/src/hosts.rs
+182 -1
View File
@@ -259,6 +259,9 @@ pub struct InternalWorkerSummary {
pub availability: protocol::subscription::SubscriptionWorkerAvailability,
/// Runtime catalog lifecycle compatibility state.
pub state: String,
/// Runtime-process reconstruction evidence used only for lifecycle coordination.
#[serde(skip)]
pub execution_reconstructed: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
pub last_seen_at: Option<String>,
@@ -770,6 +773,12 @@ pub struct WorkerProxyConnectPoint {
pub diagnostics: Vec<RuntimeDiagnostic>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RuntimeSessionAttachment {
pub media_type: String,
pub data: Vec<u8>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RuntimeRegistryError {
InvalidIdentifier {
@@ -1220,6 +1229,23 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
})
}
fn worker_session_attachment(
&self,
_workspace_id: &str,
worker_ref: &EmbeddedWorkerRef,
_session_id: &str,
_attachment_id: &str,
) -> Result<RuntimeSessionAttachment, RuntimeRegistryError> {
Err(RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id().to_string(),
code: "session_attachment_unsupported".to_string(),
message: format!(
"Session attachment reads are not supported for Worker `{}`",
worker_ref.worker_id
),
})
}
fn session_public_search(
&self,
_request: runtime_api::SessionPublicSearchRequest,
@@ -2096,6 +2122,27 @@ impl RuntimeRegistry {
})
}
pub fn worker_session_attachment(
&self,
workspace_id: &str,
worker: &RuntimeWorkerRef,
session_id: &str,
attachment_id: &str,
) -> Result<RuntimeSessionAttachment, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", &worker.runtime_id)?;
validate_backend_identifier("worker_id", &worker.worker_id)?;
validate_backend_identifier("session_id", session_id)?;
validate_backend_identifier("attachment_id", attachment_id)?;
let runtime = self.runtime(&worker.runtime_id)?;
let worker_ref =
EmbeddedWorkerRef::new(EmbeddedWorkerId::parse(&worker.worker_id).ok_or_else(
|| RuntimeRegistryError::UnknownWorker {
worker: worker.clone(),
},
)?);
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
}
pub fn session_public_search(
&self,
runtime_id: &str,
@@ -2333,6 +2380,7 @@ impl EmbeddedWorkerRuntime {
summary.execution_metadata_available,
)
.to_string(),
execution_reconstructed: false,
worker_state: summary.worker_state.clone(),
last_seen_at: None,
pinned: false,
@@ -2373,6 +2421,7 @@ impl EmbeddedWorkerRuntime {
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
state: embedded_worker_state_label(detail.status, detail.execution_metadata_available)
.to_string(),
execution_reconstructed: detail.execution_reconstructed,
worker_state: detail.worker_state.clone(),
last_seen_at: None,
pinned: false,
@@ -3017,6 +3066,46 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
.map_err(|error| error.to_string())
}
fn worker_session_attachment(
&self,
workspace_id: &str,
worker_ref: &EmbeddedWorkerRef,
session_id: &str,
attachment_id: &str,
) -> Result<RuntimeSessionAttachment, RuntimeRegistryError> {
let scope = RuntimeWorkspaceScope::new(workspace_id, "embedded-backend");
self.runtime
.worker_session_attachment_scoped(
&scope,
worker_ref,
session_id.to_string(),
attachment_id.to_string(),
)
.map(|attachment| RuntimeSessionAttachment {
media_type: attachment.media_type,
data: attachment.data,
})
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id.clone(),
code: match error {
session_store::RetainedAttachmentReadError::RetentionMissing
| session_store::RetainedAttachmentReadError::ActivePointerMissing => {
"session_attachment_expired"
}
session_store::RetainedAttachmentReadError::SessionMismatch
| session_store::RetainedAttachmentReadError::NotFound => {
"session_attachment_not_found"
}
session_store::RetainedAttachmentReadError::ResourceLimit => {
"session_attachment_too_large"
}
_ => "session_attachment_unavailable",
}
.to_string(),
message: error.to_string(),
})
}
fn session_public_search(
&self,
request: runtime_api::SessionPublicSearchRequest,
@@ -3732,7 +3821,7 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
}
if path.ends_with("/input")
|| path.ends_with("/restore")
|| path.contains("/attachments")
|| (path.contains("/attachments") && method != "GET")
|| path.contains("/workdir-attachments")
{
return "workers:input";
@@ -4059,6 +4148,64 @@ impl RemoteWorkerRuntime {
self.send_json(path, "GET", &[], self.http.get(self.endpoint(path)))
}
fn get_session_attachment(
&self,
path: &str,
) -> Result<RuntimeSessionAttachment, RuntimeDiagnostic> {
let runtime_id = self.runtime_id.clone();
let workspace_id = self.workspace_id.clone();
let bearer_token = self.bearer_token.clone();
let capability_token = match &self.workspace_authorization {
Some(authorization) => Some(authorization.issue(
"GET",
path,
workspace_runtime_operation("GET", path),
worker_id_from_remote_path(path).as_deref(),
&[],
)?),
None => None,
};
let request = self.http.get(self.endpoint(path));
run_blocking_http(move || {
let mut request = request.header(RUNTIME_WORKSPACE_SCOPE_HEADER, &workspace_id);
if let Some(token) = capability_token.as_deref().or(bearer_token.as_deref()) {
request = request.header(AUTHORIZATION, format!("Bearer {token}"));
}
let response = request
.send()
.map_err(|error| remote_reqwest_diagnostic(&runtime_id, error))?;
let status = response.status();
if !status.is_success() {
return Err(remote_http_status_diagnostic(&runtime_id, status, response));
}
let media_type = response
.headers()
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or("application/octet-stream")
.to_string();
let mut data = Vec::new();
response
.take((MAX_REMOTE_RUNTIME_RESPONSE_BYTES + 1) as u64)
.read_to_end(&mut data)
.map_err(|_| {
diagnostic(
"remote_runtime_response_read_failed",
HostDiagnosticSeverity::Error,
"Remote Runtime attachment response could not be read".to_string(),
)
})?;
if data.len() > MAX_REMOTE_RUNTIME_RESPONSE_BYTES {
return Err(diagnostic(
"remote_runtime_response_too_large",
HostDiagnosticSeverity::Error,
"Remote Runtime attachment response exceeded the allowed size".to_string(),
));
}
Ok(RuntimeSessionAttachment { media_type, data })
})
}
fn post_json<B, T>(&self, path: &str, body: &B) -> Result<T, RuntimeDiagnostic>
where
B: Serialize + ?Sized,
@@ -4226,6 +4373,7 @@ impl RemoteWorkerRuntime {
summary.execution_metadata_available,
)
.to_string(),
execution_reconstructed: false,
worker_state: summary.worker_state.clone(),
last_seen_at: None,
pinned: false,
@@ -4264,6 +4412,7 @@ impl RemoteWorkerRuntime {
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
state: embedded_worker_state_label(detail.status, detail.execution_metadata_available)
.to_string(),
execution_reconstructed: detail.execution_reconstructed,
worker_state: detail.worker_state.clone(),
last_seen_at: None,
pinned: false,
@@ -5093,6 +5242,29 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
.map_err(|diagnostic| diagnostic.message)
}
fn worker_session_attachment(
&self,
workspace_id: &str,
worker_ref: &EmbeddedWorkerRef,
session_id: &str,
attachment_id: &str,
) -> Result<RuntimeSessionAttachment, RuntimeRegistryError> {
let path = format!(
"/v1/workers/{}/sessions/{}/attachments/{}?workspace_id={}",
url_path_segment_encode(&worker_ref.worker_id.to_string()),
url_path_segment_encode(session_id),
url_path_segment_encode(attachment_id),
url_query_value_encode(workspace_id),
);
self.get_session_attachment(&path).map_err(|diagnostic| {
RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id.clone(),
code: diagnostic.code,
message: diagnostic.message,
}
})
}
fn session_public_search(
&self,
request: runtime_api::SessionPublicSearchRequest,
@@ -6106,6 +6278,7 @@ pub fn placeholder_worker(host_id: impl Into<String>) -> InternalWorkerSummary {
},
availability: protocol::subscription::SubscriptionWorkerAvailability::Unavailable,
state: "unsupported".to_string(),
execution_reconstructed: false,
worker_state: None,
last_seen_at: None,
pinned: false,
@@ -6834,6 +7007,7 @@ mod tests {
},
availability: protocol::subscription::SubscriptionWorkerAvailability::Observed,
state: "available".to_string(),
execution_reconstructed: false,
worker_state: None,
last_seen_at: None,
pinned: false,
@@ -7648,6 +7822,13 @@ mod tests {
),
"workers:input"
);
assert_eq!(
workspace_runtime_operation(
"GET",
&format!("/v1/workers/{worker_id}/sessions/session-1/attachments/attachment-1")
),
"workers:read"
);
assert_eq!(
workspace_runtime_operation("POST", SSH_HOST_KEY_PROBE_PATH),
SSH_HOST_KEY_PROBE_OPERATION
crates/workspace-server/src/latest_schema.sql
+9
View File
@@ -875,6 +875,13 @@ CREATE TABLE backend_job_attempts (
result_digest TEXT,
failure_category TEXT,
failure_detail TEXT,
worker_cleanup_state TEXT CHECK (
worker_cleanup_state IN ('pending', 'executing', 'completed', 'failed')
),
worker_cleanup_failure_category TEXT,
worker_cleanup_failure_detail TEXT,
worker_cleanup_updated_at TEXT,
worker_cleanup_completed_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
completed_at TEXT,
@@ -889,6 +896,8 @@ CREATE TABLE backend_job_attempts (
);
CREATE INDEX backend_job_attempts_recovery
ON backend_job_attempts(workspace_id, state, deadline_at);
CREATE INDEX backend_job_attempts_worker_cleanup
ON backend_job_attempts(workspace_id, worker_cleanup_state, updated_at);
CREATE TABLE backend_job_deliveries (
workspace_id TEXT NOT NULL,
delivery_id TEXT NOT NULL,
crates/workspace-server/src/profile_settings.rs
+17
View File
@@ -447,6 +447,23 @@ pub fn build_virtual_profile_config_bundle_with_prompt_projection(
Ok(Some(bundle))
}
pub(crate) fn resolve_profile_manifest_from_config_bundle(
bundle: &ConfigBundle,
selector: &str,
) -> Result<manifest::WorkerManifest> {
let archive = bundle.profile_source_archive.as_ref().ok_or_else(|| {
Error::Config(format!(
"resolved Profile {selector:?} is missing its source archive"
))
})?;
let verified = archive.verify().map_err(|error| {
Error::Config(format!("failed to verify Profile {selector:?}: {error}"))
})?;
verified
.resolve_profile(selector, Path::new("/"), "worker-launch-validation")
.map_err(|error| Error::Config(format!("failed to resolve Profile {selector:?}: {error}")))
}
fn build_virtual_profile_archive(
selector: &str,
entry: &VirtualProfileEntry,
crates/workspace-server/src/server.rs
+1298 -172
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/store.rs
+319 -8
View File
@@ -21,13 +21,14 @@ use crate::backend_job::{
BackendJobAttemptRecord, BackendJobAttemptState, BackendJobDeliveryRecord,
BackendJobDeliveryState, BackendJobRecord, BackendJobRequest, BackendJobReservation,
BackendJobResultAcceptance, BackendJobResultSubmission, BackendJobState,
BackendJobWorkerBinding, attempt_id, bounded_failure_detail, result_digest,
BackendJobWorkerBinding, BackendJobWorkerCleanupState, attempt_id, bounded_failure_detail,
result_digest,
};
use crate::workspace_deletion::WorkspaceDeletionStore;
use crate::{Error, Result};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 76;
const LATEST_SCHEMA_VERSION: i64 = 77;
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings";
const RUNTIME_BINDING_AUDIT_MIGRATION_NAME: &str = "workspace Runtime binding revision and audit";
@@ -74,6 +75,8 @@ const RUNTIME_SCOPED_WORKER_IDENTITY_MIGRATION_NAME: &str =
"Runtime-scoped durable Worker catalog identity";
const ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME: &str =
"preserve explicit observe grants for committed Worker Session archives";
const BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME: &str =
"durable terminal Backend Job Worker cleanup";
const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7;
const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline";
const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#"
@@ -251,6 +254,11 @@ const MIGRATIONS: &[Migration] = &[
name: ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME,
apply: migrate_archive_observe_grants_v75_to_v76,
},
Migration {
version: 77,
name: BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME,
apply: migrate_backend_job_worker_cleanup_v76_to_v77,
},
];
#[derive(Clone, Copy)]
@@ -1592,6 +1600,34 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
failure_detail: &str,
now: &str,
) -> Result<BackendJobAttemptRecord>;
fn recover_backend_job_worker_cleanups(
&self,
workspace_id: &str,
now: &str,
limit: usize,
) -> Result<Vec<BackendJobAttemptRecord>>;
fn list_backend_job_worker_cleanups(
&self,
workspace_id: &str,
limit: usize,
) -> Result<Vec<BackendJobAttemptRecord>>;
fn claim_backend_job_worker_cleanup(
&self,
workspace_id: &str,
job_id: &str,
attempt_id: &str,
now: &str,
) -> Result<(BackendJobAttemptRecord, bool)>;
fn finish_backend_job_worker_cleanup(
&self,
workspace_id: &str,
job_id: &str,
attempt_id: &str,
state: BackendJobWorkerCleanupState,
failure_category: Option<&str>,
failure_detail: Option<&str>,
now: &str,
) -> Result<BackendJobAttemptRecord>;
fn reserve_backend_job_delivery(
&self,
workspace_id: &str,
@@ -6648,7 +6684,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
));
}
let attempt_changed = tx.execute(
"UPDATE backend_job_attempts SET state = 'completed', result_json = ?4, result_digest = ?5, failure_category = NULL, failure_detail = NULL, completed_at = ?6, updated_at = ?6 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND state = 'dispatched'",
"UPDATE backend_job_attempts SET state = 'completed', result_json = ?4, result_digest = ?5, failure_category = NULL, failure_detail = NULL, worker_cleanup_state = 'pending', worker_cleanup_failure_category = NULL, worker_cleanup_failure_detail = NULL, worker_cleanup_updated_at = ?6, worker_cleanup_completed_at = NULL, completed_at = ?6, updated_at = ?6 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND state = 'dispatched'",
params![workspace_id, submission.job_id, submission.attempt_id, result_json, digest, now],
)?;
let job_changed = tx.execute(
@@ -6707,7 +6743,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
return Ok(attempt);
}
tx.execute(
"UPDATE backend_job_attempts SET state = ?4, failure_category = ?5, failure_detail = ?6, completed_at = ?7, updated_at = ?7 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND state IN ('reserved', 'dispatching', 'dispatched')",
"UPDATE backend_job_attempts SET state = ?4, failure_category = ?5, failure_detail = ?6, worker_cleanup_state = CASE WHEN worker_id IS NULL THEN 'completed' ELSE 'pending' END, worker_cleanup_failure_category = NULL, worker_cleanup_failure_detail = NULL, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = CASE WHEN worker_id IS NULL THEN ?7 ELSE NULL END, completed_at = ?7, updated_at = ?7 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND state IN ('reserved', 'dispatching', 'dispatched')",
params![workspace_id, job_id, attempt_id, state.as_str(), failure_category, detail, now],
)?;
if attempt.attempt == job.current_attempt && job.state == BackendJobState::Pending {
@@ -6728,6 +6764,154 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
})
}
fn recover_backend_job_worker_cleanups(
&self,
workspace_id: &str,
now: &str,
limit: usize,
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute(
"UPDATE backend_job_attempts SET worker_cleanup_state = 'pending', worker_cleanup_failure_category = 'cleanup_interrupted', worker_cleanup_failure_detail = 'Backend restart interrupted Worker cleanup; retrying canonical removal', worker_cleanup_updated_at = ?2 WHERE workspace_id = ?1 AND worker_cleanup_state = 'executing'",
params![workspace_id, now],
)?;
let mut statement = tx.prepare(
"SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('completed', 'failed', 'unknown') AND runtime_id IS NOT NULL AND worker_cleanup_state IN ('pending', 'failed') ORDER BY worker_cleanup_updated_at, created_at LIMIT ?2",
)?;
let identities = statement
.query_map(params![workspace_id, limit as i64], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
drop(statement);
let attempts = identities
.into_iter()
.map(|(job_id, attempt_id)| {
read_backend_job_attempt(&tx, workspace_id, &job_id, &attempt_id)?.ok_or_else(
|| Error::Store("recoverable Backend Job Worker cleanup disappeared".to_string()),
)
})
.collect::<Result<Vec<_>>>()?;
tx.commit()?;
Ok(attempts)
})
}
fn list_backend_job_worker_cleanups(
&self,
workspace_id: &str,
limit: usize,
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn(|conn| {
let mut statement = conn.prepare(
"SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('completed', 'failed', 'unknown') AND runtime_id IS NOT NULL AND worker_cleanup_state IN ('pending', 'failed') ORDER BY worker_cleanup_updated_at, created_at LIMIT ?2",
)?;
let identities = statement
.query_map(params![workspace_id, limit as i64], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
identities
.into_iter()
.map(|(job_id, attempt_id)| {
read_backend_job_attempt(conn, workspace_id, &job_id, &attempt_id)?.ok_or_else(
|| Error::Store("recoverable Backend Job Worker cleanup disappeared".to_string()),
)
})
.collect()
})
}
fn claim_backend_job_worker_cleanup(
&self,
workspace_id: &str,
job_id: &str,
attempt_id: &str,
now: &str,
) -> Result<(BackendJobAttemptRecord, bool)> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
if !attempt.state.terminal() || attempt.worker.is_none() {
return Err(Error::InvalidInput(
"Backend Job Worker cleanup requires a terminal bound attempt".to_string(),
));
}
if !matches!(
attempt.worker_cleanup_state,
Some(BackendJobWorkerCleanupState::Pending | BackendJobWorkerCleanupState::Failed)
) {
tx.commit()?;
return Ok((attempt, false));
}
let changed = tx.execute(
"UPDATE backend_job_attempts SET worker_cleanup_state = 'executing', worker_cleanup_failure_category = NULL, worker_cleanup_failure_detail = NULL, worker_cleanup_updated_at = ?4 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND worker_cleanup_state IN ('pending', 'failed')",
params![workspace_id, job_id, attempt_id, now],
)?;
if changed != 1 {
return Err(Error::Store(
"Backend Job Worker cleanup claim compare-and-set failed".to_string(),
));
}
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::Store("claimed Backend Job Worker cleanup disappeared".to_string()))?;
tx.commit()?;
Ok((attempt, true))
})
}
fn finish_backend_job_worker_cleanup(
&self,
workspace_id: &str,
job_id: &str,
attempt_id: &str,
state: BackendJobWorkerCleanupState,
failure_category: Option<&str>,
failure_detail: Option<&str>,
now: &str,
) -> Result<BackendJobAttemptRecord> {
if !matches!(
state,
BackendJobWorkerCleanupState::Completed | BackendJobWorkerCleanupState::Failed
) {
return Err(Error::InvalidInput(
"Backend Job Worker cleanup completion requires completed or failed state"
.to_string(),
));
}
let failure_detail = failure_detail.map(bounded_failure_detail);
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let current = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
if current.worker_cleanup_state == Some(state) {
tx.commit()?;
return Ok(current);
}
if current.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Executing) {
return Err(Error::InvalidInput(
"Backend Job Worker cleanup is not executing".to_string(),
));
}
let completed_at = (state == BackendJobWorkerCleanupState::Completed).then_some(now);
let changed = tx.execute(
"UPDATE backend_job_attempts SET worker_cleanup_state = ?4, worker_cleanup_failure_category = ?5, worker_cleanup_failure_detail = ?6, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = ?8 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND worker_cleanup_state = 'executing'",
params![workspace_id, job_id, attempt_id, state.as_str(), failure_category, failure_detail, now, completed_at],
)?;
if changed != 1 {
return Err(Error::Store(
"Backend Job Worker cleanup completion compare-and-set failed".to_string(),
));
}
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::Store("finished Backend Job Worker cleanup disappeared".to_string()))?;
tx.commit()?;
Ok(attempt)
})
}
fn reserve_backend_job_delivery(
&self,
workspace_id: &str,
@@ -10457,14 +10641,16 @@ fn read_backend_job_attempt(
attempt_id: &str,
) -> Result<Option<BackendJobAttemptRecord>> {
let row = conn.query_row(
"SELECT attempt, input_revision, state, runtime_id, worker_id, runtime_run_id, dispatched_at, deadline_at, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at FROM backend_job_attempts WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3",
"SELECT attempt, input_revision, state, runtime_id, worker_id, runtime_run_id, dispatched_at, deadline_at, result_json, result_digest, failure_category, failure_detail, worker_cleanup_state, worker_cleanup_failure_category, worker_cleanup_failure_detail, worker_cleanup_updated_at, worker_cleanup_completed_at, created_at, updated_at, completed_at FROM backend_job_attempts WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3",
params![workspace_id, job_id, attempt_id],
|row| Ok((
row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?,
row.get::<_, Option<String>>(3)?, row.get::<_, Option<String>>(4)?, row.get::<_, Option<String>>(5)?,
row.get::<_, Option<String>>(6)?, row.get::<_, String>(7)?, row.get::<_, Option<String>>(8)?,
row.get::<_, Option<String>>(9)?, row.get::<_, Option<String>>(10)?, row.get::<_, Option<String>>(11)?,
row.get::<_, String>(12)?, row.get::<_, String>(13)?, row.get::<_, Option<String>>(14)?,
row.get::<_, Option<String>>(12)?, row.get::<_, Option<String>>(13)?, row.get::<_, Option<String>>(14)?,
row.get::<_, Option<String>>(15)?, row.get::<_, Option<String>>(16)?, row.get::<_, String>(17)?,
row.get::<_, String>(18)?, row.get::<_, Option<String>>(19)?,
)),
).optional()?;
let Some((
@@ -10480,6 +10666,11 @@ fn read_backend_job_attempt(
result_digest,
failure_category,
failure_detail,
worker_cleanup_state,
worker_cleanup_failure_category,
worker_cleanup_failure_detail,
worker_cleanup_updated_at,
worker_cleanup_completed_at,
created_at,
updated_at,
completed_at,
@@ -10521,6 +10712,14 @@ fn read_backend_job_attempt(
result_digest,
failure_category,
failure_detail,
worker_cleanup_state: worker_cleanup_state
.as_deref()
.map(BackendJobWorkerCleanupState::parse)
.transpose()?,
worker_cleanup_failure_category,
worker_cleanup_failure_detail,
worker_cleanup_updated_at,
worker_cleanup_completed_at,
created_at,
updated_at,
completed_at,
@@ -14248,6 +14447,45 @@ fn migrate_archive_observe_grants_v75_to_v76(conn: &Connection) -> Result<()> {
Ok(())
}
fn migrate_backend_job_worker_cleanup_v76_to_v77(conn: &Connection) -> Result<()> {
let current = current_schema_version(conn)?;
if current != 76 {
return Err(Error::Store(format!(
"expected schema version 76 before {BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME} migration, found {current}"
)));
}
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
tx.execute_batch(
r#"
ALTER TABLE backend_job_attempts ADD COLUMN worker_cleanup_state TEXT
CHECK (worker_cleanup_state IN ('pending', 'executing', 'completed', 'failed'));
ALTER TABLE backend_job_attempts ADD COLUMN worker_cleanup_failure_category TEXT;
ALTER TABLE backend_job_attempts ADD COLUMN worker_cleanup_failure_detail TEXT;
ALTER TABLE backend_job_attempts ADD COLUMN worker_cleanup_updated_at TEXT;
ALTER TABLE backend_job_attempts ADD COLUMN worker_cleanup_completed_at TEXT;
UPDATE backend_job_attempts
SET worker_cleanup_state = CASE
WHEN runtime_id IS NULL THEN 'completed'
ELSE 'pending'
END,
worker_cleanup_updated_at = updated_at,
worker_cleanup_completed_at = CASE
WHEN runtime_id IS NULL THEN updated_at
ELSE NULL
END
WHERE state IN ('completed', 'failed', 'unknown');
CREATE INDEX backend_job_attempts_worker_cleanup
ON backend_job_attempts(workspace_id, worker_cleanup_state, updated_at);
"#,
)?;
tx.execute(
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)",
params![77_i64, BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME],
)?;
tx.commit()?;
Ok(())
}
fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
conn.execute_batch(include_str!("latest_schema.sql"))?;
@@ -16810,6 +17048,10 @@ mod tests {
version: 76,
name: ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME.to_string(),
},
WorkspaceSchemaMigrationStep {
version: 77,
name: BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME.to_string(),
},
]
);
@@ -16893,6 +17135,10 @@ mod tests {
RUNTIME_SCOPED_WORKER_IDENTITY_MIGRATION_NAME.to_string(),
),
(76, ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME.to_string()),
(
77,
BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME.to_string(),
),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -17160,6 +17406,71 @@ mod tests {
);
}
#[test]
fn backend_job_worker_cleanup_migration_queues_legacy_terminal_workers() {
let conn = Connection::open_in_memory().unwrap();
conn.execute_batch(
r#"
CREATE TABLE __yoi_schema_migrations (
version INTEGER PRIMARY KEY,
name TEXT NOT NULL
);
INSERT INTO __yoi_schema_migrations(version, name)
VALUES (76, 'workspace schema baseline');
CREATE TABLE backend_job_attempts (
workspace_id TEXT NOT NULL,
job_id TEXT NOT NULL,
attempt_id TEXT NOT NULL,
state TEXT NOT NULL,
runtime_id TEXT,
worker_id TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
INSERT INTO backend_job_attempts VALUES
('w', 'terminal-bound', 'a1', 'completed', 'runtime', 'worker', '1', '2'),
('w', 'terminal-unbound', 'a2', 'failed', NULL, NULL, '1', '3'),
('w', 'running', 'a3', 'dispatched', 'runtime', 'running-worker', '1', '4');
"#,
)
.unwrap();
migrate_backend_job_worker_cleanup_v76_to_v77(&conn).unwrap();
let states = conn
.prepare(
"SELECT job_id, worker_cleanup_state, worker_cleanup_completed_at FROM backend_job_attempts ORDER BY job_id",
)
.unwrap()
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, Option<String>>(1)?,
row.get::<_, Option<String>>(2)?,
))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert_eq!(
states,
vec![
("running".to_string(), None, None,),
(
"terminal-bound".to_string(),
Some("pending".to_string()),
None,
),
(
"terminal-unbound".to_string(),
Some("completed".to_string()),
Some("3".to_string()),
),
]
);
assert_eq!(current_schema_version(&conn).unwrap(), 77);
}
#[test]
fn migration_resumes_from_a_valid_partially_applied_chain() {
let temp = tempfile::tempdir().unwrap();
@@ -17179,7 +17490,7 @@ mod tests {
.collect::<Vec<_>>(),
vec![
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72,
73, 74, 75, 76
73, 74, 75, 76, 77
]
);
SqliteWorkspaceStore::migrate_database(&path).unwrap();
@@ -17188,7 +17499,7 @@ mod tests {
current_schema_version(&conn).unwrap(),
LATEST_SCHEMA_VERSION
);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 27);
assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 28);
assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap());
assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap());
assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap());
crates/workspace-server/src/subjektiv.rs
+42
View File
@@ -22,6 +22,7 @@ use crate::feature_storage::{
pub const SUBJEKTIV_SCHEMA_VERSION: u32 = 1;
pub const SUBJEKTIV_FEATURE_ID: &str = "subjektiv";
pub const SUBJECT_WORKER_SINGLETON_PREFIX: &str = "subjektiv:";
pub const MAX_STAGING_ANCHORS: usize = 10;
/// Deterministic surface-generation policy. Input estimates use the repository's
/// provider-independent UTF-8 byte estimate (`ceil(bytes / 4)`). The material
@@ -45,6 +46,8 @@ pub enum SubjektivError {
SubjectNotFound(String),
#[error("subject `{0}` is retired")]
SubjectRetired(String),
#[error("the selected Worker Profile does not enable subjektiv")]
ProfileDisabled,
#[error("staging candidate `{0}` was not found for this subject")]
CandidateNotFound(String),
#[error("staging candidate `{0}` already exists with different content")]
@@ -147,6 +150,23 @@ pub struct SubjectPage {
pub has_more: bool,
}
/// Backend-authorized launch material for one active subject. The singleton key
/// is derived by subjektiv and then acquired through Yoi's generic singleton path.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AuthorizedSubjectWorkerAttachment {
pub singleton_key: String,
}
pub fn subject_worker_singleton_key(subject_id: &str) -> Result<String> {
validate_label("subject id", subject_id)?;
if subject_id.trim() != subject_id {
return Err(SubjektivError::InvalidRecord(
"subject id must not contain surrounding whitespace".to_string(),
));
}
Ok(format!("{SUBJECT_WORKER_SINGLETON_PREFIX}{subject_id}"))
}
/// Immutable host-recorded history that attributes one durable Session to the
/// subject and Worker that produced it. This is archival provenance only: it is
/// deliberately not a current-Worker link or execution ownership record.
@@ -1091,6 +1111,28 @@ impl SubjektivStore {
})
}
/// Validates an explicit product launch connection against this Workspace's
/// subject store and the already-resolved Profile policy. Subject knowledge
/// alone is insufficient: only an active local record produces host launch material.
pub fn authorize_worker_attachment(
&self,
subject_id: &str,
profile: &manifest::WorkerManifest,
) -> Result<AuthorizedSubjectWorkerAttachment> {
if !profile.feature.subjektiv.profile.enabled {
return Err(SubjektivError::ProfileDisabled);
}
let subject = self
.subject(subject_id)?
.ok_or_else(|| SubjektivError::SubjectNotFound(subject_id.to_string()))?;
if subject.state != SubjectState::Active {
return Err(SubjektivError::SubjectRetired(subject_id.to_string()));
}
Ok(AuthorizedSubjectWorkerAttachment {
singleton_key: subject_worker_singleton_key(&subject.id)?,
})
}
/// Lists the first deterministic bounded page of subjects. Ordering uses
/// immutable creation time with the host-issued subject id as a total-order
/// tie breaker.