feat: replace revision counters with domain state contracts

Use content digests, key fingerprints, binding and trust identities, immutable mutation receipts, and observation tokens instead of state-update ordinals across API, storage, Runtime, and Web contracts.

Preserve historical replay and secret material through frozen migrations; preflight uncertain removals before cutover, resume provisioning receipts, validate encrypted migration candidates, and align fresh/upgraded schemas. Include reviewed regression fixes, generated artifacts, and validation evidence.
This commit is contained in:
2026-10-10 15:55:20 +09:00
parent 72ce817de4
commit c78f7f7d80
358 changed files with 25965 additions and 12069 deletions
+3 -6
View File
@@ -75,7 +75,6 @@ pub struct WorkingDirectoryRepository {
pub id: String,
pub provider: String,
pub source: server_api::RepositorySource,
pub source_revision: u64,
pub source_fingerprint: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<RepositorySelector>,
@@ -121,7 +120,7 @@ impl std::fmt::Debug for SensitiveString {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RepositorySshCredentialCandidate {
pub credential_id: String,
pub credential_revision: u64,
pub public_key_fingerprint: String,
#[serde(skip, default)]
pub private_key: SensitiveString,
}
@@ -130,7 +129,7 @@ pub struct RepositorySshCredentialCandidate {
pub struct RepositorySshMaterializationAccess {
pub credential_candidates: Vec<RepositorySshCredentialCandidate>,
pub host_trust_id: String,
pub host_trust_revision: u64,
pub host_key_fingerprint: String,
pub access: server_api::RepositoryAccessMode,
pub expires_at_epoch_seconds: u64,
pub repository_id: String,
@@ -146,7 +145,6 @@ pub struct RepositoryMaterializationContext {
pub workspace_id: String,
pub runtime_id: String,
pub operation_id: String,
pub config_revision: u64,
pub config_projection_digest: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ssh: Option<RepositorySshMaterializationAccess>,
@@ -191,10 +189,9 @@ pub struct RepositoryRefObservationRequest {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RepositoryRefObservation {
pub repository_id: String,
pub source_revision: u64,
pub source_fingerprint: String,
pub selector: String,
pub revision_ref: String,
pub resolved_ref: String,
pub observed_at_epoch_seconds: u64,
}
+3 -22
View File
@@ -43,7 +43,6 @@ impl ConfigBundle {
pub fn computed_digest(&self) -> String {
let mut lines = Vec::new();
lines.push(format!("id\0{}", self.metadata.id));
lines.push(format!("revision\0{}", self.metadata.revision));
lines.push(format!("workspace_id\0{}", self.metadata.workspace_id));
lines.push(format!("created_at\0{}", self.metadata.created_at));
lines.push(format!(
@@ -82,7 +81,7 @@ impl ConfigBundle {
if let Some(prompt_catalog) = &self.prompt_catalog {
lines.push(format!(
"prompt_catalog\0{}\0{}\0{}\0{}",
prompt_catalog.config_revision,
prompt_catalog.source_digest,
prompt_catalog.schema_fingerprint,
prompt_catalog.toolchain_fingerprint,
prompt_catalog.catalog_digest
@@ -100,12 +99,8 @@ impl ConfigBundle {
}
if let Some(handle) = &self.profile_source_archive_handle {
lines.push(format!(
"profile_archive_handle\0{}\0{}\0{}\0{}\0{}",
handle.workspace_id,
handle.resource_id,
handle.digest,
handle.revision,
handle.max_bytes
"profile_archive_handle\0{}\0{}\0{}\0{}",
handle.workspace_id, handle.resource_id, handle.digest, handle.max_bytes
));
for (selector, path) in handle
.profile_source_graph
@@ -140,7 +135,6 @@ impl ConfigBundle {
id: self.metadata.id.clone(),
digest: self.metadata.digest.clone(),
digest_algorithm: CONFIG_BUNDLE_DIGEST_ALGORITHM.to_string(),
revision: self.metadata.revision.clone(),
workspace_id: self.metadata.workspace_id.clone(),
created_at: self.metadata.created_at.clone(),
provenance: self.metadata.provenance.clone(),
@@ -182,7 +176,6 @@ impl From<ConfigBundle> for server_api::WorkspaceRuntimeConfigResponse {
prompt_catalog: prompt_catalog.map(|catalog| {
server_api::WorkspaceRuntimePromptCatalog {
templates: catalog.templates,
config_revision: catalog.config_revision,
source_digest: catalog.source_digest,
schema_fingerprint: catalog.schema_fingerprint,
toolchain_fingerprint: catalog.toolchain_fingerprint,
@@ -200,7 +193,6 @@ impl From<ConfigBundleMetadata> for server_api::WorkspaceRuntimeConfigMetadata {
Self {
id: metadata.id,
digest: metadata.digest,
revision: metadata.revision,
workspace_id: metadata.workspace_id,
created_at: metadata.created_at,
provenance: metadata.provenance.into(),
@@ -303,8 +295,6 @@ impl From<BackendResourceHandle> for server_api::WorkspaceRuntimeResourceHandle
operation: handle.operation.into(),
expires_at_unix_seconds: handle.expires_at_unix_seconds,
nonce: handle.nonce,
revision: handle.revision,
generation: handle.generation,
max_bytes: handle.max_bytes,
content_type: handle.content_type,
redaction: handle.redaction.into(),
@@ -344,7 +334,6 @@ impl From<ResourceRedactionPolicy> for server_api::WorkspaceRuntimeResourceRedac
pub struct ConfigBundleMetadata {
pub id: String,
pub digest: String,
pub revision: String,
pub workspace_id: String,
pub created_at: String,
pub provenance: ConfigBundleProvenance,
@@ -402,7 +391,6 @@ pub struct ConfigBundleSummary {
pub id: String,
pub digest: String,
pub digest_algorithm: String,
pub revision: String,
pub workspace_id: String,
pub created_at: String,
pub provenance: ConfigBundleProvenance,
@@ -422,7 +410,6 @@ pub(crate) fn validate_config_bundle(bundle: &ConfigBundle) -> Result<(), Runtim
validate_config_bundle_id(&bundle.metadata.id)?;
validate_non_empty("config bundle digest", &bundle.metadata.digest)?;
validate_digest("config bundle digest", &bundle.metadata.digest)?;
validate_non_empty("config bundle revision", &bundle.metadata.revision)?;
validate_non_empty("config bundle workspace id", &bundle.metadata.workspace_id)?;
validate_non_empty("config bundle created_at", &bundle.metadata.created_at)?;
validate_non_empty(
@@ -430,7 +417,6 @@ pub(crate) fn validate_config_bundle(bundle: &ConfigBundle) -> Result<(), Runtim
&bundle.metadata.provenance.source,
)?;
validate_boundary_text("config bundle id", &bundle.metadata.id)?;
validate_boundary_text("config bundle revision", &bundle.metadata.revision)?;
validate_boundary_text("config bundle workspace id", &bundle.metadata.workspace_id)?;
validate_boundary_text(
"config bundle provenance source",
@@ -489,7 +475,6 @@ pub(crate) fn validate_config_bundle(bundle: &ConfigBundle) -> Result<(), Runtim
("resource handle resource id", handle.resource_id.as_str()),
("resource handle digest", handle.digest.as_str()),
("resource handle nonce", handle.nonce.as_str()),
("resource handle revision", handle.revision.as_str()),
("resource handle content type", handle.content_type.as_str()),
(
"resource handle audit correlation id",
@@ -766,7 +751,6 @@ mod tests {
metadata: ConfigBundleMetadata {
id: "bundle-1".to_string(),
digest: String::new(),
revision: "rev-1".to_string(),
workspace_id: "workspace-1".to_string(),
created_at: "2026-06-26T00:00:00Z".to_string(),
provenance: ConfigBundleProvenance {
@@ -823,7 +807,6 @@ mod tests {
bundle.prompt_catalog = Some(
worker::EffectivePromptCatalog::new(
std::collections::BTreeMap::from([("default".to_string(), "hello".to_string())]),
7,
"schema",
"toolchain",
)
@@ -866,8 +849,6 @@ mod tests {
operation: crate::resource::BackendResourceOperation::FetchArchive,
expires_at_unix_seconds: 4_102_444_800,
nonce: "nonce-not-for-browser".to_string(),
revision: "rev-1".to_string(),
generation: Some(1),
max_bytes: 128,
content_type: crate::resource::PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE.to_string(),
redaction: crate::resource::ResourceRedactionPolicy::RuntimeInternalOnly,
+1 -1
View File
@@ -693,7 +693,7 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
}
/// Observe a newer immutable Workspace Prompt projection. Profile-backed
/// execution uses this as a revision notification; other backends may
/// execution uses this as a content projection notification; other backends may
/// safely ignore it.
fn observe_workspace_prompt_projection(
&self,
+284 -19
View File
@@ -21,7 +21,9 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
const SCHEMA_VERSION: u32 = 9;
const SCHEMA_VERSION: u32 = 10;
// Frozen source schemas: do not move their meaning when adding a migration.
const AGGREGATE_SCHEMA_VERSION: u32 = 9;
const PREVIOUS_SCHEMA_VERSION: u32 = 8;
const LEGACY_SCHEMA_VERSION: u32 = 7;
const OLDEST_SCHEMA_VERSION: u32 = 6;
@@ -609,8 +611,8 @@ fn plan_runtime_store_migration(
format!("Runtime store schema version {schema_version} is out of range"),
)
})?;
let staging = migration_sibling(root, "schema-v9-staging")?;
let backup = migration_sibling(root, "pre-schema-v9-backup")?;
let staging = migration_sibling(root, "schema-v10-staging")?;
let backup = migration_sibling(root, "pre-schema-v10-backup")?;
if staging.exists() || backup.exists() {
return Err(runtime_store_corrupt(
root,
@@ -639,12 +641,15 @@ fn plan_runtime_store_migration(
}
if !matches!(
current_schema_version,
OLDEST_SCHEMA_VERSION | LEGACY_SCHEMA_VERSION | PREVIOUS_SCHEMA_VERSION
OLDEST_SCHEMA_VERSION
| LEGACY_SCHEMA_VERSION
| PREVIOUS_SCHEMA_VERSION
| AGGREGATE_SCHEMA_VERSION
) {
return Err(runtime_store_corrupt(
&runtime_path,
format!(
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
),
));
}
@@ -969,11 +974,30 @@ fn validate_migrated_worker_documents(
}
fn migrate_worker_record(
identity_document: serde_json::Value,
mut identity_document: serde_json::Value,
source_schema_version: u32,
mapping: Option<&LegacyWorkerIdentityMapping>,
identity_path: &Path,
) -> Result<WorkerAggregateRecord, RuntimeError> {
if source_schema_version == AGGREGATE_SCHEMA_VERSION {
migrate_v9_restore_guard(&mut identity_document, identity_path)?;
remove_legacy_job_input_counter(&mut identity_document);
migrate_legacy_request_evidence(&mut identity_document)
.map_err(|message| runtime_store_corrupt(identity_path, message.into()))?;
identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION);
let aggregate: WorkerAggregateRecord =
serde_json::from_value(identity_document).map_err(|error| {
runtime_store_corrupt(
identity_path,
format!("decode migrated schema-v9 Worker aggregate: {error}"),
)
})?;
aggregate.clone().validate(identity_path)?;
return Ok(aggregate);
}
remove_legacy_job_input_counter(&mut identity_document);
migrate_legacy_request_evidence(&mut identity_document)
.map_err(|message| runtime_store_corrupt(identity_path, message.into()))?;
if source_schema_version == PREVIOUS_SCHEMA_VERSION {
let identity: WorkerIdentityRecord =
serde_json::from_value(identity_document).map_err(|error| {
@@ -984,8 +1008,18 @@ fn migrate_worker_record(
})?;
identity.validate_for_schema(identity_path, PREVIOUS_SCHEMA_VERSION)?;
let execution_path = identity_path.with_file_name(WORKER_EXECUTION_FILE);
let execution: WorkerExecutionRecord =
let mut execution_document: serde_json::Value =
read_bounded_json(&execution_path, "read schema-v8 Worker execution record")?;
remove_legacy_job_input_counter(&mut execution_document);
migrate_legacy_request_evidence(&mut execution_document)
.map_err(|message| runtime_store_corrupt(&execution_path, message.into()))?;
let execution: WorkerExecutionRecord =
serde_json::from_value(execution_document).map_err(|error| {
runtime_store_corrupt(
&execution_path,
format!("decode migrated schema-v8 execution: {error}"),
)
})?;
execution.validate_for_schema(&identity, &execution_path, PREVIOUS_SCHEMA_VERSION)?;
return Ok(WorkerAggregateRecord::from_v8(identity, execution));
}
@@ -1016,6 +1050,114 @@ fn migrate_worker_record(
Ok(WorkerAggregateRecord::from_v8(identity, execution))
}
// Frozen schema-v9 conversion. Preserve the exact token and all Restore owners
// and receipts; generating a fresh token here would lose retry/admission authority.
fn migrate_v9_restore_guard(
document: &mut serde_json::Value,
path: &Path,
) -> Result<(), RuntimeError> {
let Some(guard) = document.get_mut("restore_guard") else {
return Ok(());
};
let guard = guard.as_object_mut().ok_or_else(|| {
runtime_store_corrupt(path, "schema-v9 restore_guard must be an object".into())
})?;
let token = guard.remove("generation").ok_or_else(|| {
runtime_store_corrupt(path, "schema-v9 restore_guard is missing generation".into())
})?;
if !token.as_str().is_some_and(|token| !token.is_empty())
|| guard.contains_key("observation_token")
{
return Err(runtime_store_corrupt(
path,
"schema-v9 Restore observation token is invalid or ambiguous".into(),
));
}
guard.insert("observation_token".into(), token);
Ok(())
}
// Legacy Runtime Job bindings never held input JSON, so the old caller-selected
// counter cannot become a content digest. Retain Job/attempt identity and leave
// content identity unknown until the host supplies its immutable intent.
fn remove_legacy_job_input_counter(value: &mut serde_json::Value) {
match value {
serde_json::Value::Object(object) => {
if let Some(job) = object
.get_mut("backend_job")
.and_then(serde_json::Value::as_object_mut)
{
job.remove("input_revision");
}
for value in object.values_mut() {
remove_legacy_job_input_counter(value);
}
}
serde_json::Value::Array(values) => {
for value in values {
remove_legacy_job_input_counter(value);
}
}
_ => {}
}
}
// Frozen schema 6..9 request migration. Preserve concrete source/config digests,
// contents, operation ownership and fingerprints. Unknown legacy SSH key identity
// stays empty and cannot pass the materializer's live-authority validation.
fn migrate_legacy_request_evidence(value: &mut serde_json::Value) -> Result<(), &'static str> {
match value {
serde_json::Value::Object(object) => {
if object.contains_key("credential_id") && object.contains_key("credential_revision") {
let counter = object.remove("credential_revision").expect("checked field");
if !counter.as_u64().is_some_and(|value| value > 0) {
return Err("invalid legacy SSH credential evidence");
}
object
.entry("public_key_fingerprint")
.or_insert_with(|| serde_json::json!(""));
}
if object.contains_key("host_trust_id") && object.contains_key("host_trust_revision") {
let counter = object.remove("host_trust_revision").expect("checked field");
if !counter.as_u64().is_some_and(|value| value > 0) {
return Err("invalid legacy SSH host trust evidence");
}
object
.entry("host_key_fingerprint")
.or_insert_with(|| serde_json::json!(""));
}
if let Some(settings) = object
.get_mut("memory_settings")
.and_then(serde_json::Value::as_object_mut)
&& let Some(counter) = settings.remove("settings_revision")
&& !counter.as_u64().is_some_and(|value| value > 0)
{
return Err("invalid legacy Memory settings evidence");
}
if object.contains_key("source_fingerprint") {
object.remove("source_revision");
}
if object.contains_key("config_projection_digest") {
object.remove("config_revision");
}
if object.contains_key("resource_id") && object.contains_key("nonce") {
object.remove("revision");
object.remove("generation");
}
for value in object.values_mut() {
migrate_legacy_request_evidence(value)?;
}
}
serde_json::Value::Array(values) => {
for value in values {
migrate_legacy_request_evidence(value)?;
}
}
_ => {}
}
Ok(())
}
fn runtime_worker_name(worker_id: WorkerId) -> String {
format!("worker-runtime-{worker_id}")
}
@@ -1216,6 +1358,9 @@ fn migrate_runtime_document(
serde_json::Value::from(SCHEMA_VERSION),
);
object.remove("workers");
// This legacy cache is not Runtime lifecycle or restore authority. Bundles
// are reacquired from Workspace content authority, never renumbered/rehash-pinned here.
object.remove("config_bundles");
object.remove("next_worker_sequence");
let snapshot: RuntimeSnapshot = serde_json::from_value(document.clone()).map_err(|error| {
runtime_store_corrupt(
@@ -1325,8 +1470,14 @@ fn remove_runtime_migration_directory(path: &Path) -> Result<(), RuntimeError> {
}
fn recover_runtime_store_migration(root: &Path) -> Result<(), RuntimeError> {
let staging = migration_sibling(root, "schema-v9-staging")?;
let backup = migration_sibling(root, "pre-schema-v9-backup")?;
for suffix in ["schema-v9-staging", "pre-schema-v9-backup"] {
let earlier = migration_sibling(root, suffix)?;
if earlier.exists() {
return Err(runtime_store_corrupt(root, "unfinished schema-v9 migration requires explicit recovery before schema-v10 startup".into()));
}
}
let staging = migration_sibling(root, "schema-v10-staging")?;
let backup = migration_sibling(root, "pre-schema-v10-backup")?;
let root_exists = migration_directory_exists(root)?;
let staging_exists = migration_directory_exists(&staging)?;
let backup_exists = migration_directory_exists(&backup)?;
@@ -1528,8 +1679,8 @@ fn migrate_runtime_store(
if !plan.migration_required {
return Ok(plan);
}
let staging = migration_sibling(root, "schema-v9-staging")?;
let backup = migration_sibling(root, "pre-schema-v9-backup")?;
let staging = migration_sibling(root, "schema-v10-staging")?;
let backup = migration_sibling(root, "pre-schema-v10-backup")?;
if staging.exists() || backup.exists() {
return Err(runtime_store_corrupt(
root,
@@ -2875,7 +3026,7 @@ mod tests {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root);
let staging = migration_sibling(&root, "schema-v9-staging").unwrap();
let staging = migration_sibling(&root, "schema-v10-staging").unwrap();
fs::create_dir_all(&staging).unwrap();
fs::write(staging.join("partial"), b"partial").unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -2890,7 +3041,7 @@ mod tests {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root);
let backup = migration_sibling(&root, "pre-schema-v9-backup").unwrap();
let backup = migration_sibling(&root, "pre-schema-v10-backup").unwrap();
fs::rename(&root, &backup).unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -2904,10 +3055,10 @@ mod tests {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root);
let staging = migration_sibling(&root, "schema-v9-staging").unwrap();
let staging = migration_sibling(&root, "schema-v10-staging").unwrap();
copy_runtime_tree(&root, &root, &staging).unwrap();
migrate_runtime_store_in_place(&staging, "test-runtime").unwrap();
let backup = migration_sibling(&root, "pre-schema-v9-backup").unwrap();
let backup = migration_sibling(&root, "pre-schema-v10-backup").unwrap();
fs::rename(&root, &backup).unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -2922,10 +3073,10 @@ mod tests {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root);
let staging = migration_sibling(&root, "schema-v9-staging").unwrap();
let staging = migration_sibling(&root, "schema-v10-staging").unwrap();
fs::create_dir_all(&staging).unwrap();
fs::write(staging.join(RUNTIME_FILE), b"{\"schema_version\":8}").unwrap();
let backup = migration_sibling(&root, "pre-schema-v9-backup").unwrap();
let backup = migration_sibling(&root, "pre-schema-v10-backup").unwrap();
fs::rename(&root, &backup).unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -2942,7 +3093,7 @@ mod tests {
let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root);
migrate_runtime_store_in_place(&root, "test-runtime").unwrap();
let backup = migration_sibling(&root, "pre-schema-v9-backup").unwrap();
let backup = migration_sibling(&root, "pre-schema-v10-backup").unwrap();
write_empty_schema_v8_store(&backup);
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
@@ -2950,6 +3101,120 @@ mod tests {
assert!(!backup.exists());
}
#[test]
fn earlier_migration_backup_cannot_be_mistaken_for_a_fresh_runtime() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store");
let backup = migration_sibling(&root, "pre-schema-v9-backup").unwrap();
write_empty_schema_v8_store(&backup);
let error = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap_err();
assert!(error.to_string().contains("explicit recovery"));
assert!(!root.exists());
assert!(backup.join(RUNTIME_FILE).exists());
}
#[test]
fn legacy_request_migration_preserves_content_and_never_invents_key_identity() {
let mut document = serde_json::json!({"request": {
"memory_settings": {"workspace_id":"workspace-a", "language":"English", "settings_revision":2},
"repository": {"source_revision":1, "source_fingerprint":"source-content"},
"materialization": {"operation_id":"op-a", "config_revision":2, "config_projection_digest":"projection-content", "ssh": {
"host_trust_id":"host-a", "host_trust_revision":3,
"credential_candidates":[{"credential_id":"credential-a", "credential_revision":4}]
}}
}});
migrate_legacy_request_evidence(&mut document).unwrap();
let request = &document["request"];
assert_eq!(request["memory_settings"]["language"], "English");
assert!(
request["memory_settings"]
.get("settings_revision")
.is_none()
);
assert_eq!(
request["repository"]["source_fingerprint"],
"source-content"
);
assert_eq!(
request["materialization"]["config_projection_digest"],
"projection-content"
);
assert_eq!(
request["materialization"]["ssh"]["host_key_fingerprint"],
""
);
assert_eq!(
request["materialization"]["ssh"]["credential_candidates"][0]["public_key_fingerprint"],
""
);
}
#[test]
fn schema_v9_restore_token_migrates_durably_without_rotating_admission_identity() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime-store");
write_empty_schema_v8_store(&root);
let runtime_path = root.join(RUNTIME_FILE);
let mut runtime: serde_json::Value = read_json(&runtime_path, "read test Runtime").unwrap();
runtime["schema_version"] = serde_json::json!(AGGREGATE_SCHEMA_VERSION);
fs::write(&runtime_path, serde_json::to_vec(&runtime).unwrap()).unwrap();
let worker_id = WorkerId::now_v7();
let path = root
.join(WORKERS_DIR)
.join(worker_id.to_string())
.join(WORKER_FILE);
fs::create_dir_all(path.parent().unwrap()).unwrap();
let aggregate = migrate_worker_record(
schema_v7_worker_document(worker_id),
LEGACY_SCHEMA_VERSION,
None,
&path,
)
.unwrap();
let mut legacy = serde_json::to_value(&aggregate).unwrap();
let token = legacy["restore_guard"]["observation_token"].take();
legacy["restore_guard"]
.as_object_mut()
.unwrap()
.remove("observation_token");
legacy["restore_guard"]["generation"] = token.clone();
legacy["schema_version"] = serde_json::json!(AGGREGATE_SCHEMA_VERSION);
fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap();
let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
store.store.load_runtime_state().unwrap();
drop(store);
let migrated: serde_json::Value = read_json(&path, "read migrated Worker").unwrap();
assert_eq!(migrated["schema_version"], SCHEMA_VERSION);
assert_eq!(migrated["restore_guard"]["observation_token"], token);
assert!(migrated["restore_guard"].get("generation").is_none());
let store = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap();
store.store.load_runtime_state().unwrap();
assert_eq!(
read_json::<serde_json::Value>(&path, "read restarted Worker").unwrap(),
migrated
);
}
#[test]
fn frozen_restore_guard_conversion_preserves_receipts_and_rejects_ambiguous_identity() {
let path = Path::new("worker.json");
let mut document = serde_json::json!({"restore_guard": {
"generation": "opaque-admission-token", "active_request_id": "restore-a",
"owners": {"restore-a": {"fingerprint": "intent", "receipt": {"accepted": true}}}
}});
let owners = document["restore_guard"]["owners"].clone();
migrate_v9_restore_guard(&mut document, path).unwrap();
assert_eq!(document["restore_guard"]["owners"], owners);
assert_eq!(document["restore_guard"]["active_request_id"], "restore-a");
assert_eq!(
document["restore_guard"]["observation_token"],
"opaque-admission-token"
);
let mut ambiguous =
serde_json::json!({"restore_guard": {"generation": "old", "observation_token": "new"}});
assert!(migrate_v9_restore_guard(&mut ambiguous, path).is_err());
}
#[test]
fn bounded_worker_record_read_rejects_oversize_before_parse() {
let root = tempfile::tempdir().unwrap();
@@ -3047,7 +3312,7 @@ mod tests {
));
assert!(!worker_dir.join(WORKER_EXECUTION_FILE).exists());
let aggregate: WorkerAggregateRecord =
read_bounded_json(&identity_path, "read schema-v9 aggregate").unwrap();
read_bounded_json(&identity_path, "read schema-v10 aggregate").unwrap();
assert_eq!(aggregate.schema_version, SCHEMA_VERSION);
}
+198 -56
View File
@@ -413,7 +413,8 @@ pub struct RuntimeHttpWorkspacePromptProjectionRequest {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeHttpWorkspacePromptProjectionResponse {
pub workspace_id: String,
pub config_revision: u64,
pub source_digest: String,
pub projection_digest: String,
}
/// Config bundle availability response used by sync/check endpoints.
@@ -576,10 +577,10 @@ async fn post_workspace_verification_challenge(
})?;
if challenge.workspace_id != verified.workspace_id
|| challenge.runtime_id != verified.runtime_id
|| challenge.binding_revision != verified.binding_revision
|| challenge.binding_id != verified.binding_id
|| challenge.workspace_key_id != verified.issuer_key_id
|| challenge.workspace_identity_revision != verified.issuer_identity_revision
|| challenge.workspace_trust_generation != verified.trust_generation
|| challenge.workspace_public_key_fingerprint != verified.issuer_public_key_fingerprint
|| challenge.workspace_trust_id != verified.trust_id
|| challenge.runtime_id != auth.signer.runtime_id()
|| challenge.runtime_public_key_fingerprint != auth.signer.public_key_fingerprint()
{
@@ -599,6 +600,13 @@ async fn post_workspace_verification_challenge(
error.to_string(),
)
})?;
auth.verifications.begin(challenge).map_err(|error| {
RuntimeHttpRestError::new(
StatusCode::SERVICE_UNAVAILABLE,
"workspace_runtime_verification_unavailable",
error.to_string(),
)
})?;
Ok(Json(response))
}
@@ -616,17 +624,17 @@ async fn post_workspace_verification_acknowledgement(
})?;
if acknowledgement.workspace_id != verified.workspace_id
|| acknowledgement.runtime_id != verified.runtime_id
|| acknowledgement.binding_revision != verified.binding_revision
|| acknowledgement.binding_id != verified.binding_id
|| acknowledgement.workspace_key_id != verified.issuer_key_id
|| acknowledgement.workspace_identity_revision != verified.issuer_identity_revision
|| acknowledgement.workspace_trust_generation != verified.trust_generation
|| acknowledgement.workspace_public_key_fingerprint
!= verified.issuer_public_key_fingerprint
|| acknowledgement.workspace_trust_id != verified.trust_id
|| acknowledgement.runtime_id != auth.signer.runtime_id()
|| acknowledgement.runtime_public_key_fingerprint != auth.signer.public_key_fingerprint()
|| acknowledgement.expires_at <= unix_now_i64()
|| acknowledgement.binding_revision == 0
|| acknowledgement.runtime_identity_revision == 0
|| acknowledgement.workspace_identity_revision == 0
|| acknowledgement.workspace_trust_generation == 0
|| acknowledgement.binding_id.is_empty()
|| acknowledgement.workspace_public_key_fingerprint.is_empty()
|| acknowledgement.workspace_trust_id.is_empty()
|| acknowledgement.workspace_nonce.is_empty()
|| acknowledgement.runtime_nonce.is_empty()
|| acknowledgement.response_digest.len() != workspace_request_body_digest(&[]).len()
@@ -641,12 +649,11 @@ async fn post_workspace_verification_acknowledgement(
challenge_id: acknowledgement.challenge_id.clone(),
workspace_id: acknowledgement.workspace_id.clone(),
runtime_id: acknowledgement.runtime_id.clone(),
binding_revision: acknowledgement.binding_revision,
binding_id: acknowledgement.binding_id.clone(),
workspace_key_id: acknowledgement.workspace_key_id.clone(),
workspace_identity_revision: acknowledgement.workspace_identity_revision,
workspace_trust_generation: acknowledgement.workspace_trust_generation,
workspace_public_key_fingerprint: acknowledgement.workspace_public_key_fingerprint.clone(),
workspace_trust_id: acknowledgement.workspace_trust_id.clone(),
runtime_public_key_fingerprint: acknowledgement.runtime_public_key_fingerprint.clone(),
runtime_identity_revision: acknowledgement.runtime_identity_revision,
workspace_nonce: acknowledgement.workspace_nonce.clone(),
expires_at: acknowledgement.expires_at,
};
@@ -689,17 +696,23 @@ async fn post_workspace_verification_acknowledgement(
)
})?;
auth.verifications
.record(WorkspaceRuntimeVerificationRecord {
workspace_id: acknowledgement.workspace_id.clone(),
runtime_id: acknowledgement.runtime_id.clone(),
binding_revision: acknowledgement.binding_revision,
workspace_key_id: acknowledgement.workspace_key_id.clone(),
workspace_identity_revision: acknowledgement.workspace_identity_revision,
workspace_trust_generation: acknowledgement.workspace_trust_generation,
runtime_public_key_fingerprint: acknowledgement.runtime_public_key_fingerprint.clone(),
runtime_identity_revision: acknowledgement.runtime_identity_revision,
verified_at: unix_now_i64(),
})
.acknowledge(
&acknowledgement.challenge_id,
WorkspaceRuntimeVerificationRecord {
workspace_id: acknowledgement.workspace_id.clone(),
runtime_id: acknowledgement.runtime_id.clone(),
binding_id: acknowledgement.binding_id.clone(),
workspace_key_id: acknowledgement.workspace_key_id.clone(),
workspace_public_key_fingerprint: acknowledgement
.workspace_public_key_fingerprint
.clone(),
workspace_trust_id: acknowledgement.workspace_trust_id.clone(),
runtime_public_key_fingerprint: acknowledgement
.runtime_public_key_fingerprint
.clone(),
verified_at: unix_now_i64(),
},
)
.map_err(|error| {
RuntimeHttpRestError::new(
StatusCode::SERVICE_UNAVAILABLE,
@@ -711,7 +724,7 @@ async fn post_workspace_verification_acknowledgement(
challenge_id: acknowledgement.challenge_id,
workspace_id: acknowledgement.workspace_id,
runtime_id: acknowledgement.runtime_id,
binding_revision: acknowledgement.binding_revision,
binding_id: acknowledgement.binding_id.clone(),
accepted_at: unix_now_i64(),
}))
}
@@ -810,14 +823,16 @@ async fn observe_workspace_prompt_projection(
));
}
let workspace_id = request.projection.workspace_id.clone();
let config_revision = request.projection.config_revision;
let source_digest = request.projection.source_digest.clone();
let projection_digest = request.projection.projection_digest.clone();
state
.runtime
.observe_workspace_prompt_projection(request.projection)
.map_err(RuntimeHttpRestError::runtime)?;
Ok(Json(RuntimeHttpWorkspacePromptProjectionResponse {
workspace_id,
config_revision,
source_digest,
projection_digest,
}))
}
@@ -2108,7 +2123,7 @@ async fn require_runtime_auth(
let body_digest = workspace_request_body_digest(&body);
let expected = WorkspaceCapabilityExpectation {
workspace_id: &claims.issuer_workspace_id,
binding_revision: claims.binding_revision,
binding_id: &claims.binding_id,
runtime_id: workspace_auth.signer.runtime_id(),
worker_id: expected_worker_id.as_deref(),
operation: required_permission,
@@ -2145,13 +2160,13 @@ async fn require_runtime_auth(
.into_response();
}
};
if record.binding_revision != verified.binding_revision
if record.binding_id != verified.binding_id
|| record.workspace_key_id != verified.issuer_key_id
|| record.workspace_identity_revision != verified.issuer_identity_revision
|| record.workspace_trust_generation != verified.trust_generation
|| record.workspace_public_key_fingerprint
!= verified.issuer_public_key_fingerprint
|| record.workspace_trust_id != verified.trust_id
|| record.runtime_public_key_fingerprint
!= workspace_auth.signer.public_key_fingerprint()
|| record.runtime_identity_revision == 0
{
return RuntimeHttpRestError::new(
StatusCode::FORBIDDEN,
@@ -2782,9 +2797,8 @@ mod tests {
key_id: "workspace-key".to_string(),
algorithm: "ed25519".to_string(),
public_key: workspace_identity.public_key.clone(),
public_key_fingerprint: workspace_fingerprint,
identity_revision: 1,
trust_generation: 1,
public_key_fingerprint: workspace_fingerprint.clone(),
trust_id: "trust-test".into(),
state: WorkspaceIssuerTrustState::Active,
registered_at_unix: 1,
updated_at_unix: 1,
@@ -2805,12 +2819,11 @@ mod tests {
challenge_id: "challenge-1".to_string(),
workspace_id: "workspace-a".to_string(),
runtime_id: "runtime-test".to_string(),
binding_revision: 4,
binding_id: "binding-test".into(),
workspace_key_id: "workspace-key".to_string(),
workspace_identity_revision: 1,
workspace_trust_generation: 1,
workspace_public_key_fingerprint: workspace_fingerprint.clone(),
workspace_trust_id: "trust-test".into(),
runtime_public_key_fingerprint: runtime_fingerprint,
runtime_identity_revision: 1,
workspace_nonce: "workspace-nonce".to_string(),
expires_at: unix_now_i64() + 60,
};
@@ -2819,9 +2832,9 @@ mod tests {
issuer: "https://backend.test".to_string(),
issuer_workspace_id: "workspace-a".to_string(),
issuer_key_id: "workspace-key".to_string(),
issuer_identity_revision: 1,
trust_generation: 1,
binding_revision: 4,
issuer_public_key_fingerprint: workspace_fingerprint.clone(),
trust_id: "trust-test".into(),
binding_id: "binding-test".into(),
runtime_id: "runtime-test".to_string(),
worker_id: None,
operation: WORKSPACE_VERIFICATION_OPERATION.to_string(),
@@ -2870,14 +2883,15 @@ mod tests {
challenge_id: verification_response.challenge_id.clone(),
workspace_id: verification_response.workspace_id.clone(),
runtime_id: verification_response.runtime_id.clone(),
binding_revision: verification_response.binding_revision,
binding_id: verification_response.binding_id.clone(),
workspace_key_id: verification_response.workspace_key_id.clone(),
workspace_identity_revision: verification_response.workspace_identity_revision,
workspace_trust_generation: verification_response.workspace_trust_generation,
workspace_public_key_fingerprint: verification_response
.workspace_public_key_fingerprint
.clone(),
workspace_trust_id: verification_response.workspace_trust_id.clone(),
runtime_public_key_fingerprint: verification_response
.runtime_public_key_fingerprint
.clone(),
runtime_identity_revision: verification_response.runtime_identity_revision,
workspace_nonce: verification_response.workspace_nonce.clone(),
runtime_nonce: verification_response.runtime_nonce.clone(),
response_digest: workspace_request_body_digest(&response_body),
@@ -2974,13 +2988,11 @@ mod tests {
workspace_id: "workspace-b".to_string(),
source_runtime_id: "runtime-a".to_string(),
worker_id: retention_worker_id,
expected_worker_revision: "revision".to_string(),
source_created_at: "2025-01-01T00:00:00Z".to_string(),
removed_at: "2025-01-02T00:00:00Z".to_string(),
effective_profile: None,
retention_class: None,
policy_id: "policy".to_string(),
policy_revision: 1,
session_disposition: SessionDisposition::Purge,
diagnostics_disposition: DiagnosticsDisposition::Retain,
};
@@ -3018,6 +3030,141 @@ mod tests {
assert_eq!(error.error.code, "worker_not_found");
}
#[tokio::test]
async fn reenrolled_binding_trust_or_keys_require_fresh_signed_verification() {
let workspace_identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
let runtime_identity = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
let fingerprint = |key: &str| {
format!(
"sha256:{}",
crate::workspace_issuer::hex_lower(&sha2::Sha256::digest(
crate::auth::decode_public_key(key).unwrap()
))
)
};
let old_trust = WorkspaceIssuerTrustRecord::from_bundle(
server_api::WorkspacePublicIdentityBundle {
workspace_id: "workspace-a".into(),
backend_url: "https://backend.test".into(),
key_id: "workspace-key".into(),
algorithm: "ed25519".into(),
public_key: workspace_identity.public_key.clone(),
public_key_fingerprint: fingerprint(&workspace_identity.public_key),
},
"old-trust".into(),
1,
)
.unwrap();
let old_verification = WorkspaceRuntimeVerificationRecord {
workspace_id: old_trust.workspace_id.clone(),
runtime_id: "runtime-test".into(),
binding_id: "old-binding".into(),
workspace_key_id: old_trust.key_id.clone(),
workspace_public_key_fingerprint: old_trust.public_key_fingerprint.clone(),
workspace_trust_id: old_trust.trust_id.clone(),
runtime_public_key_fingerprint: fingerprint(&runtime_identity.public_key),
verified_at: 1,
};
for changed in ["binding", "trust", "workspace_key", "runtime_key"] {
let mut current = old_verification.clone();
let mut trust = old_trust.clone();
let mut workspace_key = workspace_identity.clone();
let mut runtime_key = runtime_identity.clone();
match changed {
"binding" => current.binding_id = "reenrolled-binding".into(),
"trust" => {
trust.trust_id = "reaccepted-trust".into();
current.workspace_trust_id = trust.trust_id.clone();
}
"workspace_key" => {
workspace_key = RuntimeIdentityMaterial::generate("new-workspace-key").unwrap();
trust.key_id = workspace_key.identity_id.clone();
trust.public_key = workspace_key.public_key.clone();
trust.public_key_fingerprint = fingerprint(&workspace_key.public_key);
current.workspace_key_id = trust.key_id.clone();
current.workspace_public_key_fingerprint = trust.public_key_fingerprint.clone();
}
"runtime_key" => {
runtime_key = RuntimeIdentityMaterial::generate("runtime-test").unwrap();
current.runtime_public_key_fingerprint = fingerprint(&runtime_key.public_key);
}
_ => unreachable!(),
}
let authority = Arc::new(InMemoryWorkspaceRuntimeVerificationAuthority::default());
authority.record(old_verification.clone()).unwrap();
let app = runtime_http_router_with_workspace_auth(
Runtime::new_memory(),
None,
WorkspaceRuntimeHttpAuth {
verifier: WorkspaceCapabilityVerifier::new(
vec![trust.clone()],
Arc::new(InMemoryWorkspaceClaimReplayProtection::default()),
)
.unwrap(),
signer: RuntimeVerificationSigner::from_identity(&runtime_key).unwrap(),
verifications: authority.clone(),
},
);
let claims = WorkspaceCapabilityClaims {
issuer: trust.backend_url.clone(),
issuer_workspace_id: trust.workspace_id.clone(),
issuer_key_id: trust.key_id.clone(),
issuer_public_key_fingerprint: trust.public_key_fingerprint.clone(),
trust_id: trust.trust_id.clone(),
binding_id: current.binding_id.clone(),
runtime_id: current.runtime_id.clone(),
worker_id: None,
operation: RUNTIME_PING_PERMISSION.into(),
method: "GET".into(),
path_and_query: "/v1/ping".into(),
body_digest: workspace_request_body_digest(&[]),
iat: unix_now_i64(),
exp: unix_now_i64() + 60,
jti: "stale-verification".into(),
};
for (expected, token_id) in [
(StatusCode::FORBIDDEN, "stale-verification"),
(StatusCode::OK, "fresh-verification"),
] {
if expected == StatusCode::OK {
authority.record(current.clone()).unwrap();
}
let token = issue_workspace_capability_token(
&workspace_key.signing_key().unwrap(),
&WorkspaceCapabilityClaims {
jti: token_id.into(),
..claims.clone()
},
)
.unwrap();
let response = app
.clone()
.oneshot(
Request::builder()
.uri("/v1/ping")
.header(header::AUTHORIZATION, format!("Bearer {token}"))
.header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(
status,
expected,
"{changed}: {}",
String::from_utf8_lossy(&body)
);
if expected == StatusCode::FORBIDDEN {
let error: RuntimeHttpErrorResponse = serde_json::from_slice(&body).unwrap();
assert_eq!(error.error.code, "workspace_runtime_verification_stale");
}
}
}
}
#[cfg(feature = "ws-server")]
#[test]
fn runtime_protocol_replaces_serialized_tracked_source() {
@@ -3107,7 +3254,6 @@ mod tests {
metadata: ConfigBundleMetadata {
id: "http-test-bundle".to_string(),
digest: String::new(),
revision: "test".to_string(),
workspace_id: "test-workspace".to_string(),
created_at: "test".to_string(),
provenance: ConfigBundleProvenance {
@@ -3164,7 +3310,6 @@ mod tests {
workspace_api: None,
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "local".to_string(),
settings_revision: 1,
language: "English".to_string(),
}),
subjektiv_attached: false,
@@ -4477,8 +4622,7 @@ mod tests {
algorithm: "ed25519".to_string(),
public_key: workspace_identity.public_key,
public_key_fingerprint: workspace_fingerprint,
identity_revision: 1,
trust_generation: 1,
trust_id: "trust-cleanup-test".to_string(),
state: WorkspaceIssuerTrustState::Active,
registered_at_unix: 1,
updated_at_unix: 1,
@@ -5041,7 +5185,6 @@ mod ws_tests {
metadata: ConfigBundleMetadata {
id: "ws-test-bundle".to_string(),
digest: String::new(),
revision: "test".to_string(),
workspace_id: "test".to_string(),
created_at: "test".to_string(),
provenance: ConfigBundleProvenance {
@@ -5097,7 +5240,6 @@ mod ws_tests {
workspace_api: None,
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "local".to_string(),
settings_revision: 1,
language: "English".to_string(),
}),
subjektiv_attached: false,
+71 -9
View File
@@ -905,12 +905,11 @@ fn print_workspace_trust_mutation(
WorkspaceIssuerTrustMutation::Unchanged => "unchanged",
};
println!(
"Workspace issuer trust {action}: workspace={} key={} fingerprint={} identity_revision={} trust_generation={} state={:?}",
"Workspace issuer trust {action}: workspace={} key={} fingerprint={} trust_id={} state={:?}",
record.workspace_id,
record.key_id,
record.public_key_fingerprint,
record.identity_revision,
record.trust_generation,
record.trust_id,
record.state,
);
}
@@ -936,10 +935,38 @@ fn read_runtime_auth_file(path: &Path) -> Result<RuntimeAuthFile, ProcessError>
}
let contents = String::from_utf8(contents)
.map_err(|_| ProcessError::auth("runtime auth store is corrupt"))?;
let auth: RuntimeAuthFile = toml::from_str(&contents)
let mut document: toml::Value = toml::from_str(&contents)
.map_err(|_| ProcessError::auth("runtime auth store is corrupt"))?;
let mut migrated = false;
if let Some(records) = document
.get_mut("workspace_issuers")
.and_then(toml::Value::as_array_mut)
{
for value in records {
if value.get("identity_revision").is_some() || value.get("trust_generation").is_some() {
let legacy = serde_json::to_value(&*value)
.map_err(|_| ProcessError::auth("runtime auth store is corrupt"))?;
let record =
worker_runtime::workspace_issuer::migrate_legacy_workspace_issuer_trust_record(
legacy,
)
.map_err(|_| ProcessError::auth("runtime legacy trust store is corrupt"))?;
*value = toml::Value::try_from(record).map_err(|_| {
ProcessError::auth("runtime auth migration serialization failed")
})?;
migrated = true;
}
}
}
let auth: RuntimeAuthFile = document
.try_into()
.map_err(|_| ProcessError::auth("runtime auth store is corrupt"))?;
validate_workspace_issuer_trust_records(&auth.workspace_issuers)
.map_err(|_| ProcessError::auth("runtime Workspace issuer trust store is corrupt"))?;
if migrated {
// Commit the revoked migration before exposing any issuer authority.
write_runtime_auth_file(path, &auth)?;
}
Ok(auth)
}
@@ -1398,7 +1425,6 @@ mod tests {
algorithm: "ed25519".to_string(),
public_key: identity.public_key,
public_key_fingerprint: fingerprint,
revision: 1,
})
.unwrap(),
)
@@ -1419,7 +1445,38 @@ mod tests {
let path = runtime_auth_path(&config);
let first = read_runtime_auth_file(&path).unwrap();
assert_eq!(first.workspace_issuers.len(), 1);
assert_eq!(first.workspace_issuers[0].trust_generation, 1);
assert!(!first.workspace_issuers[0].trust_id.is_empty());
// `trust-workspace show` serializes this persisted record. Owner
// registration must forward that exact enrollment ID, not derive one
// from the key fingerprint or an initial counter.
let shown = serde_json::to_value(&first.workspace_issuers[0]).unwrap();
let runtime_identity = RuntimeIdentityMaterial::generate("remote-runtime").unwrap();
let mut registration_wire = serde_json::json!({
"public_bundle": {
"identity_id": runtime_identity.identity_id,
"public_key": runtime_identity.public_key,
},
"endpoint": "https://runtime.example.test",
"workspace_trust_id": shown["trust_id"],
});
let registration: server_api::CreateRemoteRuntimeRequest =
serde_json::from_value(registration_wire.clone()).unwrap();
assert_eq!(
registration.workspace_trust_id,
first.workspace_issuers[0].trust_id
);
assert_ne!(
registration.workspace_trust_id,
first.workspace_issuers[0].public_key_fingerprint
);
registration_wire
.as_object_mut()
.unwrap()
.remove("workspace_trust_id");
assert!(
serde_json::from_value::<server_api::CreateRemoteRuntimeRequest>(registration_wire)
.is_err()
);
run_trust_workspace_command(VecDeque::from([
"add".to_string(),
@@ -1430,7 +1487,10 @@ mod tests {
]))
.unwrap();
let replay = read_runtime_auth_file(&path).unwrap();
assert_eq!(replay.workspace_issuers[0].trust_generation, 1);
assert_eq!(
replay.workspace_issuers[0].trust_id,
first.workspace_issuers[0].trust_id
);
let replacement = RuntimeIdentityMaterial::generate("WK-2").unwrap();
let public_key = decode_public_key(&replacement.public_key).unwrap();
@@ -1451,7 +1511,6 @@ mod tests {
algorithm: "ed25519".to_string(),
public_key: replacement.public_key,
public_key_fingerprint: fingerprint,
revision: 2,
})
.unwrap(),
)
@@ -1483,7 +1542,10 @@ mod tests {
]))
.unwrap();
let revoked = read_runtime_auth_file(&path).unwrap();
assert_eq!(revoked.workspace_issuers[0].trust_generation, 3);
assert_ne!(
revoked.workspace_issuers[0].trust_id,
first.workspace_issuers[0].trust_id
);
assert_eq!(
revoked.workspace_issuers[0].state,
worker_runtime::workspace_issuer::WorkspaceIssuerTrustState::Revoked
+3 -8
View File
@@ -17,7 +17,7 @@ pub const DEFAULT_BACKEND_RESOURCE_FETCH_TIMEOUT: std::time::Duration =
#[derive(Clone, Serialize, Deserialize)]
pub struct RepositorySshAccessSecretCandidate {
pub credential_id: String,
pub credential_revision: u64,
pub public_key_fingerprint: String,
pub private_key: String,
}
@@ -32,7 +32,7 @@ impl std::fmt::Debug for RepositorySshAccessSecretCandidate {
formatter
.debug_struct("RepositorySshAccessSecretCandidate")
.field("credential_id", &self.credential_id)
.field("credential_revision", &self.credential_revision)
.field("public_key_fingerprint", &self.public_key_fingerprint)
.field("private_key", &"[REDACTED]")
.finish()
}
@@ -89,9 +89,6 @@ pub struct BackendResourceHandle {
pub operation: BackendResourceOperation,
pub expires_at_unix_seconds: i64,
pub nonce: String,
pub revision: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub generation: Option<u64>,
pub max_bytes: u64,
pub content_type: String,
pub redaction: ResourceRedactionPolicy,
@@ -425,8 +422,6 @@ mod tests {
operation: BackendResourceOperation::FetchArchive,
expires_at_unix_seconds: 4_102_444_800,
nonce: "nonce-test".to_string(),
revision: sha256_hex(bytes),
generation: Some(1),
max_bytes: 1024,
content_type: PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE.to_string(),
redaction: ResourceRedactionPolicy::RuntimeInternalOnly,
@@ -477,7 +472,7 @@ mod tests {
let secret = RepositorySshAccessSecret {
credential_candidates: vec![RepositorySshAccessSecretCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 2,
public_key_fingerprint: "key-2".into(),
private_key: "PRIVATE KEY secret bytes".to_string(),
}],
known_hosts_entry: "host key secret bytes".to_string(),
-8
View File
@@ -116,13 +116,11 @@ pub struct WorkerRetentionExecutionRequest {
pub workspace_id: String,
pub source_runtime_id: String,
pub worker_id: WorkerId,
pub expected_worker_revision: String,
pub source_created_at: String,
pub removed_at: String,
pub effective_profile: Option<String>,
pub retention_class: Option<String>,
pub policy_id: String,
pub policy_revision: u64,
pub session_disposition: SessionDisposition,
pub diagnostics_disposition: DiagnosticsDisposition,
}
@@ -145,7 +143,6 @@ pub struct WorkerSessionArchiveManifest {
pub content_bytes: u64,
pub content_file_count: u64,
pub policy_id: String,
pub policy_revision: u64,
pub operation_id: String,
pub input_fingerprint: String,
}
@@ -154,7 +151,6 @@ pub struct WorkerSessionArchiveManifest {
pub struct WorkerRetentionExecutionResult {
pub operation_id: String,
pub input_fingerprint: String,
pub expected_worker_revision: String,
pub worker_id: WorkerId,
pub session_disposition: SessionDisposition,
pub diagnostics_disposition: DiagnosticsDisposition,
@@ -524,7 +520,6 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
let mut result = WorkerRetentionExecutionResult {
operation_id: request.operation_id.clone(),
input_fingerprint: request.input_fingerprint.clone(),
expected_worker_revision: request.expected_worker_revision.clone(),
worker_id: request.worker_id,
session_disposition: request.session_disposition,
diagnostics_disposition: request.diagnostics_disposition,
@@ -749,7 +744,6 @@ fn commit_session_archive(
content_bytes: bytes,
content_file_count: count,
policy_id: request.policy_id.clone(),
policy_revision: request.policy_revision,
operation_id: request.operation_id.clone(),
input_fingerprint: request.input_fingerprint.clone(),
};
@@ -1290,7 +1284,6 @@ mod tests {
WorkerRetentionExecutionRequest {
operation_id: "operation-a".to_string(),
input_fingerprint: "fingerprint-a".to_string(),
expected_worker_revision: "revision-a".to_string(),
archive_id: (disposition == SessionDisposition::Archive)
.then(|| "archive-a".to_string()),
workspace_id: "workspace-a".to_string(),
@@ -1301,7 +1294,6 @@ mod tests {
effective_profile: Some("builtin:coder".to_string()),
retention_class: None,
policy_id: "policy-a".to_string(),
policy_revision: 3,
session_disposition: disposition,
diagnostics_disposition: DiagnosticsDisposition::Purge,
}
+57 -173
View File
@@ -457,7 +457,7 @@ impl Runtime {
state
};
state.execution_backend = execution_backend;
// Initialize legacy generations durably before exposing an observation.
// Initialize legacy observation tokens durably before exposing an observation.
state.persist_workers()?;
let runtime = Self {
inner: Arc::new(Mutex::new(state)),
@@ -761,7 +761,7 @@ impl Runtime {
.zip(&ssh.credential_candidates)
.any(|(secret, metadata)| {
secret.credential_id != metadata.credential_id
|| secret.credential_revision != metadata.credential_revision
|| secret.public_key_fingerprint != metadata.public_key_fingerprint
})
{
return Err(RuntimeError::InvalidRequest(
@@ -2157,7 +2157,8 @@ impl Runtime {
&& candidate.pending_stop.is_none()
&& candidate.restore_guard.active_request_id.is_none();
if coordinate && request.preparation.is_none() && !recovering_pending {
if candidate.restore_guard.generation != request.expected_observation_token
if candidate.restore_guard.observation_token
!= request.expected_observation_token
|| candidate.restore_guard.active_request_id.is_some()
|| candidate.has_pending_lifecycle_operation()
{
@@ -2176,7 +2177,7 @@ impl Runtime {
));
}
state.ensure_running()?;
if candidate.restore_guard.generation != request.expected_observation_token
if candidate.restore_guard.observation_token != request.expected_observation_token
|| candidate.restore_guard.active_request_id.is_some()
|| (candidate.has_pending_lifecycle_operation() && !recovering_pending)
{
@@ -2283,7 +2284,7 @@ impl Runtime {
return worker.restore_guard.owners[id].request.clone();
}
runtime_api::WorkerRestoreRequest {
expected_observation_token: worker.restore_guard.generation.clone(),
expected_observation_token: worker.restore_guard.observation_token.clone(),
request_id: uuid::Uuid::now_v7().to_string(),
preparation: worker
.workspace_id
@@ -5531,7 +5532,7 @@ impl RuntimeState {
ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => Some(name.clone()),
};
Ok(SubscriptionWorker {
restore_observation_token: Some(worker.restore_guard.generation.clone()),
restore_observation_token: Some(worker.restore_guard.observation_token.clone()),
worker_id,
runtime_id: None,
resource_key: None,
@@ -5806,7 +5807,6 @@ impl RuntimeState {
InternalWorkerActivity {
status: Some(snapshot.status),
parent_session_id: snapshot.worker.parent_session_id.clone(),
revision: snapshot.revision,
},
);
for child in &snapshot.internal_workers {
@@ -5854,42 +5854,22 @@ impl RuntimeState {
fn remove_internal_worker_subtree(
activity: &mut InternalWorkerActivityProjection,
worker: &protocol::InternalWorkerRef,
revision: u64,
) {
let known_revision = activity
.workers
.get(&worker.session_id)
.map(|worker| worker.revision)
.into_iter()
.chain(
activity
.removed_workers
.get(&worker.session_id)
.map(|worker| worker.revision),
)
.max();
if known_revision.is_some_and(|known_revision| revision <= known_revision) {
return;
}
for session_id in Self::internal_worker_subtree_ids(activity, &worker.session_id) {
let active = activity.workers.remove(&session_id);
let removed = activity.removed_workers.remove(&session_id);
let (worker_revision, parent_session_id) = if session_id == worker.session_id {
(revision, worker.parent_session_id.clone())
let parent_session_id = if session_id == worker.session_id {
worker.parent_session_id.clone()
} else if let Some(active) = active {
(active.revision, active.parent_session_id)
active.parent_session_id
} else if let Some(removed) = removed {
(removed.revision, removed.parent_session_id)
removed.parent_session_id
} else {
continue;
};
activity.removed_workers.insert(
session_id,
RemovedInternalWorkerActivity {
revision: worker_revision,
parent_session_id,
},
RemovedInternalWorkerActivity { parent_session_id },
);
}
}
@@ -5897,37 +5877,25 @@ impl RuntimeState {
fn touch_internal_worker_activity(
activity: &mut InternalWorkerActivityProjection,
worker: &protocol::InternalWorkerRef,
revision: u64,
) {
activity
.workers
.entry(worker.session_id.clone())
.and_modify(|activity| {
activity.parent_session_id = worker.parent_session_id.clone();
activity.revision = revision;
})
.and_modify(|activity| activity.parent_session_id = worker.parent_session_id.clone())
.or_insert_with(|| InternalWorkerActivity {
status: None,
parent_session_id: worker.parent_session_id.clone(),
revision,
});
}
fn project_internal_worker_event(
activity: &mut InternalWorkerActivityProjection,
worker: &protocol::InternalWorkerRef,
revision: u64,
event: &protocol::Event,
) {
if activity.removed_workers.contains_key(&worker.session_id)
|| activity
.workers
.get(&worker.session_id)
.is_some_and(|current| revision <= current.revision)
{
if activity.removed_workers.contains_key(&worker.session_id) {
return;
}
match event {
protocol::Event::Snapshot {
state,
@@ -5940,7 +5908,6 @@ impl RuntimeState {
InternalWorkerActivity {
status: Some(state.catalog_status()),
parent_session_id: worker.parent_session_id.clone(),
revision,
},
);
for child in internal_workers {
@@ -5949,23 +5916,16 @@ impl RuntimeState {
}
protocol::Event::InternalWorker {
worker: nested_worker,
revision: nested_revision,
event,
} => {
Self::touch_internal_worker_activity(activity, worker, revision);
Self::project_internal_worker_event(
activity,
nested_worker,
*nested_revision,
event,
);
Self::touch_internal_worker_activity(activity, worker);
Self::project_internal_worker_event(activity, nested_worker, event);
}
protocol::Event::InternalWorkerRemoved {
worker: removed_worker,
revision: removed_revision,
} => {
Self::touch_internal_worker_activity(activity, worker, revision);
Self::remove_internal_worker_subtree(activity, removed_worker, *removed_revision);
Self::touch_internal_worker_activity(activity, worker);
Self::remove_internal_worker_subtree(activity, removed_worker);
}
protocol::Event::WorkerState { snapshot }
| protocol::Event::CommandAcknowledged {
@@ -5979,11 +5939,10 @@ impl RuntimeState {
InternalWorkerActivity {
status: Some(snapshot.catalog_status()),
parent_session_id: worker.parent_session_id.clone(),
revision,
},
);
}
_ => Self::touch_internal_worker_activity(activity, worker, revision),
_ => Self::touch_internal_worker_activity(activity, worker),
}
}
@@ -6001,15 +5960,11 @@ impl RuntimeState {
Self::internal_worker_snapshot_statuses(activity, child);
}
}
protocol::Event::InternalWorker {
worker,
revision,
event,
} => {
Self::project_internal_worker_event(activity, worker, *revision, event);
protocol::Event::InternalWorker { worker, event } => {
Self::project_internal_worker_event(activity, worker, event);
}
protocol::Event::InternalWorkerRemoved { worker, revision } => {
Self::remove_internal_worker_subtree(activity, worker, *revision);
protocol::Event::InternalWorkerRemoved { worker } => {
Self::remove_internal_worker_subtree(activity, worker);
}
_ => {}
}
@@ -6057,12 +6012,10 @@ impl RuntimeState {
struct InternalWorkerActivity {
status: Option<protocol::WorkerStatus>,
parent_session_id: Option<String>,
revision: u64,
}
#[derive(Debug, Clone)]
struct RemovedInternalWorkerActivity {
revision: u64,
parent_session_id: Option<String>,
}
@@ -6111,8 +6064,9 @@ pub(crate) struct RestoreRequestOwner {
}
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct RestoreGuard {
generation: String,
observation_token: String,
active_request_id: Option<String>,
owners: BTreeMap<String, RestoreRequestOwner>,
}
@@ -6120,7 +6074,7 @@ pub(crate) struct RestoreGuard {
impl Default for RestoreGuard {
fn default() -> Self {
Self {
generation: uuid::Uuid::now_v7().to_string(),
observation_token: uuid::Uuid::now_v7().to_string(),
active_request_id: None,
owners: BTreeMap::new(),
}
@@ -6129,7 +6083,7 @@ impl Default for RestoreGuard {
impl RestoreGuard {
fn rotate(&mut self) {
self.generation = uuid::Uuid::now_v7().to_string();
self.observation_token = uuid::Uuid::now_v7().to_string();
}
fn finish(&mut self, result: RuntimeWorkerRestoreResult) {
@@ -6199,7 +6153,7 @@ impl WorkerRecord {
fn summary(&self) -> WorkerSummary {
WorkerSummary {
restore_observation_token: Some(self.restore_guard.generation.clone()),
restore_observation_token: Some(self.restore_guard.observation_token.clone()),
worker_ref: self.worker_ref.clone(),
worker_id: self.worker_id,
status: self.status,
@@ -6217,7 +6171,7 @@ impl WorkerRecord {
fn detail(&self) -> WorkerDetail {
WorkerDetail {
restore_observation_token: Some(self.restore_guard.generation.clone()),
restore_observation_token: Some(self.restore_guard.observation_token.clone()),
worker_ref: self.worker_ref.clone(),
worker_id: self.worker_id,
status: self.status,
@@ -6740,11 +6694,6 @@ fn validate_create_workspace_scope(
snapshot.workspace_id
)));
}
if snapshot.settings_revision == 0 {
return Err(RuntimeError::InvalidRequest(
"Memory settings revision must be at least 1".to_string(),
));
}
if !manifest::is_normalized_workspace_memory_language(&snapshot.language) {
return Err(RuntimeError::InvalidRequest(
"Memory settings language must be a normalized bounded UTF-8 value".to_string(),
@@ -6953,12 +6902,10 @@ mod tests {
fn internal_worker_status_event(
worker: protocol::InternalWorkerRef,
revision: u64,
status: protocol::WorkerStatus,
) -> protocol::Event {
protocol::Event::InternalWorker {
worker,
revision,
event: Box::new(protocol::Event::WorkerState {
snapshot: status.into(),
}),
@@ -6967,13 +6914,11 @@ mod tests {
fn internal_worker_snapshot(
worker: protocol::InternalWorkerRef,
revision: u64,
status: protocol::WorkerStatus,
internal_workers: Vec<protocol::InternalWorkerSnapshot>,
) -> protocol::InternalWorkerSnapshot {
protocol::InternalWorkerSnapshot {
worker,
revision,
status,
greeting: None,
session: protocol::SessionSnapshot {
@@ -7017,7 +6962,6 @@ mod tests {
&mut activity,
&internal_worker_status_event(
internal_worker_ref("child-a", None),
1,
protocol::WorkerStatus::Running,
),
));
@@ -7025,7 +6969,6 @@ mod tests {
&mut activity,
&internal_worker_status_event(
internal_worker_ref("child-b", None),
1,
protocol::WorkerStatus::Running,
),
));
@@ -7033,7 +6976,6 @@ mod tests {
&mut activity,
&internal_worker_status_event(
internal_worker_ref("child-a", None),
2,
protocol::WorkerStatus::Idle,
),
));
@@ -7041,7 +6983,6 @@ mod tests {
&mut activity,
&internal_worker_status_event(
internal_worker_ref("child-b", None),
2,
protocol::WorkerStatus::Stopped,
),
));
@@ -7053,10 +6994,8 @@ mod tests {
let direct_child = internal_worker_ref("child", None);
let nested_running = protocol::Event::InternalWorker {
worker: direct_child.clone(),
revision: 1,
event: Box::new(internal_worker_status_event(
internal_worker_ref("grandchild", Some("child")),
1,
protocol::WorkerStatus::Running,
)),
};
@@ -7067,10 +7006,8 @@ mod tests {
let nested_idle = protocol::Event::InternalWorker {
worker: direct_child,
revision: 2,
event: Box::new(internal_worker_status_event(
internal_worker_ref("grandchild", Some("child")),
2,
protocol::WorkerStatus::Idle,
)),
};
@@ -7087,7 +7024,6 @@ mod tests {
&mut activity,
&internal_worker_status_event(
internal_worker_ref("child-a", None),
1,
protocol::WorkerStatus::Running,
),
);
@@ -7101,44 +7037,23 @@ mod tests {
}
#[test]
fn internal_worker_removal_is_revision_fenced_until_parent_snapshot() {
fn internal_worker_removal_is_session_fenced_until_parent_snapshot() {
let mut activity = InternalWorkerActivityProjection::default();
let child_a = internal_worker_ref("child-a", None);
let child_b = internal_worker_ref("child-b", None);
assert!(RuntimeState::update_internal_worker_activity(
&mut activity,
&internal_worker_status_event(child_a.clone(), 2, protocol::WorkerStatus::Running),
&internal_worker_status_event(child_a.clone(), protocol::WorkerStatus::Running),
));
assert!(!RuntimeState::update_internal_worker_activity(
&mut activity,
&internal_worker_status_event(child_b.clone(), 1, protocol::WorkerStatus::Running),
&internal_worker_status_event(child_b.clone(), protocol::WorkerStatus::Running),
));
assert!(!RuntimeState::update_internal_worker_activity(
&mut activity,
&protocol::Event::InternalWorkerRemoved {
worker: child_a.clone(),
revision: 1,
},
));
assert!(activity.workers.contains_key("child-a"));
assert!(!RuntimeState::update_internal_worker_activity(
&mut activity,
&protocol::Event::InternalWorkerRemoved {
worker: child_a.clone(),
revision: 2,
},
));
assert!(activity.workers.contains_key("child-a"));
assert!(!RuntimeState::update_internal_worker_activity(
&mut activity,
&protocol::Event::InternalWorkerRemoved {
worker: child_a,
revision: 3,
},
&protocol::Event::InternalWorkerRemoved { worker: child_a },
));
assert!(!activity.workers.contains_key("child-a"));
assert!(activity.removed_workers.contains_key("child-a"));
@@ -7148,14 +7063,13 @@ mod tests {
&mut activity,
&protocol::Event::InternalWorkerRemoved {
worker: child_b.clone(),
revision: 2,
},
));
assert!(!activity.has_running_worker());
assert!(!RuntimeState::update_internal_worker_activity(
&mut activity,
&internal_worker_status_event(child_b.clone(), 3, protocol::WorkerStatus::Running),
&internal_worker_status_event(child_b.clone(), protocol::WorkerStatus::Running),
));
assert!(!activity.workers.contains_key("child-b"));
assert!(!activity.has_running_worker());
@@ -7164,7 +7078,6 @@ mod tests {
&mut activity,
&parent_snapshot(vec![internal_worker_snapshot(
child_b,
4,
protocol::WorkerStatus::Running,
Vec::new(),
)]),
@@ -7182,16 +7095,14 @@ mod tests {
assert!(!RuntimeState::update_internal_worker_activity(
&mut activity,
&internal_worker_status_event(child.clone(), 1, protocol::WorkerStatus::Idle),
&internal_worker_status_event(child.clone(), protocol::WorkerStatus::Idle),
));
assert!(RuntimeState::update_internal_worker_activity(
&mut activity,
&protocol::Event::InternalWorker {
worker: child.clone(),
revision: 2,
event: Box::new(internal_worker_status_event(
grandchild.clone(),
1,
protocol::WorkerStatus::Running,
)),
},
@@ -7200,13 +7111,10 @@ mod tests {
&mut activity,
&protocol::Event::InternalWorker {
worker: child.clone(),
revision: 3,
event: Box::new(protocol::Event::InternalWorker {
worker: grandchild.clone(),
revision: 2,
event: Box::new(internal_worker_status_event(
great_grandchild,
1,
protocol::WorkerStatus::Running,
)),
}),
@@ -7217,10 +7125,8 @@ mod tests {
&mut activity,
&protocol::Event::InternalWorker {
worker: child.clone(),
revision: 4,
event: Box::new(protocol::Event::InternalWorkerRemoved {
worker: grandchild.clone(),
revision: 3,
}),
},
));
@@ -7235,10 +7141,8 @@ mod tests {
&mut activity,
&protocol::Event::InternalWorker {
worker: child,
revision: 5,
event: Box::new(internal_worker_status_event(
grandchild,
4,
protocol::WorkerStatus::Running,
)),
},
@@ -7338,7 +7242,6 @@ mod tests {
workspace_api: None,
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "local".to_string(),
settings_revision: 1,
language: "English".to_string(),
}),
subjektiv_attached: false,
@@ -7367,7 +7270,7 @@ mod tests {
request.backend_job = Some(crate::catalog::BackendJobExecutionBinding {
job_id: "job-1".into(),
attempt_id: "attempt-1".into(),
input_revision: Some("revision-1".into()),
input_digest: Some(format!("sha256:{}", "a".repeat(64))),
subjektiv_consolidation: false,
});
let owner = scope("local", "server-job-test");
@@ -7416,7 +7319,7 @@ mod tests {
request.backend_job = Some(crate::catalog::BackendJobExecutionBinding {
job_id: "job-1".into(),
attempt_id: "attempt-1".into(),
input_revision: None,
input_digest: None,
subjektiv_consolidation: true,
});
assert!(validate_create_worker_request(&request).is_err());
@@ -7444,7 +7347,6 @@ mod tests {
kind: server_api::RepositorySourceKind::Ssh,
uri: "ssh://git@example.test/repo.git".to_string(),
},
source_revision: 1,
source_fingerprint: "sha256:source".to_string(),
selector: None,
},
@@ -7455,16 +7357,15 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-1".to_string(),
config_revision: 1,
config_projection_digest: "sha256:projection".to_string(),
ssh: Some(RepositorySshMaterializationAccess {
credential_candidates: vec![RepositorySshCredentialCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 1,
public_key_fingerprint: "key-1".into(),
private_key: SensitiveString::new("private-key-bytes"),
}],
host_trust_id: "host-trust-1".to_string(),
host_trust_revision: 1,
host_key_fingerprint: "key-1".into(),
access: server_api::RepositoryAccessMode::ReadOnly,
expires_at_epoch_seconds: u64::MAX,
repository_id: "repository-1".to_string(),
@@ -7512,8 +7413,6 @@ mod tests {
operation: crate::resource::BackendResourceOperation::FetchOnce,
expires_at_unix_seconds: i64::MAX,
nonce: "repository-access-1".to_string(),
revision: "1".to_string(),
generation: None,
max_bytes: crate::resource::DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES,
content_type: crate::resource::REPOSITORY_SSH_ACCESS_CONTENT_TYPE.to_string(),
redaction: crate::resource::ResourceRedactionPolicy::RuntimeInternalOnly,
@@ -7544,12 +7443,12 @@ mod tests {
credential_candidates: vec![
crate::resource::RepositorySshAccessSecretCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 1,
public_key_fingerprint: "key-1".into(),
private_key: "private-key-bytes-1".to_string(),
},
crate::resource::RepositorySshAccessSecretCandidate {
credential_id: "credential-2".to_string(),
credential_revision: 3,
public_key_fingerprint: "key-3".into(),
private_key: "private-key-bytes-2".to_string(),
},
],
@@ -7567,23 +7466,22 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-1".to_string(),
config_revision: 1,
config_projection_digest: "sha256:projection".to_string(),
ssh: Some(RepositorySshMaterializationAccess {
credential_candidates: vec![
RepositorySshCredentialCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 1,
public_key_fingerprint: "key-1".into(),
private_key: SensitiveString::default(),
},
RepositorySshCredentialCandidate {
credential_id: "credential-2".to_string(),
credential_revision: 3,
public_key_fingerprint: "key-3".into(),
private_key: SensitiveString::default(),
},
],
host_trust_id: "host-trust-1".to_string(),
host_trust_revision: 1,
host_key_fingerprint: "key-1".into(),
access: server_api::RepositoryAccessMode::ReadOnly,
expires_at_epoch_seconds: u64::MAX,
repository_id: "repository-1".to_string(),
@@ -7652,12 +7550,12 @@ mod tests {
credential_candidates: vec![
crate::resource::RepositorySshAccessSecretCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 1,
public_key_fingerprint: "key-1".into(),
private_key: "create-private-key-bytes-1".to_string(),
},
crate::resource::RepositorySshAccessSecretCandidate {
credential_id: "credential-2".to_string(),
credential_revision: 3,
public_key_fingerprint: "key-3".into(),
private_key: "create-private-key-bytes-2".to_string(),
},
],
@@ -7677,7 +7575,6 @@ mod tests {
kind: server_api::RepositorySourceKind::Ssh,
uri: "ssh://git@example.test/repo.git".to_string(),
},
source_revision: 1,
source_fingerprint: "sha256:source".to_string(),
selector: None,
},
@@ -7688,23 +7585,22 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-create".to_string(),
config_revision: 1,
config_projection_digest: "sha256:projection".to_string(),
ssh: Some(RepositorySshMaterializationAccess {
credential_candidates: vec![
RepositorySshCredentialCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 1,
public_key_fingerprint: "key-1".into(),
private_key: SensitiveString::default(),
},
RepositorySshCredentialCandidate {
credential_id: "credential-2".to_string(),
credential_revision: 3,
public_key_fingerprint: "key-3".into(),
private_key: SensitiveString::default(),
},
],
host_trust_id: "host-trust-1".to_string(),
host_trust_revision: 1,
host_key_fingerprint: "key-1".into(),
access: server_api::RepositoryAccessMode::ReadOnly,
expires_at_epoch_seconds: u64::MAX,
repository_id: "repository-1".to_string(),
@@ -7751,7 +7647,6 @@ mod tests {
});
request.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: workspace_id.to_string(),
settings_revision: 1,
language: "English".to_string(),
});
request
@@ -7769,14 +7664,12 @@ mod tests {
request.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "workspace-b".to_string(),
settings_revision: 1,
language: "English".to_string(),
});
assert!(validate_create_workspace_scope(&request, Some("workspace-a")).is_err());
request.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "workspace-a".to_string(),
settings_revision: 2,
language: " english ".to_string(),
});
assert!(validate_create_workspace_scope(&request, Some("workspace-a")).is_err());
@@ -7791,7 +7684,6 @@ mod tests {
metadata: ConfigBundleMetadata {
id: "bundle-1".to_string(),
digest: String::new(),
revision: "rev-1".to_string(),
workspace_id: "workspace-1".to_string(),
created_at: "2026-06-26T00:00:00Z".to_string(),
provenance: ConfigBundleProvenance {
@@ -8350,7 +8242,6 @@ mod tests {
kind: server_api::RepositorySourceKind::LocalPath,
uri: "/unused-test-repository".into(),
},
source_revision: 1,
source_fingerprint: "sha256:test-source".into(),
selector: None,
},
@@ -8368,7 +8259,6 @@ mod tests {
workspace_id: "workspace-a".into(),
runtime_id: "runtime-1".into(),
operation_id: "operation-1".into(),
config_revision: 1,
config_projection_digest: "sha256:test-projection".into(),
ssh: None,
},
@@ -8842,7 +8732,6 @@ mod tests {
&created.worker_ref,
internal_worker_status_event(
internal_worker_ref("child-live", None),
1,
protocol::WorkerStatus::Running,
),
)
@@ -8861,7 +8750,6 @@ mod tests {
&created.worker_ref,
protocol::Event::InternalWorkerRemoved {
worker: internal_worker_ref("child-live", None),
revision: 2,
},
)
.unwrap();
@@ -8879,7 +8767,6 @@ mod tests {
&created.worker_ref,
internal_worker_status_event(
internal_worker_ref("child-live", None),
3,
protocol::WorkerStatus::Running,
),
)
@@ -8900,7 +8787,6 @@ mod tests {
&created.worker_ref,
parent_snapshot(vec![internal_worker_snapshot(
internal_worker_ref("child-live", None),
4,
protocol::WorkerStatus::Running,
Vec::new(),
)]),
@@ -9427,7 +9313,6 @@ mod tests {
bundle.prompt_catalog = Some(
worker::EffectivePromptCatalog::new(
BTreeMap::from([("default".to_string(), "workspace prompt".to_string())]),
7,
"schema",
"toolchain",
)
@@ -10287,7 +10172,6 @@ mod tests {
workspace_id: "workspace-a".into(),
runtime_id: "peer-runtime".into(),
operation_id: format!("worker-restore:{}", request.request_id),
config_revision: 1,
config_projection_digest: "snapshot".into(),
ssh: None,
},
@@ -10405,7 +10289,7 @@ mod tests {
}
#[test]
fn restore_guard_protocol_state_aba_rotates_generation() {
fn restore_guard_protocol_state_aba_rotates_observation_token() {
let (runtime, backend) = runtime_and_backend();
let worker = runtime.create_worker(task_request("state ABA")).unwrap();
let request = runtime.test_restore_request(&worker.worker_ref);
@@ -11160,7 +11044,7 @@ mod tests {
#[cfg(feature = "fs-store")]
#[test]
fn restore_guard_generation_and_terminal_receipt_survive_restart() {
fn restore_guard_observation_token_and_terminal_receipt_survive_restart() {
let dir = tempfile::tempdir().unwrap();
let options =
FsRuntimeStoreOptions::new(dir.path().join("runtime")).with_runtime_id("guard-test");
@@ -11177,7 +11061,7 @@ mod tests {
.restore_worker_operation(&worker.worker_ref, request.clone())
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let generation = runtime
let observation_token = runtime
.worker_detail(&worker.worker_ref)
.unwrap()
.restore_observation_token;
@@ -11188,7 +11072,7 @@ mod tests {
.worker_detail(&worker.worker_ref)
.unwrap()
.restore_observation_token,
generation
observation_token
);
assert_eq!(
restarted
@@ -12680,7 +12564,7 @@ mod tests {
assert!(
error
.to_string()
.contains("unsupported Runtime store schema version 2; expected 6, 7, 8, or 9")
.contains("unsupported Runtime store schema version 2; expected 6, 7, 8, 9, or 10")
);
let _ = std::fs::remove_dir_all(root);
@@ -12722,7 +12606,7 @@ mod tests {
let worker_aggregate: serde_json::Value =
serde_json::from_slice(&std::fs::read(worker_store_dir.join("worker.json")).unwrap())
.unwrap();
assert_eq!(worker_aggregate["schema_version"], serde_json::json!(9));
assert_eq!(worker_aggregate["schema_version"], serde_json::json!(10));
assert_eq!(worker_aggregate["status"], serde_json::json!("stopped"));
assert_eq!(
worker_aggregate["execution_state"]["state"],
@@ -13072,7 +12956,7 @@ mod tests {
.unwrap()
.remove("restore_guard")
.unwrap();
assert_ne!(guard["generation"], old_guard["generation"]);
assert_ne!(guard["observation_token"], old_guard["observation_token"]);
assert!(guard["active_request_id"].is_null());
assert_eq!(
guard["owners"]
@@ -13669,7 +13553,7 @@ mod tests {
);
let migrated_aggregate: serde_json::Value =
serde_json::from_slice(&std::fs::read(&worker_path).unwrap()).unwrap();
assert_eq!(migrated_aggregate["schema_version"], serde_json::json!(9));
assert_eq!(migrated_aggregate["schema_version"], serde_json::json!(10));
assert!(migrated_aggregate.get("request").is_none());
assert!(migrated_aggregate.get("execution").is_none());
assert_eq!(
@@ -374,7 +374,6 @@ fn api_archive_manifest(manifest: &DomainArchiveManifest) -> WorkerSessionArchiv
content_bytes: manifest.content_bytes,
content_file_count: manifest.content_file_count,
policy_id: manifest.policy_id.clone(),
policy_revision: manifest.policy_revision,
operation_id: manifest.operation_id.clone(),
input_fingerprint: manifest.input_fingerprint.clone(),
}
+43 -61
View File
@@ -133,7 +133,7 @@ use worker::feature::builtin::{
WorkspaceClientWorkerObservationProvider,
};
#[cfg(feature = "ws-server")]
use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session};
use worker::ipc::protocol_session::{WorkerProtocolSessionStreams, live_log_entry_event};
use worker::{
PreparedWorker, PromptCatalogSource, SegmentLogSink, SubjektivSessionAttributionLifecycle,
Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout,
@@ -142,6 +142,15 @@ use worker::{
bash_output_dir_for_worker_id,
};
#[cfg(feature = "ws-server")]
fn subscribe_worker_protocol_session(handle: &WorkerHandle) -> WorkerProtocolSessionStreams {
// Child state changes and their live events share this publication gate.
// Acquire it before subscribing, and retain it through the parent snapshot
// capture, so a pre-snapshot child event cannot replay after newer state.
let _publish_guard = handle.protocol_snapshot_publish_guard();
worker::ipc::protocol_session::subscribe_worker_protocol_session(handle)
}
const DEFAULT_BACKEND_ID: &str = "worker-crate";
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
const SPAWN_RESTORE_TASK_TIMEOUT: Duration = Duration::from_secs(60);
@@ -403,26 +412,12 @@ impl WorkspacePromptProjectionCache {
.active
.lock()
.map_err(|_| "Workspace Prompt projection cache lock was poisoned".to_string())?;
if let Some(current) = active.get(&workspace_id) {
if current.projection.config_revision > resolution.projection.config_revision {
return Ok(current.clone());
}
if current.projection.config_revision == resolution.projection.config_revision
&& (current.projection.source_digest != resolution.projection.source_digest
|| current.projection.projection_digest
!= resolution.projection.projection_digest
|| current.projection.catalog.catalog_digest
!= resolution.projection.catalog.catalog_digest
|| current.projection.catalog.schema_fingerprint
!= resolution.projection.catalog.schema_fingerprint
|| current.projection.catalog.toolchain_fingerprint
!= resolution.projection.catalog.toolchain_fingerprint)
{
return Err(format!(
"Workspace Prompt projection identity changed without a config revision transition: workspace={workspace_id} revision={}",
resolution.projection.config_revision
));
}
// Runtime receives authoritative projections in operation/arrival order.
// Equal content reuses the compiled catalog; counters are not authority.
if let Some(current) = active.get(&workspace_id)
&& current.projection == resolution.projection
{
return Ok(current.clone());
}
active.insert(workspace_id, resolution.clone());
Ok(resolution)
@@ -688,7 +683,7 @@ impl ProfileRuntimeWorkerFactory {
None => fallback,
};
// Saved policy is execution authority. Rebinding current settings here
// would hide revision and attachment mismatches.
// would hide content and attachment mismatches.
validate_worker_memory_settings(&manifest, request)?;
Ok((manifest, loader))
}
@@ -1195,11 +1190,8 @@ fn validate_worker_memory_settings(
"Workspace Worker restored without its bound Memory settings snapshot".to_string()
})?;
return Err(format!(
"Workspace Worker Memory settings snapshot mismatch: expected {} revision {}, restored {} revision {}",
expected.workspace_id,
expected.settings_revision,
actual.workspace_id,
actual.settings_revision
"Workspace Worker Memory settings snapshot mismatch: expected {} language {}, restored {} language {}",
expected.workspace_id, expected.language, actual.workspace_id, actual.language
));
}
let subjektiv_settings = manifest.feature.subjektiv.workspace_settings();
@@ -1216,11 +1208,8 @@ fn validate_worker_memory_settings(
.to_string()
})?;
return Err(format!(
"Workspace Worker subjektiv settings snapshot mismatch: expected {} revision {}, restored {} revision {}",
expected.workspace_id,
expected.settings_revision,
actual.workspace_id,
actual.settings_revision
"Workspace Worker subjektiv settings snapshot mismatch: expected {} language {}, restored {} language {}",
expected.workspace_id, expected.language, actual.workspace_id, actual.language
));
}
} else if subjektiv_settings.is_some() {
@@ -4789,7 +4778,6 @@ mod tests {
request.subjektiv_attached = true;
request.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "workspace-saved-subjektiv".to_string(),
settings_revision: 17,
language: "English".to_string(),
});
bind_workspace_memory_settings(&mut saved, &request).unwrap();
@@ -5982,7 +5970,6 @@ mod tests {
);
let mut catalog = worker::EffectivePromptCatalog::new(
templates,
17,
projection.schema_fingerprint.clone(),
projection.toolchain_fingerprint.clone(),
)
@@ -6005,7 +5992,6 @@ mod tests {
missing_instruction_bundle.prompt_catalog = Some(
worker::EffectivePromptCatalog::new(
templates,
17,
projection.schema_fingerprint.clone(),
projection.toolchain_fingerprint.clone(),
)
@@ -6115,19 +6101,19 @@ mod tests {
.with_runtime_store_dir(&runtime_store)
.with_runtime_id("runtime-subjektiv-test");
let before = persisted_files(&runtime_store);
let mut wrong_revision = request.clone();
wrong_revision
let mut wrong_settings = request.clone();
wrong_settings
.request
.memory_settings
.as_mut()
.unwrap()
.settings_revision = 18;
.language = "different language".into();
let mut detached = request.clone();
detached.request.subjektiv_attached = false;
let mut missing_settings = request.clone();
missing_settings.request.memory_settings = None;
for (invalid, expected) in [
(wrong_revision, "subjektiv settings snapshot mismatch"),
(wrong_settings, "subjektiv settings snapshot mismatch"),
(detached, "unauthorized subjektiv attachment"),
(missing_settings, "missing its trusted settings snapshot"),
] {
@@ -6660,7 +6646,6 @@ mod tests {
let cache = WorkspacePromptProjectionCache::default();
let catalog_v1 = worker::EffectivePromptCatalog::new(
BTreeMap::from([("default".to_string(), "prompt-v1".to_string())]),
8,
"schema",
"toolchain",
)
@@ -6674,7 +6659,6 @@ mod tests {
.unwrap();
let catalog_v2 = worker::EffectivePromptCatalog::new(
BTreeMap::from([("default".to_string(), "prompt-v2".to_string())]),
9,
"schema",
"toolchain",
)
@@ -6691,7 +6675,7 @@ mod tests {
cache.observe(projection_v2).unwrap();
let active = cache.active("workspace-a").unwrap().unwrap();
assert_eq!(active.projection.config_revision, 9);
assert_eq!(active.projection.source_digest, "source-v2");
assert_eq!(
active.projection.catalog.catalog_digest,
catalog_v2.catalog_digest
@@ -6699,10 +6683,9 @@ mod tests {
}
#[test]
fn workspace_prompt_projection_cache_rejects_same_revision_source_drift() {
fn workspace_prompt_projection_cache_accepts_authoritative_source_change() {
let catalog = worker::EffectivePromptCatalog::new(
BTreeMap::from([("default".to_string(), "prompt".to_string())]),
8,
"schema",
"toolchain",
)
@@ -6724,22 +6707,20 @@ mod tests {
let cache = WorkspacePromptProjectionCache::default();
cache.observe(first).unwrap();
let error = cache.observe(drifted).unwrap_err();
assert!(
error
.to_string()
.contains("without a config revision transition")
assert_eq!(
cache.observe(drifted).unwrap().projection.source_digest,
"source-b"
);
}
#[test]
fn workspace_prompt_projection_cache_rejects_same_revision_schema_drift() {
fn workspace_prompt_projection_cache_accepts_authoritative_toolchain_change() {
let templates = BTreeMap::from([("default".to_string(), "prompt".to_string())]);
let first_catalog =
worker::EffectivePromptCatalog::new(templates.clone(), 8, "schema-a", "toolchain")
worker::EffectivePromptCatalog::new(templates.clone(), "schema-a", "toolchain")
.unwrap();
let drifted_catalog =
worker::EffectivePromptCatalog::new(templates, 8, "schema-b", "toolchain").unwrap();
worker::EffectivePromptCatalog::new(templates, "schema-b", "toolchain").unwrap();
let first = worker::WorkspacePromptProjection::new(
"workspace-a",
"source-a",
@@ -6757,8 +6738,14 @@ mod tests {
let cache = WorkspacePromptProjectionCache::default();
cache.observe(first).unwrap();
let error = cache.observe(drifted).unwrap_err();
assert!(error.contains("without a config revision transition"));
assert_eq!(
cache
.observe(drifted)
.unwrap()
.projection
.schema_fingerprint,
"schema-b"
);
}
#[test]
@@ -7165,7 +7152,6 @@ mod tests {
metadata: crate::config_bundle::ConfigBundleMetadata {
id: "adapter-test-bundle".to_string(),
digest: String::new(),
revision: "test".to_string(),
workspace_id: "adapter-test".to_string(),
created_at: "test".to_string(),
provenance: crate::config_bundle::ConfigBundleProvenance {
@@ -7273,7 +7259,7 @@ mod tests {
request.backend_job = Some(crate::catalog::BackendJobExecutionBinding {
job_id: "job-1".into(),
attempt_id: "attempt-1".into(),
input_revision: None,
input_digest: None,
subjektiv_consolidation: granted,
});
for tools in [false, true] {
@@ -7329,7 +7315,6 @@ mod tests {
};
let settings = manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "workspace-subject".to_string(),
settings_revision: 3,
language: "English".to_string(),
};
let mut ordinary_request = create_request("ordinary");
@@ -7987,7 +7972,6 @@ mod tests {
kind: server_api::RepositorySourceKind::LocalPath,
uri: repo.display().to_string(),
},
source_revision: 1,
source_fingerprint: "sha256:test".to_string(),
selector: Some(RepositorySelector::from("HEAD")),
},
@@ -8197,7 +8181,6 @@ mod tests {
);
let mut effective = worker::EffectivePromptCatalog::new(
templates,
7,
projection.schema_fingerprint.clone(),
projection.toolchain_fingerprint.clone(),
)
@@ -8213,7 +8196,7 @@ mod tests {
.unwrap()
.unwrap();
assert_eq!(resolution.projection.config_revision, 7);
assert_eq!(resolution.projection.source_digest, "source-7");
assert_eq!(
resolution.catalog.notify_wrapper("restored").unwrap(),
"PENDING-LAUNCH restored"
@@ -8250,7 +8233,6 @@ mod tests {
});
request.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: "workspace-restore".to_string(),
settings_revision: 1,
language: "English".to_string(),
});
let identity = RuntimeIdentityMaterial::generate("runtime-restore").unwrap();
+7 -32
View File
@@ -736,7 +736,6 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
fn current_prompt_projection(
&self,
minimum_revision: Option<u64>,
) -> Result<Option<WorkspacePromptCatalogResolution>, WorkspaceClientError> {
let Some(cache) = self.prompt_projection_cache.as_ref() else {
return Ok(None);
@@ -744,11 +743,6 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
if let Some(resolution) = cache
.active(&self.workspace_id)
.map_err(WorkspaceClientError::Request)?
.filter(|resolution| {
minimum_revision
.map(|minimum| resolution.projection.config_revision >= minimum)
.unwrap_or(true)
})
{
return Ok(Some((*resolution).clone()));
}
@@ -763,11 +757,6 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
if let Some(resolution) = cache
.active(&self.workspace_id)
.map_err(WorkspaceClientError::Request)?
.filter(|resolution| {
minimum_revision
.map(|minimum| resolution.projection.config_revision >= minimum)
.unwrap_or(true)
})
{
return Ok(Some((*resolution).clone()));
}
@@ -807,14 +796,6 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
let resolution = cache
.observe(projection)
.map_err(WorkspaceClientError::Request)?;
if let Some(minimum_revision) = minimum_revision
&& resolution.projection.config_revision < minimum_revision
{
return Err(WorkspaceClientError::Request(format!(
"active Workspace Prompt projection is stale: required revision {minimum_revision}, got {}",
resolution.projection.config_revision
)));
}
Ok(Some((*resolution).clone()))
}
@@ -1082,7 +1063,6 @@ mod tests {
"default".to_string(),
"workspace prompt".to_string(),
)]),
3,
"schema",
"toolchain",
)
@@ -1103,22 +1083,20 @@ mod tests {
)
.with_prompt_projection_cache(cache);
let projection = client.current_prompt_projection(None).unwrap().unwrap();
let second = client.current_prompt_projection(None).unwrap().unwrap();
let projection = client.current_prompt_projection().unwrap().unwrap();
let second = client.current_prompt_projection().unwrap().unwrap();
assert_eq!(projection.projection.config_revision, 3);
assert_eq!(projection.projection.source_digest, "source-3");
assert!(Arc::ptr_eq(&projection.catalog, &second.catalog));
}
#[test]
fn prompt_projection_minimum_revision_rejects_stale_server_response() {
fn prompt_projection_cache_miss_fetches_content_identity() {
use std::io::{Read, Write};
use std::net::TcpListener;
let catalog = worker::EffectivePromptCatalog::new(
std::collections::BTreeMap::from([("default".to_string(), "stale prompt".to_string())]),
3,
"schema",
"toolchain",
)
@@ -1132,7 +1110,6 @@ mod tests {
.unwrap();
let body = serde_json::to_string(&projection).unwrap();
let cache = Arc::new(WorkspacePromptProjectionCache::default());
cache.observe(projection).unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let base_url = format!("http://{}", listener.local_addr().unwrap());
let server = std::thread::spawn(move || {
@@ -1151,10 +1128,10 @@ mod tests {
RuntimeOwnedWorkspaceClient::new("workspace-a", base_url, "runtime-a", "worker-a")
.with_prompt_projection_cache(cache);
let error = client.current_prompt_projection(Some(4)).unwrap_err();
let fetched = client.current_prompt_projection().unwrap().unwrap();
server.join().unwrap();
assert!(error.to_string().contains("required revision 4, got 3"));
assert_eq!(fetched.projection.source_digest, "source-3");
}
#[test]
@@ -1168,7 +1145,6 @@ mod tests {
"default".to_string(),
"shared prompt".to_string(),
)]),
5,
"schema",
"toolchain",
)
@@ -1211,7 +1187,7 @@ mod tests {
let barrier = barrier.clone();
std::thread::spawn(move || {
barrier.wait();
client.current_prompt_projection(None).unwrap().unwrap()
client.current_prompt_projection().unwrap().unwrap()
})
})
.collect::<Vec<_>>();
@@ -1239,7 +1215,6 @@ mod tests {
"default".to_string(),
"foreign prompt".to_string(),
)]),
4,
"schema",
"toolchain",
)
@@ -1270,7 +1245,7 @@ mod tests {
RuntimeOwnedWorkspaceClient::new("workspace-a", base_url, "runtime-a", "worker-a")
.with_prompt_projection_cache(Arc::new(WorkspacePromptProjectionCache::default()));
let error = client.current_prompt_projection(None).unwrap_err();
let error = client.current_prompt_projection().unwrap_err();
server.join().unwrap();
assert!(error.to_string().contains("scope mismatch"));
+257 -101
View File
@@ -40,6 +40,7 @@ const REPOSITORY_MAX_BYTES: u64 = 5 * 1024 * 1024 * 1024;
static NEXT_WORKING_DIRECTORY_SEQUENCE: AtomicU64 = AtomicU64::new(0);
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(try_from = "FrozenEvidenceRead")]
pub struct WorkingDirectoryEvidence {
pub repository_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -48,16 +49,61 @@ pub struct WorkingDirectoryEvidence {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub resolved_tree: Option<String>,
pub materializer_kind: MaterializerKind,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub repository_source_revision: Option<u64>,
/// SSH authority must be reacquired before opening a session, even if an old
/// record has no concrete key fingerprint. Never inferred as false from that omission.
pub repository_access_required: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub repository_source_fingerprint: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub operation_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub credential_revision: Option<u64>,
pub public_key_fingerprint: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub host_trust_revision: Option<u64>,
pub host_key_fingerprint: Option<String>,
}
// Frozen persisted evidence reader. Legacy counters only mark that SSH
// access is required; they are never converted to key identity or grants.
#[derive(Deserialize)]
struct FrozenEvidenceRead {
repository_id: String,
requested_selector: Option<String>,
resolved_commit: String,
resolved_tree: Option<String>,
materializer_kind: MaterializerKind,
repository_source_fingerprint: Option<String>,
operation_id: Option<String>,
public_key_fingerprint: Option<String>,
host_key_fingerprint: Option<String>,
repository_access_required: Option<bool>,
credential_revision: Option<u64>,
host_trust_revision: Option<u64>,
}
impl TryFrom<FrozenEvidenceRead> for WorkingDirectoryEvidence {
type Error = &'static str;
fn try_from(wire: FrozenEvidenceRead) -> Result<Self, Self::Error> {
if wire.credential_revision == Some(0) || wire.host_trust_revision == Some(0) {
return Err("legacy SSH evidence contains an invalid authority marker");
}
let repository_access_required = wire.repository_access_required.unwrap_or(false)
|| wire.credential_revision.is_some()
|| wire.host_trust_revision.is_some()
|| wire.public_key_fingerprint.is_some()
|| wire.host_key_fingerprint.is_some();
Ok(Self {
repository_id: wire.repository_id,
requested_selector: wire.requested_selector,
resolved_commit: wire.resolved_commit,
resolved_tree: wire.resolved_tree,
materializer_kind: wire.materializer_kind,
repository_access_required,
repository_source_fingerprint: wire.repository_source_fingerprint,
operation_id: wire.operation_id,
public_key_fingerprint: wire.public_key_fingerprint,
host_key_fingerprint: wire.host_key_fingerprint,
})
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -156,7 +202,7 @@ impl WorkingDirectoryBinding {
WorkingDirectoryStatusKind::Active | WorkingDirectoryStatusKind::CleanupPending
) && binding_paths_are_available(self)
{
let (current_selector, current_ref, current_tree) = binding_current_revision(self);
let (current_selector, current_ref, current_tree) = binding_current_commit(self);
summary.current_selector = current_selector;
summary.current_ref = current_ref;
summary.current_tree = current_tree;
@@ -274,7 +320,7 @@ fn binding_paths_are_available(binding: &WorkingDirectoryBinding) -> bool {
root.join(".git").is_dir()
}
fn binding_current_revision(
fn binding_current_commit(
binding: &WorkingDirectoryBinding,
) -> (Option<String>, Option<String>, Option<String>) {
let current_ref = git_stdout(binding.root(), ["rev-parse", "HEAD"])
@@ -305,13 +351,12 @@ fn binding_cleanliness(binding: &WorkingDirectoryBinding) -> String {
#[derive(Debug)]
struct PendingRepositoryAccessLease {
generation: u64,
lease_id: uuid::Uuid,
access: RepositorySshMaterializationAccess,
}
#[derive(Debug, Default)]
struct RepositoryAccessExpiryState {
next_generation: u64,
pending: HashMap<String, PendingRepositoryAccessLease>,
shutdown: bool,
}
@@ -360,23 +405,11 @@ impl RepositoryAccessExpiryScheduler {
}
}
fn next_generation(
state: &mut RepositoryAccessExpiryState,
) -> Result<u64, WorkingDirectoryDiagnostic> {
state.next_generation = state.next_generation.checked_add(1).ok_or_else(|| {
WorkingDirectoryDiagnostic::new(
"working_directory_repository_access_generation_exhausted",
"Runtime Repository access generation is exhausted",
)
})?;
Ok(state.next_generation)
}
fn store_pending(
&self,
working_directory_id: &str,
access: RepositorySshMaterializationAccess,
) -> Result<u64, WorkingDirectoryDiagnostic> {
) -> Result<uuid::Uuid, WorkingDirectoryDiagnostic> {
let mut state = self.inner.state.lock().map_err(|_| {
WorkingDirectoryDiagnostic::new(
"working_directory_repository_access_unavailable",
@@ -389,14 +422,14 @@ impl RepositoryAccessExpiryScheduler {
"Runtime Repository access state is shutting down",
));
}
let generation = Self::next_generation(&mut state)?;
let lease_id = uuid::Uuid::now_v7();
state.pending.insert(
working_directory_id.to_string(),
PendingRepositoryAccessLease { generation, access },
PendingRepositoryAccessLease { lease_id, access },
);
drop(state);
self.inner.wake.notify_one();
Ok(generation)
Ok(lease_id)
}
fn pending(
@@ -452,14 +485,14 @@ impl RepositoryAccessExpiryScheduler {
Ok(())
}
fn next_expiry(state: &RepositoryAccessExpiryState) -> Option<(String, u64, u64)> {
fn next_expiry(state: &RepositoryAccessExpiryState) -> Option<(String, uuid::Uuid, u64)> {
state
.pending
.iter()
.map(|(working_directory_id, lease)| {
(
working_directory_id.clone(),
lease.generation,
lease.lease_id,
lease.access.expires_at_epoch_seconds,
)
})
@@ -469,14 +502,14 @@ impl RepositoryAccessExpiryScheduler {
fn expire_if_current(
state: &mut RepositoryAccessExpiryState,
working_directory_id: &str,
generation: u64,
lease_id: uuid::Uuid,
now_epoch_seconds: u64,
) {
let should_remove = state
.pending
.get(working_directory_id)
.is_some_and(|lease| {
lease.generation == generation
lease.lease_id == lease_id
&& lease.access.expires_at_epoch_seconds <= now_epoch_seconds
});
if should_remove {
@@ -494,7 +527,7 @@ impl RepositoryAccessExpiryScheduler {
if state.shutdown {
return;
}
let Some((working_directory_id, generation, expires_at_epoch_seconds)) =
let Some((working_directory_id, lease_id, expires_at_epoch_seconds)) =
Self::next_expiry(&state)
else {
state = match inner.wake.wait(state) {
@@ -518,7 +551,7 @@ impl RepositoryAccessExpiryScheduler {
Self::expire_if_current(
&mut state,
&working_directory_id,
generation,
lease_id,
now_epoch_seconds,
);
break;
@@ -527,21 +560,21 @@ impl RepositoryAccessExpiryScheduler {
}
#[cfg(test)]
fn pending_generation(&self, working_directory_id: &str) -> Option<u64> {
fn pending_lease_id(&self, working_directory_id: &str) -> Option<uuid::Uuid> {
self.inner
.state
.lock()
.ok()?
.pending
.get(working_directory_id)
.map(|lease| lease.generation)
.map(|lease| lease.lease_id)
}
#[cfg(test)]
fn expire_pending_for_test(
&self,
working_directory_id: &str,
generation: u64,
lease_id: uuid::Uuid,
now_epoch_seconds: u64,
) {
let Ok(mut state) = self.inner.state.lock() else {
@@ -550,7 +583,7 @@ impl RepositoryAccessExpiryScheduler {
Self::expire_if_current(
&mut state,
working_directory_id,
generation,
lease_id,
now_epoch_seconds,
);
}
@@ -701,6 +734,26 @@ impl RuntimeGitMaterializer {
fn read_binding(
&self,
working_directory_id: &str,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
// Frozen evidence normalization can write the record. Keep it inside
// use admission so observational reads cannot race physical cleanup.
let _use_lease = self.occupancy.acquire_use(working_directory_id)?;
self.read_binding_record(working_directory_id, true)
}
fn read_binding_for_cleanup(
&self,
working_directory_id: &str,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
// Cleanup already owns exclusive admission and validates the original
// bytes. Its witnessed status write performs any canonicalization.
self.read_binding_record(working_directory_id, false)
}
fn read_binding_record(
&self,
working_directory_id: &str,
migrate: bool,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
let working_directory_root = self.working_directory_root(working_directory_id);
let path = working_directory_root.join(MATERIALIZATION_RECORD);
@@ -716,7 +769,7 @@ impl RuntimeGitMaterializer {
"working directory working_directory record is invalid; backend-private path details were omitted",
)
})?;
Ok(WorkingDirectoryBinding {
let binding = WorkingDirectoryBinding {
working_directory: record.working_directory,
root: record.root.clone(),
cwd: record.root,
@@ -726,7 +779,39 @@ impl RuntimeGitMaterializer {
occupancy: self.occupancy.clone(),
occupancy_id: working_directory_id.to_string(),
generation: self.occupancy.generation(&working_directory_id),
};
let canonical = serde_json::to_vec_pretty(&WorkingDirectoryMaterializationRecord {
working_directory: binding.working_directory.clone(),
root: binding.root.clone(),
})
.map_err(|_| {
WorkingDirectoryDiagnostic::new(
"working_directory_record_invalid",
"materialization evidence is invalid",
)
})?;
if migrate && canonical != raw {
let temporary = path.with_extension(format!("migration-{}.tmp", uuid::Uuid::now_v7()));
let result = (|| -> std::io::Result<()> {
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&temporary)?;
file.write_all(&canonical)?;
file.sync_all()?;
fs::rename(&temporary, &path)?;
fs::File::open(&binding.working_directory_root)?.sync_all()?;
Ok(())
})();
if result.is_err() {
let _ = fs::remove_file(&temporary);
return Err(WorkingDirectoryDiagnostic::new(
"working_directory_record_write_failed",
"materialization evidence migration could not be committed",
));
}
}
Ok(binding)
}
fn store_repository_access(
@@ -750,8 +835,7 @@ impl RuntimeGitMaterializer {
if binding
.working_directory
.evidence
.credential_revision
.is_some()
.repository_access_required
{
return Err(WorkingDirectoryDiagnostic::new(
"working_directory_remote_repository_access_required",
@@ -832,7 +916,6 @@ impl RuntimeGitMaterializer {
if materialization.workspace_id.trim().is_empty()
|| materialization.runtime_id.trim().is_empty()
|| materialization.operation_id.trim().is_empty()
|| materialization.config_revision == 0
|| materialization.config_projection_digest.trim().is_empty()
{
return Err(WorkingDirectoryDiagnostic::new(
@@ -1032,16 +1115,17 @@ impl RuntimeGitMaterializer {
resolved_commit,
resolved_tree,
materializer_kind: MaterializerKind::RuntimeGitClone,
repository_source_revision: Some(request.repository.source_revision),
repository_access_required: request.repository.source.kind
== server_api::RepositorySourceKind::Ssh,
repository_source_fingerprint: Some(request.repository.source_fingerprint.clone()),
operation_id: context.map(|value| value.operation_id.clone()),
credential_revision: context
public_key_fingerprint: context
.and_then(|value| value.ssh.as_ref())
.and_then(|value| value.credential_candidates.first())
.map(|candidate| candidate.credential_revision),
host_trust_revision: context
.map(|candidate| candidate.public_key_fingerprint.clone()),
host_key_fingerprint: context
.and_then(|value| value.ssh.as_ref())
.map(|value| value.host_trust_revision),
.map(|value| value.host_key_fingerprint.clone()),
},
cleanup_target: WorkingDirectoryCleanupTarget {
kind: "runtime_git_clone".to_string(),
@@ -1122,11 +1206,10 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
}
Err(error) => return Err(error),
};
if binding
if !binding
.working_directory
.evidence
.credential_revision
.is_none()
.repository_access_required
{
return Err(WorkingDirectoryDiagnostic::new(
"working_directory_repository_access_not_applicable",
@@ -1146,13 +1229,15 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
)?;
binding.working_directory.evidence.operation_id =
Some(request.materialization.operation_id.clone());
binding.working_directory.evidence.credential_revision = Some(
binding.working_directory.evidence.public_key_fingerprint = Some(
ssh.credential_candidates
.first()
.expect("validated SSH credential candidate")
.credential_revision,
.public_key_fingerprint
.clone(),
);
binding.working_directory.evidence.host_trust_revision = Some(ssh.host_trust_revision);
binding.working_directory.evidence.host_key_fingerprint =
Some(ssh.host_key_fingerprint.clone());
self.write_record(&binding)?;
self.store_repository_access(&request.working_directory_id, ssh)
}
@@ -1195,15 +1280,15 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
"Repository provider returned an ambiguous ref observation",
));
}
let (revision_ref, observed_selector) = line.split_once('\t').ok_or_else(|| {
let (resolved_ref, observed_selector) = line.split_once('\t').ok_or_else(|| {
WorkingDirectoryDiagnostic::new(
"repository_ref_response_invalid",
"Repository provider returned an invalid ref observation",
)
})?;
if observed_selector != selector
|| !matches!(revision_ref.len(), 40 | 64)
|| !revision_ref.bytes().all(|byte| byte.is_ascii_hexdigit())
|| !matches!(resolved_ref.len(), 40 | 64)
|| !resolved_ref.bytes().all(|byte| byte.is_ascii_hexdigit())
{
return Err(WorkingDirectoryDiagnostic::new(
"repository_ref_response_invalid",
@@ -1213,10 +1298,9 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
Ok(RepositoryRefObservation {
repository_id: request.repository.id.clone(),
source_revision: request.repository.source_revision,
source_fingerprint: request.repository.source_fingerprint.clone(),
selector: selector.to_string(),
revision_ref: revision_ref.to_ascii_lowercase(),
resolved_ref: resolved_ref.to_ascii_lowercase(),
observed_at_epoch_seconds: SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
@@ -2294,11 +2378,11 @@ fn validate_ssh_materialization_access(
if access.credential_candidates.is_empty()
|| access.credential_candidates.iter().any(|candidate| {
candidate.credential_id.trim().is_empty()
|| candidate.credential_revision == 0
|| candidate.public_key_fingerprint.trim().is_empty()
|| !candidate.private_key.expose().contains("PRIVATE KEY")
})
|| access.host_trust_id.trim().is_empty()
|| access.host_trust_revision == 0
|| access.host_key_fingerprint.trim().is_empty()
|| access.known_hosts_entry.expose().trim().is_empty()
{
return Err(WorkingDirectoryDiagnostic::new(
@@ -3013,8 +3097,6 @@ mod tests {
operation: crate::resource::BackendResourceOperation::FetchOnce,
expires_at_unix_seconds: i64::MAX,
nonce: "repository-access-1".to_string(),
revision: "1".to_string(),
generation: None,
max_bytes: crate::resource::DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES,
content_type: crate::resource::REPOSITORY_SSH_ACCESS_CONTENT_TYPE.to_string(),
redaction: crate::resource::ResourceRedactionPolicy::RuntimeInternalOnly,
@@ -3027,11 +3109,11 @@ mod tests {
crate::catalog::RepositorySshMaterializationAccess {
credential_candidates: vec![crate::catalog::RepositorySshCredentialCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 2,
public_key_fingerprint: "key-2".into(),
private_key: crate::catalog::SensitiveString::new("PRIVATE KEY secret bytes"),
}],
host_trust_id: "trust-1".to_string(),
host_trust_revision: 4,
host_key_fingerprint: "key-4".into(),
access: server_api::RepositoryAccessMode::ReadOnly,
expires_at_epoch_seconds: u64::MAX,
repository_id: "repo-main".to_string(),
@@ -3043,7 +3125,7 @@ mod tests {
}
#[test]
fn repository_access_expiry_fences_a_stale_pending_generation_after_refresh() {
fn repository_access_expiry_fences_a_stale_pending_lease_id_after_refresh() {
let scheduler = RepositoryAccessExpiryScheduler::new();
let working_directory_id = "working-directory-refresh";
let old_expiry = repository_access_now_epoch_seconds() + 60;
@@ -3052,9 +3134,9 @@ mod tests {
scheduler
.store_pending(working_directory_id, old_access)
.unwrap();
let old_generation = scheduler
.pending_generation(working_directory_id)
.expect("old pending generation");
let old_lease_id = scheduler
.pending_lease_id(working_directory_id)
.expect("old pending lease_id");
let barrier = Arc::new(std::sync::Barrier::new(2));
let refresh_barrier = Arc::clone(&barrier);
@@ -3069,12 +3151,12 @@ mod tests {
});
barrier.wait();
refresh.join().unwrap();
let refreshed_generation = scheduler
.pending_generation(working_directory_id)
.expect("refreshed pending generation");
assert_ne!(old_generation, refreshed_generation);
let refreshed_lease_id = scheduler
.pending_lease_id(working_directory_id)
.expect("refreshed pending lease_id");
assert_ne!(old_lease_id, refreshed_lease_id);
scheduler.expire_pending_for_test(working_directory_id, old_generation, old_expiry);
scheduler.expire_pending_for_test(working_directory_id, old_lease_id, old_expiry);
let refreshed = scheduler
.pending(working_directory_id)
.unwrap()
@@ -3175,7 +3257,6 @@ mod tests {
kind: server_api::RepositorySourceKind::LocalPath,
uri: repo.display().to_string(),
},
source_revision: 1,
source_fingerprint: "sha256:test".to_string(),
selector: Some(RepositorySelector::from("HEAD")),
},
@@ -3190,6 +3271,48 @@ mod tests {
WorkerRef::new(WorkerId::from_legacy_u64(sequence))
}
#[test]
fn legacy_ssh_evidence_requires_fresh_access_and_is_rewritten_without_counters() {
let repository = create_clean_repo();
let runtime = tempfile::tempdir().unwrap();
let materializer = RuntimeGitMaterializer::new(runtime.path());
let binding = materializer.create(&request(repository.path())).unwrap();
let id = binding.working_directory.id.clone();
let record_path = binding.working_directory_root.join(MATERIALIZATION_RECORD);
let mut record: serde_json::Value =
serde_json::from_slice(&fs::read(&record_path).unwrap()).unwrap();
let evidence = record["working_directory"]["evidence"]
.as_object_mut()
.unwrap();
evidence.remove("repository_access_required");
evidence.insert("credential_revision".into(), serde_json::json!(3));
evidence.insert("host_trust_revision".into(), serde_json::json!(2));
fs::write(&record_path, serde_json::to_vec(&record).unwrap()).unwrap();
drop(binding);
assert_eq!(
materializer
.bind_working_directory(&id, None)
.unwrap_err()
.code,
"working_directory_remote_repository_access_required"
);
let migrated: serde_json::Value =
serde_json::from_slice(&fs::read(&record_path).unwrap()).unwrap();
let evidence = &migrated["working_directory"]["evidence"];
assert_eq!(evidence["repository_access_required"], true);
assert!(evidence.get("credential_revision").is_none());
assert!(evidence.get("host_trust_revision").is_none());
assert!(evidence.get("public_key_fingerprint").is_none());
drop(materializer);
assert_eq!(
RuntimeGitMaterializer::new(runtime.path())
.bind_working_directory(&id, None)
.unwrap_err()
.code,
"working_directory_remote_repository_access_required"
);
}
#[test]
fn repository_ref_observation_reads_the_provider_fresh() {
let repo = create_clean_repo();
@@ -3207,7 +3330,7 @@ mod tests {
.observe_repository_ref(&observation_request)
.unwrap();
assert_eq!(
first.revision_ref,
first.resolved_ref,
git_stdout(repo.path(), ["rev-parse", "published"]).unwrap()
);
fs::write(repo.path().join("second.txt"), "second\n").unwrap();
@@ -3218,9 +3341,9 @@ mod tests {
let second = materializer
.observe_repository_ref(&observation_request)
.unwrap();
assert_ne!(first.revision_ref, second.revision_ref);
assert_ne!(first.resolved_ref, second.resolved_ref);
assert_eq!(
second.revision_ref,
second.resolved_ref,
git_stdout(repo.path(), ["rev-parse", "published"]).unwrap()
);
}
@@ -3290,8 +3413,8 @@ mod tests {
let still_first = materializer
.observe_repository_ref(&observation_request)
.unwrap();
assert_eq!(still_first.revision_ref, first.revision_ref);
assert_ne!(still_first.revision_ref, unpublished_second);
assert_eq!(still_first.resolved_ref, first.resolved_ref);
assert_ne!(still_first.resolved_ref, unpublished_second);
git(
&workdir,
@@ -3300,11 +3423,11 @@ mod tests {
let second = materializer
.observe_repository_ref(&observation_request)
.unwrap();
assert_eq!(second.revision_ref, unpublished_second);
assert_ne!(second.revision_ref, first.revision_ref);
assert_eq!(second.resolved_ref, unpublished_second);
assert_ne!(second.resolved_ref, first.resolved_ref);
assert_ne!(
git_stdout(&cache, ["rev-parse", "HEAD"]).unwrap(),
second.revision_ref
second.resolved_ref
);
}
@@ -3458,7 +3581,6 @@ mod tests {
let mut file_request = request(repo.path());
file_request.repository.source.kind = server_api::RepositorySourceKind::File;
file_request.repository.source.uri = format!("file://{}", repo.path().display());
file_request.repository.source_revision = 2;
file_request.repository.source_fingerprint = "sha256:file-source".to_string();
let file = materializer
.materialize(&worker_ref(3), &file_request)
@@ -3478,7 +3600,6 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-1".to_string(),
config_revision: 7,
config_projection_digest: "sha256:projection".to_string(),
ssh: None,
});
@@ -3511,8 +3632,8 @@ mod tests {
"credential-1"
);
assert_eq!(
serialized["credential_candidates"][0]["credential_revision"],
2
serialized["credential_candidates"][0]["public_key_fingerprint"],
"key-2"
);
assert!(
serialized["credential_candidates"][0]
@@ -3564,27 +3685,26 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-agent".to_string(),
config_revision: 2,
config_projection_digest: "sha256:projection".to_string(),
ssh: Some(crate::catalog::RepositorySshMaterializationAccess {
credential_candidates: vec![
crate::catalog::RepositorySshCredentialCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 1,
public_key_fingerprint: "key-1".into(),
private_key: crate::catalog::SensitiveString::new(
fs::read_to_string(&key_path).unwrap(),
),
},
crate::catalog::RepositorySshCredentialCandidate {
credential_id: "credential-2".to_string(),
credential_revision: 3,
public_key_fingerprint: "key-3".into(),
private_key: crate::catalog::SensitiveString::new(
fs::read_to_string(&fallback_key_path).unwrap(),
),
},
],
host_trust_id: "trust-1".to_string(),
host_trust_revision: 1,
host_key_fingerprint: "key-1".into(),
access: server_api::RepositoryAccessMode::ReadWrite,
expires_at_epoch_seconds: u64::MAX,
repository_id: "repo-main".to_string(),
@@ -3720,11 +3840,11 @@ mod tests {
ssh_backed_binding
.working_directory
.evidence
.credential_revision = Some(1);
.public_key_fingerprint = Some("key-1".into());
ssh_backed_binding
.working_directory
.evidence
.host_trust_revision = Some(1);
.host_key_fingerprint = Some("key-1".into());
materializer.write_record(&ssh_backed_binding).unwrap();
#[cfg(target_os = "linux")]
fail_workdir_cleanup_once(&materializer, &id);
@@ -3809,7 +3929,8 @@ mod tests {
);
let mut rotated = initial_materialization.clone();
rotated.operation_id = "operation-agent-rotated".to_string();
rotated.ssh.as_mut().unwrap().credential_candidates[0].credential_revision = 2;
rotated.ssh.as_mut().unwrap().credential_candidates[0].public_key_fingerprint =
"key-2".into();
rotated.ssh.as_mut().unwrap().access = server_api::RepositoryAccessMode::ReadOnly;
materializer
.authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest {
@@ -3819,8 +3940,8 @@ mod tests {
.unwrap();
let rebound = materializer.bind_working_directory(&id, None).unwrap();
assert_eq!(
rebound.working_directory.evidence.credential_revision,
Some(2)
rebound.working_directory.evidence.public_key_fingerprint,
Some("key-2".into())
);
let rebound_environment = rebound.command_environment();
assert_eq!(rebound_environment["YOI_REPOSITORY_ACCESS"], "read_only");
@@ -3927,7 +4048,8 @@ mod tests {
let mut read_write = rotated;
read_write.operation_id = "operation-agent-read-write".to_string();
read_write.ssh.as_mut().unwrap().credential_candidates[0].credential_revision = 3;
read_write.ssh.as_mut().unwrap().credential_candidates[0].public_key_fingerprint =
"key-3".into();
read_write.ssh.as_mut().unwrap().access = server_api::RepositoryAccessMode::ReadWrite;
let read_write_command_policy =
RepositorySshCommandPolicy::from_access(read_write.ssh.as_ref().unwrap()).unwrap();
@@ -4006,16 +4128,15 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-read-only".to_string(),
config_revision: 2,
config_projection_digest: "sha256:projection".to_string(),
ssh: Some(crate::catalog::RepositorySshMaterializationAccess {
credential_candidates: vec![crate::catalog::RepositorySshCredentialCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 1,
public_key_fingerprint: "key-1".into(),
private_key: crate::catalog::SensitiveString::new("PRIVATE KEY placeholder"),
}],
host_trust_id: "trust-1".to_string(),
host_trust_revision: 1,
host_key_fingerprint: "key-1".into(),
access: server_api::RepositoryAccessMode::ReadOnly,
expires_at_epoch_seconds: u64::MAX,
repository_id: "repo-main".to_string(),
@@ -4176,7 +4297,6 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-1".to_string(),
config_revision: 1,
config_projection_digest: "sha256:projection".to_string(),
ssh: Some(access),
});
@@ -4205,7 +4325,6 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-1".to_string(),
config_revision: 1,
config_projection_digest: "sha256:projection".to_string(),
ssh,
};
@@ -4283,11 +4402,11 @@ mod tests {
crate::catalog::RepositorySshMaterializationAccess {
credential_candidates: vec![crate::catalog::RepositorySshCredentialCandidate {
credential_id: "credential-1".to_string(),
credential_revision: 1,
public_key_fingerprint: "key-1".into(),
private_key: crate::catalog::SensitiveString::new("PRIVATE KEY placeholder"),
}],
host_trust_id: "trust-1".to_string(),
host_trust_revision: 1,
host_key_fingerprint: "key-1".into(),
access: server_api::RepositoryAccessMode::ReadOnly,
expires_at_epoch_seconds: u64::MAX,
repository_id: "repo-main".to_string(),
@@ -4321,7 +4440,6 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-1".to_string(),
config_revision: 1,
config_projection_digest: "sha256:projection".to_string(),
ssh: None,
});
@@ -4498,7 +4616,6 @@ mod tests {
workspace_id: "workspace-1".to_string(),
runtime_id: "runtime-1".to_string(),
operation_id: "operation-1".to_string(),
config_revision: 7,
config_projection_digest: "sha256:projection".to_string(),
ssh,
}
@@ -4641,6 +4758,45 @@ mod tests {
.unwrap();
}
#[cfg(target_os = "linux")]
#[test]
fn legacy_evidence_normalization_is_excluded_by_cleanup_and_cleanup_reads_do_not_rewrite() {
let repo = create_clean_repo();
let runtime_root = tempfile::tempdir().unwrap();
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
let binding = materializer.create(&request(repo.path())).unwrap();
let id = &binding.working_directory.id;
let path = binding
.working_directory_root()
.join(MATERIALIZATION_RECORD);
let mut legacy: serde_json::Value =
serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
legacy["working_directory"]["evidence"]["repository_source_revision"] =
serde_json::json!(7);
let original = serde_json::to_vec_pretty(&legacy).unwrap();
fs::write(&path, &original).unwrap();
let cleanup = materializer.occupancy.acquire_cleanup(id).unwrap();
assert_eq!(
materializer.read_binding(id).unwrap_err().code,
"working_directory_cleanup_resource_busy"
);
materializer.read_binding_for_cleanup(id).unwrap();
assert_eq!(fs::read(&path).unwrap(), original);
drop(cleanup);
materializer.read_binding(id).unwrap();
let canonical: serde_json::Value =
serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
assert!(
canonical["working_directory"]["evidence"]
.get("repository_source_revision")
.is_none()
);
materializer.cleanup_working_directory(id).unwrap();
assert!(!binding.working_directory_root().exists());
}
#[cfg(target_os = "linux")]
#[test]
fn removed_binding_cannot_activate_against_recreated_same_id() {
@@ -190,7 +190,7 @@ fn remove_with_checks(
}
} else {
let mut binding = materializer
.read_binding(id)
.read_binding_for_cleanup(id)
.map_err(|_| cause("ownership_unknown"))?;
validate_binding(materializer, id, &binding)?;
// Preflight the entire tree without following links or crossing mounts,
@@ -278,8 +278,8 @@ fn same_materialization(saved: &WorkingDirectory, current: &WorkingDirectory) ->
// These fields are refreshed by authorize_repository_access, not a new
// materialization. All repository, creation and cleanup identity stays.
directory.evidence.operation_id = None;
directory.evidence.credential_revision = None;
directory.evidence.host_trust_revision = None;
directory.evidence.public_key_fingerprint = None;
directory.evidence.host_key_fingerprint = None;
directory
};
immutable(saved) == immutable(current)
@@ -298,7 +298,7 @@ fn retry_inventory(
return Ok(before);
}
let binding = materializer
.read_binding(id)
.read_binding_for_cleanup(id)
.map_err(|_| cause("changes_present"))?;
validate_binding(materializer, id, &binding)?;
if !same_materialization(&saved.working_directory, &binding.working_directory)
@@ -1211,6 +1211,13 @@ pub(super) mod tests {
for update in ["access", "repository", "creation"] {
let (_runtime, materializer, binding) = fixture();
let id = &binding.working_directory.id;
// Model an existing SSH materialization before taking the witness.
// Refresh replaces its access proof, not its requirement for SSH authority.
let mut record = materializer.read_binding(id).unwrap();
record.working_directory.evidence.repository_access_required = true;
record.working_directory.evidence.public_key_fingerprint = Some("key-1".into());
record.working_directory.evidence.host_key_fingerprint = Some("host-key-1".into());
materializer.write_record(&record).unwrap();
fail_once(&materializer, id);
let mut record = materializer.read_binding(id).unwrap();
match update {
@@ -1218,8 +1225,8 @@ pub(super) mod tests {
// The same serialized fields changed by authorize_repository_access.
record.working_directory.evidence.operation_id =
Some("reattach-operation".into());
record.working_directory.evidence.credential_revision = Some(2);
record.working_directory.evidence.host_trust_revision = Some(2);
record.working_directory.evidence.public_key_fingerprint = Some("key-2".into());
record.working_directory.evidence.host_key_fingerprint = Some("key-2".into());
}
"repository" => {
record.working_directory.repository_id = "substitute".into();
File diff suppressed because it is too large Load Diff