From 3a3c89e0b4441d12533fe052f49ff2ed3c243325 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 13:54:25 +0900 Subject: [PATCH 01/12] feat: materialize repositories through runtime Git cache --- Cargo.lock | 2 + Cargo.toml | 1 + crates/workdir/src/lib.rs | 4 +- crates/workdir/src/local.rs | 71 +- crates/workdir/src/workspace.rs | 2 + crates/worker-runtime/Cargo.toml | 1 + crates/worker-runtime/src/catalog.rs | 52 + crates/worker-runtime/src/http_server.rs | 10 + crates/worker-runtime/src/main.rs | 4 +- crates/worker-runtime/src/worker_backend.rs | 41 +- .../worker-runtime/src/working_directory.rs | 1245 +++++++++++++++-- crates/workspace-server/Cargo.toml | 1 + .../workspace-server/src/repository_access.rs | 201 ++- crates/workspace-server/src/server.rs | 86 +- .../src/workdir_create_operations.rs | 21 +- 15 files changed, 1602 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2e4fbc7..e4f7f2bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6652,6 +6652,7 @@ dependencies = [ "workdir", "worker", "workspace-api", + "zeroize", ] [[package]] @@ -6798,6 +6799,7 @@ dependencies = [ "worker", "worker-runtime", "workspace-api", + "zeroize", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 07f33556..f6e03a24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,4 +125,5 @@ toml = "1.1" tracing = "0.1" url = "2.5" uuid = "1.23" +zeroize = "1" webauthn-rs = { version = "0.5.2", features = ["danger-allow-state-serialisation", "danger-credential-internals"] } diff --git a/crates/workdir/src/lib.rs b/crates/workdir/src/lib.rs index 0c226c57..22fb57de 100644 --- a/crates/workdir/src/lib.rs +++ b/crates/workdir/src/lib.rs @@ -28,7 +28,9 @@ pub use fs_operation::{ GlobResult, GrepOutputMode, GrepRequest, GrepResult, ListEntry, ListRequest, ListResult, ReadRequest, ReadResult, StatRequest, StatResult, WriteRequest, WriteResult, }; -pub use local::{LocalWorkdirSession, SymlinkInfo, direct_symlink, first_symlink}; +pub use local::{ + LocalWorkdirSession, SymlinkInfo, WorkdirSessionResource, direct_symlink, first_symlink, +}; pub use operation::*; /// Persistent, opaque identity of one materialized Workdir. diff --git a/crates/workdir/src/local.rs b/crates/workdir/src/local.rs index 2cfa0f09..16686cb4 100644 --- a/crates/workdir/src/local.rs +++ b/crates/workdir/src/local.rs @@ -8,7 +8,8 @@ //! `LocalWorkdirSession` is cheap to clone (`Arc` inside). Tool-specific session //! state, such as read-before-edit tracking, remains owned by the tool layer. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; +use std::fmt::Debug; #[cfg(test)] use std::io::Write as _; use std::io::{Read as _, Seek as _, SeekFrom}; @@ -228,6 +229,8 @@ struct LocalWorkdirSessionInner { next_command_id: AtomicU64, commands: Mutex>, command_telemetry: CommandTelemetry, + command_environment: BTreeMap, + resources: StdMutex>>, } impl Drop for LocalWorkdirSessionInner { @@ -242,6 +245,9 @@ impl Drop for LocalWorkdirSessionInner { } } +pub trait WorkdirSessionResource: Debug + Send + Sync {} +impl WorkdirSessionResource for T where T: Debug + Send + Sync {} + /// Scope-aware filesystem handle. Clone-cheap (`Arc` inside). /// /// The wrapped [`SharedScope`] is shared with every clone of this @@ -318,6 +324,26 @@ impl LocalWorkdirSession { cwd: PathBuf, scope: SharedScope, capabilities: WorkdirSessionCapabilities, + ) -> Self { + Self::materialized_bound_with_environment( + workdir, + root, + cwd, + scope, + capabilities, + BTreeMap::new(), + Vec::new(), + ) + } + + pub fn materialized_bound_with_environment( + workdir: Workdir, + root: PathBuf, + cwd: PathBuf, + scope: SharedScope, + capabilities: WorkdirSessionCapabilities, + command_environment: BTreeMap, + resources: Vec>, ) -> Self { Self { inner: Arc::new(LocalWorkdirSessionInner { @@ -331,6 +357,8 @@ impl LocalWorkdirSession { next_command_id: AtomicU64::new(1), commands: Mutex::new(HashMap::new()), command_telemetry: CommandTelemetry::new(), + command_environment, + resources: StdMutex::new(resources), }), } } @@ -669,9 +697,18 @@ impl WorkdirSession for LocalWorkdirSession { let (completion_tx, completion) = watch::channel(false); let command_id = handle.0.clone(); let telemetry = self.inner.command_telemetry.clone(); + let command_environment = self.inner.command_environment.clone(); let (cancel, cancel_rx) = watch::channel(false); let task = tokio::spawn(async move { - let output = run_command(cwd, request, command_id, telemetry, cancel_rx).await; + let output = run_command( + cwd, + request, + command_id, + telemetry, + command_environment, + cancel_rx, + ) + .await; let _ = completion_tx.send(true); output }); @@ -840,6 +877,9 @@ impl WorkdirSession for LocalWorkdirSession { LocalCommand::Completed(_) => {} } } + if let Ok(mut resources) = self.inner.resources.lock() { + resources.clear(); + } Ok(()) } } @@ -909,6 +949,7 @@ async fn run_command( request: CommandRequest, command_id: String, telemetry: CommandTelemetry, + command_environment: BTreeMap, mut cancel: watch::Receiver, ) -> Result { let stdout = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?; @@ -925,6 +966,7 @@ async fn run_command( .arg("-c") .arg(&request.command) .current_dir(&cwd) + .envs(command_environment) .stdin(Stdio::null()) .stdout(Stdio::from(stdout_file)) .stderr(Stdio::from(stderr_file)) @@ -2319,6 +2361,31 @@ mod tests { assert_eq!(terminal, Some((handle.0, CommandStatus::TimedOut, None))); } + #[tokio::test] + async fn closing_session_releases_runtime_resources() { + #[derive(Debug)] + struct Resource(Arc); + impl Drop for Resource { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } + + let dir = TempDir::new().unwrap(); + let released = Arc::new(AtomicBool::new(false)); + let session = LocalWorkdirSession::materialized_bound_with_environment( + Workdir::new("resource-session"), + dir.path().to_path_buf(), + dir.path().to_path_buf(), + SharedScope::new(Scope::writable(dir.path()).unwrap()), + WorkdirSessionCapabilities::ALL, + BTreeMap::from([("SSH_AUTH_SOCK".to_string(), "test-socket".to_string())]), + vec![Arc::new(Resource(released.clone()))], + ); + WorkdirSession::close(&session).await.unwrap(); + assert!(released.load(Ordering::Acquire)); + } + #[tokio::test] async fn provider_cancels_active_command() { let dir = TempDir::new().unwrap(); diff --git a/crates/workdir/src/workspace.rs b/crates/workdir/src/workspace.rs index d0646706..69bc9fe5 100644 --- a/crates/workdir/src/workspace.rs +++ b/crates/workdir/src/workspace.rs @@ -30,6 +30,8 @@ impl RuntimeWorkerRef { #[serde(rename_all = "snake_case")] pub enum MaterializerKind { #[default] + RuntimeGitCache, + /// Legacy persisted value from the pre-cache local `git worktree` materializer. LocalGitWorktree, } diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index 9171e23d..16a2cb34 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -43,6 +43,7 @@ tokio = { workspace = true, features = ["net", "rt", "sync", "time"] } toml.workspace = true url.workspace = true uuid = { workspace = true, features = ["v7"] } +zeroize.workspace = true tower = { workspace = true, features = ["util"], optional = true } worker.workspace = true workspace-api = { path = "../workspace-api" } diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index 26d592c1..fd0b8a7e 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -97,6 +97,55 @@ pub use workdir::workspace::{ WorkingDirectorySummary, }; +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SensitiveString(String); + +impl SensitiveString { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl Drop for SensitiveString { + fn drop(&mut self) { + zeroize::Zeroize::zeroize(&mut self.0); + } +} + +impl std::fmt::Debug for SensitiveString { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("[REDACTED]") + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepositorySshMaterializationAccess { + pub credential_id: String, + pub credential_revision: u64, + pub host_trust_id: String, + pub host_trust_revision: u64, + pub access: workspace_api::RepositoryAccessMode, + pub private_key: SensitiveString, + pub known_hosts_entry: SensitiveString, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepositoryMaterializationContext { + pub workspace_id: String, + pub runtime_id: String, + pub operation_id: String, + pub config_revision: u64, + pub config_projection_digest: String, + #[serde(default)] + pub cache_generation: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ssh: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkingDirectoryRequest { pub repository: WorkingDirectoryRepository, @@ -106,6 +155,9 @@ pub struct WorkingDirectoryRequest { /// Backend can create canonical registry rows before materialization. #[serde(default, skip_serializing_if = "Option::is_none")] pub backend_workdir_id: Option, + /// Backend-authored, operation-scoped repository access and cache identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub materialization: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 7236f3d3..0ed02914 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -527,9 +527,19 @@ async fn list_working_directories( async fn create_working_directory( State(state): State, + Extension(auth): Extension, body: Result, JsonRejection>, ) -> RestResult { let Json(request) = body.map_err(RuntimeHttpRestError::json_rejection)?; + if let Some(materialization) = request.materialization.as_ref() + && materialization.workspace_id != auth.workspace_id + { + return Err(RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "working_directory_materialization_workspace_mismatch", + "Repository materialization authority does not match the authenticated Workspace", + )); + } let working_directory = state .runtime .create_working_directory(request) diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index 3cd733f2..c7b8179c 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -23,7 +23,7 @@ use worker_runtime::http_server::{ RuntimeHttpServerConfig, RuntimeHttpServerError, RuntimeHttpStoreSelection, }; use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend}; -use worker_runtime::working_directory::LocalGitWorktreeMaterializer; +use worker_runtime::working_directory::RuntimeGitCacheMaterializer; use worker_runtime::{Runtime, RuntimeOptions}; fn main() -> ExitCode { @@ -192,7 +192,7 @@ fn build_runtime(config: &ProcessConfig) -> Result { let backend = Arc::new( WorkerRuntimeExecutionBackend::new(factory) .map_err(ProcessError::WorkerAdapter)? - .with_working_directory_materializer(LocalGitWorktreeMaterializer::new( + .with_working_directory_materializer(RuntimeGitCacheMaterializer::new( fs_paths.workdir_target.clone(), )), ); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index c6d69c6b..3f334145 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -706,13 +706,17 @@ fn runtime_local_workdir_session( root: &Path, cwd: &Path, scope: manifest::SharedScope, + command_environment: std::collections::BTreeMap, + resources: Vec>, ) -> WorkdirSessionHandle { - Arc::new(LocalWorkdirSession::materialized_bound( + Arc::new(LocalWorkdirSession::materialized_bound_with_environment( Workdir::new(workdir_id), root.to_path_buf(), cwd.to_path_buf(), scope, WorkdirSessionCapabilities::ALL, + command_environment, + resources, )) } @@ -893,6 +897,8 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { binding.root(), binding.cwd(), worker.scope().clone(), + binding.command_environment(), + binding.session_resources(), ))); } else { worker.bind_workdir_session(None); @@ -1071,6 +1077,8 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { binding.root(), binding.cwd(), worker.scope().clone(), + binding.command_environment(), + binding.session_resources(), ))); } else { worker.bind_workdir_session(None); @@ -1624,6 +1632,8 @@ where binding.root(), binding.cwd(), manifest::SharedScope::new(scope), + binding.command_environment(), + binding.session_resources(), )) } @@ -2142,7 +2152,7 @@ mod tests { use crate::identity::WorkerRef; use crate::management::RuntimeOptions; use crate::observation::WorkerObservationCursor; - use crate::working_directory::LocalGitWorktreeMaterializer; + use crate::working_directory::RuntimeGitCacheMaterializer; use agen::Engine; use agen::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent}; use agen::llm_client::{ClientError, LlmClient, Request}; @@ -2728,8 +2738,9 @@ mod tests { source_fingerprint: "sha256:test".to_string(), selector: Some(RepositorySelector::from("HEAD")), }, - materializer: MaterializerKind::LocalGitWorktree, + materializer: MaterializerKind::RuntimeGitCache, backend_workdir_id: None, + materialization: None, } } @@ -2853,12 +2864,16 @@ mod tests { root.path(), root.path(), manifest::SharedScope::new(Scope::writable(root.path()).unwrap()), + Default::default(), + Vec::new(), ); let restored = runtime_local_workdir_session( "working-directory-42", root.path(), root.path(), manifest::SharedScope::new(Scope::writable(root.path()).unwrap()), + Default::default(), + Vec::new(), ); assert_eq!(spawned.workdir().id().as_str(), "working-directory-42"); @@ -3330,7 +3345,7 @@ mod tests { }; let backend = WorkerRuntimeExecutionBackend::new(factory) .unwrap() - .with_working_directory_materializer(LocalGitWorktreeMaterializer::new( + .with_working_directory_materializer(RuntimeGitCacheMaterializer::new( runtime_base.path(), )); let runtime = @@ -3485,7 +3500,7 @@ mod tests { }; let backend = WorkerRuntimeExecutionBackend::new(factory) .unwrap() - .with_working_directory_materializer(LocalGitWorktreeMaterializer::new( + .with_working_directory_materializer(RuntimeGitCacheMaterializer::new( runtime_base.path(), )); let runtime = @@ -3524,7 +3539,7 @@ mod tests { let repo = create_clean_repo(); let backend = WorkerRuntimeExecutionBackend::new(FailingFactory) .unwrap() - .with_working_directory_materializer(LocalGitWorktreeMaterializer::new( + .with_working_directory_materializer(RuntimeGitCacheMaterializer::new( runtime_base.path(), )); let runtime = @@ -3560,7 +3575,7 @@ mod tests { let repo = create_clean_repo(); let backend = WorkerRuntimeExecutionBackend::new(FailingFactory) .unwrap() - .with_working_directory_materializer(LocalGitWorktreeMaterializer::new( + .with_working_directory_materializer(RuntimeGitCacheMaterializer::new( runtime_base.path(), )); let runtime = @@ -3574,9 +3589,15 @@ mod tests { assert!(format!("{error:?}").contains("spawn failed")); let working_directories_root = runtime_base.path(); - let remaining_entries = fs::read_dir(working_directories_root) - .map(|entries| entries.count()) + let remaining_workdirs = fs::read_dir(working_directories_root) + .map(|entries| { + entries + .flatten() + .filter(|entry| !entry.file_name().to_string_lossy().starts_with('.')) + .count() + }) .unwrap_or(0); - assert_eq!(remaining_entries, 0); + assert_eq!(remaining_workdirs, 0); + assert!(working_directories_root.join(".repository-cache").is_dir()); } } diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index ddf53105..7d2d7f44 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -1,17 +1,28 @@ use crate::catalog::{ - MaterializerKind, WorkingDirectoryCleanupTarget, WorkingDirectoryRequest, - WorkingDirectoryStatus, WorkingDirectoryStatusKind, WorkingDirectorySummary, + MaterializerKind, RepositorySshMaterializationAccess, WorkingDirectoryCleanupTarget, + WorkingDirectoryRequest, WorkingDirectoryStatus, WorkingDirectoryStatusKind, + WorkingDirectorySummary, }; use crate::identity::WorkerRef; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, HashMap}; use std::fs; +use std::io::Write; use std::path::{Component, Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use workdir::WorkdirSessionResource; const CHECKOUT_DIR: &str = "checkout"; const MATERIALIZATION_RECORD: &str = "materialization.json"; +const REPOSITORY_CACHE_DIR: &str = ".repository-cache"; +const REPOSITORY_ACCESS_DIR: &str = ".repository-access"; +const REPOSITORY_COMMAND_TIMEOUT: Duration = Duration::from_secs(300); +const REPOSITORY_MAX_OBJECTS: u64 = 5_000_000; +const REPOSITORY_MAX_BYTES: u64 = 5 * 1024 * 1024 * 1024; static NEXT_WORKING_DIRECTORY_SEQUENCE: AtomicU64 = AtomicU64::new(0); #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -23,6 +34,20 @@ pub struct WorkingDirectoryEvidence { #[serde(default, skip_serializing_if = "Option::is_none")] pub resolved_tree: Option, pub materializer_kind: MaterializerKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repository_source_revision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repository_source_fingerprint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repository_cache_key: Option, + #[serde(default)] + pub cache_generation: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operation_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credential_revision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub host_trust_revision: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -54,13 +79,15 @@ impl WorkingDirectory { } } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug)] pub struct WorkingDirectoryBinding { pub working_directory: WorkingDirectory, pub root: PathBuf, pub cwd: PathBuf, working_directory_root: PathBuf, source_repository_path: PathBuf, + command_environment: BTreeMap, + session_resources: Vec>, } impl WorkingDirectoryBinding { @@ -80,6 +107,14 @@ impl WorkingDirectoryBinding { &self.source_repository_path } + pub fn command_environment(&self) -> BTreeMap { + self.command_environment.clone() + } + + pub fn session_resources(&self) -> Vec> { + self.session_resources.clone() + } + pub fn status(&self) -> WorkingDirectoryStatus { let mut working_directory = self.working_directory.clone(); if working_directory.status == WorkingDirectoryStatusKind::Active @@ -200,14 +235,18 @@ fn binding_cleanliness(binding: &WorkingDirectoryBinding) -> String { } #[derive(Clone, Debug)] -pub struct LocalGitWorktreeMaterializer { +pub struct RuntimeGitCacheMaterializer { runtime_root: PathBuf, + repository_access: Arc>>, + cache_locks: Arc>>>>, } -impl LocalGitWorktreeMaterializer { +impl RuntimeGitCacheMaterializer { pub fn new(runtime_root: impl Into) -> Self { Self { runtime_root: runtime_root.into(), + repository_access: Arc::new(Mutex::new(HashMap::new())), + cache_locks: Arc::new(Mutex::new(HashMap::new())), } } @@ -223,6 +262,33 @@ impl LocalGitWorktreeMaterializer { self.runtime_root.join(working_directory_id) } + fn repository_cache_key(request: &WorkingDirectoryRequest) -> String { + let mut digest = Sha256::new(); + if let Some(materialization) = &request.materialization { + digest.update(materialization.workspace_id.as_bytes()); + digest.update([0]); + digest.update(materialization.cache_generation.to_be_bytes()); + } + digest.update(request.repository.id.as_bytes()); + digest.update([0]); + digest.update(request.repository.source.kind.as_str().as_bytes()); + digest.update([0]); + digest.update(request.repository.source_revision.to_be_bytes()); + digest.update([0]); + digest.update(request.repository.source_fingerprint.as_bytes()); + digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() + } + + fn repository_cache_path(&self, request: &WorkingDirectoryRequest) -> PathBuf { + self.runtime_root + .join(REPOSITORY_CACHE_DIR) + .join(format!("{}.git", Self::repository_cache_key(request))) + } + fn corrupted_status(&self, working_directory_id: &str) -> WorkingDirectoryStatus { WorkingDirectoryStatus { summary: WorkingDirectorySummary { @@ -232,9 +298,9 @@ impl LocalGitWorktreeMaterializer { creation_ref: None, current_selector: None, current_ref: None, - materializer_kind: MaterializerKind::LocalGitWorktree, + materializer_kind: MaterializerKind::RuntimeGitCache, cleanup_target: Some(WorkingDirectoryCleanupTarget { - kind: "local_git_worktree".to_string(), + kind: "runtime_git_cache_worktree".to_string(), working_directory_id: working_directory_id.to_string(), repository_id: "unknown".to_string(), }), @@ -294,51 +360,124 @@ impl LocalGitWorktreeMaterializer { cwd: record.root, working_directory_root, source_repository_path: record.source_repository_path, + command_environment: BTreeMap::new(), + session_resources: Vec::new(), }) } - fn materialize_with_working_directory_id( + fn bind_repository_access( &self, - working_directory_id: String, - request: &WorkingDirectoryRequest, + working_directory_id: &str, + mut binding: WorkingDirectoryBinding, ) -> Result { - validate_working_directory_id(&working_directory_id)?; - if request.materializer != MaterializerKind::LocalGitWorktree { + let access = self + .repository_access + .lock() + .map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_unavailable", + "Runtime Repository access state is unavailable", + ) + })? + .get(working_directory_id) + .cloned(); + let Some(access) = access else { + if binding + .working_directory + .evidence + .credential_revision + .is_some() + { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_remote_repository_access_required", + "SSH Repository access must be reacquired before opening the Workdir session", + )); + } + return Ok(binding); + }; + let agent = Arc::new(RepositorySshAgent::start( + &self.runtime_root, + working_directory_id, + &access, + )?); + binding.command_environment.insert( + "SSH_AUTH_SOCK".to_string(), + agent.socket.to_string_lossy().to_string(), + ); + binding.command_environment.insert( + "YOI_REPOSITORY_ACCESS".to_string(), + match access.access { + workspace_api::RepositoryAccessMode::ReadOnly => "read_only", + workspace_api::RepositoryAccessMode::ReadWrite => "read_write", + } + .to_string(), + ); + binding.session_resources.push(agent); + Ok(binding) + } + + fn validate_request( + request: &WorkingDirectoryRequest, + ) -> Result<(), WorkingDirectoryDiagnostic> { + if !matches!( + request.materializer, + MaterializerKind::RuntimeGitCache | MaterializerKind::LocalGitWorktree + ) { return Err(WorkingDirectoryDiagnostic::new( "working_directory_materializer_unsupported", - "only local_git_worktree working directory materialization is supported in v0", + "the requested working directory materializer is unsupported", )); } if request.repository.provider != "git" { return Err(WorkingDirectoryDiagnostic::new( "working_directory_repository_provider_unsupported", - format!( - "repository provider `{}` is not supported by the v0 working directory materializer", - request.repository.provider - ), + "the configured Repository provider is unsupported", )); } - let source_path = match request.repository.source.kind { - workspace_api::RepositorySourceKind::LocalPath => { - PathBuf::from(&request.repository.source.uri) + if matches!( + request.repository.source.kind, + workspace_api::RepositorySourceKind::Https | workspace_api::RepositorySourceKind::Ssh + ) { + validate_remote_source_uri(request)?; + let materialization = request.materialization.as_ref().ok_or_else(|| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_materialization_authority_required", + "remote Repository materialization requires Backend-authored operation authority", + ) + })?; + if materialization.workspace_id.trim().is_empty() + || materialization.runtime_id.trim().is_empty() + || materialization.operation_id.trim().is_empty() + || materialization.config_revision == 0 + || materialization.config_projection_digest.trim().is_empty() + { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_materialization_authority_invalid", + "remote Repository materialization authority is invalid", + )); } - workspace_api::RepositorySourceKind::File => { - url::Url::parse(&request.repository.source.uri) - .ok() - .and_then(|uri| uri.to_file_path().ok()) + } + match request.repository.source.kind { + workspace_api::RepositorySourceKind::LocalPath + | workspace_api::RepositorySourceKind::File + | workspace_api::RepositorySourceKind::Https => {} + workspace_api::RepositorySourceKind::Ssh => { + let ssh = request + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.as_ref()) .ok_or_else(|| { WorkingDirectoryDiagnostic::new( - "working_directory_repository_source_invalid", - "configured file Repository source is invalid", + "working_directory_remote_repository_access_required", + "SSH Repository materialization requires operation-scoped credential and host-trust authority", ) - })? + })?; + validate_ssh_materialization_access(ssh)?; } - workspace_api::RepositorySourceKind::Ssh - | workspace_api::RepositorySourceKind::Http - | workspace_api::RepositorySourceKind::Https => { + workspace_api::RepositorySourceKind::Http => { return Err(WorkingDirectoryDiagnostic::new( - "working_directory_remote_repository_access_required", - "remote Repository materialization requires an explicit authenticated access and trust handle", + "working_directory_insecure_repository_transport_rejected", + "plain HTTP Repository materialization is rejected", )); } workspace_api::RepositorySourceKind::Invalid => { @@ -347,39 +486,116 @@ impl LocalGitWorktreeMaterializer { "configured Repository source is invalid and cannot be materialized", )); } - }; - let source_root = git_stdout(&source_path, ["rev-parse", "--show-toplevel"]) - .map(|value| PathBuf::from(value.trim())) + } + validate_selector(request.repository.selector.as_deref().unwrap_or("HEAD")) + } + + fn ensure_repository_cache( + &self, + request: &WorkingDirectoryRequest, + ) -> Result { + Self::validate_request(request)?; + let cache_key = Self::repository_cache_key(request); + let cache_lock = self + .cache_locks + .lock() .map_err(|_| { WorkingDirectoryDiagnostic::new( - "working_directory_git_repository_unavailable", - "configured local repository is not an available Git worktree; backend-private path details were omitted", + "working_directory_repository_cache_unavailable", + "Runtime Repository cache coordination is unavailable", ) - })?; + })? + .entry(cache_key) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone(); + let _cache_guard = cache_lock.lock().map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_cache_unavailable", + "Runtime Repository cache coordination is unavailable", + ) + })?; + let cache_path = self.repository_cache_path(request); + let cache_parent = cache_path.parent().ok_or_else(|| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_cache_invalid", + "Runtime Repository cache path is invalid", + ) + })?; + fs::create_dir_all(cache_parent).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_cache_create_failed", + "Runtime Repository cache could not be created; backend-private path details were omitted", + ) + })?; - let selector = request - .repository - .selector - .as_deref() - .unwrap_or("HEAD") - .to_string(); - if selector == "HEAD" { - let status = git_stdout(&source_root, ["status", "--porcelain"])?; - if !status.trim().is_empty() { + let access = RepositoryCommandAccess::prepare(&self.runtime_root, request)?; + if cache_path.exists() { + if git_dir_stdout(&cache_path, ["rev-parse", "--is-bare-repository"])? != "true" + || git_dir_stdout(&cache_path, ["remote", "get-url", "origin"])? + != request.repository.source.uri + { return Err(WorkingDirectoryDiagnostic::new( - "working_directory_dirty_source_rejected", - "working directory materialization rejects dirty source repository state", + "working_directory_repository_cache_identity_mismatch", + "Runtime Repository cache identity does not match the requested source", )); } + let mut command = repository_git_command(request, access.as_ref()); + command + .arg("--git-dir") + .arg(&cache_path) + .args(["remote", "update", "--prune"]); + run_repository_git(command, "working_directory_repository_fetch_failed")?; + } else { + let staging = cache_path.with_extension(format!( + "staging-{}", + next_working_directory_id(&request.repository.id) + )); + if staging.exists() { + let _ = fs::remove_dir_all(&staging); + } + let mut command = repository_git_command(request, access.as_ref()); + command.args(["clone", "--mirror"]); + if request.repository.source.kind == workspace_api::RepositorySourceKind::LocalPath { + command.arg("--no-local"); + } + command.arg(&request.repository.source.uri).arg(&staging); + if let Err(error) = + run_repository_git(command, "working_directory_repository_fetch_failed") + { + let _ = fs::remove_dir_all(&staging); + return Err(error); + } + if let Err(error) = fs::rename(&staging, &cache_path) { + let _ = fs::remove_dir_all(&staging); + if !cache_path.exists() { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_cache_publish_failed", + format!( + "Runtime Repository cache could not be published: {}", + error.kind() + ), + )); + } + } } + validate_repository_cache_limits(&cache_path)?; + Ok(cache_path) + } + + fn materialize_with_working_directory_id( + &self, + working_directory_id: String, + request: &WorkingDirectoryRequest, + ) -> Result { + validate_working_directory_id(&working_directory_id)?; + let repository_cache = self.ensure_repository_cache(request)?; + let selector = request.repository.selector.as_deref().unwrap_or("HEAD"); let commit_spec = format!("{selector}^{{commit}}"); - let resolved_commit = git_stdout(&source_root, ["rev-parse", commit_spec.as_str()])? - .trim() - .to_string(); + let resolved_commit = + git_dir_stdout(&repository_cache, ["rev-parse", commit_spec.as_str()])?; let tree_spec = format!("{resolved_commit}^{{tree}}"); - let resolved_tree = git_stdout(&source_root, ["rev-parse", tree_spec.as_str()]) + let resolved_tree = git_dir_stdout(&repository_cache, ["rev-parse", tree_spec.as_str()]) .ok() - .map(|value| value.trim().to_string()) .filter(|value| !value.is_empty()); let working_directory_root = self.working_directory_root(&working_directory_id); @@ -387,38 +603,62 @@ impl LocalGitWorktreeMaterializer { if worktree_root.exists() { return Err(WorkingDirectoryDiagnostic::new( "working_directory_exists", - "working directory working_directory target already exists; cleanup or choose a new working_directory", + "working directory target already exists; cleanup or choose a new working_directory", )); } - fs::create_dir_all(worktree_root.parent().ok_or_else(|| { - WorkingDirectoryDiagnostic::new( - "working_directory_invalid_target", - "working directory working_directory target has no parent directory", - ) - })?) - .map_err(|_| { + fs::create_dir_all(&working_directory_root).map_err(|_| { WorkingDirectoryDiagnostic::new( "working_directory_create_failed", - "failed to create working directory working_directory directory; backend-private path details were omitted", + "failed to create working directory; backend-private path details were omitted", ) })?; + let mut command = isolated_git_command(); + command + .arg("--git-dir") + .arg(&repository_cache) + .args(["worktree", "add", "--detach"]) + .arg(&worktree_root) + .arg(&resolved_commit); + if let Err(error) = run_repository_git(command, "working_directory_git_failed") { + let _ = fs::remove_dir_all(&working_directory_root); + return Err(error); + } + if request + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.as_ref()) + .is_some_and(|ssh| ssh.access == workspace_api::RepositoryAccessMode::ReadOnly) + { + let mut enable_worktree_config = isolated_git_command(); + enable_worktree_config + .arg("--git-dir") + .arg(&repository_cache) + .args(["config", "extensions.worktreeConfig", "true"]); + let mut disable_push = isolated_git_command(); + disable_push.arg("-C").arg(&worktree_root).args([ + "config", + "--worktree", + "remote.origin.pushurl", + "yoi-read-only://repository-push-disabled", + ]); + if let Err(error) = run_repository_git( + enable_worktree_config, + "working_directory_repository_policy_failed", + ) + .and_then(|_| { + run_repository_git(disable_push, "working_directory_repository_policy_failed") + }) { + remove_cached_worktree(&repository_cache, &worktree_root); + let _ = fs::remove_dir_all(&working_directory_root); + return Err(error); + } + } - let workspace_worktree_root_arg = path_str(&worktree_root)?; - git_status( - &source_root, - [ - "worktree", - "add", - "--detach", - workspace_worktree_root_arg.as_str(), - resolved_commit.as_str(), - ], - )?; - + let context = request.materialization.as_ref(); let working_directory = WorkingDirectory { id: working_directory_id.clone(), repository_id: request.repository.id.clone(), - materializer_kind: MaterializerKind::LocalGitWorktree, + materializer_kind: MaterializerKind::RuntimeGitCache, evidence: WorkingDirectoryEvidence { repository_id: request.repository.id.clone(), requested_selector: request @@ -428,10 +668,23 @@ impl LocalGitWorktreeMaterializer { .map(|selector| selector.as_ref().to_string()), resolved_commit, resolved_tree, - materializer_kind: MaterializerKind::LocalGitWorktree, + materializer_kind: MaterializerKind::RuntimeGitCache, + repository_source_revision: Some(request.repository.source_revision), + repository_source_fingerprint: Some(request.repository.source_fingerprint.clone()), + repository_cache_key: Some(Self::repository_cache_key(request)), + cache_generation: context + .map(|value| value.cache_generation) + .unwrap_or_default(), + operation_id: context.map(|value| value.operation_id.clone()), + credential_revision: context + .and_then(|value| value.ssh.as_ref()) + .map(|value| value.credential_revision), + host_trust_revision: context + .and_then(|value| value.ssh.as_ref()) + .map(|value| value.host_trust_revision), }, cleanup_target: WorkingDirectoryCleanupTarget { - kind: "git_worktree".to_string(), + kind: "runtime_git_cache_worktree".to_string(), working_directory_id, repository_id: request.repository.id.clone(), }, @@ -440,16 +693,40 @@ impl LocalGitWorktreeMaterializer { let binding = WorkingDirectoryBinding { working_directory, root: worktree_root.clone(), - cwd: worktree_root, - working_directory_root, - source_repository_path: source_root, + cwd: worktree_root.clone(), + working_directory_root: working_directory_root.clone(), + source_repository_path: repository_cache.clone(), + command_environment: BTreeMap::new(), + session_resources: Vec::new(), }; - self.write_record(&binding)?; + if let Err(error) = self.write_record(&binding) { + remove_cached_worktree(&repository_cache, &worktree_root); + let _ = fs::remove_dir_all(&working_directory_root); + return Err(error); + } + if let Some(ssh) = request + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.clone()) + { + let mut repository_access = match self.repository_access.lock() { + Ok(repository_access) => repository_access, + Err(_) => { + remove_cached_worktree(&repository_cache, &worktree_root); + let _ = fs::remove_dir_all(&working_directory_root); + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_unavailable", + "Runtime Repository access state is unavailable", + )); + } + }; + repository_access.insert(binding.working_directory.id.clone(), ssh); + } Ok(binding) } } -impl WorkingDirectoryMaterializer for LocalGitWorktreeMaterializer { +impl WorkingDirectoryMaterializer for RuntimeGitCacheMaterializer { fn materialize( &self, worker_ref: &WorkerRef, @@ -483,6 +760,7 @@ impl WorkingDirectoryMaterializer for LocalGitWorktreeMaterializer { "working directory working_directory is not active", )); } + let binding = self.bind_repository_access(working_directory_id, binding)?; let cwd = validate_relative_cwd(binding.root(), relative_cwd)?; Ok(WorkingDirectoryBinding { cwd, ..binding }) } @@ -510,7 +788,9 @@ impl WorkingDirectoryMaterializer for LocalGitWorktreeMaterializer { continue; } let working_directory_id = entry.file_name().to_string_lossy().to_string(); - if validate_working_directory_id(&working_directory_id).is_err() { + if working_directory_id.starts_with('.') + || validate_working_directory_id(&working_directory_id).is_err() + { continue; } match self.read_binding(&working_directory_id) { @@ -601,9 +881,19 @@ impl WorkingDirectoryMaterializer for LocalGitWorktreeMaterializer { )); } let workspace_worktree_root_arg = path_str(&root)?; - let remove_result = git_status( - binding.source_repository_path(), - ["worktree", "remove", "--force", workspace_worktree_root_arg.as_str()], + let mut remove_command = isolated_git_command(); + remove_command + .arg("--git-dir") + .arg(binding.source_repository_path()) + .args([ + "worktree", + "remove", + "--force", + workspace_worktree_root_arg.as_str(), + ]); + let remove_result = run_repository_git( + remove_command, + "working_directory_cleanup_failed", ) .or_else(|_| { if root.exists() { @@ -625,13 +915,485 @@ impl WorkingDirectoryMaterializer for LocalGitWorktreeMaterializer { cwd: binding.cwd.clone(), working_directory_root: binding.working_directory_root.clone(), source_repository_path: binding.source_repository_path.clone(), + command_environment: BTreeMap::new(), + session_resources: Vec::new(), }; let _ = self.write_record(&updated); + } else if let Ok(mut access) = self.repository_access.lock() { + access.remove(&binding.working_directory.id); } remove_result } } +#[derive(Debug)] +struct RepositorySshAgent { + root: PathBuf, + socket: PathBuf, + child: Mutex>, +} + +impl RepositorySshAgent { + fn start( + runtime_root: &Path, + working_directory_id: &str, + access: &RepositorySshMaterializationAccess, + ) -> Result { + let root = runtime_root.join(".repository-agents").join(format!( + "{}-{}", + sanitize_path_component(working_directory_id), + next_working_directory_id("agent") + )); + fs::create_dir_all(&root).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_agent_failed", + "Runtime-managed Repository SSH agent could not be created", + ) + })?; + set_directory_owner_only(&root)?; + let socket = root.join("agent.sock"); + let mut child = Command::new("ssh-agent") + .args(["-D", "-a"]) + .arg(&socket) + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_agent_unavailable", + "Runtime-managed Repository SSH agent is unavailable", + ) + })?; + let started = Instant::now(); + loop { + if socket.exists() { + break; + } + if matches!(child.try_wait(), Ok(Some(_)) | Err(_)) + || started.elapsed() >= Duration::from_secs(2) + { + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(&root); + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_agent_failed", + "Runtime-managed Repository SSH agent could not be started", + )); + } + std::thread::sleep(Duration::from_millis(10)); + } + let agent = Self { + root, + socket, + child: Mutex::new(Some(child)), + }; + let mut add = match Command::new("ssh-add") + .arg("-") + .env("SSH_AUTH_SOCK", &agent.socket) + .env("SSH_ASKPASS", "/bin/false") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + { + Ok(add) => add, + Err(_) => { + drop(agent); + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_agent_unavailable", + "Runtime-managed Repository SSH agent is unavailable", + )); + } + }; + let write_result = add.stdin.as_mut().map_or_else( + || Err(std::io::Error::other("ssh-add stdin unavailable")), + |stdin| stdin.write_all(access.private_key.expose().as_bytes()), + ); + let status = add.wait(); + if write_result.is_err() || !matches!(status, Ok(status) if status.success()) { + drop(agent); + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_agent_failed", + "Runtime-managed Repository SSH agent rejected credential material", + )); + } + Ok(agent) + } +} + +impl Drop for RepositorySshAgent { + fn drop(&mut self) { + if let Ok(mut child) = self.child.lock() + && let Some(mut child) = child.take() + { + let _ = child.kill(); + let _ = child.wait(); + } + let _ = fs::remove_dir_all(&self.root); + } +} + +struct RepositoryCommandAccess { + root: PathBuf, + ssh_command: PathBuf, +} + +impl RepositoryCommandAccess { + fn prepare( + runtime_root: &Path, + request: &WorkingDirectoryRequest, + ) -> Result, WorkingDirectoryDiagnostic> { + let Some(ssh) = request + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.as_ref()) + else { + return Ok(None); + }; + let operation_id = request + .materialization + .as_ref() + .map(|materialization| materialization.operation_id.as_str()) + .unwrap_or("operation"); + let root = runtime_root.join(REPOSITORY_ACCESS_DIR).join(format!( + "{}-{}", + sanitize_path_component(operation_id), + next_working_directory_id(&request.repository.id) + )); + fs::create_dir_all(&root).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "operation-scoped Repository access could not be prepared", + ) + })?; + set_directory_owner_only(&root)?; + let private_key = root.join("identity"); + let known_hosts = root.join("known_hosts"); + let ssh_command = root.join("ssh-command"); + write_owner_only(&private_key, ssh.private_key.expose().as_bytes())?; + write_owner_only(&known_hosts, ssh.known_hosts_entry.expose().as_bytes())?; + let script = format!( + "#!/bin/sh\nexec ssh -F /dev/null -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile={} -i {} \"$@\"\n", + shell_quote_path(&known_hosts)?, + shell_quote_path(&private_key)?, + ); + write_owner_only(&ssh_command, script.as_bytes())?; + set_file_owner_executable(&ssh_command)?; + Ok(Some(Self { root, ssh_command })) + } +} + +impl Drop for RepositoryCommandAccess { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } +} + +fn validate_ssh_materialization_access( + access: &RepositorySshMaterializationAccess, +) -> Result<(), WorkingDirectoryDiagnostic> { + if access.credential_id.trim().is_empty() + || access.credential_revision == 0 + || access.host_trust_id.trim().is_empty() + || access.host_trust_revision == 0 + || !access.private_key.expose().contains("PRIVATE KEY") + || access.known_hosts_entry.expose().trim().is_empty() + { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_remote_repository_access_invalid", + "operation-scoped SSH credential or host-trust authority is invalid", + )); + } + Ok(()) +} + +fn validate_remote_source_uri( + request: &WorkingDirectoryRequest, +) -> Result<(), WorkingDirectoryDiagnostic> { + let url = url::Url::parse(&request.repository.source.uri).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_source_invalid", + "remote Repository source URI is invalid", + ) + })?; + let expected_scheme = match request.repository.source.kind { + workspace_api::RepositorySourceKind::Https => "https", + workspace_api::RepositorySourceKind::Ssh => "ssh", + _ => return Ok(()), + }; + if url.scheme() != expected_scheme + || url.host_str().is_none() + || url.password().is_some() + || !url.query().is_none() + || !url.fragment().is_none() + || (expected_scheme == "https" && !url.username().is_empty()) + { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_source_invalid", + "remote Repository source URI is invalid or contains forbidden credentials", + )); + } + if expected_scheme == "ssh" { + let access = request + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.as_ref()) + .ok_or_else(|| { + WorkingDirectoryDiagnostic::new( + "working_directory_remote_repository_access_required", + "SSH Repository materialization requires operation-scoped credential and host-trust authority", + ) + })?; + let host = url.host_str().unwrap_or_default(); + let known_host = if url.port().unwrap_or(22) == 22 { + format!("{host} ") + } else { + format!("[{host}]:{} ", url.port().unwrap_or(22)) + }; + if !access.known_hosts_entry.expose().starts_with(&known_host) { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_remote_repository_host_trust_mismatch", + "SSH Repository source does not match the operation-scoped host-trust authority", + )); + } + } + Ok(()) +} + +fn validate_selector(selector: &str) -> Result<(), WorkingDirectoryDiagnostic> { + let valid = !selector.is_empty() + && selector.len() <= 512 + && !selector.starts_with('-') + && !selector.ends_with('.') + && !selector.contains("..") + && !selector.contains("@{") + && !selector.contains("//") + && !selector.chars().any(|ch| { + ch.is_control() + || ch.is_whitespace() + || matches!(ch, '~' | '^' | ':' | '?' | '*' | '[' | '\\') + }); + if !valid { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_selector_invalid", + "configured Repository selector is invalid", + )); + } + Ok(()) +} + +fn isolated_git_command() -> Command { + let mut command = Command::new("git"); + command + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_ASKPASS", "/bin/false") + .env("SSH_ASKPASS", "/bin/false") + .env("LC_ALL", "C") + .args([ + "-c", + "credential.helper=", + "-c", + "core.askPass=/bin/false", + "-c", + "http.followRedirects=false", + "-c", + "protocol.ext.allow=never", + "-c", + "submodule.recurse=false", + ]); + command +} + +fn repository_git_command( + request: &WorkingDirectoryRequest, + access: Option<&RepositoryCommandAccess>, +) -> Command { + let mut command = isolated_git_command(); + let file_policy = if matches!( + request.repository.source.kind, + workspace_api::RepositorySourceKind::LocalPath | workspace_api::RepositorySourceKind::File + ) { + "always" + } else { + "never" + }; + command.args(["-c", &format!("protocol.file.allow={file_policy}")]); + if let Some(access) = access { + command.env("GIT_SSH_COMMAND", &access.ssh_command); + } + command +} + +fn run_repository_git( + mut command: Command, + code: &'static str, +) -> Result<(), WorkingDirectoryDiagnostic> { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_git_unavailable", + "Git command could not be executed; backend-private path details were omitted", + ) + })?; + let started = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait().map_err(|_| { + WorkingDirectoryDiagnostic::new( + code, + "Git Repository operation failed; credentials and backend-private path details were omitted", + ) + })? { + break status; + } + if started.elapsed() >= REPOSITORY_COMMAND_TIMEOUT { + let _ = child.kill(); + let _ = child.wait(); + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_timeout", + "Git Repository operation exceeded the Runtime time limit", + )); + } + std::thread::sleep(Duration::from_millis(25)); + }; + if status.success() { + Ok(()) + } else { + Err(WorkingDirectoryDiagnostic::new( + code, + "Git Repository operation failed; credentials and backend-private path details were omitted", + )) + } +} + +fn validate_repository_cache_limits( + repository_cache: &Path, +) -> Result<(), WorkingDirectoryDiagnostic> { + let report = git_dir_stdout(repository_cache, ["count-objects", "-v"])?; + let mut objects = 0u64; + let mut kibibytes = 0u64; + for line in report.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let value = value.trim().parse::().unwrap_or(u64::MAX); + match key { + "count" | "in-pack" | "garbage" => objects = objects.saturating_add(value), + "size" | "size-pack" | "size-garbage" => kibibytes = kibibytes.saturating_add(value), + _ => {} + } + } + if objects > REPOSITORY_MAX_OBJECTS || kibibytes.saturating_mul(1024) > REPOSITORY_MAX_BYTES { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_limit_exceeded", + "Git Repository exceeds Runtime object or storage limits", + )); + } + Ok(()) +} + +fn remove_cached_worktree(repository_cache: &Path, worktree_root: &Path) { + let mut command = isolated_git_command(); + command + .arg("--git-dir") + .arg(repository_cache) + .args(["worktree", "remove", "--force"]) + .arg(worktree_root); + let _ = run_repository_git(command, "working_directory_cleanup_failed"); +} + +fn git_dir_stdout<'a, I>( + repository_path: &Path, + args: I, +) -> Result +where + I: IntoIterator, +{ + let mut command = isolated_git_command(); + let output = command + .arg("--git-dir") + .arg(repository_path) + .args(args) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + .map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_git_unavailable", + "Git command could not be executed; backend-private path details were omitted", + ) + })?; + if !output.status.success() || output.stdout.len() > 4096 { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_selector_unresolved", + "configured Repository selector could not be resolved to a commit", + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +fn write_owner_only(path: &Path, content: &[u8]) -> Result<(), WorkingDirectoryDiagnostic> { + let mut options = fs::OpenOptions::new(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "operation-scoped Repository access could not be prepared", + ) + })?; + file.write_all(content).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "operation-scoped Repository access could not be prepared", + ) + }) +} + +fn set_directory_owner_only(path: &Path) -> Result<(), WorkingDirectoryDiagnostic> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "operation-scoped Repository access could not be prepared", + ) + })?; + } + Ok(()) +} + +fn set_file_owner_executable(path: &Path) -> Result<(), WorkingDirectoryDiagnostic> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "operation-scoped Repository access could not be prepared", + ) + })?; + } + Ok(()) +} + +fn shell_quote_path(path: &Path) -> Result { + let value = path_str(path)?; + Ok(format!("'{}'", value.replace('\'', "'\\''"))) +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] struct WorkingDirectoryMaterializationRecord { working_directory: WorkingDirectory, @@ -663,13 +1425,6 @@ where Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) } -fn git_status<'a, I>(repository_path: &Path, args: I) -> Result<(), WorkingDirectoryDiagnostic> -where - I: IntoIterator, -{ - git_stdout(repository_path, args).map(|_| ()) -} - fn path_str(path: &Path) -> Result { path.to_str().map(ToString::to_string).ok_or_else(|| { WorkingDirectoryDiagnostic::new( @@ -815,8 +1570,9 @@ mod tests { source_fingerprint: "sha256:test".to_string(), selector: Some(RepositorySelector::from("HEAD")), }, - materializer: MaterializerKind::LocalGitWorktree, + materializer: MaterializerKind::RuntimeGitCache, backend_workdir_id: None, + materialization: None, } } @@ -828,7 +1584,7 @@ mod tests { fn local_git_repo_materializes_detached_worktree_under_runtime_root() { let repo = create_clean_repo(); let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); let binding = materializer .materialize(&worker_ref(1), &request(repo.path())) .unwrap(); @@ -853,7 +1609,7 @@ mod tests { ); assert_eq!( binding.working_directory.materializer_kind, - MaterializerKind::LocalGitWorktree + MaterializerKind::RuntimeGitCache ); assert!( binding @@ -867,7 +1623,7 @@ mod tests { fn multiple_workers_materialize_distinct_paths_for_same_source_repo() { let repo = create_clean_repo(); let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); let first = materializer .materialize(&worker_ref(1), &request(repo.path())) .unwrap(); @@ -883,18 +1639,18 @@ mod tests { } #[test] - fn dirty_source_is_rejected_by_materialization() { + fn dirty_source_is_ignored_by_commit_only_materialization() { let repo = create_clean_repo(); fs::write(repo.path().join("dirty.txt"), "dirty\n").unwrap(); let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); - let error = materializer + let binding = materializer .materialize(&worker_ref(1), &request(repo.path())) - .unwrap_err(); + .unwrap(); - assert_eq!(error.code, "working_directory_dirty_source_rejected"); - assert!(error.message.contains("dirty source")); + assert!(binding.root.join("README.md").exists()); + assert!(!binding.root.join("dirty.txt").exists()); } #[test] @@ -903,7 +1659,7 @@ mod tests { git(repo.path(), &["branch", "pinned"]); fs::write(repo.path().join("dirty.txt"), "dirty\n").unwrap(); let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); let mut request = request(repo.path()); request.repository.selector = Some(RepositorySelector::from("pinned")); @@ -921,15 +1677,276 @@ mod tests { assert!(!binding.root.join("dirty.txt").exists()); } + #[test] + fn file_and_local_sources_share_the_runtime_cache_pipeline() { + let repo = create_clean_repo(); + let runtime_root = tempfile::tempdir().unwrap(); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); + let local = materializer + .materialize(&worker_ref(1), &request(repo.path())) + .unwrap(); + let second = materializer + .materialize(&worker_ref(2), &request(repo.path())) + .unwrap(); + + assert_eq!( + local.working_directory.evidence.repository_cache_key, + second.working_directory.evidence.repository_cache_key + ); + assert_eq!( + fs::read_dir(runtime_root.path().join(REPOSITORY_CACHE_DIR)) + .unwrap() + .count(), + 1 + ); + + let mut file_request = request(repo.path()); + file_request.repository.source.kind = workspace_api::RepositorySourceKind::File; + file_request.repository.source.uri = format!("file://{}", repo.path().display()); + file_request.repository.source_revision = 2; + file_request.repository.source_fingerprint = "sha256:file-source".to_string(); + let file = materializer + .materialize(&worker_ref(3), &file_request) + .unwrap(); + assert!(file.root.join("README.md").exists()); + assert_ne!( + local.working_directory.evidence.repository_cache_key, + file.working_directory.evidence.repository_cache_key + ); + } + + #[test] + fn materialization_context_is_audited_without_secret_values() { + let repo = create_clean_repo(); + let runtime_root = tempfile::tempdir().unwrap(); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); + let mut request = request(repo.path()); + request.materialization = Some(crate::catalog::RepositoryMaterializationContext { + workspace_id: "workspace-1".to_string(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-1".to_string(), + config_revision: 7, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 3, + ssh: None, + }); + + let binding = materializer.materialize(&worker_ref(1), &request).unwrap(); + assert_eq!( + binding.working_directory.evidence.operation_id.as_deref(), + Some("operation-1") + ); + assert_eq!(binding.working_directory.evidence.cache_generation, 3); + let record = fs::read_to_string( + binding + .working_directory_root() + .join(MATERIALIZATION_RECORD), + ) + .unwrap(); + assert!(!record.contains("private key")); + } + + #[test] + fn sensitive_repository_access_debug_output_is_redacted() { + let access = crate::catalog::RepositorySshMaterializationAccess { + credential_id: "credential-1".to_string(), + credential_revision: 2, + host_trust_id: "trust-1".to_string(), + host_trust_revision: 4, + access: workspace_api::RepositoryAccessMode::ReadOnly, + private_key: crate::catalog::SensitiveString::new("PRIVATE KEY secret bytes"), + known_hosts_entry: crate::catalog::SensitiveString::new("host key secret bytes"), + }; + + let debug = format!("{access:?}"); + assert!(!debug.contains("secret bytes")); + assert!(debug.contains("[REDACTED]")); + } + + #[test] + fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() { + let repo = create_clean_repo(); + let runtime_root = tempfile::tempdir().unwrap(); + let key_root = tempfile::tempdir().unwrap(); + let key_path = key_root.path().join("id_ed25519"); + let status = Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(&key_path) + .status() + .unwrap(); + assert!(status.success()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); + let mut request = request(repo.path()); + request.materialization = Some(crate::catalog::RepositoryMaterializationContext { + workspace_id: "workspace-1".to_string(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-agent".to_string(), + config_revision: 2, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 0, + ssh: Some(crate::catalog::RepositorySshMaterializationAccess { + credential_id: "credential-1".to_string(), + credential_revision: 1, + host_trust_id: "trust-1".to_string(), + host_trust_revision: 1, + access: workspace_api::RepositoryAccessMode::ReadWrite, + private_key: crate::catalog::SensitiveString::new( + fs::read_to_string(&key_path).unwrap(), + ), + known_hosts_entry: crate::catalog::SensitiveString::new( + "example.test ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIexample", + ), + }), + }); + let created = materializer.create(&request).unwrap(); + let id = created.working_directory.id; + assert_eq!(materializer.list_working_directories().unwrap().len(), 1); + assert!(!runtime_root.path().join(".repository-agents").exists()); + let binding = materializer.bind_working_directory(&id, None).unwrap(); + let socket = PathBuf::from(binding.command_environment()["SSH_AUTH_SOCK"].clone()); + assert!(socket.exists()); + assert_eq!( + binding.command_environment()["YOI_REPOSITORY_ACCESS"], + "read_write" + ); + drop(binding); + assert!(!socket.exists()); + let restored = RuntimeGitCacheMaterializer::new(runtime_root.path()); + assert_eq!( + restored.bind_working_directory(&id, None).unwrap_err().code, + "working_directory_remote_repository_access_required" + ); + } + + #[test] + fn read_only_repository_access_disables_default_push_target() { + let repo = create_clean_repo(); + let runtime_root = tempfile::tempdir().unwrap(); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); + let mut request = request(repo.path()); + request.materialization = Some(crate::catalog::RepositoryMaterializationContext { + workspace_id: "workspace-1".to_string(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-read-only".to_string(), + config_revision: 2, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 0, + ssh: Some(crate::catalog::RepositorySshMaterializationAccess { + credential_id: "credential-1".to_string(), + credential_revision: 1, + host_trust_id: "trust-1".to_string(), + host_trust_revision: 1, + access: workspace_api::RepositoryAccessMode::ReadOnly, + private_key: crate::catalog::SensitiveString::new("PRIVATE KEY placeholder"), + known_hosts_entry: crate::catalog::SensitiveString::new( + "example.test ssh-ed25519 placeholder", + ), + }), + }); + + let binding = materializer.create(&request).unwrap(); + assert_eq!( + git_stdout( + binding.root(), + ["config", "--worktree", "--get", "remote.origin.pushurl"], + ) + .unwrap(), + "yoi-read-only://repository-push-disabled" + ); + } + + #[test] + fn selector_is_not_accepted_as_a_git_option_or_refspec() { + let repo = create_clean_repo(); + let runtime_root = tempfile::tempdir().unwrap(); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); + for selector in ["--upload-pack=evil", "refs/heads/main:evil", "main@{1}"] { + let mut request = request(repo.path()); + request.repository.selector = Some(RepositorySelector::from(selector)); + assert_eq!( + materializer + .materialize(&worker_ref(1), &request) + .unwrap_err() + .code, + "working_directory_repository_selector_invalid" + ); + } + } + + #[test] + fn remote_source_rejects_uri_credentials_and_mismatched_host_trust() { + let repo = create_clean_repo(); + let runtime_root = tempfile::tempdir().unwrap(); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); + let context = |ssh| crate::catalog::RepositoryMaterializationContext { + workspace_id: "workspace-1".to_string(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-1".to_string(), + config_revision: 1, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 0, + ssh, + }; + + let mut https = request(repo.path()); + https.repository.source = workspace_api::RepositorySource { + kind: workspace_api::RepositorySourceKind::Https, + uri: "https://token@example.test/repo.git".to_string(), + }; + https.materialization = Some(context(None)); + assert_eq!( + materializer + .materialize(&worker_ref(1), &https) + .unwrap_err() + .code, + "working_directory_repository_source_invalid" + ); + + let mut ssh = request(repo.path()); + ssh.repository.source = workspace_api::RepositorySource { + kind: workspace_api::RepositorySourceKind::Ssh, + uri: "ssh://git@example.test/repo.git".to_string(), + }; + ssh.materialization = Some(context(Some( + crate::catalog::RepositorySshMaterializationAccess { + credential_id: "credential-1".to_string(), + credential_revision: 1, + host_trust_id: "trust-1".to_string(), + host_trust_revision: 1, + access: workspace_api::RepositoryAccessMode::ReadOnly, + private_key: crate::catalog::SensitiveString::new("PRIVATE KEY placeholder"), + known_hosts_entry: crate::catalog::SensitiveString::new( + "other.test ssh-ed25519 placeholder", + ), + }, + ))); + assert_eq!( + materializer + .materialize(&worker_ref(2), &ssh) + .unwrap_err() + .code, + "working_directory_remote_repository_host_trust_mismatch" + ); + } + #[test] fn unsupported_remote_and_non_git_provider_return_typed_diagnostics() { let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); let mut remote = request(Path::new(".")); remote.repository.source = workspace_api::RepositorySource { - kind: workspace_api::RepositorySourceKind::Https, - uri: "https://example.invalid/repo.git".to_string(), + kind: workspace_api::RepositorySourceKind::Ssh, + uri: "ssh://git@example.invalid/repo.git".to_string(), }; + remote.materialization = Some(crate::catalog::RepositoryMaterializationContext { + workspace_id: "workspace-1".to_string(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-1".to_string(), + config_revision: 1, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 0, + ssh: None, + }); let error = materializer .materialize(&worker_ref(1), &remote) .unwrap_err(); @@ -958,7 +1975,7 @@ mod tests { git(repo.path(), &["add", "crates/yoi/lib.rs"]); git(repo.path(), &["commit", "-m", "add crate"]); let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); let working_directory = materializer.create(&request(repo.path())).unwrap(); let bound = materializer @@ -983,7 +2000,7 @@ mod tests { fn working_directory_observes_current_selector_and_ref_without_changing_creation_evidence() { let repo = create_clean_repo(); let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); let working_directory = materializer.create(&request(repo.path())).unwrap(); let bound = materializer .bind_working_directory(&working_directory.working_directory.id, None) @@ -1012,7 +2029,7 @@ mod tests { git(repo.path(), &["add", "inside/file.txt"]); git(repo.path(), &["commit", "-m", "add inside"]); let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); let working_directory = materializer.create(&request(repo.path())).unwrap(); assert_eq!( @@ -1064,7 +2081,7 @@ mod tests { fn cleanup_working_directory_removes_worktree_and_record() { let repo = create_clean_repo(); let runtime_root = tempfile::tempdir().unwrap(); - let materializer = LocalGitWorktreeMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitCacheMaterializer::new(runtime_root.path()); let binding = materializer .materialize(&worker_ref(1), &request(repo.path())) .unwrap(); diff --git a/crates/workspace-server/Cargo.toml b/crates/workspace-server/Cargo.toml index b0c8f447..ad365732 100644 --- a/crates/workspace-server/Cargo.toml +++ b/crates/workspace-server/Cargo.toml @@ -48,6 +48,7 @@ tracing.workspace = true ts-rs = { version = "12.0.1", optional = true } url.workspace = true uuid = { workspace = true, features = ["v7"] } +zeroize.workspace = true webauthn-rs = { workspace = true } [dev-dependencies] diff --git a/crates/workspace-server/src/repository_access.rs b/crates/workspace-server/src/repository_access.rs index 921eb155..9736a07c 100644 --- a/crates/workspace-server/src/repository_access.rs +++ b/crates/workspace-server/src/repository_access.rs @@ -11,7 +11,7 @@ use ring::rand::{SecureRandom, SystemRandom}; use rusqlite::{OptionalExtension, TransactionBehavior, params}; use serde::Deserialize; use sha2::{Digest, Sha256}; -use ssh_key::{Algorithm, HashAlg, PrivateKey, PublicKey}; +use ssh_key::{Algorithm, HashAlg, LineEnding, PrivateKey, PublicKey}; use workspace_api::{ CreateRepositorySshCredentialRequest, DeleteRepositorySshCredentialRequest, DeleteRepositorySshHostTrustRequest, PutRepositorySshHostTrustRequest, RepositoryAccessMode, @@ -59,7 +59,6 @@ impl WorkspaceConfigSchemaProvider for RepositoryAccessConfigSchemaProvider { } #[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] struct VirtualWorkspaceConfig { #[serde(default)] repository_access: BTreeMap, @@ -219,6 +218,16 @@ fn project_repository_access_evaluation( }) } +#[derive(Clone)] +pub struct LeasedRepositorySshAccess { + pub credential_id: String, + pub credential_revision: u64, + pub host_trust_id: String, + pub host_trust_revision: u64, + pub private_key: zeroize::Zeroizing, + pub known_hosts_entry: String, +} + #[derive(Clone)] pub struct RepositorySecretService { store: Arc, @@ -829,6 +838,144 @@ impl RepositorySecretService { }) } + pub fn lease_ssh_materialization_access( + &self, + workspace_id: &str, + binding: &RepositorySshAccessBinding, + ) -> Result { + let credential = self + .get_credential(workspace_id, &binding.credential_id, &[])? + .ok_or_else(|| { + Error::InvalidInput(format!( + "unknown Repository SSH credential `{}`", + binding.credential_id + )) + })?; + if credential.status != "active" { + return Err(Error::InvalidInput(format!( + "Repository SSH credential `{}` is not active", + binding.credential_id + ))); + } + let host_trust = self + .get_host_trust(workspace_id, &binding.host_trust_id, &[])? + .ok_or_else(|| { + Error::InvalidInput(format!( + "unknown Repository SSH host trust `{}`", + binding.host_trust_id + )) + })?; + let (private_key, passphrase) = self.store.with_conn(|conn| { + let private_key = read_sealed_secret( + conn, + workspace_id, + &binding.credential_id, + credential.current_revision, + "private_key", + )? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Repository SSH credential `{}` is missing its private-key revision", + binding.credential_id + )) + })?; + let passphrase = read_sealed_secret( + conn, + workspace_id, + &binding.credential_id, + credential.current_revision, + "passphrase", + )?; + Ok((private_key, passphrase)) + })?; + let private_key = self.unseal( + workspace_id, + &binding.credential_id, + credential.current_revision, + "private_key", + private_key, + )?; + let passphrase = passphrase + .map(|secret| { + self.unseal( + workspace_id, + &binding.credential_id, + credential.current_revision, + "passphrase", + secret, + ) + }) + .transpose()?; + let private_key = + zeroize::Zeroizing::new(String::from_utf8(private_key).map_err(|_| { + Error::Store("Repository SSH private key plaintext is invalid".to_string()) + })?); + let passphrase = passphrase + .map(|value| { + String::from_utf8(value) + .map(zeroize::Zeroizing::new) + .map_err(|_| { + Error::Store("Repository SSH passphrase plaintext is invalid".to_string()) + }) + }) + .transpose()?; + let key = PrivateKey::from_openssh(private_key.as_str()).map_err(|_| { + Error::Store("Repository SSH private key plaintext is invalid".to_string()) + })?; + let key = if key.is_encrypted() { + key.decrypt(passphrase.as_deref().ok_or_else(|| { + Error::Store("Repository SSH passphrase revision is unavailable".to_string()) + })?) + .map_err(|_| Error::Store("Repository SSH private key decryption failed".to_string()))? + } else { + key + }; + let private_key = key + .to_openssh(LineEnding::LF) + .map_err(|_| Error::Store("Repository SSH private key encoding failed".to_string()))?; + let host = if host_trust.port == 22 { + host_trust.hostname.clone() + } else { + format!("[{}]:{}", host_trust.hostname, host_trust.port) + }; + Ok(LeasedRepositorySshAccess { + credential_id: binding.credential_id.clone(), + credential_revision: credential.current_revision, + host_trust_id: binding.host_trust_id.clone(), + host_trust_revision: host_trust.current_revision, + private_key, + known_hosts_entry: format!("{host} {}\n", host_trust.host_key), + }) + } + + fn unseal( + &self, + workspace_id: &str, + credential_id: &str, + revision: u64, + purpose: &str, + secret: SealedSecret, + ) -> Result> { + let master_key = self.master_key.as_ref().ok_or_else(|| { + Error::Store("Repository secret encryption authority is unavailable".to_string()) + })?; + let unbound = UnboundKey::new(&AES_256_GCM, master_key.as_slice()) + .map_err(|_| Error::Store("Repository secret encryption key is invalid".to_string()))?; + let key = LessSafeKey::new(unbound); + let mut plaintext = secret.ciphertext; + let aad = secret_aad(workspace_id, credential_id, revision, purpose); + let plaintext_len = key + .open_in_place( + Nonce::assume_unique_for_key(secret.nonce), + Aad::from(aad.as_bytes()), + &mut plaintext, + ) + .map_err(|_| Error::Store("Repository secret decryption failed".to_string()))? + .len(); + plaintext.truncate(plaintext_len); + Ok(plaintext) + } + fn seal( &self, workspace_id: &str, @@ -968,6 +1115,45 @@ fn insert_secret( Ok(()) } +fn read_sealed_secret( + conn: &rusqlite::Connection, + workspace_id: &str, + credential_id: &str, + revision: u64, + purpose: &str, +) -> Result> { + let row = conn + .query_row( + r#"SELECT encryption_algorithm, nonce, ciphertext + FROM server_secret_versions + WHERE workspace_id = ?1 AND secret_id = ?2 + AND revision = ?3 AND purpose = ?4"#, + params![workspace_id, credential_id, revision, purpose], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Vec>(1)?, + row.get::<_, Vec>(2)?, + )) + }, + ) + .optional()?; + let Some((algorithm, nonce, ciphertext)) = row else { + return Ok(None); + }; + if algorithm != "aes-256-gcm-v1" || nonce.len() != NONCE_BYTES { + return Err(Error::RegistryInconsistency( + "Repository secret envelope is invalid".to_string(), + )); + } + let mut nonce_bytes = [0u8; NONCE_BYTES]; + nonce_bytes.copy_from_slice(&nonce); + Ok(Some(SealedSecret { + nonce: nonce_bytes, + ciphertext, + })) +} + fn replay_credential_operation( tx: &rusqlite::Transaction<'_>, workspace_id: &str, @@ -1691,6 +1877,17 @@ mod tests { projection.bindings[0].access, RepositoryAccessMode::ReadOnly ); + let lease = service + .lease_ssh_materialization_access("workspace-a", &projection.bindings[0]) + .unwrap(); + assert_eq!(lease.credential_revision, 1); + assert_eq!(lease.host_trust_revision, 1); + assert!(lease.private_key.contains("BEGIN OPENSSH PRIVATE KEY")); + assert!( + lease + .known_hosts_entry + .starts_with("example.test ssh-ed25519 ") + ); let unknown = config_state( r#"{ diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 35919ab4..fbc7df02 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -132,8 +132,10 @@ use crate::store::{ use crate::workspace_catalog::{WorkspaceCatalogService, WorkspaceCreateRequest}; use crate::{Error, Result}; use worker_runtime::catalog::{ - ConfigBundleRef, ProfileSelector, RepositorySelector as RuntimeRepositorySelector, - WorkingDirectoryClaim, WorkingDirectoryRepository, WorkingDirectoryRequest, WorkspaceApiRef, + ConfigBundleRef, ProfileSelector, RepositoryMaterializationContext, + RepositorySelector as RuntimeRepositorySelector, RepositorySshMaterializationAccess, + SensitiveString, WorkingDirectoryClaim, WorkingDirectoryRepository, WorkingDirectoryRequest, + WorkspaceApiRef, }; use worker_runtime::config_bundle::ConfigBundle; use worker_runtime::http_server::{ @@ -8453,6 +8455,8 @@ async fn create_workspace_working_directory( &request.repository_id, selector.as_deref(), requested_runtime_id.as_deref(), + &working_directory_request.repository.source_fingerprint, + working_directory_request.repository.source_revision, ); let reserved = if let Some(existing) = api .config_store @@ -8596,6 +8600,24 @@ async fn create_workspace_working_directory( )); } + let repository_access_projection = active_repository_access_projection(api, workspace_id)?; + if let Err(error) = authorize_repository_materialization( + api, + &reserved.resolved_runtime_id, + &operation_id, + &repository_access_projection, + &mut working_directory_request, + ) { + api.config_store.finish_workdir_create_operation( + workspace_id, + &operation_id, + &request_fingerprint, + false, + Some("working_directory_remote_repository_access_required"), + &now_registry_timestamp(), + )?; + return Err(error); + } working_directory_request.backend_workdir_id = Some(reserved.working_directory_id.clone()); let existing = match api.runtime.working_directory( &reserved.resolved_runtime_id, @@ -10834,8 +10856,9 @@ fn working_directory_request_from_repository( }) .or_else(|| Some(RuntimeRepositorySelector::from("HEAD"))), }, - materializer: MaterializerKind::LocalGitWorktree, + materializer: MaterializerKind::RuntimeGitCache, backend_workdir_id: None, + materialization: None, } } @@ -13917,6 +13940,51 @@ fn validate_working_directory_claim_for_browser( Ok(()) } +fn authorize_repository_materialization( + api: &WorkspaceApi, + runtime_id: &str, + operation_id: &str, + projection: &RepositoryAccessProjection, + request: &mut WorkingDirectoryRequest, +) -> ApiResult<()> { + let ssh = if request.repository.source.kind == workspace_api::RepositorySourceKind::Ssh { + let binding = projection + .bindings + .iter() + .find(|binding| binding.repository_id == request.repository.id) + .ok_or_else(|| { + settings_bad_request( + "working_directory_remote_repository_access_required", + "SSH Repository has no active Workspace credential and host-trust binding", + ) + })?; + let lease = api + .repository_secrets + .lease_ssh_materialization_access(&api.config.workspace_id, binding)?; + Some(RepositorySshMaterializationAccess { + credential_id: lease.credential_id, + credential_revision: lease.credential_revision, + host_trust_id: lease.host_trust_id, + host_trust_revision: lease.host_trust_revision, + access: binding.access, + private_key: SensitiveString::new(lease.private_key.as_str()), + known_hosts_entry: SensitiveString::new(lease.known_hosts_entry), + }) + } else { + None + }; + request.materialization = Some(RepositoryMaterializationContext { + workspace_id: api.config.workspace_id.clone(), + runtime_id: runtime_id.to_string(), + operation_id: operation_id.to_string(), + config_revision: projection.config_revision, + config_projection_digest: projection.projection_digest.clone(), + cache_generation: 0, + ssh, + }); + Ok(()) +} + fn working_directory_request_for_browser( api: &WorkspaceApi, request: BrowserWorkingDirectoryCreateRequest, @@ -13935,8 +14003,9 @@ fn working_directory_request_for_browser( source_fingerprint: repository.source_fingerprint.clone(), selector: selector.map(RuntimeRepositorySelector), }, - materializer: MaterializerKind::LocalGitWorktree, + materializer: MaterializerKind::RuntimeGitCache, backend_workdir_id: None, + materialization: None, }) } @@ -16015,7 +16084,7 @@ mod tests { worker_runtime::execution::WorkerExecutionContext, >, >, - materializer: worker_runtime::working_directory::LocalGitWorktreeMaterializer, + materializer: worker_runtime::working_directory::RuntimeGitCacheMaterializer, spawn_failure: std::sync::Mutex>, input_failure: std::sync::Mutex>, inputs: std::sync::Mutex>, @@ -16035,7 +16104,7 @@ mod tests { ); Self { contexts: std::sync::Mutex::new(std::collections::HashMap::new()), - materializer: worker_runtime::working_directory::LocalGitWorktreeMaterializer::new( + materializer: worker_runtime::working_directory::RuntimeGitCacheMaterializer::new( std::env::temp_dir().join(unique), ), spawn_failure: std::sync::Mutex::new(None), @@ -21083,10 +21152,15 @@ mod tests { init_clean_git_workspace(dir.path()); let api = test_api(dir.path()).await; let operation_id = "provider-rejection-classification"; + let repository = api + .require_configured_workspace_repository(TEST_REPOSITORY_ID) + .unwrap(); let request_fingerprint = crate::workdir_create_operations::request_fingerprint( TEST_REPOSITORY_ID, Some("HEAD"), Some(EMBEDDED_WORKER_RUNTIME_ID), + &repository.source_fingerprint, + repository.source_revision, ); api.config_store .reserve_workdir_create_operation(&WorkdirCreateOperationRecord { diff --git a/crates/workspace-server/src/workdir_create_operations.rs b/crates/workspace-server/src/workdir_create_operations.rs index 02cc2f78..b8b0e25c 100644 --- a/crates/workspace-server/src/workdir_create_operations.rs +++ b/crates/workspace-server/src/workdir_create_operations.rs @@ -8,9 +8,16 @@ pub fn request_fingerprint( repository_id: &str, selector: Option<&str>, requested_runtime_id: Option<&str>, + repository_source_fingerprint: &str, + repository_source_revision: u64, ) -> String { let mut hasher = Sha256::new(); - for value in [Some(repository_id), selector, requested_runtime_id] { + for value in [ + Some(repository_id), + selector, + requested_runtime_id, + Some(repository_source_fingerprint), + ] { match value { Some(value) => { hasher.update([1]); @@ -20,6 +27,7 @@ pub fn request_fingerprint( None => hasher.update([0]), } } + hasher.update(repository_source_revision.to_be_bytes()); let digest = hasher.finalize(); let mut encoded = String::with_capacity(digest.len() * 2); for byte in digest { @@ -201,7 +209,13 @@ mod tests { let record = WorkdirCreateOperationRecord { workspace_id: "workspace".to_string(), operation_id: "call-1".to_string(), - request_fingerprint: request_fingerprint("main", Some("develop"), None), + request_fingerprint: request_fingerprint( + "main", + Some("develop"), + None, + "sha256:test", + 1, + ), repository_id: "main".to_string(), selector: Some("develop".to_string()), requested_runtime_id: None, @@ -234,7 +248,8 @@ mod tests { Some(record.clone()) ); let mut changed_input = record.clone(); - changed_input.request_fingerprint = request_fingerprint("main", Some("main"), None); + changed_input.request_fingerprint = + request_fingerprint("main", Some("main"), None, "sha256:test", 1); assert!( store .reserve_workdir_create_operation(&changed_input) From ffb2a34ae5fc17b5f18109b9205a83b3378c6166 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 14:39:37 +0900 Subject: [PATCH 02/12] fix: enforce repository access and cache boundaries --- crates/worker-runtime/src/catalog.rs | 7 + crates/worker-runtime/src/execution.rs | 22 +- crates/worker-runtime/src/http_server.rs | 41 +- crates/worker-runtime/src/runtime.rs | 21 +- crates/worker-runtime/src/worker_backend.rs | 15 +- .../worker-runtime/src/working_directory.rs | 369 +++++++++++++++--- crates/workspace-server/src/hosts.rs | 54 ++- crates/workspace-server/src/server.rs | 74 ++++ 8 files changed, 533 insertions(+), 70 deletions(-) diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index fd0b8a7e..da18810e 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -129,6 +129,7 @@ pub struct RepositorySshMaterializationAccess { pub host_trust_id: String, pub host_trust_revision: u64, pub access: workspace_api::RepositoryAccessMode, + pub expires_at_epoch_seconds: u64, pub private_key: SensitiveString, pub known_hosts_entry: SensitiveString, } @@ -146,6 +147,12 @@ pub struct RepositoryMaterializationContext { pub ssh: Option, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkingDirectoryRepositoryAccessRequest { + pub working_directory_id: String, + pub materialization: RepositoryMaterializationContext, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkingDirectoryRequest { pub repository: WorkingDirectoryRepository, diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index 642e23e5..c37f9792 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -1,4 +1,6 @@ -use crate::catalog::{WorkingDirectoryRequest, WorkingDirectoryStatus}; +use crate::catalog::{ + WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus, +}; use crate::config_bundle::ConfigBundle; use crate::error::RuntimeError; use crate::identity::WorkerRef; @@ -319,6 +321,16 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { )) } + fn authorize_working_directory_repository_access( + &self, + _request: &WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), WorkingDirectoryDiagnostic> { + Err(WorkingDirectoryDiagnostic::rejected( + "working_directory_repository_access_unsupported", + "Worker execution backend does not support Repository access authorization", + )) + } + fn list_working_directories(&self) -> Vec { Vec::new() } @@ -454,6 +466,14 @@ impl WorkerExecutionBackendRef { self.backend.create_working_directory(request) } + pub(crate) fn authorize_working_directory_repository_access( + &self, + request: &WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), WorkingDirectoryDiagnostic> { + self.backend + .authorize_working_directory_repository_access(request) + } + pub(crate) fn list_working_directories(&self) -> Vec { self.backend.list_working_directories() } diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 0ed02914..4c179663 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -12,7 +12,8 @@ use crate::auth::{ }; use crate::catalog::{ ConfigBundleRef, CreateWorkerRequest, WorkerDetail, WorkerLifecycleAck, WorkerSummary, - WorkingDirectoryRequest, WorkingDirectoryStatus, WorkspaceApiRef, + WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus, + WorkspaceApiRef, }; use crate::config_bundle::{ConfigBundle, ConfigBundleAvailability, ConfigBundleSummary}; use crate::error::RuntimeError; @@ -203,6 +204,10 @@ fn runtime_http_router_with_optional_auth( "/v1/working-directories", get(list_working_directories).post(create_working_directory), ) + .route( + "/v1/working-directories/repository-access", + post(authorize_working_directory_repository_access), + ) .route( "/v1/working-directories/{working_directory_id}/sessions", post(open_workdir_session), @@ -335,6 +340,11 @@ pub struct RuntimeHttpWorkingDirectoriesResponse { pub working_directories: Vec, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RuntimeHttpRepositoryAccessResponse { + pub authorized: bool, +} + /// Working directory response used by create/detail/delete endpoints. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct RuntimeHttpWorkingDirectoryResponse { @@ -513,6 +523,28 @@ async fn list_workers( Ok(Json(RuntimeHttpWorkersResponse { workers })) } +async fn authorize_working_directory_repository_access( + State(state): State, + Extension(auth): Extension, + body: Result, JsonRejection>, +) -> RestResult { + let Json(request) = body.map_err(RuntimeHttpRestError::json_rejection)?; + if request.materialization.workspace_id != auth.workspace_id { + return Err(RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "working_directory_materialization_workspace_mismatch", + "Repository access authority does not match the authenticated Workspace", + )); + } + state + .runtime + .authorize_working_directory_repository_access(request) + .map_err(RuntimeHttpRestError::runtime)?; + Ok(Json(RuntimeHttpRepositoryAccessResponse { + authorized: true, + })) +} + async fn list_working_directories( State(state): State, ) -> RestResult { @@ -1569,6 +1601,9 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s if path == "/v1/workers" && *method == Method::POST { return Some("workers:create"); } + if path == "/v1/working-directories/repository-access" && *method == Method::POST { + return Some("workdirs:operate"); + } if path.starts_with("/v1/workdir-sessions") || (path.starts_with("/v1/working-directories/") && path.ends_with("/sessions")) { @@ -2230,6 +2265,10 @@ mod tests { #[test] fn workdir_routes_require_dedicated_operation_permission() { + assert_eq!( + required_runtime_permission(&Method::POST, "/v1/working-directories/repository-access",), + Some("workdirs:operate") + ); assert_eq!( required_runtime_permission(&Method::POST, "/v1/working-directories/wd-1/sessions"), Some("workdirs:operate") diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 97ad176c..588f3338 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1,6 +1,6 @@ use crate::catalog::{ ConfigBundleRef, CreateWorkerRequest, ProfileSelector, WorkerDetail, WorkerLifecycleAck, - WorkerStatus, WorkerSummary, WorkingDirectoryRequest, + WorkerStatus, WorkerSummary, WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus as CatalogWorkingDirectoryStatus, WorkspaceApiRef, }; use crate::config_bundle::{ @@ -366,6 +366,25 @@ impl Runtime { .map_err(RuntimeError::from) } + pub fn authorize_working_directory_repository_access( + &self, + request: WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), RuntimeError> { + let backend = { + let state = self.lock()?; + state.ensure_running()?; + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::ExecutionBackendUnavailable { + message: "working directory Repository access requires an execution backend" + .to_string(), + } + })? + }; + backend + .authorize_working_directory_repository_access(&request) + .map_err(RuntimeError::from) + } + /// List Runtime-owned working directories through the attached execution backend. pub fn list_working_directories( &self, diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 3f334145..fe245c6f 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -20,7 +20,7 @@ use crate::auth::{ }; use crate::catalog::{ CreateWorkerRequest, ProfileSourceArchiveHttpRef, ProfileSourceArchiveSource, - WorkingDirectoryRequest, WorkingDirectoryStatus, + WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus, }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, @@ -1590,6 +1590,19 @@ where Ok(materializer.create(request)?.status()) } + fn authorize_working_directory_repository_access( + &self, + request: &WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), WorkingDirectoryDiagnostic> { + let materializer = self.working_directory_materializer.as_ref().ok_or_else(|| { + WorkingDirectoryDiagnostic::rejected( + "working_directory_materializer_unavailable", + "working directory Repository access requested, but no materializer is configured for this runtime backend", + ) + })?; + materializer.authorize_repository_access(request) + } + fn list_working_directories(&self) -> Vec { self.working_directory_materializer .as_ref() diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index 7d2d7f44..f2407a6e 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -1,7 +1,7 @@ use crate::catalog::{ MaterializerKind, RepositorySshMaterializationAccess, WorkingDirectoryCleanupTarget, - WorkingDirectoryRequest, WorkingDirectoryStatus, WorkingDirectoryStatusKind, - WorkingDirectorySummary, + WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus, + WorkingDirectoryStatusKind, WorkingDirectorySummary, }; use crate::identity::WorkerRef; use serde::{Deserialize, Serialize}; @@ -48,6 +48,8 @@ pub struct WorkingDirectoryEvidence { pub credential_revision: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub host_trust_revision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transport_warning: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -174,6 +176,11 @@ pub trait WorkingDirectoryMaterializer: Send + Sync + 'static { request: &WorkingDirectoryRequest, ) -> Result; + fn authorize_repository_access( + &self, + request: &WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), WorkingDirectoryDiagnostic>; + fn bind_working_directory( &self, working_directory_id: &str, @@ -379,8 +386,7 @@ impl RuntimeGitCacheMaterializer { "Runtime Repository access state is unavailable", ) })? - .get(working_directory_id) - .cloned(); + .remove(working_directory_id); let Some(access) = access else { if binding .working_directory @@ -395,11 +401,26 @@ impl RuntimeGitCacheMaterializer { } return Ok(binding); }; + validate_ssh_materialization_access(&access)?; let agent = Arc::new(RepositorySshAgent::start( &self.runtime_root, working_directory_id, &access, )?); + let weak_agent = Arc::downgrade(&agent); + let expires_at = access.expires_at_epoch_seconds; + std::thread::spawn(move || { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + if expires_at > now { + std::thread::sleep(Duration::from_secs(expires_at - now)); + } + if let Some(agent) = weak_agent.upgrade() { + agent.stop(); + } + }); binding.command_environment.insert( "SSH_AUTH_SOCK".to_string(), agent.socket.to_string_lossy().to_string(), @@ -436,7 +457,9 @@ impl RuntimeGitCacheMaterializer { } if matches!( request.repository.source.kind, - workspace_api::RepositorySourceKind::Https | workspace_api::RepositorySourceKind::Ssh + workspace_api::RepositorySourceKind::Https + | workspace_api::RepositorySourceKind::Http + | workspace_api::RepositorySourceKind::Ssh ) { validate_remote_source_uri(request)?; let materialization = request.materialization.as_ref().ok_or_else(|| { @@ -460,7 +483,8 @@ impl RuntimeGitCacheMaterializer { match request.repository.source.kind { workspace_api::RepositorySourceKind::LocalPath | workspace_api::RepositorySourceKind::File - | workspace_api::RepositorySourceKind::Https => {} + | workspace_api::RepositorySourceKind::Https + | workspace_api::RepositorySourceKind::Http => {} workspace_api::RepositorySourceKind::Ssh => { let ssh = request .materialization @@ -474,12 +498,6 @@ impl RuntimeGitCacheMaterializer { })?; validate_ssh_materialization_access(ssh)?; } - workspace_api::RepositorySourceKind::Http => { - return Err(WorkingDirectoryDiagnostic::new( - "working_directory_insecure_repository_transport_rejected", - "plain HTTP Repository materialization is rejected", - )); - } workspace_api::RepositorySourceKind::Invalid => { return Err(WorkingDirectoryDiagnostic::new( "working_directory_repository_source_invalid", @@ -539,12 +557,7 @@ impl RuntimeGitCacheMaterializer { "Runtime Repository cache identity does not match the requested source", )); } - let mut command = repository_git_command(request, access.as_ref()); - command - .arg("--git-dir") - .arg(&cache_path) - .args(["remote", "update", "--prune"]); - run_repository_git(command, "working_directory_repository_fetch_failed")?; + fetch_repository_cache(request, access.as_ref(), &cache_path)?; } else { let staging = cache_path.with_extension(format!( "staging-{}", @@ -553,15 +566,42 @@ impl RuntimeGitCacheMaterializer { if staging.exists() { let _ = fs::remove_dir_all(&staging); } - let mut command = repository_git_command(request, access.as_ref()); - command.args(["clone", "--mirror"]); - if request.repository.source.kind == workspace_api::RepositorySourceKind::LocalPath { - command.arg("--no-local"); - } - command.arg(&request.repository.source.uri).arg(&staging); - if let Err(error) = - run_repository_git(command, "working_directory_repository_fetch_failed") - { + fs::create_dir_all(&staging).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_cache_create_failed", + "Runtime Repository cache could not be created; backend-private path details were omitted", + ) + })?; + let mut init = isolated_git_command(); + init.args(["init", "--bare"]).arg(&staging); + let mut add_origin = repository_git_command(request, access.as_ref()); + add_origin + .arg("--git-dir") + .arg(&staging) + .args(["remote", "add", "origin"]) + .arg(&request.repository.source.uri); + let mut configure_fetch = isolated_git_command(); + configure_fetch.arg("--git-dir").arg(&staging).args([ + "config", + "remote.origin.fetch", + "+refs/heads/*:refs/remotes/origin/*", + ]); + let initialized = + run_repository_git(init, "working_directory_repository_cache_create_failed") + .and_then(|_| { + run_repository_git( + add_origin, + "working_directory_repository_cache_create_failed", + ) + }) + .and_then(|_| { + run_repository_git( + configure_fetch, + "working_directory_repository_cache_create_failed", + ) + }) + .and_then(|_| fetch_repository_cache(request, access.as_ref(), &staging)); + if let Err(error) = initialized { let _ = fs::remove_dir_all(&staging); return Err(error); } @@ -590,9 +630,7 @@ impl RuntimeGitCacheMaterializer { validate_working_directory_id(&working_directory_id)?; let repository_cache = self.ensure_repository_cache(request)?; let selector = request.repository.selector.as_deref().unwrap_or("HEAD"); - let commit_spec = format!("{selector}^{{commit}}"); - let resolved_commit = - git_dir_stdout(&repository_cache, ["rev-parse", commit_spec.as_str()])?; + let resolved_commit = resolve_cached_commit(&repository_cache, selector)?; let tree_spec = format!("{resolved_commit}^{{tree}}"); let resolved_tree = git_dir_stdout(&repository_cache, ["rev-parse", tree_spec.as_str()]) .ok() @@ -682,6 +720,8 @@ impl RuntimeGitCacheMaterializer { host_trust_revision: context .and_then(|value| value.ssh.as_ref()) .map(|value| value.host_trust_revision), + transport_warning: repository_transport_warning(request.repository.source.kind) + .map(str::to_string), }, cleanup_target: WorkingDirectoryCleanupTarget { kind: "runtime_git_cache_worktree".to_string(), @@ -704,24 +744,6 @@ impl RuntimeGitCacheMaterializer { let _ = fs::remove_dir_all(&working_directory_root); return Err(error); } - if let Some(ssh) = request - .materialization - .as_ref() - .and_then(|materialization| materialization.ssh.clone()) - { - let mut repository_access = match self.repository_access.lock() { - Ok(repository_access) => repository_access, - Err(_) => { - remove_cached_worktree(&repository_cache, &worktree_root); - let _ = fs::remove_dir_all(&working_directory_root); - return Err(WorkingDirectoryDiagnostic::new( - "working_directory_repository_access_unavailable", - "Runtime Repository access state is unavailable", - )); - } - }; - repository_access.insert(binding.working_directory.id.clone(), ssh); - } Ok(binding) } } @@ -747,6 +769,67 @@ impl WorkingDirectoryMaterializer for RuntimeGitCacheMaterializer { self.materialize_with_working_directory_id(working_directory_id, request) } + fn authorize_repository_access( + &self, + request: &WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), WorkingDirectoryDiagnostic> { + validate_working_directory_id(&request.working_directory_id)?; + let ssh = request.materialization.ssh.as_ref().ok_or_else(|| { + WorkingDirectoryDiagnostic::new( + "working_directory_remote_repository_access_required", + "SSH Repository access authority is required", + ) + })?; + validate_ssh_materialization_access(ssh)?; + let mut binding = self.read_binding(&request.working_directory_id)?; + if binding + .working_directory + .evidence + .credential_revision + .is_none() + { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_not_applicable", + "Workdir is not backed by an SSH Repository", + )); + } + binding.working_directory.evidence.operation_id = + Some(request.materialization.operation_id.clone()); + binding.working_directory.evidence.credential_revision = Some(ssh.credential_revision); + binding.working_directory.evidence.host_trust_revision = Some(ssh.host_trust_revision); + self.write_record(&binding)?; + let working_directory_id = request.working_directory_id.clone(); + let credential_revision = ssh.credential_revision; + let expires_at = ssh.expires_at_epoch_seconds; + self.repository_access + .lock() + .map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_unavailable", + "Runtime Repository access state is unavailable", + ) + })? + .insert(working_directory_id.clone(), ssh.clone()); + let repository_access = self.repository_access.clone(); + std::thread::spawn(move || { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + if expires_at > now { + std::thread::sleep(Duration::from_secs(expires_at - now)); + } + if let Ok(mut access) = repository_access.lock() + && access + .get(&working_directory_id) + .is_some_and(|access| access.credential_revision == credential_revision) + { + access.remove(&working_directory_id); + } + }); + Ok(()) + } + fn bind_working_directory( &self, working_directory_id: &str, @@ -1021,10 +1104,8 @@ impl RepositorySshAgent { } Ok(agent) } -} -impl Drop for RepositorySshAgent { - fn drop(&mut self) { + fn stop(&self) { if let Ok(mut child) = self.child.lock() && let Some(mut child) = child.take() { @@ -1035,9 +1116,16 @@ impl Drop for RepositorySshAgent { } } +impl Drop for RepositorySshAgent { + fn drop(&mut self) { + self.stop(); + } +} + struct RepositoryCommandAccess { root: PathBuf, ssh_command: PathBuf, + agent: RepositorySshAgent, } impl RepositoryCommandAccess { @@ -1045,6 +1133,9 @@ impl RepositoryCommandAccess { runtime_root: &Path, request: &WorkingDirectoryRequest, ) -> Result, WorkingDirectoryDiagnostic> { + if request.repository.source.kind != workspace_api::RepositorySourceKind::Ssh { + return Ok(None); + } let Some(ssh) = request .materialization .as_ref() @@ -1069,19 +1160,21 @@ impl RepositoryCommandAccess { ) })?; set_directory_owner_only(&root)?; - let private_key = root.join("identity"); let known_hosts = root.join("known_hosts"); let ssh_command = root.join("ssh-command"); - write_owner_only(&private_key, ssh.private_key.expose().as_bytes())?; write_owner_only(&known_hosts, ssh.known_hosts_entry.expose().as_bytes())?; let script = format!( - "#!/bin/sh\nexec ssh -F /dev/null -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile={} -i {} \"$@\"\n", + "#!/bin/sh\nexec ssh -F /dev/null -o BatchMode=yes -o IdentitiesOnly=no -o IdentityFile=/dev/null -o StrictHostKeyChecking=yes -o UserKnownHostsFile={} \"$@\"\n", shell_quote_path(&known_hosts)?, - shell_quote_path(&private_key)?, ); write_owner_only(&ssh_command, script.as_bytes())?; set_file_owner_executable(&ssh_command)?; - Ok(Some(Self { root, ssh_command })) + let agent = RepositorySshAgent::start(runtime_root, operation_id, ssh)?; + Ok(Some(Self { + root, + ssh_command, + agent, + })) } } @@ -1094,6 +1187,16 @@ impl Drop for RepositoryCommandAccess { fn validate_ssh_materialization_access( access: &RepositorySshMaterializationAccess, ) -> Result<(), WorkingDirectoryDiagnostic> { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + if access.expires_at_epoch_seconds <= now { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_expired", + "operation-scoped SSH credential and host-trust authority has expired", + )); + } if access.credential_id.trim().is_empty() || access.credential_revision == 0 || access.host_trust_id.trim().is_empty() @@ -1109,6 +1212,10 @@ fn validate_ssh_materialization_access( Ok(()) } +fn repository_transport_warning(kind: workspace_api::RepositorySourceKind) -> Option<&'static str> { + (kind == workspace_api::RepositorySourceKind::Http).then_some("plain_http_transport") +} + fn validate_remote_source_uri( request: &WorkingDirectoryRequest, ) -> Result<(), WorkingDirectoryDiagnostic> { @@ -1120,6 +1227,7 @@ fn validate_remote_source_uri( })?; let expected_scheme = match request.repository.source.kind { workspace_api::RepositorySourceKind::Https => "https", + workspace_api::RepositorySourceKind::Http => "http", workspace_api::RepositorySourceKind::Ssh => "ssh", _ => return Ok(()), }; @@ -1128,7 +1236,7 @@ fn validate_remote_source_uri( || url.password().is_some() || !url.query().is_none() || !url.fragment().is_none() - || (expected_scheme == "https" && !url.username().is_empty()) + || (matches!(expected_scheme, "https" | "http") && !url.username().is_empty()) { return Err(WorkingDirectoryDiagnostic::new( "working_directory_repository_source_invalid", @@ -1223,7 +1331,9 @@ fn repository_git_command( }; command.args(["-c", &format!("protocol.file.allow={file_policy}")]); if let Some(access) = access { - command.env("GIT_SSH_COMMAND", &access.ssh_command); + command + .env("GIT_SSH_COMMAND", &access.ssh_command) + .env("SSH_AUTH_SOCK", &access.agent.socket); } command } @@ -1273,6 +1383,57 @@ fn run_repository_git( } } +fn resolve_cached_commit( + repository_cache: &Path, + selector: &str, +) -> Result { + let mut candidates = Vec::new(); + if selector == "HEAD" { + candidates.push("FETCH_HEAD".to_string()); + } else if let Some(branch) = selector.strip_prefix("refs/heads/") { + candidates.push(format!("refs/remotes/origin/{branch}")); + } else if selector.starts_with("refs/") || selector.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + candidates.push(selector.to_string()); + } else { + candidates.push(format!("refs/remotes/origin/{selector}")); + candidates.push(selector.to_string()); + } + for candidate in candidates { + let spec = format!("{candidate}^{{commit}}"); + if let Ok(commit) = git_dir_stdout(repository_cache, ["rev-parse", spec.as_str()]) + && !commit.is_empty() + { + return Ok(commit); + } + } + Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_selector_unresolved", + "configured Repository selector could not be resolved to a commit", + )) +} + +fn fetch_repository_cache( + request: &WorkingDirectoryRequest, + access: Option<&RepositoryCommandAccess>, + repository_cache: &Path, +) -> Result<(), WorkingDirectoryDiagnostic> { + let mut refs = repository_git_command(request, access); + refs.arg("--git-dir").arg(repository_cache).args([ + "fetch", + "--prune", + "--tags", + "origin", + "+refs/heads/*:refs/remotes/origin/*", + ]); + let mut head = repository_git_command(request, access); + head.arg("--git-dir") + .arg(repository_cache) + .args(["fetch", "--no-tags", "origin", "HEAD"]); + run_repository_git(refs, "working_directory_repository_fetch_failed")?; + run_repository_git(head, "working_directory_repository_fetch_failed") +} + fn validate_repository_cache_limits( repository_cache: &Path, ) -> Result<(), WorkingDirectoryDiagnostic> { @@ -1693,6 +1854,21 @@ mod tests { local.working_directory.evidence.repository_cache_key, second.working_directory.evidence.repository_cache_key ); + assert_eq!( + git_dir_stdout( + local.source_repository_path(), + ["config", "--get-all", "remote.origin.fetch"], + ) + .unwrap(), + "+refs/heads/*:refs/remotes/origin/*" + ); + assert!( + git_dir_stdout( + local.source_repository_path(), + ["config", "--get", "remote.origin.mirror"], + ) + .is_err() + ); assert_eq!( fs::read_dir(runtime_root.path().join(REPOSITORY_CACHE_DIR)) .unwrap() @@ -1754,6 +1930,7 @@ mod tests { host_trust_id: "trust-1".to_string(), host_trust_revision: 4, access: workspace_api::RepositoryAccessMode::ReadOnly, + expires_at_epoch_seconds: u64::MAX, private_key: crate::catalog::SensitiveString::new("PRIVATE KEY secret bytes"), known_hosts_entry: crate::catalog::SensitiveString::new("host key secret bytes"), }; @@ -1790,6 +1967,7 @@ mod tests { host_trust_id: "trust-1".to_string(), host_trust_revision: 1, access: workspace_api::RepositoryAccessMode::ReadWrite, + expires_at_epoch_seconds: u64::MAX, private_key: crate::catalog::SensitiveString::new( fs::read_to_string(&key_path).unwrap(), ), @@ -1798,10 +1976,35 @@ mod tests { ), }), }); + let mut ssh_operation = request.clone(); + ssh_operation.repository.source = workspace_api::RepositorySource { + kind: workspace_api::RepositorySourceKind::Ssh, + uri: "ssh://git@example.test/repo.git".to_string(), + }; + let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation) + .unwrap() + .unwrap(); + assert!(!command_access.root.join("identity").exists()); + assert!(command_access.agent.socket.exists()); + let operation_socket = command_access.agent.socket.clone(); + drop(command_access); + assert!(!operation_socket.exists()); + let created = materializer.create(&request).unwrap(); let id = created.working_directory.id; assert_eq!(materializer.list_working_directories().unwrap().len(), 1); - assert!(!runtime_root.path().join(".repository-agents").exists()); + assert_eq!( + fs::read_dir(runtime_root.path().join(".repository-agents")) + .map(|entries| entries.count()) + .unwrap_or_default(), + 0 + ); + materializer + .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { + working_directory_id: id.clone(), + materialization: request.materialization.clone().unwrap(), + }) + .unwrap(); let binding = materializer.bind_working_directory(&id, None).unwrap(); let socket = PathBuf::from(binding.command_environment()["SSH_AUTH_SOCK"].clone()); assert!(socket.exists()); @@ -1811,6 +2014,40 @@ mod tests { ); drop(binding); assert!(!socket.exists()); + assert_eq!( + materializer + .bind_working_directory(&id, None) + .unwrap_err() + .code, + "working_directory_remote_repository_access_required" + ); + let mut rotated = request.materialization.clone().unwrap(); + rotated.operation_id = "operation-agent-rotated".to_string(); + rotated.ssh.as_mut().unwrap().credential_revision = 2; + materializer + .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { + working_directory_id: id.clone(), + materialization: rotated, + }) + .unwrap(); + let rebound = materializer.bind_working_directory(&id, None).unwrap(); + assert_eq!( + rebound.working_directory.evidence.credential_revision, + Some(2) + ); + drop(rebound); + let mut expired = request.materialization.clone().unwrap(); + expired.ssh.as_mut().unwrap().expires_at_epoch_seconds = 1; + assert_eq!( + materializer + .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { + working_directory_id: id.clone(), + materialization: expired, + }) + .unwrap_err() + .code, + "working_directory_repository_access_expired" + ); let restored = RuntimeGitCacheMaterializer::new(runtime_root.path()); assert_eq!( restored.bind_working_directory(&id, None).unwrap_err().code, @@ -1837,6 +2074,7 @@ mod tests { host_trust_id: "trust-1".to_string(), host_trust_revision: 1, access: workspace_api::RepositoryAccessMode::ReadOnly, + expires_at_epoch_seconds: u64::MAX, private_key: crate::catalog::SensitiveString::new("PRIVATE KEY placeholder"), known_hosts_entry: crate::catalog::SensitiveString::new( "example.test ssh-ed25519 placeholder", @@ -1902,6 +2140,18 @@ mod tests { "working_directory_repository_source_invalid" ); + let mut http = request(repo.path()); + http.repository.source = workspace_api::RepositorySource { + kind: workspace_api::RepositorySourceKind::Http, + uri: "http://example.test/repo.git".to_string(), + }; + http.materialization = Some(context(None)); + RuntimeGitCacheMaterializer::validate_request(&http).unwrap(); + assert_eq!( + repository_transport_warning(http.repository.source.kind), + Some("plain_http_transport") + ); + let mut ssh = request(repo.path()); ssh.repository.source = workspace_api::RepositorySource { kind: workspace_api::RepositorySourceKind::Ssh, @@ -1914,6 +2164,7 @@ mod tests { host_trust_id: "trust-1".to_string(), host_trust_revision: 1, access: workspace_api::RepositoryAccessMode::ReadOnly, + expires_at_epoch_seconds: u64::MAX, private_key: crate::catalog::SensitiveString::new("PRIVATE KEY placeholder"), known_hosts_entry: crate::catalog::SensitiveString::new( "other.test ssh-ed25519 placeholder", diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 87cf4261..4c67e799 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -25,8 +25,9 @@ use worker_runtime::auth::{CapabilityTokenSigner, capability_claims}; use worker_runtime::catalog::{ ConfigBundleRef, CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveHttpRef, ProfileSourceArchiveSource, WorkerDetail as EmbeddedWorkerDetail, - WorkerStatus as EmbeddedWorkerStatus, WorkingDirectoryClaim, WorkingDirectoryRequest, - WorkingDirectoryStatus, WorkingDirectorySummary, WorkspaceApiRef, + WorkerStatus as EmbeddedWorkerStatus, WorkingDirectoryClaim, + WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus, + WorkingDirectorySummary, WorkspaceApiRef, }; use worker_runtime::config_bundle::{ConfigBundle, ConfigBundleAvailability, ConfigBundleSummary}; #[cfg(test)] @@ -39,11 +40,11 @@ use worker_runtime::execution::WorkerExecutionRunState; use worker_runtime::fs_store::FsRuntimeStoreOptions; use worker_runtime::http_server::{ RuntimeHttpConfigBundleAvailabilityResponse, RuntimeHttpConfigBundleSyncRequest, - RuntimeHttpErrorResponse, RuntimeHttpSummaryResponse, RuntimeHttpWorkerCompletionsRequest, - RuntimeHttpWorkerCompletionsResponse, RuntimeHttpWorkerDeleteResponse, - RuntimeHttpWorkerInputResponse, RuntimeHttpWorkerLifecycleRequest, - RuntimeHttpWorkerLifecycleResponse, RuntimeHttpWorkerResponse, - RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse, + RuntimeHttpErrorResponse, RuntimeHttpRepositoryAccessResponse, RuntimeHttpSummaryResponse, + RuntimeHttpWorkerCompletionsRequest, RuntimeHttpWorkerCompletionsResponse, + RuntimeHttpWorkerDeleteResponse, RuntimeHttpWorkerInputResponse, + RuntimeHttpWorkerLifecycleRequest, RuntimeHttpWorkerLifecycleResponse, + RuntimeHttpWorkerResponse, RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse, RuntimeHttpWorkingDirectoriesResponse, RuntimeHttpWorkingDirectoryResponse, RuntimeHttpWorkspacePromptProjectionRequest, RuntimeHttpWorkspacePromptProjectionResponse, }; @@ -854,6 +855,16 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { } } + fn authorize_working_directory_repository_access( + &self, + _request: WorkingDirectoryRepositoryAccessRequest, + ) -> std::result::Result<(), Error> { + Err(Error::InvalidInput( + "Runtime does not support working directory Repository access authorization" + .to_string(), + )) + } + fn list_working_directories(&self) -> RuntimeList { RuntimeList::new(Vec::new(), Vec::new()) } @@ -1427,6 +1438,23 @@ impl RuntimeRegistry { Ok(runtime.create_working_directory(request)) } + pub fn authorize_working_directory_repository_access( + &self, + runtime_id: &str, + request: WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), RuntimeRegistryError> { + validate_backend_identifier("runtime_id", runtime_id)?; + validate_backend_identifier("working_directory_id", &request.working_directory_id)?; + let runtime = self.runtime(runtime_id)?; + runtime + .authorize_working_directory_repository_access(request) + .map_err(|error| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: runtime_id.to_string(), + code: "working_directory_repository_access_failed".to_string(), + message: error.to_string(), + }) + } + pub fn list_working_directories( &self, runtime_id: &str, @@ -3205,6 +3233,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { } } + fn authorize_working_directory_repository_access( + &self, + request: WorkingDirectoryRepositoryAccessRequest, + ) -> std::result::Result<(), Error> { + self.post_json::<_, RuntimeHttpRepositoryAccessResponse>( + "/v1/working-directories/repository-access", + &request, + ) + .map(|_| ()) + .map_err(|diagnostic| Error::RegistryInconsistency(diagnostic.message)) + } + fn list_working_directories(&self) -> RuntimeList { match self.get_json::("/v1/working-directories") { Ok(response) => RuntimeList::new(response.working_directories, Vec::new()), diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index fbc7df02..660b9245 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1437,6 +1437,18 @@ impl WorkspaceApi { self.validate_worker_spawn_repository_scope(&request)?; let workspace_api = self.workspace_api_ref(runtime_id); request.resolved_workspace_api = Some(workspace_api.clone()); + if let Some(working_directory) = request.resolved_working_directory.as_ref() + && let Some(access) = repository_access_request_for_workdir( + self, + runtime_id, + &working_directory.working_directory_id, + &format!("worker-spawn:{}", WorkerId::now_v7()), + )? + { + self.runtime + .authorize_working_directory_repository_access(runtime_id, access) + .map_err(RuntimeRegistryError::into_error)?; + } let attachment_reservation = request .resolved_working_directory @@ -1652,6 +1664,22 @@ impl WorkspaceApi { &self, worker: &RuntimeWorkerRef, ) -> ApiResult { + if let Some(link) = self + .store + .list_worker_workdir_links(&self.config.workspace_id, worker)? + .into_iter() + .find(|link| link.unlinked_at.is_none()) + && let Some(access) = repository_access_request_for_workdir( + self, + &worker.runtime_id, + &link.workdir_id, + &format!("worker-restore:{}", WorkerId::now_v7()), + )? + { + self.runtime + .authorize_working_directory_repository_access(&worker.runtime_id, access) + .map_err(RuntimeRegistryError::into_error)?; + } let binding = self .runtime .replace_worker_workspace_api(worker, self.workspace_api_ref(&worker.runtime_id)) @@ -13967,6 +13995,11 @@ fn authorize_repository_materialization( host_trust_id: lease.host_trust_id, host_trust_revision: lease.host_trust_revision, access: binding.access, + expires_at_epoch_seconds: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .saturating_add(300), private_key: SensitiveString::new(lease.private_key.as_str()), known_hosts_entry: SensitiveString::new(lease.known_hosts_entry), }) @@ -13985,6 +14018,47 @@ fn authorize_repository_materialization( Ok(()) } +fn repository_access_request_for_workdir( + api: &WorkspaceApi, + runtime_id: &str, + working_directory_id: &str, + operation_id: &str, +) -> ApiResult> { + let record = api + .config_store + .get_workdir_registry(&api.config.workspace_id, working_directory_id)? + .ok_or_else(|| { + settings_bad_request( + "working_directory_not_found", + "Working directory is not registered in this Workspace", + ) + })?; + if record.runtime_id != runtime_id { + return Err(settings_bad_request( + "working_directory_runtime_mismatch", + "Working directory is owned by a different Runtime", + )); + } + let repository = api.require_configured_workspace_repository(&record.repository_id)?; + if repository.source.kind != workspace_api::RepositorySourceKind::Ssh { + return Ok(None); + } + let projection = active_repository_access_projection(api, &api.config.workspace_id)?; + let mut request = working_directory_request_from_repository(&repository, None); + authorize_repository_materialization(api, runtime_id, operation_id, &projection, &mut request)?; + Ok(Some( + worker_runtime::catalog::WorkingDirectoryRepositoryAccessRequest { + working_directory_id: working_directory_id.to_string(), + materialization: request.materialization.ok_or_else(|| { + settings_bad_request( + "working_directory_remote_repository_access_required", + "SSH Repository access authority is unavailable", + ) + })?, + }, + )) +} + fn working_directory_request_for_browser( api: &WorkspaceApi, request: BrowserWorkingDirectoryCreateRequest, From 471db64bcc686ad02c338b3219792c3deb736729 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 15:21:52 +0900 Subject: [PATCH 03/12] fix: preserve repository access across retries --- .../worker-runtime/src/working_directory.rs | 61 ++++-- .../workspace-server/src/repository_access.rs | 92 ++++++-- crates/workspace-server/src/server.rs | 200 +++++++++++++++++- crates/workspace-server/src/store.rs | 133 ++++++++++-- .../src/workdir_create_operations.rs | 168 +++++++++++++-- 5 files changed, 570 insertions(+), 84 deletions(-) diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index f2407a6e..79b7817b 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -402,12 +402,13 @@ impl RuntimeGitCacheMaterializer { return Ok(binding); }; validate_ssh_materialization_access(&access)?; - let agent = Arc::new(RepositorySshAgent::start( + let command_access = Arc::new(RepositoryCommandAccess::prepare_ssh( &self.runtime_root, - working_directory_id, + &format!("attachment-{working_directory_id}"), + &binding.working_directory.repository_id, &access, )?); - let weak_agent = Arc::downgrade(&agent); + let weak_access = Arc::downgrade(&command_access); let expires_at = access.expires_at_epoch_seconds; std::thread::spawn(move || { let now = SystemTime::now() @@ -417,13 +418,17 @@ impl RuntimeGitCacheMaterializer { if expires_at > now { std::thread::sleep(Duration::from_secs(expires_at - now)); } - if let Some(agent) = weak_agent.upgrade() { - agent.stop(); + if let Some(access) = weak_access.upgrade() { + access.stop(); } }); binding.command_environment.insert( "SSH_AUTH_SOCK".to_string(), - agent.socket.to_string_lossy().to_string(), + command_access.agent.socket.to_string_lossy().to_string(), + ); + binding.command_environment.insert( + "GIT_SSH_COMMAND".to_string(), + command_access.ssh_command.to_string_lossy().to_string(), ); binding.command_environment.insert( "YOI_REPOSITORY_ACCESS".to_string(), @@ -433,7 +438,7 @@ impl RuntimeGitCacheMaterializer { } .to_string(), ); - binding.session_resources.push(agent); + binding.session_resources.push(command_access); Ok(binding) } @@ -1122,6 +1127,7 @@ impl Drop for RepositorySshAgent { } } +#[derive(Debug)] struct RepositoryCommandAccess { root: PathBuf, ssh_command: PathBuf, @@ -1148,10 +1154,24 @@ impl RepositoryCommandAccess { .as_ref() .map(|materialization| materialization.operation_id.as_str()) .unwrap_or("operation"); + Ok(Some(Self::prepare_ssh( + runtime_root, + operation_id, + &request.repository.id, + ssh, + )?)) + } + + fn prepare_ssh( + runtime_root: &Path, + operation_id: &str, + repository_id: &str, + ssh: &RepositorySshMaterializationAccess, + ) -> Result { let root = runtime_root.join(REPOSITORY_ACCESS_DIR).join(format!( "{}-{}", sanitize_path_component(operation_id), - next_working_directory_id(&request.repository.id) + next_working_directory_id(repository_id) )); fs::create_dir_all(&root).map_err(|_| { WorkingDirectoryDiagnostic::new( @@ -1170,17 +1190,22 @@ impl RepositoryCommandAccess { write_owner_only(&ssh_command, script.as_bytes())?; set_file_owner_executable(&ssh_command)?; let agent = RepositorySshAgent::start(runtime_root, operation_id, ssh)?; - Ok(Some(Self { + Ok(Self { root, ssh_command, agent, - })) + }) + } + + fn stop(&self) { + self.agent.stop(); + let _ = fs::remove_dir_all(&self.root); } } impl Drop for RepositoryCommandAccess { fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.root); + self.stop(); } } @@ -2006,14 +2031,18 @@ mod tests { }) .unwrap(); let binding = materializer.bind_working_directory(&id, None).unwrap(); - let socket = PathBuf::from(binding.command_environment()["SSH_AUTH_SOCK"].clone()); + let environment = binding.command_environment(); + let socket = PathBuf::from(environment["SSH_AUTH_SOCK"].clone()); + let ssh_command = PathBuf::from(environment["GIT_SSH_COMMAND"].clone()); assert!(socket.exists()); - assert_eq!( - binding.command_environment()["YOI_REPOSITORY_ACCESS"], - "read_write" - ); + assert!(ssh_command.exists()); + let ssh_policy = fs::read_to_string(&ssh_command).unwrap(); + assert!(ssh_policy.contains("StrictHostKeyChecking=yes")); + assert!(ssh_policy.contains("UserKnownHostsFile=")); + assert_eq!(environment["YOI_REPOSITORY_ACCESS"], "read_write"); drop(binding); assert!(!socket.exists()); + assert!(!ssh_command.exists()); assert_eq!( materializer .bind_working_directory(&id, None) diff --git a/crates/workspace-server/src/repository_access.rs b/crates/workspace-server/src/repository_access.rs index 9736a07c..9c642ac9 100644 --- a/crates/workspace-server/src/repository_access.rs +++ b/crates/workspace-server/src/repository_access.rs @@ -865,33 +865,73 @@ impl RepositorySecretService { binding.host_trust_id )) })?; - let (private_key, passphrase) = self.store.with_conn(|conn| { + self.lease_ssh_materialization_access_revision( + workspace_id, + &binding.credential_id, + credential.current_revision, + &binding.host_trust_id, + host_trust.current_revision, + ) + } + + pub fn lease_ssh_materialization_access_revision( + &self, + workspace_id: &str, + credential_id: &str, + credential_revision: u64, + host_trust_id: &str, + host_trust_revision: u64, + ) -> Result { + let (private_key, passphrase, hostname, port, host_key) = self.store.with_conn(|conn| { let private_key = read_sealed_secret( conn, workspace_id, - &binding.credential_id, - credential.current_revision, + credential_id, + credential_revision, "private_key", )? .ok_or_else(|| { Error::RegistryInconsistency(format!( - "Repository SSH credential `{}` is missing its private-key revision", - binding.credential_id + "Repository SSH credential `{credential_id}` revision {credential_revision} is unavailable" )) })?; let passphrase = read_sealed_secret( conn, workspace_id, - &binding.credential_id, - credential.current_revision, + credential_id, + credential_revision, "passphrase", )?; - Ok((private_key, passphrase)) + let (hostname, port, host_key) = conn + .query_row( + r#"SELECT h.hostname, h.port, v.host_key + FROM repository_ssh_host_trusts h + JOIN repository_ssh_host_trust_revisions v + ON v.workspace_id = h.workspace_id + AND v.host_trust_id = h.host_trust_id + WHERE h.workspace_id = ?1 AND h.host_trust_id = ?2 + AND v.revision = ?3"#, + params![workspace_id, host_trust_id, host_trust_revision as i64], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)? as u16, + row.get::<_, String>(2)?, + )) + }, + ) + .optional()? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Repository SSH host trust `{host_trust_id}` revision {host_trust_revision} is unavailable" + )) + })?; + Ok((private_key, passphrase, hostname, port, host_key)) })?; let private_key = self.unseal( workspace_id, - &binding.credential_id, - credential.current_revision, + credential_id, + credential_revision, "private_key", private_key, )?; @@ -899,8 +939,8 @@ impl RepositorySecretService { .map(|secret| { self.unseal( workspace_id, - &binding.credential_id, - credential.current_revision, + credential_id, + credential_revision, "passphrase", secret, ) @@ -933,18 +973,18 @@ impl RepositorySecretService { let private_key = key .to_openssh(LineEnding::LF) .map_err(|_| Error::Store("Repository SSH private key encoding failed".to_string()))?; - let host = if host_trust.port == 22 { - host_trust.hostname.clone() + let host = if port == 22 { + hostname } else { - format!("[{}]:{}", host_trust.hostname, host_trust.port) + format!("[{hostname}]:{port}") }; Ok(LeasedRepositorySshAccess { - credential_id: binding.credential_id.clone(), - credential_revision: credential.current_revision, - host_trust_id: binding.host_trust_id.clone(), - host_trust_revision: host_trust.current_revision, + credential_id: credential_id.to_string(), + credential_revision, + host_trust_id: host_trust_id.to_string(), + host_trust_revision, private_key, - known_hosts_entry: format!("{host} {}\n", host_trust.host_key), + known_hosts_entry: format!("{host} {host_key}\n"), }) } @@ -1888,6 +1928,18 @@ mod tests { .known_hosts_entry .starts_with("example.test ssh-ed25519 ") ); + let exact = service + .lease_ssh_materialization_access_revision( + "workspace-a", + "deploy", + lease.credential_revision, + "example", + lease.host_trust_revision, + ) + .unwrap(); + assert_eq!(exact.credential_revision, lease.credential_revision); + assert_eq!(exact.host_trust_revision, lease.host_trust_revision); + assert_eq!(exact.known_hosts_entry, lease.known_hosts_entry); let unknown = config_state( r#"{ diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 660b9245..bdc953df 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -8474,6 +8474,34 @@ async fn create_workspace_working_directory( ) .into()); } + if let Some(existing) = api + .config_store + .load_workdir_create_operation(workspace_id, &operation_id)? + && let (Some(kind), Some(uri), Some(revision), Some(fingerprint)) = ( + existing.source_kind.as_deref(), + existing.source_uri, + existing.source_revision, + existing.source_fingerprint, + ) + { + let kind = match kind { + "local_path" => workspace_api::RepositorySourceKind::LocalPath, + "file" => workspace_api::RepositorySourceKind::File, + "https" => workspace_api::RepositorySourceKind::Https, + "http" => workspace_api::RepositorySourceKind::Http, + "ssh" => workspace_api::RepositorySourceKind::Ssh, + "invalid" => workspace_api::RepositorySourceKind::Invalid, + _ => { + return Err(settings_bad_request( + "working_directory_repository_source_invalid", + "persisted Workdir create Repository source kind is invalid", + )); + } + }; + working_directory_request.repository.source = workspace_api::RepositorySource { kind, uri }; + working_directory_request.repository.source_revision = revision; + working_directory_request.repository.source_fingerprint = fingerprint; + } let selector = working_directory_request .repository .selector @@ -8538,6 +8566,28 @@ async fn create_workspace_working_directory( resolved_runtime_id, config_revision: runtime_projection.config_revision, config_projection_digest: runtime_projection.projection_digest, + source_kind: Some( + working_directory_request + .repository + .source + .kind + .as_str() + .to_string(), + ), + source_uri: Some(working_directory_request.repository.source.uri.clone()), + source_revision: Some(working_directory_request.repository.source_revision), + source_fingerprint: Some( + working_directory_request + .repository + .source_fingerprint + .clone(), + ), + credential_id: None, + credential_revision: None, + host_trust_id: None, + host_trust_revision: None, + repository_access_mode: None, + cache_generation: 0, working_directory_id: next_backend_workdir_id(&request.repository_id), state: "pending".to_string(), failure: None, @@ -8628,12 +8678,10 @@ async fn create_workspace_working_directory( )); } - let repository_access_projection = active_repository_access_projection(api, workspace_id)?; - if let Err(error) = authorize_repository_materialization( + if let Err(error) = authorize_repository_materialization_operation( api, - &reserved.resolved_runtime_id, - &operation_id, - &repository_access_projection, + &reserved, + &request_fingerprint, &mut working_directory_request, ) { api.config_store.finish_workdir_create_operation( @@ -13968,6 +14016,132 @@ fn validate_working_directory_claim_for_browser( Ok(()) } +fn authorize_repository_materialization_operation( + api: &WorkspaceApi, + operation: &WorkdirCreateOperationRecord, + request_fingerprint: &str, + request: &mut WorkingDirectoryRequest, +) -> ApiResult<()> { + let context = if request.repository.source.kind == workspace_api::RepositorySourceKind::Ssh { + if let ( + Some(credential_id), + Some(credential_revision), + Some(host_trust_id), + Some(host_trust_revision), + Some(access_mode), + ) = ( + operation.credential_id.as_deref(), + operation.credential_revision, + operation.host_trust_id.as_deref(), + operation.host_trust_revision, + operation.repository_access_mode.as_deref(), + ) { + let lease = api + .repository_secrets + .lease_ssh_materialization_access_revision( + &api.config.workspace_id, + credential_id, + credential_revision, + host_trust_id, + host_trust_revision, + )?; + let access = match access_mode { + "read_only" => workspace_api::RepositoryAccessMode::ReadOnly, + "read_write" => workspace_api::RepositoryAccessMode::ReadWrite, + _ => { + return Err(settings_bad_request( + "working_directory_repository_access_invalid", + "persisted Repository access mode is invalid", + )); + } + }; + RepositoryMaterializationContext { + workspace_id: api.config.workspace_id.clone(), + runtime_id: operation.resolved_runtime_id.clone(), + operation_id: operation.operation_id.clone(), + config_revision: operation.config_revision, + config_projection_digest: operation.config_projection_digest.clone(), + cache_generation: operation.cache_generation, + ssh: Some(RepositorySshMaterializationAccess { + credential_id: lease.credential_id, + credential_revision: lease.credential_revision, + host_trust_id: lease.host_trust_id, + host_trust_revision: lease.host_trust_revision, + access, + expires_at_epoch_seconds: repository_access_expiry(), + private_key: SensitiveString::new(lease.private_key.as_str()), + known_hosts_entry: SensitiveString::new(lease.known_hosts_entry), + }), + } + } else { + let projection = active_repository_access_projection(api, &api.config.workspace_id)?; + if projection.config_revision != operation.config_revision + || projection.projection_digest != operation.config_projection_digest + { + return Err(settings_bad_request( + "working_directory_repository_access_revision_changed", + "Workspace Repository access revision changed before operation reservation was bound", + )); + } + authorize_repository_materialization( + api, + &operation.resolved_runtime_id, + &operation.operation_id, + &projection, + request, + )?; + let context = request.materialization.take().ok_or_else(|| { + settings_bad_request( + "working_directory_remote_repository_access_required", + "SSH Repository access authority is unavailable", + ) + })?; + let ssh = context.ssh.as_ref().ok_or_else(|| { + settings_bad_request( + "working_directory_remote_repository_access_required", + "SSH Repository access authority is unavailable", + ) + })?; + api.config_store.bind_workdir_create_repository_access( + &api.config.workspace_id, + &operation.operation_id, + request_fingerprint, + &ssh.credential_id, + ssh.credential_revision, + &ssh.host_trust_id, + ssh.host_trust_revision, + match ssh.access { + workspace_api::RepositoryAccessMode::ReadOnly => "read_only", + workspace_api::RepositoryAccessMode::ReadWrite => "read_write", + }, + context.cache_generation, + &now_registry_timestamp(), + )?; + context + } + } else { + RepositoryMaterializationContext { + workspace_id: api.config.workspace_id.clone(), + runtime_id: operation.resolved_runtime_id.clone(), + operation_id: operation.operation_id.clone(), + config_revision: operation.config_revision, + config_projection_digest: operation.config_projection_digest.clone(), + cache_generation: operation.cache_generation, + ssh: None, + } + }; + request.materialization = Some(context); + Ok(()) +} + +fn repository_access_expiry() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .saturating_add(300) +} + fn authorize_repository_materialization( api: &WorkspaceApi, runtime_id: &str, @@ -13995,11 +14169,7 @@ fn authorize_repository_materialization( host_trust_id: lease.host_trust_id, host_trust_revision: lease.host_trust_revision, access: binding.access, - expires_at_epoch_seconds: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - .saturating_add(300), + expires_at_epoch_seconds: repository_access_expiry(), private_key: SensitiveString::new(lease.private_key.as_str()), known_hosts_entry: SensitiveString::new(lease.known_hosts_entry), }) @@ -21247,6 +21417,16 @@ mod tests { resolved_runtime_id: EMBEDDED_WORKER_RUNTIME_ID.to_string(), config_revision: 1, config_projection_digest: "sha256:test".to_string(), + source_kind: Some("local_path".to_string()), + source_uri: Some("/tmp/repo".to_string()), + source_revision: Some(1), + source_fingerprint: Some("sha256:test".to_string()), + credential_id: None, + credential_revision: None, + host_trust_id: None, + host_trust_revision: None, + repository_access_mode: None, + cache_generation: 0, working_directory_id: "workdir-provider-rejection".to_string(), state: "pending".to_string(), failure: None, diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 694ebf24..effe1a3e 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -257,6 +257,11 @@ const MIGRATIONS: &[Migration] = &[ name: "create Workspace Repository SSH secret authority", apply: create_repository_ssh_secret_authority, }, + Migration { + version: 47, + name: "bind Workdir create repository access evidence", + apply: bind_workdir_create_repository_access_evidence, + }, ]; struct Migration { @@ -590,6 +595,16 @@ pub struct WorkdirCreateOperationRecord { pub resolved_runtime_id: String, pub config_revision: u64, pub config_projection_digest: String, + pub source_kind: Option, + pub source_uri: Option, + pub source_revision: Option, + pub source_fingerprint: Option, + pub credential_id: Option, + pub credential_revision: Option, + pub host_trust_id: Option, + pub host_trust_revision: Option, + pub repository_access_mode: Option, + pub cache_generation: u64, pub working_directory_id: String, pub state: String, pub failure: Option, @@ -6812,6 +6827,25 @@ fn create_repository_ssh_secret_authority(conn: &Connection) -> Result<()> { Ok(()) } +fn bind_workdir_create_repository_access_evidence(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + ALTER TABLE workdir_create_operations ADD COLUMN source_kind TEXT; + ALTER TABLE workdir_create_operations ADD COLUMN source_uri TEXT; + ALTER TABLE workdir_create_operations ADD COLUMN source_revision INTEGER; + ALTER TABLE workdir_create_operations ADD COLUMN source_fingerprint TEXT; + ALTER TABLE workdir_create_operations ADD COLUMN credential_id TEXT; + ALTER TABLE workdir_create_operations ADD COLUMN credential_revision INTEGER; + ALTER TABLE workdir_create_operations ADD COLUMN host_trust_id TEXT; + ALTER TABLE workdir_create_operations ADD COLUMN host_trust_revision INTEGER; + ALTER TABLE workdir_create_operations ADD COLUMN repository_access_mode TEXT; + ALTER TABLE workdir_create_operations + ADD COLUMN cache_generation INTEGER NOT NULL DEFAULT 0; + "#, + )?; + Ok(()) +} + fn create_workspace_catalog_operations(conn: &Connection) -> Result<()> { conn.execute_batch( r#" @@ -9697,7 +9731,7 @@ mod tests { apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); let remote = conn .query_row( "SELECT source_kind, source_uri, source_revision, source_fingerprint, observed_status \ @@ -9775,7 +9809,7 @@ mod tests { let before = std::fs::read(&path).unwrap(); let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap(); assert_eq!(plan.current_schema_version, 36); - assert_eq!(plan.target_schema_version, 46); + assert_eq!(plan.target_schema_version, 47); assert!(plan.migration_required); assert_eq!(plan.worker_count, 1); assert_eq!(plan.mappings[0].legacy_worker_id, 7); @@ -9789,7 +9823,7 @@ mod tests { store .with_conn(|conn| { assert!(table_exists(conn, "worker_diagnostics_archives")?); - assert_eq!(current_schema_version(conn)?, 46); + assert_eq!(current_schema_version(conn)?, 47); Ok(()) }) .unwrap(); @@ -9925,7 +9959,7 @@ mod tests { ), ] ); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); let foreign_key_error: Option = conn .query_row("PRAGMA foreign_key_check", [], |row| row.get(0)) .optional() @@ -10054,7 +10088,7 @@ INSERT INTO worker_orphan_diagnostics ( apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); assert!(!table_exists(&conn, "worker_control_delegation_operations").unwrap()); let controller_worker_id: String = conn .query_row( @@ -10172,7 +10206,7 @@ INSERT INTO worker_orphan_diagnostics ( apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); assert!(table_exists(&conn, "worker_workdir_attachment_reservations").unwrap()); } @@ -10190,7 +10224,7 @@ INSERT INTO worker_orphan_diagnostics ( apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); let settings = conn .query_row( "SELECT settings_revision, language FROM workspace_memory_settings \ @@ -10231,7 +10265,7 @@ CREATE TABLE flow_events (event_id TEXT PRIMARY KEY); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); assert!(table_exists(&conn, "flow_sources").unwrap()); assert!(table_exists(&conn, "flow_source_revisions").unwrap()); assert!(!table_exists(&conn, "flow_instances").unwrap()); @@ -10298,7 +10332,7 @@ INSERT INTO worker_workdir_attachment_reservations ( apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); let repositories_sql: String = conn .query_row( "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'repositories'", @@ -10481,7 +10515,7 @@ INSERT INTO workdir_registry ( let db = dir.path().join("control-plane.sqlite"); let store = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 46); + assert_eq!(store.schema_version().await.unwrap(), 47); assert!( !store .with_conn(|conn| table_exists(conn, "worker_workspace_credentials")) @@ -10498,7 +10532,7 @@ INSERT INTO workdir_registry ( store.upsert_workspace(&record).await.unwrap(); let reopened = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(reopened.schema_version().await.unwrap(), 46); + assert_eq!(reopened.schema_version().await.unwrap(), 47); assert_eq!( reopened.get_workspace("local-dev").await.unwrap(), Some(record) @@ -11252,7 +11286,7 @@ INSERT INTO worker_registry ( let migrated = SqliteWorkspaceStore::open(&db_path).unwrap(); migrated .with_conn(|conn| { - assert_eq!(current_schema_version(conn)?, 46); + assert_eq!(current_schema_version(conn)?, 47); assert_eq!( conn.query_row("PRAGMA foreign_keys", [], |row| row.get::<_, i64>(0))?, 1, @@ -11609,13 +11643,13 @@ INSERT INTO worker_registry ( DROP TABLE repository_ssh_host_trust_revisions; DROP TABLE repository_ssh_host_trusts; DROP TABLE workdir_create_operations; - DELETE FROM __yoi_schema_migrations WHERE version IN (45, 46);", + DELETE FROM __yoi_schema_migrations WHERE version IN (45, 46, 47);", ) .unwrap(); assert_eq!(current_schema_version(&conn).unwrap(), 44); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); assert!(table_exists(&conn, "workdir_create_operations").unwrap()); let columns = table_columns(&conn, "workdir_create_operations").unwrap(); for required in [ @@ -11647,13 +11681,23 @@ INSERT INTO worker_registry ( DROP TABLE repository_ssh_credentials; DROP TABLE repository_ssh_host_trust_revisions; DROP TABLE repository_ssh_host_trusts; - DELETE FROM __yoi_schema_migrations WHERE version = 46;", + ALTER TABLE workdir_create_operations DROP COLUMN source_kind; + ALTER TABLE workdir_create_operations DROP COLUMN source_uri; + ALTER TABLE workdir_create_operations DROP COLUMN source_revision; + ALTER TABLE workdir_create_operations DROP COLUMN source_fingerprint; + ALTER TABLE workdir_create_operations DROP COLUMN credential_id; + ALTER TABLE workdir_create_operations DROP COLUMN credential_revision; + ALTER TABLE workdir_create_operations DROP COLUMN host_trust_id; + ALTER TABLE workdir_create_operations DROP COLUMN host_trust_revision; + ALTER TABLE workdir_create_operations DROP COLUMN repository_access_mode; + ALTER TABLE workdir_create_operations DROP COLUMN cache_generation; + DELETE FROM __yoi_schema_migrations WHERE version IN (46, 47);", ) .unwrap(); assert_eq!(current_schema_version(&conn).unwrap(), 45); apply_migrations(&conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); for table in [ "repository_ssh_credentials", "repository_ssh_credential_revisions", @@ -11672,19 +11716,62 @@ INSERT INTO worker_registry ( assert!(foreign_key_error.is_none()); } + #[test] + fn schema_v47_binds_workdir_create_repository_access_evidence() { + let conn = Connection::open_in_memory().unwrap(); + configure_sqlite(&conn).unwrap(); + apply_migrations(&conn).unwrap(); + conn.execute_batch( + "ALTER TABLE workdir_create_operations DROP COLUMN source_kind; + ALTER TABLE workdir_create_operations DROP COLUMN source_uri; + ALTER TABLE workdir_create_operations DROP COLUMN source_revision; + ALTER TABLE workdir_create_operations DROP COLUMN source_fingerprint; + ALTER TABLE workdir_create_operations DROP COLUMN credential_id; + ALTER TABLE workdir_create_operations DROP COLUMN credential_revision; + ALTER TABLE workdir_create_operations DROP COLUMN host_trust_id; + ALTER TABLE workdir_create_operations DROP COLUMN host_trust_revision; + ALTER TABLE workdir_create_operations DROP COLUMN repository_access_mode; + ALTER TABLE workdir_create_operations DROP COLUMN cache_generation; + DELETE FROM __yoi_schema_migrations WHERE version = 47;", + ) + .unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 46); + + apply_migrations(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 47); + let columns = table_columns(&conn, "workdir_create_operations").unwrap(); + for required in [ + "source_kind", + "source_uri", + "source_revision", + "source_fingerprint", + "credential_id", + "credential_revision", + "host_trust_id", + "host_trust_revision", + "repository_access_mode", + "cache_generation", + ] { + assert!( + columns.iter().any(|column| column == required), + "missing column {required}" + ); + } + } + #[test] fn server_refuses_a_database_from_a_newer_schema_generation() { let conn = Connection::open_in_memory().unwrap(); configure_sqlite(&conn).unwrap(); apply_migrations(&conn).unwrap(); conn.execute( - "INSERT INTO __yoi_schema_migrations (version, name) VALUES (47, 'future')", + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (48, 'future')", [], ) .unwrap(); let error = apply_migrations(&conn).unwrap_err().to_string(); - assert!(error.contains("schema version 47 is newer"), "{error}"); + assert!(error.contains("schema version 48 is newer"), "{error}"); assert!(error.contains("refusing to serve"), "{error}"); } @@ -11905,7 +11992,7 @@ VALUES ('workspace-b', 'ticket-b', 'related', 'ticket-a', NULL, 'tester', '2026- apply_migrations(&mut conn).unwrap(); - assert_eq!(current_schema_version(&conn).unwrap(), 46); + assert_eq!(current_schema_version(&conn).unwrap(), 47); let workspace_id: Option = conn .query_row( "SELECT workspace_id FROM trusted_runtime_records WHERE runtime_id = 'runtime-a'", @@ -12528,7 +12615,7 @@ WHERE workspace_id = 'workspace-a' .unwrap(); let store = SqliteWorkspaceStore::from_connection(conn).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 46); + assert_eq!(store.schema_version().await.unwrap(), 47); store .with_conn(|conn| { @@ -12717,7 +12804,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn repository_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 46); + assert_eq!(store.schema_version().await.unwrap(), 47); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -12795,7 +12882,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn memory_authority_records_round_trip_and_close_staging() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 46); + assert_eq!(store.schema_version().await.unwrap(), 47); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: None, @@ -13202,7 +13289,7 @@ CREATE TABLE ticket_assignment_operations ( #[tokio::test] async fn account_and_login_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 46); + assert_eq!(store.schema_version().await.unwrap(), 47); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { account_id: "acct-user-alice".to_string(), diff --git a/crates/workspace-server/src/workdir_create_operations.rs b/crates/workspace-server/src/workdir_create_operations.rs index b8b0e25c..81e5cdbd 100644 --- a/crates/workspace-server/src/workdir_create_operations.rs +++ b/crates/workspace-server/src/workdir_create_operations.rs @@ -48,9 +48,10 @@ impl SqliteWorkspaceStore { r#"INSERT OR IGNORE INTO workdir_create_operations ( workspace_id, operation_id, request_fingerprint, repository_id, selector, requested_runtime_id, resolved_runtime_id, config_revision, - config_projection_digest, working_directory_id, state, failure, + config_projection_digest, source_kind, source_uri, source_revision, + source_fingerprint, working_directory_id, state, failure, created_at, updated_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"#, + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)"#, params![ record.workspace_id, record.operation_id, @@ -61,6 +62,10 @@ impl SqliteWorkspaceStore { record.resolved_runtime_id, record.config_revision as i64, record.config_projection_digest, + record.source_kind, + record.source_uri, + record.source_revision.map(|revision| revision as i64), + record.source_fingerprint, record.working_directory_id, record.state, record.failure, @@ -87,6 +92,81 @@ impl SqliteWorkspaceStore { }) } + pub fn bind_workdir_create_repository_access( + &self, + workspace_id: &str, + operation_id: &str, + request_fingerprint: &str, + credential_id: &str, + credential_revision: u64, + host_trust_id: &str, + host_trust_revision: u64, + repository_access_mode: &str, + cache_generation: u64, + now: &str, + ) -> Result { + self.with_conn_mut(|conn| { + let operation = read_workdir_create_operation(conn, workspace_id, operation_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workdir create operation `{operation_id}` disappeared before Repository access binding" + )) + })?; + if operation.request_fingerprint != request_fingerprint { + return Err(Error::InvalidInput(format!( + "Workdir create operation `{operation_id}` was reused with different input" + ))); + } + if let Some(existing) = operation.credential_id.as_deref() { + if existing != credential_id + || operation.credential_revision != Some(credential_revision) + || operation.host_trust_id.as_deref() != Some(host_trust_id) + || operation.host_trust_revision != Some(host_trust_revision) + || operation.repository_access_mode.as_deref() + != Some(repository_access_mode) + || operation.cache_generation != cache_generation + { + return Err(Error::InvalidInput(format!( + "Workdir create operation `{operation_id}` Repository access evidence changed" + ))); + } + return Ok(operation); + } + conn.execute( + r#"UPDATE workdir_create_operations + SET credential_id = ?4, credential_revision = ?5, + host_trust_id = ?6, host_trust_revision = ?7, + repository_access_mode = ?8, cache_generation = ?9, + updated_at = ?10 + WHERE workspace_id = ?1 AND operation_id = ?2 + AND request_fingerprint = ?3 AND credential_id IS NULL"#, + params![ + workspace_id, + operation_id, + request_fingerprint, + credential_id, + i64::try_from(credential_revision).map_err(|_| Error::InvalidInput( + "credential revision is out of range".to_string() + ))?, + host_trust_id, + i64::try_from(host_trust_revision).map_err(|_| Error::InvalidInput( + "host-trust revision is out of range".to_string() + ))?, + repository_access_mode, + i64::try_from(cache_generation).map_err(|_| Error::InvalidInput( + "cache generation is out of range".to_string() + ))?, + now, + ], + )?; + read_workdir_create_operation(conn, workspace_id, operation_id)?.ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Workdir create operation `{operation_id}` disappeared after Repository access binding" + )) + }) + }) + } + pub fn finish_workdir_create_operation( &self, workspace_id: &str, @@ -141,7 +221,10 @@ fn read_workdir_create_operation( conn.query_row( r#"SELECT workspace_id, operation_id, request_fingerprint, repository_id, selector, requested_runtime_id, resolved_runtime_id, config_revision, - config_projection_digest, working_directory_id, state, failure, + config_projection_digest, source_kind, source_uri, source_revision, + source_fingerprint, credential_id, credential_revision, + host_trust_id, host_trust_revision, repository_access_mode, + cache_generation, working_directory_id, state, failure, created_at, updated_at FROM workdir_create_operations WHERE workspace_id = ?1 AND operation_id = ?2"#, @@ -157,11 +240,21 @@ fn read_workdir_create_operation( resolved_runtime_id: row.get(6)?, config_revision: row.get::<_, i64>(7)? as u64, config_projection_digest: row.get(8)?, - working_directory_id: row.get(9)?, - state: row.get(10)?, - failure: row.get(11)?, - created_at: row.get(12)?, - updated_at: row.get(13)?, + source_kind: row.get(9)?, + source_uri: row.get(10)?, + source_revision: row.get::<_, Option>(11)?.map(|value| value as u64), + source_fingerprint: row.get(12)?, + credential_id: row.get(13)?, + credential_revision: row.get::<_, Option>(14)?.map(|value| value as u64), + host_trust_id: row.get(15)?, + host_trust_revision: row.get::<_, Option>(16)?.map(|value| value as u64), + repository_access_mode: row.get(17)?, + cache_generation: row.get::<_, i64>(18)? as u64, + working_directory_id: row.get(19)?, + state: row.get(20)?, + failure: row.get(21)?, + created_at: row.get(22)?, + updated_at: row.get(23)?, }) }, ) @@ -222,6 +315,16 @@ mod tests { resolved_runtime_id: "arcadia".to_string(), config_revision: 7, config_projection_digest: "sha256:projection".to_string(), + source_kind: Some("local_path".to_string()), + source_uri: Some("/tmp/repo".to_string()), + source_revision: Some(1), + source_fingerprint: Some("sha256:source".to_string()), + credential_id: None, + credential_revision: None, + host_trust_id: None, + host_trust_revision: None, + repository_access_mode: None, + cache_generation: 0, working_directory_id: "wd-1".to_string(), state: "pending".to_string(), failure: None, @@ -232,20 +335,55 @@ mod tests { store.reserve_workdir_create_operation(&record).unwrap(), record ); + let bound = store + .bind_workdir_create_repository_access( + "workspace", + "call-1", + &record.request_fingerprint, + "credential-1", + 3, + "trust-1", + 5, + "read_only", + 2, + "2026-08-24T00:00:01Z", + ) + .unwrap(); + assert_eq!(bound.credential_id.as_deref(), Some("credential-1")); + assert_eq!(bound.credential_revision, Some(3)); + assert_eq!(bound.host_trust_revision, Some(5)); + assert_eq!(bound.cache_generation, 2); + assert!( + store + .bind_workdir_create_repository_access( + "workspace", + "call-1", + &record.request_fingerprint, + "credential-1", + 4, + "trust-1", + 5, + "read_only", + 2, + "2026-08-24T00:00:02Z", + ) + .is_err() + ); let mut changed_resolution = record.clone(); changed_resolution.resolved_runtime_id = "other".to_string(); changed_resolution.config_revision = 8; - assert_eq!( - store - .reserve_workdir_create_operation(&changed_resolution) - .unwrap(), - record - ); + changed_resolution.source_uri = Some("ssh://git@other.test/repo.git".to_string()); + changed_resolution.source_revision = Some(9); + let replayed = store + .reserve_workdir_create_operation(&changed_resolution) + .unwrap(); + assert_eq!(replayed, bound); + assert_eq!(replayed.source_uri.as_deref(), Some("/tmp/repo")); assert_eq!( store .load_workdir_create_operation("workspace", "call-1") .unwrap(), - Some(record.clone()) + Some(bound.clone()) ); let mut changed_input = record.clone(); changed_input.request_fingerprint = From 3cdcbb47bfa0b9ca1d3a99976845119a978fab4c Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 16:05:14 +0900 Subject: [PATCH 04/12] fix: preserve selector retries and workdir observations --- crates/workdir/src/workspace.rs | 21 ++++ .../worker-runtime/src/working_directory.rs | 41 ++++++- crates/workspace-server/src/server.rs | 101 +++++++++++++----- crates/workspace-server/src/store.rs | 62 +++++++++-- .../src/workdir_create_operations.rs | 27 +++++ 5 files changed, 213 insertions(+), 39 deletions(-) diff --git a/crates/workdir/src/workspace.rs b/crates/workdir/src/workspace.rs index 69bc9fe5..e76fdf76 100644 --- a/crates/workdir/src/workspace.rs +++ b/crates/workdir/src/workspace.rs @@ -111,6 +111,8 @@ pub struct WorkingDirectoryProvenance { pub creation_selector: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub creation_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub creation_tree: Option, pub materializer_kind: MaterializerKind, #[serde(default, skip_serializing_if = "Option::is_none")] pub cleanup_target: Option, @@ -124,6 +126,10 @@ pub struct WorkingDirectoryCurrentObservation { pub current_selector: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub current_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_tree: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_at_epoch_seconds: Option, pub status: WorkingDirectoryStatusKind, #[serde(default, skip_serializing_if = "Option::is_none")] pub cleanliness: Option, @@ -143,9 +149,15 @@ pub struct WorkingDirectorySummary { #[serde(default, skip_serializing_if = "Option::is_none")] pub creation_ref: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub creation_tree: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub current_selector: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub current_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_tree: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_at_epoch_seconds: Option, pub materializer_kind: MaterializerKind, #[serde(default, skip_serializing_if = "Option::is_none")] pub cleanup_target: Option, @@ -168,6 +180,7 @@ impl WorkingDirectorySummary { WorkingDirectoryProvenance { creation_selector: self.creation_selector.clone(), creation_ref: self.creation_ref.clone(), + creation_tree: self.creation_tree.clone(), materializer_kind: self.materializer_kind.clone(), cleanup_target: self.cleanup_target.clone(), } @@ -177,6 +190,8 @@ impl WorkingDirectorySummary { WorkingDirectoryCurrentObservation { current_selector: self.current_selector.clone(), current_ref: self.current_ref.clone(), + current_tree: self.current_tree.clone(), + observed_at_epoch_seconds: self.observed_at_epoch_seconds, status: self.status.clone(), cleanliness: self.cleanliness.clone(), primary_worker_id: self.primary_worker_id.clone(), @@ -249,8 +264,11 @@ mod tests { repository_id: "repo".to_string(), creation_selector: Some("develop".to_string()), creation_ref: Some("abc123".to_string()), + creation_tree: Some("tree123".to_string()), current_selector: Some("work/ticket".to_string()), current_ref: Some("def456".to_string()), + current_tree: Some("tree456".to_string()), + observed_at_epoch_seconds: Some(1_777_777_777), materializer_kind: MaterializerKind::LocalGitWorktree, cleanup_target: Some(WorkingDirectoryCleanupTarget { kind: "git_worktree".to_string(), @@ -271,8 +289,11 @@ mod tests { repository_id: "repo".to_string(), creation_selector: None, creation_ref: None, + creation_tree: None, current_selector: None, current_ref: Some("987fed".to_string()), + current_tree: None, + observed_at_epoch_seconds: None, materializer_kind: MaterializerKind::LocalGitWorktree, cleanup_target: None, status: WorkingDirectoryStatusKind::Active, diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index 79b7817b..63104766 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -69,8 +69,11 @@ impl WorkingDirectory { repository_id: self.repository_id.clone(), creation_selector: self.evidence.requested_selector.clone(), creation_ref: Some(self.evidence.resolved_commit.clone()), + creation_tree: self.evidence.resolved_tree.clone(), current_selector: None, current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, materializer_kind: self.materializer_kind.clone(), cleanup_target: Some(self.cleanup_target.clone()), status: self.status.clone(), @@ -126,9 +129,16 @@ impl WorkingDirectoryBinding { } let mut summary = working_directory.status_summary(); summary.cleanliness = if summary.status == WorkingDirectoryStatusKind::Active { - let (current_selector, current_ref) = binding_current_revision(self); + let (current_selector, current_ref, current_tree) = binding_current_revision(self); summary.current_selector = current_selector; summary.current_ref = current_ref; + summary.current_tree = current_tree; + summary.observed_at_epoch_seconds = Some( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ); Some(binding_cleanliness(self)) } else { Some("unknown".to_string()) @@ -217,12 +227,14 @@ fn binding_paths_are_available(binding: &WorkingDirectoryBinding) -> bool { source_repository_path.is_dir() } -fn binding_current_revision(binding: &WorkingDirectoryBinding) -> (Option, Option) { +fn binding_current_revision( + binding: &WorkingDirectoryBinding, +) -> (Option, Option, Option) { let current_ref = git_stdout(binding.root(), ["rev-parse", "HEAD"]) .ok() .filter(|value| !value.is_empty()); if current_ref.is_none() { - return (None, None); + return (None, None, None); } let current_selector = git_stdout( binding.root(), @@ -230,7 +242,10 @@ fn binding_current_revision(binding: &WorkingDirectoryBinding) -> (Option String { @@ -303,8 +318,11 @@ impl RuntimeGitCacheMaterializer { repository_id: "unknown".to_string(), creation_selector: None, creation_ref: None, + creation_tree: None, current_selector: None, current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, materializer_kind: MaterializerKind::RuntimeGitCache, cleanup_target: Some(WorkingDirectoryCleanupTarget { kind: "runtime_git_cache_worktree".to_string(), @@ -2299,6 +2317,21 @@ mod tests { assert_eq!(summary.creation_ref.as_deref(), Some(initial_ref.as_str())); assert_eq!(summary.current_selector.as_deref(), Some("observed-branch")); assert_ne!(summary.current_ref.as_deref(), Some(initial_ref.as_str())); + assert!(summary.creation_tree.is_some()); + assert!(summary.current_tree.is_some()); + assert_ne!(summary.current_tree, summary.creation_tree); + assert!(summary.observed_at_epoch_seconds.is_some()); + assert_eq!(summary.cleanliness.as_deref(), Some("clean")); + + fs::write(bound.root.join("dirty.txt"), "dirty\n").unwrap(); + let dirty = materializer.list_working_directories().unwrap()[0] + .summary + .clone(); + assert_eq!(dirty.creation_ref, summary.creation_ref); + assert_eq!(dirty.current_ref, summary.current_ref); + assert_eq!(dirty.current_tree, summary.current_tree); + assert_eq!(dirty.cleanliness.as_deref(), Some("dirty")); + assert!(dirty.observed_at_epoch_seconds.is_some()); } #[test] diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index bdc953df..d5cd876a 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -8477,30 +8477,39 @@ async fn create_workspace_working_directory( if let Some(existing) = api .config_store .load_workdir_create_operation(workspace_id, &operation_id)? - && let (Some(kind), Some(uri), Some(revision), Some(fingerprint)) = ( - existing.source_kind.as_deref(), - existing.source_uri, - existing.source_revision, - existing.source_fingerprint, - ) { - let kind = match kind { - "local_path" => workspace_api::RepositorySourceKind::LocalPath, - "file" => workspace_api::RepositorySourceKind::File, - "https" => workspace_api::RepositorySourceKind::Https, - "http" => workspace_api::RepositorySourceKind::Http, - "ssh" => workspace_api::RepositorySourceKind::Ssh, - "invalid" => workspace_api::RepositorySourceKind::Invalid, - _ => { - return Err(settings_bad_request( - "working_directory_repository_source_invalid", - "persisted Workdir create Repository source kind is invalid", - )); - } - }; - working_directory_request.repository.source = workspace_api::RepositorySource { kind, uri }; - working_directory_request.repository.source_revision = revision; - working_directory_request.repository.source_fingerprint = fingerprint; + working_directory_request.repository.selector = + crate::workdir_create_operations::selector_for_retry( + request.selector.as_deref(), + existing.selector.as_deref(), + working_directory_request.repository.selector.as_deref(), + ) + .map(RuntimeRepositorySelector::from); + if let (Some(kind), Some(uri), Some(revision), Some(fingerprint)) = ( + existing.source_kind.as_deref(), + existing.source_uri.clone(), + existing.source_revision, + existing.source_fingerprint.clone(), + ) { + let kind = match kind { + "local_path" => workspace_api::RepositorySourceKind::LocalPath, + "file" => workspace_api::RepositorySourceKind::File, + "https" => workspace_api::RepositorySourceKind::Https, + "http" => workspace_api::RepositorySourceKind::Http, + "ssh" => workspace_api::RepositorySourceKind::Ssh, + "invalid" => workspace_api::RepositorySourceKind::Invalid, + _ => { + return Err(settings_bad_request( + "working_directory_repository_source_invalid", + "persisted Workdir create Repository source kind is invalid", + )); + } + }; + working_directory_request.repository.source = + workspace_api::RepositorySource { kind, uri }; + working_directory_request.repository.source_revision = revision; + working_directory_request.repository.source_fingerprint = fingerprint; + } } let selector = working_directory_request .repository @@ -13704,8 +13713,11 @@ fn upsert_pending_backend_workdir( .as_ref() .map(|selector| selector.as_ref().to_string()), creation_ref: None, + creation_tree: None, current_selector: None, current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, materialization_status: "pending".to_string(), cleanliness: "unknown".to_string(), created_at: timestamp.clone(), @@ -13853,8 +13865,11 @@ fn workdir_record_from_summary( repository_id: summary.repository_id.clone(), creation_selector: summary.creation_selector.clone(), creation_ref: summary.creation_ref.clone(), + creation_tree: summary.creation_tree.clone(), current_selector: summary.current_selector.clone(), current_ref: summary.current_ref.clone(), + current_tree: summary.current_tree.clone(), + observed_at_epoch_seconds: summary.observed_at_epoch_seconds, materialization_status: match summary.status { WorkingDirectoryStatusKind::Active => "present", WorkingDirectoryStatusKind::CleanupPending => "pending", @@ -13891,6 +13906,21 @@ fn preserve_workdir_identity_for_corrupted_summary( if record.creation_ref.is_none() { record.creation_ref = existing.creation_ref.clone(); } + if record.creation_tree.is_none() { + record.creation_tree = existing.creation_tree.clone(); + } + if record.current_selector.is_none() { + record.current_selector = existing.current_selector.clone(); + } + if record.current_ref.is_none() { + record.current_ref = existing.current_ref.clone(); + } + if record.current_tree.is_none() { + record.current_tree = existing.current_tree.clone(); + } + if record.observed_at_epoch_seconds.is_none() { + record.observed_at_epoch_seconds = existing.observed_at_epoch_seconds; + } } fn workdir_summary_from_record(record: &WorkdirRegistryRecord) -> WorkingDirectorySummary { @@ -13907,11 +13937,14 @@ fn workdir_summary_from_record(record: &WorkdirRegistryRecord) -> WorkingDirecto repository_id: record.repository_id.clone(), creation_selector: record.creation_selector.clone(), creation_ref: record.creation_ref.clone(), + creation_tree: record.creation_tree.clone(), current_selector: record.current_selector.clone(), current_ref: record.current_ref.clone(), - materializer_kind: MaterializerKind::LocalGitWorktree, + current_tree: record.current_tree.clone(), + observed_at_epoch_seconds: record.observed_at_epoch_seconds, + materializer_kind: MaterializerKind::RuntimeGitCache, cleanup_target: Some(WorkingDirectoryCleanupTarget { - kind: "local_git_worktree".to_string(), + kind: "runtime_git_cache_worktree".to_string(), working_directory_id: record.workdir_id.clone(), repository_id: record.repository_id.clone(), }), @@ -15109,8 +15142,11 @@ mod tests { repository_id: "repo".to_string(), creation_selector: Some("develop".to_string()), creation_ref: Some("abcdef".to_string()), + creation_tree: Some("tree-creation".to_string()), current_selector: None, current_ref: Some("fedcba".to_string()), + current_tree: Some("tree-current".to_string()), + observed_at_epoch_seconds: Some(3), materialization_status: "missing".to_string(), cleanliness: "clean".to_string(), created_at: "1".to_string(), @@ -16168,8 +16204,11 @@ mod tests { repository_id: "repo".to_string(), creation_selector: None, creation_ref: None, + creation_tree: None, current_selector: None, current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, materialization_status: "present".to_string(), cleanliness: "clean".to_string(), created_at: "1".to_string(), @@ -16184,8 +16223,11 @@ mod tests { repository_id: "repo".to_string(), creation_selector: None, creation_ref: None, + creation_tree: None, current_selector: None, current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, materialization_status: "present".to_string(), cleanliness: "unknown".to_string(), created_at: "1".to_string(), @@ -16257,8 +16299,11 @@ mod tests { repository_id: "repo".to_string(), creation_selector: None, creation_ref: None, + creation_tree: None, current_selector: None, current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, materialization_status: "present".to_string(), cleanliness: "unknown".to_string(), created_at: "1".to_string(), @@ -20586,8 +20631,11 @@ mod tests { repository_id: "repo-test".to_string(), creation_selector: Some("HEAD".to_string()), creation_ref: None, + creation_tree: None, current_selector: None, current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, materialization_status: status.to_string(), cleanliness: cleanliness.to_string(), created_at: now.clone(), @@ -23967,8 +24015,11 @@ VALUES ('0192f0e8-4d84-7d6e-a000-000000000001', 'ticket', 3); repository_id: "main".to_string(), creation_selector: None, creation_ref: None, + creation_tree: None, current_selector: Some("work/ticket".to_string()), current_ref: Some("abc123".to_string()), + current_tree: Some("tree123".to_string()), + observed_at_epoch_seconds: Some(1_777_777_777), materializer_kind: MaterializerKind::LocalGitWorktree, cleanup_target: None, status: WorkingDirectoryStatusKind::Active, diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index effe1a3e..073b9fa8 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -620,8 +620,11 @@ pub struct WorkdirRegistryRecord { pub repository_id: String, pub creation_selector: Option, pub creation_ref: Option, + pub creation_tree: Option, pub current_selector: Option, pub current_ref: Option, + pub current_tree: Option, + pub observed_at_epoch_seconds: Option, pub materialization_status: String, pub cleanliness: String, pub created_at: String, @@ -4627,16 +4630,20 @@ impl ControlPlaneStore for SqliteWorkspaceStore { conn.execute( r#"INSERT INTO workdir_registry ( workspace_id, workdir_id, runtime_id, repository_id, - creation_selector, creation_ref, current_selector, current_ref, + creation_selector, creation_ref, creation_tree, + current_selector, current_ref, current_tree, observed_at_epoch_seconds, materialization_status, cleanliness, created_at, updated_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15) ON CONFLICT(workspace_id, workdir_id) DO UPDATE SET runtime_id = excluded.runtime_id, repository_id = excluded.repository_id, creation_selector = excluded.creation_selector, creation_ref = excluded.creation_ref, + creation_tree = excluded.creation_tree, current_selector = excluded.current_selector, current_ref = excluded.current_ref, + current_tree = excluded.current_tree, + observed_at_epoch_seconds = excluded.observed_at_epoch_seconds, materialization_status = excluded.materialization_status, cleanliness = excluded.cleanliness, updated_at = excluded.updated_at"#, @@ -4647,8 +4654,11 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record.repository_id, record.creation_selector, record.creation_ref, + record.creation_tree, record.current_selector, record.current_ref, + record.current_tree, + record.observed_at_epoch_seconds.map(|value| value as i64), record.materialization_status, record.cleanliness, record.created_at, @@ -5868,7 +5878,8 @@ fn require_expected_ticket_assignment( fn workdir_registry_select_sql(where_clause: &str) -> String { format!( "SELECT workspace_id, workdir_id, runtime_id, repository_id, \ - creation_selector, creation_ref, current_selector, current_ref, \ + creation_selector, creation_ref, creation_tree, \ + current_selector, current_ref, current_tree, observed_at_epoch_seconds, \ materialization_status, cleanliness, created_at, updated_at \ FROM workdir_registry {where_clause}" ) @@ -5884,12 +5895,15 @@ fn read_workdir_registry_record( repository_id: row.get(3)?, creation_selector: row.get(4)?, creation_ref: row.get(5)?, - current_selector: row.get(6)?, - current_ref: row.get(7)?, - materialization_status: row.get(8)?, - cleanliness: row.get(9)?, - created_at: row.get(10)?, - updated_at: row.get(11)?, + creation_tree: row.get(6)?, + current_selector: row.get(7)?, + current_ref: row.get(8)?, + current_tree: row.get(9)?, + observed_at_epoch_seconds: row.get::<_, Option>(10)?.map(|value| value as u64), + materialization_status: row.get(11)?, + cleanliness: row.get(12)?, + created_at: row.get(13)?, + updated_at: row.get(14)?, }) } @@ -6830,6 +6844,9 @@ fn create_repository_ssh_secret_authority(conn: &Connection) -> Result<()> { fn bind_workdir_create_repository_access_evidence(conn: &Connection) -> Result<()> { conn.execute_batch( r#" + ALTER TABLE workdir_registry ADD COLUMN creation_tree TEXT; + ALTER TABLE workdir_registry ADD COLUMN current_tree TEXT; + ALTER TABLE workdir_registry ADD COLUMN observed_at_epoch_seconds INTEGER; ALTER TABLE workdir_create_operations ADD COLUMN source_kind TEXT; ALTER TABLE workdir_create_operations ADD COLUMN source_uri TEXT; ALTER TABLE workdir_create_operations ADD COLUMN source_revision INTEGER; @@ -11643,6 +11660,9 @@ INSERT INTO worker_registry ( DROP TABLE repository_ssh_host_trust_revisions; DROP TABLE repository_ssh_host_trusts; DROP TABLE workdir_create_operations; + ALTER TABLE workdir_registry DROP COLUMN creation_tree; + ALTER TABLE workdir_registry DROP COLUMN current_tree; + ALTER TABLE workdir_registry DROP COLUMN observed_at_epoch_seconds; DELETE FROM __yoi_schema_migrations WHERE version IN (45, 46, 47);", ) .unwrap(); @@ -11681,6 +11701,9 @@ INSERT INTO worker_registry ( DROP TABLE repository_ssh_credentials; DROP TABLE repository_ssh_host_trust_revisions; DROP TABLE repository_ssh_host_trusts; + ALTER TABLE workdir_registry DROP COLUMN creation_tree; + ALTER TABLE workdir_registry DROP COLUMN current_tree; + ALTER TABLE workdir_registry DROP COLUMN observed_at_epoch_seconds; ALTER TABLE workdir_create_operations DROP COLUMN source_kind; ALTER TABLE workdir_create_operations DROP COLUMN source_uri; ALTER TABLE workdir_create_operations DROP COLUMN source_revision; @@ -11722,7 +11745,10 @@ INSERT INTO worker_registry ( configure_sqlite(&conn).unwrap(); apply_migrations(&conn).unwrap(); conn.execute_batch( - "ALTER TABLE workdir_create_operations DROP COLUMN source_kind; + "ALTER TABLE workdir_registry DROP COLUMN creation_tree; + ALTER TABLE workdir_registry DROP COLUMN current_tree; + ALTER TABLE workdir_registry DROP COLUMN observed_at_epoch_seconds; + ALTER TABLE workdir_create_operations DROP COLUMN source_kind; ALTER TABLE workdir_create_operations DROP COLUMN source_uri; ALTER TABLE workdir_create_operations DROP COLUMN source_revision; ALTER TABLE workdir_create_operations DROP COLUMN source_fingerprint; @@ -11757,6 +11783,13 @@ INSERT INTO worker_registry ( "missing column {required}" ); } + let workdir_columns = table_columns(&conn, "workdir_registry").unwrap(); + for required in ["creation_tree", "current_tree", "observed_at_epoch_seconds"] { + assert!( + workdir_columns.iter().any(|column| column == required), + "missing column {required}" + ); + } } #[test] @@ -12537,6 +12570,9 @@ WHERE workspace_id = 'workspace-a' "updated_at", "current_selector", "current_ref", + "creation_tree", + "current_tree", + "observed_at_epoch_seconds", ], ); assert_columns( @@ -13028,8 +13064,11 @@ CREATE TABLE ticket_assignment_operations ( repository_id: "repo".to_string(), creation_selector: Some("develop".to_string()), creation_ref: Some("abcdef".to_string()), + creation_tree: Some("tree-creation".to_string()), current_selector: None, current_ref: Some("abcdef".to_string()), + current_tree: Some("tree-current".to_string()), + observed_at_epoch_seconds: Some(1_777_777_777), materialization_status: "not_found".to_string(), cleanliness: "clean".to_string(), created_at: "2".to_string(), @@ -13043,8 +13082,11 @@ CREATE TABLE ticket_assignment_operations ( repository_id: "repo".to_string(), creation_selector: Some("feature".to_string()), creation_ref: Some("123456".to_string()), + creation_tree: None, current_selector: Some("feature".to_string()), current_ref: Some("123456".to_string()), + current_tree: None, + observed_at_epoch_seconds: None, materialization_status: "present".to_string(), cleanliness: "unknown".to_string(), created_at: "3".to_string(), diff --git a/crates/workspace-server/src/workdir_create_operations.rs b/crates/workspace-server/src/workdir_create_operations.rs index 81e5cdbd..1a48c8dd 100644 --- a/crates/workspace-server/src/workdir_create_operations.rs +++ b/crates/workspace-server/src/workdir_create_operations.rs @@ -4,6 +4,17 @@ use sha2::{Digest, Sha256}; use crate::store::WorkdirCreateOperationRecord; use crate::{Error, Result, SqliteWorkspaceStore}; +pub fn selector_for_retry( + explicit_selector: Option<&str>, + persisted_selector: Option<&str>, + current_default_selector: Option<&str>, +) -> Option { + explicit_selector + .or(persisted_selector) + .or(current_default_selector) + .map(str::to_string) +} + pub fn request_fingerprint( repository_id: &str, selector: Option<&str>, @@ -267,6 +278,22 @@ mod tests { use super::*; use crate::store::{ControlPlaneStore, RepositoryRecord, WorkspaceRecord}; + #[test] + fn retry_selector_keeps_persisted_default_but_honors_explicit_input() { + assert_eq!( + selector_for_retry(None, Some("develop"), Some("main")), + Some("develop".to_string()) + ); + assert_eq!( + selector_for_retry(Some("release"), Some("develop"), Some("main")), + Some("release".to_string()) + ); + assert_eq!( + selector_for_retry(None, None, Some("main")), + Some("main".to_string()) + ); + } + #[test] fn retry_keeps_resolved_config_evidence_and_rejects_changed_input() { let store = SqliteWorkspaceStore::in_memory().unwrap(); From 1873e18f8e92bf9ba391093baf55c1956175dfa5 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 16:10:39 +0900 Subject: [PATCH 05/12] fix: enforce attachment access transitions --- .../worker-runtime/src/working_directory.rs | 122 ++++++++++++++---- 1 file changed, 96 insertions(+), 26 deletions(-) diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index 63104766..e2038199 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -420,6 +420,11 @@ impl RuntimeGitCacheMaterializer { return Ok(binding); }; validate_ssh_materialization_access(&access)?; + apply_worktree_access_policy( + binding.source_repository_path(), + binding.root(), + access.access, + )?; let command_access = Arc::new(RepositoryCommandAccess::prepare_ssh( &self.runtime_root, &format!("attachment-{working_directory_id}"), @@ -684,35 +689,16 @@ impl RuntimeGitCacheMaterializer { let _ = fs::remove_dir_all(&working_directory_root); return Err(error); } - if request + if let Some(ssh) = request .materialization .as_ref() .and_then(|materialization| materialization.ssh.as_ref()) - .is_some_and(|ssh| ssh.access == workspace_api::RepositoryAccessMode::ReadOnly) + && let Err(error) = + apply_worktree_access_policy(&repository_cache, &worktree_root, ssh.access) { - let mut enable_worktree_config = isolated_git_command(); - enable_worktree_config - .arg("--git-dir") - .arg(&repository_cache) - .args(["config", "extensions.worktreeConfig", "true"]); - let mut disable_push = isolated_git_command(); - disable_push.arg("-C").arg(&worktree_root).args([ - "config", - "--worktree", - "remote.origin.pushurl", - "yoi-read-only://repository-push-disabled", - ]); - if let Err(error) = run_repository_git( - enable_worktree_config, - "working_directory_repository_policy_failed", - ) - .and_then(|_| { - run_repository_git(disable_push, "working_directory_repository_policy_failed") - }) { - remove_cached_worktree(&repository_cache, &worktree_root); - let _ = fs::remove_dir_all(&working_directory_root); - return Err(error); - } + remove_cached_worktree(&repository_cache, &worktree_root); + let _ = fs::remove_dir_all(&working_directory_root); + return Err(error); } let context = request.materialization.as_ref(); @@ -1477,6 +1463,52 @@ fn fetch_repository_cache( run_repository_git(head, "working_directory_repository_fetch_failed") } +fn apply_worktree_access_policy( + repository_cache: &Path, + worktree_root: &Path, + access: workspace_api::RepositoryAccessMode, +) -> Result<(), WorkingDirectoryDiagnostic> { + let mut enable_worktree_config = isolated_git_command(); + enable_worktree_config + .arg("--git-dir") + .arg(repository_cache) + .args(["config", "extensions.worktreeConfig", "true"]); + run_repository_git( + enable_worktree_config, + "working_directory_repository_policy_failed", + )?; + match access { + workspace_api::RepositoryAccessMode::ReadOnly => { + let mut disable_push = isolated_git_command(); + disable_push.arg("-C").arg(worktree_root).args([ + "config", + "--worktree", + "remote.origin.pushurl", + "yoi-read-only://repository-push-disabled", + ]); + run_repository_git(disable_push, "working_directory_repository_policy_failed") + } + workspace_api::RepositoryAccessMode::ReadWrite => { + if git_stdout( + worktree_root, + ["config", "--worktree", "--get-all", "remote.origin.pushurl"], + ) + .is_err() + { + return Ok(()); + } + let mut enable_push = isolated_git_command(); + enable_push.arg("-C").arg(worktree_root).args([ + "config", + "--worktree", + "--unset-all", + "remote.origin.pushurl", + ]); + run_repository_git(enable_push, "working_directory_repository_policy_failed") + } + } +} + fn validate_repository_cache_limits( repository_cache: &Path, ) -> Result<(), WorkingDirectoryDiagnostic> { @@ -2071,10 +2103,11 @@ mod tests { let mut rotated = request.materialization.clone().unwrap(); rotated.operation_id = "operation-agent-rotated".to_string(); rotated.ssh.as_mut().unwrap().credential_revision = 2; + rotated.ssh.as_mut().unwrap().access = workspace_api::RepositoryAccessMode::ReadOnly; materializer .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { working_directory_id: id.clone(), - materialization: rotated, + materialization: rotated.clone(), }) .unwrap(); let rebound = materializer.bind_working_directory(&id, None).unwrap(); @@ -2082,7 +2115,44 @@ mod tests { rebound.working_directory.evidence.credential_revision, Some(2) ); + assert_eq!( + rebound.command_environment()["YOI_REPOSITORY_ACCESS"], + "read_only" + ); + assert_eq!( + git_stdout( + rebound.root(), + ["config", "--worktree", "--get", "remote.origin.pushurl"], + ) + .unwrap(), + "yoi-read-only://repository-push-disabled" + ); drop(rebound); + + let mut read_write = rotated; + read_write.operation_id = "operation-agent-read-write".to_string(); + read_write.ssh.as_mut().unwrap().credential_revision = 3; + read_write.ssh.as_mut().unwrap().access = workspace_api::RepositoryAccessMode::ReadWrite; + materializer + .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { + working_directory_id: id.clone(), + materialization: read_write, + }) + .unwrap(); + let rebound = materializer.bind_working_directory(&id, None).unwrap(); + assert_eq!( + rebound.command_environment()["YOI_REPOSITORY_ACCESS"], + "read_write" + ); + assert!( + git_stdout( + rebound.root(), + ["config", "--worktree", "--get", "remote.origin.pushurl"], + ) + .is_err() + ); + drop(rebound); + let mut expired = request.materialization.clone().unwrap(); expired.ssh.as_mut().unwrap().expires_at_epoch_seconds = 1; assert_eq!( From 108d5b14d70727ad1ffaa2ab5848e29aa83148f4 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 16:23:48 +0900 Subject: [PATCH 06/12] fix: withhold write-capable credentials from read-only sessions --- .../worker-runtime/src/working_directory.rs | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index e2038199..e7636e9d 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -445,10 +445,12 @@ impl RuntimeGitCacheMaterializer { access.stop(); } }); - binding.command_environment.insert( - "SSH_AUTH_SOCK".to_string(), - command_access.agent.socket.to_string_lossy().to_string(), - ); + if access.access == workspace_api::RepositoryAccessMode::ReadWrite { + binding.command_environment.insert( + "SSH_AUTH_SOCK".to_string(), + command_access.agent.socket.to_string_lossy().to_string(), + ); + } binding.command_environment.insert( "GIT_SSH_COMMAND".to_string(), command_access.ssh_command.to_string_lossy().to_string(), @@ -1187,13 +1189,19 @@ impl RepositoryCommandAccess { let known_hosts = root.join("known_hosts"); let ssh_command = root.join("ssh-command"); write_owner_only(&known_hosts, ssh.known_hosts_entry.expose().as_bytes())?; + let agent = RepositorySshAgent::start(runtime_root, operation_id, ssh)?; + let read_only_guard = if ssh.access == workspace_api::RepositoryAccessMode::ReadOnly { + "case \" $* \" in\n *\" -G \"*|*\" git-upload-pack \"*|*\" git-upload-archive \"*) ;;\n *) echo 'read-only Repository SSH operation denied' >&2; exit 126 ;;\nesac\n" + } else { + "" + }; let script = format!( - "#!/bin/sh\nexec ssh -F /dev/null -o BatchMode=yes -o IdentitiesOnly=no -o IdentityFile=/dev/null -o StrictHostKeyChecking=yes -o UserKnownHostsFile={} \"$@\"\n", + "#!/bin/sh\n{read_only_guard}export SSH_AUTH_SOCK={}\nexec ssh -F /dev/null -o BatchMode=yes -o IdentitiesOnly=no -o IdentityFile=/dev/null -o StrictHostKeyChecking=yes -o UserKnownHostsFile={} \"$@\"\n", + shell_quote_path(&agent.socket)?, shell_quote_path(&known_hosts)?, ); write_owner_only(&ssh_command, script.as_bytes())?; set_file_owner_executable(&ssh_command)?; - let agent = RepositorySshAgent::start(runtime_root, operation_id, ssh)?; Ok(Self { root, ssh_command, @@ -2115,10 +2123,16 @@ mod tests { rebound.working_directory.evidence.credential_revision, Some(2) ); - assert_eq!( - rebound.command_environment()["YOI_REPOSITORY_ACCESS"], - "read_only" - ); + let rebound_environment = rebound.command_environment(); + assert_eq!(rebound_environment["YOI_REPOSITORY_ACCESS"], "read_only"); + assert!(!rebound_environment.contains_key("SSH_AUTH_SOCK")); + let read_only_ssh = &rebound_environment["GIT_SSH_COMMAND"]; + let denied = Command::new(read_only_ssh) + .args(["example.test", "git-receive-pack 'repo.git'"]) + .env_remove("SSH_AUTH_SOCK") + .status() + .unwrap(); + assert_eq!(denied.code(), Some(126)); assert_eq!( git_stdout( rebound.root(), @@ -2144,6 +2158,7 @@ mod tests { rebound.command_environment()["YOI_REPOSITORY_ACCESS"], "read_write" ); + assert!(rebound.command_environment().contains_key("SSH_AUTH_SOCK")); assert!( git_stdout( rebound.root(), From df34533765c29e8d53676cc884f9f8b483a4216f Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 16:48:37 +0900 Subject: [PATCH 07/12] fix: broker read-only repository SSH operations --- crates/worker-runtime/src/main.rs | 13 + .../worker-runtime/src/working_directory.rs | 511 +++++++++++++++++- 2 files changed, 506 insertions(+), 18 deletions(-) diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index c7b8179c..3e5d1e3c 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -27,6 +27,19 @@ use worker_runtime::working_directory::RuntimeGitCacheMaterializer; use worker_runtime::{Runtime, RuntimeOptions}; fn main() -> ExitCode { + let mut arguments = std::env::args().skip(1).collect::>(); + if arguments.first().map(String::as_str) == Some("__repository-read-only-ssh") { + arguments.remove(0); + return match worker_runtime::working_directory::run_repository_read_only_ssh_client( + &arguments, + ) { + Ok(status) => ExitCode::from(u8::try_from(status).unwrap_or(1)), + Err(error) => { + eprintln!("{error}"); + ExitCode::from(1) + } + }; + } match run() { Ok(()) => ExitCode::SUCCESS, Err(error) => { diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index e7636e9d..adfbe04a 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -8,11 +8,16 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, HashMap}; use std::fs; -use std::io::Write; +use std::io::{BufRead, BufReader, Read, Write}; +#[cfg(unix)] +use std::net::Shutdown; +#[cfg(unix)] +use std::os::unix::net::{UnixListener, UnixStream}; use std::path::{Component, Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use workdir::WorkdirSessionResource; @@ -1133,11 +1138,419 @@ impl Drop for RepositorySshAgent { } } +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(tag = "channel", rename_all = "snake_case")] +enum RepositorySshBrokerHeader { + Data { + request_id: String, + args: Vec, + }, + Stderr { + request_id: String, + }, + Status { + request_id: String, + }, +} + +#[derive(Default)] +struct PendingRepositorySshRequest { + created_at: Option, + args: Option>, + data: Option, + stderr: Option, + status: Option, +} + +impl std::fmt::Debug for PendingRepositorySshRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PendingRepositorySshRequest") + .field("args", &self.args) + .field("data_ready", &self.data.is_some()) + .field("stderr_ready", &self.stderr.is_some()) + .field("status_ready", &self.status.is_some()) + .finish() + } +} + +#[derive(Debug)] +struct RepositoryReadOnlySshBroker { + socket: PathBuf, + stopped: Arc, + thread: Mutex>>, +} + +impl RepositoryReadOnlySshBroker { + fn start( + root: &Path, + agent: Arc, + known_hosts: PathBuf, + ) -> Result { + let socket = root.join("b.sock"); + let listener = UnixListener::bind(&socket).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "read-only Repository SSH broker could not be prepared", + ) + })?; + listener.set_nonblocking(true).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "read-only Repository SSH broker could not be prepared", + ) + })?; + let stopped = Arc::new(AtomicBool::new(false)); + let broker_stopped = Arc::clone(&stopped); + let thread = std::thread::spawn(move || { + let mut pending = HashMap::::new(); + while !broker_stopped.load(Ordering::Acquire) { + pending.retain(|_, request| { + request + .created_at + .is_some_and(|created_at| created_at.elapsed() < Duration::from_secs(5)) + }); + match listener.accept() { + Ok((stream, _)) => { + if let Some((request_id, request)) = + receive_repository_ssh_channel(stream, &mut pending) + { + let request_agent = Arc::clone(&agent); + let request_known_hosts = known_hosts.clone(); + std::thread::spawn(move || { + run_brokered_read_only_ssh( + request, + &request_agent.socket, + &request_known_hosts, + ); + }); + pending.remove(&request_id); + } + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(_) => break, + } + } + }); + Ok(Self { + socket, + stopped, + thread: Mutex::new(Some(thread)), + }) + } + + fn stop(&self) { + if self.stopped.swap(true, Ordering::AcqRel) { + return; + } + let _ = UnixStream::connect(&self.socket); + if let Ok(mut thread) = self.thread.lock() + && let Some(thread) = thread.take() + { + let _ = thread.join(); + } + let _ = fs::remove_file(&self.socket); + } +} + +impl Drop for RepositoryReadOnlySshBroker { + fn drop(&mut self) { + self.stop(); + } +} + +fn receive_repository_ssh_channel( + stream: UnixStream, + pending: &mut HashMap, +) -> Option<(String, PendingRepositorySshRequest)> { + let mut header = String::new(); + if BufReader::new(stream.try_clone().ok()?) + .take(8_193) + .read_line(&mut header) + .ok()? + == 0 + || header.len() > 8_192 + { + return None; + } + let header: RepositorySshBrokerHeader = serde_json::from_str(header.trim_end()).ok()?; + let request_id = match &header { + RepositorySshBrokerHeader::Data { request_id, .. } + | RepositorySshBrokerHeader::Stderr { request_id } + | RepositorySshBrokerHeader::Status { request_id } => request_id.clone(), + }; + if request_id.is_empty() || request_id.len() > 128 { + return None; + } + if !pending.contains_key(&request_id) && pending.len() >= 16 { + return None; + } + let request = pending.entry(request_id.clone()).or_default(); + request.created_at.get_or_insert_with(Instant::now); + match header { + RepositorySshBrokerHeader::Data { args, .. } => { + request.args = Some(args); + request.data = Some(stream); + } + RepositorySshBrokerHeader::Stderr { .. } => request.stderr = Some(stream), + RepositorySshBrokerHeader::Status { .. } => request.status = Some(stream), + } + if request.args.is_some() + && request.data.is_some() + && request.stderr.is_some() + && request.status.is_some() + { + pending + .remove(&request_id) + .map(|request| (request_id, request)) + } else { + None + } +} + +fn run_brokered_read_only_ssh( + mut request: PendingRepositorySshRequest, + agent_socket: &Path, + known_hosts: &Path, +) { + let args = request.args.take().unwrap_or_default(); + let mut data = request.data.take().expect("complete broker request data"); + let mut stderr_stream = request + .stderr + .take() + .expect("complete broker request stderr"); + let mut status_stream = request + .status + .take() + .expect("complete broker request status"); + let status = if !validate_read_only_ssh_args(&args) { + let _ = stderr_stream.write_all(b"read-only Repository SSH operation denied\n"); + let _ = stderr_stream.shutdown(Shutdown::Write); + let _ = data.shutdown(Shutdown::Both); + 126 + } else if args.iter().any(|argument| argument == "-G") { + let _ = stderr_stream.shutdown(Shutdown::Write); + let _ = data.shutdown(Shutdown::Both); + 0 + } else { + let mut command = Command::new("ssh"); + command + .args(["-F", "/dev/null"]) + .args(["-o", "BatchMode=yes"]) + .args(["-o", "IdentitiesOnly=no"]) + .args(["-o", "IdentityFile=/dev/null"]) + .args(["-o", "IdentityAgent=SSH_AUTH_SOCK"]) + .args(["-o", "StrictHostKeyChecking=yes"]) + .arg("-o") + .arg(format!( + "UserKnownHostsFile={}", + known_hosts.to_string_lossy() + )) + .args(["-o", "ClearAllForwardings=yes"]) + .args(["-o", "PermitLocalCommand=no"]) + .args(&args) + .env("SSH_AUTH_SOCK", agent_socket) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + match command.spawn() { + Ok(mut child) => { + let mut child_stdin = child.stdin.take().expect("piped ssh stdin"); + let mut input = data.try_clone().expect("clone broker data stream"); + let input_thread = std::thread::spawn(move || { + let _ = std::io::copy(&mut input, &mut child_stdin); + }); + let mut child_stdout = child.stdout.take().expect("piped ssh stdout"); + let output_thread = std::thread::spawn(move || { + let _ = std::io::copy(&mut child_stdout, &mut data); + let _ = data.shutdown(Shutdown::Write); + }); + let mut child_stderr = child.stderr.take().expect("piped ssh stderr"); + let error_thread = std::thread::spawn(move || { + let _ = std::io::copy(&mut child_stderr, &mut stderr_stream); + let _ = stderr_stream.shutdown(Shutdown::Write); + }); + let status = child + .wait() + .ok() + .and_then(|status| status.code()) + .unwrap_or(1); + let _ = input_thread.join(); + let _ = output_thread.join(); + let _ = error_thread.join(); + status + } + Err(_) => 1, + } + }; + let _ = writeln!(status_stream, "{status}"); + let _ = status_stream.shutdown(Shutdown::Write); +} + +fn validate_read_only_ssh_args(args: &[String]) -> bool { + let mut index = 0; + let mut probe = false; + let mut positional = Vec::new(); + while index < args.len() { + match args[index].as_str() { + "-G" => probe = true, + "-4" | "-6" | "-v" | "-vv" | "-vvv" => {} + "-p" => { + index += 1; + if index >= args.len() + || args[index].is_empty() + || !args[index].bytes().all(|byte| byte.is_ascii_digit()) + { + return false; + } + } + "-l" => { + index += 1; + if index >= args.len() || !is_safe_ssh_destination(&args[index]) { + return false; + } + } + "-o" => { + index += 1; + if index >= args.len() || !is_safe_git_ssh_option(&args[index]) { + return false; + } + } + option if option.starts_with("-o") => { + if !is_safe_git_ssh_option(&option[2..]) { + return false; + } + } + option if option.starts_with('-') => return false, + value => positional.push(value), + } + index += 1; + } + if probe { + positional.len() == 1 && is_safe_ssh_destination(positional[0]) + } else { + positional.len() == 2 + && is_safe_ssh_destination(positional[0]) + && is_safe_read_only_git_command(positional[1]) + } +} + +fn is_safe_git_ssh_option(option: &str) -> bool { + option == "SendEnv=GIT_PROTOCOL" + || option + .strip_prefix("SetEnv=GIT_PROTOCOL=") + .is_some_and(|value| { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-')) + }) +} + +fn is_safe_ssh_destination(value: &str) -> bool { + !value.is_empty() + && value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!(byte, b'.' | b'-' | b'_' | b'@' | b':' | b'[' | b']') + }) +} + +fn is_safe_read_only_git_command(command: &str) -> bool { + let Some(argument) = command + .strip_prefix("git-upload-pack ") + .or_else(|| command.strip_prefix("git-upload-archive ")) + else { + return false; + }; + let argument = argument + .strip_prefix('\'') + .and_then(|argument| argument.strip_suffix('\'')) + .unwrap_or(argument); + !argument.is_empty() + && argument.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'/' | b'.' | b'-' | b'_' | b'@' | b':' | b'+' | b'~' | b' ' + ) + }) +} + +pub fn run_repository_read_only_ssh_client(arguments: &[String]) -> Result { + let (socket, ssh_args) = arguments + .split_first() + .ok_or_else(|| "read-only Repository SSH broker socket is missing".to_string())?; + let request_id = format!( + "{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + let mut data = connect_repository_ssh_broker_channel( + socket, + &RepositorySshBrokerHeader::Data { + request_id: request_id.clone(), + args: ssh_args.to_vec(), + }, + )?; + let mut stderr_stream = connect_repository_ssh_broker_channel( + socket, + &RepositorySshBrokerHeader::Stderr { + request_id: request_id.clone(), + }, + )?; + let mut status_stream = connect_repository_ssh_broker_channel( + socket, + &RepositorySshBrokerHeader::Status { request_id }, + )?; + let mut input = data + .try_clone() + .map_err(|_| "read-only Repository SSH broker input failed".to_string())?; + let input_thread = std::thread::spawn(move || { + let _ = std::io::copy(&mut std::io::stdin(), &mut input); + let _ = input.shutdown(Shutdown::Write); + }); + let error_thread = std::thread::spawn(move || { + let _ = std::io::copy(&mut stderr_stream, &mut std::io::stderr()); + }); + std::io::copy(&mut data, &mut std::io::stdout()) + .map_err(|_| "read-only Repository SSH broker output failed".to_string())?; + let _ = input_thread.join(); + let _ = error_thread.join(); + let mut status = String::new(); + status_stream + .read_to_string(&mut status) + .map_err(|_| "read-only Repository SSH broker status failed".to_string())?; + status + .trim() + .parse::() + .map_err(|_| "read-only Repository SSH broker status was invalid".to_string()) +} + +fn connect_repository_ssh_broker_channel( + socket: &str, + header: &RepositorySshBrokerHeader, +) -> Result { + let mut stream = UnixStream::connect(socket) + .map_err(|_| "read-only Repository SSH broker is unavailable".to_string())?; + serde_json::to_writer(&mut stream, header) + .map_err(|_| "read-only Repository SSH broker request failed".to_string())?; + stream + .write_all(b"\n") + .map_err(|_| "read-only Repository SSH broker request failed".to_string())?; + Ok(stream) +} + #[derive(Debug)] struct RepositoryCommandAccess { root: PathBuf, ssh_command: PathBuf, - agent: RepositorySshAgent, + agent: Arc, + read_only_broker: Option, } impl RepositoryCommandAccess { @@ -1189,27 +1602,49 @@ impl RepositoryCommandAccess { let known_hosts = root.join("known_hosts"); let ssh_command = root.join("ssh-command"); write_owner_only(&known_hosts, ssh.known_hosts_entry.expose().as_bytes())?; - let agent = RepositorySshAgent::start(runtime_root, operation_id, ssh)?; - let read_only_guard = if ssh.access == workspace_api::RepositoryAccessMode::ReadOnly { - "case \" $* \" in\n *\" -G \"*|*\" git-upload-pack \"*|*\" git-upload-archive \"*) ;;\n *) echo 'read-only Repository SSH operation denied' >&2; exit 126 ;;\nesac\n" + let agent = Arc::new(RepositorySshAgent::start(runtime_root, operation_id, ssh)?); + let read_only_broker = if ssh.access == workspace_api::RepositoryAccessMode::ReadOnly { + Some(RepositoryReadOnlySshBroker::start( + &root, + Arc::clone(&agent), + known_hosts.clone(), + )?) } else { - "" + None + }; + let script = if let Some(broker) = read_only_broker.as_ref() { + let executable = std::env::current_exe().map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "read-only Repository SSH broker client could not be resolved", + ) + })?; + format!( + "#!/bin/sh\nexec {} __repository-read-only-ssh {} \"$@\"\n", + shell_quote_path(&executable)?, + shell_quote_path(&broker.socket)?, + ) + } else { + format!( + "#!/bin/sh\nexport SSH_AUTH_SOCK={}\nexec ssh -F /dev/null -o BatchMode=yes -o IdentitiesOnly=no -o IdentityFile=/dev/null -o StrictHostKeyChecking=yes -o UserKnownHostsFile={} \"$@\"\n", + shell_quote_path(&agent.socket)?, + shell_quote_path(&known_hosts)?, + ) }; - let script = format!( - "#!/bin/sh\n{read_only_guard}export SSH_AUTH_SOCK={}\nexec ssh -F /dev/null -o BatchMode=yes -o IdentitiesOnly=no -o IdentityFile=/dev/null -o StrictHostKeyChecking=yes -o UserKnownHostsFile={} \"$@\"\n", - shell_quote_path(&agent.socket)?, - shell_quote_path(&known_hosts)?, - ); write_owner_only(&ssh_command, script.as_bytes())?; set_file_owner_executable(&ssh_command)?; Ok(Self { root, ssh_command, agent, + read_only_broker, }) } fn stop(&self) { + if let Some(broker) = self.read_only_broker.as_ref() { + broker.stop(); + } self.agent.stop(); let _ = fs::remove_dir_all(&self.root); } @@ -2127,12 +2562,51 @@ mod tests { assert_eq!(rebound_environment["YOI_REPOSITORY_ACCESS"], "read_only"); assert!(!rebound_environment.contains_key("SSH_AUTH_SOCK")); let read_only_ssh = &rebound_environment["GIT_SSH_COMMAND"]; - let denied = Command::new(read_only_ssh) - .args(["example.test", "git-receive-pack 'repo.git'"]) - .env_remove("SSH_AUTH_SOCK") - .status() - .unwrap(); - assert_eq!(denied.code(), Some(126)); + let read_only_policy = fs::read_to_string(read_only_ssh).unwrap(); + assert!(read_only_policy.contains("__repository-read-only-ssh")); + assert!(!read_only_policy.contains("SSH_AUTH_SOCK")); + assert!(!read_only_policy.contains(".repository-agents")); + assert!(!read_only_policy.contains("known_hosts")); + assert!(!validate_read_only_ssh_args(&[ + "example.test".to_string(), + "git-receive-pack 'repo.git'".to_string(), + ])); + let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR)) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.path().join("b.sock")) + .find(|path| path.exists()) + .expect("read-only broker socket"); + assert_eq!( + run_repository_read_only_ssh_client(&[ + broker_socket.to_string_lossy().to_string(), + "-G".to_string(), + "example.test".to_string(), + ]) + .unwrap(), + 0 + ); + assert_eq!( + run_repository_read_only_ssh_client(&[ + broker_socket.to_string_lossy().to_string(), + "example.test".to_string(), + "git-receive-pack 'repo.git'".to_string(), + ]) + .unwrap(), + 126 + ); + assert!(!validate_read_only_ssh_args(&[ + "-o".to_string(), + "ProxyCommand=sh -c exploit".to_string(), + "example.test".to_string(), + "git-upload-pack 'repo.git'".to_string(), + ])); + assert!(validate_read_only_ssh_args(&[ + "-o".to_string(), + "SendEnv=GIT_PROTOCOL".to_string(), + "example.test".to_string(), + "git-upload-pack 'repo.git'".to_string(), + ])); assert_eq!( git_stdout( rebound.root(), @@ -2142,6 +2616,7 @@ mod tests { "yoi-read-only://repository-push-disabled" ); drop(rebound); + assert!(!broker_socket.exists()); let mut read_write = rotated; read_write.operation_id = "operation-agent-read-write".to_string(); From b644971d453b487c131a0276d42b8caa37f3f654 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 17:23:43 +0900 Subject: [PATCH 08/12] fix: preauthorize repository access without persisting secrets --- crates/worker-runtime/src/runtime.rs | 77 ++++++- .../worker-runtime/src/working_directory.rs | 196 ++++++++++++++---- crates/workspace-server/src/server.rs | 84 ++++++++ 3 files changed, 313 insertions(+), 44 deletions(-) diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 588f3338..17d302d1 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -585,12 +585,13 @@ impl Runtime { let worker_id = request.worker_id; let worker_ref = WorkerRef::new(worker_id); + let durable_request = durable_create_worker_request(&request); let record = WorkerRecord { worker_ref: worker_ref.clone(), worker_id: worker_id.clone(), status: WorkerStatus::Stopped, workspace_id: scope.map(|scope| scope.workspace_id.clone()), - request: request.clone(), + request: durable_request, run_generation: 1, working_directory: None, execution_handle: None, @@ -2733,6 +2734,16 @@ fn worker_status_from_run_state(run_state: WorkerExecutionRunState) -> WorkerSta } } +fn durable_create_worker_request(request: &CreateWorkerRequest) -> CreateWorkerRequest { + let mut durable = request.clone(); + if let Some(working_directory) = durable.working_directory_request.as_mut() + && let Some(materialization) = working_directory.materialization.as_mut() + { + materialization.ssh = None; + } + durable +} + fn requested_primary_workdir_id(request: &CreateWorkerRequest) -> Option<&str> { request .working_directory @@ -2903,7 +2914,9 @@ fn subscription_worker_state(status: WorkerStatus) -> SubscriptionWorkerState { mod tests { use super::*; use crate::catalog::{ - ConfigBundleRef, ProfileSelector, WorkingDirectoryClaim, WorkspaceApiRef, + ConfigBundleRef, MaterializerKind, ProfileSelector, RepositoryMaterializationContext, + RepositorySshMaterializationAccess, SensitiveString, WorkingDirectoryClaim, + WorkingDirectoryRepository, WorkingDirectoryRequest, WorkspaceApiRef, }; use crate::config_bundle::{ ConfigBundle, ConfigBundleMetadata, ConfigBundleProvenance, ConfigDeclaration, @@ -3134,6 +3147,66 @@ mod tests { } } + #[test] + fn durable_worker_request_omits_repository_credentials() { + let mut request = task_request("worker-secret-redaction"); + request.working_directory_request = Some(WorkingDirectoryRequest { + repository: WorkingDirectoryRepository { + id: "repository-1".to_string(), + provider: "git".to_string(), + source: workspace_api::RepositorySource { + kind: workspace_api::RepositorySourceKind::Ssh, + uri: "ssh://git@example.test/repo.git".to_string(), + }, + source_revision: 1, + source_fingerprint: "sha256:source".to_string(), + selector: None, + }, + materializer: MaterializerKind::RuntimeGitCache, + backend_workdir_id: Some("working-directory-1".to_string()), + materialization: Some(RepositoryMaterializationContext { + workspace_id: "workspace-1".to_string(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-1".to_string(), + config_revision: 1, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 0, + ssh: Some(RepositorySshMaterializationAccess { + credential_id: "credential-1".to_string(), + credential_revision: 1, + host_trust_id: "host-trust-1".to_string(), + host_trust_revision: 1, + access: workspace_api::RepositoryAccessMode::ReadOnly, + expires_at_epoch_seconds: u64::MAX, + private_key: SensitiveString::new("private-key-bytes"), + known_hosts_entry: SensitiveString::new("known-hosts-entry"), + }), + }), + }); + + let durable = durable_create_worker_request(&request); + + assert!( + request + .working_directory_request + .as_ref() + .and_then(|working_directory| working_directory.materialization.as_ref()) + .and_then(|materialization| materialization.ssh.as_ref()) + .is_some() + ); + assert!( + durable + .working_directory_request + .as_ref() + .and_then(|working_directory| working_directory.materialization.as_ref()) + .and_then(|materialization| materialization.ssh.as_ref()) + .is_none() + ); + let serialized = serde_json::to_string(&durable).unwrap(); + assert!(!serialized.contains("private-key-bytes")); + assert!(!serialized.contains("known-hosts-entry")); + } + fn scoped_task_request(objective: &str, workspace_id: &str) -> CreateWorkerRequest { let mut request = task_request(objective); request.workspace_api = Some(WorkspaceApiRef { diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index adfbe04a..375faaaf 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -395,6 +395,44 @@ impl RuntimeGitCacheMaterializer { }) } + fn cache_repository_access( + &self, + working_directory_id: &str, + ssh: &RepositorySshMaterializationAccess, + ) -> Result<(), WorkingDirectoryDiagnostic> { + validate_ssh_materialization_access(ssh)?; + let working_directory_id = working_directory_id.to_string(); + let credential_revision = ssh.credential_revision; + let expires_at = ssh.expires_at_epoch_seconds; + self.repository_access + .lock() + .map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_unavailable", + "Runtime Repository access state is unavailable", + ) + })? + .insert(working_directory_id.clone(), ssh.clone()); + let repository_access = self.repository_access.clone(); + std::thread::spawn(move || { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + if expires_at > now { + std::thread::sleep(Duration::from_secs(expires_at - now)); + } + if let Ok(mut access) = repository_access.lock() + && access + .get(&working_directory_id) + .is_some_and(|access| access.credential_revision == credential_revision) + { + access.remove(&working_directory_id); + } + }); + Ok(()) + } + fn bind_repository_access( &self, working_directory_id: &str, @@ -657,13 +695,62 @@ impl RuntimeGitCacheMaterializer { Ok(cache_path) } + fn request_with_authorized_repository_access( + &self, + working_directory_id: &str, + request: &WorkingDirectoryRequest, + ) -> Result { + let mut request = request.clone(); + if request.repository.provider != "git" + || request.repository.source.kind != workspace_api::RepositorySourceKind::Ssh + || request + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.as_ref()) + .is_some() + { + return Ok(request); + } + let access = self + .repository_access + .lock() + .map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_unavailable", + "Runtime Repository access state is unavailable", + ) + })? + .get(working_directory_id) + .cloned() + .ok_or_else(|| { + WorkingDirectoryDiagnostic::new( + "working_directory_remote_repository_access_required", + "SSH Repository materialization requires pre-authorized credential and host-trust authority", + ) + })?; + validate_ssh_materialization_access(&access)?; + request + .materialization + .as_mut() + .ok_or_else(|| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_materialization_authority_required", + "remote Repository materialization requires Backend-authored operation authority", + ) + })? + .ssh = Some(access); + Ok(request) + } + fn materialize_with_working_directory_id( &self, working_directory_id: String, request: &WorkingDirectoryRequest, ) -> Result { validate_working_directory_id(&working_directory_id)?; - let repository_cache = self.ensure_repository_cache(request)?; + let request = + self.request_with_authorized_repository_access(&working_directory_id, request)?; + let repository_cache = self.ensure_repository_cache(&request)?; let selector = request.repository.selector.as_deref().unwrap_or("HEAD"); let resolved_commit = resolve_cached_commit(&repository_cache, selector)?; let tree_spec = format!("{resolved_commit}^{{tree}}"); @@ -725,7 +812,7 @@ impl RuntimeGitCacheMaterializer { materializer_kind: MaterializerKind::RuntimeGitCache, repository_source_revision: Some(request.repository.source_revision), repository_source_fingerprint: Some(request.repository.source_fingerprint.clone()), - repository_cache_key: Some(Self::repository_cache_key(request)), + repository_cache_key: Some(Self::repository_cache_key(&request)), cache_generation: context .map(|value| value.cache_generation) .unwrap_or_default(), @@ -741,7 +828,7 @@ impl RuntimeGitCacheMaterializer { }, cleanup_target: WorkingDirectoryCleanupTarget { kind: "runtime_git_cache_worktree".to_string(), - working_directory_id, + working_directory_id: working_directory_id.clone(), repository_id: request.repository.id.clone(), }, status: WorkingDirectoryStatusKind::Active, @@ -760,6 +847,16 @@ impl RuntimeGitCacheMaterializer { let _ = fs::remove_dir_all(&working_directory_root); return Err(error); } + if let Some(ssh) = request + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.as_ref()) + && let Err(error) = self.cache_repository_access(&working_directory_id, ssh) + { + remove_cached_worktree(&repository_cache, &worktree_root); + let _ = fs::remove_dir_all(&working_directory_root); + return Err(error); + } Ok(binding) } } @@ -797,7 +894,13 @@ impl WorkingDirectoryMaterializer for RuntimeGitCacheMaterializer { ) })?; validate_ssh_materialization_access(ssh)?; - let mut binding = self.read_binding(&request.working_directory_id)?; + let mut binding = match self.read_binding(&request.working_directory_id) { + Ok(binding) => binding, + Err(error) if error.code == "working_directory_not_found" => { + return self.cache_repository_access(&request.working_directory_id, ssh); + } + Err(error) => return Err(error), + }; if binding .working_directory .evidence @@ -814,36 +917,7 @@ impl WorkingDirectoryMaterializer for RuntimeGitCacheMaterializer { binding.working_directory.evidence.credential_revision = Some(ssh.credential_revision); binding.working_directory.evidence.host_trust_revision = Some(ssh.host_trust_revision); self.write_record(&binding)?; - let working_directory_id = request.working_directory_id.clone(); - let credential_revision = ssh.credential_revision; - let expires_at = ssh.expires_at_epoch_seconds; - self.repository_access - .lock() - .map_err(|_| { - WorkingDirectoryDiagnostic::new( - "working_directory_repository_access_unavailable", - "Runtime Repository access state is unavailable", - ) - })? - .insert(working_directory_id.clone(), ssh.clone()); - let repository_access = self.repository_access.clone(); - std::thread::spawn(move || { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - if expires_at > now { - std::thread::sleep(Duration::from_secs(expires_at - now)); - } - if let Ok(mut access) = repository_access.lock() - && access - .get(&working_directory_id) - .is_some_and(|access| access.credential_revision == credential_revision) - { - access.remove(&working_directory_id); - } - }); - Ok(()) + self.cache_repository_access(&request.working_directory_id, ssh) } fn bind_working_directory( @@ -2508,8 +2582,52 @@ mod tests { drop(command_access); assert!(!operation_socket.exists()); + let initial_materialization = request.materialization.clone().unwrap(); + let working_directory_id = "working-directory-agent".to_string(); + materializer + .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { + working_directory_id: working_directory_id.clone(), + materialization: initial_materialization.clone(), + }) + .unwrap(); + request.backend_workdir_id = Some(working_directory_id.clone()); + request.materialization.as_mut().unwrap().ssh = None; + let mut authorized_ssh_request = request.clone(); + authorized_ssh_request.repository.source = workspace_api::RepositorySource { + kind: workspace_api::RepositorySourceKind::Ssh, + uri: "ssh://git@example.test/repo.git".to_string(), + }; + assert!( + materializer + .request_with_authorized_repository_access( + &working_directory_id, + &authorized_ssh_request, + ) + .unwrap() + .materialization + .unwrap() + .ssh + .is_some() + ); let created = materializer.create(&request).unwrap(); let id = created.working_directory.id; + assert_eq!(id, working_directory_id); + let mut ssh_backed_binding = materializer.read_binding(&id).unwrap(); + ssh_backed_binding + .working_directory + .evidence + .credential_revision = Some(1); + ssh_backed_binding + .working_directory + .evidence + .host_trust_revision = Some(1); + materializer.write_record(&ssh_backed_binding).unwrap(); + materializer + .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { + working_directory_id: id.clone(), + materialization: initial_materialization.clone(), + }) + .unwrap(); assert_eq!(materializer.list_working_directories().unwrap().len(), 1); assert_eq!( fs::read_dir(runtime_root.path().join(".repository-agents")) @@ -2517,12 +2635,6 @@ mod tests { .unwrap_or_default(), 0 ); - materializer - .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { - working_directory_id: id.clone(), - materialization: request.materialization.clone().unwrap(), - }) - .unwrap(); let binding = materializer.bind_working_directory(&id, None).unwrap(); let environment = binding.command_environment(); let socket = PathBuf::from(environment["SSH_AUTH_SOCK"].clone()); @@ -2543,7 +2655,7 @@ mod tests { .code, "working_directory_remote_repository_access_required" ); - let mut rotated = request.materialization.clone().unwrap(); + let mut rotated = initial_materialization.clone(); rotated.operation_id = "operation-agent-rotated".to_string(); rotated.ssh.as_mut().unwrap().credential_revision = 2; rotated.ssh.as_mut().unwrap().access = workspace_api::RepositoryAccessMode::ReadOnly; @@ -2643,7 +2755,7 @@ mod tests { ); drop(rebound); - let mut expired = request.materialization.clone().unwrap(); + let mut expired = initial_materialization; expired.ssh.as_mut().unwrap().expires_at_epoch_seconds = 1; assert_eq!( materializer diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index d5cd876a..eb324532 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1227,6 +1227,35 @@ pub async fn build_workspace_server_router( ))) } +fn take_new_workdir_repository_access( + request: &mut WorkerSpawnRequest, +) -> ApiResult> { + let Some(working_directory) = request.resolved_working_directory_request.as_mut() else { + return Ok(None); + }; + let Some(materialization) = working_directory.materialization.as_mut() else { + return Ok(None); + }; + if materialization.ssh.is_none() { + return Ok(None); + } + let working_directory_id = working_directory + .backend_workdir_id + .clone() + .ok_or_else(|| { + Error::Config( + "repository access authorization requires a Backend WorkingDirectory id" + .to_string(), + ) + })?; + let access = worker_runtime::catalog::WorkingDirectoryRepositoryAccessRequest { + working_directory_id, + materialization: materialization.clone(), + }; + materialization.ssh = None; + Ok(Some(access)) +} + impl WorkspaceApi { pub fn with_config_schema_provider( mut self, @@ -1437,6 +1466,11 @@ impl WorkspaceApi { self.validate_worker_spawn_repository_scope(&request)?; let workspace_api = self.workspace_api_ref(runtime_id); request.resolved_workspace_api = Some(workspace_api.clone()); + if let Some(access) = take_new_workdir_repository_access(&mut request)? { + self.runtime + .authorize_working_directory_repository_access(runtime_id, access) + .map_err(RuntimeRegistryError::into_error)?; + } if let Some(working_directory) = request.resolved_working_directory.as_ref() && let Some(access) = repository_access_request_for_workdir( self, @@ -15387,6 +15421,56 @@ mod tests { api.validate_worker_spawn_repository_scope(&workdir_flow_launch) .is_err() ); + + let mut repository_access_launch = workdir_flow_launch; + let working_directory = repository_access_launch + .resolved_working_directory_request + .as_mut() + .unwrap(); + working_directory.backend_workdir_id = Some("working-directory-1".to_string()); + working_directory.materialization = + Some(worker_runtime::catalog::RepositoryMaterializationContext { + workspace_id: api.config.workspace_id.clone(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-1".to_string(), + config_revision: 1, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 0, + ssh: Some( + worker_runtime::catalog::RepositorySshMaterializationAccess { + credential_id: "credential-1".to_string(), + credential_revision: 1, + host_trust_id: "host-trust-1".to_string(), + host_trust_revision: 1, + access: workspace_api::RepositoryAccessMode::ReadOnly, + expires_at_epoch_seconds: u64::MAX, + private_key: worker_runtime::catalog::SensitiveString::new( + "private-key-bytes", + ), + known_hosts_entry: worker_runtime::catalog::SensitiveString::new( + "known-hosts-entry", + ), + }, + ), + }); + + let access = take_new_workdir_repository_access(&mut repository_access_launch) + .unwrap() + .unwrap(); + + assert_eq!(access.working_directory_id, "working-directory-1"); + assert!(access.materialization.ssh.is_some()); + assert!( + repository_access_launch + .resolved_working_directory_request + .as_ref() + .and_then(|request| request.materialization.as_ref()) + .and_then(|materialization| materialization.ssh.as_ref()) + .is_none() + ); + let serialized = serde_json::to_string(&repository_access_launch).unwrap(); + assert!(!serialized.contains("private-key-bytes")); + assert!(!serialized.contains("known-hosts-entry")); } #[test] From 4b132a21e9a3558e0738f09c180c4eba7a1d05df Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 18:28:22 +0900 Subject: [PATCH 09/12] fix: bind repository SSH secrets to one-shot resources --- crates/worker-runtime/src/catalog.rs | 12 + crates/worker-runtime/src/http_server.rs | 3 +- crates/worker-runtime/src/main.rs | 29 ++- crates/worker-runtime/src/resource.rs | 61 ++++- crates/worker-runtime/src/runtime.rs | 221 +++++++++++++++++ .../worker-runtime/src/working_directory.rs | 98 ++++++++ crates/workspace-server/src/hosts.rs | 2 +- .../workspace-server/src/resource_broker.rs | 232 +++++++++++++++--- crates/workspace-server/src/server.rs | 102 +++++++- 9 files changed, 699 insertions(+), 61 deletions(-) diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index da18810e..44453575 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -116,6 +116,12 @@ impl Drop for SensitiveString { } } +impl Default for SensitiveString { + fn default() -> Self { + Self(String::new()) + } +} + impl std::fmt::Debug for SensitiveString { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter.write_str("[REDACTED]") @@ -130,7 +136,13 @@ pub struct RepositorySshMaterializationAccess { pub host_trust_revision: u64, pub access: workspace_api::RepositoryAccessMode, pub expires_at_epoch_seconds: u64, + pub repository_id: String, + pub repository_source_fingerprint: String, + pub repository_uri: String, + pub secret_resource: crate::resource::BackendResourceHandle, + #[serde(skip, default)] pub private_key: SensitiveString, + #[serde(skip, default)] pub known_hosts_entry: SensitiveString, } diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 4c179663..57adb0d9 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -538,7 +538,8 @@ async fn authorize_working_directory_repository_access( } state .runtime - .authorize_working_directory_repository_access(request) + .authorize_working_directory_repository_access_from_resource(request) + .await .map_err(RuntimeHttpRestError::runtime)?; Ok(Json(RuntimeHttpRepositoryAccessResponse { authorized: true, diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index 3e5d1e3c..d4ce88be 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -182,6 +182,9 @@ fn build_runtime(config: &ProcessConfig) -> Result { factory = factory.with_remote_worker_mutation_identity(identity); } } + let mut backend_resource_client: Option< + Arc, + > = None; if let Some(endpoint) = config.backend_resource_endpoint.clone() { let identity = runtime_auth.identity.as_ref().ok_or_else(|| { ProcessError::Auth( @@ -194,13 +197,15 @@ fn build_runtime(config: &ProcessConfig) -> Result { .to_owned(), )); }; - factory = factory.with_resource_client(Arc::new( + let client = Arc::new( worker_runtime::resource::HttpBackendResourceClient::new( endpoint, config.backend_resource_token.clone(), ) .with_runtime_request_source(identity, trusted_server.server_id.clone()), - )); + ); + factory = factory.with_resource_client(client.clone()); + backend_resource_client = Some(client); } let backend = Arc::new( WorkerRuntimeExecutionBackend::new(factory) @@ -210,10 +215,10 @@ fn build_runtime(config: &ProcessConfig) -> Result { )), ); - match &config.http.store { + let runtime = match &config.http.store { RuntimeHttpStoreSelection::Memory => { Runtime::with_execution_backend(runtime_options_from_http(&config.http), backend) - .map_err(ProcessError::Runtime) + .map_err(ProcessError::Runtime)? } RuntimeHttpStoreSelection::Fs { root } => { let mut options = FsRuntimeStoreOptions::new(root.clone()).with_runtime_id( @@ -226,12 +231,20 @@ fn build_runtime(config: &ProcessConfig) -> Result { ); options.display_name = config.http.display_name.clone(); Runtime::with_fs_store_and_execution_backend(options, backend) - .map_err(ProcessError::Runtime) + .map_err(ProcessError::Runtime)? } - _ => Err(ProcessError::usage( - "unsupported Runtime catalog store selection".to_string(), - )), + _ => { + return Err(ProcessError::usage( + "unsupported Runtime catalog store selection".to_string(), + )); + } + }; + if let Some(client) = backend_resource_client { + runtime + .install_backend_resource_client(client) + .map_err(ProcessError::Runtime)?; } + Ok(runtime) } fn runtime_options_from_http(config: &RuntimeHttpServerConfig) -> RuntimeOptions { diff --git a/crates/worker-runtime/src/resource.rs b/crates/worker-runtime/src/resource.rs index 60046fab..10234548 100644 --- a/crates/worker-runtime/src/resource.rs +++ b/crates/worker-runtime/src/resource.rs @@ -11,18 +11,46 @@ use std::sync::Mutex; pub const PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE: &str = "application/vnd.yoi.profile-source-archive+tar"; +pub const REPOSITORY_SSH_ACCESS_CONTENT_TYPE: &str = + "application/vnd.yoi.repository-ssh-access+json"; pub const DEFAULT_PROFILE_SOURCE_ARCHIVE_MAX_BYTES: u64 = 2 * 1024 * 1024; +pub const DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES: u64 = 64 * 1024; + +#[derive(Clone, Serialize, Deserialize)] +pub struct RepositorySshAccessSecret { + pub private_key: String, + pub known_hosts_entry: String, +} + +impl Drop for RepositorySshAccessSecret { + fn drop(&mut self) { + zeroize::Zeroize::zeroize(&mut self.private_key); + zeroize::Zeroize::zeroize(&mut self.known_hosts_entry); + } +} + +impl std::fmt::Debug for RepositorySshAccessSecret { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RepositorySshAccessSecret") + .field("private_key", &"[REDACTED]") + .field("known_hosts_entry", &"[REDACTED]") + .finish() + } +} #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum BackendResourceKind { ProfileSourceArchive, + RepositorySshAccess, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum BackendResourceOperation { FetchArchive, + FetchOnce, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -66,7 +94,7 @@ pub struct BackendResourceFetchRequest { pub audit_correlation_id: String, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct BackendResourceFetchResponse { pub kind: BackendResourceKind, pub resource_id: String, @@ -76,6 +104,29 @@ pub struct BackendResourceFetchResponse { pub audit_correlation_id: String, } +impl std::fmt::Debug for BackendResourceFetchResponse { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("BackendResourceFetchResponse") + .field("kind", &self.kind) + .field("resource_id", &self.resource_id) + .field("digest", &self.digest) + .field("content_type", &self.content_type) + .field( + "bytes", + &format_args!("[REDACTED; {} bytes]", self.bytes.len()), + ) + .field("audit_correlation_id", &self.audit_correlation_id) + .finish() + } +} + +impl Drop for BackendResourceFetchResponse { + fn drop(&mut self) { + self.bytes.fill(0); + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)] #[serde(tag = "code", rename_all = "snake_case")] pub enum BackendResourceError { @@ -248,7 +299,7 @@ pub fn build_profile_source_archive_fetch_request( pub fn profile_source_archive_from_response( handle: &BackendResourceHandle, - response: BackendResourceFetchResponse, + mut response: BackendResourceFetchResponse, ) -> Result { if handle.kind != BackendResourceKind::ProfileSourceArchive || response.kind != BackendResourceKind::ProfileSourceArchive @@ -263,7 +314,7 @@ pub fn profile_source_archive_from_response( if response.content_type != handle.content_type { return Err(BackendResourceError::ContentTypeMismatch { expected: handle.content_type.clone(), - actual: response.content_type, + actual: response.content_type.clone(), }); } let actual_bytes = response.bytes.len() as u64; @@ -278,7 +329,7 @@ pub fn profile_source_archive_from_response( return Err(BackendResourceError::DigestMismatch { expected: handle.digest.clone(), actual: if response.digest != handle.digest { - response.digest + response.digest.clone() } else { actual_digest }, @@ -296,7 +347,7 @@ pub fn profile_source_archive_from_response( } })?, }, - content: response.bytes, + content: std::mem::take(&mut response.bytes), }) } diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 17d302d1..a84ad4de 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -26,6 +26,10 @@ use crate::management::{ }; #[cfg(feature = "ws-server")] use crate::observation::{WorkerObservationCursor, WorkerObservationEvent}; +use crate::resource::{ + BackendResourceClient, BackendResourceError, BackendResourceFetchRequest, BackendResourceKind, + REPOSITORY_SSH_ACCESS_CONTENT_TYPE, RepositorySshAccessSecret, +}; #[cfg(feature = "fs-store")] use crate::retention::{ FsWorkerRetentionProvider, WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, @@ -172,6 +176,14 @@ impl Runtime { Ok(runtime) } + pub fn install_backend_resource_client( + &self, + client: Arc, + ) -> Result<(), RuntimeError> { + self.lock()?.backend_resource_client = Some(BackendResourceClientRef(client)); + Ok(()) + } + /// Create or restore a filesystem-backed Runtime. /// /// The store is scoped by `options.root`; if the directory already exists, @@ -385,6 +397,61 @@ impl Runtime { .map_err(RuntimeError::from) } + pub async fn authorize_working_directory_repository_access_from_resource( + &self, + mut request: WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), RuntimeError> { + let ssh = request.materialization.ssh.as_mut().ok_or_else(|| { + RuntimeError::InvalidRequest("Repository SSH access metadata is missing".to_string()) + })?; + if ssh.private_key.expose().is_empty() || ssh.known_hosts_entry.expose().is_empty() { + let (client, runtime_id) = { + let state = self.lock()?; + let client = state.backend_resource_client.clone().ok_or_else(|| { + RuntimeError::InvalidRequest( + "Backend Repository access resource client is unavailable".to_string(), + ) + })?; + let runtime_id = state.runtime_identity.clone().ok_or_else(|| { + RuntimeError::InvalidRequest("Runtime identity is unavailable".to_string()) + })?; + (client, runtime_id) + }; + let mut response = client + .0 + .fetch_resource(BackendResourceFetchRequest { + handle: ssh.secret_resource.clone(), + runtime_id, + worker_id: None, + audit_correlation_id: ssh.secret_resource.audit_correlation_id.clone(), + }) + .await + .map_err(repository_resource_error)?; + if response.kind != BackendResourceKind::RepositorySshAccess + || response.content_type != REPOSITORY_SSH_ACCESS_CONTENT_TYPE + || response.resource_id != ssh.secret_resource.resource_id + || response.digest != ssh.secret_resource.digest + || response.bytes.len() as u64 > ssh.secret_resource.max_bytes + { + return Err(RuntimeError::InvalidRequest( + "Backend Repository SSH access resource response was invalid".to_string(), + )); + } + let secret = serde_json::from_slice::(&response.bytes); + response.bytes.fill(0); + let mut secret = secret.map_err(|_| { + RuntimeError::InvalidRequest( + "Backend Repository SSH access resource payload was invalid".to_string(), + ) + })?; + ssh.private_key = + crate::catalog::SensitiveString::new(std::mem::take(&mut secret.private_key)); + ssh.known_hosts_entry = + crate::catalog::SensitiveString::new(std::mem::take(&mut secret.known_hosts_entry)); + } + self.authorize_working_directory_repository_access(request) + } + /// List Runtime-owned working directories through the attached execution backend. pub fn list_working_directories( &self, @@ -1862,6 +1929,15 @@ struct SubscriptionSink { lagged: Arc, } +#[derive(Clone)] +struct BackendResourceClientRef(Arc); + +impl std::fmt::Debug for BackendResourceClientRef { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("BackendResourceClientRef(..)") + } +} + #[derive(Debug)] struct RuntimeState { display_name: Option, @@ -1873,6 +1949,7 @@ struct RuntimeState { persistence: RuntimePersistence, status: RuntimeStatus, execution_backend: Option, + backend_resource_client: Option, #[cfg(feature = "fs-store")] next_diagnostic_id: u64, workers: BTreeMap, @@ -1900,6 +1977,7 @@ impl RuntimeState { persistence: RuntimePersistence::Memory, status: RuntimeStatus::Running, execution_backend: None, + backend_resource_client: None, #[cfg(feature = "fs-store")] next_diagnostic_id: 1, workers: BTreeMap::new(), @@ -1928,6 +2006,7 @@ impl RuntimeState { persistence: RuntimePersistence::Fs(store), status: RuntimeStatus::Running, execution_backend: None, + backend_resource_client: None, #[cfg(feature = "fs-store")] next_diagnostic_id: 1, workers: BTreeMap::new(), @@ -1979,6 +2058,7 @@ impl RuntimeState { persistence: RuntimePersistence::Fs(store), status: persisted.status, execution_backend: None, + backend_resource_client: None, next_diagnostic_id, workers, config_bundles: BTreeMap::new(), @@ -2734,6 +2814,23 @@ fn worker_status_from_run_state(run_state: WorkerExecutionRunState) -> WorkerSta } } +fn repository_resource_error(error: BackendResourceError) -> RuntimeError { + let category = match error { + BackendResourceError::Expired => "expired", + BackendResourceError::Unauthorized { .. } => "unauthorized", + BackendResourceError::UnsupportedKind => "unsupported_kind", + BackendResourceError::MissingResource => "missing_resource", + BackendResourceError::Oversized { .. } => "oversized", + BackendResourceError::DigestMismatch { .. } => "digest_mismatch", + BackendResourceError::ContentTypeMismatch { .. } => "content_type_mismatch", + BackendResourceError::InvalidResponse { .. } => "invalid_response", + BackendResourceError::Transport { .. } => "transport", + }; + RuntimeError::InvalidRequest(format!( + "Backend Repository SSH access resource fetch failed: {category}" + )) +} + fn durable_create_worker_request(request: &CreateWorkerRequest) -> CreateWorkerRequest { let mut durable = request.clone(); if let Some(working_directory) = durable.working_directory_request.as_mut() @@ -2926,6 +3023,8 @@ mod tests { WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle, WorkerExecutionRestoreRequest, WorkerExecutionRunState, }; + use crate::working_directory::WorkingDirectoryDiagnostic; + use async_trait::async_trait; use std::collections::BTreeMap; #[cfg(feature = "fs-store")] use std::sync::atomic::{AtomicU64, Ordering}; @@ -3178,6 +3277,10 @@ mod tests { host_trust_revision: 1, access: workspace_api::RepositoryAccessMode::ReadOnly, expires_at_epoch_seconds: u64::MAX, + repository_id: "repository-1".to_string(), + repository_source_fingerprint: "sha256:source".to_string(), + repository_uri: "ssh://git@example.test/repo.git".to_string(), + secret_resource: repository_resource_handle(), private_key: SensitiveString::new("private-key-bytes"), known_hosts_entry: SensitiveString::new("known-hosts-entry"), }), @@ -3207,6 +3310,94 @@ mod tests { assert!(!serialized.contains("known-hosts-entry")); } + fn repository_resource_handle() -> crate::resource::BackendResourceHandle { + crate::resource::BackendResourceHandle { + kind: crate::resource::BackendResourceKind::RepositorySshAccess, + workspace_id: "workspace-1".to_string(), + scope_id: Some("repository-ssh-access".to_string()), + runtime_id: Some("runtime-1".to_string()), + worker_id: None, + resource_id: "repository-access-1".to_string(), + digest: "opaque:repository-access-1".to_string(), + operation: crate::resource::BackendResourceOperation::FetchOnce, + expires_at_unix_seconds: i64::MAX, + nonce: "repository-access-1".to_string(), + revision: "1".to_string(), + generation: None, + max_bytes: crate::resource::DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES, + content_type: crate::resource::REPOSITORY_SSH_ACCESS_CONTENT_TYPE.to_string(), + redaction: crate::resource::ResourceRedactionPolicy::RuntimeInternalOnly, + audit_correlation_id: "repository-access-1".to_string(), + profile_source_graph: None, + } + } + + #[tokio::test] + async fn repository_access_resource_is_fetched_before_provider_authorization() { + let (runtime, backend) = runtime_and_backend(); + runtime.bind_runtime_identity("runtime-1").unwrap(); + let handle = repository_resource_handle(); + runtime + .install_backend_resource_client(Arc::new(TestRepositoryResourceClient { + response: Mutex::new(Some(crate::resource::BackendResourceFetchResponse { + kind: crate::resource::BackendResourceKind::RepositorySshAccess, + resource_id: handle.resource_id.clone(), + digest: handle.digest.clone(), + content_type: crate::resource::REPOSITORY_SSH_ACCESS_CONTENT_TYPE.to_string(), + bytes: serde_json::to_vec(&RepositorySshAccessSecret { + private_key: "private-key-bytes".to_string(), + known_hosts_entry: "known-hosts-entry".to_string(), + }) + .unwrap(), + audit_correlation_id: handle.audit_correlation_id.clone(), + })), + })) + .unwrap(); + let request = WorkingDirectoryRepositoryAccessRequest { + working_directory_id: "working-directory-1".to_string(), + materialization: RepositoryMaterializationContext { + workspace_id: "workspace-1".to_string(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-1".to_string(), + config_revision: 1, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 0, + ssh: Some(RepositorySshMaterializationAccess { + credential_id: "credential-1".to_string(), + credential_revision: 1, + host_trust_id: "host-trust-1".to_string(), + host_trust_revision: 1, + access: workspace_api::RepositoryAccessMode::ReadOnly, + expires_at_epoch_seconds: u64::MAX, + repository_id: "repository-1".to_string(), + repository_source_fingerprint: "sha256:source".to_string(), + repository_uri: "ssh://git@example.test/repo.git".to_string(), + secret_resource: handle, + private_key: SensitiveString::default(), + known_hosts_entry: SensitiveString::default(), + }), + }, + }; + + let replay = request.clone(); + runtime + .authorize_working_directory_repository_access_from_resource(request) + .await + .unwrap(); + assert!( + runtime + .authorize_working_directory_repository_access_from_resource(replay) + .await + .is_err() + ); + + let accesses = backend.repository_accesses.lock().unwrap(); + assert_eq!(accesses.len(), 1); + let access = accesses[0].materialization.ssh.as_ref().unwrap(); + assert_eq!(access.private_key.expose(), "private-key-bytes"); + assert_eq!(access.known_hosts_entry.expose(), "known-hosts-entry"); + } + fn scoped_task_request(objective: &str, workspace_id: &str) -> CreateWorkerRequest { let mut request = task_request(objective); request.workspace_api = Some(WorkspaceApiRef { @@ -3288,6 +3479,7 @@ mod tests { config_bundles: Mutex>>, contexts: Mutex>, dispatched_inputs: Mutex>, + repository_accesses: Mutex>, preserve_commit_ack_submission_id: AtomicBool, #[cfg(feature = "ws-server")] snapshots: Mutex>, @@ -3328,6 +3520,17 @@ mod tests { "test-execution-backend" } + fn authorize_working_directory_repository_access( + &self, + request: &WorkingDirectoryRepositoryAccessRequest, + ) -> Result<(), WorkingDirectoryDiagnostic> { + self.repository_accesses + .lock() + .unwrap() + .push(request.clone()); + Ok(()) + } + fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { self.run_generations .lock() @@ -3435,6 +3638,24 @@ mod tests { } } + struct TestRepositoryResourceClient { + response: Mutex>, + } + + #[async_trait] + impl BackendResourceClient for TestRepositoryResourceClient { + async fn fetch_resource( + &self, + _request: BackendResourceFetchRequest, + ) -> Result { + self.response + .lock() + .unwrap() + .take() + .ok_or(BackendResourceError::MissingResource) + } + } + fn runtime_with_backend() -> Runtime { let runtime = Runtime::with_execution_backend( RuntimeOptions::default(), diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index 375faaaf..edaa8f98 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -729,6 +729,12 @@ impl RuntimeGitCacheMaterializer { ) })?; validate_ssh_materialization_access(&access)?; + validate_repository_access_binding( + &access, + &request.repository.id, + &request.repository.source_fingerprint, + Some(request.repository.source.uri.as_str()), + )?; request .materialization .as_mut() @@ -912,6 +918,17 @@ impl WorkingDirectoryMaterializer for RuntimeGitCacheMaterializer { "Workdir is not backed by an SSH Repository", )); } + validate_repository_access_binding( + ssh, + &binding.working_directory.evidence.repository_id, + binding + .working_directory + .evidence + .repository_source_fingerprint + .as_deref() + .unwrap_or_default(), + None, + )?; binding.working_directory.evidence.operation_id = Some(request.materialization.operation_id.clone()); binding.working_directory.evidence.credential_revision = Some(ssh.credential_revision); @@ -1730,6 +1747,24 @@ impl Drop for RepositoryCommandAccess { } } +fn validate_repository_access_binding( + access: &RepositorySshMaterializationAccess, + repository_id: &str, + repository_source_fingerprint: &str, + repository_uri: Option<&str>, +) -> Result<(), WorkingDirectoryDiagnostic> { + if access.repository_id != repository_id + || access.repository_source_fingerprint != repository_source_fingerprint + || repository_uri.is_some_and(|repository_uri| access.repository_uri != repository_uri) + { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_binding_mismatch", + "Repository SSH access authority does not match the requested Repository source", + )); + } + Ok(()) +} + fn validate_ssh_materialization_access( access: &RepositorySshMaterializationAccess, ) -> Result<(), WorkingDirectoryDiagnostic> { @@ -2286,6 +2321,28 @@ mod tests { use crate::catalog::{RepositorySelector, WorkingDirectoryRepository}; use crate::identity::{WorkerId, WorkerRef}; + fn repository_resource_handle() -> crate::resource::BackendResourceHandle { + crate::resource::BackendResourceHandle { + kind: crate::resource::BackendResourceKind::RepositorySshAccess, + workspace_id: "workspace-1".to_string(), + scope_id: Some("repository-ssh-access".to_string()), + runtime_id: Some("runtime-1".to_string()), + worker_id: None, + resource_id: "repository-access-1".to_string(), + digest: "opaque:repository-access-1".to_string(), + operation: crate::resource::BackendResourceOperation::FetchOnce, + expires_at_unix_seconds: i64::MAX, + nonce: "repository-access-1".to_string(), + revision: "1".to_string(), + generation: None, + max_bytes: crate::resource::DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES, + content_type: crate::resource::REPOSITORY_SSH_ACCESS_CONTENT_TYPE.to_string(), + redaction: crate::resource::ResourceRedactionPolicy::RuntimeInternalOnly, + audit_correlation_id: "repository-access-1".to_string(), + profile_source_graph: None, + } + } + fn git(path: &Path, args: &[&str]) { let status = Command::new("git") .arg("-C") @@ -2523,6 +2580,10 @@ mod tests { host_trust_revision: 4, access: workspace_api::RepositoryAccessMode::ReadOnly, expires_at_epoch_seconds: u64::MAX, + repository_id: "repo-main".to_string(), + repository_source_fingerprint: "sha256:test".to_string(), + repository_uri: "ssh://git@example.test/repo.git".to_string(), + secret_resource: repository_resource_handle(), private_key: crate::catalog::SensitiveString::new("PRIVATE KEY secret bytes"), known_hosts_entry: crate::catalog::SensitiveString::new("host key secret bytes"), }; @@ -2560,6 +2621,10 @@ mod tests { host_trust_revision: 1, access: workspace_api::RepositoryAccessMode::ReadWrite, expires_at_epoch_seconds: u64::MAX, + repository_id: "repo-main".to_string(), + repository_source_fingerprint: "sha256:test".to_string(), + repository_uri: "ssh://git@example.test/repo.git".to_string(), + secret_resource: repository_resource_handle(), private_key: crate::catalog::SensitiveString::new( fs::read_to_string(&key_path).unwrap(), ), @@ -2609,6 +2674,31 @@ mod tests { .ssh .is_some() ); + let mut mismatched_source_request = authorized_ssh_request.clone(); + mismatched_source_request.repository.source.uri = + "ssh://git@example.test/other.git".to_string(); + assert_eq!( + materializer + .request_with_authorized_repository_access( + &working_directory_id, + &mismatched_source_request, + ) + .unwrap_err() + .code, + "working_directory_repository_access_binding_mismatch" + ); + let mut mismatched_repository_request = authorized_ssh_request.clone(); + mismatched_repository_request.repository.id = "repo-other".to_string(); + assert_eq!( + materializer + .request_with_authorized_repository_access( + &working_directory_id, + &mismatched_repository_request, + ) + .unwrap_err() + .code, + "working_directory_repository_access_binding_mismatch" + ); let created = materializer.create(&request).unwrap(); let id = created.working_directory.id; assert_eq!(id, working_directory_id); @@ -2794,6 +2884,10 @@ mod tests { host_trust_revision: 1, access: workspace_api::RepositoryAccessMode::ReadOnly, expires_at_epoch_seconds: u64::MAX, + repository_id: "repo-main".to_string(), + repository_source_fingerprint: "sha256:test".to_string(), + repository_uri: "ssh://git@example.test/repo.git".to_string(), + secret_resource: repository_resource_handle(), private_key: crate::catalog::SensitiveString::new("PRIVATE KEY placeholder"), known_hosts_entry: crate::catalog::SensitiveString::new( "example.test ssh-ed25519 placeholder", @@ -2884,6 +2978,10 @@ mod tests { host_trust_revision: 1, access: workspace_api::RepositoryAccessMode::ReadOnly, expires_at_epoch_seconds: u64::MAX, + repository_id: "repo-main".to_string(), + repository_source_fingerprint: "sha256:test".to_string(), + repository_uri: "ssh://git@example.test/repo.git".to_string(), + secret_resource: repository_resource_handle(), private_key: crate::catalog::SensitiveString::new("PRIVATE KEY placeholder"), known_hosts_entry: crate::catalog::SensitiveString::new( "other.test ssh-ed25519 placeholder", diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 4c67e799..18dd47b5 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -4500,7 +4500,7 @@ mod tests { let handle = bundle.profile_source_archive_handle.as_ref().unwrap(); assert!(bundle.profile_source_archive.is_none()); let response = broker - .fetch_profile_source_archive(worker_runtime::resource::BackendResourceFetchRequest { + .fetch_resource(worker_runtime::resource::BackendResourceFetchRequest { handle: handle.clone(), runtime_id: runtime_id.to_string(), worker_id: None, diff --git a/crates/workspace-server/src/resource_broker.rs b/crates/workspace-server/src/resource_broker.rs index 2549b52e..b619eaf9 100644 --- a/crates/workspace-server/src/resource_broker.rs +++ b/crates/workspace-server/src/resource_broker.rs @@ -9,7 +9,8 @@ use worker_runtime::resource::{ BackendResourceClient, BackendResourceError, BackendResourceFetchRequest, BackendResourceFetchResponse, BackendResourceHandle, BackendResourceKind, BackendResourceOperation, DEFAULT_PROFILE_SOURCE_ARCHIVE_MAX_BYTES, - PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE, ResourceRedactionPolicy, + DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES, PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE, + REPOSITORY_SSH_ACCESS_CONTENT_TYPE, RepositorySshAccessSecret, ResourceRedactionPolicy, }; #[derive(Clone, Default)] @@ -29,7 +30,34 @@ struct StoredResource { runtime_id: Option, worker: Option, handle: BackendResourceHandle, - archive: ProfileSourceArchive, + bytes: Vec, + archive: Option, + one_shot: bool, +} + +impl StoredResource { + fn byte_len(&self) -> usize { + self.archive + .as_ref() + .map(|archive| archive.content.len()) + .unwrap_or_else(|| self.bytes.len()) + } + + fn take_bytes(&mut self) -> Vec { + self.archive + .as_mut() + .map(|archive| std::mem::take(&mut archive.content)) + .unwrap_or_else(|| std::mem::take(&mut self.bytes)) + } +} + +impl Drop for StoredResource { + fn drop(&mut self) { + self.bytes.fill(0); + if let Some(archive) = self.archive.as_mut() { + archive.content.fill(0); + } + } } impl BackendResourceBroker { @@ -73,7 +101,9 @@ impl BackendResourceBroker { runtime_id, worker, handle: handle.clone(), - archive, + bytes: Vec::new(), + archive: Some(archive), + one_shot: false, }; if let Ok(mut resources) = self.resources.lock() { resources.insert(nonce, stored); @@ -81,6 +111,84 @@ impl BackendResourceBroker { handle } + pub fn issue_repository_ssh_access_handle( + &self, + workspace_id: impl Into, + runtime_id: &str, + resource_id: impl Into, + revision: impl Into, + expires_at_unix_seconds: i64, + secret: RepositorySshAccessSecret, + ) -> Result { + let bytes = + serde_json::to_vec(&secret).map_err(|error| BackendResourceError::InvalidResponse { + message: error.to_string(), + })?; + if bytes.len() as u64 > DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES { + return Err(BackendResourceError::Oversized { + max_bytes: DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES, + actual_bytes: bytes.len() as u64, + }); + } + let workspace_id = workspace_id.into(); + let resource_id = resource_id.into(); + let revision = revision.into(); + let nonce = Uuid::now_v7().to_string(); + let handle = BackendResourceHandle { + kind: BackendResourceKind::RepositorySshAccess, + workspace_id, + scope_id: Some("repository-ssh-access".to_string()), + runtime_id: Some(runtime_id.to_string()), + worker_id: None, + resource_id, + digest: format!("opaque:{nonce}"), + operation: BackendResourceOperation::FetchOnce, + expires_at_unix_seconds, + nonce: nonce.clone(), + revision, + generation: None, + max_bytes: DEFAULT_REPOSITORY_SSH_ACCESS_MAX_BYTES, + content_type: REPOSITORY_SSH_ACCESS_CONTENT_TYPE.to_string(), + redaction: ResourceRedactionPolicy::RuntimeInternalOnly, + audit_correlation_id: format!("repository-ssh-access-{nonce}"), + profile_source_graph: None, + }; + let stored = StoredResource { + runtime_id: Some(runtime_id.to_string()), + worker: None, + handle: handle.clone(), + bytes, + archive: None, + one_shot: true, + }; + let resource_key = nonce.clone(); + self.resources + .lock() + .map_err(|_| BackendResourceError::Transport { + message: "resource broker lock poisoned".to_string(), + })? + .insert(resource_key.clone(), stored); + if expires_at_unix_seconds != i64::MAX { + let resources = self.resources.clone(); + std::thread::spawn(move || { + let now = Utc::now().timestamp(); + if expires_at_unix_seconds > now { + std::thread::sleep(std::time::Duration::from_secs( + (expires_at_unix_seconds - now) as u64, + )); + } + if let Ok(mut resources) = resources.lock() + && resources + .get(&resource_key) + .is_some_and(|stored| stored.handle.nonce == resource_key) + { + resources.remove(&resource_key); + } + }); + } + Ok(handle) + } + pub fn profile_source_archive( &self, digest: &str, @@ -90,20 +198,21 @@ impl BackendResourceBroker { .ok()? .values() .find(|resource| resource.handle.digest == digest) - .map(|resource| resource.archive.clone()) + .and_then(|resource| resource.archive.clone()) } - pub fn fetch_profile_source_archive( + pub fn fetch_resource( &self, request: BackendResourceFetchRequest, ) -> Result { verify_handle_shape(&request.handle)?; - let stored = self + let mut resources = self .resources .lock() .map_err(|_| BackendResourceError::Transport { message: "resource broker lock poisoned".to_string(), - })? + })?; + let mut stored = resources .get(&request.handle.nonce) .cloned() .ok_or(BackendResourceError::MissingResource)?; @@ -111,7 +220,7 @@ impl BackendResourceBroker { if stored.handle.expires_at_unix_seconds < Utc::now().timestamp() { return Err(BackendResourceError::Expired); } - let actual_bytes = stored.archive.content.len() as u64; + let actual_bytes = stored.byte_len() as u64; if actual_bytes > stored.handle.max_bytes { return Err(BackendResourceError::Oversized { max_bytes: stored.handle.max_bytes, @@ -139,12 +248,15 @@ impl BackendResourceBroker { }); } } + if stored.one_shot { + resources.remove(&request.handle.nonce); + } Ok(BackendResourceFetchResponse { - kind: BackendResourceKind::ProfileSourceArchive, - resource_id: stored.archive.reference.id, - digest: stored.archive.reference.digest, - content_type: PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE.to_string(), - bytes: stored.archive.content, + kind: stored.handle.kind.clone(), + resource_id: stored.handle.resource_id.clone(), + digest: stored.handle.digest.clone(), + content_type: stored.handle.content_type.clone(), + bytes: stored.take_bytes(), audit_correlation_id: request.audit_correlation_id, }) } @@ -156,24 +268,38 @@ impl BackendResourceClient for BackendResourceBroker { &self, request: BackendResourceFetchRequest, ) -> Result { - self.fetch_profile_source_archive(request) + self.fetch_resource(request) } } fn verify_handle_shape(handle: &BackendResourceHandle) -> Result<(), BackendResourceError> { - if handle.kind != BackendResourceKind::ProfileSourceArchive { - return Err(BackendResourceError::UnsupportedKind); - } - if handle.operation != BackendResourceOperation::FetchArchive { - return Err(BackendResourceError::Unauthorized { - message: "resource handle operation is not fetch_archive".to_string(), - }); - } - if handle.content_type != PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE { - return Err(BackendResourceError::ContentTypeMismatch { - expected: PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE.to_string(), - actual: handle.content_type.clone(), - }); + match handle.kind { + BackendResourceKind::ProfileSourceArchive => { + if handle.operation != BackendResourceOperation::FetchArchive { + return Err(BackendResourceError::Unauthorized { + message: "resource handle operation is not fetch_archive".to_string(), + }); + } + if handle.content_type != PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE { + return Err(BackendResourceError::ContentTypeMismatch { + expected: PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE.to_string(), + actual: handle.content_type.clone(), + }); + } + } + BackendResourceKind::RepositorySshAccess => { + if handle.operation != BackendResourceOperation::FetchOnce { + return Err(BackendResourceError::Unauthorized { + message: "resource handle operation is not fetch_once".to_string(), + }); + } + if handle.content_type != REPOSITORY_SSH_ACCESS_CONTENT_TYPE { + return Err(BackendResourceError::ContentTypeMismatch { + expected: REPOSITORY_SSH_ACCESS_CONTENT_TYPE.to_string(), + actual: handle.content_type.clone(), + }); + } + } } Ok(()) } @@ -249,7 +375,7 @@ mod tests { archive(), ); let response = broker - .fetch_profile_source_archive(BackendResourceFetchRequest { + .fetch_resource(BackendResourceFetchRequest { handle: handle.clone(), runtime_id: runtime_id.to_string(), worker_id: None, @@ -260,6 +386,46 @@ mod tests { assert_eq!(response.content_type, PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE); } + #[test] + fn repository_ssh_access_resource_is_runtime_bound_and_one_shot() { + let broker = BackendResourceBroker::default(); + let handle = broker + .issue_repository_ssh_access_handle( + "workspace-test", + "runtime-test", + "repository-access-test", + "1", + i64::MAX, + RepositorySshAccessSecret { + private_key: "private-key-bytes".to_string(), + known_hosts_entry: "known-hosts-entry".to_string(), + }, + ) + .unwrap(); + let unauthorized = broker + .fetch_resource(request(handle.clone(), "runtime-other", None)) + .unwrap_err(); + assert!(matches!( + unauthorized, + BackendResourceError::Unauthorized { .. } + )); + + let response = broker + .fetch_resource(request(handle.clone(), "runtime-test", None)) + .unwrap(); + assert_eq!(response.kind, BackendResourceKind::RepositorySshAccess); + assert_eq!(response.content_type, REPOSITORY_SSH_ACCESS_CONTENT_TYPE); + let debug = format!("{response:?}"); + assert!(!debug.contains("private-key-bytes")); + assert!(debug.contains("REDACTED")); + let secret: RepositorySshAccessSecret = serde_json::from_slice(&response.bytes).unwrap(); + assert_eq!(secret.private_key, "private-key-bytes"); + assert!(matches!( + broker.fetch_resource(request(handle, "runtime-test", None)), + Err(BackendResourceError::MissingResource) + )); + } + #[test] fn broker_rejects_runtime_mismatch() { let broker = BackendResourceBroker::default(); @@ -270,7 +436,7 @@ mod tests { archive(), ); let err = broker - .fetch_profile_source_archive(request(handle, "runtime-b", None)) + .fetch_resource(request(handle, "runtime-b", None)) .unwrap_err(); assert!(matches!(err, BackendResourceError::Unauthorized { .. })); } @@ -287,7 +453,7 @@ mod tests { archive(), ); let err = broker - .fetch_profile_source_archive(request(handle, runtime_id, Some(&worker_b.worker_id))) + .fetch_resource(request(handle, runtime_id, Some(&worker_b.worker_id))) .unwrap_err(); assert!(matches!(err, BackendResourceError::Unauthorized { .. })); } @@ -312,7 +478,7 @@ mod tests { let mut extended = handle; extended.expires_at_unix_seconds = 4_102_444_800; let err = broker - .fetch_profile_source_archive(request(extended, &runtime_id, None)) + .fetch_resource(request(extended, &runtime_id, None)) .unwrap_err(); assert!(matches!(err, BackendResourceError::Expired)); } @@ -328,7 +494,7 @@ mod tests { ); handle.scope_id = Some("tampered-scope".to_string()); let err = broker - .fetch_profile_source_archive(request(handle, &runtime_id, None)) + .fetch_resource(request(handle, &runtime_id, None)) .unwrap_err(); assert!(matches!(err, BackendResourceError::Unauthorized { .. })); } @@ -345,7 +511,7 @@ mod tests { ); handle.max_bytes = DEFAULT_PROFILE_SOURCE_ARCHIVE_MAX_BYTES + 1024; let err = broker - .fetch_profile_source_archive(request(handle, &runtime_id, None)) + .fetch_resource(request(handle, &runtime_id, None)) .unwrap_err(); assert!(matches!(err, BackendResourceError::Oversized { .. })); } diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index eb324532..be3d96a5 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -11504,7 +11504,7 @@ async fn scoped_post_internal_runtime_resource_fetch( )); } api.resource_broker - .fetch_profile_source_archive(request) + .fetch_resource(request) .map(Json) .map_err(|error| (backend_resource_error_status(&error), Json(error))) } @@ -14122,6 +14122,29 @@ fn authorize_repository_materialization_operation( )); } }; + let expires_at_epoch_seconds = repository_access_expiry(); + let secret_resource = api + .resource_broker + .issue_repository_ssh_access_handle( + &api.config.workspace_id, + &operation.resolved_runtime_id, + format!("repository-ssh-access:{}", operation.operation_id), + format!( + "credential:{}:host-trust:{}", + lease.credential_revision, lease.host_trust_revision + ), + i64::try_from(expires_at_epoch_seconds).unwrap_or(i64::MAX), + worker_runtime::resource::RepositorySshAccessSecret { + private_key: lease.private_key.as_str().to_string(), + known_hosts_entry: lease.known_hosts_entry.clone(), + }, + ) + .map_err(|_| { + settings_bad_request( + "working_directory_repository_access_resource_failed", + "Repository SSH access resource could not be issued", + ) + })?; RepositoryMaterializationContext { workspace_id: api.config.workspace_id.clone(), runtime_id: operation.resolved_runtime_id.clone(), @@ -14135,9 +14158,13 @@ fn authorize_repository_materialization_operation( host_trust_id: lease.host_trust_id, host_trust_revision: lease.host_trust_revision, access, - expires_at_epoch_seconds: repository_access_expiry(), - private_key: SensitiveString::new(lease.private_key.as_str()), - known_hosts_entry: SensitiveString::new(lease.known_hosts_entry), + expires_at_epoch_seconds, + repository_id: request.repository.id.clone(), + repository_source_fingerprint: request.repository.source_fingerprint.clone(), + repository_uri: request.repository.source.uri.clone(), + secret_resource, + private_key: SensitiveString::default(), + known_hosts_entry: SensitiveString::default(), }), } } else { @@ -14230,15 +14257,42 @@ fn authorize_repository_materialization( let lease = api .repository_secrets .lease_ssh_materialization_access(&api.config.workspace_id, binding)?; + let expires_at_epoch_seconds = repository_access_expiry(); + let secret_resource = api + .resource_broker + .issue_repository_ssh_access_handle( + &api.config.workspace_id, + runtime_id, + format!("repository-ssh-access:{operation_id}"), + format!( + "credential:{}:host-trust:{}", + lease.credential_revision, lease.host_trust_revision + ), + i64::try_from(expires_at_epoch_seconds).unwrap_or(i64::MAX), + worker_runtime::resource::RepositorySshAccessSecret { + private_key: lease.private_key.as_str().to_string(), + known_hosts_entry: lease.known_hosts_entry.clone(), + }, + ) + .map_err(|_| { + settings_bad_request( + "working_directory_repository_access_resource_failed", + "Repository SSH access resource could not be issued", + ) + })?; Some(RepositorySshMaterializationAccess { credential_id: lease.credential_id, credential_revision: lease.credential_revision, host_trust_id: lease.host_trust_id, host_trust_revision: lease.host_trust_revision, access: binding.access, - expires_at_epoch_seconds: repository_access_expiry(), - private_key: SensitiveString::new(lease.private_key.as_str()), - known_hosts_entry: SensitiveString::new(lease.known_hosts_entry), + expires_at_epoch_seconds, + repository_id: request.repository.id.clone(), + repository_source_fingerprint: request.repository.source_fingerprint.clone(), + repository_uri: request.repository.source.uri.clone(), + secret_resource, + private_key: SensitiveString::default(), + known_hosts_entry: SensitiveString::default(), }) } else { None @@ -15423,6 +15477,20 @@ mod tests { ); let mut repository_access_launch = workdir_flow_launch; + let secret_resource = api + .resource_broker + .issue_repository_ssh_access_handle( + &api.config.workspace_id, + "runtime-1", + "repository-access-test", + "1", + i64::MAX, + worker_runtime::resource::RepositorySshAccessSecret { + private_key: "private-key-bytes".to_string(), + known_hosts_entry: "known-hosts-entry".to_string(), + }, + ) + .unwrap(); let working_directory = repository_access_launch .resolved_working_directory_request .as_mut() @@ -15444,12 +15512,15 @@ mod tests { host_trust_revision: 1, access: workspace_api::RepositoryAccessMode::ReadOnly, expires_at_epoch_seconds: u64::MAX, - private_key: worker_runtime::catalog::SensitiveString::new( - "private-key-bytes", - ), - known_hosts_entry: worker_runtime::catalog::SensitiveString::new( - "known-hosts-entry", - ), + repository_id: working_directory.repository.id.clone(), + repository_source_fingerprint: working_directory + .repository + .source_fingerprint + .clone(), + repository_uri: working_directory.repository.source.uri.clone(), + secret_resource, + private_key: worker_runtime::catalog::SensitiveString::default(), + known_hosts_entry: worker_runtime::catalog::SensitiveString::default(), }, ), }); @@ -15460,6 +15531,11 @@ mod tests { assert_eq!(access.working_directory_id, "working-directory-1"); assert!(access.materialization.ssh.is_some()); + let serialized_access = serde_json::to_string(&access).unwrap(); + assert!(!serialized_access.contains("private-key-bytes")); + assert!(!serialized_access.contains("known-hosts-entry")); + assert!(!serialized_access.contains("private_key")); + assert!(!serialized_access.contains("known_hosts_entry")); assert!( repository_access_launch .resolved_working_directory_request From 4ebc465e8dad3dcc3e1a96e8348fb79de0f481c2 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 19:02:20 +0900 Subject: [PATCH 10/12] fix: constrain repository SSH commands to authorized source --- crates/worker-runtime/src/http_server.rs | 3 +- crates/worker-runtime/src/main.rs | 6 +- crates/worker-runtime/src/runtime.rs | 223 ++++++++-- .../worker-runtime/src/working_directory.rs | 404 +++++++++++++----- 4 files changed, 471 insertions(+), 165 deletions(-) diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 57adb0d9..20b9e442 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -575,7 +575,8 @@ async fn create_working_directory( } let working_directory = state .runtime - .create_working_directory(request) + .create_working_directory_from_resource(request) + .await .map_err(RuntimeHttpRestError::runtime)?; Ok(Json(RuntimeHttpWorkingDirectoryResponse { working_directory, diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index d4ce88be..a07c2bb2 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -28,11 +28,9 @@ use worker_runtime::{Runtime, RuntimeOptions}; fn main() -> ExitCode { let mut arguments = std::env::args().skip(1).collect::>(); - if arguments.first().map(String::as_str) == Some("__repository-read-only-ssh") { + if arguments.first().map(String::as_str) == Some("__repository-ssh") { arguments.remove(0); - return match worker_runtime::working_directory::run_repository_read_only_ssh_client( - &arguments, - ) { + return match worker_runtime::working_directory::run_repository_ssh_client(&arguments) { Ok(status) => ExitCode::from(u8::try_from(status).unwrap_or(1)), Err(error) => { eprintln!("{error}"); diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index a84ad4de..5bc2ad5e 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -378,6 +378,20 @@ impl Runtime { .map_err(RuntimeError::from) } + pub async fn create_working_directory_from_resource( + &self, + mut request: WorkingDirectoryRequest, + ) -> Result { + if let Some(ssh) = request + .materialization + .as_mut() + .and_then(|materialization| materialization.ssh.as_mut()) + { + self.resolve_repository_access_resource(ssh).await?; + } + self.create_working_directory(request) + } + pub fn authorize_working_directory_repository_access( &self, request: WorkingDirectoryRepositoryAccessRequest, @@ -397,6 +411,59 @@ impl Runtime { .map_err(RuntimeError::from) } + async fn resolve_repository_access_resource( + &self, + ssh: &mut crate::catalog::RepositorySshMaterializationAccess, + ) -> Result<(), RuntimeError> { + if !ssh.private_key.expose().is_empty() && !ssh.known_hosts_entry.expose().is_empty() { + return Ok(()); + } + let (client, runtime_id) = { + let state = self.lock()?; + let client = state.backend_resource_client.clone().ok_or_else(|| { + RuntimeError::InvalidRequest( + "Backend Repository access resource client is unavailable".to_string(), + ) + })?; + let runtime_id = state.runtime_identity.clone().ok_or_else(|| { + RuntimeError::InvalidRequest("Runtime identity is unavailable".to_string()) + })?; + (client, runtime_id) + }; + let mut response = client + .0 + .fetch_resource(BackendResourceFetchRequest { + handle: ssh.secret_resource.clone(), + runtime_id, + worker_id: None, + audit_correlation_id: ssh.secret_resource.audit_correlation_id.clone(), + }) + .await + .map_err(repository_resource_error)?; + if response.kind != BackendResourceKind::RepositorySshAccess + || response.content_type != REPOSITORY_SSH_ACCESS_CONTENT_TYPE + || response.resource_id != ssh.secret_resource.resource_id + || response.digest != ssh.secret_resource.digest + || response.bytes.len() as u64 > ssh.secret_resource.max_bytes + { + return Err(RuntimeError::InvalidRequest( + "Backend Repository SSH access resource response was invalid".to_string(), + )); + } + let secret = serde_json::from_slice::(&response.bytes); + response.bytes.fill(0); + let mut secret = secret.map_err(|_| { + RuntimeError::InvalidRequest( + "Backend Repository SSH access resource payload was invalid".to_string(), + ) + })?; + ssh.private_key = + crate::catalog::SensitiveString::new(std::mem::take(&mut secret.private_key)); + ssh.known_hosts_entry = + crate::catalog::SensitiveString::new(std::mem::take(&mut secret.known_hosts_entry)); + Ok(()) + } + pub async fn authorize_working_directory_repository_access_from_resource( &self, mut request: WorkingDirectoryRepositoryAccessRequest, @@ -404,51 +471,7 @@ impl Runtime { let ssh = request.materialization.ssh.as_mut().ok_or_else(|| { RuntimeError::InvalidRequest("Repository SSH access metadata is missing".to_string()) })?; - if ssh.private_key.expose().is_empty() || ssh.known_hosts_entry.expose().is_empty() { - let (client, runtime_id) = { - let state = self.lock()?; - let client = state.backend_resource_client.clone().ok_or_else(|| { - RuntimeError::InvalidRequest( - "Backend Repository access resource client is unavailable".to_string(), - ) - })?; - let runtime_id = state.runtime_identity.clone().ok_or_else(|| { - RuntimeError::InvalidRequest("Runtime identity is unavailable".to_string()) - })?; - (client, runtime_id) - }; - let mut response = client - .0 - .fetch_resource(BackendResourceFetchRequest { - handle: ssh.secret_resource.clone(), - runtime_id, - worker_id: None, - audit_correlation_id: ssh.secret_resource.audit_correlation_id.clone(), - }) - .await - .map_err(repository_resource_error)?; - if response.kind != BackendResourceKind::RepositorySshAccess - || response.content_type != REPOSITORY_SSH_ACCESS_CONTENT_TYPE - || response.resource_id != ssh.secret_resource.resource_id - || response.digest != ssh.secret_resource.digest - || response.bytes.len() as u64 > ssh.secret_resource.max_bytes - { - return Err(RuntimeError::InvalidRequest( - "Backend Repository SSH access resource response was invalid".to_string(), - )); - } - let secret = serde_json::from_slice::(&response.bytes); - response.bytes.fill(0); - let mut secret = secret.map_err(|_| { - RuntimeError::InvalidRequest( - "Backend Repository SSH access resource payload was invalid".to_string(), - ) - })?; - ssh.private_key = - crate::catalog::SensitiveString::new(std::mem::take(&mut secret.private_key)); - ssh.known_hosts_entry = - crate::catalog::SensitiveString::new(std::mem::take(&mut secret.known_hosts_entry)); - } + self.resolve_repository_access_resource(ssh).await?; self.authorize_working_directory_repository_access(request) } @@ -3335,6 +3358,9 @@ mod tests { #[tokio::test] async fn repository_access_resource_is_fetched_before_provider_authorization() { let (runtime, backend) = runtime_and_backend(); + backend + .repository_access_available + .store(true, Ordering::SeqCst); runtime.bind_runtime_identity("runtime-1").unwrap(); let handle = repository_resource_handle(); runtime @@ -3398,6 +3424,88 @@ mod tests { assert_eq!(access.known_hosts_entry.expose(), "known-hosts-entry"); } + #[tokio::test] + async fn working_directory_create_fetches_repository_access_before_provider_call() { + let (runtime, backend) = runtime_and_backend(); + backend + .repository_access_available + .store(true, Ordering::SeqCst); + runtime.bind_runtime_identity("runtime-1").unwrap(); + let handle = repository_resource_handle(); + runtime + .install_backend_resource_client(Arc::new(TestRepositoryResourceClient { + response: Mutex::new(Some(crate::resource::BackendResourceFetchResponse { + kind: crate::resource::BackendResourceKind::RepositorySshAccess, + resource_id: handle.resource_id.clone(), + digest: handle.digest.clone(), + content_type: crate::resource::REPOSITORY_SSH_ACCESS_CONTENT_TYPE.to_string(), + bytes: serde_json::to_vec(&RepositorySshAccessSecret { + private_key: "create-private-key-bytes".to_string(), + known_hosts_entry: "create-known-hosts-entry".to_string(), + }) + .unwrap(), + audit_correlation_id: handle.audit_correlation_id.clone(), + })), + })) + .unwrap(); + let request = WorkingDirectoryRequest { + repository: WorkingDirectoryRepository { + id: "repository-1".to_string(), + provider: "git".to_string(), + source: workspace_api::RepositorySource { + kind: workspace_api::RepositorySourceKind::Ssh, + uri: "ssh://git@example.test/repo.git".to_string(), + }, + source_revision: 1, + source_fingerprint: "sha256:source".to_string(), + selector: None, + }, + materializer: MaterializerKind::RuntimeGitCache, + backend_workdir_id: Some("working-directory-1".to_string()), + materialization: Some(RepositoryMaterializationContext { + workspace_id: "workspace-1".to_string(), + runtime_id: "runtime-1".to_string(), + operation_id: "operation-create".to_string(), + config_revision: 1, + config_projection_digest: "sha256:projection".to_string(), + cache_generation: 0, + ssh: Some(RepositorySshMaterializationAccess { + credential_id: "credential-1".to_string(), + credential_revision: 1, + host_trust_id: "host-trust-1".to_string(), + host_trust_revision: 1, + access: workspace_api::RepositoryAccessMode::ReadOnly, + expires_at_epoch_seconds: u64::MAX, + repository_id: "repository-1".to_string(), + repository_source_fingerprint: "sha256:source".to_string(), + repository_uri: "ssh://git@example.test/repo.git".to_string(), + secret_resource: handle, + private_key: SensitiveString::default(), + known_hosts_entry: SensitiveString::default(), + }), + }), + }; + + assert!( + runtime + .create_working_directory_from_resource(request) + .await + .is_err() + ); + + let requests = backend.working_directory_requests.lock().unwrap(); + let access = requests[0] + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.as_ref()) + .unwrap(); + assert_eq!(access.private_key.expose(), "create-private-key-bytes"); + assert_eq!( + access.known_hosts_entry.expose(), + "create-known-hosts-entry" + ); + } + fn scoped_task_request(objective: &str, workspace_id: &str) -> CreateWorkerRequest { let mut request = task_request(objective); request.workspace_api = Some(WorkspaceApiRef { @@ -3480,6 +3588,8 @@ mod tests { contexts: Mutex>, dispatched_inputs: Mutex>, repository_accesses: Mutex>, + repository_access_available: AtomicBool, + working_directory_requests: Mutex>, preserve_commit_ack_submission_id: AtomicBool, #[cfg(feature = "ws-server")] snapshots: Mutex>, @@ -3520,6 +3630,20 @@ mod tests { "test-execution-backend" } + fn create_working_directory( + &self, + request: &WorkingDirectoryRequest, + ) -> Result { + self.working_directory_requests + .lock() + .unwrap() + .push(request.clone()); + Err(WorkingDirectoryDiagnostic::rejected( + "working_directory_unsupported", + "Worker execution backend does not support working directory materialization", + )) + } + fn authorize_working_directory_repository_access( &self, request: &WorkingDirectoryRepositoryAccessRequest, @@ -3528,7 +3652,14 @@ mod tests { .lock() .unwrap() .push(request.clone()); - Ok(()) + if self.repository_access_available.load(Ordering::SeqCst) { + Ok(()) + } else { + Err(WorkingDirectoryDiagnostic::rejected( + "working_directory_repository_access_unsupported", + "Worker execution backend does not support Repository access authorization", + )) + } } fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index edaa8f98..06702830 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -488,12 +488,9 @@ impl RuntimeGitCacheMaterializer { access.stop(); } }); - if access.access == workspace_api::RepositoryAccessMode::ReadWrite { - binding.command_environment.insert( - "SSH_AUTH_SOCK".to_string(), - command_access.agent.socket.to_string_lossy().to_string(), - ); - } + binding + .command_environment + .insert("SSH_AUTH_SOCK".to_string(), "/dev/null".to_string()); binding.command_environment.insert( "GIT_SSH_COMMAND".to_string(), command_access.ssh_command.to_string_lossy().to_string(), @@ -703,14 +700,23 @@ impl RuntimeGitCacheMaterializer { let mut request = request.clone(); if request.repository.provider != "git" || request.repository.source.kind != workspace_api::RepositorySourceKind::Ssh - || request - .materialization - .as_ref() - .and_then(|materialization| materialization.ssh.as_ref()) - .is_some() { return Ok(request); } + if let Some(access) = request + .materialization + .as_ref() + .and_then(|materialization| materialization.ssh.as_ref()) + { + validate_ssh_materialization_access(access)?; + validate_repository_access_binding( + access, + &request.repository.id, + &request.repository.source_fingerprint, + Some(request.repository.source.uri.as_str()), + )?; + return Ok(request); + } let access = self .repository_access .lock() @@ -1266,29 +1272,30 @@ impl std::fmt::Debug for PendingRepositorySshRequest { } #[derive(Debug)] -struct RepositoryReadOnlySshBroker { +struct RepositorySshBroker { socket: PathBuf, stopped: Arc, thread: Mutex>>, } -impl RepositoryReadOnlySshBroker { +impl RepositorySshBroker { fn start( root: &Path, agent: Arc, known_hosts: PathBuf, + policy: RepositorySshCommandPolicy, ) -> Result { let socket = root.join("b.sock"); let listener = UnixListener::bind(&socket).map_err(|_| { WorkingDirectoryDiagnostic::new( "working_directory_repository_access_setup_failed", - "read-only Repository SSH broker could not be prepared", + "Repository SSH broker could not be prepared", ) })?; listener.set_nonblocking(true).map_err(|_| { WorkingDirectoryDiagnostic::new( "working_directory_repository_access_setup_failed", - "read-only Repository SSH broker could not be prepared", + "Repository SSH broker could not be prepared", ) })?; let stopped = Arc::new(AtomicBool::new(false)); @@ -1308,11 +1315,13 @@ impl RepositoryReadOnlySshBroker { { let request_agent = Arc::clone(&agent); let request_known_hosts = known_hosts.clone(); + let request_policy = policy.clone(); std::thread::spawn(move || { - run_brokered_read_only_ssh( + run_brokered_repository_ssh( request, &request_agent.socket, &request_known_hosts, + &request_policy, ); }); pending.remove(&request_id); @@ -1346,7 +1355,7 @@ impl RepositoryReadOnlySshBroker { } } -impl Drop for RepositoryReadOnlySshBroker { +impl Drop for RepositorySshBroker { fn drop(&mut self) { self.stop(); } @@ -1401,10 +1410,11 @@ fn receive_repository_ssh_channel( } } -fn run_brokered_read_only_ssh( +fn run_brokered_repository_ssh( mut request: PendingRepositorySshRequest, agent_socket: &Path, known_hosts: &Path, + policy: &RepositorySshCommandPolicy, ) { let args = request.args.take().unwrap_or_default(); let mut data = request.data.take().expect("complete broker request data"); @@ -1416,8 +1426,8 @@ fn run_brokered_read_only_ssh( .status .take() .expect("complete broker request status"); - let status = if !validate_read_only_ssh_args(&args) { - let _ = stderr_stream.write_all(b"read-only Repository SSH operation denied\n"); + let status = if !validate_repository_ssh_args(&args, policy) { + let _ = stderr_stream.write_all(b"Repository SSH operation denied\n"); let _ = stderr_stream.shutdown(Shutdown::Write); let _ = data.shutdown(Shutdown::Both); 126 @@ -1480,28 +1490,114 @@ fn run_brokered_read_only_ssh( let _ = status_stream.shutdown(Shutdown::Write); } -fn validate_read_only_ssh_args(args: &[String]) -> bool { +#[derive(Clone, Debug)] +struct RepositorySshCommandPolicy { + host: String, + username: Option, + port: Option, + repository_path: String, + access: workspace_api::RepositoryAccessMode, +} + +impl RepositorySshCommandPolicy { + fn from_access( + access: &RepositorySshMaterializationAccess, + ) -> Result { + let uri = url::Url::parse(&access.repository_uri).map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_binding_mismatch", + "Repository SSH access URI is invalid", + ) + })?; + if uri.scheme() != "ssh" + || uri.password().is_some() + || uri.query().is_some() + || uri.fragment().is_some() + { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_binding_mismatch", + "Repository SSH access URI is not an authorized SSH endpoint", + )); + } + let host = uri + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(|| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_binding_mismatch", + "Repository SSH access host is missing", + ) + })?; + let username = (!uri.username().is_empty()).then(|| uri.username().to_string()); + if username + .as_deref() + .is_some_and(|username| !is_safe_ssh_destination(username)) + || uri.path().is_empty() + || !is_safe_repository_path(uri.path()) + { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_binding_mismatch", + "Repository SSH access endpoint or path is invalid", + )); + } + Ok(Self { + host: host.to_string(), + username, + port: uri.port(), + repository_path: uri.path().to_string(), + access: access.access, + }) + } + + fn destination_matches(&self, destination: &str, login: Option<&str>) -> bool { + let host = if self.host.contains(':') { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + match self.username.as_deref() { + Some(username) => { + (login.is_none() && destination == format!("{username}@{host}")) + || (login == Some(username) && destination == host) + } + None => login.is_none() && destination == host, + } + } +} + +fn validate_repository_ssh_args(args: &[String], policy: &RepositorySshCommandPolicy) -> bool { let mut index = 0; let mut probe = false; + let mut port = None; + let mut login = None; let mut positional = Vec::new(); while index < args.len() { match args[index].as_str() { "-G" => probe = true, "-4" | "-6" | "-v" | "-vv" | "-vvv" => {} "-p" => { - index += 1; - if index >= args.len() - || args[index].is_empty() - || !args[index].bytes().all(|byte| byte.is_ascii_digit()) - { + if port.is_some() { return false; } + index += 1; + let Some(value) = args.get(index).and_then(|value| value.parse::().ok()) + else { + return false; + }; + port = Some(value); } "-l" => { - index += 1; - if index >= args.len() || !is_safe_ssh_destination(&args[index]) { + if login.is_some() { return false; } + index += 1; + let Some(value) = args + .get(index) + .filter(|value| is_safe_ssh_destination(value)) + else { + return false; + }; + login = Some(value.as_str()); } "-o" => { index += 1; @@ -1519,12 +1615,16 @@ fn validate_read_only_ssh_args(args: &[String]) -> bool { } index += 1; } + if port != policy.port + || positional.is_empty() + || !policy.destination_matches(positional[0], login) + { + return false; + } if probe { - positional.len() == 1 && is_safe_ssh_destination(positional[0]) + positional.len() == 1 } else { - positional.len() == 2 - && is_safe_ssh_destination(positional[0]) - && is_safe_read_only_git_command(positional[1]) + positional.len() == 2 && repository_command_matches(positional[1], policy) } } @@ -1548,31 +1648,43 @@ fn is_safe_ssh_destination(value: &str) -> bool { }) } -fn is_safe_read_only_git_command(command: &str) -> bool { - let Some(argument) = command - .strip_prefix("git-upload-pack ") - .or_else(|| command.strip_prefix("git-upload-archive ")) - else { - return false; - }; - let argument = argument - .strip_prefix('\'') - .and_then(|argument| argument.strip_suffix('\'')) - .unwrap_or(argument); - !argument.is_empty() - && argument.bytes().all(|byte| { +fn is_safe_repository_path(path: &str) -> bool { + !path.is_empty() + && path.bytes().all(|byte| { byte.is_ascii_alphanumeric() || matches!( byte, - b'/' | b'.' | b'-' | b'_' | b'@' | b':' | b'+' | b'~' | b' ' + b'/' | b'.' | b'-' | b'_' | b'@' | b':' | b'+' | b'~' | b' ' | b'%' ) }) } -pub fn run_repository_read_only_ssh_client(arguments: &[String]) -> Result { +fn repository_command_matches(command: &str, policy: &RepositorySshCommandPolicy) -> bool { + let (operation, argument) = if let Some(argument) = command.strip_prefix("git-upload-pack ") { + ("upload-pack", argument) + } else if let Some(argument) = command.strip_prefix("git-upload-archive ") { + ("upload-archive", argument) + } else if let Some(argument) = command.strip_prefix("git-receive-pack ") { + ("receive-pack", argument) + } else { + return false; + }; + if operation == "receive-pack" + && policy.access != workspace_api::RepositoryAccessMode::ReadWrite + { + return false; + } + let argument = argument + .strip_prefix('\'') + .and_then(|argument| argument.strip_suffix('\'')) + .unwrap_or(argument); + is_safe_repository_path(argument) && argument == policy.repository_path +} + +pub fn run_repository_ssh_client(arguments: &[String]) -> Result { let (socket, ssh_args) = arguments .split_first() - .ok_or_else(|| "read-only Repository SSH broker socket is missing".to_string())?; + .ok_or_else(|| "Repository SSH broker socket is missing".to_string())?; let request_id = format!( "{}-{}", std::process::id(), @@ -1600,7 +1712,7 @@ pub fn run_repository_read_only_ssh_client(arguments: &[String]) -> Result Result() - .map_err(|_| "read-only Repository SSH broker status was invalid".to_string()) + .map_err(|_| "Repository SSH broker status was invalid".to_string()) } fn connect_repository_ssh_broker_channel( @@ -1627,12 +1739,12 @@ fn connect_repository_ssh_broker_channel( header: &RepositorySshBrokerHeader, ) -> Result { let mut stream = UnixStream::connect(socket) - .map_err(|_| "read-only Repository SSH broker is unavailable".to_string())?; + .map_err(|_| "Repository SSH broker is unavailable".to_string())?; serde_json::to_writer(&mut stream, header) - .map_err(|_| "read-only Repository SSH broker request failed".to_string())?; + .map_err(|_| "Repository SSH broker request failed".to_string())?; stream .write_all(b"\n") - .map_err(|_| "read-only Repository SSH broker request failed".to_string())?; + .map_err(|_| "Repository SSH broker request failed".to_string())?; Ok(stream) } @@ -1641,7 +1753,7 @@ struct RepositoryCommandAccess { root: PathBuf, ssh_command: PathBuf, agent: Arc, - read_only_broker: Option, + ssh_broker: Option, } impl RepositoryCommandAccess { @@ -1694,46 +1806,37 @@ impl RepositoryCommandAccess { let ssh_command = root.join("ssh-command"); write_owner_only(&known_hosts, ssh.known_hosts_entry.expose().as_bytes())?; let agent = Arc::new(RepositorySshAgent::start(runtime_root, operation_id, ssh)?); - let read_only_broker = if ssh.access == workspace_api::RepositoryAccessMode::ReadOnly { - Some(RepositoryReadOnlySshBroker::start( - &root, - Arc::clone(&agent), - known_hosts.clone(), - )?) - } else { - None - }; - let script = if let Some(broker) = read_only_broker.as_ref() { - let executable = std::env::current_exe().map_err(|_| { - WorkingDirectoryDiagnostic::new( - "working_directory_repository_access_setup_failed", - "read-only Repository SSH broker client could not be resolved", - ) - })?; - format!( - "#!/bin/sh\nexec {} __repository-read-only-ssh {} \"$@\"\n", - shell_quote_path(&executable)?, - shell_quote_path(&broker.socket)?, + let policy = RepositorySshCommandPolicy::from_access(ssh)?; + let ssh_broker = Some(RepositorySshBroker::start( + &root, + Arc::clone(&agent), + known_hosts.clone(), + policy, + )?); + let broker = ssh_broker.as_ref().expect("Repository SSH broker"); + let executable = std::env::current_exe().map_err(|_| { + WorkingDirectoryDiagnostic::new( + "working_directory_repository_access_setup_failed", + "Repository SSH broker client could not be resolved", ) - } else { - format!( - "#!/bin/sh\nexport SSH_AUTH_SOCK={}\nexec ssh -F /dev/null -o BatchMode=yes -o IdentitiesOnly=no -o IdentityFile=/dev/null -o StrictHostKeyChecking=yes -o UserKnownHostsFile={} \"$@\"\n", - shell_quote_path(&agent.socket)?, - shell_quote_path(&known_hosts)?, - ) - }; + })?; + let script = format!( + "#!/bin/sh\nexec {} __repository-ssh {} \"$@\"\n", + shell_quote_path(&executable)?, + shell_quote_path(&broker.socket)?, + ); write_owner_only(&ssh_command, script.as_bytes())?; set_file_owner_executable(&ssh_command)?; Ok(Self { root, ssh_command, agent, - read_only_broker, + ssh_broker, }) } fn stop(&self) { - if let Some(broker) = self.read_only_broker.as_ref() { + if let Some(broker) = self.ssh_broker.as_ref() { broker.stop(); } self.agent.stop(); @@ -2727,17 +2830,27 @@ mod tests { ); let binding = materializer.bind_working_directory(&id, None).unwrap(); let environment = binding.command_environment(); - let socket = PathBuf::from(environment["SSH_AUTH_SOCK"].clone()); + assert_eq!(environment["SSH_AUTH_SOCK"], "/dev/null"); let ssh_command = PathBuf::from(environment["GIT_SSH_COMMAND"].clone()); - assert!(socket.exists()); assert!(ssh_command.exists()); let ssh_policy = fs::read_to_string(&ssh_command).unwrap(); - assert!(ssh_policy.contains("StrictHostKeyChecking=yes")); - assert!(ssh_policy.contains("UserKnownHostsFile=")); + assert!(ssh_policy.contains("__repository-ssh")); + assert!(!ssh_policy.contains("SSH_AUTH_SOCK")); + assert_eq!( + fs::read_dir(runtime_root.path().join(".repository-agents")) + .map(|entries| entries.count()) + .unwrap_or_default(), + 1 + ); assert_eq!(environment["YOI_REPOSITORY_ACCESS"], "read_write"); drop(binding); - assert!(!socket.exists()); assert!(!ssh_command.exists()); + assert_eq!( + fs::read_dir(runtime_root.path().join(".repository-agents")) + .map(|entries| entries.count()) + .unwrap_or_default(), + 0 + ); assert_eq!( materializer .bind_working_directory(&id, None) @@ -2762,17 +2875,22 @@ mod tests { ); let rebound_environment = rebound.command_environment(); assert_eq!(rebound_environment["YOI_REPOSITORY_ACCESS"], "read_only"); - assert!(!rebound_environment.contains_key("SSH_AUTH_SOCK")); + assert_eq!(rebound_environment["SSH_AUTH_SOCK"], "/dev/null"); let read_only_ssh = &rebound_environment["GIT_SSH_COMMAND"]; let read_only_policy = fs::read_to_string(read_only_ssh).unwrap(); - assert!(read_only_policy.contains("__repository-read-only-ssh")); + assert!(read_only_policy.contains("__repository-ssh")); assert!(!read_only_policy.contains("SSH_AUTH_SOCK")); assert!(!read_only_policy.contains(".repository-agents")); assert!(!read_only_policy.contains("known_hosts")); - assert!(!validate_read_only_ssh_args(&[ - "example.test".to_string(), - "git-receive-pack 'repo.git'".to_string(), - ])); + let read_only_command_policy = + RepositorySshCommandPolicy::from_access(rotated.ssh.as_ref().unwrap()).unwrap(); + assert!(!validate_repository_ssh_args( + &[ + "git@example.test".to_string(), + "git-receive-pack '/repo.git'".to_string(), + ], + &read_only_command_policy, + )); let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR)) .unwrap() .filter_map(Result::ok) @@ -2780,35 +2898,77 @@ mod tests { .find(|path| path.exists()) .expect("read-only broker socket"); assert_eq!( - run_repository_read_only_ssh_client(&[ + run_repository_ssh_client(&[ broker_socket.to_string_lossy().to_string(), "-G".to_string(), - "example.test".to_string(), + "git@example.test".to_string(), ]) .unwrap(), 0 ); assert_eq!( - run_repository_read_only_ssh_client(&[ + run_repository_ssh_client(&[ broker_socket.to_string_lossy().to_string(), - "example.test".to_string(), - "git-receive-pack 'repo.git'".to_string(), + "git@example.test".to_string(), + "git-receive-pack '/repo.git'".to_string(), ]) .unwrap(), 126 ); - assert!(!validate_read_only_ssh_args(&[ - "-o".to_string(), - "ProxyCommand=sh -c exploit".to_string(), - "example.test".to_string(), - "git-upload-pack 'repo.git'".to_string(), - ])); - assert!(validate_read_only_ssh_args(&[ - "-o".to_string(), - "SendEnv=GIT_PROTOCOL".to_string(), - "example.test".to_string(), - "git-upload-pack 'repo.git'".to_string(), - ])); + assert!(!validate_repository_ssh_args( + &[ + "-o".to_string(), + "ProxyCommand=sh -c exploit".to_string(), + "git@example.test".to_string(), + "git-upload-pack '/repo.git'".to_string(), + ], + &read_only_command_policy, + )); + assert!(validate_repository_ssh_args( + &[ + "-o".to_string(), + "SendEnv=GIT_PROTOCOL".to_string(), + "git@example.test".to_string(), + "git-upload-pack '/repo.git'".to_string(), + ], + &read_only_command_policy, + )); + assert!(!validate_repository_ssh_args( + &[ + "git@other.test".to_string(), + "git-upload-pack '/repo.git'".to_string(), + ], + &read_only_command_policy, + )); + let mut port_bound_access = rotated.ssh.as_ref().unwrap().clone(); + port_bound_access.repository_uri = "ssh://git@example.test:2222/repo.git".to_string(); + let port_bound_policy = + RepositorySshCommandPolicy::from_access(&port_bound_access).unwrap(); + assert!(validate_repository_ssh_args( + &[ + "-p".to_string(), + "2222".to_string(), + "git@example.test".to_string(), + "git-upload-pack '/repo.git'".to_string(), + ], + &port_bound_policy, + )); + assert!(!validate_repository_ssh_args( + &[ + "-p".to_string(), + "22".to_string(), + "git@example.test".to_string(), + "git-upload-pack '/repo.git'".to_string(), + ], + &port_bound_policy, + )); + assert!(!validate_repository_ssh_args( + &[ + "git@example.test".to_string(), + "git-upload-pack '/other.git'".to_string(), + ], + &read_only_command_policy, + )); assert_eq!( git_stdout( rebound.root(), @@ -2824,6 +2984,8 @@ mod tests { read_write.operation_id = "operation-agent-read-write".to_string(); read_write.ssh.as_mut().unwrap().credential_revision = 3; read_write.ssh.as_mut().unwrap().access = workspace_api::RepositoryAccessMode::ReadWrite; + let read_write_command_policy = + RepositorySshCommandPolicy::from_access(read_write.ssh.as_ref().unwrap()).unwrap(); materializer .authorize_repository_access(&WorkingDirectoryRepositoryAccessRequest { working_directory_id: id.clone(), @@ -2835,7 +2997,21 @@ mod tests { rebound.command_environment()["YOI_REPOSITORY_ACCESS"], "read_write" ); - assert!(rebound.command_environment().contains_key("SSH_AUTH_SOCK")); + assert_eq!(rebound.command_environment()["SSH_AUTH_SOCK"], "/dev/null"); + assert!(validate_repository_ssh_args( + &[ + "git@example.test".to_string(), + "git-receive-pack '/repo.git'".to_string(), + ], + &read_write_command_policy, + )); + assert!(!validate_repository_ssh_args( + &[ + "git@example.test".to_string(), + "git-receive-pack '/other.git'".to_string(), + ], + &read_write_command_policy, + )); assert!( git_stdout( rebound.root(), From 29c2fb8e0626e84203b1e2b37b6a3ba2e81c0c97 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Aug 2026 19:27:10 +0900 Subject: [PATCH 11/12] fix: refresh repository access for worker workdirs --- crates/workspace-server/src/server.rs | 89 +++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index be3d96a5..9db95dfc 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -6504,15 +6504,6 @@ async fn open_current_worker_workdir_session_locked( worker: &RuntimeWorkerRef, link: &WorkerWorkdirLinkRecord, ) -> Result { - if let Some(session) = api - .workdir_sessions - .lock() - .expect("Workdir session registry lock poisoned") - .get(worker) - .cloned() - { - return Ok(session); - } let workdir = api .store .list_workdir_registry(&api.config.workspace_id, 10_000)? @@ -6526,6 +6517,40 @@ async fn open_current_worker_workdir_session_locked( link.workdir_id ), })?; + let repository = api + .require_configured_workspace_repository(&workdir.repository_id) + .map_err(|error| error.error)?; + let repository_requires_access = + repository.source.kind == workspace_api::RepositorySourceKind::Ssh; + if repository_requires_access { + close_current_worker_session_locked(api, worker).await?; + let access = repository_access_request_for_workdir( + api, + &workdir.runtime_id, + &workdir.workdir_id, + &format!( + "workdir-session:{}:{}:{}", + worker.runtime_id, worker.worker_id, workdir.workdir_id + ), + ) + .map_err(|error| error.error)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: workdir.runtime_id.clone(), + code: "working_directory_remote_repository_access_required".to_string(), + message: "current Repository SSH access authority is unavailable".to_string(), + })?; + api.runtime + .authorize_working_directory_repository_access(&workdir.runtime_id, access) + .map_err(RuntimeRegistryError::into_error)?; + } else if let Some(session) = api + .workdir_sessions + .lock() + .expect("Workdir session registry lock poisoned") + .get(worker) + .cloned() + { + return Ok(session); + } let owner_worker_id = runtime_local_owner_worker_id(worker, &workdir.runtime_id); let session = api .runtime @@ -12071,14 +12096,31 @@ async fn create_runtime_worker( .as_ref() .map(|working_directory| configured_working_directory_request(&api, working_directory)) .transpose()?; + let repository_operation_id = request + .resolved_control_operation + .as_ref() + .map(|operation| operation.operation_id.clone()) + .or_else(|| { + request + .ticket_assignment + .as_ref() + .map(|assignment| assignment.operation_id.clone()) + }); let prepared_workdir_id = if let Some(working_directory_request) = request.resolved_working_directory_request.as_mut() { - Some(upsert_pending_backend_workdir( + let workdir_id = + upsert_pending_backend_workdir(&api, &runtime_id, working_directory_request)?; + let operation_id = repository_operation_id + .clone() + .unwrap_or_else(|| format!("worker-spawn-workdir:{workdir_id}")); + authorize_worker_spawn_workdir_materialization( &api, &runtime_id, + &operation_id, working_directory_request, - )?) + )?; + Some(workdir_id) } else { request .resolved_working_directory @@ -14236,6 +14278,16 @@ fn repository_access_expiry() -> u64 { .saturating_add(300) } +fn authorize_worker_spawn_workdir_materialization( + api: &WorkspaceApi, + runtime_id: &str, + operation_id: &str, + request: &mut WorkingDirectoryRequest, +) -> ApiResult<()> { + let projection = active_repository_access_projection(api, &api.config.workspace_id)?; + authorize_repository_materialization(api, runtime_id, operation_id, &projection, request) +} + fn authorize_repository_materialization( api: &WorkspaceApi, runtime_id: &str, @@ -15445,6 +15497,21 @@ mod tests { .is_err() ); + let mut local_workdir_request = + working_directory_request_from_repository(&api.config.repositories[0], Some("HEAD")); + authorize_worker_spawn_workdir_materialization( + &api, + "runtime-1", + "worker-spawn-operation", + &mut local_workdir_request, + ) + .unwrap(); + let materialization = local_workdir_request.materialization.unwrap(); + assert_eq!(materialization.workspace_id, api.config.workspace_id); + assert_eq!(materialization.runtime_id, "runtime-1"); + assert_eq!(materialization.operation_id, "worker-spawn-operation"); + assert!(materialization.ssh.is_none()); + let mut foreign_repository = api.config.repositories[0].clone(); foreign_repository.id = "foreign".to_string(); let workdir_flow_launch = WorkerSpawnRequest { From 08be5e85e4df54237217d8f504cc7dabdb59329d Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 27 Aug 2026 08:09:22 +0900 Subject: [PATCH 12/12] fix: preserve command sessions across access refresh --- crates/workspace-server/src/server.rs | 626 +++++++++++++++++++++----- 1 file changed, 512 insertions(+), 114 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 9db95dfc..7cd0beb7 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -44,7 +44,6 @@ use webauthn_rs::prelude::{ PublicKeyCredential, RegisterPublicKeyCredential, RequestChallengeResponse, Webauthn, WebauthnBuilder, }; -use workdir::WorkdirSessionHandle; use workdir::http::{WorkdirSessionOperation, WorkdirSessionOperationResult}; use workdir::workspace::{ MaterializerKind, WorkingDirectoryCleanupTarget, @@ -54,6 +53,7 @@ use workdir::workspace::{ WorkingDirectoryStatusKind, WorkingDirectorySummary, WorkspaceWorkdirSessionFence, WorkspaceWorkdirSessionOperationRequest, }; +use workdir::{CommandHandle, WorkdirSessionHandle}; use worker::feature::builtin::{WorkerObservationSubject, WorkerObservationSubjectRef}; use worker_runtime::resource::{BackendResourceError, BackendResourceFetchRequest}; use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend}; @@ -347,6 +347,165 @@ static EMBEDDED_RUNTIME_REQUEST_IDENTITY: std::sync::LazyLock< .expect("embedded Runtime request identity generation must succeed") }); +#[derive(Clone)] +struct WorkdirCommandSession { + source: WorkdirSessionHandle, + provider_handle: CommandHandle, + delegations: Vec, +} + +enum RegisteredWorkdirSession { + Attachment { + worker: RuntimeWorkerRef, + session: WorkdirSessionHandle, + }, + Command { + worker: RuntimeWorkerRef, + external_handle: CommandHandle, + session: WorkdirCommandSession, + }, +} + +#[derive(Default)] +struct WorkdirSessionRegistry { + attachments: HashMap, + commands: HashMap<(RuntimeWorkerRef, CommandHandle), WorkdirCommandSession>, +} + +impl WorkdirSessionRegistry { + fn attachment(&self, worker: &RuntimeWorkerRef) -> Option { + self.attachments.get(worker).cloned() + } + + fn insert_attachment( + &mut self, + worker: RuntimeWorkerRef, + session: WorkdirSessionHandle, + ) -> Option { + self.attachments.insert(worker, session) + } + + fn remove_attachment(&mut self, worker: &RuntimeWorkerRef) -> Option { + self.attachments.remove(worker) + } + + fn register_command( + &mut self, + worker: RuntimeWorkerRef, + source: WorkdirSessionHandle, + provider_handle: CommandHandle, + delegations: Vec, + ) -> CommandHandle { + let external_handle = loop { + let candidate = CommandHandle(Uuid::now_v7().to_string()); + if !self + .commands + .contains_key(&(worker.clone(), candidate.clone())) + { + break candidate; + } + }; + self.commands.insert( + (worker, external_handle.clone()), + WorkdirCommandSession { + source, + provider_handle, + delegations, + }, + ); + external_handle + } + + fn command( + &self, + worker: &RuntimeWorkerRef, + external_handle: &CommandHandle, + ) -> Option { + self.commands + .get(&(worker.clone(), external_handle.clone())) + .cloned() + } + + fn take_worker(&mut self, worker: &RuntimeWorkerRef) -> Vec { + let mut sessions = Vec::new(); + if let Some(session) = self.attachments.remove(worker) { + sessions.push(RegisteredWorkdirSession::Attachment { + worker: worker.clone(), + session, + }); + } + let command_handles = self + .commands + .keys() + .filter(|(owner, _)| owner == worker) + .map(|(_, handle)| handle.clone()) + .collect::>(); + for external_handle in command_handles { + if let Some(session) = self + .commands + .remove(&(worker.clone(), external_handle.clone())) + { + sessions.push(RegisteredWorkdirSession::Command { + worker: worker.clone(), + external_handle, + session, + }); + } + } + sessions + } + + fn restore(&mut self, registered: RegisteredWorkdirSession) { + match registered { + RegisteredWorkdirSession::Attachment { worker, session } => { + self.attachments.insert(worker, session); + } + RegisteredWorkdirSession::Command { + worker, + external_handle, + session, + } => { + self.commands.insert((worker, external_handle), session); + } + } + } +} + +async fn close_worker_workdir_sessions( + registry: &Arc>, + worker: &RuntimeWorkerRef, +) -> std::result::Result<(), String> { + let registered = registry + .lock() + .map_err(|_| "Workdir session registry was poisoned".to_string())? + .take_worker(worker); + let mut failed = Vec::new(); + let mut errors = Vec::new(); + for item in registered { + let session = match &item { + RegisteredWorkdirSession::Attachment { session, .. } => session, + RegisteredWorkdirSession::Command { session, .. } => &session.source, + }; + if let Err(error) = session.close().await { + errors.push(error.to_string()); + failed.push(item); + } + } + if !failed.is_empty() { + let mut sessions = registry + .lock() + .map_err(|_| "Workdir session registry was poisoned".to_string())?; + for item in failed { + sessions.restore(item); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } +} + #[derive(Clone)] pub struct WorkspaceApi { pub(crate) config: ServerConfig, @@ -363,7 +522,7 @@ pub struct WorkspaceApi { observation_proxy: BackendObservationProxy, runtime_subscription_broker: RuntimeSubscriptionBroker, resource_broker: BackendResourceBroker, - workdir_sessions: Arc>>, + workdir_sessions: Arc>, workdir_session_locks: Arc>>>>, worker_remove_locks: Arc>>>>, worker_control_locks: Arc>>>>, @@ -389,7 +548,7 @@ struct WorkspaceWorkerRemoveExecutor { workspace_id: String, store: Arc, runtime: Weak, - workdir_sessions: Arc>>, + workdir_sessions: Arc>, workdir_session_locks: Arc>>>>, worker_remove_locks: Arc>>>>, worker_control_locks: Arc>>>>, @@ -560,31 +719,21 @@ impl WorkspaceWorkerRemoveExecutor { } else { prepared }; - let session = { - self.workdir_sessions - .lock() - .map_err(|_| "Workdir session registry was poisoned".to_string())? - .get(&target) - .cloned() - }; - if let Some(session) = session { - if session.close().await.is_err() { - let _ = self.store.fail_worker_removal( - &self.workspace_id, - &prepared.plan.operation_id, - &prepared.plan.input_fingerprint, - "workdir_session_close_failed", - ); - return Ok(worker_remove_error_response( - StatusCode::SERVICE_UNAVAILABLE, - "attachment_close_failed", - "Worker Workdir session could not be closed; removal can be retried", - )); - } - self.workdir_sessions - .lock() - .map_err(|_| "Workdir session registry was poisoned".to_string())? - .remove(&target); + if close_worker_workdir_sessions(&self.workdir_sessions, &target) + .await + .is_err() + { + let _ = self.store.fail_worker_removal( + &self.workspace_id, + &prepared.plan.operation_id, + &prepared.plan.input_fingerprint, + "workdir_session_close_failed", + ); + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "attachment_close_failed", + "Worker Workdir sessions could not be closed; removal can be retried", + )); } if self .store @@ -666,30 +815,21 @@ impl WorkspaceWorkerRemoveExecutor { Err(error) => return Ok(worker_retention_error_response(error)), }; - let session = self - .workdir_sessions - .lock() - .map_err(|_| "Workdir session registry was poisoned".to_string())? - .get(&target) - .cloned(); - if let Some(session) = session { - if session.close().await.is_err() { - let _ = self.store.fail_worker_removal( - &self.workspace_id, - &plan.operation_id, - &plan.input_fingerprint, - "workdir_session_close_failed", - ); - return Ok(worker_remove_error_response( - StatusCode::SERVICE_UNAVAILABLE, - "attachment_close_failed", - "Worker Workdir session could not be closed; removal can be retried", - )); - } - self.workdir_sessions - .lock() - .map_err(|_| "Workdir session registry was poisoned".to_string())? - .remove(&target); + if close_worker_workdir_sessions(&self.workdir_sessions, &target) + .await + .is_err() + { + let _ = self.store.fail_worker_removal( + &self.workspace_id, + &plan.operation_id, + &plan.input_fingerprint, + "workdir_session_close_failed", + ); + return Ok(worker_remove_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "attachment_close_failed", + "Worker Workdir sessions could not be closed; removal can be retried", + )); } if let Err(_) = self.store.detach_worker_workdir( @@ -1424,7 +1564,7 @@ impl WorkspaceApi { observation_proxy, runtime_subscription_broker, resource_broker, - workdir_sessions: Arc::new(Mutex::new(HashMap::new())), + workdir_sessions: Arc::new(Mutex::new(WorkdirSessionRegistry::default())), workdir_session_locks: Arc::new(Mutex::new(HashMap::new())), worker_remove_locks: Arc::new(Mutex::new(HashMap::new())), worker_control_locks: Arc::new(Mutex::new(HashMap::new())), @@ -6523,7 +6663,7 @@ async fn open_current_worker_workdir_session_locked( let repository_requires_access = repository.source.kind == workspace_api::RepositorySourceKind::Ssh; if repository_requires_access { - close_current_worker_session_locked(api, worker).await?; + close_current_worker_attachment_session_locked(api, worker).await?; let access = repository_access_request_for_workdir( api, &workdir.runtime_id, @@ -6546,8 +6686,7 @@ async fn open_current_worker_workdir_session_locked( .workdir_sessions .lock() .expect("Workdir session registry lock poisoned") - .get(worker) - .cloned() + .attachment(worker) { return Ok(session); } @@ -6557,24 +6696,13 @@ async fn open_current_worker_workdir_session_locked( .open_workdir_session(&workdir.runtime_id, &workdir.workdir_id, owner_worker_id) .await .map_err(|error| error.into_error())?; - let key = worker.clone(); - let (selected, unused) = { - let mut sessions = api - .workdir_sessions - .lock() - .expect("Workdir session registry lock poisoned"); - match sessions.entry(key) { - std::collections::hash_map::Entry::Occupied(entry) => { - (entry.get().clone(), Some(session)) - } - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert(session.clone()); - (session, None) - } - } - }; - if let Some(unused) = unused { - unused + let replaced = api + .workdir_sessions + .lock() + .expect("Workdir session registry lock poisoned") + .insert_attachment(worker.clone(), session.clone()); + if let Some(replaced) = replaced { + replaced .close() .await .map_err(|error| Error::RuntimeOperationFailed { @@ -6583,35 +6711,45 @@ async fn open_current_worker_workdir_session_locked( message: error.to_string(), })?; } - Ok(selected) + Ok(session) +} + +async fn close_current_worker_attachment_session_locked( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> Result<()> { + let session = api + .workdir_sessions + .lock() + .expect("Workdir session registry lock poisoned") + .remove_attachment(worker); + if let Some(session) = session { + if let Err(error) = session.close().await { + api.workdir_sessions + .lock() + .expect("Workdir session registry lock poisoned") + .insert_attachment(worker.clone(), session); + return Err(Error::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "workdir_session_close_failed".to_string(), + message: error.to_string(), + }); + } + } + Ok(()) } async fn close_current_worker_session_locked( api: &WorkspaceApi, worker: &RuntimeWorkerRef, ) -> Result<()> { - let key = worker.clone(); - let session = api - .workdir_sessions - .lock() - .expect("Workdir session registry lock poisoned") - .get(&key) - .cloned(); - if let Some(session) = session { - session - .close() - .await - .map_err(|error| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "workdir_session_close_failed".to_string(), - message: error.to_string(), - })?; - api.workdir_sessions - .lock() - .expect("Workdir session registry lock poisoned") - .remove(&key); - } - Ok(()) + close_worker_workdir_sessions(&api.workdir_sessions, worker) + .await + .map_err(|message| Error::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "workdir_session_close_failed".to_string(), + message, + }) } async fn scoped_attach_current_worker_workdir( @@ -6721,6 +6859,16 @@ fn validate_current_worker_workdir_session_fence( } } +fn validated_current_worker_attachment( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + expected_session_fence: Option<&str>, +) -> ApiResult { + let link = current_worker_active_attachment(api, worker)?; + validate_current_worker_workdir_session_fence(&link, expected_session_fence)?; + Ok(link) +} + async fn scoped_execute_current_worker_workdir_operation( State(api): State, AxumPath(path): AxumPath, @@ -6729,29 +6877,175 @@ async fn scoped_execute_current_worker_workdir_operation( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; - let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; - let link = current_worker_active_attachment(&api, &worker)?; - validate_current_worker_workdir_session_fence( - &link, - request.expected_session_fence.as_deref(), - )?; - let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; - let applied = workdir::apply_delegation_chain(source, request.delegations) + let expected_session_fence = request.expected_session_fence; + let delegations = request.delegations; + let result = match request.operation { + WorkdirSessionOperation::CommandStart(command) => { + let session_lock = current_worker_session_lock(&api, &worker); + let _session_guard = session_lock.lock().await; + let link = validated_current_worker_attachment( + &api, + &worker, + expected_session_fence.as_deref(), + )?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; + let applied = + apply_current_worker_delegations(&worker, source.clone(), delegations.clone()) + .await?; + let provider_handle = applied + .scoped_session + .start_command(command) + .await + .map_err(|error| current_worker_workdir_operation_error(&worker, error))?; + let registered_source = api + .workdir_sessions + .lock() + .expect("Workdir session registry lock poisoned") + .remove_attachment(&worker) + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "workdir_session_registration_failed".to_string(), + message: "started command session was not registered as the active attachment session" + .to_string(), + })?; + let external_handle = api + .workdir_sessions + .lock() + .expect("Workdir session registry lock poisoned") + .register_command( + worker.clone(), + registered_source, + provider_handle, + delegations, + ); + WorkdirSessionOperationResult::CommandStart(external_handle) + } + WorkdirSessionOperation::CommandStatus(external_handle) => { + let (session, provider_handle) = current_worker_command_session( + &api, + &worker, + &external_handle, + &delegations, + expected_session_fence.as_deref(), + ) + .await?; + session + .scoped_session + .command_status(provider_handle) + .await + .map(WorkdirSessionOperationResult::CommandStatus) + .map_err(|error| current_worker_workdir_operation_error(&worker, error))? + } + WorkdirSessionOperation::CommandOutput(mut output) => { + let (session, provider_handle) = current_worker_command_session( + &api, + &worker, + &output.handle, + &delegations, + expected_session_fence.as_deref(), + ) + .await?; + output.handle = provider_handle; + session + .scoped_session + .command_output(output) + .await + .map(WorkdirSessionOperationResult::CommandOutput) + .map_err(|error| current_worker_workdir_operation_error(&worker, error))? + } + WorkdirSessionOperation::CommandCancel(external_handle) => { + let (session, provider_handle) = current_worker_command_session( + &api, + &worker, + &external_handle, + &delegations, + expected_session_fence.as_deref(), + ) + .await?; + session + .scoped_session + .cancel_command(provider_handle) + .await + .map(|()| WorkdirSessionOperationResult::CommandCancel) + .map_err(|error| current_worker_workdir_operation_error(&worker, error))? + } + operation @ (WorkdirSessionOperation::Stat(_) + | WorkdirSessionOperation::Read(_) + | WorkdirSessionOperation::Write(_) + | WorkdirSessionOperation::Edit(_) + | WorkdirSessionOperation::List(_) + | WorkdirSessionOperation::Glob(_) + | WorkdirSessionOperation::Grep(_)) => { + let session_lock = current_worker_session_lock(&api, &worker); + let _session_guard = session_lock.lock().await; + let link = validated_current_worker_attachment( + &api, + &worker, + expected_session_fence.as_deref(), + )?; + let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; + let applied = apply_current_worker_delegations(&worker, source, delegations).await?; + execute_workdir_session_operation(&applied.scoped_session, operation) + .await + .map_err(|error| current_worker_workdir_operation_error(&worker, error))? + } + }; + Ok(Json(result)) +} + +async fn apply_current_worker_delegations( + worker: &RuntimeWorkerRef, + source: WorkdirSessionHandle, + delegations: Vec, +) -> Result { + workdir::apply_delegation_chain(source, delegations) .await .map_err(|error| Error::RuntimeOperationFailed { runtime_id: worker.runtime_id.clone(), code: "workdir_session_delegation_failed".to_string(), message: error.to_string(), + }) +} + +async fn current_worker_command_session( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + external_handle: &CommandHandle, + delegations: &[workdir::WorkdirDelegationRequest], + expected_session_fence: Option<&str>, +) -> ApiResult<(workdir::AppliedWorkdirDelegation, CommandHandle)> { + let _link = validated_current_worker_attachment(api, worker, expected_session_fence)?; + let command = api + .workdir_sessions + .lock() + .expect("Workdir session registry lock poisoned") + .command(worker, external_handle) + .ok_or_else(|| { + ApiError::from(current_worker_workdir_operation_error( + worker, + workdir::WorkdirError::UnknownCommand(external_handle.0.clone()), + )) })?; - let result = execute_workdir_session_operation(&applied.scoped_session, request.operation) - .await - .map_err(|error| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "workdir_session_operation_failed".to_string(), - message: error.to_string(), - })?; - Ok(Json(result)) + if command.delegations != delegations { + return Err(Error::WorkdirAttachmentConflict( + "command lifecycle delegation differs from CommandStart".to_string(), + ) + .into()); + } + let session = + apply_current_worker_delegations(worker, command.source, command.delegations).await?; + Ok((session, command.provider_handle)) +} + +fn current_worker_workdir_operation_error( + worker: &RuntimeWorkerRef, + error: workdir::WorkdirError, +) -> Error { + Error::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "workdir_session_operation_failed".to_string(), + message: error.to_string(), + } } async fn execute_workdir_session_operation( @@ -14979,6 +15273,110 @@ mod tests { SqliteWorkspaceStore, TrustedRuntimeRecord, UserRecord, WorkspaceRecord, }; + #[tokio::test] + async fn command_session_survives_attachment_refresh_until_worker_revocation() { + let directory = tempfile::tempdir().unwrap(); + let worker = RuntimeWorkerRef { + runtime_id: "runtime-command-session".to_string(), + worker_id: "worker-command-session".to_string(), + }; + let source: WorkdirSessionHandle = Arc::new(workdir::LocalWorkdirSession::new( + manifest::Scope::writable(directory.path()).unwrap(), + directory.path().to_path_buf(), + )); + let provider_handle = source + .start_command(workdir::CommandRequest { + command: "printf ready; sleep 30".to_string(), + timeout_secs: 60, + output_limit: 4096, + tool_call_id: Some("tool-call-command-session".to_string()), + }) + .await + .unwrap(); + + let mut registry = WorkdirSessionRegistry::default(); + registry.insert_attachment(worker.clone(), source.clone()); + let registered_source = registry.remove_attachment(&worker).unwrap(); + let external_handle = registry.register_command( + worker.clone(), + registered_source, + provider_handle.clone(), + Vec::new(), + ); + assert_ne!(external_handle, provider_handle); + + let refreshed: WorkdirSessionHandle = Arc::new(workdir::LocalWorkdirSession::new( + manifest::Scope::writable(directory.path()).unwrap(), + directory.path().to_path_buf(), + )); + registry.insert_attachment(worker.clone(), refreshed); + registry + .remove_attachment(&worker) + .unwrap() + .close() + .await + .unwrap(); + + let command = registry.command(&worker, &external_handle).unwrap(); + let output = command + .source + .command_output(workdir::CommandOutputRequest { + handle: command.provider_handle.clone(), + cursor: 0, + limit: 4096, + wait: false, + }) + .await + .unwrap(); + assert!(matches!(output.status, workdir::CommandStatus::Running)); + assert!(matches!( + command + .source + .command_status(command.provider_handle.clone()) + .await + .unwrap(), + workdir::CommandStatus::Running + )); + let waiting_source = command.source.clone(); + let waiting_handle = command.provider_handle.clone(); + let waiting = tokio::spawn(async move { + waiting_source + .command_output(workdir::CommandOutputRequest { + handle: waiting_handle, + cursor: output.next_cursor.unwrap_or(0), + limit: 4096, + wait: true, + }) + .await + }); + tokio::task::yield_now().await; + command + .source + .cancel_command(command.provider_handle.clone()) + .await + .unwrap(); + let terminal = tokio::time::timeout(std::time::Duration::from_secs(2), waiting) + .await + .expect("cancel should unblock a waiting output read") + .unwrap() + .unwrap(); + assert!(!matches!(terminal.status, workdir::CommandStatus::Running)); + + let registered = registry.take_worker(&worker); + assert_eq!(registered.len(), 1); + let RegisteredWorkdirSession::Command { session, .. } = ®istered[0] else { + panic!("expected retained command session"); + }; + session.source.close().await.unwrap(); + assert!( + session + .source + .command_status(session.provider_handle.clone()) + .await + .is_err() + ); + } + fn seed_test_api_token(store: &dyn ControlPlaneStore, suffix: &str) -> String { let account_id = format!("account-{suffix}"); let user_id = format!("user-{suffix}");