Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: distribute latest workspace config to runtimes

Browse Source
This commit is contained in:
Hare
2026-09-07 02:40:03 +09:00
parent 31f7d39647
commit f1baea1705
11 changed files with 656 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/worker-runtime/src/catalog.rs
+4 -14
View File
@@ -15,32 +15,22 @@ pub enum ProfileSelector {
Named(String),
}
/// Runtime fetch/caching metadata for a Backend-authored Decodal profile source archive.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProfileSourceArchiveHttpRef {
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub etag: Option<String>,
pub archive: ProfileSourceArchiveRef,
}
/// Profile source material available to a Runtime during Worker creation.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ProfileSourceArchiveSource {
/// Backend-internal embedded runtimes may receive already-built archive bytes.
Embedded { archive: ProfileSourceArchive },
/// Standalone runtimes fetch/cache the tar archive over HTTP.
Http {
location: ProfileSourceArchiveHttpRef,
},
/// Standalone runtimes resolve this immutable archive from the latest
/// Workspace Config bundle before creating the Worker.
WorkspaceConfig { archive: ProfileSourceArchiveRef },
}
impl ProfileSourceArchiveSource {
pub fn reference(&self) -> ProfileSourceArchiveRef {
match self {
Self::Embedded { archive } => archive.reference.clone(),
Self::Http { location } => location.archive.clone(),
Self::WorkspaceConfig { archive } => archive.clone(),
}
}
}
crates/worker-runtime/src/config_bundle.rs
+5
View File
@@ -9,6 +9,11 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
pub const CONFIG_BUNDLE_DIGEST_ALGORITHM: &str = "sha256";
pub const WORKSPACE_CONFIG_ETAG_PREFIX: &str = "workspace-config:";
pub fn workspace_config_etag(digest: &str) -> String {
format!("\"{WORKSPACE_CONFIG_ETAG_PREFIX}{digest}\"")
}
/// Backend-synced Profile/config bundle stored by a Runtime.
///
crates/worker-runtime/src/execution.rs
+32 -1
View File
@@ -1,6 +1,7 @@
use crate::catalog::{
RepositoryRefObservation, RepositoryRefObservationRequest,
ConfigBundleRef, ProfileSelector, RepositoryRefObservation, RepositoryRefObservationRequest,
WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus,
WorkspaceApiRef,
};
use crate::config_bundle::ConfigBundle;
use crate::error::RuntimeError;
@@ -265,6 +266,22 @@ pub struct WorkerExecutionRestoreRequest {
pub config_bundle: Option<ConfigBundle>,
}
/// Runtime-side request to refresh the latest Workspace Config before Worker creation.
#[derive(Clone, Debug)]
pub struct WorkspaceConfigFetchRequest {
pub workspace_api: WorkspaceApiRef,
pub profile: ProfileSelector,
pub expected: ConfigBundleRef,
pub cached: Option<ConfigBundleRef>,
}
/// Result of a conditional Workspace Config fetch.
#[derive(Clone, Debug)]
pub enum WorkspaceConfigFetchResult {
NotModified,
Modified(ConfigBundle),
}
/// Backend outcome for Worker spawn/restore operations.
#[derive(Clone, Debug)]
pub enum WorkerExecutionSpawnResult {
@@ -291,6 +308,13 @@ impl WorkerExecutionSpawnResult {
pub trait WorkerExecutionBackend: Send + Sync + 'static {
fn backend_id(&self) -> &str;
fn fetch_workspace_config(
&self,
_request: WorkspaceConfigFetchRequest,
) -> Result<WorkspaceConfigFetchResult, String> {
Err("execution backend does not support Workspace Config fetching".to_string())
}
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult;
fn restore_worker(
@@ -472,6 +496,13 @@ impl WorkerExecutionBackendRef {
})
}
pub(crate) fn fetch_workspace_config(
&self,
request: WorkspaceConfigFetchRequest,
) -> Result<WorkspaceConfigFetchResult, String> {
self.backend.fetch_workspace_config(request)
}
pub(crate) fn spawn_worker(
&self,
request: WorkerExecutionSpawnRequest,
crates/worker-runtime/src/http_server.rs
+28 -28
View File
@@ -2694,22 +2694,22 @@ mod tests {
create_fingerprint: "test-create".to_string(),
profile,
display_name: None,
profile_source: crate::catalog::ProfileSourceArchiveSource::Http {
location: crate::catalog::ProfileSourceArchiveHttpRef {
url: "http://127.0.0.1/profile-source.tar".to_string(),
etag: None,
archive: crate::profile_archive::ProfileSourceArchiveRef {
profile_source: crate::catalog::ProfileSourceArchiveSource::Embedded {
archive: crate::profile_archive::ProfileSourceArchive::build(
crate::profile_archive::ProfileSourceArchiveInput {
id: "test-profile-source".to_string(),
digest: "test-digest".to_string(),
size_bytes: 0,
source_graph: crate::profile_archive::ProfileSourceGraphSummary {
source_count: 0,
total_source_bytes: 0,
entrypoints: std::collections::BTreeMap::new(),
import_count: 0,
},
entrypoints: std::collections::BTreeMap::from([(
"builtin:coder".to_string(),
"profiles/coder.dcdl".to_string(),
)]),
imports: std::collections::BTreeMap::new(),
sources: std::collections::BTreeMap::from([(
"profiles/coder.dcdl".to_string(),
"{}".to_string(),
)]),
},
},
)
.unwrap(),
},
config_bundle: Some(ConfigBundleRef {
id: bundle.metadata.id,
@@ -3312,22 +3312,22 @@ mod ws_tests {
create_fingerprint: "test-create".to_string(),
profile: ProfileSelector::Builtin("builtin:companion".to_string()),
display_name: None,
profile_source: crate::catalog::ProfileSourceArchiveSource::Http {
location: crate::catalog::ProfileSourceArchiveHttpRef {
url: "http://127.0.0.1/profile-source.tar".to_string(),
etag: None,
archive: crate::profile_archive::ProfileSourceArchiveRef {
profile_source: crate::catalog::ProfileSourceArchiveSource::Embedded {
archive: crate::profile_archive::ProfileSourceArchive::build(
crate::profile_archive::ProfileSourceArchiveInput {
id: "test-profile-source".to_string(),
digest: "test-digest".to_string(),
size_bytes: 0,
source_graph: crate::profile_archive::ProfileSourceGraphSummary {
source_count: 0,
total_source_bytes: 0,
entrypoints: std::collections::BTreeMap::new(),
import_count: 0,
},
entrypoints: std::collections::BTreeMap::from([(
"builtin:coder".to_string(),
"profiles/coder.dcdl".to_string(),
)]),
imports: std::collections::BTreeMap::new(),
sources: std::collections::BTreeMap::from([(
"profiles/coder.dcdl".to_string(),
"{}".to_string(),
)]),
},
},
)
.unwrap(),
},
config_bundle: Some(ConfigBundleRef {
id: bundle.metadata.id,
crates/worker-runtime/src/resource.rs
-19
View File
@@ -6,8 +6,6 @@ use crate::identity::WorkerId;
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Mutex;
pub const PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE: &str =
"application/vnd.yoi.profile-source-archive+tar";
@@ -266,23 +264,6 @@ impl BackendResourceClient for HttpBackendResourceClient {
}
}
#[derive(Default, Debug)]
pub struct ProfileSourceArchiveCache {
archives: Mutex<HashMap<String, ProfileSourceArchive>>,
}
impl ProfileSourceArchiveCache {
pub fn get(&self, digest: &str) -> Option<ProfileSourceArchive> {
self.archives.lock().ok()?.get(digest).cloned()
}
pub fn insert(&self, archive: ProfileSourceArchive) {
if let Ok(mut archives) = self.archives.lock() {
archives.insert(archive.reference.digest.clone(), archive);
}
}
}
pub fn build_profile_source_archive_fetch_request(
handle: BackendResourceHandle,
runtime_id: &str,
crates/worker-runtime/src/runtime.rs
+228 -29
View File
@@ -1,8 +1,9 @@
use crate::catalog::{
ConfigBundleRef, CreateWorkerRequest, ProfileSelector, RepositoryRefObservation,
RepositoryRefObservationRequest, WorkerDetail, WorkerLifecycleAck, WorkerRestoreIntent,
WorkerStatus, WorkerSummary, WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest,
WorkingDirectoryStatus as CatalogWorkingDirectoryStatus, WorkspaceApiRef,
ConfigBundleRef, CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveSource,
RepositoryRefObservation, RepositoryRefObservationRequest, WorkerDetail, WorkerLifecycleAck,
WorkerRestoreIntent, WorkerStatus, WorkerSummary, WorkingDirectoryRepositoryAccessRequest,
WorkingDirectoryRequest, WorkingDirectoryStatus as CatalogWorkingDirectoryStatus,
WorkspaceApiRef,
};
use crate::config_bundle::{
ConfigBundle, ConfigBundleAvailability, ConfigBundleSummary, validate_config_bundle,
@@ -14,7 +15,7 @@ use crate::execution::WorkerExecutionRestoreRequest;
use crate::execution::{
WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle,
WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionSpawnRequest,
WorkerExecutionSpawnResult,
WorkerExecutionSpawnResult, WorkspaceConfigFetchRequest, WorkspaceConfigFetchResult,
};
#[cfg(feature = "fs-store")]
use crate::fs_store::{
@@ -643,11 +644,147 @@ impl Runtime {
self.create_worker_with_workspace(request, Some(scope))
}
fn existing_worker_for_create(
&self,
request: &CreateWorkerRequest,
scope: Option<&RuntimeWorkspaceScope>,
) -> Result<Option<WorkerDetail>, RuntimeError> {
let mut state = self.lock()?;
state.ensure_running()?;
validate_create_worker_request(request)?;
validate_create_workspace_scope(request, scope.map(|scope| scope.workspace_id.as_str()))?;
if let Some(scope) = scope {
state.ensure_workspace_owner(scope, false)?;
}
let workspace_id = scope.map(|scope| scope.workspace_id.as_str());
if let Some(existing) = state.workers.get(&request.worker_id) {
if existing.workspace_id.as_deref() != workspace_id {
return Err(RuntimeError::InvalidRequest(format!(
"worker {} already belongs to another Workspace scope",
request.worker_id
)));
}
if existing.request.create_fingerprint != request.create_fingerprint {
return Err(RuntimeError::InvalidRequest(format!(
"worker {} was already created with a different fingerprint",
request.worker_id
)));
}
return Ok(Some(existing.detail()));
}
Ok(None)
}
fn refresh_workspace_config(&self, request: &CreateWorkerRequest) -> Result<(), RuntimeError> {
let ProfileSourceArchiveSource::WorkspaceConfig {
archive: expected_archive,
} = &request.profile_source
else {
return Ok(());
};
let expected = request.config_bundle.as_ref().ok_or_else(|| {
RuntimeError::InvalidRequest(
"Workspace Config profile source requires a config_bundle reference".to_string(),
)
})?;
let workspace_api = request.workspace_api.clone().ok_or_else(|| {
RuntimeError::InvalidRequest(
"Workspace Config profile source requires a Workspace API reference".to_string(),
)
})?;
let profile_key = match &request.profile {
ProfileSelector::Builtin(value) | ProfileSelector::Named(value) => value.clone(),
};
let cache_key = format!(
"{}\u{0}{}\u{0}{}",
workspace_api.base_url, workspace_api.workspace_id, profile_key
);
let fetch_gate = {
let mut state = self.lock()?;
state
.workspace_config_fetch_gates
.entry(cache_key.clone())
.or_insert_with(|| Arc::new(Mutex::new(())))
.clone()
};
let _fetch_guard = fetch_gate.lock().map_err(|_| RuntimeError::StatePoisoned)?;
let (backend, cached) = {
let state = self.lock()?;
let cached_reference = state
.workspace_config_latest
.get(&cache_key)
.unwrap_or(expected);
let cached = state
.config_bundles
.get(&cached_reference.id)
.filter(|bundle| bundle.metadata.digest == cached_reference.digest)
.map(|bundle| ConfigBundleRef {
id: bundle.metadata.id.clone(),
digest: bundle.metadata.digest.clone(),
});
(state.execution_backend.clone(), cached)
};
let backend = backend.ok_or_else(|| RuntimeError::ExecutionBackendUnavailable {
message: "Workspace Config refresh requires an execution backend".to_string(),
})?;
let fetched = backend
.fetch_workspace_config(WorkspaceConfigFetchRequest {
workspace_api,
profile: request.profile.clone(),
expected: expected.clone(),
cached: cached.clone(),
})
.map_err(|message| {
RuntimeError::InvalidRequest(format!(
"failed to refresh latest Workspace Config: {message}"
))
})?;
let resolved = match fetched {
WorkspaceConfigFetchResult::NotModified => cached.ok_or_else(|| {
RuntimeError::InvalidRequest(
"latest Workspace Config returned not-modified without a cached bundle"
.to_string(),
)
})?,
WorkspaceConfigFetchResult::Modified(bundle) => {
self.store_config_bundle(bundle)?.reference
}
};
if &resolved != expected {
return Err(RuntimeError::InvalidRequest(format!(
"latest Workspace Config changed while Worker creation was being prepared: expected '{}@{}', got '{}@{}'",
expected.id, expected.digest, resolved.id, resolved.digest
)));
}
let mut state = self.lock()?;
let bundle = state.config_bundles.get(&resolved.id).ok_or_else(|| {
RuntimeError::InvalidRequest(
"latest Workspace Config was not available after refresh".to_string(),
)
})?;
let archive = bundle.profile_source_archive.as_ref().ok_or_else(|| {
RuntimeError::InvalidRequest(
"latest Workspace Config is missing its profile source archive".to_string(),
)
})?;
if &archive.reference != expected_archive {
return Err(RuntimeError::InvalidRequest(
"latest Workspace Config profile source archive reference mismatch".to_string(),
));
}
state.workspace_config_latest.insert(cache_key, resolved);
Ok(())
}
fn create_worker_with_workspace(
&self,
request: CreateWorkerRequest,
scope: Option<&RuntimeWorkspaceScope>,
) -> Result<WorkerDetail, RuntimeError> {
if let Some(existing) = self.existing_worker_for_create(&request, scope)? {
return Ok(existing);
}
self.refresh_workspace_config(&request)?;
let (backend, worker_ref, spawn_request) = {
let mut state = self.lock()?;
state.ensure_running()?;
@@ -2170,6 +2307,8 @@ struct RuntimeState {
workers: BTreeMap<WorkerId, WorkerRecord>,
workspace_owners: BTreeMap<String, String>,
config_bundles: BTreeMap<String, ConfigBundle>,
workspace_config_latest: BTreeMap<String, ConfigBundleRef>,
workspace_config_fetch_gates: BTreeMap<String, Arc<Mutex<()>>>,
diagnostics: Vec<RuntimeDiagnostic>,
subscription_revision: u64,
worker_subject_revisions: BTreeMap<WorkerId, u64>,
@@ -2198,6 +2337,8 @@ impl RuntimeState {
workers: BTreeMap::new(),
workspace_owners: BTreeMap::new(),
config_bundles: BTreeMap::new(),
workspace_config_latest: BTreeMap::new(),
workspace_config_fetch_gates: BTreeMap::new(),
diagnostics: Vec::new(),
subscription_revision: 0,
worker_subject_revisions: BTreeMap::new(),
@@ -2227,6 +2368,8 @@ impl RuntimeState {
workers: BTreeMap::new(),
workspace_owners: BTreeMap::new(),
config_bundles: BTreeMap::new(),
workspace_config_latest: BTreeMap::new(),
workspace_config_fetch_gates: BTreeMap::new(),
diagnostics: Vec::new(),
subscription_revision: 0,
worker_subject_revisions: BTreeMap::new(),
@@ -2286,6 +2429,8 @@ impl RuntimeState {
next_diagnostic_id,
workers,
config_bundles: BTreeMap::new(),
workspace_config_latest: BTreeMap::new(),
workspace_config_fetch_gates: BTreeMap::new(),
workspace_owners: persisted.workspace_owners,
diagnostics,
subscription_revision: 0,
@@ -3126,15 +3271,10 @@ fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), R
RuntimeError::InvalidRequest(format!("profile_source archive is invalid: {err}"))
})?;
}
crate::catalog::ProfileSourceArchiveSource::Http { location } => {
if location.url.trim().is_empty() {
crate::catalog::ProfileSourceArchiveSource::WorkspaceConfig { archive } => {
if archive.digest.trim().is_empty() {
return Err(RuntimeError::InvalidRequest(
"profile_source.location.url must not be empty".to_string(),
));
}
if location.archive.digest.trim().is_empty() {
return Err(RuntimeError::InvalidRequest(
"profile_source.location.archive.digest must not be empty".to_string(),
"profile_source.archive.digest must not be empty".to_string(),
));
}
}
@@ -3494,6 +3634,21 @@ mod tests {
assert!(validate_create_worker_request(&request).is_ok());
}
fn test_profile_source_archive() -> crate::profile_archive::ProfileSourceArchive {
crate::profile_archive::ProfileSourceArchive::build(
crate::profile_archive::ProfileSourceArchiveInput {
id: "test-profile-source".to_string(),
entrypoints: BTreeMap::from([(
"builtin:coder".to_string(),
"profiles/coder.dcdl".to_string(),
)]),
imports: BTreeMap::new(),
sources: BTreeMap::from([("profiles/coder.dcdl".to_string(), "{}".to_string())]),
},
)
.unwrap()
}
fn task_request(_objective: &str) -> CreateWorkerRequest {
let profile = ProfileSelector::Builtin("builtin:coder".to_string());
let bundle = test_bundle_for_profile(profile.clone());
@@ -3502,22 +3657,8 @@ mod tests {
create_fingerprint: "test-create".to_string(),
profile,
display_name: None,
profile_source: crate::catalog::ProfileSourceArchiveSource::Http {
location: crate::catalog::ProfileSourceArchiveHttpRef {
url: "http://127.0.0.1/profile-source.tar".to_string(),
etag: None,
archive: crate::profile_archive::ProfileSourceArchiveRef {
id: "test-profile-source".to_string(),
digest: "test-digest".to_string(),
size_bytes: 0,
source_graph: crate::profile_archive::ProfileSourceGraphSummary {
source_count: 0,
total_source_bytes: 0,
entrypoints: BTreeMap::new(),
import_count: 0,
},
},
},
profile_source: crate::catalog::ProfileSourceArchiveSource::Embedded {
archive: test_profile_source_archive(),
},
config_bundle: Some(ConfigBundleRef {
id: bundle.metadata.id,
@@ -3853,6 +3994,8 @@ mod tests {
restore_count: Mutex<u64>,
run_generations: Mutex<Vec<u64>>,
config_bundles: Mutex<Vec<Option<ConfigBundle>>>,
workspace_config_fetches: Mutex<Vec<WorkspaceConfigFetchRequest>>,
workspace_config_results: Mutex<Vec<WorkspaceConfigFetchResult>>,
contexts: Mutex<BTreeMap<WorkerId, WorkerExecutionContext>>,
dispatched_inputs: Mutex<Vec<WorkerInput>>,
repository_accesses: Mutex<Vec<WorkingDirectoryRepositoryAccessRequest>>,
@@ -3898,6 +4041,18 @@ mod tests {
"test-execution-backend"
}
fn fetch_workspace_config(
&self,
request: WorkspaceConfigFetchRequest,
) -> Result<WorkspaceConfigFetchResult, String> {
self.workspace_config_fetches.lock().unwrap().push(request);
let mut results = self.workspace_config_results.lock().unwrap();
if results.is_empty() {
return Err("no Workspace Config fetch result configured".to_string());
}
Ok(results.remove(0))
}
fn create_working_directory(
&self,
request: &WorkingDirectoryRequest,
@@ -4585,6 +4740,50 @@ mod tests {
);
}
#[test]
fn remote_create_refreshes_workspace_config_and_revalidates_cached_etag() {
let backend = Arc::new(TestExecutionBackend::default());
let runtime =
Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap();
let mut bundle = test_bundle();
let archive = test_profile_source_archive();
bundle.profile_source_archive = Some(archive.clone());
bundle = bundle.with_computed_digest();
backend.workspace_config_results.lock().unwrap().extend([
WorkspaceConfigFetchResult::Modified(bundle.clone()),
WorkspaceConfigFetchResult::NotModified,
]);
let archive = archive.reference;
let request = |objective: &str| {
let mut request = bundled_task_request(objective, &bundle);
request.profile_source = ProfileSourceArchiveSource::WorkspaceConfig {
archive: archive.clone(),
};
request.workspace_api = Some(WorkspaceApiRef {
workspace_id: "workspace-test".to_string(),
base_url: "https://workspace.example".to_string(),
});
request
};
let first = request("first refresh");
let second = request("cached refresh");
runtime.create_worker(first).unwrap();
runtime.create_worker(second.clone()).unwrap();
runtime.create_worker(second).unwrap();
let fetches = backend.workspace_config_fetches.lock().unwrap();
assert_eq!(fetches.len(), 2);
assert_eq!(fetches[0].cached, None);
assert_eq!(
fetches[1].cached,
Some(ConfigBundleRef {
id: bundle.metadata.id.clone(),
digest: bundle.metadata.digest.clone(),
})
);
}
#[test]
fn restore_does_not_require_recorded_config_bundle() {
let (runtime, backend) = runtime_and_backend();
crates/worker-runtime/src/worker_backend.rs
+126 -112
View File
@@ -19,18 +19,19 @@ use crate::auth::{
RuntimeIdentityMaterial, RuntimeRequestSourceSigner, unix_now_seconds,
};
use crate::catalog::{
CreateWorkerRequest, ProfileSourceArchiveHttpRef, ProfileSourceArchiveSource,
RepositoryRefObservation, RepositoryRefObservationRequest,
WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus,
CreateWorkerRequest, ProfileSourceArchiveSource, RepositoryRefObservation,
RepositoryRefObservationRequest, WorkingDirectoryRepositoryAccessRequest,
WorkingDirectoryRequest, WorkingDirectoryStatus,
};
use crate::config_bundle::{ConfigBundle, workspace_config_etag};
use crate::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest,
WorkerExecutionSpawnResult,
WorkerExecutionSpawnResult, WorkspaceConfigFetchRequest, WorkspaceConfigFetchResult,
};
use crate::identity::WorkerRef;
use crate::interaction::{WorkerInput, WorkerInputKind};
use crate::resource::{BackendResourceClient, ProfileSourceArchiveCache};
use crate::resource::BackendResourceClient;
use crate::worker_source::{
EmbeddedWorkerMutationDispatcher, RuntimeOwnedWorkspaceClient, RuntimeWorkerMutationForwarder,
};
@@ -38,6 +39,8 @@ use crate::working_directory::{
WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer,
};
use async_trait::async_trait;
#[cfg(feature = "http-server")]
use futures::StreamExt;
#[cfg(test)]
use protocol::WorkerStatus;
use protocol::{Event, Method, Segment, WorkerCommandEnvelope};
@@ -86,6 +89,8 @@ use worker::{
const DEFAULT_BACKEND_ID: &str = "worker-crate";
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
const WORKSPACE_CONFIG_HTTP_TIMEOUT: Duration = Duration::from_secs(8);
const MAX_WORKSPACE_CONFIG_RESPONSE_BYTES: usize = 72 * 1024 * 1024;
// Keep this below the adapter task timeout so a failed acknowledgement task
// returns a typed execution error instead of leaving the outer waiter to time out.
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9);
@@ -107,6 +112,13 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static {
Ok(())
}
async fn fetch_workspace_config(
&self,
_request: WorkspaceConfigFetchRequest,
) -> Result<WorkspaceConfigFetchResult, String> {
Err("Runtime Worker factory does not support Workspace Config fetching".to_string())
}
async fn spawn_controller(
&self,
request: WorkerExecutionSpawnRequest,
@@ -307,7 +319,6 @@ pub struct ProfileRuntimeWorkerFactory {
profile_base_dir: PathBuf,
worker_aggregate_root: Option<PathBuf>,
resource_client: Option<Arc<dyn BackendResourceClient>>,
profile_archive_cache: Arc<ProfileSourceArchiveCache>,
prompt_projection_cache: Arc<WorkspacePromptProjectionCache>,
runtime_id: Option<String>,
worker_mutation_identity: Option<RuntimeIdentityMaterial>,
@@ -324,7 +335,6 @@ impl ProfileRuntimeWorkerFactory {
profile_base_dir,
worker_aggregate_root: None,
resource_client: None,
profile_archive_cache: Arc::new(ProfileSourceArchiveCache::default()),
prompt_projection_cache: Arc::new(WorkspacePromptProjectionCache::default()),
runtime_id: None,
worker_mutation_identity: None,
@@ -486,62 +496,28 @@ impl ProfileRuntimeWorkerFactory {
async fn resolve_profile_source_archive(
&self,
source: &ProfileSourceArchiveSource,
request_audience: Option<&str>,
config_bundle: Option<&ConfigBundle>,
) -> Result<crate::profile_archive::VerifiedProfileSourceArchive, String> {
match source {
ProfileSourceArchiveSource::Embedded { archive } => archive
.verify()
.map_err(|err| format!("failed to verify embedded profile source archive: {err}")),
ProfileSourceArchiveSource::Http { location } => {
self.fetch_profile_source_archive(location, request_audience)
.await
}
}
}
async fn fetch_profile_source_archive(
&self,
location: &ProfileSourceArchiveHttpRef,
request_audience: Option<&str>,
) -> Result<crate::profile_archive::VerifiedProfileSourceArchive, String> {
if let Some(cached) = self.profile_archive_cache.get(&location.archive.digest) {
let response = fetch_profile_source_archive_http(
location,
Some(&location.archive.digest),
self.worker_mutation_identity.as_ref(),
self.runtime_request_audience
.as_deref()
.or(request_audience),
)
.await?;
if let Some(fetched) = response {
self.profile_archive_cache.insert(fetched.clone());
fetched.verify().map_err(|err| {
format!("failed to verify fetched profile source archive: {err}")
ProfileSourceArchiveSource::WorkspaceConfig { archive } => {
let bundle = config_bundle.ok_or_else(|| {
"Workspace Config profile source requires a resolved Config Bundle".to_string()
})?;
let embedded = bundle.profile_source_archive.as_ref().ok_or_else(|| {
"resolved Workspace Config is missing its profile source archive".to_string()
})?;
if &embedded.reference != archive {
return Err(
"Workspace Config profile source archive reference mismatch".to_string()
);
}
embedded.verify().map_err(|err| {
format!("failed to verify Workspace Config profile source archive: {err}")
})
} else {
cached
.verify()
.map_err(|err| format!("failed to verify cached profile source archive: {err}"))
}
} else {
let archive = fetch_profile_source_archive_http(
location,
None,
self.worker_mutation_identity.as_ref(),
self.runtime_request_audience
.as_deref()
.or(request_audience),
)
.await?
.ok_or_else(|| {
"profile source archive HTTP revalidation returned 304 without a cached archive"
.to_string()
})?;
self.profile_archive_cache.insert(archive.clone());
archive
.verify()
.map_err(|err| format!("failed to verify fetched profile source archive: {err}"))
}
}
}
@@ -627,91 +603,112 @@ impl RuntimeWorkspaceBackendRef {
}
#[cfg(feature = "http-server")]
async fn fetch_profile_source_archive_http(
location: &ProfileSourceArchiveHttpRef,
cached_digest: Option<&str>,
async fn fetch_workspace_config_http(
request: &WorkspaceConfigFetchRequest,
identity: Option<&RuntimeIdentityMaterial>,
audience: Option<&str>,
) -> Result<Option<crate::profile_archive::ProfileSourceArchive>, String> {
let client = reqwest::Client::new();
let url = reqwest::Url::parse(&location.url)
.map_err(|error| format!("profile source archive URL is invalid: {error}"))?;
) -> Result<WorkspaceConfigFetchResult, String> {
let mut url = reqwest::Url::parse(&request.workspace_api.base_url)
.map_err(|error| format!("Workspace API base URL is invalid: {error}"))?;
url.set_path(&format!(
"/api/w/{}/runtime-config",
request.workspace_api.workspace_id
));
let profile = match &request.profile {
crate::catalog::ProfileSelector::Builtin(value)
| crate::catalog::ProfileSelector::Named(value) => value.clone(),
};
url.query_pairs_mut().append_pair("profile", &profile);
let path = url.path().to_owned();
let workspace_id = path
.split('/')
.collect::<Vec<_>>()
.windows(2)
.find_map(|parts| (parts[0] == "w").then_some(parts[1]))
.filter(|value| !value.is_empty())
.ok_or_else(|| "profile source archive URL is not workspace-scoped".to_owned())?;
let mut request = client.get(url);
let request_target = match url.query() {
Some(query) => format!("{path}?{query}"),
None => path.clone(),
};
let client = reqwest::Client::builder()
.connect_timeout(Duration::from_secs(3))
.timeout(WORKSPACE_CONFIG_HTTP_TIMEOUT)
.build()
.map_err(|error| format!("failed to build Workspace Config HTTP client: {error}"))?;
let mut http_request = client.get(url);
if let Some(identity) = identity {
let audience = audience.ok_or_else(|| {
"profile source archive request proof audience is unavailable".to_owned()
})?;
let audience = audience
.ok_or_else(|| "Workspace Config request proof audience is unavailable".to_owned())?;
let proof = RuntimeRequestSourceSigner::from_identity(identity)
.issue(
audience,
workspace_id,
&request.workspace_api.workspace_id,
None,
BACKEND_RESOURCE_FETCH_PERMISSION,
"GET",
&path,
&request_target,
b"",
i64::try_from(unix_now_seconds()).unwrap_or(i64::MAX),
30,
)
.map_err(|error| error.to_string())?;
request = request.header(RUNTIME_REQUEST_SOURCE_PROOF_HEADER, proof);
http_request = http_request.header(RUNTIME_REQUEST_SOURCE_PROOF_HEADER, proof);
}
if cached_digest == Some(location.archive.digest.as_str()) {
if let Some(etag) = location.etag.as_deref() {
request = request.header(reqwest::header::IF_NONE_MATCH, etag);
}
if let Some(cached) = request.cached.as_ref() {
http_request = http_request.header(
reqwest::header::IF_NONE_MATCH,
workspace_config_etag(&cached.digest),
);
}
let response = request
let response = http_request
.send()
.await
.map_err(|err| format!("failed to fetch profile source archive: {err}"))?;
.map_err(|error| format!("failed to fetch latest Workspace Config: {error}"))?;
if response.status() == reqwest::StatusCode::NOT_MODIFIED {
return Ok(None);
return Ok(WorkspaceConfigFetchResult::NotModified);
}
if !response.status().is_success() {
let status = response.status();
return Err(format!(
"profile source archive fetch failed with HTTP {status}"
"latest Workspace Config fetch failed with HTTP {}",
response.status()
));
}
let bytes = response
.bytes()
.await
.map_err(|err| format!("failed to read profile source archive response: {err}"))?
.to_vec();
let archive = crate::profile_archive::ProfileSourceArchive {
reference: location.archive.clone(),
content: bytes,
};
if archive.content.len() as u64 != archive.reference.size_bytes {
if response
.content_length()
.is_some_and(|size| size > MAX_WORKSPACE_CONFIG_RESPONSE_BYTES as u64)
{
return Err("latest Workspace Config response exceeds the size limit".to_string());
}
let response_etag = response
.headers()
.get(reqwest::header::ETAG)
.and_then(|value| value.to_str().ok())
.map(str::to_string)
.ok_or_else(|| "latest Workspace Config response is missing its ETag".to_string())?;
let mut body = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk =
chunk.map_err(|error| format!("failed to read latest Workspace Config: {error}"))?;
if body.len().saturating_add(chunk.len()) > MAX_WORKSPACE_CONFIG_RESPONSE_BYTES {
return Err("latest Workspace Config response exceeds the size limit".to_string());
}
body.extend_from_slice(&chunk);
}
let bundle = serde_json::from_slice::<ConfigBundle>(&body)
.map_err(|error| format!("failed to decode latest Workspace Config: {error}"))?;
let expected_etag = workspace_config_etag(&bundle.metadata.digest);
if response_etag != expected_etag {
return Err(format!(
"profile source archive size mismatch: expected {}, got {}",
archive.reference.size_bytes,
archive.content.len()
"latest Workspace Config ETag mismatch: expected {expected_etag}, got {response_etag}"
));
}
Ok(Some(archive))
Ok(WorkspaceConfigFetchResult::Modified(bundle))
}
#[cfg(not(feature = "http-server"))]
async fn fetch_profile_source_archive_http(
_location: &ProfileSourceArchiveHttpRef,
_cached_digest: Option<&str>,
async fn fetch_workspace_config_http(
_request: &WorkspaceConfigFetchRequest,
_identity: Option<&RuntimeIdentityMaterial>,
_audience: Option<&str>,
) -> Result<Option<crate::profile_archive::ProfileSourceArchive>, String> {
Err(
"HTTP profile source archive fetch requires the worker-runtime http-server feature"
.to_string(),
)
) -> Result<WorkspaceConfigFetchResult, String> {
Err("Workspace Config fetch requires the worker-runtime http-server feature".to_string())
}
fn runtime_local_workdir_session(
@@ -803,6 +800,18 @@ fn validate_worker_memory_settings(
#[async_trait]
impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
async fn fetch_workspace_config(
&self,
request: WorkspaceConfigFetchRequest,
) -> Result<WorkspaceConfigFetchResult, String> {
fetch_workspace_config_http(
&request,
self.worker_mutation_identity.as_ref(),
self.runtime_request_audience.as_deref(),
)
.await
}
fn observe_workspace_prompt_projection(
&self,
projection: worker::WorkspacePromptProjection,
@@ -856,10 +865,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
let archive = self
.resolve_profile_source_archive(
&request.request.profile_source,
request
.workspace_scope
.as_ref()
.map(|scope| scope.server_id.as_str()),
request.config_bundle.as_ref(),
)
.await?;
let (mut manifest, mut loader) = {
@@ -1557,6 +1563,14 @@ where
&self.backend_id
}
fn fetch_workspace_config(
&self,
request: WorkspaceConfigFetchRequest,
) -> Result<WorkspaceConfigFetchResult, String> {
let factory = self.factory.clone();
self.run_on_adapter_runtime(async move { factory.fetch_workspace_config(request).await })
}
fn observe_workspace_prompt_projection(
&self,
projection: worker::WorkspacePromptProjection,