feat: add backend resource fetch api

This commit is contained in:
2026-07-08 20:18:38 +09:00
parent 1a2097b10d
commit 57e96d3be8
16 changed files with 859 additions and 41 deletions
+2 -1
View File
@@ -14,7 +14,7 @@ required-features = ["ws-server", "fs-store"]
[features]
default = []
fs-store = []
http-server = ["dep:axum", "dep:tower"]
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
[dependencies]
@@ -28,6 +28,7 @@ serde = { workspace = true, features = ["derive"] }
session-store.workspace = true
sha2.workspace = true
serde_json.workspace = true
reqwest = { version = "0.13", optional = true, default-features = false, features = ["json", "rustls"] }
tar.workspace = true
thiserror = { workspace = true }
tokio = { workspace = true, features = ["net", "rt", "sync", "time"] }
+105 -1
View File
@@ -4,6 +4,7 @@ use crate::profile_archive::{
ProfileArchiveError, ProfileSourceArchive, ProfileSourceArchiveRef,
VerifiedProfileSourceArchive,
};
use crate::resource::{BackendResourceHandle, validate_resource_handle_text};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
@@ -24,6 +25,8 @@ pub struct ConfigBundle {
pub declarations: Vec<ConfigDeclaration>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profile_source_archive: Option<ProfileSourceArchive>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profile_source_archive_handle: Option<BackendResourceHandle>,
}
impl ConfigBundle {
@@ -75,6 +78,27 @@ impl ConfigBundle {
lines.push(format!("profile_archive_entrypoint\0{selector}\0{path}"));
}
}
if let Some(handle) = &self.profile_source_archive_handle {
lines.push(format!(
"profile_archive_handle\0{}\0{}\0{}\0{}\0{}",
handle.workspace_id,
handle.resource_id,
handle.digest,
handle.revision,
handle.max_bytes
));
for (selector, path) in handle
.profile_source_graph
.as_ref()
.map(|graph| &graph.entrypoints)
.into_iter()
.flatten()
{
lines.push(format!(
"profile_archive_handle_entrypoint\0{selector}\0{path}"
));
}
}
lines.sort();
let mut hasher = Sha256::new();
@@ -105,7 +129,12 @@ impl ConfigBundle {
profile_source_archive: self
.profile_source_archive
.as_ref()
.map(|archive| archive.reference.source_graph.clone()),
.map(|archive| archive.reference.source_graph.clone())
.or_else(|| {
self.profile_source_archive_handle
.as_ref()
.and_then(|handle| handle.profile_source_graph.clone())
}),
}
}
@@ -253,6 +282,33 @@ pub(crate) fn validate_config_bundle(bundle: &ConfigBundle) -> Result<(), Runtim
RuntimeError::InvalidRequest(format!("invalid profile source archive: {err}"))
})?;
}
if let Some(handle) = &bundle.profile_source_archive_handle {
for (label, value) in [
("resource handle workspace id", handle.workspace_id.as_str()),
("resource handle resource id", handle.resource_id.as_str()),
("resource handle digest", handle.digest.as_str()),
("resource handle nonce", handle.nonce.as_str()),
("resource handle revision", handle.revision.as_str()),
("resource handle content type", handle.content_type.as_str()),
(
"resource handle audit correlation id",
handle.audit_correlation_id.as_str(),
),
] {
validate_resource_handle_text(label, value).map_err(RuntimeError::InvalidRequest)?;
}
if !handle.digest.starts_with("sha256:") {
return Err(RuntimeError::InvalidRequest(
"resource handle digest must use sha256:<hex>".to_string(),
));
}
if handle.profile_source_graph.is_none() {
return Err(RuntimeError::InvalidRequest(
"profile source archive resource handle must include source graph summary"
.to_string(),
));
}
}
Ok(())
}
@@ -529,6 +585,7 @@ mod tests {
reference: reference.to_string(),
}],
profile_source_archive: None,
profile_source_archive_handle: None,
}
.with_computed_digest()
}
@@ -560,6 +617,53 @@ mod tests {
validate_config_bundle(&bundle_with_declaration("vault:team.api-key")).unwrap();
}
#[test]
fn bundle_summary_redacts_runtime_internal_resource_handle() {
let mut bundle = bundle_with_declaration("secret:github-token");
let source_graph = crate::profile_archive::ProfileSourceGraphSummary {
entrypoints: std::collections::BTreeMap::from([(
"default".to_string(),
"profiles/default.dcdl".to_string(),
)]),
source_count: 1,
import_count: 0,
total_source_bytes: 16,
};
bundle.profile_source_archive_handle = Some(crate::resource::BackendResourceHandle {
kind: crate::resource::BackendResourceKind::ProfileSourceArchive,
workspace_id: "workspace-1".to_string(),
scope_id: Some("scope-1".to_string()),
runtime_id: Some("runtime-1".to_string()),
worker_id: Some("worker-1".to_string()),
resource_id: "profile-source-archive:test".to_string(),
digest: "sha256:0123456789abcdef".to_string(),
operation: crate::resource::BackendResourceOperation::FetchArchive,
expires_at_unix_seconds: 4_102_444_800,
nonce: "nonce-not-for-browser".to_string(),
revision: "rev-1".to_string(),
generation: Some(1),
max_bytes: 128,
content_type: crate::resource::PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE.to_string(),
redaction: crate::resource::ResourceRedactionPolicy::RuntimeInternalOnly,
audit_correlation_id: "audit-1".to_string(),
profile_source_graph: Some(source_graph),
});
let rendered = serde_json::to_string(&bundle.summary()).unwrap();
assert!(rendered.contains("profile_source_archive"));
for forbidden in [
"nonce-not-for-browser",
"profile-source-archive:test",
"sha256:0123456789abcdef",
"runtime-1",
"worker-1",
] {
assert!(
!rendered.contains(forbidden),
"leaked {forbidden}: {rendered}"
);
}
}
#[test]
fn rejects_unsafe_bundle_ids_and_refs() {
for id in ["bundle/1", "bundle?x", "bundle&x", "bundle#x", " bundle"] {
+2
View File
@@ -850,6 +850,7 @@ mod tests {
}],
declarations: Vec::new(),
profile_source_archive: None,
profile_source_archive_handle: None,
}
.with_computed_digest()
}
@@ -1137,6 +1138,7 @@ mod ws_tests {
}],
declarations: Vec::new(),
profile_source_archive: None,
profile_source_archive_handle: None,
}
.with_computed_digest()
}
+1
View File
@@ -20,6 +20,7 @@ pub mod interaction;
pub mod management;
pub mod observation;
pub mod profile_archive;
pub mod resource;
mod runtime;
pub mod worker_backend;
pub mod working_directory;
+21
View File
@@ -78,6 +78,14 @@ fn build_runtime(config: &ProcessConfig) -> Result<Runtime, ProcessError> {
if let Some(runtime_base_dir) = config.worker_runtime_base_dir.clone() {
factory = factory.with_runtime_base_dir(runtime_base_dir);
}
if let Some(endpoint) = config.backend_resource_endpoint.clone() {
factory = factory.with_resource_client(Arc::new(
worker_runtime::resource::HttpBackendResourceClient::new(
endpoint,
config.backend_resource_token.clone(),
),
));
}
if let Some(profile) = config.profile.clone() {
factory = factory.with_profile(profile);
}
@@ -173,6 +181,13 @@ where
config.worker_runtime_base_dir =
Some(PathBuf::from(take_value(&flag, inline_value, &mut args)?));
}
"--backend-resource-endpoint" => {
config.backend_resource_endpoint =
Some(take_value(&flag, inline_value, &mut args)?);
}
"--backend-resource-token" => {
config.backend_resource_token = Some(take_value(&flag, inline_value, &mut args)?);
}
"--profile" => {
config.profile = Some(take_value(&flag, inline_value, &mut args)?);
}
@@ -317,6 +332,8 @@ struct ProcessConfig {
worker_store_dir: Option<PathBuf>,
worker_metadata_dir: Option<PathBuf>,
worker_runtime_base_dir: Option<PathBuf>,
backend_resource_endpoint: Option<String>,
backend_resource_token: Option<String>,
profile: Option<String>,
}
@@ -330,6 +347,8 @@ impl ProcessConfig {
worker_store_dir: None,
worker_metadata_dir: None,
worker_runtime_base_dir: None,
backend_resource_endpoint: None,
backend_resource_token: None,
profile: None,
})
}
@@ -405,6 +424,8 @@ Options:\n\
--worker-store-dir <PATH> Worker session store directory\n\
--worker-metadata-dir <PATH> Worker metadata directory\n\
--worker-runtime-base-dir <PATH> Worker controller runtime directory\n\
--backend-resource-endpoint <URL> Internal Backend resource fetch endpoint for resource handles\n\
--backend-resource-token <TOKEN> Optional bearer token for the Backend resource fetch endpoint\n\
--store <memory|fs> Runtime catalog store selection (default: memory)\n\
--fs-root <PATH> Runtime catalog filesystem store root\n\
--local-token <TOKEN> Minimal local bearer token placeholder\n\
+1 -1
View File
@@ -472,7 +472,7 @@ fn validate_archive_path(path: &str) -> Result<(), ProfileArchiveError> {
Ok(())
}
fn sha256_hex(data: &[u8]) -> String {
pub fn sha256_hex(data: &[u8]) -> String {
let digest = Sha256::digest(data);
let mut out = String::from("sha256:");
for byte in digest.as_slice() {
+256
View File
@@ -0,0 +1,256 @@
use crate::identity::{RuntimeId, WorkerId};
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Mutex;
pub const PROFILE_SOURCE_ARCHIVE_CONTENT_TYPE: &str =
"application/vnd.yoi.profile-source-archive+tar";
pub const DEFAULT_PROFILE_SOURCE_ARCHIVE_MAX_BYTES: u64 = 2 * 1024 * 1024;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum BackendResourceKind {
ProfileSourceArchive,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum BackendResourceOperation {
FetchArchive,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BackendResourceHandle {
pub kind: BackendResourceKind,
pub workspace_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scope_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub runtime_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_id: Option<String>,
pub resource_id: String,
pub digest: String,
pub operation: BackendResourceOperation,
pub expires_at_unix_seconds: i64,
pub nonce: String,
pub revision: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub generation: Option<u64>,
pub max_bytes: u64,
pub content_type: String,
pub redaction: ResourceRedactionPolicy,
pub audit_correlation_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profile_source_graph: Option<crate::profile_archive::ProfileSourceGraphSummary>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ResourceRedactionPolicy {
RuntimeInternalOnly,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BackendResourceFetchRequest {
pub handle: BackendResourceHandle,
pub runtime_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_id: Option<String>,
pub audit_correlation_id: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BackendResourceFetchResponse {
pub kind: BackendResourceKind,
pub resource_id: String,
pub digest: String,
pub content_type: String,
pub bytes: Vec<u8>,
pub audit_correlation_id: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)]
#[serde(tag = "code", rename_all = "snake_case")]
pub enum BackendResourceError {
#[error("backend resource handle is expired")]
Expired,
#[error("backend resource handle is unauthorized: {message}")]
Unauthorized { message: String },
#[error("backend resource kind is unsupported")]
UnsupportedKind,
#[error("backend resource is missing")]
MissingResource,
#[error("backend resource digest mismatch: expected {expected}, got {actual}")]
DigestMismatch { expected: String, actual: String },
#[error("backend resource response is oversized: limit {max_bytes}, actual {actual_bytes}")]
Oversized { max_bytes: u64, actual_bytes: u64 },
#[error("backend resource content type mismatch: expected {expected}, got {actual}")]
ContentTypeMismatch { expected: String, actual: String },
#[error("backend resource transport failed: {message}")]
Transport { message: String },
#[error("backend resource response is invalid: {message}")]
InvalidResponse { message: String },
}
#[async_trait]
pub trait BackendResourceClient: Send + Sync + 'static {
async fn fetch_resource(
&self,
request: BackendResourceFetchRequest,
) -> Result<BackendResourceFetchResponse, BackendResourceError>;
}
#[cfg(feature = "http-server")]
#[derive(Clone, Debug)]
pub struct HttpBackendResourceClient {
endpoint: String,
bearer_token: Option<String>,
client: reqwest::Client,
}
#[cfg(feature = "http-server")]
impl HttpBackendResourceClient {
pub fn new(endpoint: impl Into<String>, bearer_token: Option<String>) -> Self {
Self {
endpoint: endpoint.into(),
bearer_token,
client: reqwest::Client::new(),
}
}
}
#[cfg(feature = "http-server")]
#[async_trait]
impl BackendResourceClient for HttpBackendResourceClient {
async fn fetch_resource(
&self,
request: BackendResourceFetchRequest,
) -> Result<BackendResourceFetchResponse, BackendResourceError> {
let builder = self.client.post(&self.endpoint).json(&request);
let builder = if let Some(token) = self.bearer_token.as_deref() {
builder.bearer_auth(token)
} else {
builder
};
let response = builder
.send()
.await
.map_err(|err| BackendResourceError::Transport {
message: err.to_string(),
})?;
if response.status().is_success() {
response
.json::<BackendResourceFetchResponse>()
.await
.map_err(|err| BackendResourceError::InvalidResponse {
message: err.to_string(),
})
} else {
let status = response.status();
match response.json::<BackendResourceError>().await {
Ok(error) => Err(error),
Err(err) => Err(BackendResourceError::Transport {
message: format!("backend resource fetch failed with HTTP {status}: {err}"),
}),
}
}
}
}
#[derive(Default, Debug)]
pub struct ProfileSourceArchiveCache {
archives: Mutex<HashMap<String, ProfileSourceArchive>>,
}
impl ProfileSourceArchiveCache {
pub fn get(&self, digest: &str) -> Option<ProfileSourceArchive> {
self.archives.lock().ok()?.get(digest).cloned()
}
pub fn insert(&self, archive: ProfileSourceArchive) {
if let Ok(mut archives) = self.archives.lock() {
archives.insert(archive.reference.digest.clone(), archive);
}
}
}
pub fn build_profile_source_archive_fetch_request(
handle: BackendResourceHandle,
runtime_id: &RuntimeId,
worker_id: Option<&WorkerId>,
) -> BackendResourceFetchRequest {
let audit_correlation_id = handle.audit_correlation_id.clone();
BackendResourceFetchRequest {
handle,
runtime_id: runtime_id.as_str().to_string(),
worker_id: worker_id.map(|id| id.as_str().to_string()),
audit_correlation_id,
}
}
pub fn profile_source_archive_from_response(
handle: &BackendResourceHandle,
response: BackendResourceFetchResponse,
) -> Result<ProfileSourceArchive, BackendResourceError> {
if handle.kind != BackendResourceKind::ProfileSourceArchive
|| response.kind != BackendResourceKind::ProfileSourceArchive
{
return Err(BackendResourceError::UnsupportedKind);
}
if handle.operation != BackendResourceOperation::FetchArchive {
return Err(BackendResourceError::Unauthorized {
message: "resource handle operation is not fetch_archive".to_string(),
});
}
if response.content_type != handle.content_type {
return Err(BackendResourceError::ContentTypeMismatch {
expected: handle.content_type.clone(),
actual: response.content_type,
});
}
let actual_bytes = response.bytes.len() as u64;
if actual_bytes > handle.max_bytes {
return Err(BackendResourceError::Oversized {
max_bytes: handle.max_bytes,
actual_bytes,
});
}
let actual_digest = sha256_hex(&response.bytes);
if actual_digest != handle.digest || response.digest != handle.digest {
return Err(BackendResourceError::DigestMismatch {
expected: handle.digest.clone(),
actual: if response.digest != handle.digest {
response.digest
} else {
actual_digest
},
});
}
Ok(ProfileSourceArchive {
reference: ProfileSourceArchiveRef {
id: handle.resource_id.clone(),
digest: handle.digest.clone(),
size_bytes: actual_bytes,
source_graph: handle.profile_source_graph.clone().ok_or_else(|| {
BackendResourceError::InvalidResponse {
message: "profile source archive handle omitted source graph summary"
.to_string(),
}
})?,
},
content: response.bytes,
})
}
pub fn validate_resource_handle_text(label: &str, value: &str) -> Result<(), String> {
if value.trim().is_empty() {
return Err(format!("{label} must not be empty"));
}
if value.len() > 256 || value.contains('\0') || value.contains('\n') || value.contains('\r') {
return Err(format!("{label} contains unsupported boundary text"));
}
Ok(())
}
+1
View File
@@ -1628,6 +1628,7 @@ mod tests {
reference: "capability:read".to_string(),
}],
profile_source_archive: None,
profile_source_archive_handle: None,
}
.with_computed_digest()
}
+68 -9
View File
@@ -20,6 +20,10 @@ use crate::execution::{
WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
};
use crate::interaction::{WorkerInput, WorkerInputKind};
use crate::resource::{
BackendResourceClient, BackendResourceError, ProfileSourceArchiveCache,
build_profile_source_archive_fetch_request, profile_source_archive_from_response,
};
use crate::working_directory::{WorkingDirectoryBinding, WorkingDirectoryMaterializer};
use async_trait::async_trait;
use manifest::paths;
@@ -46,7 +50,7 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static {
/// Production factory that resolves a normal Worker profile and spawns it under
/// `WorkerController`.
#[derive(Debug, Clone)]
#[derive(Clone)]
pub struct ProfileRuntimeWorkerFactory {
profile_base_dir: PathBuf,
cwd: PathBuf,
@@ -54,6 +58,8 @@ pub struct ProfileRuntimeWorkerFactory {
worker_metadata_dir: Option<PathBuf>,
profile: Option<String>,
runtime_base_dir: Option<PathBuf>,
resource_client: Option<Arc<dyn BackendResourceClient>>,
profile_archive_cache: Arc<ProfileSourceArchiveCache>,
}
impl ProfileRuntimeWorkerFactory {
@@ -66,6 +72,8 @@ impl ProfileRuntimeWorkerFactory {
worker_metadata_dir: None,
profile: None,
runtime_base_dir: None,
resource_client: None,
profile_archive_cache: Arc::new(ProfileSourceArchiveCache::default()),
}
}
@@ -96,6 +104,11 @@ impl ProfileRuntimeWorkerFactory {
self
}
pub fn with_resource_client(mut self, resource_client: Arc<dyn BackendResourceClient>) -> Self {
self.resource_client = Some(resource_client);
self
}
fn store_dir(&self) -> Result<PathBuf, String> {
self.store_dir
.clone()
@@ -152,6 +165,56 @@ impl ProfileRuntimeWorkerFactory {
}
Self::runtime_profile_value(&request.request.profile)
}
async fn resolve_profile_source_archive(
&self,
bundle: &crate::config_bundle::ConfigBundle,
request: &WorkerExecutionSpawnRequest,
) -> Result<crate::profile_archive::VerifiedProfileSourceArchive, String> {
if let Some(archive) = verified_profile_source_archive(bundle)
.map_err(|err| format!("failed to verify profile source archive: {err}"))?
{
return Ok(archive);
}
let handle = bundle
.profile_source_archive_handle
.clone()
.ok_or_else(|| {
format!(
"config bundle {} does not contain a ProfileSourceArchive resource handle",
bundle.metadata.id
)
})?;
if let Some(cached) = self.profile_archive_cache.get(&handle.digest) {
return cached
.verify()
.map_err(|err| format!("failed to verify cached profile source archive: {err}"));
}
let client = self.resource_client.as_ref().ok_or_else(|| {
format!(
"config bundle {} requires a Backend resource client for profile source archive fetch",
bundle.metadata.id
)
})?;
let fetch_request = build_profile_source_archive_fetch_request(
handle.clone(),
&request.worker_ref.runtime_id,
Some(&request.worker_ref.worker_id),
);
let response = client
.fetch_resource(fetch_request)
.await
.map_err(format_backend_resource_error)?;
let archive = profile_source_archive_from_response(&handle, response)
.map_err(format_backend_resource_error)?;
self.profile_archive_cache.insert(archive.clone());
archive
.verify()
.map_err(|err| format!("failed to verify fetched profile source archive: {err}"))
}
}
fn format_backend_resource_error(error: BackendResourceError) -> String {
format!("backend resource fetch failed: {error}")
}
#[async_trait]
@@ -174,14 +237,9 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
.unwrap_or_else(|| self.cwd.clone());
let (mut manifest, loader) = if let Some(bundle) = request.config_bundle.as_ref() {
let selector = profile.as_deref().unwrap_or("builtin:default");
let archive = verified_profile_source_archive(bundle)
.map_err(|err| format!("failed to verify profile source archive: {err}"))?
.ok_or_else(|| {
format!(
"config bundle {} does not contain a ProfileSourceArchive",
bundle.metadata.id
)
})?;
let archive = self
.resolve_profile_source_archive(bundle, &request)
.await?;
let manifest = archive
.resolve_profile(selector, &worker_root, &worker_name)
.map_err(|err| format!("failed to resolve profile source archive: {err}"))?;
@@ -791,6 +849,7 @@ mod tests {
}],
declarations: Vec::new(),
profile_source_archive: None,
profile_source_archive_handle: None,
}
.with_computed_digest()
}