feat: add decodal profile archives
This commit is contained in:
@@ -0,0 +1,544 @@
|
||||
use decodal::{Engine, LoadedSource, SourceLoader};
|
||||
use manifest::{ProfileSource, WorkerManifest, resolve_profile_artifact_value};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::{Cursor, Read, Write};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use tar::{Archive, Builder, Header};
|
||||
|
||||
const MANIFEST_PATH: &str = "manifest.json";
|
||||
const MAX_SOURCES: usize = 64;
|
||||
const MAX_SOURCE_BYTES: u64 = 256 * 1024;
|
||||
const MAX_TOTAL_BYTES: u64 = 1024 * 1024;
|
||||
const MAX_PATH_DEPTH: usize = 8;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ProfileArchiveError {
|
||||
#[error("profile source archive has unsupported schema version {0}")]
|
||||
UnsupportedSchema(u32),
|
||||
#[error("profile source archive path must be relative and confined: {0}")]
|
||||
UnsafePath(String),
|
||||
#[error("profile source archive contains unsupported entry type at {0}")]
|
||||
UnsupportedEntry(String),
|
||||
#[error("profile source archive limit exceeded: {0}")]
|
||||
LimitExceeded(&'static str),
|
||||
#[error("profile source archive missing manifest.json")]
|
||||
MissingManifest,
|
||||
#[error("profile source archive missing source {0}")]
|
||||
MissingSource(String),
|
||||
#[error(
|
||||
"profile source archive source digest mismatch for {path}: expected {expected}, got {actual}"
|
||||
)]
|
||||
SourceDigestMismatch {
|
||||
path: String,
|
||||
expected: String,
|
||||
actual: String,
|
||||
},
|
||||
#[error("profile source archive digest mismatch for {id}: expected {expected}, got {actual}")]
|
||||
ArchiveDigestMismatch {
|
||||
id: String,
|
||||
expected: String,
|
||||
actual: String,
|
||||
},
|
||||
#[error("profile source archive has unsupported source kind {kind} at {path}")]
|
||||
UnsupportedSource { path: String, kind: String },
|
||||
#[error("profile source archive has no entrypoint for selector {0}")]
|
||||
MissingEntrypoint(String),
|
||||
#[error("profile source archive import is not declared in manifest import map: {0}")]
|
||||
ImportNotDeclared(String),
|
||||
#[error("failed to read profile source archive: {0}")]
|
||||
Io(String),
|
||||
#[error("failed to parse profile source archive manifest: {0}")]
|
||||
Json(String),
|
||||
#[error("failed to evaluate Decodal profile source {path}: {message}")]
|
||||
Decodal { path: String, message: String },
|
||||
#[error("failed to resolve archive profile artifact {profile_source}: {message}")]
|
||||
Profile {
|
||||
profile_source: String,
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ProfileSourceArchiveRef {
|
||||
pub id: String,
|
||||
pub digest: String,
|
||||
pub size_bytes: u64,
|
||||
pub source_graph: ProfileSourceGraphSummary,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ProfileSourceGraphSummary {
|
||||
pub source_count: usize,
|
||||
pub total_source_bytes: u64,
|
||||
pub entrypoints: BTreeMap<String, String>,
|
||||
pub import_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ProfileSourceArchive {
|
||||
pub reference: ProfileSourceArchiveRef,
|
||||
pub content: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ProfileSourceArchiveManifest {
|
||||
pub schema_version: u32,
|
||||
pub id: String,
|
||||
pub entrypoints: BTreeMap<String, String>,
|
||||
pub imports: BTreeMap<String, String>,
|
||||
pub sources: Vec<ProfileSourceArchiveSource>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ProfileSourceArchiveSource {
|
||||
pub path: String,
|
||||
pub kind: String,
|
||||
pub digest: String,
|
||||
pub size_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ProfileSourceArchiveInput {
|
||||
pub id: String,
|
||||
pub entrypoints: BTreeMap<String, String>,
|
||||
pub imports: BTreeMap<String, String>,
|
||||
pub sources: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl ProfileSourceArchive {
|
||||
pub fn build(input: ProfileSourceArchiveInput) -> Result<Self, ProfileArchiveError> {
|
||||
if input.sources.len() > MAX_SOURCES {
|
||||
return Err(ProfileArchiveError::LimitExceeded("source count"));
|
||||
}
|
||||
let mut total_source_bytes = 0u64;
|
||||
let mut source_meta = Vec::new();
|
||||
for (path, source) in &input.sources {
|
||||
validate_archive_path(path)?;
|
||||
let size = source.as_bytes().len() as u64;
|
||||
if size > MAX_SOURCE_BYTES {
|
||||
return Err(ProfileArchiveError::LimitExceeded("source bytes"));
|
||||
}
|
||||
total_source_bytes = total_source_bytes
|
||||
.checked_add(size)
|
||||
.ok_or(ProfileArchiveError::LimitExceeded("total source bytes"))?;
|
||||
if total_source_bytes > MAX_TOTAL_BYTES {
|
||||
return Err(ProfileArchiveError::LimitExceeded("total source bytes"));
|
||||
}
|
||||
source_meta.push(ProfileSourceArchiveSource {
|
||||
path: path.clone(),
|
||||
kind: "decodal".to_string(),
|
||||
digest: sha256_hex(source.as_bytes()),
|
||||
size_bytes: size,
|
||||
});
|
||||
}
|
||||
for path in input.entrypoints.values().chain(input.imports.values()) {
|
||||
validate_archive_path(path)?;
|
||||
if !input.sources.contains_key(path) {
|
||||
return Err(ProfileArchiveError::MissingSource(path.clone()));
|
||||
}
|
||||
}
|
||||
let manifest = ProfileSourceArchiveManifest {
|
||||
schema_version: 1,
|
||||
id: input.id.clone(),
|
||||
entrypoints: input.entrypoints.clone(),
|
||||
imports: input.imports.clone(),
|
||||
sources: source_meta,
|
||||
};
|
||||
let manifest_json = serde_json::to_vec_pretty(&manifest)
|
||||
.map_err(|err| ProfileArchiveError::Json(err.to_string()))?;
|
||||
let mut bytes = Vec::new();
|
||||
{
|
||||
let mut builder = Builder::new(&mut bytes);
|
||||
append_bytes(&mut builder, MANIFEST_PATH, &manifest_json)?;
|
||||
for (path, source) in &input.sources {
|
||||
append_bytes(&mut builder, path, source.as_bytes())?;
|
||||
}
|
||||
builder
|
||||
.finish()
|
||||
.map_err(|err| ProfileArchiveError::Io(err.to_string()))?;
|
||||
}
|
||||
let digest = sha256_hex(&bytes);
|
||||
let reference = ProfileSourceArchiveRef {
|
||||
id: input.id,
|
||||
digest,
|
||||
size_bytes: bytes.len() as u64,
|
||||
source_graph: ProfileSourceGraphSummary {
|
||||
source_count: input.sources.len(),
|
||||
total_source_bytes,
|
||||
entrypoints: input.entrypoints,
|
||||
import_count: manifest.imports.len(),
|
||||
},
|
||||
};
|
||||
Ok(Self {
|
||||
reference,
|
||||
content: bytes,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn verify(&self) -> Result<VerifiedProfileSourceArchive, ProfileArchiveError> {
|
||||
let digest = sha256_hex(&self.content);
|
||||
if digest != self.reference.digest {
|
||||
return Err(ProfileArchiveError::ArchiveDigestMismatch {
|
||||
id: self.reference.id.clone(),
|
||||
expected: self.reference.digest.clone(),
|
||||
actual: digest,
|
||||
});
|
||||
}
|
||||
VerifiedProfileSourceArchive::from_bytes(self.reference.clone(), &self.content)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VerifiedProfileSourceArchive {
|
||||
reference: ProfileSourceArchiveRef,
|
||||
manifest: ProfileSourceArchiveManifest,
|
||||
sources: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl VerifiedProfileSourceArchive {
|
||||
pub fn from_bytes(
|
||||
reference: ProfileSourceArchiveRef,
|
||||
bytes: &[u8],
|
||||
) -> Result<Self, ProfileArchiveError> {
|
||||
if bytes.len() as u64 > MAX_TOTAL_BYTES + 64 * 1024 {
|
||||
return Err(ProfileArchiveError::LimitExceeded("archive bytes"));
|
||||
}
|
||||
let mut archive = Archive::new(Cursor::new(bytes));
|
||||
let mut manifest: Option<ProfileSourceArchiveManifest> = None;
|
||||
let mut entries = BTreeMap::new();
|
||||
for entry in archive
|
||||
.entries()
|
||||
.map_err(|err| ProfileArchiveError::Io(err.to_string()))?
|
||||
{
|
||||
let mut entry = entry.map_err(|err| ProfileArchiveError::Io(err.to_string()))?;
|
||||
let path = entry
|
||||
.path()
|
||||
.map_err(|err| ProfileArchiveError::Io(err.to_string()))?
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
validate_archive_path(&path)?;
|
||||
if !entry.header().entry_type().is_file() {
|
||||
return Err(ProfileArchiveError::UnsupportedEntry(path));
|
||||
}
|
||||
let size = entry.size();
|
||||
if size > MAX_SOURCE_BYTES && path != MANIFEST_PATH {
|
||||
return Err(ProfileArchiveError::LimitExceeded("source bytes"));
|
||||
}
|
||||
let mut data = Vec::new();
|
||||
entry
|
||||
.read_to_end(&mut data)
|
||||
.map_err(|err| ProfileArchiveError::Io(err.to_string()))?;
|
||||
if path == MANIFEST_PATH {
|
||||
manifest = Some(
|
||||
serde_json::from_slice(&data)
|
||||
.map_err(|err| ProfileArchiveError::Json(err.to_string()))?,
|
||||
);
|
||||
} else {
|
||||
let source = String::from_utf8(data)
|
||||
.map_err(|err| ProfileArchiveError::Io(err.to_string()))?;
|
||||
entries.insert(path, source);
|
||||
}
|
||||
if entries.len() > MAX_SOURCES {
|
||||
return Err(ProfileArchiveError::LimitExceeded("source count"));
|
||||
}
|
||||
}
|
||||
let manifest = manifest.ok_or(ProfileArchiveError::MissingManifest)?;
|
||||
if manifest.schema_version != 1 {
|
||||
return Err(ProfileArchiveError::UnsupportedSchema(
|
||||
manifest.schema_version,
|
||||
));
|
||||
}
|
||||
if manifest.id != reference.id {
|
||||
return Err(ProfileArchiveError::ArchiveDigestMismatch {
|
||||
id: reference.id,
|
||||
expected: "matching archive id".to_string(),
|
||||
actual: manifest.id,
|
||||
});
|
||||
}
|
||||
let mut total = 0u64;
|
||||
for source in &manifest.sources {
|
||||
validate_archive_path(&source.path)?;
|
||||
if source.kind != "decodal" {
|
||||
return Err(ProfileArchiveError::UnsupportedSource {
|
||||
path: source.path.clone(),
|
||||
kind: source.kind.clone(),
|
||||
});
|
||||
}
|
||||
let content = entries
|
||||
.get(&source.path)
|
||||
.ok_or_else(|| ProfileArchiveError::MissingSource(source.path.clone()))?;
|
||||
let digest = sha256_hex(content.as_bytes());
|
||||
if digest != source.digest {
|
||||
return Err(ProfileArchiveError::SourceDigestMismatch {
|
||||
path: source.path.clone(),
|
||||
expected: source.digest.clone(),
|
||||
actual: digest,
|
||||
});
|
||||
}
|
||||
let size = content.as_bytes().len() as u64;
|
||||
if size != source.size_bytes {
|
||||
return Err(ProfileArchiveError::LimitExceeded(
|
||||
"source byte metadata mismatch",
|
||||
));
|
||||
}
|
||||
total += size;
|
||||
}
|
||||
if total != reference.source_graph.total_source_bytes
|
||||
|| manifest.sources.len() != reference.source_graph.source_count
|
||||
{
|
||||
return Err(ProfileArchiveError::LimitExceeded(
|
||||
"source graph metadata mismatch",
|
||||
));
|
||||
}
|
||||
for path in manifest
|
||||
.entrypoints
|
||||
.values()
|
||||
.chain(manifest.imports.values())
|
||||
{
|
||||
validate_archive_path(path)?;
|
||||
if !entries.contains_key(path) {
|
||||
return Err(ProfileArchiveError::MissingSource(path.clone()));
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
reference,
|
||||
manifest,
|
||||
sources: entries,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn reference(&self) -> &ProfileSourceArchiveRef {
|
||||
&self.reference
|
||||
}
|
||||
|
||||
pub fn resolve_profile(
|
||||
&self,
|
||||
selector: &str,
|
||||
worker_root: &Path,
|
||||
worker_name: &str,
|
||||
) -> Result<WorkerManifest, ProfileArchiveError> {
|
||||
let path = self
|
||||
.manifest
|
||||
.entrypoints
|
||||
.get(selector)
|
||||
.or_else(|| self.manifest.entrypoints.get("default"))
|
||||
.ok_or_else(|| ProfileArchiveError::MissingEntrypoint(selector.to_string()))?;
|
||||
let mut engine = Engine::new(ArchiveSourceLoader { archive: self });
|
||||
let source = self
|
||||
.sources
|
||||
.get(path)
|
||||
.ok_or_else(|| ProfileArchiveError::MissingSource(path.clone()))?;
|
||||
let module = engine.add_root_source(path, path, source).map_err(|err| {
|
||||
ProfileArchiveError::Decodal {
|
||||
path: path.clone(),
|
||||
message: format!("{err:?}"),
|
||||
}
|
||||
})?;
|
||||
let value = engine
|
||||
.eval_module(module)
|
||||
.map_err(|err| ProfileArchiveError::Decodal {
|
||||
path: path.clone(),
|
||||
message: format!("{err:?}"),
|
||||
})?;
|
||||
let data = engine
|
||||
.materialize(&value)
|
||||
.map_err(|err| ProfileArchiveError::Decodal {
|
||||
path: path.clone(),
|
||||
message: format!("{err:?}"),
|
||||
})?;
|
||||
let json = decodal_data_to_json(&data);
|
||||
let resolved = resolve_profile_artifact_value(
|
||||
json,
|
||||
ProfileSource::Archive {
|
||||
archive_id: self.reference.id.clone(),
|
||||
source: path.clone(),
|
||||
},
|
||||
worker_root,
|
||||
worker_name,
|
||||
)
|
||||
.map_err(|err| ProfileArchiveError::Profile {
|
||||
profile_source: path.clone(),
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
Ok(resolved.manifest)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ArchiveSourceLoader<'a> {
|
||||
archive: &'a VerifiedProfileSourceArchive,
|
||||
}
|
||||
|
||||
impl<'a> ArchiveSourceLoader<'a> {
|
||||
pub fn new(archive: &'a VerifiedProfileSourceArchive) -> Self {
|
||||
Self { archive }
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceLoader for ArchiveSourceLoader<'_> {
|
||||
fn load(
|
||||
&mut self,
|
||||
_current_key: Option<&str>,
|
||||
specifier: &str,
|
||||
) -> decodal::Result<LoadedSource> {
|
||||
let path = self
|
||||
.archive
|
||||
.manifest
|
||||
.imports
|
||||
.get(specifier)
|
||||
.ok_or_else(|| {
|
||||
decodal::Diagnostic::new(
|
||||
decodal::DiagnosticKind::Import,
|
||||
decodal::Span::default(),
|
||||
format!("archive import not declared: {specifier}"),
|
||||
)
|
||||
})?;
|
||||
let source = self.archive.sources.get(path).ok_or_else(|| {
|
||||
decodal::Diagnostic::new(
|
||||
decodal::DiagnosticKind::Import,
|
||||
decodal::Span::default(),
|
||||
format!("archive source missing: {path}"),
|
||||
)
|
||||
})?;
|
||||
Ok(LoadedSource {
|
||||
key: path.clone(),
|
||||
name: path.clone(),
|
||||
source: source.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn decodal_data_to_json(data: &decodal::Data) -> serde_json::Value {
|
||||
match data {
|
||||
decodal::Data::Bool(value) => serde_json::Value::Bool(*value),
|
||||
decodal::Data::Int(value) => serde_json::Value::Number(serde_json::Number::from(*value)),
|
||||
decodal::Data::Float(value) => serde_json::Number::from_f64(*value)
|
||||
.map(serde_json::Value::Number)
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
decodal::Data::String(value) => serde_json::Value::String(value.clone()),
|
||||
decodal::Data::Array(values) => {
|
||||
serde_json::Value::Array(values.iter().map(decodal_data_to_json).collect())
|
||||
}
|
||||
decodal::Data::Object(fields) => serde_json::Value::Object(
|
||||
fields
|
||||
.iter()
|
||||
.map(|field| (field.name.clone(), decodal_data_to_json(&field.value)))
|
||||
.collect(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn append_bytes<W: Write>(
|
||||
builder: &mut Builder<W>,
|
||||
path: &str,
|
||||
data: &[u8],
|
||||
) -> Result<(), ProfileArchiveError> {
|
||||
validate_archive_path(path)?;
|
||||
let mut header = Header::new_gnu();
|
||||
header.set_size(data.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, path, Cursor::new(data))
|
||||
.map_err(|err| ProfileArchiveError::Io(err.to_string()))
|
||||
}
|
||||
|
||||
fn validate_archive_path(path: &str) -> Result<(), ProfileArchiveError> {
|
||||
if path.is_empty() || path == MANIFEST_PATH {
|
||||
return if path == MANIFEST_PATH {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(ProfileArchiveError::UnsafePath(path.to_string()))
|
||||
};
|
||||
}
|
||||
let path_ref = Path::new(path);
|
||||
if path_ref.is_absolute() {
|
||||
return Err(ProfileArchiveError::UnsafePath(path.to_string()));
|
||||
}
|
||||
let mut depth = 0usize;
|
||||
for component in path_ref.components() {
|
||||
match component {
|
||||
Component::Normal(_) => depth += 1,
|
||||
_ => return Err(ProfileArchiveError::UnsafePath(path.to_string())),
|
||||
}
|
||||
}
|
||||
if depth == 0 || depth > MAX_PATH_DEPTH {
|
||||
return Err(ProfileArchiveError::UnsafePath(path.to_string()));
|
||||
}
|
||||
let normalized = PathBuf::from(path).to_string_lossy().replace('\\', "/");
|
||||
if normalized != path {
|
||||
return Err(ProfileArchiveError::UnsafePath(path.to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
let mut out = String::from("sha256:");
|
||||
for byte in digest.as_slice() {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(out, "{byte:02x}");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample_archive() -> ProfileSourceArchive {
|
||||
let mut entrypoints = BTreeMap::new();
|
||||
entrypoints.insert("default".to_string(), "profiles/default.dcdl".to_string());
|
||||
let mut sources = BTreeMap::new();
|
||||
sources.insert(
|
||||
"profiles/default.dcdl".to_string(),
|
||||
r#"{
|
||||
slug = "default";
|
||||
description = "Default";
|
||||
scope = "workspace_read";
|
||||
}"#
|
||||
.to_string(),
|
||||
);
|
||||
ProfileSourceArchive::build(ProfileSourceArchiveInput {
|
||||
id: "builtin".to_string(),
|
||||
entrypoints,
|
||||
imports: BTreeMap::new(),
|
||||
sources,
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_and_verify_archive() {
|
||||
let archive = sample_archive();
|
||||
let verified = archive.verify().unwrap();
|
||||
assert_eq!(verified.reference().source_graph.source_count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_digest_mismatch() {
|
||||
let mut archive = sample_archive();
|
||||
archive.reference.digest = "sha256:bad".to_string();
|
||||
assert!(archive.verify().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_profile_from_archive_without_filesystem_loader() {
|
||||
let archive = sample_archive();
|
||||
let verified = archive.verify().unwrap();
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let manifest = verified
|
||||
.resolve_profile("default", root.path(), "archive-worker")
|
||||
.unwrap();
|
||||
assert_eq!(manifest.worker.name, "archive-worker");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_loader_rejects_undeclared_import() {
|
||||
let archive = sample_archive();
|
||||
let verified = archive.verify().unwrap();
|
||||
let mut loader = ArchiveSourceLoader::new(&verified);
|
||||
assert!(loader.load(None, "missing").is_err());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user