diff --git a/Cargo.lock b/Cargo.lock index 2b4ab9aa..aa6c75cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,6 +599,7 @@ dependencies = [ "pretty_assertions", "serde", "serde_json", + "serde_yaml", "sha2 0.11.0", "thiserror 2.0.18", "ts-rs", diff --git a/crates/config-source/Cargo.toml b/crates/config-source/Cargo.toml index d86dc5d5..ead1f4ce 100644 --- a/crates/config-source/Cargo.toml +++ b/crates/config-source/Cargo.toml @@ -11,6 +11,7 @@ decodal-language-service.workspace = true decodal-language-tools.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true +serde_yaml.workspace = true sha2.workspace = true thiserror.workspace = true ts-rs = "12.0.1" diff --git a/crates/config-source/src/lib.rs b/crates/config-source/src/lib.rs index 4a10f183..084da87e 100644 --- a/crates/config-source/src/lib.rs +++ b/crates/config-source/src/lib.rs @@ -3,7 +3,7 @@ use std::fmt; use decodal::{ Data, Diagnostic, DiagnosticKind, Engine, HostEnvironment, ImportCandidate, ImportLoader, - LoadedImport, Span, + LoadedImport, Span, Value, }; use decodal_language_service::{CompletionResult, LanguageService}; use serde::{Deserialize, Serialize}; @@ -91,6 +91,151 @@ impl ConfigContentType { } } +/// Stable value projection used when a virtual config source imports Markdown. +/// +/// Frontmatter delimiters are transport syntax and are intentionally absent from +/// `content`; unknown frontmatter keys stay in `frontmatter` without a +/// domain-specific parser interpreting them. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MarkdownDocumentProjection { + pub frontmatter: serde_json::Map, + pub content: String, +} + +/// Parse one Markdown source into the common virtual-config import shape. +/// +/// Files without a leading YAML frontmatter delimiter produce an empty +/// frontmatter object and preserve the complete file as `content`. +pub fn project_markdown_document(source: &str) -> Result { + let Some(after_opening) = source + .strip_prefix("---\n") + .or_else(|| source.strip_prefix("---\r\n")) + else { + return Ok(MarkdownDocumentProjection { + frontmatter: serde_json::Map::new(), + content: source.to_string(), + }); + }; + + let mut frontmatter_end = None; + let mut offset = 0usize; + for line_with_ending in after_opening.split_inclusive('\n') { + let line = line_with_ending.trim_end_matches(['\r', '\n']); + if line == "---" { + frontmatter_end = Some((offset, offset + line_with_ending.len())); + break; + } + offset += line_with_ending.len(); + } + if frontmatter_end.is_none() && after_opening.ends_with("---") { + let start = after_opening.len() - 3; + if start == 0 || after_opening[..start].ends_with('\n') { + frontmatter_end = Some((start, after_opening.len())); + } + } + let Some((frontmatter_end, content_start)) = frontmatter_end else { + return Err("opening YAML frontmatter delimiter has no closing delimiter".to_string()); + }; + + let frontmatter_source = &after_opening[..frontmatter_end]; + let frontmatter = if frontmatter_source.trim().is_empty() { + serde_json::Map::new() + } else { + let yaml = serde_yaml::from_str::(frontmatter_source) + .map_err(|error| format!("invalid YAML frontmatter: {error}"))?; + let value = yaml_to_json(yaml)?; + value + .as_object() + .cloned() + .ok_or_else(|| "YAML frontmatter must be a mapping".to_string())? + }; + + Ok(MarkdownDocumentProjection { + frontmatter, + content: after_opening[content_start..].to_string(), + }) +} + +fn yaml_to_json(value: serde_yaml::Value) -> Result { + match value { + serde_yaml::Value::Null => Ok(serde_json::Value::Null), + serde_yaml::Value::Bool(value) => Ok(serde_json::Value::Bool(value)), + serde_yaml::Value::Number(value) => { + if let Some(value) = value.as_i64() { + Ok(serde_json::Value::Number(value.into())) + } else if let Some(value) = value.as_u64() { + Ok(serde_json::Value::Number(value.into())) + } else if let Some(value) = value.as_f64() { + serde_json::Number::from_f64(value) + .map(serde_json::Value::Number) + .ok_or_else(|| "YAML frontmatter contains a non-finite number".to_string()) + } else { + Err("YAML frontmatter contains an unsupported number".to_string()) + } + } + serde_yaml::Value::String(value) => Ok(serde_json::Value::String(value)), + serde_yaml::Value::Sequence(values) => values + .into_iter() + .map(yaml_to_json) + .collect::, _>>() + .map(serde_json::Value::Array), + serde_yaml::Value::Mapping(values) => { + let mut object = serde_json::Map::new(); + for (key, value) in values { + let serde_yaml::Value::String(key) = key else { + return Err("YAML frontmatter mapping keys must be strings".to_string()); + }; + object.insert(key, yaml_to_json(value)?); + } + Ok(serde_json::Value::Object(object)) + } + serde_yaml::Value::Tagged(_) => Err("YAML frontmatter tags are not supported".to_string()), + } +} + +fn markdown_projection_to_value(projection: MarkdownDocumentProjection) -> Result { + Ok(Value::object([ + ( + "frontmatter", + json_to_decodal_value(serde_json::Value::Object(projection.frontmatter))?, + ), + ("content", Value::string(projection.content)), + ])) +} + +fn json_to_decodal_value(value: serde_json::Value) -> Result { + match value { + serde_json::Value::Null => Err( + "YAML null values cannot be represented as concrete Decodal import values".to_string(), + ), + serde_json::Value::Bool(value) => Ok(Value::bool(value)), + serde_json::Value::Number(value) => { + if let Some(value) = value.as_i64() { + Ok(Value::int(value)) + } else if let Some(value) = value.as_u64() { + i64::try_from(value) + .map(Value::int) + .map_err(|_| "YAML integer exceeds the Decodal i64 range".to_string()) + } else if let Some(value) = value.as_f64() { + Ok(Value::float(value)) + } else { + Err("JSON number cannot be represented as a Decodal value".to_string()) + } + } + serde_json::Value::String(value) => Ok(Value::string(value)), + serde_json::Value::Array(values) => values + .into_iter() + .map(json_to_decodal_value) + .collect::, _>>() + .map(Value::array), + serde_json::Value::Object(values) => values + .into_iter() + .map(|(name, value)| Ok((name, json_to_decodal_value(value)?))) + .collect::, _>>() + .map(Value::object), + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ts_rs::TS)] pub struct ConfigEntry { pub path: VirtualPath, @@ -853,6 +998,23 @@ impl ImportLoader for SnapshotImportLoader { format!("virtual config import is missing: {path}"), ) })?; + if path.as_str().ends_with(".md") { + let projection = project_markdown_document(&entry.content).map_err(|message| { + Diagnostic::new( + DiagnosticKind::Import, + Span::default(), + format!("failed to import Markdown `{path}`: {message}"), + ) + })?; + let value = markdown_projection_to_value(projection).map_err(|message| { + Diagnostic::new( + DiagnosticKind::Import, + Span::default(), + format!("failed to import Markdown `{path}`: {message}"), + ) + })?; + return Ok(LoadedImport::value(path.as_str(), value)); + } Ok(LoadedImport::source( path.as_str(), path.as_str(), @@ -1147,6 +1309,10 @@ mod tests { ConfigEntry::new(path(path_value), ConfigContentType::Decodal, content).unwrap() } + fn text_entry(path_value: &str, content: &str) -> ConfigEntry { + ConfigEntry::new(path(path_value), ConfigContentType::Text, content).unwrap() + } + #[test] fn virtual_paths_reject_ambiguous_or_escaping_forms() { for invalid in ["", "/root.dcdl", "a//b", "a/./b", "a/../b", "a\\b", "a\0b"] { @@ -1522,6 +1688,97 @@ mod tests { assert_ne!(empty.fingerprint, configured.fingerprint); } + #[test] + fn markdown_import_projects_frontmatter_and_content_as_a_value() { + let markdown = concat!( + "---\n", + "name: debug-rust\n", + "description: Debug Rust failures\n", + "custom-authority: no\n", + "allowed-tools: Read Grep\n", + "metadata:\n owner: platform\n", + "---\n", + "# Debug Rust\n", + ); + let snapshot = ConfigTreeSnapshot::from_entries( + 3, + [ + entry( + "main.dcdl", + r#"{ skill = import "./skills/debug-rust/SKILL.md" as { frontmatter = { name = String; description = String; ...Unknown }; content = String; }; }"#, + ), + text_entry("skills/debug-rust/SKILL.md", markdown), + ], + ) + .unwrap(); + let result = SnapshotEnvironment::new(snapshot.clone()) + .evaluate_contract(&ToolchainContract::new(1, vec![path("main.dcdl")], 1)) + .unwrap(); + let skill = &result.projections[0].data_json["skill"]; + assert_eq!(skill["frontmatter"]["name"], "debug-rust"); + assert_eq!(skill["frontmatter"]["custom-authority"], "no"); + assert_eq!(skill["frontmatter"]["allowed-tools"], "Read Grep"); + assert_eq!(skill["frontmatter"]["metadata"]["owner"], "platform"); + assert_eq!(skill["content"], "# Debug Rust\n"); + assert_eq!( + snapshot.entries[&path("skills/debug-rust/SKILL.md")].content_digest, + digest_bytes(markdown.as_bytes()) + ); + } + + #[test] + fn markdown_import_without_frontmatter_preserves_the_whole_body() { + let source = "# Plain skill\nKeep --- inside the body.\n"; + assert_eq!( + project_markdown_document(source).unwrap(), + MarkdownDocumentProjection { + frontmatter: serde_json::Map::new(), + content: source.to_string(), + } + ); + let snapshot = ConfigTreeSnapshot::from_entries( + 1, + [ + entry("main.dcdl", r#"import "./skills/plain/SKILL.md""#), + text_entry("skills/plain/SKILL.md", source), + ], + ) + .unwrap(); + let result = SnapshotEnvironment::new(snapshot) + .evaluate_contract(&ToolchainContract::new(1, vec![path("main.dcdl")], 1)) + .unwrap(); + assert_eq!( + result.projections[0].data_json["frontmatter"], + serde_json::json!({}) + ); + assert_eq!(result.projections[0].data_json["content"], source); + } + + #[test] + fn malformed_markdown_frontmatter_is_an_import_diagnostic() { + assert_eq!( + project_markdown_document("---\nname: missing-close\nbody\n").unwrap_err(), + "opening YAML frontmatter delimiter has no closing delimiter" + ); + let snapshot = ConfigTreeSnapshot::from_entries( + 1, + [ + entry("main.dcdl", r#"import "./skills/broken/SKILL.md""#), + text_entry( + "skills/broken/SKILL.md", + "---\nname: [unterminated\n---\nbody\n", + ), + ], + ) + .unwrap(); + let diagnostics = SnapshotEnvironment::new(snapshot) + .evaluate_contract(&ToolchainContract::new(1, vec![path("main.dcdl")], 1)) + .unwrap_err(); + assert_eq!(diagnostics[0].kind, "import"); + assert!(diagnostics[0].message.contains("invalid YAML frontmatter")); + assert!(diagnostics[0].message.contains("skills/broken/SKILL.md")); + } + #[test] fn host_environment_evaluation_uses_only_snapshot_imports() { let snapshot = ConfigTreeSnapshot::from_entries( diff --git a/web/workspace/src/lib/workspace/config-source/generated/config_source_wasm_bg.wasm b/web/workspace/src/lib/workspace/config-source/generated/config_source_wasm_bg.wasm index f492dbba..98a0c035 100644 Binary files a/web/workspace/src/lib/workspace/config-source/generated/config_source_wasm_bg.wasm and b/web/workspace/src/lib/workspace/config-source/generated/config_source_wasm_bg.wasm differ diff --git a/web/workspace/test/config-source/wasm-parity.test.ts b/web/workspace/test/config-source/wasm-parity.test.ts index 34e5ea5c..41df078b 100644 --- a/web/workspace/test/config-source/wasm-parity.test.ts +++ b/web/workspace/test/config-source/wasm-parity.test.ts @@ -146,6 +146,48 @@ const schemaContract: ToolchainContract = { fingerprint: await toolchainFingerprint(["main.dcdl"], schemaBundle), }; +const markdownSnapshot: ConfigTreeSnapshot = { + revision: 8, + digest: "sha256:markdown-tree", + entries: { + "main.dcdl": { + path: "main.dcdl", + content_type: "decodal", + content: + `{ skill = import "./skills/debug-rust/SKILL.md" as { frontmatter = { name = String; description = String; ...Unknown }; content = String; }; }`, + content_digest: "sha256:markdown-main", + }, + "skills/debug-rust/SKILL.md": { + path: "skills/debug-rust/SKILL.md", + content_type: "text", + content: + "---\nname: debug-rust\ndescription: Debug Rust\ncustom-authority: no\n---\n# Debug Rust\n", + content_digest: "sha256:markdown-skill", + }, + }, +}; +const markdownContract: ToolchainContract = { + ...contract, + entrypoints: ["main.dcdl"], + fingerprint: await toolchainFingerprint(["main.dcdl"], emptySchemaBundle), +}; + +Deno.test("generated WASM evaluates Markdown with the shared Skill projection", () => { + const result = evaluate_snapshot(markdownSnapshot, markdownContract) as { + projections: Array<{ data_json: Record }>; + }; + assertEquals(result.projections[0].data_json, { + skill: { + frontmatter: { + "custom-authority": "no", + description: "Debug Rust", + name: "debug-rust", + }, + content: "# Debug Rust\n", + }, + }); +}); + Deno.test("generated WASM applies Decodal 0.4 typed maps and explicit object rest", () => { const result = evaluate_snapshot( schemaSnapshot(