diff --git a/README.md b/README.md index fc36662..be0dd53 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Decodal is a small deterministic DSL for describing, composing, validating, and It is designed around a lightweight Rust library: -- host-supplied imports through `SourceLoader` +- host-supplied source and structured-value imports through `ImportLoader` - no filesystem access in the library core - concrete and abstract values with constraints and defaults - deterministic expression evaluation @@ -13,7 +13,7 @@ It is designed around a lightweight Rust library: ## Library crate -Embedded hosts should depend on `decodal` and provide imports with a `SourceLoader`. +Embedded hosts should depend on `decodal` and provide imports with an `ImportLoader`. ```toml [dependencies] diff --git a/crates/decodal-cli/src/main.rs b/crates/decodal-cli/src/main.rs index f040243..423a8ed 100644 --- a/crates/decodal-cli/src/main.rs +++ b/crates/decodal-cli/src/main.rs @@ -5,8 +5,8 @@ use std::{ }; use decodal::{ - Data, Diagnostic, DiagnosticKind, Engine, LoadedSource, SourceId, SourceLoader, Span, - format_diagnostic_with, + Data, Diagnostic, DiagnosticKind, Engine, ImportLoader, LoadedImport, LoadedSource, SourceId, + Span, format_diagnostic_with, }; fn main() -> ExitCode { @@ -95,12 +95,12 @@ fn read_root_source(path: &str) -> Result { #[derive(Debug, Clone, Copy)] struct FsLoader; -impl SourceLoader for FsLoader { +impl ImportLoader for FsLoader { fn load( &mut self, current_key: Option<&str>, specifier: &str, - ) -> Result { + ) -> Result { let path = Path::new(specifier); let path = if path.is_absolute() { PathBuf::from(path) @@ -112,7 +112,7 @@ impl SourceLoader for FsLoader { } else { PathBuf::from(path) }; - load_path(&path) + load_path(&path).map(LoadedImport::Source) } } diff --git a/crates/decodal-core/examples/structured_import.rs b/crates/decodal-core/examples/structured_import.rs new file mode 100644 index 0000000..b0d12a8 --- /dev/null +++ b/crates/decodal-core/examples/structured_import.rs @@ -0,0 +1,87 @@ +use decodal::{ + Data, Diagnostic, DiagnosticKind, Engine, HostValue, ImportLoader, LoadedImport, Span, +}; + +const POST: &str = r#"--- +title: Hello +draft: false +--- +# Hello + +This body stays as Markdown. +"#; + +struct ContentLoader; + +impl ImportLoader for ContentLoader { + fn load( + &mut self, + _current_key: Option<&str>, + specifier: &str, + ) -> decodal::Result { + if specifier != "./post.md" { + return Err(Diagnostic::new( + DiagnosticKind::Import, + Span::default(), + format!("unknown content import `{specifier}`"), + )); + } + Ok(LoadedImport::value( + "content/post.md", + parse_markdown(POST)?, + )) + } +} + +fn parse_markdown(source: &str) -> decodal::Result { + let source = source.strip_prefix("---\n").ok_or_else(frontmatter_error)?; + let (frontmatter, body) = source.split_once("\n---\n").ok_or_else(frontmatter_error)?; + let mut fields = Vec::new(); + for line in frontmatter.lines() { + let (name, value) = line.split_once(':').ok_or_else(frontmatter_error)?; + let value = match value.trim() { + "true" => HostValue::bool(true), + "false" => HostValue::bool(false), + value => HostValue::string(value), + }; + fields.push((name.trim(), value)); + } + Ok(HostValue::object([ + ("frontmatter", HostValue::object(fields)), + ("body", HostValue::string(body)), + ])) +} + +fn frontmatter_error() -> Diagnostic { + Diagnostic::new( + DiagnosticKind::Import, + Span::default(), + "invalid Markdown frontmatter", + ) +} + +fn main() -> decodal::Result<()> { + let mut engine = Engine::new(ContentLoader); + let module = engine.add_root_source( + "main.dcdl", + "main.dcdl", + r#" + let post = import "./post.md"; + in { + title = post.frontmatter.title; + draft = post.frontmatter.draft; + body = post.body; + } + "#, + )?; + let value = engine.eval_module(module)?; + let data = engine.materialize(&value)?; + + let Data::Object(fields) = data else { + panic!("expected imported Markdown to produce an object") + }; + assert_eq!(fields[0].value, Data::String(String::from("Hello"))); + assert_eq!(fields[1].value, Data::Bool(false)); + assert!(matches!(fields[2].value, Data::String(_))); + Ok(()) +} diff --git a/crates/decodal-core/src/eval.rs b/crates/decodal-core/src/eval.rs index 6a061e3..13dc5d5 100644 --- a/crates/decodal-core/src/eval.rs +++ b/crates/decodal-core/src/eval.rs @@ -6,7 +6,7 @@ use crate::{ constraints::normalize_constraints, diagnostic::{Diagnostic, DiagnosticKind, Result}, embedding::HostValue, - module::{EmptyLoader, LoadedSource, Module, SourceLoader}, + module::{EmptyLoader, ImportLoader, LoadedImport, LoadedSource, LoadedValue, Module}, parse_source_with_source_id, runtime::{ AbstractValue, Binding, ConcreteValue, Constraint, ConstraintEntry, Data, DataField, Env, @@ -19,10 +19,16 @@ pub struct Engine { loader: L, prelude_env: EnvId, modules: Vec, + imported_values: Vec, thunks: Vec, envs: Vec, } +struct ImportedValue { + key: String, + root: ThunkId, +} + impl Engine { pub fn from_parse(ast: Ast, root: ExprId) -> Self { let mut this = Self::new(EmptyLoader); @@ -38,12 +44,13 @@ impl Engine { } } -impl Engine { +impl Engine { pub fn new(loader: L) -> Self { Self { loader, prelude_env: EnvId(0), modules: Vec::new(), + imported_values: Vec::new(), thunks: Vec::new(), envs: vec![Env { parent: None, @@ -236,10 +243,37 @@ impl Engine { .map(|index| ModuleId(index as u32)) } - fn load_import(&mut self, current: ModuleId, specifier: &str) -> Result { + fn eval_import(&mut self, current: ModuleId, specifier: &str) -> Result { let current_key = self.modules[current.0 as usize].key.clone(); - let LoadedSource { key, name, source } = self.loader.load(Some(¤t_key), specifier)?; - self.add_source(key, name, &source) + let loaded = self.loader.load(Some(¤t_key), specifier)?; + match loaded { + LoadedImport::Source(LoadedSource { key, name, source }) => { + if let Some(value) = self.find_imported_value(&key) { + return self.force(value); + } + let module = self.add_source(key, name, &source)?; + self.eval_module(module) + } + LoadedImport::Value(LoadedValue { key, value }) => { + if let Some(module) = self.find_module(&key) { + return self.eval_module(module); + } + if let Some(value) = self.find_imported_value(&key) { + return self.force(value); + } + let value = self.internalize_host_value(value)?; + let root = self.add_value_thunk(value); + self.imported_values.push(ImportedValue { key, root }); + self.force(root) + } + } + } + + fn find_imported_value(&self, key: &str) -> Option { + self.imported_values + .iter() + .find(|value| value.key == key) + .map(|value| value.root) } fn eval_expr(&mut self, reference: ExprRef, env: EnvId) -> Result { @@ -303,10 +337,7 @@ impl Engine { let_env, ) } - Expr::Import(specifier) => { - let module = self.load_import(reference.module, &specifier)?; - self.eval_module(module) - } + Expr::Import(specifier) => self.eval_import(reference.module, &specifier), Expr::Path { base, field } => { let base = self.eval_expr( ExprRef { @@ -1964,8 +1995,8 @@ mod tests { sources: Vec<(String, String)>, } - impl SourceLoader for MapLoader { - fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> Result { + impl ImportLoader for MapLoader { + fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> Result { let source = self .sources .iter() @@ -1974,11 +2005,11 @@ mod tests { .ok_or_else(|| { Diagnostic::new(DiagnosticKind::Import, Span::default(), "missing source") })?; - Ok(LoadedSource { + Ok(LoadedImport::Source(LoadedSource { key: specifier.into(), name: specifier.into(), source, - }) + })) } } @@ -2003,6 +2034,125 @@ mod tests { assert_eq!(fields[0].value, Data::Int(9000)); } + struct MarkdownLoader { + source: String, + } + + impl ImportLoader for MarkdownLoader { + fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> Result { + if specifier != "./post.md" { + return Err(Diagnostic::new( + DiagnosticKind::Import, + Span::default(), + format!("unknown content import `{specifier}`"), + )); + } + let value = parse_test_markdown(&self.source)?; + Ok(LoadedImport::value("content/post.md", value)) + } + } + + fn parse_test_markdown(source: &str) -> Result { + let source = source.strip_prefix("---\n").ok_or_else(|| { + Diagnostic::new( + DiagnosticKind::Import, + Span::default(), + "markdown frontmatter must start with `---`", + ) + })?; + let (frontmatter, body) = source.split_once("\n---\n").ok_or_else(|| { + Diagnostic::new( + DiagnosticKind::Import, + Span::default(), + "markdown frontmatter is not terminated", + ) + })?; + let mut fields = Vec::new(); + for line in frontmatter.lines() { + let (name, value) = line.split_once(':').ok_or_else(|| { + Diagnostic::new( + DiagnosticKind::Import, + Span::default(), + "invalid markdown frontmatter field", + ) + })?; + let value = match value.trim() { + "true" => HostValue::bool(true), + "false" => HostValue::bool(false), + value => HostValue::string(value.trim_matches('"')), + }; + fields.push((name.trim(), value)); + } + Ok(HostValue::object([ + ("frontmatter", HostValue::object(fields)), + ("body", HostValue::string(body)), + ])) + } + + #[test] + fn imports_host_structured_content() { + let mut engine = Engine::new(MarkdownLoader { + source: String::from("---\ntitle: \"Hello\"\ndraft: false\n---\n# Hello\n\nBody text."), + }); + let module = engine + .add_root_source( + "main.dcdl", + "main.dcdl", + r#" + let + Post = { + frontmatter = { + title = String; + draft = Bool; + }; + body = String; + }; + post = Post & import "./post.md"; + in { + title = post.frontmatter.title; + draft = post.frontmatter.draft; + body = post.body; + } + "#, + ) + .unwrap(); + let value = engine.eval_module(module).unwrap(); + let data = engine.materialize(&value).unwrap(); + let Data::Object(fields) = data else { panic!() }; + assert_eq!(fields[0].value, Data::String(String::from("Hello"))); + assert_eq!(fields[1].value, Data::Bool(false)); + assert_eq!( + fields[2].value, + Data::String(String::from("# Hello\n\nBody text.")) + ); + } + + struct StructuredAliasLoader; + + impl ImportLoader for StructuredAliasLoader { + fn load(&mut self, _current_key: Option<&str>, _specifier: &str) -> Result { + Ok(LoadedImport::value( + "shared-value", + HostValue::int_type().default_int(1)?, + )) + } + } + + #[test] + fn structured_imports_share_cached_runtime_value_by_key() { + let mut engine = Engine::new(StructuredAliasLoader); + let module = engine + .add_root_source( + "main.dcdl", + "main.dcdl", + r#"(import "./first.data") & (import "./second.data")"#, + ) + .unwrap(); + let value = engine.eval_module(module).unwrap(); + let data = engine.materialize(&value).unwrap(); + assert_eq!(data, Data::Int(1)); + } + #[test] fn top_level_fields_are_recursive_module_scope() { let mut engine = Engine::new(EmptyLoader); diff --git a/crates/decodal-core/src/lib.rs b/crates/decodal-core/src/lib.rs index 27900be..4eebbb4 100644 --- a/crates/decodal-core/src/lib.rs +++ b/crates/decodal-core/src/lib.rs @@ -21,7 +21,7 @@ pub use decodal_derive::Decodal; pub use diagnostic::{Diagnostic, DiagnosticKind, Result}; pub use embedding::{HostField, HostValue}; pub use eval::{Engine, format_diagnostic_with}; -pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader}; +pub use module::{EmptyLoader, ImportLoader, LoadedImport, LoadedSource, LoadedValue, Module}; pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id}; pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue}; pub use span::{SourceId, Span}; diff --git a/crates/decodal-core/src/module.rs b/crates/decodal-core/src/module.rs index c7e209a..70b5f8d 100644 --- a/crates/decodal-core/src/module.rs +++ b/crates/decodal-core/src/module.rs @@ -1,7 +1,7 @@ use alloc::string::String; use crate::{ - Ast, ExprId, SourceForm, SourceId, + Ast, ExprId, HostValue, SourceForm, SourceId, runtime::{EnvId, ThunkId}, }; @@ -24,15 +24,48 @@ pub struct LoadedSource { pub source: String, } -pub trait SourceLoader { - fn load(&mut self, current_key: Option<&str>, specifier: &str) -> crate::Result; +#[derive(Debug, Clone)] +pub struct LoadedValue { + pub key: String, + pub value: HostValue, +} + +#[derive(Debug, Clone)] +pub enum LoadedImport { + Source(LoadedSource), + Value(LoadedValue), +} + +impl LoadedImport { + pub fn source( + key: impl Into, + name: impl Into, + source: impl Into, + ) -> Self { + Self::Source(LoadedSource { + key: key.into(), + name: name.into(), + source: source.into(), + }) + } + + pub fn value(key: impl Into, value: HostValue) -> Self { + Self::Value(LoadedValue { + key: key.into(), + value, + }) + } +} + +pub trait ImportLoader { + fn load(&mut self, current_key: Option<&str>, specifier: &str) -> crate::Result; } #[derive(Debug, Clone, Copy, Default)] pub struct EmptyLoader; -impl SourceLoader for EmptyLoader { - fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> crate::Result { +impl ImportLoader for EmptyLoader { + fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> crate::Result { Err(crate::Diagnostic::new( crate::DiagnosticKind::Import, crate::Span::default(), diff --git a/crates/decodal-wasm/src/lib.rs b/crates/decodal-wasm/src/lib.rs index 1c5f7af..5a2705f 100644 --- a/crates/decodal-wasm/src/lib.rs +++ b/crates/decodal-wasm/src/lib.rs @@ -1,8 +1,8 @@ use std::collections::BTreeMap; use decodal::{ - Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader, - Span, format_diagnostic_with, + Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, ImportLoader, LoadedImport, + LoadedSource, SourceId, Span, format_diagnostic_with, }; use wasm_bindgen::prelude::*; @@ -76,12 +76,12 @@ struct VirtualLoader { files: BTreeMap, } -impl SourceLoader for VirtualLoader { +impl ImportLoader for VirtualLoader { fn load( &mut self, current_key: Option<&str>, specifier: &str, - ) -> decodal::Result { + ) -> decodal::Result { let key = resolve_import(current_key, specifier).ok_or_else(|| { Diagnostic::new( DiagnosticKind::Import, @@ -96,11 +96,11 @@ impl SourceLoader for VirtualLoader { format!("import `{specifier}` resolved to `{key}`, but that file does not exist"), ) })?; - Ok(LoadedSource { + Ok(LoadedImport::Source(LoadedSource { key: key.clone(), name: key, source, - }) + })) } } diff --git a/doc/manual/souce/design/embedding-api.md b/doc/manual/souce/design/embedding-api.md index 8b5df2c..5028ceb 100644 --- a/doc/manual/souce/design/embedding-api.md +++ b/doc/manual/souce/design/embedding-api.md @@ -1,7 +1,7 @@ # Embedding API Decodal core can be embedded without giving the core crate access to a filesystem. -The host supplies imported sources through `SourceLoader` and may also provide global bindings through the host prelude API. +The host supplies imports through `ImportLoader` and may also provide global bindings through the host prelude API. ## Host prelude @@ -131,11 +131,32 @@ Supported field attributes are intentionally small: The derive does not add host callbacks or reflection. It only generates schema construction and typed decoding code. -## SourceLoader and prelude together +## ImportLoader and prelude together -`SourceLoader` and host prelude bindings are independent mechanisms. +`ImportLoader` and host prelude bindings are independent mechanisms. -- Use `SourceLoader` when user sources should explicitly import host-provided modules. +- Use `ImportLoader` when user sources should explicitly import host-provided sources or values. - Use prelude bindings when host-provided schemas or constants should be globally available. Both mechanisms share the same runtime evaluator, thunk model, and materialization rules. + +## Structured imports + +`ImportLoader::load` returns either `LoadedImport::Source` or `LoadedImport::Value`. +The value variant carries a `HostValue`, allowing a host to parse non-Decodal resources such as Markdown into an application-specific structure. + +```text +import "./post.md" + -> host parses content + -> LoadedImport::Value( + { frontmatter: {...}, body: "..." } + ) + -> Engine internalizes HostValue + -> normal composition and materialization +``` + +The core does not select content types or bundle Markdown/frontmatter parsers. +The loader owns path resolution, media or extension dispatch, parsing rules, and parse diagnostics. +The stable loader key is also used to cache structured imports. + +`load` is the single import hook: loaders dispatch by extension, media type, or another host-defined rule and return the appropriate variant directly. diff --git a/doc/manual/souce/design/execution-pipeline.md b/doc/manual/souce/design/execution-pipeline.md index 1d40e29..a14860e 100644 --- a/doc/manual/souce/design/execution-pipeline.md +++ b/doc/manual/souce/design/execution-pipeline.md @@ -66,14 +66,16 @@ ModuleRegistry: 処理系は、まず root module を parse / desugar して registry に登録する。 import 先 module は、この段階で全て読み込む必要はない。 -import expression が評価されたとき、処理系は `SourceLoader` に現在の module key と import specifier を渡す。 -loader は module key、表示名、source text を返す。 -module registry は key が未登録なら対象 module を parse / desugar して登録する。 -登録された module は module root thunk を持つ。 -同じ module が複数回 import された場合は、同じ `ModuleId` を返す。 +import expression が評価されたとき、処理系は `ImportLoader::load` に現在の module key と import specifier を渡す。 +loader は module key、表示名、および DCDL source text または構造化済み `HostValue` を返す。 +DCDL source の場合、module registry は key が未登録なら対象 module を parse / desugar して登録する。 +登録された source module は module root thunk を持つ。 +構造化値の場合、host value を runtime value に internalize した root thunk を key でキャッシュする。 +同じ key が複数回 import された場合は、同じ source module または structured root thunk を使う。 -つまり import は module を即時評価しない。 +つまり source import は module を即時評価しない。 module を読み込み、module root を thunk として登録するだけにする。 +structured import も root value を thunk として保持し、object field や array item の既存 thunk model を利用する。 AST の `ExprId` は module-local である。 そのため runtime が保持する式参照は `ExprRef { module, expr }` として module-qualified にする。 diff --git a/doc/manual/souce/design/runtime-model.md b/doc/manual/souce/design/runtime-model.md index 7db2877..69d41d7 100644 --- a/doc/manual/souce/design/runtime-model.md +++ b/doc/manual/souce/design/runtime-model.md @@ -104,7 +104,7 @@ Concrete(Int(8000)) ## Runtime scope Decodal runtime は application runtime ではなく、pure value evaluator である。 -同じ source、同じ import sources、同じ host globals が与えられた場合、評価結果は決定的である。 +同じ source、同じ import results、同じ host globals が与えられた場合、評価結果は決定的である。 runtime が扱う責務は以下に限る。 @@ -114,7 +114,7 @@ runtime が扱う責務は以下に限る。 - materialize 時に constraint を検証する。 runtime は filesystem、network、environment variable、time、random、mutation を扱わない。 -core における import は host supplied source を受け取る境界であり、filesystem access ではない。 +core における import は host supplied source または structured value を受け取る境界であり、filesystem access ではない。 ## Constraint diff --git a/doc/manual/souce/language/modules-and-imports.md b/doc/manual/souce/language/modules-and-imports.md index f28273c..78a4509 100644 --- a/doc/manual/souce/language/modules-and-imports.md +++ b/doc/manual/souce/language/modules-and-imports.md @@ -32,15 +32,77 @@ result = schema; 通常の object literal 内の field は、その object 内の sibling field を暗黙には識別子として参照できない。 object 内の値を参照する場合は、外側で束縛された値や明示的な path reference を使う。 -## SourceLoader +## ImportLoader -`import` specifier の解決は処理系 core ではなく host 側の `SourceLoader` が行う。 +`import` specifier の解決は処理系 core ではなく host 側の `ImportLoader` が行う。 CLI では、specifier を現在の module path からの相対 path として解決する。 組み込み利用では、resource table や static source map など、filesystem 以外の loader を使える。 module cache の key は loader が返す安定 key を使う。 CLI では canonical path を key とする。 +### 構造化 import + +`ImportLoader::load` は DCDL source または host が構築した `HostValue` を import 結果として返す。 +Markdown、JSON、TOML などの解釈規則は core に固定せず、loader がファイル種別を判定して構造化する。 + +```rust +use decodal::{HostValue, ImportLoader, LoadedImport}; + +struct ContentLoader; + +impl ImportLoader for ContentLoader { + fn load( + &mut self, + current_key: Option<&str>, + specifier: &str, + ) -> decodal::Result { + if specifier.ends_with(".md") { + let markdown = read_content(current_key, specifier)?; + let parsed = parse_frontmatter(&markdown)?; + return Ok(LoadedImport::value( + parsed.key, + HostValue::object([ + ("frontmatter", parsed.frontmatter), + ("body", HostValue::string(parsed.body)), + ]), + )); + } + + let source = read_dcdl(current_key, specifier)?; + Ok(LoadedImport::source( + source.key, + specifier, + source.text, + )) + } +} +``` + +`read_content` と `parse_frontmatter` は host 独自の処理であり、Decodal core は Markdown や YAML parser に依存しない。 +上の loader を使うと、DCDL 側から次のように扱える。 + +```dcdl +Post = { + frontmatter = { + title = String; + draft = Bool default false; + }; + body = String; +}; + +post = Post & import "./hello.md"; +title = post.frontmatter.title; +body = post.body; +``` + +`LoadedImport::Value` は通常の concrete runtime value に internalize される。 +そのため、path reference、object composition、constraint validation、materialize は source 由来の値と同じ規則を使う。 +安定した `key` が同じ構造化 import は、engine 内で同じ値としてキャッシュされる。 + +`load` が唯一の import hook である。 +loader は拡張子、media type、または host 独自の規則で振り分け、対応する `LoadedImport` variant を直接返す。 + ## 循環 import モジュール間に循環参照があっても、必要なフィールドの依存関係が循環していなければ評価できる。 @@ -93,4 +155,5 @@ Module func - ファイルが読めない。 - import 先の構文解析に失敗する。 - import 先の評価で必要な値がエラーになる。 +- host による非 DCDL content の読み込みまたは構造化に失敗する。 - 実装が禁止する import 循環に該当する。 diff --git a/packages/decodal-wasm/decodal_wasm_bg.wasm b/packages/decodal-wasm/decodal_wasm_bg.wasm index 2a2cb52..8139ed9 100644 Binary files a/packages/decodal-wasm/decodal_wasm_bg.wasm and b/packages/decodal-wasm/decodal_wasm_bg.wasm differ