Implement module loading and imports

crates/decodal-cli/src/main.rs

@@ -1,6 +1,10 @@
-use std::{env, fs, process::ExitCode};
+use std::{
+    env, fs,
+    path::{Path, PathBuf},
+    process::ExitCode,
+};
 
-use decodal_core::{Data, Diagnostic, Engine, parse_source};
+use decodal_core::{Data, Diagnostic, DiagnosticKind, Engine, LoadedSource, SourceLoader, Span};
 
 fn main() -> ExitCode {
     match run() {
@@ -26,34 +30,36 @@ fn run() -> Result<(), Diagnostic> {
         }
         "check" => {
             let path = args.next().unwrap_or_else(|| String::from("-"));
-            let source = read_source(&path)?;
+            let data = materialize_path(&path)?;
-            let parsed = parse_source(&source)?;
+            drop(data);
-            let mut engine = Engine::from_parse(parsed.ast, parsed.root);
-            let value = engine.eval_root()?;
-            engine.materialize(&value)?;
             println!("ok");
             Ok(())
         }
         "eval" | "materialize" => {
             let path = args.next().unwrap_or_else(|| String::from("-"));
-            materialize_file(&path)
+            let data = materialize_path(&path)?;
+            print_data(&data, 0);
+            println!();
+            Ok(())
+        }
+        path => {
+            let data = materialize_path(path)?;
+            print_data(&data, 0);
+            println!();
+            Ok(())
         }
-        path => materialize_file(path),
     }
 }
 
-fn materialize_file(path: &str) -> Result<(), Diagnostic> {
+fn materialize_path(path: &str) -> Result<Data, Diagnostic> {
-    let source = read_source(path)?;
+    let root = read_root_source(path)?;
-    let parsed = parse_source(&source)?;
+    let mut engine = Engine::new(FsLoader);
-    let mut engine = Engine::from_parse(parsed.ast, parsed.root);
+    let module = engine.add_root_source(root.key, root.name, &root.source)?;
-    let value = engine.eval_root()?;
+    let value = engine.eval_module(module)?;
-    let data = engine.materialize(&value)?;
+    engine.materialize(&value)
-    print_data(&data, 0);
-    println!();
-    Ok(())
 }
 
-fn read_source(path: &str) -> Result<String, Diagnostic> {
+fn read_root_source(path: &str) -> Result<LoadedSource, Diagnostic> {
     if path == "-" {
         use std::io::Read;
         let mut source = String::new();
@@ -61,23 +67,68 @@ fn read_source(path: &str) -> Result<String, Diagnostic> {
             .read_to_string(&mut source)
             .map_err(|error| {
                 Diagnostic::new(
-                    decodal_core::DiagnosticKind::Import,
+                    DiagnosticKind::Import,
-                    decodal_core::Span::default(),
+                    Span::default(),
                     format!("failed to read stdin: {error}"),
                 )
             })?;
-        Ok(source)
+        Ok(LoadedSource {
-    } else {
+            key: String::from("<stdin>"),
-        fs::read_to_string(path).map_err(|error| {
+            name: String::from("<stdin>"),
-            Diagnostic::new(
+            source,
-                decodal_core::DiagnosticKind::Import,
-                decodal_core::Span::default(),
-                format!("failed to read `{path}`: {error}"),
-            )
         })
+    } else {
+        load_path(Path::new(path))
     }
 }
 
+#[derive(Debug, Clone, Copy)]
+struct FsLoader;
+
+impl SourceLoader for FsLoader {
+    fn load(
+        &mut self,
+        current_key: Option<&str>,
+        specifier: &str,
+    ) -> Result<LoadedSource, Diagnostic> {
+        let path = Path::new(specifier);
+        let path = if path.is_absolute() {
+            PathBuf::from(path)
+        } else if let Some(current_key) = current_key.filter(|key| *key != "<stdin>") {
+            Path::new(current_key)
+                .parent()
+                .unwrap_or_else(|| Path::new("."))
+                .join(path)
+        } else {
+            PathBuf::from(path)
+        };
+        load_path(&path)
+    }
+}
+
+fn load_path(path: &Path) -> Result<LoadedSource, Diagnostic> {
+    let canonical = path.canonicalize().map_err(|error| {
+        Diagnostic::new(
+            DiagnosticKind::Import,
+            Span::default(),
+            format!("failed to resolve `{}`: {error}", path.display()),
+        )
+    })?;
+    let source = fs::read_to_string(&canonical).map_err(|error| {
+        Diagnostic::new(
+            DiagnosticKind::Import,
+            Span::default(),
+            format!("failed to read `{}`: {error}", canonical.display()),
+        )
+    })?;
+    let key = canonical.to_string_lossy().into_owned();
+    Ok(LoadedSource {
+        name: key.clone(),
+        key,
+        source,
+    })
+}
+
 fn print_help() {
     println!("Decodal - Deferred Constraint Data Language");
     println!();
@@ -90,8 +141,9 @@ fn print_help() {
 
 fn print_diagnostic(error: &Diagnostic) {
     eprintln!(
-        "error[{kind:?}] {start}..{end}: {message}",
+        "error[{kind:?}] {source}:{start}..{end}: {message}",
         kind = error.kind,
+        source = error.span.source.0,
         start = error.span.start,
         end = error.span.end,
         message = error.message,

crates/decodal-core/src/eval.rs

@@ -1,40 +1,66 @@
 use alloc::{format, string::String, vec, vec::Vec};
 
 use crate::{
-    Span,
+    ExprId, SourceForm, SourceId, Span,
-    ast::{Ast, BinaryOp, CompareOp, Expr, ExprId, Field, Literal},
+    ast::{Ast, BinaryOp, CompareOp, Expr, Field, Literal},
     diagnostic::{Diagnostic, DiagnosticKind, Result},
+    module::{EmptyLoader, LoadedSource, Module, SourceLoader},
+    parse_source_with_source_id,
     runtime::{
-        AbstractValue, Binding, ConcreteValue, Constraint, Data, DataField, Env, EnvId,
+        AbstractValue, Binding, ConcreteValue, Constraint, Data, DataField, Env, EnvId, ExprRef,
-        FunctionParam, FunctionValue, LiteralValue, ObjectField, ObjectValue, PrimitiveType,
+        FunctionParam, FunctionValue, LiteralValue, ModuleId, ObjectField, ObjectValue,
-        RuntimeValue, Thunk, ThunkId, ThunkKind, ThunkState,
+        PrimitiveType, RuntimeValue, Thunk, ThunkId, ThunkKind, ThunkState,
     },
 };
 
-pub struct Engine {
+pub struct Engine<L = EmptyLoader> {
-    ast: Ast,
+    loader: L,
-    root: ExprId,
+    modules: Vec<Module>,
     thunks: Vec<Thunk>,
     envs: Vec<Env>,
 }
 
-impl Engine {
+impl Engine<EmptyLoader> {
     pub fn from_parse(ast: Ast, root: ExprId) -> Self {
-        let mut this = Self {
+        let mut this = Self::new(EmptyLoader);
+        this.register_parsed(
+            String::from("<memory>"),
+            String::from("<memory>"),
+            SourceId(0),
             ast,
             root,
+            SourceForm::Expr,
+        );
+        this
+    }
+}
+
+impl<L: SourceLoader> Engine<L> {
+    pub fn new(loader: L) -> Self {
+        Self {
+            loader,
+            modules: Vec::new(),
             thunks: Vec::new(),
             envs: Vec::new(),
-        };
+        }
-        this.envs.push(Env {
+    }
-            parent: None,
+
-            bindings: Vec::new(),
+    pub fn add_root_source(
-        });
+        &mut self,
-        this
+        key: impl Into<String>,
+        name: impl Into<String>,
+        source: &str,
+    ) -> Result<ModuleId> {
+        self.add_source(key.into(), name.into(), source)
+    }
+
+    pub fn eval_module(&mut self, module: ModuleId) -> Result<RuntimeValue> {
+        let thunk = self.modules[module.0 as usize].root_thunk;
+        self.force(thunk)
     }
 
     pub fn eval_root(&mut self) -> Result<RuntimeValue> {
-        self.eval_expr(self.root, EnvId(0))
+        self.eval_module(ModuleId(0))
     }
 
     pub fn materialize(&mut self, value: &RuntimeValue) -> Result<Data> {
@@ -84,36 +110,131 @@ impl Engine {
         }
     }
 
-    fn eval_expr(&mut self, id: ExprId, env: EnvId) -> Result<RuntimeValue> {
+    fn add_source(&mut self, key: String, name: String, source: &str) -> Result<ModuleId> {
-        let span = self.ast.span(id);
+        if let Some(id) = self.find_module(&key) {
-        let expr = self.ast.get(id).expr.clone();
+            return Ok(id);
+        }
+        let source_id = SourceId(self.modules.len() as u32);
+        let parsed = parse_source_with_source_id(source_id, source)?;
+        Ok(self.register_parsed(
+            key,
+            name,
+            source_id,
+            parsed.ast,
+            parsed.root,
+            parsed.source_form,
+        ))
+    }
+
+    fn register_parsed(
+        &mut self,
+        key: String,
+        name: String,
+        source: SourceId,
+        ast: Ast,
+        root: ExprId,
+        source_form: SourceForm,
+    ) -> ModuleId {
+        let module = ModuleId(self.modules.len() as u32);
+        let root_env = self.new_env(None);
+        let root_thunk = if source_form == SourceForm::Fields {
+            if let Expr::Object(fields) = ast.get(root).expr.clone() {
+                let object = self
+                    .build_object(module, &fields, root_env)
+                    .expect("module object construction should not fail for parsed fields");
+                for field in &object.fields {
+                    self.bind(root_env, field.name.clone(), field.value);
+                }
+                self.add_value_thunk(RuntimeValue::Concrete(ConcreteValue::Object(object)))
+            } else {
+                self.add_expr_thunk(ExprRef { module, expr: root }, root_env)
+            }
+        } else {
+            self.add_expr_thunk(ExprRef { module, expr: root }, root_env)
+        };
+        self.modules.push(Module {
+            key,
+            name,
+            source,
+            ast,
+            root,
+            source_form,
+            root_env,
+            root_thunk,
+        });
+        module
+    }
+
+    fn find_module(&self, key: &str) -> Option<ModuleId> {
+        self.modules
+            .iter()
+            .position(|module| module.key == key)
+            .map(|index| ModuleId(index as u32))
+    }
+
+    fn load_import(&mut self, current: ModuleId, specifier: &str) -> Result<ModuleId> {
+        let current_key = self.modules[current.0 as usize].key.clone();
+        let LoadedSource { key, name, source } = self.loader.load(Some(&current_key), specifier)?;
+        self.add_source(key, name, &source)
+    }
+
+    fn eval_expr(&mut self, reference: ExprRef, env: EnvId) -> Result<RuntimeValue> {
+        let span = self.expr_span(reference);
+        let expr = self.expr(reference).clone();
         match expr {
             Expr::Literal(literal) => Ok(RuntimeValue::Concrete(literal_to_concrete(literal))),
             Expr::Ident(name) => self.eval_ident(&name, env, span),
-            Expr::Object(fields) => self.eval_object(&fields, env),
+            Expr::Object(fields) => self
+                .build_object(reference.module, &fields, env)
+                .map(|object| RuntimeValue::Concrete(ConcreteValue::Object(object))),
             Expr::Array(items) => {
                 let thunks = items
                     .into_iter()
-                    .map(|item| self.add_expr_thunk(item, env))
+                    .map(|item| {
+                        self.add_expr_thunk(
+                            ExprRef {
+                                module: reference.module,
+                                expr: item,
+                            },
+                            env,
+                        )
+                    })
                     .collect();
                 Ok(RuntimeValue::Concrete(ConcreteValue::Array(thunks)))
             }
             Expr::Let { bindings, body } => {
-                let let_env = self.new_child_env(env);
+                let let_env = self.new_env(Some(env));
                 for binding in bindings {
                     let name = field_name(&binding)?;
-                    let thunk = self.add_expr_thunk(binding.value, let_env);
+                    let thunk = self.add_expr_thunk(
+                        ExprRef {
+                            module: reference.module,
+                            expr: binding.value,
+                        },
+                        let_env,
+                    );
                     self.bind(let_env, name, thunk);
                 }
-                self.eval_expr(body, let_env)
+                self.eval_expr(
+                    ExprRef {
+                        module: reference.module,
+                        expr: body,
+                    },
+                    let_env,
+                )
+            }
+            Expr::Import(specifier) => {
+                let module = self.load_import(reference.module, &specifier)?;
+                self.eval_module(module)
             }
-            Expr::Import(_) => Err(Diagnostic::new(
-                DiagnosticKind::UnsupportedFeature,
-                span,
-                "import evaluation is not implemented yet",
-            )),
             Expr::Path { base, field } => {
-                let base = self.eval_expr(base, env)?;
+                let base = self.eval_expr(
+                    ExprRef {
+                        module: reference.module,
+                        expr: base,
+                    },
+                    env,
+                )?;
                 let RuntimeValue::Concrete(ConcreteValue::Object(object)) = base else {
                     return Err(Diagnostic::new(
                         DiagnosticKind::TypeMismatch,
@@ -135,24 +256,59 @@ impl Engine {
                 };
                 self.force(thunk)
             }
-            Expr::Call { callee, args } => self.eval_call(callee, args, env, span),
+            Expr::Call { callee, args } => self.eval_call(
+                ExprRef {
+                    module: reference.module,
+                    expr: callee,
+                },
+                args,
+                reference.module,
+                env,
+                span,
+            ),
             Expr::Function { params, body } => {
                 let params = params
                     .into_iter()
                     .map(|param| FunctionParam {
                         name: param.name,
-                        constraint: param.constraint,
+                        constraint: param.constraint.map(|expr| ExprRef {
+                            module: reference.module,
+                            expr,
+                        }),
                     })
                     .collect();
                 Ok(RuntimeValue::Concrete(ConcreteValue::Function(
-                    FunctionValue { params, body, env },
+                    FunctionValue {
+                        params,
+                        body: ExprRef {
+                            module: reference.module,
+                            expr: body,
+                        },
+                        env,
+                    },
                 )))
             }
             Expr::Match { scrutinee, arms } => {
-                let value = self.eval_expr(scrutinee, env)?;
+                let value = self.eval_expr(
+                    ExprRef {
+                        module: reference.module,
+                        expr: scrutinee,
+                    },
+                    env,
+                )?;
                 for arm in arms {
-                    if self.matches_pattern(&value, arm.pattern, env)? {
+                    let pattern = ExprRef {
-                        return self.eval_expr(arm.body, env);
+                        module: reference.module,
+                        expr: arm.pattern,
+                    };
+                    if self.matches_pattern(&value, pattern, env)? {
+                        return self.eval_expr(
+                            ExprRef {
+                                module: reference.module,
+                                expr: arm.body,
+                            },
+                            env,
+                        );
                     }
                 }
                 Err(Diagnostic::new(
@@ -162,25 +318,53 @@ impl Engine {
                 ))
             }
             Expr::Binary { op, lhs, rhs } => {
-                let lhs = self.eval_expr(lhs, env)?;
+                let lhs = self.eval_expr(
-                let rhs = self.eval_expr(rhs, env)?;
+                    ExprRef {
+                        module: reference.module,
+                        expr: lhs,
+                    },
+                    env,
+                )?;
+                let rhs = self.eval_expr(
+                    ExprRef {
+                        module: reference.module,
+                        expr: rhs,
+                    },
+                    env,
+                )?;
                 match op {
                     BinaryOp::And => self.compose_and(lhs, rhs, span),
                     BinaryOp::Patch => self.patch(lhs, rhs),
                 }
             }
             Expr::Default { base, fallback } => {
-                let base = self.eval_expr(base, env)?;
+                let base = self.eval_expr(
+                    ExprRef {
+                        module: reference.module,
+                        expr: base,
+                    },
+                    env,
+                )?;
                 match base {
                     RuntimeValue::Abstract(mut abstract_value) => {
-                        abstract_value.default = Some(self.add_expr_thunk(fallback, env));
+                        abstract_value.default = Some(self.add_expr_thunk(
+                            ExprRef {
+                                module: reference.module,
+                                expr: fallback,
+                            },
+                            env,
+                        ));
                         Ok(RuntimeValue::Abstract(abstract_value))
                     }
                     concrete @ RuntimeValue::Concrete(_) => Ok(concrete),
                 }
             }
             Expr::CompareConstraint { op, value } => {
-                let value = self.eval_expr(value, env)?;
+                let value_ref = ExprRef {
+                    module: reference.module,
+                    expr: value,
+                };
+                let value = self.eval_expr(value_ref, env)?;
                 let value = literal_value_from_runtime(&value).ok_or_else(|| {
                     Diagnostic::new(
                         DiagnosticKind::TypeMismatch,
@@ -228,17 +412,30 @@ impl Engine {
         ))
     }
 
-    fn eval_object(&mut self, fields: &[Field], env: EnvId) -> Result<RuntimeValue> {
+    fn build_object(
+        &mut self,
+        module: ModuleId,
+        fields: &[Field],
+        env: EnvId,
+    ) -> Result<ObjectValue> {
         let mut object = ObjectValue { fields: Vec::new() };
         for field in fields {
-            self.insert_field(&mut object, &field.path, field.value, env, field.span)?;
+            self.insert_field(
+                &mut object,
+                module,
+                &field.path,
+                field.value,
+                env,
+                field.span,
+            )?;
         }
-        Ok(RuntimeValue::Concrete(ConcreteValue::Object(object)))
+        Ok(object)
     }
 
     fn insert_field(
         &mut self,
         object: &mut ObjectValue,
+        module: ModuleId,
         path: &[String],
         expr: ExprId,
         env: EnvId,
@@ -252,7 +449,7 @@ impl Engine {
             ));
         }
         if path.len() == 1 {
-            let value = self.add_expr_thunk(expr, env);
+            let value = self.add_expr_thunk(ExprRef { module, expr }, env);
             if object.fields.iter().any(|field| field.name == path[0]) {
                 return Err(Diagnostic::new(
                     DiagnosticKind::Conflict,
@@ -277,14 +474,14 @@ impl Engine {
                     format!("field `{name}` is already defined as a non-object"),
                 ));
             };
-            self.insert_field(&mut nested, &path[1..], expr, env, span)?;
+            self.insert_field(&mut nested, module, &path[1..], expr, env, span)?;
             object.fields[index].value =
                 self.add_value_thunk(RuntimeValue::Concrete(ConcreteValue::Object(nested)));
             return Ok(());
         }
 
         let mut nested = ObjectValue { fields: Vec::new() };
-        self.insert_field(&mut nested, &path[1..], expr, env, span)?;
+        self.insert_field(&mut nested, module, &path[1..], expr, env, span)?;
         object.fields.push(ObjectField {
             name: name.clone(),
             value: self.add_value_thunk(RuntimeValue::Concrete(ConcreteValue::Object(nested))),
@@ -294,8 +491,9 @@ impl Engine {
 
     fn eval_call(
         &mut self,
-        callee: ExprId,
+        callee: ExprRef,
         args: Vec<ExprId>,
+        caller_module: ModuleId,
         caller_env: EnvId,
         span: Span,
     ) -> Result<RuntimeValue> {
@@ -314,12 +512,16 @@ impl Engine {
                 "function call argument count mismatch",
             ));
         }
-        let call_env = self.new_child_env(function.env);
+        let call_env = self.new_env(Some(function.env));
         for (param, arg) in function.params.iter().zip(args) {
+            let arg_ref = ExprRef {
+                module: caller_module,
+                expr: arg,
+            };
             let arg_thunk = if let Some(constraint) = param.constraint {
-                self.add_constrained_thunk(constraint, function.env, arg, caller_env)
+                self.add_constrained_thunk(constraint, function.env, arg_ref, caller_env)
             } else {
-                self.add_expr_thunk(arg, caller_env)
+                self.add_expr_thunk(arg_ref, caller_env)
             };
             self.bind(call_env, param.name.clone(), arg_thunk);
         }
@@ -329,10 +531,10 @@ impl Engine {
     fn matches_pattern(
         &mut self,
         value: &RuntimeValue,
-        pattern: ExprId,
+        pattern: ExprRef,
         env: EnvId,
     ) -> Result<bool> {
-        match self.ast.get(pattern).expr.clone() {
+        match self.expr(pattern).clone() {
             Expr::Wildcard => Ok(true),
             Expr::CompareConstraint { .. } | Expr::RegexConstraint(_) | Expr::Ident(_) => {
                 let constraint = self.eval_expr(pattern, env)?;
@@ -341,7 +543,7 @@ impl Engine {
                         .ensure_satisfies(
                             value,
                             &abstract_value.constraints,
-                            self.ast.span(pattern),
+                            self.expr_span(pattern),
                         )
                         .map(|_| true)
                         .or_else(|diag| match diag.kind {
@@ -529,7 +731,7 @@ impl Engine {
                 value,
                 value_env,
             } => {
-                let span = self.ast.span(value);
+                let span = self.expr_span(value);
                 let constraint = self.eval_expr(constraint, constraint_env)?;
                 let value = self.eval_expr(value, value_env)?;
                 self.compose_and(constraint, value, span)
@@ -548,15 +750,15 @@ impl Engine {
         }
     }
 
-    fn add_expr_thunk(&mut self, expr: ExprId, env: EnvId) -> ThunkId {
+    fn add_expr_thunk(&mut self, expr: ExprRef, env: EnvId) -> ThunkId {
         self.add_thunk(ThunkKind::Expr { expr, env })
     }
 
     fn add_constrained_thunk(
         &mut self,
-        constraint: ExprId,
+        constraint: ExprRef,
         constraint_env: EnvId,
-        value: ExprId,
+        value: ExprRef,
         value_env: EnvId,
     ) -> ThunkId {
         self.add_thunk(ThunkKind::Constrained {
@@ -580,10 +782,10 @@ impl Engine {
         id
     }
 
-    fn new_child_env(&mut self, parent: EnvId) -> EnvId {
+    fn new_env(&mut self, parent: Option<EnvId>) -> EnvId {
         let id = EnvId(self.envs.len() as u32);
         self.envs.push(Env {
-            parent: Some(parent),
+            parent,
             bindings: Vec::new(),
         });
         id
@@ -611,6 +813,19 @@ impl Engine {
         }
         None
     }
+
+    fn expr(&self, reference: ExprRef) -> &Expr {
+        &self.modules[reference.module.0 as usize]
+            .ast
+            .get(reference.expr)
+            .expr
+    }
+
+    fn expr_span(&self, reference: ExprRef) -> Span {
+        self.modules[reference.module.0 as usize]
+            .ast
+            .span(reference.expr)
+    }
 }
 
 fn field_name(field: &Field) -> Result<String> {
@@ -733,7 +948,7 @@ fn merge_default(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::parse_source;
+    use crate::{LoadedSource, parse_source};
 
     fn eval_data(source: &str) -> Data {
         let parsed = parse_source(source).unwrap();
@@ -769,4 +984,65 @@ mod tests {
         let mut engine = Engine::from_parse(parsed.ast, parsed.root);
         assert!(engine.eval_root().is_err());
     }
+
+    #[derive(Default)]
+    struct MapLoader {
+        sources: Vec<(String, String)>,
+    }
+
+    impl SourceLoader for MapLoader {
+        fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> Result<LoadedSource> {
+            let source = self
+                .sources
+                .iter()
+                .find(|(key, _)| key == specifier)
+                .map(|(_, source)| source.clone())
+                .ok_or_else(|| {
+                    Diagnostic::new(DiagnosticKind::Import, Span::default(), "missing source")
+                })?;
+            Ok(LoadedSource {
+                key: specifier.into(),
+                name: specifier.into(),
+                source,
+            })
+        }
+    }
+
+    #[test]
+    fn imports_module_on_demand() {
+        let mut engine = Engine::new(MapLoader {
+            sources: vec![(
+                String::from("dep"),
+                String::from("schema = { port = Int default 8080; }"),
+            )],
+        });
+        let module = engine
+            .add_root_source(
+                "main",
+                "main",
+                r#"(import "dep").schema & { port = 9000; }"#,
+            )
+            .unwrap();
+        let value = engine.eval_module(module).unwrap();
+        let data = engine.materialize(&value).unwrap();
+        let Data::Object(fields) = data else { panic!() };
+        assert_eq!(fields[0].value, Data::Int(9000));
+    }
+
+    #[test]
+    fn top_level_fields_are_recursive_module_scope() {
+        let mut engine = Engine::new(EmptyLoader);
+        let module = engine
+            .add_root_source(
+                "main",
+                "main",
+                "schema = { port = Int default 8080; }; result = schema;",
+            )
+            .unwrap();
+        let value = engine.eval_module(module).unwrap();
+        let data = engine.materialize(&value).unwrap();
+        let Data::Object(fields) = data else { panic!() };
+        assert_eq!(fields[1].name, "result");
+        assert!(matches!(fields[1].value, Data::Object(_)));
+    }
 }

crates/decodal-core/src/lexer.rs

@@ -1,6 +1,6 @@
 use alloc::{string::String, vec::Vec};
 
-use crate::{Diagnostic, Span, diagnostic::Result};
+use crate::{Diagnostic, SourceId, Span, diagnostic::Result};
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct Token {
@@ -45,6 +45,7 @@ pub enum TokenKind {
 }
 
 pub struct Lexer<'a> {
+    source_id: SourceId,
     source: &'a str,
     bytes: &'a [u8],
     pos: usize,
@@ -52,7 +53,12 @@ pub struct Lexer<'a> {
 
 impl<'a> Lexer<'a> {
     pub fn new(source: &'a str) -> Self {
+        Self::with_source_id(SourceId(0), source)
+    }
+
+    pub fn with_source_id(source_id: SourceId, source: &'a str) -> Self {
         Self {
+            source_id,
             source,
             bytes: source.as_bytes(),
             pos: 0,
@@ -77,7 +83,7 @@ impl<'a> Lexer<'a> {
         let Some(ch) = self.peek() else {
             return Ok(Token {
                 kind: TokenKind::Eof,
-                span: Span::empty(self.pos),
+                span: self.empty_span(self.pos),
             });
         };
 
@@ -167,7 +173,7 @@ impl<'a> Lexer<'a> {
             c if is_ident_start(c) => self.lex_ident_or_keyword(),
             _ => {
                 return Err(Diagnostic::syntax(
-                    Span::new(start, start + 1),
+                    self.span(start, start + 1),
                     "unexpected character",
                 ));
             }
@@ -175,7 +181,7 @@ impl<'a> Lexer<'a> {
 
         Ok(Token {
             kind,
-            span: Span::new(start, self.pos),
+            span: self.span(start, self.pos),
         })
     }
 
@@ -208,7 +214,7 @@ impl<'a> Lexer<'a> {
                 b'\\' => {
                     let Some(escaped) = self.peek() else {
                         return Err(Diagnostic::syntax(
-                            Span::new(start, self.pos),
+                            self.span(start, self.pos),
                             "unterminated escape",
                         ));
                     };
@@ -227,7 +233,7 @@ impl<'a> Lexer<'a> {
             }
         }
         Err(Diagnostic::syntax(
-            Span::new(start, self.pos),
+            self.span(start, self.pos),
             "unterminated string",
         ))
     }
@@ -252,7 +258,7 @@ impl<'a> Lexer<'a> {
             }
         }
         Err(Diagnostic::syntax(
-            Span::new(start, self.pos),
+            self.span(start, self.pos),
             "unterminated regex",
         ))
     }
@@ -273,12 +279,12 @@ impl<'a> Lexer<'a> {
         let text = &self.source[start..self.pos];
         if is_float {
             text.parse::<f64>().map(TokenKind::Float).map_err(|_| {
-                Diagnostic::syntax(Span::new(start, self.pos), "invalid float literal")
+                Diagnostic::syntax(self.span(start, self.pos), "invalid float literal")
             })
         } else {
             text.parse::<i64>()
                 .map(TokenKind::Int)
-                .map_err(|_| Diagnostic::syntax(Span::new(start, self.pos), "invalid int literal"))
+                .map_err(|_| Diagnostic::syntax(self.span(start, self.pos), "invalid int literal"))
         }
     }
 
@@ -301,6 +307,14 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    fn span(&self, start: usize, end: usize) -> Span {
+        Span::new(self.source_id, start, end)
+    }
+
+    fn empty_span(&self, offset: usize) -> Span {
+        Span::empty(self.source_id, offset)
+    }
+
     fn peek(&self) -> Option<u8> {
         self.bytes.get(self.pos).copied()
     }

crates/decodal-core/src/lib.rs

@@ -6,6 +6,7 @@ pub mod ast;
 pub mod diagnostic;
 pub mod eval;
 pub mod lexer;
+pub mod module;
 pub mod parser;
 pub mod runtime;
 pub mod span;
@@ -14,9 +15,10 @@ pub use ast::{Ast, BinaryOp, CompareOp, Expr, ExprId, Field, Literal, Param};
 pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
 pub use eval::Engine;
 pub use lexer::{Lexer, Token, TokenKind};
-pub use parser::{ParseOutput, Parser, parse_source};
+pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
-pub use runtime::{Data, RuntimeValue};
+pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
-pub use span::Span;
+pub use runtime::{Data, ExprRef, ModuleId, RuntimeValue};
+pub use span::{SourceId, Span};
 
 pub fn version() -> &'static str {
     env!("CARGO_PKG_VERSION")

crates/decodal-core/src/module.rs

@@ -0,0 +1,42 @@
+use alloc::string::String;
+
+use crate::{
+    Ast, ExprId, SourceForm, SourceId,
+    runtime::{EnvId, ThunkId},
+};
+
+#[derive(Debug, Clone)]
+pub struct Module {
+    pub key: String,
+    pub name: String,
+    pub source: SourceId,
+    pub ast: Ast,
+    pub root: ExprId,
+    pub source_form: SourceForm,
+    pub root_env: EnvId,
+    pub root_thunk: ThunkId,
+}
+
+#[derive(Debug, Clone)]
+pub struct LoadedSource {
+    pub key: String,
+    pub name: String,
+    pub source: String,
+}
+
+pub trait SourceLoader {
+    fn load(&mut self, current_key: Option<&str>, specifier: &str) -> crate::Result<LoadedSource>;
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct EmptyLoader;
+
+impl SourceLoader for EmptyLoader {
+    fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> crate::Result<LoadedSource> {
+        Err(crate::Diagnostic::new(
+            crate::DiagnosticKind::Import,
+            crate::Span::default(),
+            alloc::format!("no source loader is configured for import `{specifier}`"),
+        ))
+    }
+}

crates/decodal-core/src/parser.rs

@@ -1,7 +1,7 @@
 use alloc::{string::String, vec::Vec};
 
 use crate::{
-    Span,
+    SourceId, Span,
     ast::{Ast, BinaryOp, CompareOp, Expr, ExprId, Field, Literal, MatchArm, Param},
     diagnostic::{Diagnostic, Result},
     lexer::{Lexer, Token, TokenKind},
@@ -11,10 +11,21 @@ use crate::{
 pub struct ParseOutput {
     pub ast: Ast,
     pub root: ExprId,
+    pub source_form: SourceForm,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SourceForm {
+    Expr,
+    Fields,
 }
 
 pub fn parse_source(source: &str) -> Result<ParseOutput> {
-    let tokens = Lexer::new(source).tokenize()?;
+    parse_source_with_source_id(SourceId(0), source)
+}
+
+pub fn parse_source_with_source_id(source_id: SourceId, source: &str) -> Result<ParseOutput> {
+    let tokens = Lexer::with_source_id(source_id, source).tokenize()?;
     Parser::new(tokens).parse()
 }
 
@@ -34,7 +45,7 @@ impl Parser {
     }
 
     pub fn parse(mut self) -> Result<ParseOutput> {
-        let root = if self.starts_field() {
+        let (root, source_form) = if self.starts_field() {
             let fields = self.parse_fields_until_eof()?;
             let span = fields
                 .first()
@@ -43,16 +54,20 @@ impl Parser {
                         .iter()
                         .fold(f.span, |acc, field| acc.join(field.span))
                 })
-                .unwrap_or_else(|| Span::empty(0));
+                .unwrap_or_else(|| self.peek().span);
-            self.ast.push(Expr::Object(fields), span)
+            (
+                self.ast.push(Expr::Object(fields), span),
+                SourceForm::Fields,
+            )
         } else {
             let expr = self.parse_expr(0)?;
             self.expect_eof()?;
-            expr
+            (expr, SourceForm::Expr)
         };
         Ok(ParseOutput {
             ast: self.ast,
             root,
+            source_form,
         })
     }
 

crates/decodal-core/src/runtime.rs

@@ -2,6 +2,15 @@ use alloc::{string::String, vec::Vec};
 
 use crate::{ExprId, ast::CompareOp};
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct ModuleId(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct ExprRef {
+    pub module: ModuleId,
+    pub expr: ExprId,
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub enum RuntimeValue {
     Concrete(ConcreteValue),
@@ -33,14 +42,14 @@ pub struct ObjectField {
 #[derive(Debug, Clone, PartialEq)]
 pub struct FunctionValue {
     pub params: Vec<FunctionParam>,
-    pub body: ExprId,
+    pub body: ExprRef,
     pub env: EnvId,
 }
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct FunctionParam {
     pub name: String,
-    pub constraint: Option<ExprId>,
+    pub constraint: Option<ExprRef>,
 }
 
 #[derive(Debug, Clone, PartialEq)]
@@ -104,13 +113,13 @@ pub struct Thunk {
 #[derive(Debug, Clone)]
 pub enum ThunkKind {
     Expr {
-        expr: ExprId,
+        expr: ExprRef,
         env: EnvId,
     },
     Constrained {
-        constraint: ExprId,
+        constraint: ExprRef,
         constraint_env: EnvId,
-        value: ExprId,
+        value: ExprRef,
         value_env: EnvId,
     },
     Value(RuntimeValue),

crates/decodal-core/src/span.rs

@@ -1,23 +1,29 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub struct SourceId(pub u32);
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub struct Span {
+    pub source: SourceId,
     pub start: u32,
     pub end: u32,
 }
 
 impl Span {
-    pub const fn new(start: usize, end: usize) -> Self {
+    pub const fn new(source: SourceId, start: usize, end: usize) -> Self {
         Self {
+            source,
             start: start as u32,
             end: end as u32,
         }
     }
 
-    pub const fn empty(offset: usize) -> Self {
+    pub const fn empty(source: SourceId, offset: usize) -> Self {
-        Self::new(offset, offset)
+        Self::new(source, offset, offset)
     }
 
     pub fn join(self, other: Self) -> Self {
         Self {
+            source: self.source,
             start: self.start.min(other.start),
             end: self.end.max(other.end),
         }

doc/manual/souce/design/execution-pipeline.md

@@ -54,23 +54,30 @@ desugar は、意味論を単純にするための表層構文変換を行う。
 
 ## module registry
 
-module registry は、読み込んだ module を canonical path で管理する。
+module registry は、読み込んだ module を loader が返す安定 key で管理する。
+CLI では canonical path を key とする。
+組み込み利用では、resource name や static source table の key を使える。
 
 ```text
 ModuleRegistry:
-  CanonicalPath -> ModuleId
+  ModuleKey -> ModuleId
 ```
 
 処理系は、まず root module を parse / desugar して registry に登録する。
 import 先 module は、この段階で全て読み込む必要はない。
 
-import expression が評価されたとき、module registry は path を解決し、未登録なら対象 module を parse / desugar して登録する。
+import expression が評価されたとき、処理系は `SourceLoader` に現在の module key と import specifier を渡す。
+loader は module key、表示名、source text を返す。
+module registry は key が未登録なら対象 module を parse / desugar して登録する。
 登録された module は module root thunk を持つ。
 同じ module が複数回 import された場合は、同じ `ModuleId` を返す。
 
 つまり import は module を即時評価しない。
 module を読み込み、module root を thunk として登録するだけにする。
 
+AST の `ExprId` は module-local である。
+そのため runtime が保持する式参照は `ExprRef { module, expr }` として module-qualified にする。
+
 ## demand-driven evaluation
 
 評価器は、必要になった thunk だけを force する。

doc/manual/souce/design/thunk-and-lazy-evaluation.md

@@ -7,11 +7,16 @@ thunk は、まだ評価していない式をあとで評価できるように
 
 ```text
 Thunk {
-  expr: ExprId
+  expr: ExprRef
   env: EnvId
   state: ThunkState
 }
 
+ExprRef {
+  module: ModuleId
+  expr: ExprId
+}
+
 ThunkState =
   Unevaluated
   Evaluating
@@ -19,7 +24,8 @@ ThunkState =
   Error(Diagnostic)
 ```
 
-`expr` は評価対象の AST node を指す。
+`expr` は評価対象の AST node を module-qualified に指す。
+`ExprId` は module-local な ID なので、runtime では `ModuleId` と組み合わせた `ExprRef` を保持する。
 `env` は、その式を評価するときに使う lexical environment を指す。
 
 式だけではなく environment も保持するのは、遅延評価された式が定義時の名前解決文脈を必要とするためである。

doc/manual/souce/language/modules-and-imports.md

@@ -17,6 +17,29 @@ import 先はモジュール単位で読み込まれる。
 ただし、モジュール全体を即時評価する必要はない。
 各フィールドは thunk として保持され、必要になったときだけ評価される。
 
+top-level に field 定義列を書いた module は、recursive module scope を作る。
+つまり、top-level field は同じ module の他の top-level field から識別子として参照できる。
+
+```dcdl
+schema = {
+    hoge = String;
+};
+
+result = schema;
+```
+
+この場合、`result` の右辺の `schema` は同じ module の top-level field `schema` を参照する。
+通常の object literal 内の field を暗黙に recursive scope にするかは別仕様とする。
+
+## SourceLoader
+
+`import` specifier の解決は処理系 core ではなく host 側の `SourceLoader` が行う。
+CLI では、specifier を現在の module path からの相対 path として解決する。
+組み込み利用では、resource table や static source map など、filesystem 以外の loader を使える。
+
+module cache の key は loader が返す安定 key を使う。
+CLI では canonical path を key とする。
+
 ## 循環 import
 
 モジュール間に循環参照があっても、必要なフィールドの依存関係が循環していなければ評価できる。
@@ -42,7 +65,7 @@ import 先はモジュール単位で読み込まれる。
 }
 ```
 
-`func.n` は `main.n` を import しているが、参照しているのは `main.schema` である。
+`func.dcdl` は `main.dcdl` を import しているが、参照しているのは `main.schema` である。
 `main.schema` が `main.result` に依存していなければ、この循環 import は成立する。
 
 ## import の評価単位

examples/import/main.dcdl

@@ -0,0 +1,6 @@
+let
+  schema = import "./schema.dcdl";
+in
+  schema.MyConfig & {
+    host = "localhost";
+  }

examples/import/schema.dcdl

@@ -0,0 +1,4 @@
+MyConfig = {
+  host = String;
+  port = Int & > 443 default 8080;
+};