Implement module loading and imports

crates/decodal-core/src/lexer.rs

@@ -1,6 +1,6 @@
 use alloc::{string::String, vec::Vec};
 
-use crate::{Diagnostic, Span, diagnostic::Result};
+use crate::{Diagnostic, SourceId, Span, diagnostic::Result};
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct Token {
@@ -45,6 +45,7 @@ pub enum TokenKind {
 }
 
 pub struct Lexer<'a> {
+    source_id: SourceId,
     source: &'a str,
     bytes: &'a [u8],
     pos: usize,
@@ -52,7 +53,12 @@ pub struct Lexer<'a> {
 
 impl<'a> Lexer<'a> {
     pub fn new(source: &'a str) -> Self {
+        Self::with_source_id(SourceId(0), source)
+    }
+
+    pub fn with_source_id(source_id: SourceId, source: &'a str) -> Self {
         Self {
+            source_id,
             source,
             bytes: source.as_bytes(),
             pos: 0,
@@ -77,7 +83,7 @@ impl<'a> Lexer<'a> {
         let Some(ch) = self.peek() else {
             return Ok(Token {
                 kind: TokenKind::Eof,
-                span: Span::empty(self.pos),
+                span: self.empty_span(self.pos),
             });
         };
 
@@ -167,7 +173,7 @@ impl<'a> Lexer<'a> {
             c if is_ident_start(c) => self.lex_ident_or_keyword(),
             _ => {
                 return Err(Diagnostic::syntax(
-                    Span::new(start, start + 1),
+                    self.span(start, start + 1),
                     "unexpected character",
                 ));
             }
@@ -175,7 +181,7 @@ impl<'a> Lexer<'a> {
 
         Ok(Token {
             kind,
-            span: Span::new(start, self.pos),
+            span: self.span(start, self.pos),
         })
     }
 
@@ -208,7 +214,7 @@ impl<'a> Lexer<'a> {
                 b'\\' => {
                     let Some(escaped) = self.peek() else {
                         return Err(Diagnostic::syntax(
-                            Span::new(start, self.pos),
+                            self.span(start, self.pos),
                             "unterminated escape",
                         ));
                     };
@@ -227,7 +233,7 @@ impl<'a> Lexer<'a> {
             }
         }
         Err(Diagnostic::syntax(
-            Span::new(start, self.pos),
+            self.span(start, self.pos),
             "unterminated string",
         ))
     }
@@ -252,7 +258,7 @@ impl<'a> Lexer<'a> {
             }
         }
         Err(Diagnostic::syntax(
-            Span::new(start, self.pos),
+            self.span(start, self.pos),
             "unterminated regex",
         ))
     }
@@ -273,12 +279,12 @@ impl<'a> Lexer<'a> {
         let text = &self.source[start..self.pos];
         if is_float {
             text.parse::<f64>().map(TokenKind::Float).map_err(|_| {
-                Diagnostic::syntax(Span::new(start, self.pos), "invalid float literal")
+                Diagnostic::syntax(self.span(start, self.pos), "invalid float literal")
             })
         } else {
             text.parse::<i64>()
                 .map(TokenKind::Int)
-                .map_err(|_| Diagnostic::syntax(Span::new(start, self.pos), "invalid int literal"))
+                .map_err(|_| Diagnostic::syntax(self.span(start, self.pos), "invalid int literal"))
         }
     }
 
@@ -301,6 +307,14 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    fn span(&self, start: usize, end: usize) -> Span {
+        Span::new(self.source_id, start, end)
+    }
+
+    fn empty_span(&self, offset: usize) -> Span {
+        Span::empty(self.source_id, offset)
+    }
+
     fn peek(&self) -> Option<u8> {
         self.bytes.get(self.pos).copied()
     }