Unify formatting on canonical syntax

crates/decodal-core/src/lexer.rs

@@ -52,9 +52,25 @@ pub enum TokenKind {
     Gte,
     Lt,
     Lte,
+    /// A line comment, including its leading `#` and excluding its newline.
+    Comment,
     Eof,
 }
 
+/// Tokenizes source while retaining comments and source spans for tooling.
+///
+/// Whitespace remains available through the gaps between adjacent token spans,
+/// making this a lossless syntax view when paired with the original source.
+pub fn tokenize_source(source: &str) -> Result<Vec<Token>> {
+    tokenize_source_with_source_id(SourceId(0), source)
+}
+
+/// Tokenizes source with a caller-provided source identifier while retaining
+/// comments and source spans for tooling.
+pub fn tokenize_source_with_source_id(source_id: SourceId, source: &str) -> Result<Vec<Token>> {
+    Lexer::with_source_id(source_id, source).tokenize_with_comments()
+}
+
 pub struct Lexer<'a> {
     source_id: SourceId,
     source: &'a str,
@@ -78,10 +94,26 @@ impl<'a> Lexer<'a> {
     }
 
     pub fn tokenize(mut self) -> Result<Vec<Token>> {
+        self.tokenize_impl(false)
+    }
+
+    fn tokenize_with_comments(mut self) -> Result<Vec<Token>> {
+        self.tokenize_impl(true)
+    }
+
+    fn tokenize_impl(&mut self, include_comments: bool) -> Result<Vec<Token>> {
         let mut tokens = Vec::new();
         let mut previous = None;
         loop {
-            let token = self.next_token(previous.as_ref())?;
+            self.skip_whitespace();
+            if self.peek() == Some(b'#') {
+                let comment = self.lex_comment();
+                if include_comments {
+                    tokens.push(comment);
+                }
+                continue;
+            }
+            let token = self.next_non_ws_token(previous.as_ref())?;
             let is_eof = token.kind == TokenKind::Eof;
             if !is_eof {
                 previous = Some(token.kind.clone());
@@ -93,11 +125,6 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
-        self.skip_ws_and_comments();
-        self.next_non_ws_token(previous)
-    }
-
     fn next_non_ws_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
         let start = self.pos;
         let Some(ch) = self.peek() else {
@@ -259,21 +286,23 @@ impl<'a> Lexer<'a> {
         })
     }
 
-    fn skip_ws_and_comments(&mut self) {
-        loop {
-            while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
-                self.pos += 1;
+    fn skip_whitespace(&mut self) {
+        while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
+            self.pos += 1;
+        }
+    }
+
+    fn lex_comment(&mut self) -> Token {
+        let start = self.pos;
+        while let Some(c) = self.peek() {
+            if c == b'\n' {
+                break;
             }
-            if self.peek() == Some(b'#') {
-                while let Some(c) = self.peek() {
-                    self.pos += 1;
-                    if c == b'\n' {
-                        break;
-                    }
-                }
-                continue;
-            }
-            break;
+            self.pos += 1;
+        }
+        Token {
+            kind: TokenKind::Comment,
+            span: self.span(start, self.pos),
         }
     }
 
@@ -453,4 +482,30 @@ mod tests {
         assert!(matches!(tokens[2].kind, TokenKind::Ident(_)));
         assert_eq!(tokens[3].kind, TokenKind::RBracket);
     }
+
+    #[test]
+    fn tooling_tokens_retain_comments_and_whitespace_gaps() {
+        let source = "value = 1; # trailing\n\n# leading\nnext = 2;";
+        let tokens = tokenize_source(source).unwrap();
+        let comments = tokens
+            .iter()
+            .filter(|token| token.kind == TokenKind::Comment)
+            .collect::<Vec<_>>();
+        assert_eq!(comments.len(), 2);
+        assert_eq!(
+            &source[comments[0].span.start as usize..comments[0].span.end as usize],
+            "# trailing"
+        );
+        assert!(
+            source[comments[0].span.end as usize..comments[1].span.start as usize].contains("\n\n")
+        );
+
+        let source_id = SourceId(7);
+        let identified = tokenize_source_with_source_id(source_id, "# comment").unwrap();
+        assert!(
+            identified
+                .iter()
+                .all(|token| token.span.source == source_id)
+        );
+    }
 }