Unify formatting on canonical syntax

crates/decodal-core/src/ast.rs

@@ -70,6 +70,13 @@ pub enum Expr {
         params: Vec<Param>,
         body: ExprId,
     },
+    /// An explicitly parenthesized expression.
+    ///
+    /// Keeping this node preserves the delimiters and their trivia for source
+    /// tools while evaluation remains equivalent to evaluating `expr`.
+    Parenthesized {
+        expr: ExprId,
+    },
     Match {
         scrutinee: ExprId,
         arms: Vec<MatchArm>,

crates/decodal-core/src/eval.rs

@@ -471,6 +471,13 @@ impl<L: ImportLoader> Engine<L> {
                     },
                 )))
             }
+            Expr::Parenthesized { expr } => self.eval_expr(
+                ExprRef {
+                    module: reference.module,
+                    expr,
+                },
+                env,
+            ),
             Expr::Match { scrutinee, arms } => {
                 let value = self.eval_expr(
                     ExprRef {
@@ -827,6 +834,14 @@ impl<L: ImportLoader> Engine<L> {
     ) -> Result<bool> {
         match self.expr(pattern).clone() {
             Expr::Wildcard => Ok(true),
+            Expr::Parenthesized { expr } => self.matches_pattern(
+                value,
+                ExprRef {
+                    module: pattern.module,
+                    expr,
+                },
+                env,
+            ),
             Expr::ArrayConstraint { .. }
             | Expr::CompareConstraint { .. }
             | Expr::RegexConstraint(_)
@@ -2043,6 +2058,13 @@ mod tests {
         assert_eq!(fields[4].value, Data::Int(-2));
     }
 
+    #[test]
+    fn parenthesized_wildcard_remains_a_match_pattern() {
+        let data = eval_data("result = match 1 { (_): 2; };");
+        let Data::Object(fields) = data else { panic!() };
+        assert_eq!(fields[0].value, Data::Int(2));
+    }
+
     #[test]
     fn arithmetic_can_feed_constraints() {
         let data = eval_data("port = Int & > 4000 + 42 default 8080;");

crates/decodal-core/src/lexer.rs

@@ -52,9 +52,25 @@ pub enum TokenKind {
     Gte,
     Lt,
     Lte,
+    /// A line comment, including its leading `#` and excluding its newline.
+    Comment,
     Eof,
 }
 
+/// Tokenizes source while retaining comments and source spans for tooling.
+///
+/// Whitespace remains available through the gaps between adjacent token spans,
+/// making this a lossless syntax view when paired with the original source.
+pub fn tokenize_source(source: &str) -> Result<Vec<Token>> {
+    tokenize_source_with_source_id(SourceId(0), source)
+}
+
+/// Tokenizes source with a caller-provided source identifier while retaining
+/// comments and source spans for tooling.
+pub fn tokenize_source_with_source_id(source_id: SourceId, source: &str) -> Result<Vec<Token>> {
+    Lexer::with_source_id(source_id, source).tokenize_with_comments()
+}
+
 pub struct Lexer<'a> {
     source_id: SourceId,
     source: &'a str,
@@ -78,10 +94,26 @@ impl<'a> Lexer<'a> {
     }
 
     pub fn tokenize(mut self) -> Result<Vec<Token>> {
+        self.tokenize_impl(false)
+    }
+
+    fn tokenize_with_comments(mut self) -> Result<Vec<Token>> {
+        self.tokenize_impl(true)
+    }
+
+    fn tokenize_impl(&mut self, include_comments: bool) -> Result<Vec<Token>> {
         let mut tokens = Vec::new();
         let mut previous = None;
         loop {
-            let token = self.next_token(previous.as_ref())?;
+            self.skip_whitespace();
+            if self.peek() == Some(b'#') {
+                let comment = self.lex_comment();
+                if include_comments {
+                    tokens.push(comment);
+                }
+                continue;
+            }
+            let token = self.next_non_ws_token(previous.as_ref())?;
             let is_eof = token.kind == TokenKind::Eof;
             if !is_eof {
                 previous = Some(token.kind.clone());
@@ -93,11 +125,6 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
-        self.skip_ws_and_comments();
-        self.next_non_ws_token(previous)
-    }
-
     fn next_non_ws_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
         let start = self.pos;
         let Some(ch) = self.peek() else {
@@ -259,21 +286,23 @@ impl<'a> Lexer<'a> {
         })
     }
 
-    fn skip_ws_and_comments(&mut self) {
-        loop {
-            while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
-                self.pos += 1;
+    fn skip_whitespace(&mut self) {
+        while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
+            self.pos += 1;
+        }
+    }
+
+    fn lex_comment(&mut self) -> Token {
+        let start = self.pos;
+        while let Some(c) = self.peek() {
+            if c == b'\n' {
+                break;
             }
-            if self.peek() == Some(b'#') {
-                while let Some(c) = self.peek() {
-                    self.pos += 1;
-                    if c == b'\n' {
-                        break;
-                    }
-                }
-                continue;
-            }
-            break;
+            self.pos += 1;
+        }
+        Token {
+            kind: TokenKind::Comment,
+            span: self.span(start, self.pos),
         }
     }
 
@@ -453,4 +482,30 @@ mod tests {
         assert!(matches!(tokens[2].kind, TokenKind::Ident(_)));
         assert_eq!(tokens[3].kind, TokenKind::RBracket);
     }
+
+    #[test]
+    fn tooling_tokens_retain_comments_and_whitespace_gaps() {
+        let source = "value = 1; # trailing\n\n# leading\nnext = 2;";
+        let tokens = tokenize_source(source).unwrap();
+        let comments = tokens
+            .iter()
+            .filter(|token| token.kind == TokenKind::Comment)
+            .collect::<Vec<_>>();
+        assert_eq!(comments.len(), 2);
+        assert_eq!(
+            &source[comments[0].span.start as usize..comments[0].span.end as usize],
+            "# trailing"
+        );
+        assert!(
+            source[comments[0].span.end as usize..comments[1].span.start as usize].contains("\n\n")
+        );
+
+        let source_id = SourceId(7);
+        let identified = tokenize_source_with_source_id(source_id, "# comment").unwrap();
+        assert!(
+            identified
+                .iter()
+                .all(|token| token.span.source == source_id)
+        );
+    }
 }

crates/decodal-core/src/lib.rs

@@ -23,6 +23,10 @@ pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
 pub use embedding::{HostField, HostValue};
 pub use environment::HostEnvironment;
 pub use eval::{Engine, format_diagnostic_with};
+pub use lexer::{
+    Token as SyntaxToken, TokenKind as SyntaxTokenKind, tokenize_source,
+    tokenize_source_with_source_id,
+};
 pub use module::{
     EmptyLoader, ImportCandidate, ImportLoader, LoadedImport, LoadedSource, LoadedValue, Module,
 };

crates/decodal-core/src/parser.rs

@@ -38,7 +38,10 @@ pub struct Parser {
 impl Parser {
     pub fn new(tokens: Vec<Token>) -> Self {
         Self {
-            tokens,
+            tokens: tokens
+                .into_iter()
+                .filter(|token| token.kind != TokenKind::Comment)
+                .collect(),
             pos: 0,
             ast: Ast::new(),
         }
@@ -380,8 +383,9 @@ impl Parser {
         }
 
         let expr = self.parse_expr(0)?;
-        self.expect_kind(&TokenKind::RParen, "expected ')' after expression")?;
-        Ok(expr)
+        let end_span = self.expect_kind(&TokenKind::RParen, "expected ')' after expression")?;
+        let span = start_span.join(end_span);
+        Ok(self.ast.push(Expr::Parenthesized { expr }, span))
     }
 
     fn parse_params_after_lparen(&mut self) -> Result<Vec<Param>> {
@@ -692,7 +696,24 @@ mod tests {
         let Expr::ArrayConstraint { item } = parsed.ast.get(parsed.root).expr else {
             panic!()
         };
-        assert!(matches!(parsed.ast.get(item).expr, Expr::Binary { .. }));
+        let Expr::Parenthesized { expr } = parsed.ast.get(item).expr else {
+            panic!()
+        };
+        assert!(matches!(parsed.ast.get(expr).expr, Expr::Binary { .. }));
+        assert_eq!(parsed.ast.span(item), Span::new(SourceId(0), 4, 22));
+    }
+
+    #[test]
+    fn parser_accepts_the_public_lossless_token_stream() {
+        let tokens = crate::tokenize_source("value = (# note\n 1);").unwrap();
+        let parsed = Parser::new(tokens).parse().unwrap();
+        let Expr::Object(fields) = &parsed.ast.get(parsed.root).expr else {
+            panic!()
+        };
+        assert!(matches!(
+            parsed.ast.get(fields[0].value).expr,
+            Expr::Parenthesized { .. }
+        ));
     }
 
     #[test]

crates/decodal-language-tools/Cargo.toml

@@ -7,7 +7,7 @@ license.workspace = true
 repository.workspace = true
 readme.workspace = true
 description = "Source-level language tooling for Decodal."
-keywords = ["decodal", "formatter", "lsp", "tree-sitter"]
+keywords = ["decodal", "formatter", "lsp", "language-tools"]
 categories = ["development-tools", "text-processing"]
 
 [lib]
@@ -16,8 +16,6 @@ crate-type = ["cdylib", "rlib"]
 [dependencies]
 decodal = { version = "0.1.3", path = "../decodal-core" }
 serde_json.workspace = true
-tree-sitter = "0.22.6"
-tree-sitter-decodal = { version = "0.1.0", path = "../../editors/tree-sitter-decodal" }
 wasm-bindgen.workspace = true
 
 [package.metadata.wasm-pack.profile.release]