Expose tokenizer for web highlighting

crates/decodal-core/src/lexer.rs

@@ -54,6 +54,66 @@ pub enum TokenKind {
     Eof,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PublicToken {
+    pub kind: PublicTokenKind,
+    pub span: Span,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PublicTokenKind {
+    Ident,
+    Int,
+    Float,
+    String,
+    Regex,
+    True,
+    False,
+    Let,
+    In,
+    Match,
+    Import,
+    Default,
+    Underscore,
+    LBrace,
+    RBrace,
+    LBracket,
+    RBracket,
+    LParen,
+    RParen,
+    Semicolon,
+    Comma,
+    Dot,
+    Colon,
+    Equal,
+    EqualEqual,
+    Bang,
+    BangEqual,
+    Arrow,
+    Amp,
+    AmpAmp,
+    PipePipe,
+    Plus,
+    PlusPlus,
+    Minus,
+    Star,
+    Slash,
+    SlashSlash,
+    Gt,
+    Gte,
+    Lt,
+    Lte,
+    Comment,
+}
+
+pub fn tokenize_source(source: &str) -> Result<Vec<PublicToken>> {
+    tokenize_source_with_id(SourceId(0), source)
+}
+
+pub fn tokenize_source_with_id(source_id: SourceId, source: &str) -> Result<Vec<PublicToken>> {
+    Lexer::with_source_id(source_id, source).tokenize_public()
+}
+
 pub struct Lexer<'a> {
     source_id: SourceId,
     source: &'a str,
@@ -91,8 +151,26 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    pub fn tokenize_public(mut self) -> Result<Vec<PublicToken>> {
+        let mut tokens = Vec::new();
+        let mut previous = None;
+        loop {
+            let Some(token) = self.next_public_token(previous.as_ref())? else {
+                return Ok(tokens);
+            };
+            if token.kind != PublicTokenKind::Comment {
+                previous = Some(token.kind);
+            }
+            tokens.push(token);
+        }
+    }
+
     fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
         self.skip_ws_and_comments();
+        self.next_non_ws_token(previous)
+    }
+
+    fn next_non_ws_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
         let start = self.pos;
         let Some(ch) = self.peek() else {
             return Ok(Token {
@@ -242,6 +320,42 @@ impl<'a> Lexer<'a> {
         })
     }
 
+    fn next_public_token(
+        &mut self,
+        previous: Option<&PublicTokenKind>,
+    ) -> Result<Option<PublicToken>> {
+        self.skip_ws();
+        let start = self.pos;
+        let Some(ch) = self.peek() else {
+            return Ok(None);
+        };
+        if ch == b'#' {
+            self.pos += 1;
+            while !matches!(self.peek(), None | Some(b'\n')) {
+                self.pos += 1;
+            }
+            return Ok(Some(PublicToken {
+                kind: PublicTokenKind::Comment,
+                span: self.span(start, self.pos),
+            }));
+        }
+        let previous_token = previous.map(public_kind_to_token_kind);
+        let token = self.next_non_ws_token(previous_token.as_ref())?;
+        if token.kind == TokenKind::Eof {
+            return Ok(None);
+        }
+        Ok(Some(PublicToken {
+            kind: PublicTokenKind::from_token_kind(&token.kind),
+            span: token.span,
+        }))
+    }
+
+    fn skip_ws(&mut self) {
+        while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
+            self.pos += 1;
+        }
+    }
+
     fn skip_ws_and_comments(&mut self) {
         loop {
             while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
@@ -390,6 +504,102 @@ impl<'a> Lexer<'a> {
     }
 }
 
+impl PublicTokenKind {
+    fn from_token_kind(kind: &TokenKind) -> Self {
+        match kind {
+            TokenKind::Ident(_) => Self::Ident,
+            TokenKind::Int(_) => Self::Int,
+            TokenKind::Float(_) => Self::Float,
+            TokenKind::String(_) => Self::String,
+            TokenKind::Regex(_) => Self::Regex,
+            TokenKind::True => Self::True,
+            TokenKind::False => Self::False,
+            TokenKind::Let => Self::Let,
+            TokenKind::In => Self::In,
+            TokenKind::Match => Self::Match,
+            TokenKind::Import => Self::Import,
+            TokenKind::Default => Self::Default,
+            TokenKind::Underscore => Self::Underscore,
+            TokenKind::LBrace => Self::LBrace,
+            TokenKind::RBrace => Self::RBrace,
+            TokenKind::LBracket => Self::LBracket,
+            TokenKind::RBracket => Self::RBracket,
+            TokenKind::LParen => Self::LParen,
+            TokenKind::RParen => Self::RParen,
+            TokenKind::Semicolon => Self::Semicolon,
+            TokenKind::Comma => Self::Comma,
+            TokenKind::Dot => Self::Dot,
+            TokenKind::Colon => Self::Colon,
+            TokenKind::Equal => Self::Equal,
+            TokenKind::EqualEqual => Self::EqualEqual,
+            TokenKind::Bang => Self::Bang,
+            TokenKind::BangEqual => Self::BangEqual,
+            TokenKind::Arrow => Self::Arrow,
+            TokenKind::Amp => Self::Amp,
+            TokenKind::AmpAmp => Self::AmpAmp,
+            TokenKind::PipePipe => Self::PipePipe,
+            TokenKind::Plus => Self::Plus,
+            TokenKind::PlusPlus => Self::PlusPlus,
+            TokenKind::Minus => Self::Minus,
+            TokenKind::Star => Self::Star,
+            TokenKind::Slash => Self::Slash,
+            TokenKind::SlashSlash => Self::SlashSlash,
+            TokenKind::Gt => Self::Gt,
+            TokenKind::Gte => Self::Gte,
+            TokenKind::Lt => Self::Lt,
+            TokenKind::Lte => Self::Lte,
+            TokenKind::Eof => unreachable!(),
+        }
+    }
+}
+
+fn public_kind_to_token_kind(kind: &PublicTokenKind) -> TokenKind {
+    match kind {
+        PublicTokenKind::Ident => TokenKind::Ident(String::new()),
+        PublicTokenKind::Int => TokenKind::Int(0),
+        PublicTokenKind::Float => TokenKind::Float(0.0),
+        PublicTokenKind::String => TokenKind::String(String::new()),
+        PublicTokenKind::Regex => TokenKind::Regex(String::new()),
+        PublicTokenKind::True => TokenKind::True,
+        PublicTokenKind::False => TokenKind::False,
+        PublicTokenKind::Let => TokenKind::Let,
+        PublicTokenKind::In => TokenKind::In,
+        PublicTokenKind::Match => TokenKind::Match,
+        PublicTokenKind::Import => TokenKind::Import,
+        PublicTokenKind::Default => TokenKind::Default,
+        PublicTokenKind::Underscore => TokenKind::Underscore,
+        PublicTokenKind::LBrace => TokenKind::LBrace,
+        PublicTokenKind::RBrace => TokenKind::RBrace,
+        PublicTokenKind::LBracket => TokenKind::LBracket,
+        PublicTokenKind::RBracket => TokenKind::RBracket,
+        PublicTokenKind::LParen => TokenKind::LParen,
+        PublicTokenKind::RParen => TokenKind::RParen,
+        PublicTokenKind::Semicolon => TokenKind::Semicolon,
+        PublicTokenKind::Comma => TokenKind::Comma,
+        PublicTokenKind::Dot => TokenKind::Dot,
+        PublicTokenKind::Colon => TokenKind::Colon,
+        PublicTokenKind::Equal => TokenKind::Equal,
+        PublicTokenKind::EqualEqual => TokenKind::EqualEqual,
+        PublicTokenKind::Bang => TokenKind::Bang,
+        PublicTokenKind::BangEqual => TokenKind::BangEqual,
+        PublicTokenKind::Arrow => TokenKind::Arrow,
+        PublicTokenKind::Amp => TokenKind::Amp,
+        PublicTokenKind::AmpAmp => TokenKind::AmpAmp,
+        PublicTokenKind::PipePipe => TokenKind::PipePipe,
+        PublicTokenKind::Plus => TokenKind::Plus,
+        PublicTokenKind::PlusPlus => TokenKind::PlusPlus,
+        PublicTokenKind::Minus => TokenKind::Minus,
+        PublicTokenKind::Star => TokenKind::Star,
+        PublicTokenKind::Slash => TokenKind::Slash,
+        PublicTokenKind::SlashSlash => TokenKind::SlashSlash,
+        PublicTokenKind::Gt => TokenKind::Gt,
+        PublicTokenKind::Gte => TokenKind::Gte,
+        PublicTokenKind::Lt => TokenKind::Lt,
+        PublicTokenKind::Lte => TokenKind::Lte,
+        PublicTokenKind::Comment => TokenKind::Underscore,
+    }
+}
+
 fn is_ident_start(c: u8) -> bool {
     c.is_ascii_alphabetic()
 }
@@ -427,4 +637,19 @@ mod tests {
         assert_eq!(tokens[3].kind, TokenKind::Amp);
         assert_eq!(tokens[4].kind, TokenKind::Gte);
     }
+
+    #[test]
+    fn public_tokenization_keeps_comments_and_spans() {
+        let source = "# hello\nport = [1] ++ [2];";
+        let tokens = tokenize_source(source).unwrap();
+        assert_eq!(tokens[0].kind, PublicTokenKind::Comment);
+        let comment = &source[tokens[0].span.start as usize..tokens[0].span.end as usize];
+        assert_eq!(comment, "# hello");
+        assert_eq!(tokens[1].kind, PublicTokenKind::Ident);
+        assert!(
+            tokens
+                .iter()
+                .any(|token| token.kind == PublicTokenKind::PlusPlus)
+        );
+    }
 }

crates/decodal-core/src/lib.rs

@@ -21,7 +21,9 @@ pub use decodal_derive::Decodal;
 pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
 pub use embedding::{HostField, HostValue};
 pub use eval::{Engine, format_diagnostic_with};
-pub use lexer::{Lexer, Token, TokenKind};
+pub use lexer::{
+    Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id,
+};
 pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
 pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
 pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};

crates/decodal-wasm/src/lib.rs

@@ -1,8 +1,8 @@
 use std::collections::BTreeMap;
 
 use decodal::{
-    Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader,
+    Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, PublicTokenKind, SourceId,
-    Span, format_diagnostic_with,
+    SourceLoader, Span, format_diagnostic_with, tokenize_source,
 };
 use wasm_bindgen::prelude::*;
 
@@ -16,6 +16,33 @@ pub fn evaluate_project(entry: &str, files_json: &str) -> String {
     encode_result(evaluate_project_inner(entry, files_json))
 }
 
+#[wasm_bindgen(js_name = tokenizeSource)]
+pub fn tokenize_source_json(source: &str) -> String {
+    match tokenize_source(source) {
+        Ok(tokens) => {
+            let mut out = String::from("{\"ok\":true,\"tokens\":[");
+            for (index, token) in tokens.iter().enumerate() {
+                if index > 0 {
+                    out.push(',');
+                }
+                out.push_str("{\"kind\":");
+                out.push_str(&json_string(public_token_kind_name(token.kind)));
+                out.push_str(",\"start\":");
+                out.push_str(&token.span.start.to_string());
+                out.push_str(",\"end\":");
+                out.push_str(&token.span.end.to_string());
+                out.push('}');
+            }
+            out.push_str("]}");
+            out
+        }
+        Err(error) => format!(
+            "{{\"ok\":false,\"error\":{}}}",
+            json_string(&format_diagnostic_with_root(&error, "playground"))
+        ),
+    }
+}
+
 fn encode_result(result: Result<String, String>) -> String {
     match result {
         Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)),
@@ -145,6 +172,53 @@ fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str
     })
 }
 
+fn public_token_kind_name(kind: PublicTokenKind) -> &'static str {
+    match kind {
+        PublicTokenKind::Ident => "ident",
+        PublicTokenKind::Int => "int",
+        PublicTokenKind::Float => "float",
+        PublicTokenKind::String => "string",
+        PublicTokenKind::Regex => "regex",
+        PublicTokenKind::True => "true",
+        PublicTokenKind::False => "false",
+        PublicTokenKind::Let => "let",
+        PublicTokenKind::In => "in",
+        PublicTokenKind::Match => "match",
+        PublicTokenKind::Import => "import",
+        PublicTokenKind::Default => "default",
+        PublicTokenKind::Underscore => "underscore",
+        PublicTokenKind::LBrace => "l_brace",
+        PublicTokenKind::RBrace => "r_brace",
+        PublicTokenKind::LBracket => "l_bracket",
+        PublicTokenKind::RBracket => "r_bracket",
+        PublicTokenKind::LParen => "l_paren",
+        PublicTokenKind::RParen => "r_paren",
+        PublicTokenKind::Semicolon => "semicolon",
+        PublicTokenKind::Comma => "comma",
+        PublicTokenKind::Dot => "dot",
+        PublicTokenKind::Colon => "colon",
+        PublicTokenKind::Equal => "equal",
+        PublicTokenKind::EqualEqual => "equal_equal",
+        PublicTokenKind::Bang => "bang",
+        PublicTokenKind::BangEqual => "bang_equal",
+        PublicTokenKind::Arrow => "arrow",
+        PublicTokenKind::Amp => "amp",
+        PublicTokenKind::AmpAmp => "amp_amp",
+        PublicTokenKind::PipePipe => "pipe_pipe",
+        PublicTokenKind::Plus => "plus",
+        PublicTokenKind::PlusPlus => "plus_plus",
+        PublicTokenKind::Minus => "minus",
+        PublicTokenKind::Star => "star",
+        PublicTokenKind::Slash => "slash",
+        PublicTokenKind::SlashSlash => "slash_slash",
+        PublicTokenKind::Gt => "gt",
+        PublicTokenKind::Gte => "gte",
+        PublicTokenKind::Lt => "lt",
+        PublicTokenKind::Lte => "lte",
+        PublicTokenKind::Comment => "comment",
+    }
+}
+
 fn format_data(data: &Data, indent: usize) -> String {
     match data {
         Data::String(value) => json_string(value),
@@ -231,6 +305,13 @@ mod tests {
         );
     }
 
+    #[test]
+    fn tokenizes_source_for_web() {
+        let output = super::tokenize_source_json("# hi\nport = [1] ++ [2];");
+        assert!(output.contains("\"kind\":\"comment\""));
+        assert!(output.contains("\"kind\":\"plus_plus\""));
+    }
+
     #[test]
     fn evaluates_project_imports() {
         let files = r#"{

doc/manual/souce/development.md

@@ -60,6 +60,7 @@ site/decodal-site/src/pages/docs/[...slug].astro
 site/decodal-site/src/pages/playground.astro
 site/decodal-site/src/layouts/ManualLayout.astro
 site/decodal-site/src/lib/docs.js
+site/decodal-site/src/lib/highlight.js
 crates/decodal-wasm/src/lib.rs
 ```
 
@@ -80,6 +81,9 @@ site/decodal-site/src/wasm/
 
 These generated files are committed so the site can be built without requiring every consumer to regenerate the wasm package first.
 
+The playground editor uses `tokenizeSource` from `decodal-wasm` for token spans and maps those tokens to HTML classes in `src/lib/highlight.js`.
+The documentation build still uses the lightweight JavaScript fallback highlighter so Astro can render Markdown without initializing WASM at build time.
+
 To run the site locally:
 
 ```sh

site/decodal-site/src/lib/highlight.js

@@ -25,6 +25,66 @@ export function highlightCode(code, language = '') {
   return escapeHtml(code);
 }
 
+export function highlightDecodalTokens(source, tokens) {
+  let html = '';
+  let cursor = 0;
+  for (const token of tokens) {
+    if (token.start > cursor) html += escapeHtml(source.slice(cursor, token.start));
+    const text = source.slice(token.start, token.end);
+    const kind = tokenClass(token.kind, text);
+    html += kind ? `<span class="${kind}">${escapeHtml(text)}</span>` : escapeHtml(text);
+    cursor = token.end;
+  }
+  if (cursor < source.length) html += escapeHtml(source.slice(cursor));
+  return html;
+}
+
+function tokenClass(kind, text) {
+  if (['let', 'in', 'fn', 'match', 'import', 'default'].includes(kind)) return 'tok-keyword';
+  if (kind === 'ident' && ['String', 'Int', 'Float', 'Bool', 'Array'].includes(text)) return 'tok-type';
+  if (kind === 'ident') return '';
+  if (['true', 'false'].includes(kind)) return 'tok-literal';
+  if (['int', 'float'].includes(kind)) return 'tok-number';
+  if (kind === 'string') return 'tok-string';
+  if (kind === 'regex') return 'tok-regex';
+  if (kind === 'comment') return 'tok-comment';
+  if (
+    [
+      'equal',
+      'equal_equal',
+      'bang',
+      'bang_equal',
+      'arrow',
+      'amp',
+      'amp_amp',
+      'pipe_pipe',
+      'plus',
+      'plus_plus',
+      'minus',
+      'star',
+      'slash',
+      'slash_slash',
+      'gt',
+      'gte',
+      'lt',
+      'lte',
+      'dot',
+      'colon',
+      'semicolon',
+      'comma',
+      'l_brace',
+      'r_brace',
+      'l_bracket',
+      'r_bracket',
+      'l_paren',
+      'r_paren',
+    ].includes(kind)
+  ) {
+    return 'tok-operator';
+  }
+  return '';
+}
+
 export function highlightDecodal(source) {
   let html = '';
   let index = 0;

site/decodal-site/src/scripts/playground.js

@@ -1,5 +1,5 @@
-import init, { evaluateProject } from '../wasm/decodal_wasm.js';
+import init, { evaluateProject, tokenizeSource } from '../wasm/decodal_wasm.js';
-import { highlightDecodal } from '../lib/highlight.js';
+import { highlightDecodal, highlightDecodalTokens } from '../lib/highlight.js';
 import { playgroundExamples } from './playground-examples.js';
 
 const STORAGE_KEY = 'decodal-playground-project-v1';
@@ -18,6 +18,7 @@ const exampleSelect = document.getElementById('example-select');
 const loadExample = document.getElementById('load-example');
 
 const project = loadProject();
+let wasmReady = false;
 
 for (const example of playgroundExamples) {
   const option = document.createElement('option');
@@ -97,7 +98,18 @@ function setActiveFile(path) {
 }
 
 function updateHighlight() {
-  sourceHighlight.innerHTML = `${highlightDecodal(source.value)}\n`;
+  sourceHighlight.innerHTML = `${highlightSource(source.value)}\n`;
+}
+
+function highlightSource(value) {
+  if (!wasmReady) return highlightDecodal(value);
+  try {
+    const result = JSON.parse(tokenizeSource(value));
+    if (result.ok) return highlightDecodalTokens(value, result.tokens);
+  } catch (_error) {
+    // Fall back to the lightweight JavaScript highlighter.
+  }
+  return highlightDecodal(value);
 }
 
 function syncHighlightScroll() {
@@ -165,6 +177,8 @@ function compareNodes(a, b) {
 
 try {
   await init();
+  wasmReady = true;
+  updateHighlight();
   run.disabled = false;
   status.textContent = '';
   execute();

site/decodal-site/src/wasm/decodal_wasm.d.ts

@@ -5,12 +5,15 @@ export function evaluate(source: string): string;
 
 export function evaluateProject(entry: string, files_json: string): string;
 
+export function tokenizeSource(source: string): string;
+
 export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
 
 export interface InitOutput {
     readonly memory: WebAssembly.Memory;
     readonly evaluate: (a: number, b: number) => [number, number];
     readonly evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
+    readonly tokenizeSource: (a: number, b: number) => [number, number];
     readonly __wbindgen_externrefs: WebAssembly.Table;
     readonly __wbindgen_malloc: (a: number, b: number) => number;
     readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;

site/decodal-site/src/wasm/decodal_wasm.js

@@ -40,6 +40,25 @@ export function evaluateProject(entry, files_json) {
         wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
     }
 }
+
+/**
+ * @param {string} source
+ * @returns {string}
+ */
+export function tokenizeSource(source) {
+    let deferred2_0;
+    let deferred2_1;
+    try {
+        const ptr0 = passStringToWasm0(source, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.tokenizeSource(ptr0, len0);
+        deferred2_0 = ret[0];
+        deferred2_1 = ret[1];
+        return getStringFromWasm0(ret[0], ret[1]);
+    } finally {
+        wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+    }
+}
 function __wbg_get_imports() {
     const import0 = {
         __proto__: null,

site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts

@@ -3,6 +3,7 @@
 export const memory: WebAssembly.Memory;
 export const evaluate: (a: number, b: number) => [number, number];
 export const evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
+export const tokenizeSource: (a: number, b: number) => [number, number];
 export const __wbindgen_externrefs: WebAssembly.Table;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;