diff --git a/crates/decodal-core/src/lexer.rs b/crates/decodal-core/src/lexer.rs index 496919a..8d43efb 100644 --- a/crates/decodal-core/src/lexer.rs +++ b/crates/decodal-core/src/lexer.rs @@ -54,6 +54,66 @@ pub enum TokenKind { Eof, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicToken { + pub kind: PublicTokenKind, + pub span: Span, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PublicTokenKind { + Ident, + Int, + Float, + String, + Regex, + True, + False, + Let, + In, + Match, + Import, + Default, + Underscore, + LBrace, + RBrace, + LBracket, + RBracket, + LParen, + RParen, + Semicolon, + Comma, + Dot, + Colon, + Equal, + EqualEqual, + Bang, + BangEqual, + Arrow, + Amp, + AmpAmp, + PipePipe, + Plus, + PlusPlus, + Minus, + Star, + Slash, + SlashSlash, + Gt, + Gte, + Lt, + Lte, + Comment, +} + +pub fn tokenize_source(source: &str) -> Result> { + tokenize_source_with_id(SourceId(0), source) +} + +pub fn tokenize_source_with_id(source_id: SourceId, source: &str) -> Result> { + Lexer::with_source_id(source_id, source).tokenize_public() +} + pub struct Lexer<'a> { source_id: SourceId, source: &'a str, @@ -91,8 +151,26 @@ impl<'a> Lexer<'a> { } } + pub fn tokenize_public(mut self) -> Result> { + let mut tokens = Vec::new(); + let mut previous = None; + loop { + let Some(token) = self.next_public_token(previous.as_ref())? else { + return Ok(tokens); + }; + if token.kind != PublicTokenKind::Comment { + previous = Some(token.kind); + } + tokens.push(token); + } + } + fn next_token(&mut self, previous: Option<&TokenKind>) -> Result { self.skip_ws_and_comments(); + self.next_non_ws_token(previous) + } + + fn next_non_ws_token(&mut self, previous: Option<&TokenKind>) -> Result { let start = self.pos; let Some(ch) = self.peek() else { return Ok(Token { @@ -242,6 +320,42 @@ impl<'a> Lexer<'a> { }) } + fn next_public_token( + &mut self, + previous: Option<&PublicTokenKind>, + ) -> Result> { + self.skip_ws(); + let start = self.pos; + let Some(ch) = self.peek() else { + return Ok(None); + }; + if ch == b'#' { + self.pos += 1; + while !matches!(self.peek(), None | Some(b'\n')) { + self.pos += 1; + } + return Ok(Some(PublicToken { + kind: PublicTokenKind::Comment, + span: self.span(start, self.pos), + })); + } + let previous_token = previous.map(public_kind_to_token_kind); + let token = self.next_non_ws_token(previous_token.as_ref())?; + if token.kind == TokenKind::Eof { + return Ok(None); + } + Ok(Some(PublicToken { + kind: PublicTokenKind::from_token_kind(&token.kind), + span: token.span, + })) + } + + fn skip_ws(&mut self) { + while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) { + self.pos += 1; + } + } + fn skip_ws_and_comments(&mut self) { loop { while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) { @@ -390,6 +504,102 @@ impl<'a> Lexer<'a> { } } +impl PublicTokenKind { + fn from_token_kind(kind: &TokenKind) -> Self { + match kind { + TokenKind::Ident(_) => Self::Ident, + TokenKind::Int(_) => Self::Int, + TokenKind::Float(_) => Self::Float, + TokenKind::String(_) => Self::String, + TokenKind::Regex(_) => Self::Regex, + TokenKind::True => Self::True, + TokenKind::False => Self::False, + TokenKind::Let => Self::Let, + TokenKind::In => Self::In, + TokenKind::Match => Self::Match, + TokenKind::Import => Self::Import, + TokenKind::Default => Self::Default, + TokenKind::Underscore => Self::Underscore, + TokenKind::LBrace => Self::LBrace, + TokenKind::RBrace => Self::RBrace, + TokenKind::LBracket => Self::LBracket, + TokenKind::RBracket => Self::RBracket, + TokenKind::LParen => Self::LParen, + TokenKind::RParen => Self::RParen, + TokenKind::Semicolon => Self::Semicolon, + TokenKind::Comma => Self::Comma, + TokenKind::Dot => Self::Dot, + TokenKind::Colon => Self::Colon, + TokenKind::Equal => Self::Equal, + TokenKind::EqualEqual => Self::EqualEqual, + TokenKind::Bang => Self::Bang, + TokenKind::BangEqual => Self::BangEqual, + TokenKind::Arrow => Self::Arrow, + TokenKind::Amp => Self::Amp, + TokenKind::AmpAmp => Self::AmpAmp, + TokenKind::PipePipe => Self::PipePipe, + TokenKind::Plus => Self::Plus, + TokenKind::PlusPlus => Self::PlusPlus, + TokenKind::Minus => Self::Minus, + TokenKind::Star => Self::Star, + TokenKind::Slash => Self::Slash, + TokenKind::SlashSlash => Self::SlashSlash, + TokenKind::Gt => Self::Gt, + TokenKind::Gte => Self::Gte, + TokenKind::Lt => Self::Lt, + TokenKind::Lte => Self::Lte, + TokenKind::Eof => unreachable!(), + } + } +} + +fn public_kind_to_token_kind(kind: &PublicTokenKind) -> TokenKind { + match kind { + PublicTokenKind::Ident => TokenKind::Ident(String::new()), + PublicTokenKind::Int => TokenKind::Int(0), + PublicTokenKind::Float => TokenKind::Float(0.0), + PublicTokenKind::String => TokenKind::String(String::new()), + PublicTokenKind::Regex => TokenKind::Regex(String::new()), + PublicTokenKind::True => TokenKind::True, + PublicTokenKind::False => TokenKind::False, + PublicTokenKind::Let => TokenKind::Let, + PublicTokenKind::In => TokenKind::In, + PublicTokenKind::Match => TokenKind::Match, + PublicTokenKind::Import => TokenKind::Import, + PublicTokenKind::Default => TokenKind::Default, + PublicTokenKind::Underscore => TokenKind::Underscore, + PublicTokenKind::LBrace => TokenKind::LBrace, + PublicTokenKind::RBrace => TokenKind::RBrace, + PublicTokenKind::LBracket => TokenKind::LBracket, + PublicTokenKind::RBracket => TokenKind::RBracket, + PublicTokenKind::LParen => TokenKind::LParen, + PublicTokenKind::RParen => TokenKind::RParen, + PublicTokenKind::Semicolon => TokenKind::Semicolon, + PublicTokenKind::Comma => TokenKind::Comma, + PublicTokenKind::Dot => TokenKind::Dot, + PublicTokenKind::Colon => TokenKind::Colon, + PublicTokenKind::Equal => TokenKind::Equal, + PublicTokenKind::EqualEqual => TokenKind::EqualEqual, + PublicTokenKind::Bang => TokenKind::Bang, + PublicTokenKind::BangEqual => TokenKind::BangEqual, + PublicTokenKind::Arrow => TokenKind::Arrow, + PublicTokenKind::Amp => TokenKind::Amp, + PublicTokenKind::AmpAmp => TokenKind::AmpAmp, + PublicTokenKind::PipePipe => TokenKind::PipePipe, + PublicTokenKind::Plus => TokenKind::Plus, + PublicTokenKind::PlusPlus => TokenKind::PlusPlus, + PublicTokenKind::Minus => TokenKind::Minus, + PublicTokenKind::Star => TokenKind::Star, + PublicTokenKind::Slash => TokenKind::Slash, + PublicTokenKind::SlashSlash => TokenKind::SlashSlash, + PublicTokenKind::Gt => TokenKind::Gt, + PublicTokenKind::Gte => TokenKind::Gte, + PublicTokenKind::Lt => TokenKind::Lt, + PublicTokenKind::Lte => TokenKind::Lte, + PublicTokenKind::Comment => TokenKind::Underscore, + } +} + fn is_ident_start(c: u8) -> bool { c.is_ascii_alphabetic() } @@ -427,4 +637,19 @@ mod tests { assert_eq!(tokens[3].kind, TokenKind::Amp); assert_eq!(tokens[4].kind, TokenKind::Gte); } + + #[test] + fn public_tokenization_keeps_comments_and_spans() { + let source = "# hello\nport = [1] ++ [2];"; + let tokens = tokenize_source(source).unwrap(); + assert_eq!(tokens[0].kind, PublicTokenKind::Comment); + let comment = &source[tokens[0].span.start as usize..tokens[0].span.end as usize]; + assert_eq!(comment, "# hello"); + assert_eq!(tokens[1].kind, PublicTokenKind::Ident); + assert!( + tokens + .iter() + .any(|token| token.kind == PublicTokenKind::PlusPlus) + ); + } } diff --git a/crates/decodal-core/src/lib.rs b/crates/decodal-core/src/lib.rs index 0f7c47d..69a611c 100644 --- a/crates/decodal-core/src/lib.rs +++ b/crates/decodal-core/src/lib.rs @@ -21,7 +21,9 @@ pub use decodal_derive::Decodal; pub use diagnostic::{Diagnostic, DiagnosticKind, Result}; pub use embedding::{HostField, HostValue}; pub use eval::{Engine, format_diagnostic_with}; -pub use lexer::{Lexer, Token, TokenKind}; +pub use lexer::{ + Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id, +}; pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader}; pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id}; pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue}; diff --git a/crates/decodal-wasm/src/lib.rs b/crates/decodal-wasm/src/lib.rs index 1c5f7af..8e938eb 100644 --- a/crates/decodal-wasm/src/lib.rs +++ b/crates/decodal-wasm/src/lib.rs @@ -1,8 +1,8 @@ use std::collections::BTreeMap; use decodal::{ - Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader, - Span, format_diagnostic_with, + Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, PublicTokenKind, SourceId, + SourceLoader, Span, format_diagnostic_with, tokenize_source, }; use wasm_bindgen::prelude::*; @@ -16,6 +16,33 @@ pub fn evaluate_project(entry: &str, files_json: &str) -> String { encode_result(evaluate_project_inner(entry, files_json)) } +#[wasm_bindgen(js_name = tokenizeSource)] +pub fn tokenize_source_json(source: &str) -> String { + match tokenize_source(source) { + Ok(tokens) => { + let mut out = String::from("{\"ok\":true,\"tokens\":["); + for (index, token) in tokens.iter().enumerate() { + if index > 0 { + out.push(','); + } + out.push_str("{\"kind\":"); + out.push_str(&json_string(public_token_kind_name(token.kind))); + out.push_str(",\"start\":"); + out.push_str(&token.span.start.to_string()); + out.push_str(",\"end\":"); + out.push_str(&token.span.end.to_string()); + out.push('}'); + } + out.push_str("]}"); + out + } + Err(error) => format!( + "{{\"ok\":false,\"error\":{}}}", + json_string(&format_diagnostic_with_root(&error, "playground")) + ), + } +} + fn encode_result(result: Result) -> String { match result { Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)), @@ -145,6 +172,53 @@ fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str }) } +fn public_token_kind_name(kind: PublicTokenKind) -> &'static str { + match kind { + PublicTokenKind::Ident => "ident", + PublicTokenKind::Int => "int", + PublicTokenKind::Float => "float", + PublicTokenKind::String => "string", + PublicTokenKind::Regex => "regex", + PublicTokenKind::True => "true", + PublicTokenKind::False => "false", + PublicTokenKind::Let => "let", + PublicTokenKind::In => "in", + PublicTokenKind::Match => "match", + PublicTokenKind::Import => "import", + PublicTokenKind::Default => "default", + PublicTokenKind::Underscore => "underscore", + PublicTokenKind::LBrace => "l_brace", + PublicTokenKind::RBrace => "r_brace", + PublicTokenKind::LBracket => "l_bracket", + PublicTokenKind::RBracket => "r_bracket", + PublicTokenKind::LParen => "l_paren", + PublicTokenKind::RParen => "r_paren", + PublicTokenKind::Semicolon => "semicolon", + PublicTokenKind::Comma => "comma", + PublicTokenKind::Dot => "dot", + PublicTokenKind::Colon => "colon", + PublicTokenKind::Equal => "equal", + PublicTokenKind::EqualEqual => "equal_equal", + PublicTokenKind::Bang => "bang", + PublicTokenKind::BangEqual => "bang_equal", + PublicTokenKind::Arrow => "arrow", + PublicTokenKind::Amp => "amp", + PublicTokenKind::AmpAmp => "amp_amp", + PublicTokenKind::PipePipe => "pipe_pipe", + PublicTokenKind::Plus => "plus", + PublicTokenKind::PlusPlus => "plus_plus", + PublicTokenKind::Minus => "minus", + PublicTokenKind::Star => "star", + PublicTokenKind::Slash => "slash", + PublicTokenKind::SlashSlash => "slash_slash", + PublicTokenKind::Gt => "gt", + PublicTokenKind::Gte => "gte", + PublicTokenKind::Lt => "lt", + PublicTokenKind::Lte => "lte", + PublicTokenKind::Comment => "comment", + } +} + fn format_data(data: &Data, indent: usize) -> String { match data { Data::String(value) => json_string(value), @@ -231,6 +305,13 @@ mod tests { ); } + #[test] + fn tokenizes_source_for_web() { + let output = super::tokenize_source_json("# hi\nport = [1] ++ [2];"); + assert!(output.contains("\"kind\":\"comment\"")); + assert!(output.contains("\"kind\":\"plus_plus\"")); + } + #[test] fn evaluates_project_imports() { let files = r#"{ diff --git a/doc/manual/souce/development.md b/doc/manual/souce/development.md index 4f2a3cc..b4ea1ee 100644 --- a/doc/manual/souce/development.md +++ b/doc/manual/souce/development.md @@ -60,6 +60,7 @@ site/decodal-site/src/pages/docs/[...slug].astro site/decodal-site/src/pages/playground.astro site/decodal-site/src/layouts/ManualLayout.astro site/decodal-site/src/lib/docs.js +site/decodal-site/src/lib/highlight.js crates/decodal-wasm/src/lib.rs ``` @@ -80,6 +81,9 @@ site/decodal-site/src/wasm/ These generated files are committed so the site can be built without requiring every consumer to regenerate the wasm package first. +The playground editor uses `tokenizeSource` from `decodal-wasm` for token spans and maps those tokens to HTML classes in `src/lib/highlight.js`. +The documentation build still uses the lightweight JavaScript fallback highlighter so Astro can render Markdown without initializing WASM at build time. + To run the site locally: ```sh diff --git a/site/decodal-site/src/lib/highlight.js b/site/decodal-site/src/lib/highlight.js index aa5e7c3..7cfcf1a 100644 --- a/site/decodal-site/src/lib/highlight.js +++ b/site/decodal-site/src/lib/highlight.js @@ -25,6 +25,66 @@ export function highlightCode(code, language = '') { return escapeHtml(code); } +export function highlightDecodalTokens(source, tokens) { + let html = ''; + let cursor = 0; + for (const token of tokens) { + if (token.start > cursor) html += escapeHtml(source.slice(cursor, token.start)); + const text = source.slice(token.start, token.end); + const kind = tokenClass(token.kind, text); + html += kind ? `${escapeHtml(text)}` : escapeHtml(text); + cursor = token.end; + } + if (cursor < source.length) html += escapeHtml(source.slice(cursor)); + return html; +} + +function tokenClass(kind, text) { + if (['let', 'in', 'fn', 'match', 'import', 'default'].includes(kind)) return 'tok-keyword'; + if (kind === 'ident' && ['String', 'Int', 'Float', 'Bool', 'Array'].includes(text)) return 'tok-type'; + if (kind === 'ident') return ''; + if (['true', 'false'].includes(kind)) return 'tok-literal'; + if (['int', 'float'].includes(kind)) return 'tok-number'; + if (kind === 'string') return 'tok-string'; + if (kind === 'regex') return 'tok-regex'; + if (kind === 'comment') return 'tok-comment'; + if ( + [ + 'equal', + 'equal_equal', + 'bang', + 'bang_equal', + 'arrow', + 'amp', + 'amp_amp', + 'pipe_pipe', + 'plus', + 'plus_plus', + 'minus', + 'star', + 'slash', + 'slash_slash', + 'gt', + 'gte', + 'lt', + 'lte', + 'dot', + 'colon', + 'semicolon', + 'comma', + 'l_brace', + 'r_brace', + 'l_bracket', + 'r_bracket', + 'l_paren', + 'r_paren', + ].includes(kind) + ) { + return 'tok-operator'; + } + return ''; +} + export function highlightDecodal(source) { let html = ''; let index = 0; diff --git a/site/decodal-site/src/scripts/playground.js b/site/decodal-site/src/scripts/playground.js index b0b79b1..1d9c2b0 100644 --- a/site/decodal-site/src/scripts/playground.js +++ b/site/decodal-site/src/scripts/playground.js @@ -1,5 +1,5 @@ -import init, { evaluateProject } from '../wasm/decodal_wasm.js'; -import { highlightDecodal } from '../lib/highlight.js'; +import init, { evaluateProject, tokenizeSource } from '../wasm/decodal_wasm.js'; +import { highlightDecodal, highlightDecodalTokens } from '../lib/highlight.js'; import { playgroundExamples } from './playground-examples.js'; const STORAGE_KEY = 'decodal-playground-project-v1'; @@ -18,6 +18,7 @@ const exampleSelect = document.getElementById('example-select'); const loadExample = document.getElementById('load-example'); const project = loadProject(); +let wasmReady = false; for (const example of playgroundExamples) { const option = document.createElement('option'); @@ -97,7 +98,18 @@ function setActiveFile(path) { } function updateHighlight() { - sourceHighlight.innerHTML = `${highlightDecodal(source.value)}\n`; + sourceHighlight.innerHTML = `${highlightSource(source.value)}\n`; +} + +function highlightSource(value) { + if (!wasmReady) return highlightDecodal(value); + try { + const result = JSON.parse(tokenizeSource(value)); + if (result.ok) return highlightDecodalTokens(value, result.tokens); + } catch (_error) { + // Fall back to the lightweight JavaScript highlighter. + } + return highlightDecodal(value); } function syncHighlightScroll() { @@ -165,6 +177,8 @@ function compareNodes(a, b) { try { await init(); + wasmReady = true; + updateHighlight(); run.disabled = false; status.textContent = ''; execute(); diff --git a/site/decodal-site/src/wasm/decodal_wasm.d.ts b/site/decodal-site/src/wasm/decodal_wasm.d.ts index e88b72f..59b9615 100644 --- a/site/decodal-site/src/wasm/decodal_wasm.d.ts +++ b/site/decodal-site/src/wasm/decodal_wasm.d.ts @@ -5,12 +5,15 @@ export function evaluate(source: string): string; export function evaluateProject(entry: string, files_json: string): string; +export function tokenizeSource(source: string): string; + export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; export interface InitOutput { readonly memory: WebAssembly.Memory; readonly evaluate: (a: number, b: number) => [number, number]; readonly evaluateProject: (a: number, b: number, c: number, d: number) => [number, number]; + readonly tokenizeSource: (a: number, b: number) => [number, number]; readonly __wbindgen_externrefs: WebAssembly.Table; readonly __wbindgen_malloc: (a: number, b: number) => number; readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; diff --git a/site/decodal-site/src/wasm/decodal_wasm.js b/site/decodal-site/src/wasm/decodal_wasm.js index 800a76f..0f2ae92 100644 --- a/site/decodal-site/src/wasm/decodal_wasm.js +++ b/site/decodal-site/src/wasm/decodal_wasm.js @@ -40,6 +40,25 @@ export function evaluateProject(entry, files_json) { wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); } } + +/** + * @param {string} source + * @returns {string} + */ +export function tokenizeSource(source) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(source, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.tokenizeSource(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} function __wbg_get_imports() { const import0 = { __proto__: null, diff --git a/site/decodal-site/src/wasm/decodal_wasm_bg.wasm b/site/decodal-site/src/wasm/decodal_wasm_bg.wasm index b2f4a2f..ff3605f 100644 Binary files a/site/decodal-site/src/wasm/decodal_wasm_bg.wasm and b/site/decodal-site/src/wasm/decodal_wasm_bg.wasm differ diff --git a/site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts b/site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts index 4f894cc..7af5cbb 100644 --- a/site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts +++ b/site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts @@ -3,6 +3,7 @@ export const memory: WebAssembly.Memory; export const evaluate: (a: number, b: number) => [number, number]; export const evaluateProject: (a: number, b: number, c: number, d: number) => [number, number]; +export const tokenizeSource: (a: number, b: number) => [number, number]; export const __wbindgen_externrefs: WebAssembly.Table; export const __wbindgen_malloc: (a: number, b: number) => number; export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;