Hare/Decodal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expose tokenizer for web highlighting

Browse Source
This commit is contained in:
Keisuke Hirata 2026-07-08 19:50:04 +09:00
parent fe127428f9
commit a07f4c48aa
No known key found for this signature in database
10 changed files with 415 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
crates/decodal-core/src/lexer.rs
View File

@ -54,6 +54,66 @@ pub enum TokenKind {
Eof,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PublicToken {
pub kind: PublicTokenKind,
pub span: Span,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublicTokenKind {
Ident,
Int,
Float,
String,
Regex,
True,
False,
Let,
In,
Match,
Import,
Default,
Underscore,
LBrace,
RBrace,
LBracket,
RBracket,
LParen,
RParen,
Semicolon,
Comma,
Dot,
Colon,
Equal,
EqualEqual,
Bang,
BangEqual,
Arrow,
Amp,
AmpAmp,
PipePipe,
Plus,
PlusPlus,
Minus,
Star,
Slash,
SlashSlash,
Gt,
Gte,
Lt,
Lte,
Comment,
}
pub fn tokenize_source(source: &str) -> Result<Vec<PublicToken>> {
tokenize_source_with_id(SourceId(0), source)
}
pub fn tokenize_source_with_id(source_id: SourceId, source: &str) -> Result<Vec<PublicToken>> {
Lexer::with_source_id(source_id, source).tokenize_public()
}
pub struct Lexer<'a> {
source_id: SourceId,
source: &'a str,
@ -91,8 +151,26 @@ impl<'a> Lexer<'a> {
}
}
pub fn tokenize_public(mut self) -> Result<Vec<PublicToken>> {
let mut tokens = Vec::new();
let mut previous = None;
loop {
let Some(token) = self.next_public_token(previous.as_ref())? else {
return Ok(tokens);
};
if token.kind != PublicTokenKind::Comment {
previous = Some(token.kind);
}
tokens.push(token);
}
}
fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
self.skip_ws_and_comments();
self.next_non_ws_token(previous)
}
fn next_non_ws_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
let start = self.pos;
let Some(ch) = self.peek() else {
return Ok(Token {
@ -242,6 +320,42 @@ impl<'a> Lexer<'a> {
})
}
fn next_public_token(
&mut self,
previous: Option<&PublicTokenKind>,
) -> Result<Option<PublicToken>> {
self.skip_ws();
let start = self.pos;
let Some(ch) = self.peek() else {
return Ok(None);
};
if ch == b'#' {
self.pos += 1;
while !matches!(self.peek(), None | Some(b'\n')) {
self.pos += 1;
}
return Ok(Some(PublicToken {
kind: PublicTokenKind::Comment,
span: self.span(start, self.pos),
}));
}
let previous_token = previous.map(public_kind_to_token_kind);
let token = self.next_non_ws_token(previous_token.as_ref())?;
if token.kind == TokenKind::Eof {
return Ok(None);
}
Ok(Some(PublicToken {
kind: PublicTokenKind::from_token_kind(&token.kind),
span: token.span,
}))
}
fn skip_ws(&mut self) {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
self.pos += 1;
}
}
fn skip_ws_and_comments(&mut self) {
loop {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
@ -390,6 +504,102 @@ impl<'a> Lexer<'a> {
}
}
impl PublicTokenKind {
fn from_token_kind(kind: &TokenKind) -> Self {
match kind {
TokenKind::Ident(_) => Self::Ident,
TokenKind::Int(_) => Self::Int,
TokenKind::Float(_) => Self::Float,
TokenKind::String(_) => Self::String,
TokenKind::Regex(_) => Self::Regex,
TokenKind::True => Self::True,
TokenKind::False => Self::False,
TokenKind::Let => Self::Let,
TokenKind::In => Self::In,
TokenKind::Match => Self::Match,
TokenKind::Import => Self::Import,
TokenKind::Default => Self::Default,
TokenKind::Underscore => Self::Underscore,
TokenKind::LBrace => Self::LBrace,
TokenKind::RBrace => Self::RBrace,
TokenKind::LBracket => Self::LBracket,
TokenKind::RBracket => Self::RBracket,
TokenKind::LParen => Self::LParen,
TokenKind::RParen => Self::RParen,
TokenKind::Semicolon => Self::Semicolon,
TokenKind::Comma => Self::Comma,
TokenKind::Dot => Self::Dot,
TokenKind::Colon => Self::Colon,
TokenKind::Equal => Self::Equal,
TokenKind::EqualEqual => Self::EqualEqual,
TokenKind::Bang => Self::Bang,
TokenKind::BangEqual => Self::BangEqual,
TokenKind::Arrow => Self::Arrow,
TokenKind::Amp => Self::Amp,
TokenKind::AmpAmp => Self::AmpAmp,
TokenKind::PipePipe => Self::PipePipe,
TokenKind::Plus => Self::Plus,
TokenKind::PlusPlus => Self::PlusPlus,
TokenKind::Minus => Self::Minus,
TokenKind::Star => Self::Star,
TokenKind::Slash => Self::Slash,
TokenKind::SlashSlash => Self::SlashSlash,
TokenKind::Gt => Self::Gt,
TokenKind::Gte => Self::Gte,
TokenKind::Lt => Self::Lt,
TokenKind::Lte => Self::Lte,
TokenKind::Eof => unreachable!(),
}
}
}
fn public_kind_to_token_kind(kind: &PublicTokenKind) -> TokenKind {
match kind {
PublicTokenKind::Ident => TokenKind::Ident(String::new()),
PublicTokenKind::Int => TokenKind::Int(0),
PublicTokenKind::Float => TokenKind::Float(0.0),
PublicTokenKind::String => TokenKind::String(String::new()),
PublicTokenKind::Regex => TokenKind::Regex(String::new()),
PublicTokenKind::True => TokenKind::True,
PublicTokenKind::False => TokenKind::False,
PublicTokenKind::Let => TokenKind::Let,
PublicTokenKind::In => TokenKind::In,
PublicTokenKind::Match => TokenKind::Match,
PublicTokenKind::Import => TokenKind::Import,
PublicTokenKind::Default => TokenKind::Default,
PublicTokenKind::Underscore => TokenKind::Underscore,
PublicTokenKind::LBrace => TokenKind::LBrace,
PublicTokenKind::RBrace => TokenKind::RBrace,
PublicTokenKind::LBracket => TokenKind::LBracket,
PublicTokenKind::RBracket => TokenKind::RBracket,
PublicTokenKind::LParen => TokenKind::LParen,
PublicTokenKind::RParen => TokenKind::RParen,
PublicTokenKind::Semicolon => TokenKind::Semicolon,
PublicTokenKind::Comma => TokenKind::Comma,
PublicTokenKind::Dot => TokenKind::Dot,
PublicTokenKind::Colon => TokenKind::Colon,
PublicTokenKind::Equal => TokenKind::Equal,
PublicTokenKind::EqualEqual => TokenKind::EqualEqual,
PublicTokenKind::Bang => TokenKind::Bang,
PublicTokenKind::BangEqual => TokenKind::BangEqual,
PublicTokenKind::Arrow => TokenKind::Arrow,
PublicTokenKind::Amp => TokenKind::Amp,
PublicTokenKind::AmpAmp => TokenKind::AmpAmp,
PublicTokenKind::PipePipe => TokenKind::PipePipe,
PublicTokenKind::Plus => TokenKind::Plus,
PublicTokenKind::PlusPlus => TokenKind::PlusPlus,
PublicTokenKind::Minus => TokenKind::Minus,
PublicTokenKind::Star => TokenKind::Star,
PublicTokenKind::Slash => TokenKind::Slash,
PublicTokenKind::SlashSlash => TokenKind::SlashSlash,
PublicTokenKind::Gt => TokenKind::Gt,
PublicTokenKind::Gte => TokenKind::Gte,
PublicTokenKind::Lt => TokenKind::Lt,
PublicTokenKind::Lte => TokenKind::Lte,
PublicTokenKind::Comment => TokenKind::Underscore,
}
}
fn is_ident_start(c: u8) -> bool {
c.is_ascii_alphabetic()
}
@ -427,4 +637,19 @@ mod tests {
assert_eq!(tokens[3].kind, TokenKind::Amp);
assert_eq!(tokens[4].kind, TokenKind::Gte);
}
#[test]
fn public_tokenization_keeps_comments_and_spans() {
let source = "# hello\nport = [1] ++ [2];";
let tokens = tokenize_source(source).unwrap();
assert_eq!(tokens[0].kind, PublicTokenKind::Comment);
let comment = &source[tokens[0].span.start as usize..tokens[0].span.end as usize];
assert_eq!(comment, "# hello");
assert_eq!(tokens[1].kind, PublicTokenKind::Ident);
assert!(
tokens
.iter()
.any(|token| token.kind == PublicTokenKind::PlusPlus)
);
}
}

4
crates/decodal-core/src/lib.rs
View File

@ -21,7 +21,9 @@ pub use decodal_derive::Decodal;
pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
pub use embedding::{HostField, HostValue};
pub use eval::{Engine, format_diagnostic_with};
pub use lexer::{Lexer, Token, TokenKind};
pub use lexer::{
Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id,
};
pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};

85
crates/decodal-wasm/src/lib.rs
View File

@ -1,8 +1,8 @@
use std::collections::BTreeMap;
use decodal::{
Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader,
Span, format_diagnostic_with,
Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, PublicTokenKind, SourceId,
SourceLoader, Span, format_diagnostic_with, tokenize_source,
};
use wasm_bindgen::prelude::*;
@ -16,6 +16,33 @@ pub fn evaluate_project(entry: &str, files_json: &str) -> String {
encode_result(evaluate_project_inner(entry, files_json))
}
#[wasm_bindgen(js_name = tokenizeSource)]
pub fn tokenize_source_json(source: &str) -> String {
match tokenize_source(source) {
Ok(tokens) => {
let mut out = String::from("{\"ok\":true,\"tokens\":[");
for (index, token) in tokens.iter().enumerate() {
if index > 0 {
out.push(',');
}
out.push_str("{\"kind\":");
out.push_str(&json_string(public_token_kind_name(token.kind)));
out.push_str(",\"start\":");
out.push_str(&token.span.start.to_string());
out.push_str(",\"end\":");
out.push_str(&token.span.end.to_string());
out.push('}');
}
out.push_str("]}");
out
}
Err(error) => format!(
"{{\"ok\":false,\"error\":{}}}",
json_string(&format_diagnostic_with_root(&error, "playground"))
),
}
}
fn encode_result(result: Result<String, String>) -> String {
match result {
Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)),
@ -145,6 +172,53 @@ fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str
})
}
fn public_token_kind_name(kind: PublicTokenKind) -> &'static str {
match kind {
PublicTokenKind::Ident => "ident",
PublicTokenKind::Int => "int",
PublicTokenKind::Float => "float",
PublicTokenKind::String => "string",
PublicTokenKind::Regex => "regex",
PublicTokenKind::True => "true",
PublicTokenKind::False => "false",
PublicTokenKind::Let => "let",
PublicTokenKind::In => "in",
PublicTokenKind::Match => "match",
PublicTokenKind::Import => "import",
PublicTokenKind::Default => "default",
PublicTokenKind::Underscore => "underscore",
PublicTokenKind::LBrace => "l_brace",
PublicTokenKind::RBrace => "r_brace",
PublicTokenKind::LBracket => "l_bracket",
PublicTokenKind::RBracket => "r_bracket",
PublicTokenKind::LParen => "l_paren",
PublicTokenKind::RParen => "r_paren",
PublicTokenKind::Semicolon => "semicolon",
PublicTokenKind::Comma => "comma",
PublicTokenKind::Dot => "dot",
PublicTokenKind::Colon => "colon",
PublicTokenKind::Equal => "equal",
PublicTokenKind::EqualEqual => "equal_equal",
PublicTokenKind::Bang => "bang",
PublicTokenKind::BangEqual => "bang_equal",
PublicTokenKind::Arrow => "arrow",
PublicTokenKind::Amp => "amp",
PublicTokenKind::AmpAmp => "amp_amp",
PublicTokenKind::PipePipe => "pipe_pipe",
PublicTokenKind::Plus => "plus",
PublicTokenKind::PlusPlus => "plus_plus",
PublicTokenKind::Minus => "minus",
PublicTokenKind::Star => "star",
PublicTokenKind::Slash => "slash",
PublicTokenKind::SlashSlash => "slash_slash",
PublicTokenKind::Gt => "gt",
PublicTokenKind::Gte => "gte",
PublicTokenKind::Lt => "lt",
PublicTokenKind::Lte => "lte",
PublicTokenKind::Comment => "comment",
}
}
fn format_data(data: &Data, indent: usize) -> String {
match data {
Data::String(value) => json_string(value),
@ -231,6 +305,13 @@ mod tests {
);
}
#[test]
fn tokenizes_source_for_web() {
let output = super::tokenize_source_json("# hi\nport = [1] ++ [2];");
assert!(output.contains("\"kind\":\"comment\""));
assert!(output.contains("\"kind\":\"plus_plus\""));
}
#[test]
fn evaluates_project_imports() {
let files = r#"{

4
doc/manual/souce/development.md
View File

@ -60,6 +60,7 @@ site/decodal-site/src/pages/docs/[...slug].astro
site/decodal-site/src/pages/playground.astro
site/decodal-site/src/layouts/ManualLayout.astro
site/decodal-site/src/lib/docs.js
site/decodal-site/src/lib/highlight.js
crates/decodal-wasm/src/lib.rs
```
@ -80,6 +81,9 @@ site/decodal-site/src/wasm/
These generated files are committed so the site can be built without requiring every consumer to regenerate the wasm package first.
The playground editor uses `tokenizeSource` from `decodal-wasm` for token spans and maps those tokens to HTML classes in `src/lib/highlight.js`.
The documentation build still uses the lightweight JavaScript fallback highlighter so Astro can render Markdown without initializing WASM at build time.
To run the site locally:
```sh

60
site/decodal-site/src/lib/highlight.js
View File

@ -25,6 +25,66 @@ export function highlightCode(code, language = '') {
return escapeHtml(code);
}
export function highlightDecodalTokens(source, tokens) {
let html = '';
let cursor = 0;
for (const token of tokens) {
if (token.start > cursor) html += escapeHtml(source.slice(cursor, token.start));
const text = source.slice(token.start, token.end);
const kind = tokenClass(token.kind, text);
html += kind ? `<span class="${kind}">${escapeHtml(text)}</span>` : escapeHtml(text);
cursor = token.end;
}
if (cursor < source.length) html += escapeHtml(source.slice(cursor));
return html;
}
function tokenClass(kind, text) {
if (['let', 'in', 'fn', 'match', 'import', 'default'].includes(kind)) return 'tok-keyword';
if (kind === 'ident' && ['String', 'Int', 'Float', 'Bool', 'Array'].includes(text)) return 'tok-type';
if (kind === 'ident') return '';
if (['true', 'false'].includes(kind)) return 'tok-literal';
if (['int', 'float'].includes(kind)) return 'tok-number';
if (kind === 'string') return 'tok-string';
if (kind === 'regex') return 'tok-regex';
if (kind === 'comment') return 'tok-comment';
if (
[
'equal',
'equal_equal',
'bang',
'bang_equal',
'arrow',
'amp',
'amp_amp',
'pipe_pipe',
'plus',
'plus_plus',
'minus',
'star',
'slash',
'slash_slash',
'gt',
'gte',
'lt',
'lte',
'dot',
'colon',
'semicolon',
'comma',
'l_brace',
'r_brace',
'l_bracket',
'r_bracket',
'l_paren',
'r_paren',
].includes(kind)
) {
return 'tok-operator';
}
return '';
}
export function highlightDecodal(source) {
let html = '';
let index = 0;

20
site/decodal-site/src/scripts/playground.js
View File

@ -1,5 +1,5 @@
import init, { evaluateProject } from '../wasm/decodal_wasm.js';
import { highlightDecodal } from '../lib/highlight.js';
import init, { evaluateProject, tokenizeSource } from '../wasm/decodal_wasm.js';
import { highlightDecodal, highlightDecodalTokens } from '../lib/highlight.js';
import { playgroundExamples } from './playground-examples.js';
const STORAGE_KEY = 'decodal-playground-project-v1';
@ -18,6 +18,7 @@ const exampleSelect = document.getElementById('example-select');
const loadExample = document.getElementById('load-example');
const project = loadProject();
let wasmReady = false;
for (const example of playgroundExamples) {
const option = document.createElement('option');
@ -97,7 +98,18 @@ function setActiveFile(path) {
}
function updateHighlight() {
sourceHighlight.innerHTML = `${highlightDecodal(source.value)}\n`;
sourceHighlight.innerHTML = `${highlightSource(source.value)}\n`;
}
function highlightSource(value) {
if (!wasmReady) return highlightDecodal(value);
try {
const result = JSON.parse(tokenizeSource(value));
if (result.ok) return highlightDecodalTokens(value, result.tokens);
} catch (_error) {
// Fall back to the lightweight JavaScript highlighter.
}
return highlightDecodal(value);
}
function syncHighlightScroll() {
@ -165,6 +177,8 @@ function compareNodes(a, b) {
try {
await init();
wasmReady = true;
updateHighlight();
run.disabled = false;
status.textContent = '';
execute();

3
site/decodal-site/src/wasm/decodal_wasm.d.ts vendored
View File

@ -5,12 +5,15 @@ export function evaluate(source: string): string;
export function evaluateProject(entry: string, files_json: string): string;
export function tokenizeSource(source: string): string;
export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
export interface InitOutput {
readonly memory: WebAssembly.Memory;
readonly evaluate: (a: number, b: number) => [number, number];
readonly evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
readonly tokenizeSource: (a: number, b: number) => [number, number];
readonly __wbindgen_externrefs: WebAssembly.Table;
readonly __wbindgen_malloc: (a: number, b: number) => number;
readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;

19
site/decodal-site/src/wasm/decodal_wasm.js
View File

@ -40,6 +40,25 @@ export function evaluateProject(entry, files_json) {
wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
}
}
/**
* @param {string} source
* @returns {string}
*/
export function tokenizeSource(source) {
let deferred2_0;
let deferred2_1;
try {
const ptr0 = passStringToWasm0(source, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.tokenizeSource(ptr0, len0);
deferred2_0 = ret[0];
deferred2_1 = ret[1];
return getStringFromWasm0(ret[0], ret[1]);
} finally {
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
}
}
function __wbg_get_imports() {
const import0 = {
__proto__: null,

BIN
site/decodal-site/src/wasm/decodal_wasm_bg.wasm
View File

Binary file not shown.

1
site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts vendored
View File

@ -3,6 +3,7 @@
export const memory: WebAssembly.Memory;
export const evaluate: (a: number, b: number) => [number, number];
export const evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
export const tokenizeSource: (a: number, b: number) => [number, number];
export const __wbindgen_externrefs: WebAssembly.Table;
export const __wbindgen_malloc: (a: number, b: number) => number;
export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;