Expose tokenizer for web highlighting

site/decodal-site/src/lib/highlight.js

@@ -25,6 +25,66 @@ export function highlightCode(code, language = '') {
   return escapeHtml(code);
 }
 
+export function highlightDecodalTokens(source, tokens) {
+  let html = '';
+  let cursor = 0;
+  for (const token of tokens) {
+    if (token.start > cursor) html += escapeHtml(source.slice(cursor, token.start));
+    const text = source.slice(token.start, token.end);
+    const kind = tokenClass(token.kind, text);
+    html += kind ? `<span class="${kind}">${escapeHtml(text)}</span>` : escapeHtml(text);
+    cursor = token.end;
+  }
+  if (cursor < source.length) html += escapeHtml(source.slice(cursor));
+  return html;
+}
+
+function tokenClass(kind, text) {
+  if (['let', 'in', 'fn', 'match', 'import', 'default'].includes(kind)) return 'tok-keyword';
+  if (kind === 'ident' && ['String', 'Int', 'Float', 'Bool', 'Array'].includes(text)) return 'tok-type';
+  if (kind === 'ident') return '';
+  if (['true', 'false'].includes(kind)) return 'tok-literal';
+  if (['int', 'float'].includes(kind)) return 'tok-number';
+  if (kind === 'string') return 'tok-string';
+  if (kind === 'regex') return 'tok-regex';
+  if (kind === 'comment') return 'tok-comment';
+  if (
+    [
+      'equal',
+      'equal_equal',
+      'bang',
+      'bang_equal',
+      'arrow',
+      'amp',
+      'amp_amp',
+      'pipe_pipe',
+      'plus',
+      'plus_plus',
+      'minus',
+      'star',
+      'slash',
+      'slash_slash',
+      'gt',
+      'gte',
+      'lt',
+      'lte',
+      'dot',
+      'colon',
+      'semicolon',
+      'comma',
+      'l_brace',
+      'r_brace',
+      'l_bracket',
+      'r_bracket',
+      'l_paren',
+      'r_paren',
+    ].includes(kind)
+  ) {
+    return 'tok-operator';
+  }
+  return '';
+}
+
 export function highlightDecodal(source) {
   let html = '';
   let index = 0;

site/decodal-site/src/scripts/playground.js

@@ -1,5 +1,5 @@
-import init, { evaluateProject } from '../wasm/decodal_wasm.js';
-import { highlightDecodal } from '../lib/highlight.js';
+import init, { evaluateProject, tokenizeSource } from '../wasm/decodal_wasm.js';
+import { highlightDecodal, highlightDecodalTokens } from '../lib/highlight.js';
 import { playgroundExamples } from './playground-examples.js';
 
 const STORAGE_KEY = 'decodal-playground-project-v1';
@@ -18,6 +18,7 @@ const exampleSelect = document.getElementById('example-select');
 const loadExample = document.getElementById('load-example');
 
 const project = loadProject();
+let wasmReady = false;
 
 for (const example of playgroundExamples) {
   const option = document.createElement('option');
@@ -97,7 +98,18 @@ function setActiveFile(path) {
 }
 
 function updateHighlight() {
-  sourceHighlight.innerHTML = `${highlightDecodal(source.value)}\n`;
+  sourceHighlight.innerHTML = `${highlightSource(source.value)}\n`;
+}
+
+function highlightSource(value) {
+  if (!wasmReady) return highlightDecodal(value);
+  try {
+    const result = JSON.parse(tokenizeSource(value));
+    if (result.ok) return highlightDecodalTokens(value, result.tokens);
+  } catch (_error) {
+    // Fall back to the lightweight JavaScript highlighter.
+  }
+  return highlightDecodal(value);
 }
 
 function syncHighlightScroll() {
@@ -165,6 +177,8 @@ function compareNodes(a, b) {
 
 try {
   await init();
+  wasmReady = true;
+  updateHighlight();
   run.disabled = false;
   status.textContent = '';
   execute();

site/decodal-site/src/wasm/decodal_wasm.d.ts

@@ -5,12 +5,15 @@ export function evaluate(source: string): string;
 
 export function evaluateProject(entry: string, files_json: string): string;
 
+export function tokenizeSource(source: string): string;
+
 export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
 
 export interface InitOutput {
     readonly memory: WebAssembly.Memory;
     readonly evaluate: (a: number, b: number) => [number, number];
     readonly evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
+    readonly tokenizeSource: (a: number, b: number) => [number, number];
     readonly __wbindgen_externrefs: WebAssembly.Table;
     readonly __wbindgen_malloc: (a: number, b: number) => number;
     readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;

site/decodal-site/src/wasm/decodal_wasm.js

@@ -40,6 +40,25 @@ export function evaluateProject(entry, files_json) {
         wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
     }
 }
+
+/**
+ * @param {string} source
+ * @returns {string}
+ */
+export function tokenizeSource(source) {
+    let deferred2_0;
+    let deferred2_1;
+    try {
+        const ptr0 = passStringToWasm0(source, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.tokenizeSource(ptr0, len0);
+        deferred2_0 = ret[0];
+        deferred2_1 = ret[1];
+        return getStringFromWasm0(ret[0], ret[1]);
+    } finally {
+        wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+    }
+}
 function __wbg_get_imports() {
     const import0 = {
         __proto__: null,

site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts

@@ -3,6 +3,7 @@
 export const memory: WebAssembly.Memory;
 export const evaluate: (a: number, b: number) => [number, number];
 export const evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
+export const tokenizeSource: (a: number, b: number) => [number, number];
 export const __wbindgen_externrefs: WebAssembly.Table;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;