Remove public tokenizer API

crates/decodal-core/src/lexer.rs

@@ -54,66 +54,6 @@ pub enum TokenKind {
     Eof,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct PublicToken {
-    pub kind: PublicTokenKind,
-    pub span: Span,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum PublicTokenKind {
-    Ident,
-    Int,
-    Float,
-    String,
-    Regex,
-    True,
-    False,
-    Let,
-    In,
-    Match,
-    Import,
-    Default,
-    Underscore,
-    LBrace,
-    RBrace,
-    LBracket,
-    RBracket,
-    LParen,
-    RParen,
-    Semicolon,
-    Comma,
-    Dot,
-    Colon,
-    Equal,
-    EqualEqual,
-    Bang,
-    BangEqual,
-    Arrow,
-    Amp,
-    AmpAmp,
-    PipePipe,
-    Plus,
-    PlusPlus,
-    Minus,
-    Star,
-    Slash,
-    SlashSlash,
-    Gt,
-    Gte,
-    Lt,
-    Lte,
-    Comment,
-}
-
-pub fn tokenize_source(source: &str) -> Result<Vec<PublicToken>> {
-    tokenize_source_with_id(SourceId(0), source)
-}
-
-pub fn tokenize_source_with_id(source_id: SourceId, source: &str) -> Result<Vec<PublicToken>> {
-    Lexer::with_source_id(source_id, source).tokenize_public()
-}
-
 pub struct Lexer<'a> {
     source_id: SourceId,
     source: &'a str,
@@ -122,6 +62,7 @@ pub struct Lexer<'a> {
 }
 
 impl<'a> Lexer<'a> {
+    #[cfg(test)]
     pub fn new(source: &'a str) -> Self {
         Self::with_source_id(SourceId(0), source)
     }
@@ -151,20 +92,6 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    pub fn tokenize_public(mut self) -> Result<Vec<PublicToken>> {
-        let mut tokens = Vec::new();
-        let mut previous = None;
-        loop {
-            let Some(token) = self.next_public_token(previous.as_ref())? else {
-                return Ok(tokens);
-            };
-            if token.kind != PublicTokenKind::Comment {
-                previous = Some(token.kind);
-            }
-            tokens.push(token);
-        }
-    }
-
     fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
         self.skip_ws_and_comments();
         self.next_non_ws_token(previous)
@@ -320,42 +247,6 @@ impl<'a> Lexer<'a> {
         })
     }
 
-    fn next_public_token(
-        &mut self,
-        previous: Option<&PublicTokenKind>,
-    ) -> Result<Option<PublicToken>> {
-        self.skip_ws();
-        let start = self.pos;
-        let Some(ch) = self.peek() else {
-            return Ok(None);
-        };
-        if ch == b'#' {
-            self.pos += 1;
-            while !matches!(self.peek(), None | Some(b'\n')) {
-                self.pos += 1;
-            }
-            return Ok(Some(PublicToken {
-                kind: PublicTokenKind::Comment,
-                span: self.span(start, self.pos),
-            }));
-        }
-        let previous_token = previous.map(public_kind_to_token_kind);
-        let token = self.next_non_ws_token(previous_token.as_ref())?;
-        if token.kind == TokenKind::Eof {
-            return Ok(None);
-        }
-        Ok(Some(PublicToken {
-            kind: PublicTokenKind::from_token_kind(&token.kind),
-            span: token.span,
-        }))
-    }
-
-    fn skip_ws(&mut self) {
-        while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
-            self.pos += 1;
-        }
-    }
-
     fn skip_ws_and_comments(&mut self) {
         loop {
             while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
@@ -504,102 +395,6 @@ impl<'a> Lexer<'a> {
     }
 }
 
-impl PublicTokenKind {
-    fn from_token_kind(kind: &TokenKind) -> Self {
-        match kind {
-            TokenKind::Ident(_) => Self::Ident,
-            TokenKind::Int(_) => Self::Int,
-            TokenKind::Float(_) => Self::Float,
-            TokenKind::String(_) => Self::String,
-            TokenKind::Regex(_) => Self::Regex,
-            TokenKind::True => Self::True,
-            TokenKind::False => Self::False,
-            TokenKind::Let => Self::Let,
-            TokenKind::In => Self::In,
-            TokenKind::Match => Self::Match,
-            TokenKind::Import => Self::Import,
-            TokenKind::Default => Self::Default,
-            TokenKind::Underscore => Self::Underscore,
-            TokenKind::LBrace => Self::LBrace,
-            TokenKind::RBrace => Self::RBrace,
-            TokenKind::LBracket => Self::LBracket,
-            TokenKind::RBracket => Self::RBracket,
-            TokenKind::LParen => Self::LParen,
-            TokenKind::RParen => Self::RParen,
-            TokenKind::Semicolon => Self::Semicolon,
-            TokenKind::Comma => Self::Comma,
-            TokenKind::Dot => Self::Dot,
-            TokenKind::Colon => Self::Colon,
-            TokenKind::Equal => Self::Equal,
-            TokenKind::EqualEqual => Self::EqualEqual,
-            TokenKind::Bang => Self::Bang,
-            TokenKind::BangEqual => Self::BangEqual,
-            TokenKind::Arrow => Self::Arrow,
-            TokenKind::Amp => Self::Amp,
-            TokenKind::AmpAmp => Self::AmpAmp,
-            TokenKind::PipePipe => Self::PipePipe,
-            TokenKind::Plus => Self::Plus,
-            TokenKind::PlusPlus => Self::PlusPlus,
-            TokenKind::Minus => Self::Minus,
-            TokenKind::Star => Self::Star,
-            TokenKind::Slash => Self::Slash,
-            TokenKind::SlashSlash => Self::SlashSlash,
-            TokenKind::Gt => Self::Gt,
-            TokenKind::Gte => Self::Gte,
-            TokenKind::Lt => Self::Lt,
-            TokenKind::Lte => Self::Lte,
-            TokenKind::Eof => unreachable!(),
-        }
-    }
-}
-
-fn public_kind_to_token_kind(kind: &PublicTokenKind) -> TokenKind {
-    match kind {
-        PublicTokenKind::Ident => TokenKind::Ident(String::new()),
-        PublicTokenKind::Int => TokenKind::Int(0),
-        PublicTokenKind::Float => TokenKind::Float(0.0),
-        PublicTokenKind::String => TokenKind::String(String::new()),
-        PublicTokenKind::Regex => TokenKind::Regex(String::new()),
-        PublicTokenKind::True => TokenKind::True,
-        PublicTokenKind::False => TokenKind::False,
-        PublicTokenKind::Let => TokenKind::Let,
-        PublicTokenKind::In => TokenKind::In,
-        PublicTokenKind::Match => TokenKind::Match,
-        PublicTokenKind::Import => TokenKind::Import,
-        PublicTokenKind::Default => TokenKind::Default,
-        PublicTokenKind::Underscore => TokenKind::Underscore,
-        PublicTokenKind::LBrace => TokenKind::LBrace,
-        PublicTokenKind::RBrace => TokenKind::RBrace,
-        PublicTokenKind::LBracket => TokenKind::LBracket,
-        PublicTokenKind::RBracket => TokenKind::RBracket,
-        PublicTokenKind::LParen => TokenKind::LParen,
-        PublicTokenKind::RParen => TokenKind::RParen,
-        PublicTokenKind::Semicolon => TokenKind::Semicolon,
-        PublicTokenKind::Comma => TokenKind::Comma,
-        PublicTokenKind::Dot => TokenKind::Dot,
-        PublicTokenKind::Colon => TokenKind::Colon,
-        PublicTokenKind::Equal => TokenKind::Equal,
-        PublicTokenKind::EqualEqual => TokenKind::EqualEqual,
-        PublicTokenKind::Bang => TokenKind::Bang,
-        PublicTokenKind::BangEqual => TokenKind::BangEqual,
-        PublicTokenKind::Arrow => TokenKind::Arrow,
-        PublicTokenKind::Amp => TokenKind::Amp,
-        PublicTokenKind::AmpAmp => TokenKind::AmpAmp,
-        PublicTokenKind::PipePipe => TokenKind::PipePipe,
-        PublicTokenKind::Plus => TokenKind::Plus,
-        PublicTokenKind::PlusPlus => TokenKind::PlusPlus,
-        PublicTokenKind::Minus => TokenKind::Minus,
-        PublicTokenKind::Star => TokenKind::Star,
-        PublicTokenKind::Slash => TokenKind::Slash,
-        PublicTokenKind::SlashSlash => TokenKind::SlashSlash,
-        PublicTokenKind::Gt => TokenKind::Gt,
-        PublicTokenKind::Gte => TokenKind::Gte,
-        PublicTokenKind::Lt => TokenKind::Lt,
-        PublicTokenKind::Lte => TokenKind::Lte,
-        PublicTokenKind::Comment => TokenKind::Underscore,
-    }
-}
-
 fn is_ident_start(c: u8) -> bool {
     c.is_ascii_alphabetic()
 }
@@ -637,19 +432,4 @@ mod tests {
         assert_eq!(tokens[3].kind, TokenKind::Amp);
         assert_eq!(tokens[4].kind, TokenKind::Gte);
     }
-
-    #[test]
-    fn public_tokenization_keeps_comments_and_spans() {
-        let source = "# hello\nport = [1] ++ [2];";
-        let tokens = tokenize_source(source).unwrap();
-        assert_eq!(tokens[0].kind, PublicTokenKind::Comment);
-        let comment = &source[tokens[0].span.start as usize..tokens[0].span.end as usize];
-        assert_eq!(comment, "# hello");
-        assert_eq!(tokens[1].kind, PublicTokenKind::Ident);
-        assert!(
-            tokens
-                .iter()
-                .any(|token| token.kind == PublicTokenKind::PlusPlus)
-        );
-    }
 }

crates/decodal-core/src/lib.rs

@@ -7,7 +7,7 @@ pub mod constraints;
 pub mod diagnostic;
 pub mod embedding;
 pub mod eval;
-pub mod lexer;
+mod lexer;
 pub mod module;
 pub mod parser;
 pub mod runtime;
@@ -21,9 +21,6 @@ pub use decodal_derive::Decodal;
 pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
 pub use embedding::{HostField, HostValue};
 pub use eval::{Engine, format_diagnostic_with};
-pub use lexer::{
-    Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id,
-};
 pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
 pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
 pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};

crates/decodal-wasm/src/lib.rs

@@ -1,8 +1,8 @@
 use std::collections::BTreeMap;
 
 use decodal::{
-    Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, PublicTokenKind, SourceId,
+    Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader,
-    SourceLoader, Span, format_diagnostic_with, tokenize_source,
+    Span, format_diagnostic_with,
 };
 use wasm_bindgen::prelude::*;
 
@@ -16,33 +16,6 @@ pub fn evaluate_project(entry: &str, files_json: &str) -> String {
     encode_result(evaluate_project_inner(entry, files_json))
 }
 
-#[wasm_bindgen(js_name = tokenizeSource)]
-pub fn tokenize_source_json(source: &str) -> String {
-    match tokenize_source(source) {
-        Ok(tokens) => {
-            let mut out = String::from("{\"ok\":true,\"tokens\":[");
-            for (index, token) in tokens.iter().enumerate() {
-                if index > 0 {
-                    out.push(',');
-                }
-                out.push_str("{\"kind\":");
-                out.push_str(&json_string(public_token_kind_name(token.kind)));
-                out.push_str(",\"start\":");
-                out.push_str(&token.span.start.to_string());
-                out.push_str(",\"end\":");
-                out.push_str(&token.span.end.to_string());
-                out.push('}');
-            }
-            out.push_str("]}");
-            out
-        }
-        Err(error) => format!(
-            "{{\"ok\":false,\"error\":{}}}",
-            json_string(&format_diagnostic_with_root(&error, "playground"))
-        ),
-    }
-}
-
 fn encode_result(result: Result<String, String>) -> String {
     match result {
         Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)),
@@ -172,53 +145,6 @@ fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str
     })
 }
 
-fn public_token_kind_name(kind: PublicTokenKind) -> &'static str {
-    match kind {
-        PublicTokenKind::Ident => "ident",
-        PublicTokenKind::Int => "int",
-        PublicTokenKind::Float => "float",
-        PublicTokenKind::String => "string",
-        PublicTokenKind::Regex => "regex",
-        PublicTokenKind::True => "true",
-        PublicTokenKind::False => "false",
-        PublicTokenKind::Let => "let",
-        PublicTokenKind::In => "in",
-        PublicTokenKind::Match => "match",
-        PublicTokenKind::Import => "import",
-        PublicTokenKind::Default => "default",
-        PublicTokenKind::Underscore => "underscore",
-        PublicTokenKind::LBrace => "l_brace",
-        PublicTokenKind::RBrace => "r_brace",
-        PublicTokenKind::LBracket => "l_bracket",
-        PublicTokenKind::RBracket => "r_bracket",
-        PublicTokenKind::LParen => "l_paren",
-        PublicTokenKind::RParen => "r_paren",
-        PublicTokenKind::Semicolon => "semicolon",
-        PublicTokenKind::Comma => "comma",
-        PublicTokenKind::Dot => "dot",
-        PublicTokenKind::Colon => "colon",
-        PublicTokenKind::Equal => "equal",
-        PublicTokenKind::EqualEqual => "equal_equal",
-        PublicTokenKind::Bang => "bang",
-        PublicTokenKind::BangEqual => "bang_equal",
-        PublicTokenKind::Arrow => "arrow",
-        PublicTokenKind::Amp => "amp",
-        PublicTokenKind::AmpAmp => "amp_amp",
-        PublicTokenKind::PipePipe => "pipe_pipe",
-        PublicTokenKind::Plus => "plus",
-        PublicTokenKind::PlusPlus => "plus_plus",
-        PublicTokenKind::Minus => "minus",
-        PublicTokenKind::Star => "star",
-        PublicTokenKind::Slash => "slash",
-        PublicTokenKind::SlashSlash => "slash_slash",
-        PublicTokenKind::Gt => "gt",
-        PublicTokenKind::Gte => "gte",
-        PublicTokenKind::Lt => "lt",
-        PublicTokenKind::Lte => "lte",
-        PublicTokenKind::Comment => "comment",
-    }
-}
-
 fn format_data(data: &Data, indent: usize) -> String {
     match data {
         Data::String(value) => json_string(value),
@@ -305,13 +231,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn tokenizes_source_for_web() {
-        let output = super::tokenize_source_json("# hi\nport = [1] ++ [2];");
-        assert!(output.contains("\"kind\":\"comment\""));
-        assert!(output.contains("\"kind\":\"plus_plus\""));
-    }
-
     #[test]
     fn evaluates_project_imports() {
         let files = r#"{

doc/manual/souce/language/grammar.md

@@ -27,7 +27,6 @@ regex_character  = escape | ? any character except "/", "\\", or newline ? ;
 ```
 
 Whitespace and comments separate tokens and are otherwise ignored by the parser.
-Comments are retained by public tokenizer APIs for editor tooling.
 
 ## Syntactic grammar
 

site/decodal-site/src/wasm/decodal_wasm.d.ts

@@ -5,15 +5,12 @@ export function evaluate(source: string): string;
 
 export function evaluateProject(entry: string, files_json: string): string;
 
-export function tokenizeSource(source: string): string;
-
 export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
 
 export interface InitOutput {
     readonly memory: WebAssembly.Memory;
     readonly evaluate: (a: number, b: number) => [number, number];
     readonly evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
-    readonly tokenizeSource: (a: number, b: number) => [number, number];
     readonly __wbindgen_externrefs: WebAssembly.Table;
     readonly __wbindgen_malloc: (a: number, b: number) => number;
     readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;

site/decodal-site/src/wasm/decodal_wasm.js

@@ -40,25 +40,6 @@ export function evaluateProject(entry, files_json) {
         wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
     }
 }
-
-/**
- * @param {string} source
- * @returns {string}
- */
-export function tokenizeSource(source) {
-    let deferred2_0;
-    let deferred2_1;
-    try {
-        const ptr0 = passStringToWasm0(source, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.tokenizeSource(ptr0, len0);
-        deferred2_0 = ret[0];
-        deferred2_1 = ret[1];
-        return getStringFromWasm0(ret[0], ret[1]);
-    } finally {
-        wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
-    }
-}
 function __wbg_get_imports() {
     const import0 = {
         __proto__: null,

site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts

@@ -3,7 +3,6 @@
 export const memory: WebAssembly.Memory;
 export const evaluate: (a: number, b: number) => [number, number];
 export const evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
-export const tokenizeSource: (a: number, b: number) => [number, number];
 export const __wbindgen_externrefs: WebAssembly.Table;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;