diff --git a/crates/decodal-core/src/lexer.rs b/crates/decodal-core/src/lexer.rs index 8d43efb..09dd429 100644 --- a/crates/decodal-core/src/lexer.rs +++ b/crates/decodal-core/src/lexer.rs @@ -54,66 +54,6 @@ pub enum TokenKind { Eof, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PublicToken { - pub kind: PublicTokenKind, - pub span: Span, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PublicTokenKind { - Ident, - Int, - Float, - String, - Regex, - True, - False, - Let, - In, - Match, - Import, - Default, - Underscore, - LBrace, - RBrace, - LBracket, - RBracket, - LParen, - RParen, - Semicolon, - Comma, - Dot, - Colon, - Equal, - EqualEqual, - Bang, - BangEqual, - Arrow, - Amp, - AmpAmp, - PipePipe, - Plus, - PlusPlus, - Minus, - Star, - Slash, - SlashSlash, - Gt, - Gte, - Lt, - Lte, - Comment, -} - -pub fn tokenize_source(source: &str) -> Result> { - tokenize_source_with_id(SourceId(0), source) -} - -pub fn tokenize_source_with_id(source_id: SourceId, source: &str) -> Result> { - Lexer::with_source_id(source_id, source).tokenize_public() -} - pub struct Lexer<'a> { source_id: SourceId, source: &'a str, @@ -122,6 +62,7 @@ pub struct Lexer<'a> { } impl<'a> Lexer<'a> { + #[cfg(test)] pub fn new(source: &'a str) -> Self { Self::with_source_id(SourceId(0), source) } @@ -151,20 +92,6 @@ impl<'a> Lexer<'a> { } } - pub fn tokenize_public(mut self) -> Result> { - let mut tokens = Vec::new(); - let mut previous = None; - loop { - let Some(token) = self.next_public_token(previous.as_ref())? else { - return Ok(tokens); - }; - if token.kind != PublicTokenKind::Comment { - previous = Some(token.kind); - } - tokens.push(token); - } - } - fn next_token(&mut self, previous: Option<&TokenKind>) -> Result { self.skip_ws_and_comments(); self.next_non_ws_token(previous) @@ -320,42 +247,6 @@ impl<'a> Lexer<'a> { }) } - fn next_public_token( - &mut self, - previous: Option<&PublicTokenKind>, - ) -> Result> { - self.skip_ws(); - let start = self.pos; - let Some(ch) = self.peek() else { - return Ok(None); - }; - if ch == b'#' { - self.pos += 1; - while !matches!(self.peek(), None | Some(b'\n')) { - self.pos += 1; - } - return Ok(Some(PublicToken { - kind: PublicTokenKind::Comment, - span: self.span(start, self.pos), - })); - } - let previous_token = previous.map(public_kind_to_token_kind); - let token = self.next_non_ws_token(previous_token.as_ref())?; - if token.kind == TokenKind::Eof { - return Ok(None); - } - Ok(Some(PublicToken { - kind: PublicTokenKind::from_token_kind(&token.kind), - span: token.span, - })) - } - - fn skip_ws(&mut self) { - while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) { - self.pos += 1; - } - } - fn skip_ws_and_comments(&mut self) { loop { while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) { @@ -504,102 +395,6 @@ impl<'a> Lexer<'a> { } } -impl PublicTokenKind { - fn from_token_kind(kind: &TokenKind) -> Self { - match kind { - TokenKind::Ident(_) => Self::Ident, - TokenKind::Int(_) => Self::Int, - TokenKind::Float(_) => Self::Float, - TokenKind::String(_) => Self::String, - TokenKind::Regex(_) => Self::Regex, - TokenKind::True => Self::True, - TokenKind::False => Self::False, - TokenKind::Let => Self::Let, - TokenKind::In => Self::In, - TokenKind::Match => Self::Match, - TokenKind::Import => Self::Import, - TokenKind::Default => Self::Default, - TokenKind::Underscore => Self::Underscore, - TokenKind::LBrace => Self::LBrace, - TokenKind::RBrace => Self::RBrace, - TokenKind::LBracket => Self::LBracket, - TokenKind::RBracket => Self::RBracket, - TokenKind::LParen => Self::LParen, - TokenKind::RParen => Self::RParen, - TokenKind::Semicolon => Self::Semicolon, - TokenKind::Comma => Self::Comma, - TokenKind::Dot => Self::Dot, - TokenKind::Colon => Self::Colon, - TokenKind::Equal => Self::Equal, - TokenKind::EqualEqual => Self::EqualEqual, - TokenKind::Bang => Self::Bang, - TokenKind::BangEqual => Self::BangEqual, - TokenKind::Arrow => Self::Arrow, - TokenKind::Amp => Self::Amp, - TokenKind::AmpAmp => Self::AmpAmp, - TokenKind::PipePipe => Self::PipePipe, - TokenKind::Plus => Self::Plus, - TokenKind::PlusPlus => Self::PlusPlus, - TokenKind::Minus => Self::Minus, - TokenKind::Star => Self::Star, - TokenKind::Slash => Self::Slash, - TokenKind::SlashSlash => Self::SlashSlash, - TokenKind::Gt => Self::Gt, - TokenKind::Gte => Self::Gte, - TokenKind::Lt => Self::Lt, - TokenKind::Lte => Self::Lte, - TokenKind::Eof => unreachable!(), - } - } -} - -fn public_kind_to_token_kind(kind: &PublicTokenKind) -> TokenKind { - match kind { - PublicTokenKind::Ident => TokenKind::Ident(String::new()), - PublicTokenKind::Int => TokenKind::Int(0), - PublicTokenKind::Float => TokenKind::Float(0.0), - PublicTokenKind::String => TokenKind::String(String::new()), - PublicTokenKind::Regex => TokenKind::Regex(String::new()), - PublicTokenKind::True => TokenKind::True, - PublicTokenKind::False => TokenKind::False, - PublicTokenKind::Let => TokenKind::Let, - PublicTokenKind::In => TokenKind::In, - PublicTokenKind::Match => TokenKind::Match, - PublicTokenKind::Import => TokenKind::Import, - PublicTokenKind::Default => TokenKind::Default, - PublicTokenKind::Underscore => TokenKind::Underscore, - PublicTokenKind::LBrace => TokenKind::LBrace, - PublicTokenKind::RBrace => TokenKind::RBrace, - PublicTokenKind::LBracket => TokenKind::LBracket, - PublicTokenKind::RBracket => TokenKind::RBracket, - PublicTokenKind::LParen => TokenKind::LParen, - PublicTokenKind::RParen => TokenKind::RParen, - PublicTokenKind::Semicolon => TokenKind::Semicolon, - PublicTokenKind::Comma => TokenKind::Comma, - PublicTokenKind::Dot => TokenKind::Dot, - PublicTokenKind::Colon => TokenKind::Colon, - PublicTokenKind::Equal => TokenKind::Equal, - PublicTokenKind::EqualEqual => TokenKind::EqualEqual, - PublicTokenKind::Bang => TokenKind::Bang, - PublicTokenKind::BangEqual => TokenKind::BangEqual, - PublicTokenKind::Arrow => TokenKind::Arrow, - PublicTokenKind::Amp => TokenKind::Amp, - PublicTokenKind::AmpAmp => TokenKind::AmpAmp, - PublicTokenKind::PipePipe => TokenKind::PipePipe, - PublicTokenKind::Plus => TokenKind::Plus, - PublicTokenKind::PlusPlus => TokenKind::PlusPlus, - PublicTokenKind::Minus => TokenKind::Minus, - PublicTokenKind::Star => TokenKind::Star, - PublicTokenKind::Slash => TokenKind::Slash, - PublicTokenKind::SlashSlash => TokenKind::SlashSlash, - PublicTokenKind::Gt => TokenKind::Gt, - PublicTokenKind::Gte => TokenKind::Gte, - PublicTokenKind::Lt => TokenKind::Lt, - PublicTokenKind::Lte => TokenKind::Lte, - PublicTokenKind::Comment => TokenKind::Underscore, - } -} - fn is_ident_start(c: u8) -> bool { c.is_ascii_alphabetic() } @@ -637,19 +432,4 @@ mod tests { assert_eq!(tokens[3].kind, TokenKind::Amp); assert_eq!(tokens[4].kind, TokenKind::Gte); } - - #[test] - fn public_tokenization_keeps_comments_and_spans() { - let source = "# hello\nport = [1] ++ [2];"; - let tokens = tokenize_source(source).unwrap(); - assert_eq!(tokens[0].kind, PublicTokenKind::Comment); - let comment = &source[tokens[0].span.start as usize..tokens[0].span.end as usize]; - assert_eq!(comment, "# hello"); - assert_eq!(tokens[1].kind, PublicTokenKind::Ident); - assert!( - tokens - .iter() - .any(|token| token.kind == PublicTokenKind::PlusPlus) - ); - } } diff --git a/crates/decodal-core/src/lib.rs b/crates/decodal-core/src/lib.rs index 69a611c..27900be 100644 --- a/crates/decodal-core/src/lib.rs +++ b/crates/decodal-core/src/lib.rs @@ -7,7 +7,7 @@ pub mod constraints; pub mod diagnostic; pub mod embedding; pub mod eval; -pub mod lexer; +mod lexer; pub mod module; pub mod parser; pub mod runtime; @@ -21,9 +21,6 @@ pub use decodal_derive::Decodal; pub use diagnostic::{Diagnostic, DiagnosticKind, Result}; pub use embedding::{HostField, HostValue}; pub use eval::{Engine, format_diagnostic_with}; -pub use lexer::{ - Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id, -}; pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader}; pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id}; pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue}; diff --git a/crates/decodal-wasm/src/lib.rs b/crates/decodal-wasm/src/lib.rs index 8e938eb..1c5f7af 100644 --- a/crates/decodal-wasm/src/lib.rs +++ b/crates/decodal-wasm/src/lib.rs @@ -1,8 +1,8 @@ use std::collections::BTreeMap; use decodal::{ - Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, PublicTokenKind, SourceId, - SourceLoader, Span, format_diagnostic_with, tokenize_source, + Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader, + Span, format_diagnostic_with, }; use wasm_bindgen::prelude::*; @@ -16,33 +16,6 @@ pub fn evaluate_project(entry: &str, files_json: &str) -> String { encode_result(evaluate_project_inner(entry, files_json)) } -#[wasm_bindgen(js_name = tokenizeSource)] -pub fn tokenize_source_json(source: &str) -> String { - match tokenize_source(source) { - Ok(tokens) => { - let mut out = String::from("{\"ok\":true,\"tokens\":["); - for (index, token) in tokens.iter().enumerate() { - if index > 0 { - out.push(','); - } - out.push_str("{\"kind\":"); - out.push_str(&json_string(public_token_kind_name(token.kind))); - out.push_str(",\"start\":"); - out.push_str(&token.span.start.to_string()); - out.push_str(",\"end\":"); - out.push_str(&token.span.end.to_string()); - out.push('}'); - } - out.push_str("]}"); - out - } - Err(error) => format!( - "{{\"ok\":false,\"error\":{}}}", - json_string(&format_diagnostic_with_root(&error, "playground")) - ), - } -} - fn encode_result(result: Result) -> String { match result { Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)), @@ -172,53 +145,6 @@ fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str }) } -fn public_token_kind_name(kind: PublicTokenKind) -> &'static str { - match kind { - PublicTokenKind::Ident => "ident", - PublicTokenKind::Int => "int", - PublicTokenKind::Float => "float", - PublicTokenKind::String => "string", - PublicTokenKind::Regex => "regex", - PublicTokenKind::True => "true", - PublicTokenKind::False => "false", - PublicTokenKind::Let => "let", - PublicTokenKind::In => "in", - PublicTokenKind::Match => "match", - PublicTokenKind::Import => "import", - PublicTokenKind::Default => "default", - PublicTokenKind::Underscore => "underscore", - PublicTokenKind::LBrace => "l_brace", - PublicTokenKind::RBrace => "r_brace", - PublicTokenKind::LBracket => "l_bracket", - PublicTokenKind::RBracket => "r_bracket", - PublicTokenKind::LParen => "l_paren", - PublicTokenKind::RParen => "r_paren", - PublicTokenKind::Semicolon => "semicolon", - PublicTokenKind::Comma => "comma", - PublicTokenKind::Dot => "dot", - PublicTokenKind::Colon => "colon", - PublicTokenKind::Equal => "equal", - PublicTokenKind::EqualEqual => "equal_equal", - PublicTokenKind::Bang => "bang", - PublicTokenKind::BangEqual => "bang_equal", - PublicTokenKind::Arrow => "arrow", - PublicTokenKind::Amp => "amp", - PublicTokenKind::AmpAmp => "amp_amp", - PublicTokenKind::PipePipe => "pipe_pipe", - PublicTokenKind::Plus => "plus", - PublicTokenKind::PlusPlus => "plus_plus", - PublicTokenKind::Minus => "minus", - PublicTokenKind::Star => "star", - PublicTokenKind::Slash => "slash", - PublicTokenKind::SlashSlash => "slash_slash", - PublicTokenKind::Gt => "gt", - PublicTokenKind::Gte => "gte", - PublicTokenKind::Lt => "lt", - PublicTokenKind::Lte => "lte", - PublicTokenKind::Comment => "comment", - } -} - fn format_data(data: &Data, indent: usize) -> String { match data { Data::String(value) => json_string(value), @@ -305,13 +231,6 @@ mod tests { ); } - #[test] - fn tokenizes_source_for_web() { - let output = super::tokenize_source_json("# hi\nport = [1] ++ [2];"); - assert!(output.contains("\"kind\":\"comment\"")); - assert!(output.contains("\"kind\":\"plus_plus\"")); - } - #[test] fn evaluates_project_imports() { let files = r#"{ diff --git a/doc/manual/souce/language/grammar.md b/doc/manual/souce/language/grammar.md index 08bbbbe..fdcee91 100644 --- a/doc/manual/souce/language/grammar.md +++ b/doc/manual/souce/language/grammar.md @@ -27,7 +27,6 @@ regex_character = escape | ? any character except "/", "\\", or newline ? ; ``` Whitespace and comments separate tokens and are otherwise ignored by the parser. -Comments are retained by public tokenizer APIs for editor tooling. ## Syntactic grammar diff --git a/site/decodal-site/src/wasm/decodal_wasm.d.ts b/site/decodal-site/src/wasm/decodal_wasm.d.ts index 59b9615..e88b72f 100644 --- a/site/decodal-site/src/wasm/decodal_wasm.d.ts +++ b/site/decodal-site/src/wasm/decodal_wasm.d.ts @@ -5,15 +5,12 @@ export function evaluate(source: string): string; export function evaluateProject(entry: string, files_json: string): string; -export function tokenizeSource(source: string): string; - export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; export interface InitOutput { readonly memory: WebAssembly.Memory; readonly evaluate: (a: number, b: number) => [number, number]; readonly evaluateProject: (a: number, b: number, c: number, d: number) => [number, number]; - readonly tokenizeSource: (a: number, b: number) => [number, number]; readonly __wbindgen_externrefs: WebAssembly.Table; readonly __wbindgen_malloc: (a: number, b: number) => number; readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; diff --git a/site/decodal-site/src/wasm/decodal_wasm.js b/site/decodal-site/src/wasm/decodal_wasm.js index 0f2ae92..800a76f 100644 --- a/site/decodal-site/src/wasm/decodal_wasm.js +++ b/site/decodal-site/src/wasm/decodal_wasm.js @@ -40,25 +40,6 @@ export function evaluateProject(entry, files_json) { wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); } } - -/** - * @param {string} source - * @returns {string} - */ -export function tokenizeSource(source) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(source, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.tokenizeSource(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} function __wbg_get_imports() { const import0 = { __proto__: null, diff --git a/site/decodal-site/src/wasm/decodal_wasm_bg.wasm b/site/decodal-site/src/wasm/decodal_wasm_bg.wasm index ff3605f..8e76bca 100644 Binary files a/site/decodal-site/src/wasm/decodal_wasm_bg.wasm and b/site/decodal-site/src/wasm/decodal_wasm_bg.wasm differ diff --git a/site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts b/site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts index 7af5cbb..4f894cc 100644 --- a/site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts +++ b/site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts @@ -3,7 +3,6 @@ export const memory: WebAssembly.Memory; export const evaluate: (a: number, b: number) => [number, number]; export const evaluateProject: (a: number, b: number, c: number, d: number) => [number, number]; -export const tokenizeSource: (a: number, b: number) => [number, number]; export const __wbindgen_externrefs: WebAssembly.Table; export const __wbindgen_malloc: (a: number, b: number) => number; export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;