Hare/Decodal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove public tokenizer API

Browse Source
This commit is contained in:
Keisuke Hirata 2026-07-09 01:09:40 +09:00
parent c4332ffb9d
commit 26d8495631
No known key found for this signature in database
8 changed files with 4 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

222
crates/decodal-core/src/lexer.rs
View File

@ -54,66 +54,6 @@ pub enum TokenKind {
Eof,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PublicToken {
pub kind: PublicTokenKind,
pub span: Span,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublicTokenKind {
Ident,
Int,
Float,
String,
Regex,
True,
False,
Let,
In,
Match,
Import,
Default,
Underscore,
LBrace,
RBrace,
LBracket,
RBracket,
LParen,
RParen,
Semicolon,
Comma,
Dot,
Colon,
Equal,
EqualEqual,
Bang,
BangEqual,
Arrow,
Amp,
AmpAmp,
PipePipe,
Plus,
PlusPlus,
Minus,
Star,
Slash,
SlashSlash,
Gt,
Gte,
Lt,
Lte,
Comment,
}
pub fn tokenize_source(source: &str) -> Result<Vec<PublicToken>> {
tokenize_source_with_id(SourceId(0), source)
}
pub fn tokenize_source_with_id(source_id: SourceId, source: &str) -> Result<Vec<PublicToken>> {
Lexer::with_source_id(source_id, source).tokenize_public()
}
pub struct Lexer<'a> {
source_id: SourceId,
source: &'a str,
@ -122,6 +62,7 @@ pub struct Lexer<'a> {
}
impl<'a> Lexer<'a> {
#[cfg(test)]
pub fn new(source: &'a str) -> Self {
Self::with_source_id(SourceId(0), source)
}
@ -151,20 +92,6 @@ impl<'a> Lexer<'a> {
}
}
pub fn tokenize_public(mut self) -> Result<Vec<PublicToken>> {
let mut tokens = Vec::new();
let mut previous = None;
loop {
let Some(token) = self.next_public_token(previous.as_ref())? else {
return Ok(tokens);
};
if token.kind != PublicTokenKind::Comment {
previous = Some(token.kind);
}
tokens.push(token);
}
}
fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
self.skip_ws_and_comments();
self.next_non_ws_token(previous)
@ -320,42 +247,6 @@ impl<'a> Lexer<'a> {
})
}
fn next_public_token(
&mut self,
previous: Option<&PublicTokenKind>,
) -> Result<Option<PublicToken>> {
self.skip_ws();
let start = self.pos;
let Some(ch) = self.peek() else {
return Ok(None);
};
if ch == b'#' {
self.pos += 1;
while !matches!(self.peek(), None | Some(b'\n')) {
self.pos += 1;
}
return Ok(Some(PublicToken {
kind: PublicTokenKind::Comment,
span: self.span(start, self.pos),
}));
}
let previous_token = previous.map(public_kind_to_token_kind);
let token = self.next_non_ws_token(previous_token.as_ref())?;
if token.kind == TokenKind::Eof {
return Ok(None);
}
Ok(Some(PublicToken {
kind: PublicTokenKind::from_token_kind(&token.kind),
span: token.span,
}))
}
fn skip_ws(&mut self) {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
self.pos += 1;
}
}
fn skip_ws_and_comments(&mut self) {
loop {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
@ -504,102 +395,6 @@ impl<'a> Lexer<'a> {
}
}
impl PublicTokenKind {
fn from_token_kind(kind: &TokenKind) -> Self {
match kind {
TokenKind::Ident(_) => Self::Ident,
TokenKind::Int(_) => Self::Int,
TokenKind::Float(_) => Self::Float,
TokenKind::String(_) => Self::String,
TokenKind::Regex(_) => Self::Regex,
TokenKind::True => Self::True,
TokenKind::False => Self::False,
TokenKind::Let => Self::Let,
TokenKind::In => Self::In,
TokenKind::Match => Self::Match,
TokenKind::Import => Self::Import,
TokenKind::Default => Self::Default,
TokenKind::Underscore => Self::Underscore,
TokenKind::LBrace => Self::LBrace,
TokenKind::RBrace => Self::RBrace,
TokenKind::LBracket => Self::LBracket,
TokenKind::RBracket => Self::RBracket,
TokenKind::LParen => Self::LParen,
TokenKind::RParen => Self::RParen,
TokenKind::Semicolon => Self::Semicolon,
TokenKind::Comma => Self::Comma,
TokenKind::Dot => Self::Dot,
TokenKind::Colon => Self::Colon,
TokenKind::Equal => Self::Equal,
TokenKind::EqualEqual => Self::EqualEqual,
TokenKind::Bang => Self::Bang,
TokenKind::BangEqual => Self::BangEqual,
TokenKind::Arrow => Self::Arrow,
TokenKind::Amp => Self::Amp,
TokenKind::AmpAmp => Self::AmpAmp,
TokenKind::PipePipe => Self::PipePipe,
TokenKind::Plus => Self::Plus,
TokenKind::PlusPlus => Self::PlusPlus,
TokenKind::Minus => Self::Minus,
TokenKind::Star => Self::Star,
TokenKind::Slash => Self::Slash,
TokenKind::SlashSlash => Self::SlashSlash,
TokenKind::Gt => Self::Gt,
TokenKind::Gte => Self::Gte,
TokenKind::Lt => Self::Lt,
TokenKind::Lte => Self::Lte,
TokenKind::Eof => unreachable!(),
}
}
}
fn public_kind_to_token_kind(kind: &PublicTokenKind) -> TokenKind {
match kind {
PublicTokenKind::Ident => TokenKind::Ident(String::new()),
PublicTokenKind::Int => TokenKind::Int(0),
PublicTokenKind::Float => TokenKind::Float(0.0),
PublicTokenKind::String => TokenKind::String(String::new()),
PublicTokenKind::Regex => TokenKind::Regex(String::new()),
PublicTokenKind::True => TokenKind::True,
PublicTokenKind::False => TokenKind::False,
PublicTokenKind::Let => TokenKind::Let,
PublicTokenKind::In => TokenKind::In,
PublicTokenKind::Match => TokenKind::Match,
PublicTokenKind::Import => TokenKind::Import,
PublicTokenKind::Default => TokenKind::Default,
PublicTokenKind::Underscore => TokenKind::Underscore,
PublicTokenKind::LBrace => TokenKind::LBrace,
PublicTokenKind::RBrace => TokenKind::RBrace,
PublicTokenKind::LBracket => TokenKind::LBracket,
PublicTokenKind::RBracket => TokenKind::RBracket,
PublicTokenKind::LParen => TokenKind::LParen,
PublicTokenKind::RParen => TokenKind::RParen,
PublicTokenKind::Semicolon => TokenKind::Semicolon,
PublicTokenKind::Comma => TokenKind::Comma,
PublicTokenKind::Dot => TokenKind::Dot,
PublicTokenKind::Colon => TokenKind::Colon,
PublicTokenKind::Equal => TokenKind::Equal,
PublicTokenKind::EqualEqual => TokenKind::EqualEqual,
PublicTokenKind::Bang => TokenKind::Bang,
PublicTokenKind::BangEqual => TokenKind::BangEqual,
PublicTokenKind::Arrow => TokenKind::Arrow,
PublicTokenKind::Amp => TokenKind::Amp,
PublicTokenKind::AmpAmp => TokenKind::AmpAmp,
PublicTokenKind::PipePipe => TokenKind::PipePipe,
PublicTokenKind::Plus => TokenKind::Plus,
PublicTokenKind::PlusPlus => TokenKind::PlusPlus,
PublicTokenKind::Minus => TokenKind::Minus,
PublicTokenKind::Star => TokenKind::Star,
PublicTokenKind::Slash => TokenKind::Slash,
PublicTokenKind::SlashSlash => TokenKind::SlashSlash,
PublicTokenKind::Gt => TokenKind::Gt,
PublicTokenKind::Gte => TokenKind::Gte,
PublicTokenKind::Lt => TokenKind::Lt,
PublicTokenKind::Lte => TokenKind::Lte,
PublicTokenKind::Comment => TokenKind::Underscore,
}
}
fn is_ident_start(c: u8) -> bool {
c.is_ascii_alphabetic()
}
@ -637,19 +432,4 @@ mod tests {
assert_eq!(tokens[3].kind, TokenKind::Amp);
assert_eq!(tokens[4].kind, TokenKind::Gte);
}
#[test]
fn public_tokenization_keeps_comments_and_spans() {
let source = "# hello\nport = [1] ++ [2];";
let tokens = tokenize_source(source).unwrap();
assert_eq!(tokens[0].kind, PublicTokenKind::Comment);
let comment = &source[tokens[0].span.start as usize..tokens[0].span.end as usize];
assert_eq!(comment, "# hello");
assert_eq!(tokens[1].kind, PublicTokenKind::Ident);
assert!(
tokens
.iter()
.any(|token| token.kind == PublicTokenKind::PlusPlus)
);
}
}

5
crates/decodal-core/src/lib.rs
View File

@ -7,7 +7,7 @@ pub mod constraints;
pub mod diagnostic;
pub mod embedding;
pub mod eval;
pub mod lexer;
mod lexer;
pub mod module;
pub mod parser;
pub mod runtime;
@ -21,9 +21,6 @@ pub use decodal_derive::Decodal;
pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
pub use embedding::{HostField, HostValue};
pub use eval::{Engine, format_diagnostic_with};
pub use lexer::{
Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id,
};
pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};

85
crates/decodal-wasm/src/lib.rs
View File

@ -1,8 +1,8 @@
use std::collections::BTreeMap;
use decodal::{
Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, PublicTokenKind, SourceId,
SourceLoader, Span, format_diagnostic_with, tokenize_source,
Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader,
Span, format_diagnostic_with,
};
use wasm_bindgen::prelude::*;
@ -16,33 +16,6 @@ pub fn evaluate_project(entry: &str, files_json: &str) -> String {
encode_result(evaluate_project_inner(entry, files_json))
}
#[wasm_bindgen(js_name = tokenizeSource)]
pub fn tokenize_source_json(source: &str) -> String {
match tokenize_source(source) {
Ok(tokens) => {
let mut out = String::from("{\"ok\":true,\"tokens\":[");
for (index, token) in tokens.iter().enumerate() {
if index > 0 {
out.push(',');
}
out.push_str("{\"kind\":");
out.push_str(&json_string(public_token_kind_name(token.kind)));
out.push_str(",\"start\":");
out.push_str(&token.span.start.to_string());
out.push_str(",\"end\":");
out.push_str(&token.span.end.to_string());
out.push('}');
}
out.push_str("]}");
out
}
Err(error) => format!(
"{{\"ok\":false,\"error\":{}}}",
json_string(&format_diagnostic_with_root(&error, "playground"))
),
}
}
fn encode_result(result: Result<String, String>) -> String {
match result {
Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)),
@ -172,53 +145,6 @@ fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str
})
}
fn public_token_kind_name(kind: PublicTokenKind) -> &'static str {
match kind {
PublicTokenKind::Ident => "ident",
PublicTokenKind::Int => "int",
PublicTokenKind::Float => "float",
PublicTokenKind::String => "string",
PublicTokenKind::Regex => "regex",
PublicTokenKind::True => "true",
PublicTokenKind::False => "false",
PublicTokenKind::Let => "let",
PublicTokenKind::In => "in",
PublicTokenKind::Match => "match",
PublicTokenKind::Import => "import",
PublicTokenKind::Default => "default",
PublicTokenKind::Underscore => "underscore",
PublicTokenKind::LBrace => "l_brace",
PublicTokenKind::RBrace => "r_brace",
PublicTokenKind::LBracket => "l_bracket",
PublicTokenKind::RBracket => "r_bracket",
PublicTokenKind::LParen => "l_paren",
PublicTokenKind::RParen => "r_paren",
PublicTokenKind::Semicolon => "semicolon",
PublicTokenKind::Comma => "comma",
PublicTokenKind::Dot => "dot",
PublicTokenKind::Colon => "colon",
PublicTokenKind::Equal => "equal",
PublicTokenKind::EqualEqual => "equal_equal",
PublicTokenKind::Bang => "bang",
PublicTokenKind::BangEqual => "bang_equal",
PublicTokenKind::Arrow => "arrow",
PublicTokenKind::Amp => "amp",
PublicTokenKind::AmpAmp => "amp_amp",
PublicTokenKind::PipePipe => "pipe_pipe",
PublicTokenKind::Plus => "plus",
PublicTokenKind::PlusPlus => "plus_plus",
PublicTokenKind::Minus => "minus",
PublicTokenKind::Star => "star",
PublicTokenKind::Slash => "slash",
PublicTokenKind::SlashSlash => "slash_slash",
PublicTokenKind::Gt => "gt",
PublicTokenKind::Gte => "gte",
PublicTokenKind::Lt => "lt",
PublicTokenKind::Lte => "lte",
PublicTokenKind::Comment => "comment",
}
}
fn format_data(data: &Data, indent: usize) -> String {
match data {
Data::String(value) => json_string(value),
@ -305,13 +231,6 @@ mod tests {
);
}
#[test]
fn tokenizes_source_for_web() {
let output = super::tokenize_source_json("# hi\nport = [1] ++ [2];");
assert!(output.contains("\"kind\":\"comment\""));
assert!(output.contains("\"kind\":\"plus_plus\""));
}
#[test]
fn evaluates_project_imports() {
let files = r#"{

1
doc/manual/souce/language/grammar.md
View File

@ -27,7 +27,6 @@ regex_character = escape | ? any character except "/", "\\", or newline ? ;
```
Whitespace and comments separate tokens and are otherwise ignored by the parser.
Comments are retained by public tokenizer APIs for editor tooling.
## Syntactic grammar

3
site/decodal-site/src/wasm/decodal_wasm.d.ts vendored
View File

@ -5,15 +5,12 @@ export function evaluate(source: string): string;
export function evaluateProject(entry: string, files_json: string): string;
export function tokenizeSource(source: string): string;
export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
export interface InitOutput {
readonly memory: WebAssembly.Memory;
readonly evaluate: (a: number, b: number) => [number, number];
readonly evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
readonly tokenizeSource: (a: number, b: number) => [number, number];
readonly __wbindgen_externrefs: WebAssembly.Table;
readonly __wbindgen_malloc: (a: number, b: number) => number;
readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;

19
site/decodal-site/src/wasm/decodal_wasm.js
View File

@ -40,25 +40,6 @@ export function evaluateProject(entry, files_json) {
wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
}
}
/**
* @param {string} source
* @returns {string}
*/
export function tokenizeSource(source) {
let deferred2_0;
let deferred2_1;
try {
const ptr0 = passStringToWasm0(source, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.tokenizeSource(ptr0, len0);
deferred2_0 = ret[0];
deferred2_1 = ret[1];
return getStringFromWasm0(ret[0], ret[1]);
} finally {
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
}
}
function __wbg_get_imports() {
const import0 = {
__proto__: null,

BIN
site/decodal-site/src/wasm/decodal_wasm_bg.wasm
View File

Binary file not shown.

1
site/decodal-site/src/wasm/decodal_wasm_bg.wasm.d.ts vendored
View File

@ -3,7 +3,6 @@
export const memory: WebAssembly.Memory;
export const evaluate: (a: number, b: number) => [number, number];
export const evaluateProject: (a: number, b: number, c: number, d: number) => [number, number];
export const tokenizeSource: (a: number, b: number) => [number, number];
export const __wbindgen_externrefs: WebAssembly.Table;
export const __wbindgen_malloc: (a: number, b: number) => number;
export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;