Hare/Decodal
1
0
Fork 0
Code Issues Pull Requests Packages Releases Activity

Expose tokenizer for web highlighting

Browse Source
This commit is contained in:
Hare
2026-07-08 19:50:04 +09:00
parent fe127428f9
commit a07f4c48aa
10 changed files with 415 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/decodal-core/src/lexer.rs
+225
View File
@@ -54,6 +54,66 @@ pub enum TokenKind {
Eof,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PublicToken {
pub kind: PublicTokenKind,
pub span: Span,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublicTokenKind {
Ident,
Int,
Float,
String,
Regex,
True,
False,
Let,
In,
Match,
Import,
Default,
Underscore,
LBrace,
RBrace,
LBracket,
RBracket,
LParen,
RParen,
Semicolon,
Comma,
Dot,
Colon,
Equal,
EqualEqual,
Bang,
BangEqual,
Arrow,
Amp,
AmpAmp,
PipePipe,
Plus,
PlusPlus,
Minus,
Star,
Slash,
SlashSlash,
Gt,
Gte,
Lt,
Lte,
Comment,
}
pub fn tokenize_source(source: &str) -> Result<Vec<PublicToken>> {
tokenize_source_with_id(SourceId(0), source)
}
pub fn tokenize_source_with_id(source_id: SourceId, source: &str) -> Result<Vec<PublicToken>> {
Lexer::with_source_id(source_id, source).tokenize_public()
}
pub struct Lexer<'a> {
source_id: SourceId,
source: &'a str,
@@ -91,8 +151,26 @@ impl<'a> Lexer<'a> {
}
}
pub fn tokenize_public(mut self) -> Result<Vec<PublicToken>> {
let mut tokens = Vec::new();
let mut previous = None;
loop {
let Some(token) = self.next_public_token(previous.as_ref())? else {
return Ok(tokens);
};
if token.kind != PublicTokenKind::Comment {
previous = Some(token.kind);
}
tokens.push(token);
}
}
fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
self.skip_ws_and_comments();
self.next_non_ws_token(previous)
}
fn next_non_ws_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
let start = self.pos;
let Some(ch) = self.peek() else {
return Ok(Token {
@@ -242,6 +320,42 @@ impl<'a> Lexer<'a> {
})
}
fn next_public_token(
&mut self,
previous: Option<&PublicTokenKind>,
) -> Result<Option<PublicToken>> {
self.skip_ws();
let start = self.pos;
let Some(ch) = self.peek() else {
return Ok(None);
};
if ch == b'#' {
self.pos += 1;
while !matches!(self.peek(), None | Some(b'\n')) {
self.pos += 1;
}
return Ok(Some(PublicToken {
kind: PublicTokenKind::Comment,
span: self.span(start, self.pos),
}));
}
let previous_token = previous.map(public_kind_to_token_kind);
let token = self.next_non_ws_token(previous_token.as_ref())?;
if token.kind == TokenKind::Eof {
return Ok(None);
}
Ok(Some(PublicToken {
kind: PublicTokenKind::from_token_kind(&token.kind),
span: token.span,
}))
}
fn skip_ws(&mut self) {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
self.pos += 1;
}
}
fn skip_ws_and_comments(&mut self) {
loop {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
@@ -390,6 +504,102 @@ impl<'a> Lexer<'a> {
}
}
impl PublicTokenKind {
fn from_token_kind(kind: &TokenKind) -> Self {
match kind {
TokenKind::Ident(_) => Self::Ident,
TokenKind::Int(_) => Self::Int,
TokenKind::Float(_) => Self::Float,
TokenKind::String(_) => Self::String,
TokenKind::Regex(_) => Self::Regex,
TokenKind::True => Self::True,
TokenKind::False => Self::False,
TokenKind::Let => Self::Let,
TokenKind::In => Self::In,
TokenKind::Match => Self::Match,
TokenKind::Import => Self::Import,
TokenKind::Default => Self::Default,
TokenKind::Underscore => Self::Underscore,
TokenKind::LBrace => Self::LBrace,
TokenKind::RBrace => Self::RBrace,
TokenKind::LBracket => Self::LBracket,
TokenKind::RBracket => Self::RBracket,
TokenKind::LParen => Self::LParen,
TokenKind::RParen => Self::RParen,
TokenKind::Semicolon => Self::Semicolon,
TokenKind::Comma => Self::Comma,
TokenKind::Dot => Self::Dot,
TokenKind::Colon => Self::Colon,
TokenKind::Equal => Self::Equal,
TokenKind::EqualEqual => Self::EqualEqual,
TokenKind::Bang => Self::Bang,
TokenKind::BangEqual => Self::BangEqual,
TokenKind::Arrow => Self::Arrow,
TokenKind::Amp => Self::Amp,
TokenKind::AmpAmp => Self::AmpAmp,
TokenKind::PipePipe => Self::PipePipe,
TokenKind::Plus => Self::Plus,
TokenKind::PlusPlus => Self::PlusPlus,
TokenKind::Minus => Self::Minus,
TokenKind::Star => Self::Star,
TokenKind::Slash => Self::Slash,
TokenKind::SlashSlash => Self::SlashSlash,
TokenKind::Gt => Self::Gt,
TokenKind::Gte => Self::Gte,
TokenKind::Lt => Self::Lt,
TokenKind::Lte => Self::Lte,
TokenKind::Eof => unreachable!(),
}
}
}
fn public_kind_to_token_kind(kind: &PublicTokenKind) -> TokenKind {
match kind {
PublicTokenKind::Ident => TokenKind::Ident(String::new()),
PublicTokenKind::Int => TokenKind::Int(0),
PublicTokenKind::Float => TokenKind::Float(0.0),
PublicTokenKind::String => TokenKind::String(String::new()),
PublicTokenKind::Regex => TokenKind::Regex(String::new()),
PublicTokenKind::True => TokenKind::True,
PublicTokenKind::False => TokenKind::False,
PublicTokenKind::Let => TokenKind::Let,
PublicTokenKind::In => TokenKind::In,
PublicTokenKind::Match => TokenKind::Match,
PublicTokenKind::Import => TokenKind::Import,
PublicTokenKind::Default => TokenKind::Default,
PublicTokenKind::Underscore => TokenKind::Underscore,
PublicTokenKind::LBrace => TokenKind::LBrace,
PublicTokenKind::RBrace => TokenKind::RBrace,
PublicTokenKind::LBracket => TokenKind::LBracket,
PublicTokenKind::RBracket => TokenKind::RBracket,
PublicTokenKind::LParen => TokenKind::LParen,
PublicTokenKind::RParen => TokenKind::RParen,
PublicTokenKind::Semicolon => TokenKind::Semicolon,
PublicTokenKind::Comma => TokenKind::Comma,
PublicTokenKind::Dot => TokenKind::Dot,
PublicTokenKind::Colon => TokenKind::Colon,
PublicTokenKind::Equal => TokenKind::Equal,
PublicTokenKind::EqualEqual => TokenKind::EqualEqual,
PublicTokenKind::Bang => TokenKind::Bang,
PublicTokenKind::BangEqual => TokenKind::BangEqual,
PublicTokenKind::Arrow => TokenKind::Arrow,
PublicTokenKind::Amp => TokenKind::Amp,
PublicTokenKind::AmpAmp => TokenKind::AmpAmp,
PublicTokenKind::PipePipe => TokenKind::PipePipe,
PublicTokenKind::Plus => TokenKind::Plus,
PublicTokenKind::PlusPlus => TokenKind::PlusPlus,
PublicTokenKind::Minus => TokenKind::Minus,
PublicTokenKind::Star => TokenKind::Star,
PublicTokenKind::Slash => TokenKind::Slash,
PublicTokenKind::SlashSlash => TokenKind::SlashSlash,
PublicTokenKind::Gt => TokenKind::Gt,
PublicTokenKind::Gte => TokenKind::Gte,
PublicTokenKind::Lt => TokenKind::Lt,
PublicTokenKind::Lte => TokenKind::Lte,
PublicTokenKind::Comment => TokenKind::Underscore,
}
}
fn is_ident_start(c: u8) -> bool {
c.is_ascii_alphabetic()
}
@@ -427,4 +637,19 @@ mod tests {
assert_eq!(tokens[3].kind, TokenKind::Amp);
assert_eq!(tokens[4].kind, TokenKind::Gte);
}
#[test]
fn public_tokenization_keeps_comments_and_spans() {
let source = "# hello\nport = [1] ++ [2];";
let tokens = tokenize_source(source).unwrap();
assert_eq!(tokens[0].kind, PublicTokenKind::Comment);
let comment = &source[tokens[0].span.start as usize..tokens[0].span.end as usize];
assert_eq!(comment, "# hello");
assert_eq!(tokens[1].kind, PublicTokenKind::Ident);
assert!(
tokens
.iter()
.any(|token| token.kind == PublicTokenKind::PlusPlus)
);
}
}
crates/decodal-core/src/lib.rs
+3 -1
View File
@@ -21,7 +21,9 @@ pub use decodal_derive::Decodal;
pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
pub use embedding::{HostField, HostValue};
pub use eval::{Engine, format_diagnostic_with};
pub use lexer::{Lexer, Token, TokenKind};
pub use lexer::{
Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id,
};
pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};
crates/decodal-wasm/src/lib.rs
+83 -2
View File
@@ -1,8 +1,8 @@
use std::collections::BTreeMap;
use decodal::{
Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader,
Span, format_diagnostic_with,
Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, PublicTokenKind, SourceId,
SourceLoader, Span, format_diagnostic_with, tokenize_source,
};
use wasm_bindgen::prelude::*;
@@ -16,6 +16,33 @@ pub fn evaluate_project(entry: &str, files_json: &str) -> String {
encode_result(evaluate_project_inner(entry, files_json))
}
#[wasm_bindgen(js_name = tokenizeSource)]
pub fn tokenize_source_json(source: &str) -> String {
match tokenize_source(source) {
Ok(tokens) => {
let mut out = String::from("{\"ok\":true,\"tokens\":[");
for (index, token) in tokens.iter().enumerate() {
if index > 0 {
out.push(',');
}
out.push_str("{\"kind\":");
out.push_str(&json_string(public_token_kind_name(token.kind)));
out.push_str(",\"start\":");
out.push_str(&token.span.start.to_string());
out.push_str(",\"end\":");
out.push_str(&token.span.end.to_string());
out.push('}');
}
out.push_str("]}");
out
}
Err(error) => format!(
"{{\"ok\":false,\"error\":{}}}",
json_string(&format_diagnostic_with_root(&error, "playground"))
),
}
}
fn encode_result(result: Result<String, String>) -> String {
match result {
Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)),
@@ -145,6 +172,53 @@ fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str
})
}
fn public_token_kind_name(kind: PublicTokenKind) -> &'static str {
match kind {
PublicTokenKind::Ident => "ident",
PublicTokenKind::Int => "int",
PublicTokenKind::Float => "float",
PublicTokenKind::String => "string",
PublicTokenKind::Regex => "regex",
PublicTokenKind::True => "true",
PublicTokenKind::False => "false",
PublicTokenKind::Let => "let",
PublicTokenKind::In => "in",
PublicTokenKind::Match => "match",
PublicTokenKind::Import => "import",
PublicTokenKind::Default => "default",
PublicTokenKind::Underscore => "underscore",
PublicTokenKind::LBrace => "l_brace",
PublicTokenKind::RBrace => "r_brace",
PublicTokenKind::LBracket => "l_bracket",
PublicTokenKind::RBracket => "r_bracket",
PublicTokenKind::LParen => "l_paren",
PublicTokenKind::RParen => "r_paren",
PublicTokenKind::Semicolon => "semicolon",
PublicTokenKind::Comma => "comma",
PublicTokenKind::Dot => "dot",
PublicTokenKind::Colon => "colon",
PublicTokenKind::Equal => "equal",
PublicTokenKind::EqualEqual => "equal_equal",
PublicTokenKind::Bang => "bang",
PublicTokenKind::BangEqual => "bang_equal",
PublicTokenKind::Arrow => "arrow",
PublicTokenKind::Amp => "amp",
PublicTokenKind::AmpAmp => "amp_amp",
PublicTokenKind::PipePipe => "pipe_pipe",
PublicTokenKind::Plus => "plus",
PublicTokenKind::PlusPlus => "plus_plus",
PublicTokenKind::Minus => "minus",
PublicTokenKind::Star => "star",
PublicTokenKind::Slash => "slash",
PublicTokenKind::SlashSlash => "slash_slash",
PublicTokenKind::Gt => "gt",
PublicTokenKind::Gte => "gte",
PublicTokenKind::Lt => "lt",
PublicTokenKind::Lte => "lte",
PublicTokenKind::Comment => "comment",
}
}
fn format_data(data: &Data, indent: usize) -> String {
match data {
Data::String(value) => json_string(value),
@@ -231,6 +305,13 @@ mod tests {
);
}
#[test]
fn tokenizes_source_for_web() {
let output = super::tokenize_source_json("# hi\nport = [1] ++ [2];");
assert!(output.contains("\"kind\":\"comment\""));
assert!(output.contains("\"kind\":\"plus_plus\""));
}
#[test]
fn evaluates_project_imports() {
let files = r#"{