Hare/Decodal
1
0
Fork 0
Code Issues Pull Requests Packages Releases Activity

Remove public tokenizer API

Browse Source
This commit is contained in:
Hare
2026-07-09 01:09:40 +09:00
parent c4332ffb9d
commit 26d8495631
8 changed files with 4 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/decodal-core/src/lexer.rs
+1 -221
View File
@@ -54,66 +54,6 @@ pub enum TokenKind {
Eof,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PublicToken {
pub kind: PublicTokenKind,
pub span: Span,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublicTokenKind {
Ident,
Int,
Float,
String,
Regex,
True,
False,
Let,
In,
Match,
Import,
Default,
Underscore,
LBrace,
RBrace,
LBracket,
RBracket,
LParen,
RParen,
Semicolon,
Comma,
Dot,
Colon,
Equal,
EqualEqual,
Bang,
BangEqual,
Arrow,
Amp,
AmpAmp,
PipePipe,
Plus,
PlusPlus,
Minus,
Star,
Slash,
SlashSlash,
Gt,
Gte,
Lt,
Lte,
Comment,
}
pub fn tokenize_source(source: &str) -> Result<Vec<PublicToken>> {
tokenize_source_with_id(SourceId(0), source)
}
pub fn tokenize_source_with_id(source_id: SourceId, source: &str) -> Result<Vec<PublicToken>> {
Lexer::with_source_id(source_id, source).tokenize_public()
}
pub struct Lexer<'a> {
source_id: SourceId,
source: &'a str,
@@ -122,6 +62,7 @@ pub struct Lexer<'a> {
}
impl<'a> Lexer<'a> {
#[cfg(test)]
pub fn new(source: &'a str) -> Self {
Self::with_source_id(SourceId(0), source)
}
@@ -151,20 +92,6 @@ impl<'a> Lexer<'a> {
}
}
pub fn tokenize_public(mut self) -> Result<Vec<PublicToken>> {
let mut tokens = Vec::new();
let mut previous = None;
loop {
let Some(token) = self.next_public_token(previous.as_ref())? else {
return Ok(tokens);
};
if token.kind != PublicTokenKind::Comment {
previous = Some(token.kind);
}
tokens.push(token);
}
}
fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
self.skip_ws_and_comments();
self.next_non_ws_token(previous)
@@ -320,42 +247,6 @@ impl<'a> Lexer<'a> {
})
}
fn next_public_token(
&mut self,
previous: Option<&PublicTokenKind>,
) -> Result<Option<PublicToken>> {
self.skip_ws();
let start = self.pos;
let Some(ch) = self.peek() else {
return Ok(None);
};
if ch == b'#' {
self.pos += 1;
while !matches!(self.peek(), None | Some(b'\n')) {
self.pos += 1;
}
return Ok(Some(PublicToken {
kind: PublicTokenKind::Comment,
span: self.span(start, self.pos),
}));
}
let previous_token = previous.map(public_kind_to_token_kind);
let token = self.next_non_ws_token(previous_token.as_ref())?;
if token.kind == TokenKind::Eof {
return Ok(None);
}
Ok(Some(PublicToken {
kind: PublicTokenKind::from_token_kind(&token.kind),
span: token.span,
}))
}
fn skip_ws(&mut self) {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
self.pos += 1;
}
}
fn skip_ws_and_comments(&mut self) {
loop {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
@@ -504,102 +395,6 @@ impl<'a> Lexer<'a> {
}
}
impl PublicTokenKind {
fn from_token_kind(kind: &TokenKind) -> Self {
match kind {
TokenKind::Ident(_) => Self::Ident,
TokenKind::Int(_) => Self::Int,
TokenKind::Float(_) => Self::Float,
TokenKind::String(_) => Self::String,
TokenKind::Regex(_) => Self::Regex,
TokenKind::True => Self::True,
TokenKind::False => Self::False,
TokenKind::Let => Self::Let,
TokenKind::In => Self::In,
TokenKind::Match => Self::Match,
TokenKind::Import => Self::Import,
TokenKind::Default => Self::Default,
TokenKind::Underscore => Self::Underscore,
TokenKind::LBrace => Self::LBrace,
TokenKind::RBrace => Self::RBrace,
TokenKind::LBracket => Self::LBracket,
TokenKind::RBracket => Self::RBracket,
TokenKind::LParen => Self::LParen,
TokenKind::RParen => Self::RParen,
TokenKind::Semicolon => Self::Semicolon,
TokenKind::Comma => Self::Comma,
TokenKind::Dot => Self::Dot,
TokenKind::Colon => Self::Colon,
TokenKind::Equal => Self::Equal,
TokenKind::EqualEqual => Self::EqualEqual,
TokenKind::Bang => Self::Bang,
TokenKind::BangEqual => Self::BangEqual,
TokenKind::Arrow => Self::Arrow,
TokenKind::Amp => Self::Amp,
TokenKind::AmpAmp => Self::AmpAmp,
TokenKind::PipePipe => Self::PipePipe,
TokenKind::Plus => Self::Plus,
TokenKind::PlusPlus => Self::PlusPlus,
TokenKind::Minus => Self::Minus,
TokenKind::Star => Self::Star,
TokenKind::Slash => Self::Slash,
TokenKind::SlashSlash => Self::SlashSlash,
TokenKind::Gt => Self::Gt,
TokenKind::Gte => Self::Gte,
TokenKind::Lt => Self::Lt,
TokenKind::Lte => Self::Lte,
TokenKind::Eof => unreachable!(),
}
}
}
fn public_kind_to_token_kind(kind: &PublicTokenKind) -> TokenKind {
match kind {
PublicTokenKind::Ident => TokenKind::Ident(String::new()),
PublicTokenKind::Int => TokenKind::Int(0),
PublicTokenKind::Float => TokenKind::Float(0.0),
PublicTokenKind::String => TokenKind::String(String::new()),
PublicTokenKind::Regex => TokenKind::Regex(String::new()),
PublicTokenKind::True => TokenKind::True,
PublicTokenKind::False => TokenKind::False,
PublicTokenKind::Let => TokenKind::Let,
PublicTokenKind::In => TokenKind::In,
PublicTokenKind::Match => TokenKind::Match,
PublicTokenKind::Import => TokenKind::Import,
PublicTokenKind::Default => TokenKind::Default,
PublicTokenKind::Underscore => TokenKind::Underscore,
PublicTokenKind::LBrace => TokenKind::LBrace,
PublicTokenKind::RBrace => TokenKind::RBrace,
PublicTokenKind::LBracket => TokenKind::LBracket,
PublicTokenKind::RBracket => TokenKind::RBracket,
PublicTokenKind::LParen => TokenKind::LParen,
PublicTokenKind::RParen => TokenKind::RParen,
PublicTokenKind::Semicolon => TokenKind::Semicolon,
PublicTokenKind::Comma => TokenKind::Comma,
PublicTokenKind::Dot => TokenKind::Dot,
PublicTokenKind::Colon => TokenKind::Colon,
PublicTokenKind::Equal => TokenKind::Equal,
PublicTokenKind::EqualEqual => TokenKind::EqualEqual,
PublicTokenKind::Bang => TokenKind::Bang,
PublicTokenKind::BangEqual => TokenKind::BangEqual,
PublicTokenKind::Arrow => TokenKind::Arrow,
PublicTokenKind::Amp => TokenKind::Amp,
PublicTokenKind::AmpAmp => TokenKind::AmpAmp,
PublicTokenKind::PipePipe => TokenKind::PipePipe,
PublicTokenKind::Plus => TokenKind::Plus,
PublicTokenKind::PlusPlus => TokenKind::PlusPlus,
PublicTokenKind::Minus => TokenKind::Minus,
PublicTokenKind::Star => TokenKind::Star,
PublicTokenKind::Slash => TokenKind::Slash,
PublicTokenKind::SlashSlash => TokenKind::SlashSlash,
PublicTokenKind::Gt => TokenKind::Gt,
PublicTokenKind::Gte => TokenKind::Gte,
PublicTokenKind::Lt => TokenKind::Lt,
PublicTokenKind::Lte => TokenKind::Lte,
PublicTokenKind::Comment => TokenKind::Underscore,
}
}
fn is_ident_start(c: u8) -> bool {
c.is_ascii_alphabetic()
}
@@ -637,19 +432,4 @@ mod tests {
assert_eq!(tokens[3].kind, TokenKind::Amp);
assert_eq!(tokens[4].kind, TokenKind::Gte);
}
#[test]
fn public_tokenization_keeps_comments_and_spans() {
let source = "# hello\nport = [1] ++ [2];";
let tokens = tokenize_source(source).unwrap();
assert_eq!(tokens[0].kind, PublicTokenKind::Comment);
let comment = &source[tokens[0].span.start as usize..tokens[0].span.end as usize];
assert_eq!(comment, "# hello");
assert_eq!(tokens[1].kind, PublicTokenKind::Ident);
assert!(
tokens
.iter()
.any(|token| token.kind == PublicTokenKind::PlusPlus)
);
}
}
crates/decodal-core/src/lib.rs
+1 -4
View File
@@ -7,7 +7,7 @@ pub mod constraints;
pub mod diagnostic;
pub mod embedding;
pub mod eval;
pub mod lexer;
mod lexer;
pub mod module;
pub mod parser;
pub mod runtime;
@@ -21,9 +21,6 @@ pub use decodal_derive::Decodal;
pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
pub use embedding::{HostField, HostValue};
pub use eval::{Engine, format_diagnostic_with};
pub use lexer::{
Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id,
};
pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};