Expose tokenizer for web highlighting

2026-07-08 19:50:04 +09:00
parent fe127428f9
commit a07f4c48aa
10 changed files with 415 additions and 6 deletions
@@ -21,7 +21,9 @@ pub use decodal_derive::Decodal;
 pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
 pub use embedding::{HostField, HostValue};
 pub use eval::{Engine, format_diagnostic_with};
-pub use lexer::{Lexer, Token, TokenKind};
+pub use lexer::{
+    Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id,
+};
 pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
 pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
 pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};