Remove public tokenizer API

crates/decodal-core/src/lib.rs

@@ -7,7 +7,7 @@ pub mod constraints;
 pub mod diagnostic;
 pub mod embedding;
 pub mod eval;
-pub mod lexer;
+mod lexer;
 pub mod module;
 pub mod parser;
 pub mod runtime;
@@ -21,9 +21,6 @@ pub use decodal_derive::Decodal;
 pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
 pub use embedding::{HostField, HostValue};
 pub use eval::{Engine, format_diagnostic_with};
-pub use lexer::{
-    Lexer, PublicToken, PublicTokenKind, Token, TokenKind, tokenize_source, tokenize_source_with_id,
-};
 pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
 pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
 pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};