Expose tokenizer for web highlighting

crates/decodal-wasm/src/lib.rs

@@ -1,8 +1,8 @@
 use std::collections::BTreeMap;
 
 use decodal::{
-    Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader,
-    Span, format_diagnostic_with,
+    Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, PublicTokenKind, SourceId,
+    SourceLoader, Span, format_diagnostic_with, tokenize_source,
 };
 use wasm_bindgen::prelude::*;
 
@@ -16,6 +16,33 @@ pub fn evaluate_project(entry: &str, files_json: &str) -> String {
     encode_result(evaluate_project_inner(entry, files_json))
 }
 
+#[wasm_bindgen(js_name = tokenizeSource)]
+pub fn tokenize_source_json(source: &str) -> String {
+    match tokenize_source(source) {
+        Ok(tokens) => {
+            let mut out = String::from("{\"ok\":true,\"tokens\":[");
+            for (index, token) in tokens.iter().enumerate() {
+                if index > 0 {
+                    out.push(',');
+                }
+                out.push_str("{\"kind\":");
+                out.push_str(&json_string(public_token_kind_name(token.kind)));
+                out.push_str(",\"start\":");
+                out.push_str(&token.span.start.to_string());
+                out.push_str(",\"end\":");
+                out.push_str(&token.span.end.to_string());
+                out.push('}');
+            }
+            out.push_str("]}");
+            out
+        }
+        Err(error) => format!(
+            "{{\"ok\":false,\"error\":{}}}",
+            json_string(&format_diagnostic_with_root(&error, "playground"))
+        ),
+    }
+}
+
 fn encode_result(result: Result<String, String>) -> String {
     match result {
         Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)),
@@ -145,6 +172,53 @@ fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str
     })
 }
 
+fn public_token_kind_name(kind: PublicTokenKind) -> &'static str {
+    match kind {
+        PublicTokenKind::Ident => "ident",
+        PublicTokenKind::Int => "int",
+        PublicTokenKind::Float => "float",
+        PublicTokenKind::String => "string",
+        PublicTokenKind::Regex => "regex",
+        PublicTokenKind::True => "true",
+        PublicTokenKind::False => "false",
+        PublicTokenKind::Let => "let",
+        PublicTokenKind::In => "in",
+        PublicTokenKind::Match => "match",
+        PublicTokenKind::Import => "import",
+        PublicTokenKind::Default => "default",
+        PublicTokenKind::Underscore => "underscore",
+        PublicTokenKind::LBrace => "l_brace",
+        PublicTokenKind::RBrace => "r_brace",
+        PublicTokenKind::LBracket => "l_bracket",
+        PublicTokenKind::RBracket => "r_bracket",
+        PublicTokenKind::LParen => "l_paren",
+        PublicTokenKind::RParen => "r_paren",
+        PublicTokenKind::Semicolon => "semicolon",
+        PublicTokenKind::Comma => "comma",
+        PublicTokenKind::Dot => "dot",
+        PublicTokenKind::Colon => "colon",
+        PublicTokenKind::Equal => "equal",
+        PublicTokenKind::EqualEqual => "equal_equal",
+        PublicTokenKind::Bang => "bang",
+        PublicTokenKind::BangEqual => "bang_equal",
+        PublicTokenKind::Arrow => "arrow",
+        PublicTokenKind::Amp => "amp",
+        PublicTokenKind::AmpAmp => "amp_amp",
+        PublicTokenKind::PipePipe => "pipe_pipe",
+        PublicTokenKind::Plus => "plus",
+        PublicTokenKind::PlusPlus => "plus_plus",
+        PublicTokenKind::Minus => "minus",
+        PublicTokenKind::Star => "star",
+        PublicTokenKind::Slash => "slash",
+        PublicTokenKind::SlashSlash => "slash_slash",
+        PublicTokenKind::Gt => "gt",
+        PublicTokenKind::Gte => "gte",
+        PublicTokenKind::Lt => "lt",
+        PublicTokenKind::Lte => "lte",
+        PublicTokenKind::Comment => "comment",
+    }
+}
+
 fn format_data(data: &Data, indent: usize) -> String {
     match data {
         Data::String(value) => json_string(value),
@@ -231,6 +305,13 @@ mod tests {
         );
     }
 
+    #[test]
+    fn tokenizes_source_for_web() {
+        let output = super::tokenize_source_json("# hi\nport = [1] ++ [2];");
+        assert!(output.contains("\"kind\":\"comment\""));
+        assert!(output.contains("\"kind\":\"plus_plus\""));
+    }
+
     #[test]
     fn evaluates_project_imports() {
         let files = r#"{