Add language tools package

This commit is contained in:
2026-07-09 18:34:41 +09:00
parent ac99c4cb4e
commit 0e2a7b35d2
20 changed files with 1434 additions and 16 deletions
Generated
+51
View File
@@ -17,6 +17,16 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "cc"
version = "1.2.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996"
dependencies = [
"find-msvc-tools",
"shlex",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -48,6 +58,17 @@ dependencies = [
"syn",
]
[[package]]
name = "decodal-language-tools"
version = "0.1.2"
dependencies = [
"decodal",
"serde_json",
"tree-sitter",
"tree-sitter-decodal",
"wasm-bindgen",
]
[[package]]
name = "decodal-wasm"
version = "0.1.2"
@@ -57,6 +78,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "itoa"
version = "1.0.18"
@@ -170,6 +197,12 @@ dependencies = [
"zmij",
]
[[package]]
name = "shlex"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
[[package]]
name = "syn"
version = "2.0.117"
@@ -181,6 +214,24 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tree-sitter"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df7cc499ceadd4dcdf7ec6d4cbc34ece92c3fa07821e287aedecd4416c516dca"
dependencies = [
"cc",
"regex",
]
[[package]]
name = "tree-sitter-decodal"
version = "0.0.1"
dependencies = [
"cc",
"tree-sitter",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
+1
View File
@@ -4,6 +4,7 @@ members = [
"crates/decodal-cli",
"crates/decodal-wasm",
"crates/decodal-derive",
"crates/decodal-language-tools",
]
resolver = "2"
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "decodal-language-tools"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
readme.workspace = true
description = "Source-level language tooling for Decodal."
keywords = ["decodal", "formatter", "lsp", "tree-sitter"]
categories = ["development-tools", "text-processing"]
publish = false
[lib]
crate-type = ["cdylib", "rlib"]
[dependencies]
decodal = { version = "0.1.2", path = "../decodal-core" }
serde_json.workspace = true
tree-sitter = "0.22.6"
tree-sitter-decodal = { path = "../../editors/tree-sitter-decodal" }
wasm-bindgen.workspace = true
[package.metadata.wasm-pack.profile.release]
wasm-opt = false
+851
View File
@@ -0,0 +1,851 @@
use std::{error::Error, fmt};
#[cfg(not(target_arch = "wasm32"))]
use tree_sitter::{Node, Parser, TreeCursor};
use wasm_bindgen::prelude::*;
const INDENT: usize = 4;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FormatError {
message: String,
}
impl FormatError {
fn new(message: impl Into<String>) -> Self {
Self {
message: message.into(),
}
}
pub fn message(&self) -> &str {
&self.message
}
}
impl fmt::Display for FormatError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl Error for FormatError {}
pub fn format_source(source: &str) -> Result<String, FormatError> {
decodal::parse_source(source)
.map_err(|diagnostic| FormatError::new(format!("parse error: {}", diagnostic.message)))?;
format_source_impl(source)
}
#[cfg(not(target_arch = "wasm32"))]
fn format_source_impl(source: &str) -> Result<String, FormatError> {
let mut parser = Parser::new();
parser
.set_language(&tree_sitter_decodal::language())
.map_err(|error| {
FormatError::new(format!("failed to load tree-sitter grammar: {error}"))
})?;
let tree = parser
.parse(source, None)
.ok_or_else(|| FormatError::new("tree-sitter failed to parse source"))?;
let root = tree.root_node();
if root.has_error() {
return Err(FormatError::new(
"parse error: source contains tree-sitter errors",
));
}
let mut formatter = Formatter::new(source);
let mut out = formatter.format_source_file(root);
if !out.ends_with('\n') {
out.push('\n');
}
Ok(out)
}
#[wasm_bindgen(js_name = formatSource)]
pub fn format_source_json(source: &str) -> String {
match format_source(source) {
Ok(formatted) => serde_json::json!({ "ok": true, "source": formatted }).to_string(),
Err(error) => serde_json::json!({ "ok": false, "error": error.message() }).to_string(),
}
}
#[cfg(target_arch = "wasm32")]
fn format_source_impl(source: &str) -> Result<String, FormatError> {
Ok(LexicalFormatter::new(source).format())
}
#[cfg(target_arch = "wasm32")]
struct LexicalFormatter<'a> {
chars: core::iter::Peekable<core::str::Chars<'a>>,
out: String,
indent: usize,
line_start: bool,
}
#[cfg(target_arch = "wasm32")]
impl<'a> LexicalFormatter<'a> {
fn new(source: &'a str) -> Self {
Self {
chars: source.chars().peekable(),
out: String::new(),
indent: 0,
line_start: true,
}
}
fn format(mut self) -> String {
while let Some(ch) = self.chars.next() {
match ch {
' ' | '\t' | '\r' | '\n' => self.consume_whitespace(ch),
'#' => self.write_comment(),
'"' => self.write_string(),
'{' => self.open_block('{'),
'[' => self.open_block('['),
'}' => self.close_block('}'),
']' => self.close_block(']'),
';' => self.end_statement(),
',' => self.write_comma(),
'.' => self.write_compact('.'),
':' => self.write_spaced_operator(":"),
'(' => self.write_compact('('),
')' => self.write_compact(')'),
'+' | '-' | '*' | '/' | '=' | '!' | '&' | '|' | '>' | '<' => {
self.write_operator(ch)
}
_ => self.write_word(ch),
}
}
self.trim_trailing_spaces();
if !self.out.ends_with('\n') {
self.out.push('\n');
}
self.out
}
fn consume_whitespace(&mut self, ch: char) {
if ch == '\n' && self.out.ends_with('\n') && !self.out.ends_with("\n\n") {
self.out.push('\n');
self.line_start = true;
}
}
fn write_comment(&mut self) {
if self.line_start {
self.write_indent();
} else {
self.ensure_space();
}
self.out.push('#');
for ch in self.chars.by_ref() {
if ch == '\n' {
break;
}
self.out.push(ch);
}
self.trim_trailing_spaces();
self.out.push('\n');
self.line_start = true;
}
fn write_string(&mut self) {
self.write_indent_if_needed();
self.ensure_word_boundary();
self.out.push('"');
let mut escaped = false;
for ch in self.chars.by_ref() {
self.out.push(ch);
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == '"' {
break;
}
}
}
fn open_block(&mut self, ch: char) {
self.trim_trailing_spaces();
self.out.push(' ');
self.out.push(ch);
self.out.push('\n');
self.indent += INDENT;
self.line_start = true;
}
fn close_block(&mut self, ch: char) {
self.trim_trailing_spaces();
if !self.out.ends_with('\n') {
self.out.push('\n');
}
self.indent = self.indent.saturating_sub(INDENT);
self.write_indent();
self.out.push(ch);
self.line_start = false;
}
fn end_statement(&mut self) {
self.trim_trailing_spaces();
self.out.push(';');
self.out.push('\n');
self.line_start = true;
}
fn write_comma(&mut self) {
self.trim_trailing_spaces();
self.out.push(',');
self.out.push(' ');
self.line_start = false;
}
fn write_operator(&mut self, ch: char) {
let mut op = String::new();
op.push(ch);
if let Some(next) = self.chars.peek().copied() {
let two_char = matches!(
(ch, next),
('=', '=')
| ('!', '=')
| ('>', '=')
| ('<', '=')
| ('+', '+')
| ('/', '/')
| ('&', '&')
| ('|', '|')
| ('=', '>')
);
if two_char {
op.push(next);
self.chars.next();
}
}
self.write_spaced_operator(&op);
}
fn write_spaced_operator(&mut self, op: &str) {
self.trim_trailing_spaces();
self.ensure_space();
self.out.push_str(op);
self.out.push(' ');
self.line_start = false;
}
fn write_compact(&mut self, ch: char) {
self.trim_trailing_spaces();
self.write_indent_if_needed();
self.out.push(ch);
self.line_start = false;
}
fn write_word(&mut self, first: char) {
self.write_indent_if_needed();
self.ensure_word_boundary();
self.out.push(first);
while let Some(next) = self.chars.peek().copied() {
if next.is_alphanumeric() || next == '_' {
self.out.push(next);
self.chars.next();
} else {
break;
}
}
self.line_start = false;
}
fn write_indent_if_needed(&mut self) {
if self.line_start {
self.write_indent();
self.line_start = false;
}
}
fn write_indent(&mut self) {
for _ in 0..self.indent {
self.out.push(' ');
}
}
fn ensure_word_boundary(&mut self) {
let needs_space = self
.out
.chars()
.last()
.is_some_and(|ch| ch.is_alphanumeric() || matches!(ch, '_' | '"'));
if needs_space {
self.out.push(' ');
}
}
fn ensure_space(&mut self) {
let needs_space = self
.out
.chars()
.last()
.is_some_and(|ch| !matches!(ch, ' ' | '\n' | '.' | '(' | '['));
if needs_space {
self.out.push(' ');
}
}
fn trim_trailing_spaces(&mut self) {
while self.out.ends_with(' ') || self.out.ends_with('\t') {
self.out.pop();
}
}
}
#[cfg(not(target_arch = "wasm32"))]
struct Formatter<'a> {
source: &'a str,
}
#[cfg(not(target_arch = "wasm32"))]
impl<'a> Formatter<'a> {
fn new(source: &'a str) -> Self {
Self { source }
}
fn format_source_file(&mut self, node: Node<'a>) -> String {
let mut out = String::new();
let children = named_children(node);
self.write_statement_list(&mut out, &children, 0);
out
}
fn write_statement_list(&mut self, out: &mut String, children: &[Node<'a>], indent: usize) {
let mut pending_comments = Vec::new();
let mut pending_blank = false;
let mut previous_end = 0;
let mut previous_statement_end_row = None;
for &child in children {
let is_trailing_comment = child.kind() == "comment"
&& previous_statement_end_row == Some(child.start_position().row);
if previous_statement_end_row.is_some() && !is_trailing_comment {
if !out.ends_with('\n') {
out.push('\n');
}
previous_statement_end_row = None;
}
if previous_end > 0 && has_blank_line(self.slice(previous_end, child.start_byte())) {
pending_blank = true;
}
if child.kind() == "comment" {
if is_trailing_comment {
if !out.ends_with(' ') {
out.push(' ');
}
out.push_str(self.raw_trimmed(child));
out.push('\n');
previous_statement_end_row = None;
} else {
pending_comments.push(child);
}
previous_end = child.end_byte();
continue;
}
if pending_blank && !out.is_empty() && !out.ends_with("\n\n") {
out.push('\n');
}
pending_blank = false;
for comment in pending_comments.drain(..) {
write_indent(out, indent);
out.push_str(self.raw_trimmed(comment));
out.push('\n');
}
write_indent(out, indent);
self.write_statement(out, child, indent);
previous_statement_end_row = Some(child.end_position().row);
previous_end = child.end_byte();
}
if previous_statement_end_row.is_some() && !out.ends_with('\n') {
out.push('\n');
}
if pending_blank && !out.is_empty() && !out.ends_with("\n\n") {
out.push('\n');
}
for comment in pending_comments {
write_indent(out, indent);
out.push_str(self.raw_trimmed(comment));
out.push('\n');
}
}
fn write_statement(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
match node.kind() {
"field_definition" => {
self.write_field_definition(out, node, indent);
out.push(';');
}
_ => {
self.write_expr(out, node, indent, 0);
out.push(';');
}
}
}
fn write_field_definition(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
if let Some(path) = node.child_by_field_name("path") {
self.write_field_path(out, path);
} else {
out.push_str(self.raw_trimmed(node));
return;
}
out.push_str(" = ");
if let Some(value) = node.child_by_field_name("value") {
self.write_expr(out, value, indent, 0);
}
}
fn write_field_path(&mut self, out: &mut String, node: Node<'a>) {
let identifiers: Vec<_> = named_children(node)
.into_iter()
.filter(|child| child.kind() == "identifier")
.collect();
if identifiers.is_empty() {
out.push_str(self.raw_trimmed(node));
return;
}
for (index, identifier) in identifiers.into_iter().enumerate() {
if index > 0 {
out.push('.');
}
out.push_str(self.raw_trimmed(identifier));
}
}
fn write_expr(&mut self, out: &mut String, node: Node<'a>, indent: usize, parent_prec: u8) {
if expression_contains_comment(node)
&& !matches!(
node.kind(),
"object" | "array" | "let_expression" | "match_expression"
)
{
out.push_str(self.raw_trimmed(node));
return;
}
let prec = self.precedence(node);
let parenthesize = prec < parent_prec;
if parenthesize {
out.push('(');
}
match node.kind() {
"literal" => {
if let Some(child) = named_children(node).first().copied() {
out.push_str(self.raw_trimmed(child));
} else {
out.push_str(self.raw_trimmed(node));
}
}
"identifier" | "string" | "integer" | "float" | "boolean" | "regex_literal" => {
out.push_str(self.raw_trimmed(node));
}
"comparison_constraint" => self.write_comparison_constraint(out, node, indent),
"object" => self.write_object(out, node, indent),
"array" => self.write_array(out, node, indent),
"let_expression" => self.write_let(out, node, indent),
"function_expression" => self.write_function(out, node, indent),
"match_expression" => self.write_match(out, node, indent),
"import_expression" => self.write_import(out, node),
"parenthesized_expression" => {
out.push('(');
if let Some(expr) = named_children(node)
.into_iter()
.find(|child| child.kind() != "comment")
{
self.write_expr(out, expr, indent, 0);
}
out.push(')');
}
"call_expression" => self.write_call(out, node, indent),
"path_expression" => self.write_path(out, node, indent),
"unary_expression" => self.write_unary(out, node, indent),
"binary_expression" | "default_expression" => {
self.write_binary_like(out, node, indent, prec)
}
_ => out.push_str(self.raw_trimmed(node)),
}
if parenthesize {
out.push(')');
}
}
fn write_object(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
let children = named_children(node);
if children.is_empty() {
out.push_str("{}");
return;
}
out.push('{');
out.push('\n');
self.write_statement_list(out, &children, indent + INDENT);
write_indent(out, indent);
out.push('}');
}
fn write_array(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
let children = named_children(node);
if children.is_empty() {
out.push_str("[]");
return;
}
let has_comment = children.iter().any(|child| child.kind() == "comment");
let inline = !has_comment && children.iter().all(|child| is_inline_expr(*child));
if inline {
out.push('[');
for (index, child) in children.into_iter().enumerate() {
if index > 0 {
out.push_str(", ");
}
self.write_expr(out, child, indent, 0);
}
out.push(']');
return;
}
out.push('[');
out.push('\n');
let mut pending_comments = Vec::new();
for child in children {
if child.kind() == "comment" {
pending_comments.push(child);
continue;
}
for comment in pending_comments.drain(..) {
write_indent(out, indent + INDENT);
out.push_str(self.raw_trimmed(comment));
out.push('\n');
}
write_indent(out, indent + INDENT);
self.write_expr(out, child, indent + INDENT, 0);
out.push_str(",\n");
}
for comment in pending_comments {
write_indent(out, indent + INDENT);
out.push_str(self.raw_trimmed(comment));
out.push('\n');
}
write_indent(out, indent);
out.push(']');
}
fn write_let(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
let body = node.child_by_field_name("body");
out.push_str("let");
out.push('\n');
let children: Vec<_> = named_children(node)
.into_iter()
.filter(|child| Some(*child) != body)
.collect();
self.write_statement_list(out, &children, indent + INDENT);
write_indent(out, indent);
out.push_str("in");
if let Some(body) = body {
out.push('\n');
write_indent(out, indent + INDENT);
self.write_expr(out, body, indent + INDENT, 0);
}
}
fn write_function(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
let body = node.child_by_field_name("body");
let params: Vec<_> = named_children(node)
.into_iter()
.filter(|child| child.kind() == "parameter")
.collect();
out.push('(');
for (index, param) in params.into_iter().enumerate() {
if index > 0 {
out.push_str(", ");
}
self.write_parameter(out, param, indent);
}
out.push_str(") =>");
if let Some(body) = body {
if is_inline_expr(body) {
out.push(' ');
self.write_expr(out, body, indent, 0);
} else {
out.push('\n');
write_indent(out, indent + INDENT);
self.write_expr(out, body, indent + INDENT, 0);
}
}
}
fn write_parameter(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
if let Some(name) = node.child_by_field_name("name") {
out.push_str(self.raw_trimmed(name));
}
if let Some(constraint) = node.child_by_field_name("constraint") {
out.push_str(": ");
self.write_expr(out, constraint, indent, 0);
}
}
fn write_match(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
out.push_str("match ");
if let Some(scrutinee) = node.child_by_field_name("scrutinee") {
self.write_expr(out, scrutinee, indent, 0);
}
out.push_str(" {");
let children: Vec<_> = named_children(node)
.into_iter()
.filter(|child| child.kind() == "match_arm" || child.kind() == "comment")
.collect();
if !children.is_empty() {
out.push('\n');
self.write_match_arms(out, &children, indent + INDENT);
write_indent(out, indent);
}
out.push('}');
}
fn write_match_arms(&mut self, out: &mut String, children: &[Node<'a>], indent: usize) {
let mut pending_comments = Vec::new();
for &child in children {
if child.kind() == "comment" {
pending_comments.push(child);
continue;
}
for comment in pending_comments.drain(..) {
write_indent(out, indent);
out.push_str(self.raw_trimmed(comment));
out.push('\n');
}
write_indent(out, indent);
if let Some(pattern) = child.child_by_field_name("pattern") {
self.write_expr(out, pattern, indent, 0);
}
out.push_str(": ");
if let Some(body) = child.child_by_field_name("body") {
self.write_expr(out, body, indent, 0);
}
out.push_str(";\n");
}
for comment in pending_comments {
write_indent(out, indent);
out.push_str(self.raw_trimmed(comment));
out.push('\n');
}
}
fn write_import(&mut self, out: &mut String, node: Node<'a>) {
out.push_str("import ");
if let Some(specifier) = node.child_by_field_name("specifier") {
out.push_str(self.raw_trimmed(specifier));
}
}
fn write_call(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
let callee = node.child_by_field_name("function");
if let Some(callee) = callee {
self.write_expr(out, callee, indent, self.precedence(node));
}
out.push('(');
let args: Vec<_> = named_children(node)
.into_iter()
.filter(|child| Some(*child) != callee && child.kind() != "comment")
.collect();
for (index, arg) in args.into_iter().enumerate() {
if index > 0 {
out.push_str(", ");
}
self.write_expr(out, arg, indent, 0);
}
out.push(')');
}
fn write_path(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
if let Some(object) = node.child_by_field_name("object") {
self.write_expr(out, object, indent, self.precedence(node));
}
out.push('.');
if let Some(field) = node.child_by_field_name("field") {
out.push_str(self.raw_trimmed(field));
}
}
fn write_unary(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
if let Some(operand) = node.child_by_field_name("operand") {
let operator = self.slice(node.start_byte(), operand.start_byte()).trim();
out.push_str(operator);
self.write_expr(out, operand, indent, self.precedence(node));
} else {
out.push_str(self.raw_trimmed(node));
}
}
fn write_comparison_constraint(&mut self, out: &mut String, node: Node<'a>, indent: usize) {
if let Some(value) = node.child_by_field_name("value") {
let operator = self.slice(node.start_byte(), value.start_byte()).trim();
out.push_str(operator);
out.push(' ');
self.write_expr(out, value, indent, self.precedence(node));
} else {
out.push_str(self.raw_trimmed(node));
}
}
fn write_binary_like(&mut self, out: &mut String, node: Node<'a>, indent: usize, prec: u8) {
let left = node
.child_by_field_name("left")
.or_else(|| node.child_by_field_name("base"));
let right = node
.child_by_field_name("right")
.or_else(|| node.child_by_field_name("fallback"));
let (Some(left), Some(right)) = (left, right) else {
out.push_str(self.raw_trimmed(node));
return;
};
self.write_expr(out, left, indent, prec);
out.push(' ');
if node.kind() == "default_expression" {
out.push_str("default");
} else {
out.push_str(self.slice(left.end_byte(), right.start_byte()).trim());
}
out.push(' ');
self.write_expr(out, right, indent, prec + 1);
}
fn precedence(&self, node: Node<'a>) -> u8 {
match node.kind() {
"default_expression" => 1,
"binary_expression" => self.binary_precedence(node),
"unary_expression" | "comparison_constraint" => 17,
"call_expression" | "path_expression" => 19,
_ => 20,
}
}
fn binary_precedence(&self, node: Node<'a>) -> u8 {
let Some(left) = node.child_by_field_name("left") else {
return 10;
};
let Some(right) = node.child_by_field_name("right") else {
return 10;
};
match self.slice(left.end_byte(), right.start_byte()).trim() {
"//" => 3,
"&" => 5,
"||" => 7,
"&&" => 9,
"==" | "!=" | ">" | ">=" | "<" | "<=" => 11,
"++" => 12,
"+" | "-" => 13,
"*" | "/" => 15,
_ => 10,
}
}
fn raw_trimmed(&self, node: Node<'a>) -> &'a str {
self.slice(node.start_byte(), node.end_byte()).trim()
}
fn slice(&self, start: usize, end: usize) -> &'a str {
&self.source[start..end]
}
}
#[cfg(not(target_arch = "wasm32"))]
fn named_children<'a>(node: Node<'a>) -> Vec<Node<'a>> {
let mut cursor: TreeCursor<'a> = node.walk();
node.named_children(&mut cursor).collect()
}
#[cfg(not(target_arch = "wasm32"))]
fn expression_contains_comment(node: Node<'_>) -> bool {
if node.kind() == "comment" {
return true;
}
named_children(node)
.into_iter()
.any(expression_contains_comment)
}
#[cfg(not(target_arch = "wasm32"))]
fn is_inline_expr(node: Node<'_>) -> bool {
match node.kind() {
"literal"
| "identifier"
| "string"
| "integer"
| "float"
| "boolean"
| "regex_literal"
| "comparison_constraint"
| "import_expression" => true,
"path_expression"
| "call_expression"
| "unary_expression"
| "binary_expression"
| "default_expression"
| "parenthesized_expression" => {
!expression_contains_comment(node)
&& named_children(node).into_iter().all(is_inline_expr)
}
"array" => {
!expression_contains_comment(node)
&& named_children(node).into_iter().all(is_inline_expr)
}
_ => false,
}
}
#[cfg(not(target_arch = "wasm32"))]
fn has_blank_line(text: &str) -> bool {
text.bytes().filter(|byte| *byte == b'\n').count() >= 2
}
#[cfg(not(target_arch = "wasm32"))]
fn write_indent(out: &mut String, indent: usize) {
for _ in 0..indent {
out.push(' ');
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn formats_fields_and_preserves_comments() {
let source =
"Server={\n# host\nhost=String default \"localhost\"; # trailing\n\nport=Int&>0;\n};";
let formatted = format_source(source).unwrap();
assert_eq!(
formatted,
"Server = {\n # host\n host = String default \"localhost\"; # trailing\n\n port = Int & > 0;\n};\n"
);
}
#[test]
fn formats_arrays_and_match_comments() {
let source = "result=match x{# c\n_: [1,{a=2;}];};";
let formatted = format_source(source).unwrap();
assert_eq!(
formatted,
"result = match x {\n # c\n _: [\n 1,\n {\n a = 2;\n },\n ];\n};\n"
);
}
#[test]
fn wasm_export_returns_json() {
let output = format_source_json("value={a=1;};");
assert!(output.contains("\"ok\":true"));
assert!(output.contains("value = {\\n a = 1;\\n};\\n"));
}
}
+19
View File
@@ -39,6 +39,24 @@ The JSR package is `@hare/decodal-wasm`.
The generated files in `packages/decodal-wasm/` are committed so the site can build without requiring every consumer to run `wasm-pack` first.
The WebAssembly package is for execution, not syntax highlighting.
## Language tools
Source-level tooling lives in the Rust language tools crate and its generated JavaScript/WebAssembly package.
This component is responsible for operations that must preserve source text details such as comments and whitespace.
Important paths:
```text
crates/decodal-language-tools/
packages/decodal-language-tools/
```
The npm package is `decodal-language-tools`.
The JSR package is `@hare/decodal-language-tools`.
The current language tools package exposes the formatter.
Future LSP functionality should build on the same source-level tooling layer rather than on the runtime core.
## Web editor components
The Web playground editor uses CodeMirror 6 with a generated Lezer parser.
@@ -94,6 +112,7 @@ Consumers that need syntax information should use the component that matches the
- Rust execution and embedding: `decodal`
- Browser execution: `decodal-wasm`
- Source formatting and future LSP tooling: `decodal-language-tools`
- Web editor syntax: Lezer / CodeMirror
- General editor syntax: Tree-sitter
+29 -11
View File
@@ -24,8 +24,8 @@ cargo run -q -p decodal-cli --features regex -- examples/regex/main.dcdl
The primary crates.io package is `decodal`, which contains the embeddable library.
`decodal-derive` provides optional derive macros for Rust struct integration and is published only when the derive crate changes.
Workspace support crates such as `decodal-cli` and the Rust source crate `decodal-wasm` are not published to crates.io.
The generated WebAssembly package under `packages/decodal-wasm/` is published to npm and JSR.
Workspace support crates such as `decodal-cli`, the Rust source crate `decodal-wasm`, and `decodal-language-tools` are not published to crates.io.
The generated WebAssembly packages under `packages/decodal-wasm/` and `packages/decodal-language-tools/` are published to npm and JSR.
The project is dual licensed as `MIT OR Apache-2.0`.
```sh
@@ -58,7 +58,7 @@ site/decodal-site/
```
The site imports Markdown files from `doc/manual/souce/` and renders them as mdBook-style pages.
The playground loads `decodal-wasm` and evaluates DCDL entirely in the browser.
The playground loads `decodal-wasm` for evaluation and `decodal-language-tools` for source formatting.
Important files:
@@ -74,25 +74,25 @@ packages/decodal-codemirror/src/decodal.js
packages/decodal-codemirror/src/decodal-parser.js
packages/decodal-wasm/decodal_wasm.js
packages/decodal-wasm/decodal_wasm_bg.wasm
packages/decodal-language-tools/decodal_language_tools.js
packages/decodal-language-tools/decodal_language_tools_bg.wasm
crates/decodal-wasm/src/lib.rs
crates/decodal-language-tools/src/lib.rs
```
Build the WebAssembly package before building the site:
Build the WebAssembly packages before building the site:
```sh
cd site/decodal-site
npm install
npm run build:wasm
npm run build:packages
npm run build
```
`npm run build:wasm` writes generated files into:
`npm run build:wasm` writes generated runtime files into `packages/decodal-wasm/`.
`npm run build:tools` writes generated language tools files into `packages/decodal-language-tools/`.
```text
packages/decodal-wasm/
```
These generated files are committed so the site can be built without requiring every consumer to regenerate the wasm package first.
These generated files are committed so the site can be built without requiring every consumer to regenerate the wasm packages first.
Publish the generated WebAssembly package from its package directory:
@@ -113,6 +113,24 @@ The npm package name is `decodal-wasm`.
The JSR package name is `@hare/decodal-wasm`.
JSR currently expects a single SPDX license identifier in `jsr.json`; the WebAssembly package metadata uses `MIT` while the Rust workspace remains dual licensed as `MIT OR Apache-2.0`.
Publish the generated language tools package from its package directory:
```sh
cd packages/decodal-language-tools
npm publish
npx jsr publish
```
Run dry-runs first when preparing a release:
```sh
npm pack --dry-run
npx jsr publish --dry-run
```
The npm package name is `decodal-language-tools`.
The JSR package name is `@hare/decodal-language-tools`.
The playground editor uses the `decodal-codemirror` package with the generated Lezer parser in `packages/decodal-codemirror/src/decodal-parser.js`.
The canonical grammar is documented in `doc/manual/souce/language/grammar.md`; regenerate the Lezer parser when that grammar or `editors/lezer-decodal/decodal.grammar` changes.
@@ -0,0 +1 @@
# wasm-pack output is committed for the browser language tools package.
+39
View File
@@ -0,0 +1,39 @@
# decodal-language-tools
Source-level language tooling for Decodal.
This package is generated from the Rust crate in `crates/decodal-language-tools` with `wasm-pack`.
It currently exposes a comment-preserving formatter that is shared by the official playground and future editor/LSP tooling.
Use `decodal-codemirror` separately when you need CodeMirror 6 language support.
Use `decodal-wasm` separately when you need to evaluate Decodal in a browser.
## Install
```sh
npm install decodal-language-tools
```
JSR:
```sh
deno add jsr:@hare/decodal-language-tools
```
## Usage
```js
import init, { formatSource } from 'decodal-language-tools';
await init();
const result = JSON.parse(formatSource('Server={# host\nhost=String default "localhost";};'));
if (result.ok) {
console.log(result.source);
} else {
console.error(result.error);
}
```
The exported functions return JSON strings so callers can handle diagnostics without depending on Rust data structures.
@@ -0,0 +1,38 @@
/* tslint:disable */
/* eslint-disable */
export function formatSource(source: string): string;
export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
export interface InitOutput {
readonly memory: WebAssembly.Memory;
readonly formatSource: (a: number, b: number) => [number, number];
readonly __wbindgen_externrefs: WebAssembly.Table;
readonly __wbindgen_malloc: (a: number, b: number) => number;
readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
readonly __wbindgen_free: (a: number, b: number, c: number) => void;
readonly __wbindgen_start: () => void;
}
export type SyncInitInput = BufferSource | WebAssembly.Module;
/**
* Instantiates the given `module`, which can either be bytes or
* a precompiled `WebAssembly.Module`.
*
* @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
*
* @returns {InitOutput}
*/
export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
/**
* If `module_or_path` is {RequestInfo} or {URL}, makes a request and
* for everything else, calls `WebAssembly.instantiate` directly.
*
* @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
*
* @returns {Promise<InitOutput>}
*/
export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;
@@ -0,0 +1,209 @@
/* @ts-self-types="./decodal_language_tools.d.ts" */
/**
* @param {string} source
* @returns {string}
*/
export function formatSource(source) {
let deferred2_0;
let deferred2_1;
try {
const ptr0 = passStringToWasm0(source, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.formatSource(ptr0, len0);
deferred2_0 = ret[0];
deferred2_1 = ret[1];
return getStringFromWasm0(ret[0], ret[1]);
} finally {
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
}
}
function __wbg_get_imports() {
const import0 = {
__proto__: null,
__wbindgen_init_externref_table: function() {
const table = wasm.__wbindgen_externrefs;
const offset = table.grow(4);
table.set(0, undefined);
table.set(offset + 0, undefined);
table.set(offset + 1, null);
table.set(offset + 2, true);
table.set(offset + 3, false);
},
};
return {
__proto__: null,
"./decodal_language_tools_bg.js": import0,
};
}
function getStringFromWasm0(ptr, len) {
return decodeText(ptr >>> 0, len);
}
let cachedUint8ArrayMemory0 = null;
function getUint8ArrayMemory0() {
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8ArrayMemory0;
}
function passStringToWasm0(arg, malloc, realloc) {
if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length, 1) >>> 0;
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}
let len = arg.length;
let ptr = malloc(len, 1) >>> 0;
const mem = getUint8ArrayMemory0();
let offset = 0;
for (; offset < len; offset++) {
const code = arg.charCodeAt(offset);
if (code > 0x7F) break;
mem[ptr + offset] = code;
}
if (offset !== len) {
if (offset !== 0) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
const ret = cachedTextEncoder.encodeInto(arg, view);
offset += ret.written;
ptr = realloc(ptr, len, offset, 1) >>> 0;
}
WASM_VECTOR_LEN = offset;
return ptr;
}
let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
cachedTextDecoder.decode();
const MAX_SAFARI_DECODE_BYTES = 2146435072;
let numBytesDecoded = 0;
function decodeText(ptr, len) {
numBytesDecoded += len;
if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
cachedTextDecoder.decode();
numBytesDecoded = len;
}
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
}
const cachedTextEncoder = new TextEncoder();
if (!('encodeInto' in cachedTextEncoder)) {
cachedTextEncoder.encodeInto = function (arg, view) {
const buf = cachedTextEncoder.encode(arg);
view.set(buf);
return {
read: arg.length,
written: buf.length
};
};
}
let WASM_VECTOR_LEN = 0;
let wasmModule, wasmInstance, wasm;
function __wbg_finalize_init(instance, module) {
wasmInstance = instance;
wasm = instance.exports;
wasmModule = module;
cachedUint8ArrayMemory0 = null;
wasm.__wbindgen_start();
return wasm;
}
async function __wbg_load(module, imports) {
if (typeof Response === 'function' && module instanceof Response) {
if (typeof WebAssembly.instantiateStreaming === 'function') {
try {
return await WebAssembly.instantiateStreaming(module, imports);
} catch (e) {
const validResponse = module.ok && expectedResponseType(module.type);
if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') {
console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
} else { throw e; }
}
}
const bytes = await module.arrayBuffer();
return await WebAssembly.instantiate(bytes, imports);
} else {
const instance = await WebAssembly.instantiate(module, imports);
if (instance instanceof WebAssembly.Instance) {
return { instance, module };
} else {
return instance;
}
}
function expectedResponseType(type) {
switch (type) {
case 'basic': case 'cors': case 'default': return true;
}
return false;
}
}
function initSync(module) {
if (wasm !== undefined) return wasm;
if (module !== undefined) {
if (Object.getPrototypeOf(module) === Object.prototype) {
({module} = module)
} else {
console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
}
}
const imports = __wbg_get_imports();
if (!(module instanceof WebAssembly.Module)) {
module = new WebAssembly.Module(module);
}
const instance = new WebAssembly.Instance(module, imports);
return __wbg_finalize_init(instance, module);
}
async function __wbg_init(module_or_path) {
if (wasm !== undefined) return wasm;
if (module_or_path !== undefined) {
if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
({module_or_path} = module_or_path)
} else {
console.warn('using deprecated parameters for the initialization function; pass a single object instead')
}
}
if (module_or_path === undefined) {
module_or_path = new URL('decodal_language_tools_bg.wasm', import.meta.url);
}
const imports = __wbg_get_imports();
if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
module_or_path = fetch(module_or_path);
}
const { instance, module } = await __wbg_load(await module_or_path, imports);
return __wbg_finalize_init(instance, module);
}
export { initSync, __wbg_init as default };
@@ -0,0 +1,9 @@
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export const formatSource: (a: number, b: number) => [number, number];
export const __wbindgen_externrefs: WebAssembly.Table;
export const __wbindgen_malloc: (a: number, b: number) => number;
export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
export const __wbindgen_free: (a: number, b: number, c: number) => void;
export const __wbindgen_start: () => void;
+15
View File
@@ -0,0 +1,15 @@
{
"name": "@hare/decodal-language-tools",
"version": "0.1.2",
"license": "MIT",
"exports": "./decodal_language_tools.js",
"publish": {
"include": [
"README.md",
"decodal_language_tools.js",
"decodal_language_tools.d.ts",
"decodal_language_tools_bg.wasm",
"package.json"
]
}
}
@@ -0,0 +1,27 @@
{
"name": "decodal-language-tools",
"type": "module",
"description": "Source-level language tooling for Decodal.",
"version": "0.1.2",
"license": "MIT OR Apache-2.0",
"repository": {
"type": "git",
"url": "https://gitea.hareworks.net/Hare/Decodal"
},
"files": [
"decodal_language_tools_bg.wasm",
"decodal_language_tools.js",
"decodal_language_tools.d.ts"
],
"main": "decodal_language_tools.js",
"types": "decodal_language_tools.d.ts",
"sideEffects": [
"./snippets/*"
],
"keywords": [
"decodal",
"formatter",
"lsp",
"tree-sitter"
]
}
+9
View File
@@ -16,6 +16,7 @@
"astro": "^4.16.18",
"codemirror": "^6.0.2",
"decodal-codemirror": "file:../../packages/decodal-codemirror",
"decodal-language-tools": "file:../../packages/decodal-language-tools",
"decodal-wasm": "file:../../packages/decodal-wasm",
"marked": "^12.0.2"
},
@@ -33,6 +34,10 @@
"@lezer/lr": "^1.4.10"
}
},
"../../packages/decodal-language-tools": {
"version": "0.1.2",
"license": "MIT OR Apache-2.0"
},
"../../packages/decodal-wasm": {
"version": "0.1.2",
"license": "MIT OR Apache-2.0"
@@ -3255,6 +3260,10 @@
"resolved": "../../packages/decodal-codemirror",
"link": true
},
"node_modules/decodal-language-tools": {
"resolved": "../../packages/decodal-language-tools",
"link": true
},
"node_modules/decodal-wasm": {
"resolved": "../../packages/decodal-wasm",
"link": true
+4 -1
View File
@@ -9,7 +9,9 @@
"preview": "astro preview --host 0.0.0.0",
"build:wasm": "wasm-pack build ../../crates/decodal-wasm --target web --out-dir ../../packages/decodal-wasm --release && node scripts/prepare-wasm-package.mjs",
"deploy": "npm run build && node scripts/deploy-pages.mjs",
"deploy:wasm": "npm run build:wasm && npm run deploy"
"deploy:wasm": "npm run build:wasm && npm run deploy",
"build:tools": "wasm-pack build ../../crates/decodal-language-tools --target web --out-dir ../../packages/decodal-language-tools --release && node scripts/prepare-language-tools-package.mjs",
"build:packages": "npm run build:wasm && npm run build:tools"
},
"dependencies": {
"@astrojs/check": "^0.9.4",
@@ -20,6 +22,7 @@
"astro": "^4.16.18",
"codemirror": "^6.0.2",
"decodal-codemirror": "file:../../packages/decodal-codemirror",
"decodal-language-tools": "file:../../packages/decodal-language-tools",
"decodal-wasm": "file:../../packages/decodal-wasm",
"marked": "^12.0.2"
},
@@ -0,0 +1,76 @@
import { writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
const packageDir = resolve(import.meta.dirname, '../../../packages/decodal-language-tools');
writeFileSync(
resolve(packageDir, '.gitignore'),
'# wasm-pack output is committed for the browser language tools package.\n',
);
writeFileSync(
resolve(packageDir, 'README.md'),
`# decodal-language-tools
Source-level language tooling for Decodal.
This package is generated from the Rust crate in \`crates/decodal-language-tools\` with \`wasm-pack\`.
It currently exposes a comment-preserving formatter that is shared by the official playground and future editor/LSP tooling.
Use \`decodal-codemirror\` separately when you need CodeMirror 6 language support.
Use \`decodal-wasm\` separately when you need to evaluate Decodal in a browser.
## Install
\`\`\`sh
npm install decodal-language-tools
\`\`\`
JSR:
\`\`\`sh
deno add jsr:@hare/decodal-language-tools
\`\`\`
## Usage
\`\`\`js
import init, { formatSource } from 'decodal-language-tools';
await init();
const result = JSON.parse(formatSource('Server={# host\\nhost=String default "localhost";};'));
if (result.ok) {
console.log(result.source);
} else {
console.error(result.error);
}
\`\`\`
The exported functions return JSON strings so callers can handle diagnostics without depending on Rust data structures.
`,
);
writeFileSync(
resolve(packageDir, 'jsr.json'),
JSON.stringify(
{
name: '@hare/decodal-language-tools',
version: '0.1.2',
license: 'MIT',
exports: './decodal_language_tools.js',
publish: {
include: [
'README.md',
'decodal_language_tools.js',
'decodal_language_tools.d.ts',
'decodal_language_tools_bg.wasm',
'package.json',
],
},
},
null,
2,
) + '\n',
);
+10 -2
View File
@@ -40,6 +40,14 @@ Production = Server & {
<span>JSR package</span>
<strong>jsr.io/@hare/decodal-wasm</strong>
</a>
<a href="https://www.npmjs.com/package/decodal-language-tools">
<span>Language tools package</span>
<strong>npmjs.com/package/decodal-language-tools</strong>
</a>
<a href="https://jsr.io/@hare/decodal-language-tools@0.1.2">
<span>Language tools on JSR</span>
<strong>jsr.io/@hare/decodal-language-tools</strong>
</a>
<a href="https://www.npmjs.com/package/decodal-codemirror">
<span>CodeMirror package</span>
<strong>npmjs.com/package/decodal-codemirror</strong>
@@ -62,8 +70,8 @@ Production = Server & {
<article class="home-card">
<h2>Where it runs</h2>
<p>
Use the Rust crate for runtime embedding, the WASM package in browsers, CodeMirror/Lezer for the Web editor,
and Tree-sitter for general editor integration.
Use the Rust crate for runtime embedding, the WASM package in browsers, language-tools for formatting,
CodeMirror/Lezer for the Web editor, and Tree-sitter for general editor integration.
</p>
</article>
<article class="home-card">
@@ -15,6 +15,7 @@ import ManualLayout from '../layouts/ManualLayout.astro';
</label>
<button id="load-example" type="button">Load</button>
<p id="status" class="status">Loading WASM...</p>
<button id="format" disabled>Format</button>
<button id="run" disabled>Run</button>
</div>
</div>
+20 -2
View File
@@ -1,4 +1,5 @@
import init, { evaluateProject } from 'decodal-wasm';
import initRuntime, { evaluateProject } from 'decodal-wasm';
import initTools, { formatSource } from 'decodal-language-tools';
import { EditorView, basicSetup } from 'codemirror';
import { keymap } from '@codemirror/view';
import { decodal } from 'decodal-codemirror';
@@ -10,6 +11,7 @@ const starterProject = playgroundExamples[0];
const editorHost = document.getElementById('editor');
const output = document.getElementById('output');
const run = document.getElementById('run');
const formatButton = document.getElementById('format');
const status = document.getElementById('status');
const fileTree = document.getElementById('file-tree');
const activeFile = document.getElementById('active-file');
@@ -176,6 +178,20 @@ function execute() {
output.classList.toggle('error', !result.ok);
}
function formatActiveFile() {
const result = JSON.parse(formatSource(getEditorText()));
if (!result.ok) {
output.textContent = result.error;
output.classList.add('error');
return;
}
setEditorText(result.source);
project.files[project.activePath] = result.source;
saveProject();
output.textContent = 'Formatted.';
output.classList.remove('error');
}
function renderFileTree() {
const tree = buildTree(Object.keys(project.files).sort());
fileTree.replaceChildren(renderTreeList(tree.children));
@@ -227,8 +243,9 @@ function compareNodes(a, b) {
}
try {
await init();
await Promise.all([initRuntime(), initTools()]);
run.disabled = false;
formatButton.disabled = false;
status.textContent = '';
execute();
} catch (error) {
@@ -236,6 +253,7 @@ try {
}
run.addEventListener('click', execute);
formatButton.addEventListener('click', formatActiveFile);
loadExample.addEventListener('click', () => {
const example = playgroundExamples.find((item) => item.id === exampleSelect.value);
if (!example) return;