Files
Decodal/crates/decodal-language-tools/src/lib.rs
T

839 lines
29 KiB
Rust

use std::{error::Error, fmt};
use decodal::{
Ast, BinaryOp, CompareOp, Expr, ExprId, Field, ObjectRest, Param, SourceForm, Span,
SyntaxToken, SyntaxTokenKind,
ast::{MatchArm, UnaryOp},
parse_source, tokenize_source,
};
use wasm_bindgen::prelude::*;
const INDENT: usize = 4;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FormatError {
message: String,
}
impl FormatError {
fn new(message: impl Into<String>) -> Self {
Self {
message: message.into(),
}
}
pub fn message(&self) -> &str {
&self.message
}
}
impl fmt::Display for FormatError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl Error for FormatError {}
/// Formats a complete Decodal source with the canonical parser and syntax tokens.
///
/// Native and WebAssembly callers execute this exact implementation. Comments
/// are recovered from the lossless token stream while expression structure and
/// precedence come from the same AST used by evaluation.
pub fn format_source(source: &str) -> Result<String, FormatError> {
let parsed = parse_source(source)
.map_err(|diagnostic| FormatError::new(format!("parse error: {}", diagnostic.message)))?;
let tokens = tokenize_source(source)
.map_err(|diagnostic| FormatError::new(format!("lex error: {}", diagnostic.message)))?;
Ok(Formatter::new(source, &parsed.ast, tokens).format(parsed.root, parsed.source_form))
}
#[wasm_bindgen(js_name = formatSource)]
pub fn format_source_json(source: &str) -> String {
match format_source(source) {
Ok(formatted) => serde_json::json!({ "ok": true, "source": formatted }).to_string(),
Err(error) => serde_json::json!({ "ok": false, "error": error.message() }).to_string(),
}
}
struct Formatter<'a> {
source: &'a str,
ast: &'a Ast,
tokens: Vec<SyntaxToken>,
comments: Vec<Span>,
}
impl<'a> Formatter<'a> {
fn new(source: &'a str, ast: &'a Ast, tokens: Vec<SyntaxToken>) -> Self {
let comments = tokens
.iter()
.filter(|token| token.kind == SyntaxTokenKind::Comment)
.map(|token| token.span)
.collect();
Self {
source,
ast,
tokens,
comments,
}
}
fn format(&self, root: ExprId, source_form: SourceForm) -> String {
let mut out = String::new();
if source_form == SourceForm::Fields {
let Expr::Object { fields, .. } = &self.ast.get(root).expr else {
unreachable!("field-form sources parse to an object")
};
self.write_field_list(&mut out, fields, 0, self.source.len(), 0);
} else {
let span = self.ast.span(root);
self.write_between(&mut out, 0, span.start as usize, None, 0);
self.write_expr(&mut out, root, 0, 0);
self.write_between(
&mut out,
span.end as usize,
self.source.len(),
Some(span.end as usize),
0,
);
}
trim_trailing_whitespace(&mut out);
out.push('\n');
out
}
fn write_field_list(
&self,
out: &mut String,
fields: &[Field],
start: usize,
end: usize,
indent: usize,
) {
let mut cursor = start;
let mut previous_end = None;
for field in fields {
let field_start = field.span.start as usize;
self.write_between(out, cursor, field_start, previous_end, indent);
write_indent(out, indent);
self.write_field(out, field, indent);
out.push(';');
cursor = field.span.end as usize;
previous_end = Some(cursor);
}
self.write_between(out, cursor, end, previous_end, indent);
}
fn write_field(&self, out: &mut String, field: &Field, indent: usize) {
let value_start = self.ast.span(field.value).start as usize;
if self.has_comment_between(field.span.start as usize, value_start) {
out.push_str(self.raw(field.span));
return;
}
for (index, part) in field.path.iter().enumerate() {
if index > 0 {
out.push('.');
}
out.push_str(part);
}
out.push_str(" = ");
self.write_expr(out, field.value, indent, 0);
}
fn write_expr(&self, out: &mut String, id: ExprId, indent: usize, parent_prec: u8) {
let node = self.ast.get(id);
if self.has_comment(node.span)
&& !matches!(
node.expr,
Expr::Object { .. } | Expr::Array(_) | Expr::Let { .. } | Expr::Match { .. }
)
{
out.push_str(self.raw(node.span));
return;
}
let precedence = self.precedence(id);
let parenthesize = precedence < parent_prec;
if parenthesize {
out.push('(');
}
match &node.expr {
Expr::Literal(_) | Expr::Ident(_) | Expr::RegexConstraint(_) | Expr::Wildcard => {
out.push_str(self.raw(node.span));
}
Expr::Object { fields, rest } => {
self.write_object(out, node.span, fields, rest.as_ref(), indent)
}
Expr::Array(items) => self.write_array(out, node.span, items, indent),
Expr::ArrayConstraint { item } => {
out.push_str("[...");
self.write_expr(out, *item, indent, 0);
out.push(']');
}
Expr::MapConstraint { value } => {
out.push_str("{...");
self.write_expr(out, *value, indent, 0);
out.push('}');
}
Expr::Let { bindings, body } => self.write_let(out, node.span, bindings, *body, indent),
Expr::Import(_) => {
out.push_str("import ");
let raw = self.raw(node.span);
out.push_str(raw.strip_prefix("import").unwrap_or(raw).trim());
}
Expr::Path { base, field } => {
self.write_expr(out, *base, indent, precedence);
out.push('.');
out.push_str(field);
}
Expr::Call { callee, args } => {
self.write_expr(out, *callee, indent, precedence);
out.push('(');
for (index, argument) in args.iter().enumerate() {
if index > 0 {
out.push_str(", ");
}
self.write_expr(out, *argument, indent, 0);
}
out.push(')');
}
Expr::Function { params, body } => self.write_function(out, params, *body, indent),
Expr::Parenthesized { expr } => {
out.push('(');
self.write_expr(out, *expr, indent, 0);
out.push(')');
}
Expr::Match { scrutinee, arms } => {
self.write_match(out, node.span, *scrutinee, arms, indent)
}
Expr::Unary { op, expr } => {
out.push_str(match op {
UnaryOp::Neg => "-",
UnaryOp::Not => "!",
});
self.write_expr(out, *expr, indent, precedence);
}
Expr::Binary { op, lhs, rhs } => {
self.write_expr(out, *lhs, indent, precedence);
out.push(' ');
out.push_str(binary_operator(*op));
out.push(' ');
self.write_expr(out, *rhs, indent, precedence + 1);
}
Expr::Default { base, fallback } => {
self.write_expr(out, *base, indent, precedence);
out.push_str(" default ");
self.write_expr(out, *fallback, indent, precedence + 1);
}
Expr::As { narrower, wider } => {
self.write_expr(out, *narrower, indent, precedence);
out.push_str(" as ");
self.write_expr(out, *wider, indent, precedence + 1);
}
Expr::CompareConstraint { op, value } => {
out.push_str(compare_operator(*op));
out.push(' ');
self.write_expr(out, *value, indent, precedence);
}
}
if parenthesize {
out.push(')');
}
}
fn write_object(
&self,
out: &mut String,
span: Span,
fields: &[Field],
rest: Option<&ObjectRest>,
indent: usize,
) {
let (start, end) =
self.delimited_range(span, SyntaxTokenKind::LBrace, SyntaxTokenKind::RBrace);
if fields.is_empty() && rest.is_none() && !self.has_comment_between(start, end) {
out.push_str("{}");
return;
}
out.push_str("{\n");
let field_end = rest.map_or(end, |rest| rest.span.start as usize);
self.write_field_list(out, fields, start, field_end, indent + INDENT);
if let Some(rest) = rest {
write_indent(out, indent + INDENT);
out.push_str("...");
let value_start = self.ast.span(rest.value).start as usize;
let ellipsis = self
.tokens_between(rest.span.start as usize, value_start)
.find(|token| token.kind == SyntaxTokenKind::Ellipsis)
.expect("parsed object rest constraints contain `...`");
if self.has_comment_between(ellipsis.span.end as usize, value_start) {
self.write_between(
out,
ellipsis.span.end as usize,
value_start,
Some(ellipsis.span.end as usize),
indent + INDENT,
);
write_indent(out, indent + INDENT);
}
self.write_expr(out, rest.value, indent + INDENT, 0);
self.write_between(
out,
rest.span.end as usize,
end,
Some(rest.span.end as usize),
indent + INDENT,
);
}
write_indent(out, indent);
out.push('}');
}
fn write_array(&self, out: &mut String, span: Span, items: &[ExprId], indent: usize) {
let (start, end) =
self.delimited_range(span, SyntaxTokenKind::LBracket, SyntaxTokenKind::RBracket);
if items.is_empty() && !self.has_comment_between(start, end) {
out.push_str("[]");
return;
}
let inline = !self.has_comment_between(start, end)
&& items.iter().all(|item| self.is_inline_expr(*item));
if inline {
out.push('[');
for (index, item) in items.iter().enumerate() {
if index > 0 {
out.push_str(", ");
}
self.write_expr(out, *item, indent, 0);
}
out.push(']');
return;
}
out.push_str("[\n");
let mut cursor = start;
let mut previous_end = None;
for item in items {
let item_span = self.ast.span(*item);
self.write_between(
out,
cursor,
item_span.start as usize,
previous_end,
indent + INDENT,
);
write_indent(out, indent + INDENT);
self.write_expr(out, *item, indent + INDENT, 0);
out.push(',');
cursor = item_span.end as usize;
previous_end = Some(cursor);
}
self.write_between(out, cursor, end, previous_end, indent + INDENT);
write_indent(out, indent);
out.push(']');
}
fn write_let(
&self,
out: &mut String,
span: Span,
bindings: &[Field],
body: ExprId,
indent: usize,
) {
let body_span = self.ast.span(body);
let in_token = self
.tokens_between(span.start as usize, body_span.start as usize)
.rev()
.find(|token| token.kind == SyntaxTokenKind::In)
.expect("parsed let expressions contain `in`");
let let_token = self
.tokens_between(span.start as usize, span.end as usize)
.find(|token| token.kind == SyntaxTokenKind::Let)
.expect("parsed let expressions contain `let`");
out.push_str("let\n");
self.write_field_list(
out,
bindings,
let_token.span.end as usize,
in_token.span.start as usize,
indent + INDENT,
);
write_indent(out, indent);
out.push_str("in");
self.write_between(
out,
in_token.span.end as usize,
body_span.start as usize,
Some(in_token.span.end as usize),
indent + INDENT,
);
write_indent(out, indent + INDENT);
self.write_expr(out, body, indent + INDENT, 0);
}
fn write_function(&self, out: &mut String, params: &[Param], body: ExprId, indent: usize) {
out.push('(');
for (index, parameter) in params.iter().enumerate() {
if index > 0 {
out.push_str(", ");
}
out.push_str(&parameter.name);
if let Some(constraint) = parameter.constraint {
out.push_str(": ");
self.write_expr(out, constraint, indent, 0);
}
}
out.push_str(") =>");
if self.is_inline_expr(body) {
out.push(' ');
self.write_expr(out, body, indent, 0);
} else {
out.push('\n');
write_indent(out, indent + INDENT);
self.write_expr(out, body, indent + INDENT, 0);
}
}
fn write_match(
&self,
out: &mut String,
span: Span,
scrutinee: ExprId,
arms: &[MatchArm],
indent: usize,
) {
out.push_str("match ");
self.write_expr(out, scrutinee, indent, 0);
let scrutinee_end = self.ast.span(scrutinee).end as usize;
let open = self
.tokens_between(scrutinee_end, span.end as usize)
.find(|token| token.kind == SyntaxTokenKind::LBrace)
.expect("parsed match expressions contain `{`");
let close = self
.tokens_between(open.span.end as usize, span.end as usize)
.rev()
.find(|token| token.kind == SyntaxTokenKind::RBrace)
.expect("parsed match expressions contain `}`");
let start = open.span.end as usize;
let end = close.span.start as usize;
if self.has_comment_between(scrutinee_end, open.span.start as usize) {
self.write_between(
out,
scrutinee_end,
open.span.start as usize,
Some(scrutinee_end),
indent,
);
write_indent(out, indent);
out.push('{');
} else {
out.push_str(" {");
}
if arms.is_empty() && !self.has_comment_between(start, end) {
out.push('}');
return;
}
out.push('\n');
let mut cursor = start;
let mut previous_end = None;
for arm in arms {
self.write_between(
out,
cursor,
arm.span.start as usize,
previous_end,
indent + INDENT,
);
write_indent(out, indent + INDENT);
if self.has_comment_between(
self.ast.span(arm.pattern).end as usize,
self.ast.span(arm.body).start as usize,
) {
out.push_str(self.raw(arm.span));
} else {
self.write_expr(out, arm.pattern, indent + INDENT, 0);
out.push_str(": ");
self.write_expr(out, arm.body, indent + INDENT, 0);
}
out.push(';');
cursor = arm.span.end as usize;
previous_end = Some(cursor);
}
self.write_between(out, cursor, end, previous_end, indent + INDENT);
write_indent(out, indent);
out.push('}');
}
fn write_between(
&self,
out: &mut String,
start: usize,
end: usize,
previous_end: Option<usize>,
indent: usize,
) {
let mut cursor = start.min(end);
for comment in self.comments_between(start, end) {
let comment_start = comment.start as usize;
let trailing = previous_end.is_some_and(|previous| {
!out.ends_with('\n') && self.same_line(previous, comment_start)
});
if trailing {
out.push(' ');
out.push_str(self.raw(*comment));
out.push('\n');
} else {
ensure_line_break(out);
if self.has_blank_line(cursor, comment_start) {
ensure_blank_line(out);
}
write_indent(out, indent);
out.push_str(self.raw(*comment));
out.push('\n');
}
cursor = comment.end as usize;
}
if previous_end.is_some() {
ensure_line_break(out);
}
if self.has_blank_line(cursor, end) {
ensure_blank_line(out);
}
}
fn precedence(&self, id: ExprId) -> u8 {
match &self.ast.get(id).expr {
Expr::As { .. } => 0,
Expr::Default { .. } => 1,
Expr::Binary { op, .. } => match op {
BinaryOp::Patch => 3,
BinaryOp::And => 5,
BinaryOp::LogicalOr => 7,
BinaryOp::LogicalAnd => 9,
BinaryOp::Equal
| BinaryOp::NotEqual
| BinaryOp::Greater
| BinaryOp::GreaterEqual
| BinaryOp::Less
| BinaryOp::LessEqual => 11,
BinaryOp::Concat => 12,
BinaryOp::Add | BinaryOp::Sub => 13,
BinaryOp::Mul | BinaryOp::Div => 15,
},
Expr::Unary { .. } | Expr::CompareConstraint { .. } => 17,
Expr::Call { .. } | Expr::Path { .. } => 19,
Expr::Parenthesized { .. } => 20,
_ => 20,
}
}
fn is_inline_expr(&self, id: ExprId) -> bool {
let node = self.ast.get(id);
if self.has_comment(node.span) {
return false;
}
match &node.expr {
Expr::Literal(_)
| Expr::Ident(_)
| Expr::RegexConstraint(_)
| Expr::Wildcard
| Expr::Import(_)
| Expr::CompareConstraint { .. } => true,
Expr::Path { base, .. } => self.is_inline_expr(*base),
Expr::Call { callee, args } => {
self.is_inline_expr(*callee)
&& args.iter().all(|argument| self.is_inline_expr(*argument))
}
Expr::Unary { expr, .. } => self.is_inline_expr(*expr),
Expr::Binary { lhs, rhs, .. } => self.is_inline_expr(*lhs) && self.is_inline_expr(*rhs),
Expr::Default { base, fallback } => {
self.is_inline_expr(*base) && self.is_inline_expr(*fallback)
}
Expr::Array(items) => items.iter().all(|item| self.is_inline_expr(*item)),
Expr::ArrayConstraint { item } => self.is_inline_expr(*item),
Expr::MapConstraint { value } => self.is_inline_expr(*value),
Expr::Parenthesized { expr } => self.is_inline_expr(*expr),
Expr::As { narrower, wider } => {
self.is_inline_expr(*narrower) && self.is_inline_expr(*wider)
}
Expr::Object { .. } | Expr::Let { .. } | Expr::Function { .. } | Expr::Match { .. } => {
false
}
}
}
fn delimited_range(
&self,
span: Span,
open_kind: SyntaxTokenKind,
close_kind: SyntaxTokenKind,
) -> (usize, usize) {
let open = self
.tokens_between(span.start as usize, span.end as usize)
.find(|token| same_kind(&token.kind, &open_kind))
.expect("parsed delimited expressions contain an opening token");
let close = self
.tokens_between(open.span.end as usize, span.end as usize)
.rev()
.find(|token| same_kind(&token.kind, &close_kind))
.expect("parsed delimited expressions contain a closing token");
(open.span.end as usize, close.span.start as usize)
}
fn tokens_between(
&self,
start: usize,
end: usize,
) -> impl DoubleEndedIterator<Item = &SyntaxToken> {
self.tokens.iter().filter(move |token| {
token.span.start as usize >= start && token.span.end as usize <= end
})
}
fn comments_between(&self, start: usize, end: usize) -> impl Iterator<Item = &Span> {
self.comments
.iter()
.filter(move |comment| comment.start as usize >= start && comment.end as usize <= end)
}
fn has_comment(&self, span: Span) -> bool {
self.has_comment_between(span.start as usize, span.end as usize)
}
fn has_comment_between(&self, start: usize, end: usize) -> bool {
self.comments_between(start, end).next().is_some()
}
fn same_line(&self, start: usize, end: usize) -> bool {
!self.slice(start, end).contains('\n')
}
fn has_blank_line(&self, start: usize, end: usize) -> bool {
self.slice(start, end)
.bytes()
.filter(|byte| *byte == b'\n')
.count()
>= 2
}
fn raw(&self, span: Span) -> &'a str {
self.slice(span.start as usize, span.end as usize).trim()
}
fn slice(&self, start: usize, end: usize) -> &'a str {
&self.source[start.min(self.source.len())..end.min(self.source.len())]
}
}
fn binary_operator(operator: BinaryOp) -> &'static str {
match operator {
BinaryOp::Add => "+",
BinaryOp::Sub => "-",
BinaryOp::Mul => "*",
BinaryOp::Div => "/",
BinaryOp::Concat => "++",
BinaryOp::Equal => "==",
BinaryOp::NotEqual => "!=",
BinaryOp::Greater => ">",
BinaryOp::GreaterEqual => ">=",
BinaryOp::Less => "<",
BinaryOp::LessEqual => "<=",
BinaryOp::LogicalAnd => "&&",
BinaryOp::LogicalOr => "||",
BinaryOp::And => "&",
BinaryOp::Patch => "//",
}
}
fn compare_operator(operator: CompareOp) -> &'static str {
match operator {
CompareOp::Gt => ">",
CompareOp::Gte => ">=",
CompareOp::Lt => "<",
CompareOp::Lte => "<=",
CompareOp::Eq => "==",
}
}
fn same_kind(left: &SyntaxTokenKind, right: &SyntaxTokenKind) -> bool {
core::mem::discriminant(left) == core::mem::discriminant(right)
}
fn write_indent(out: &mut String, indent: usize) {
out.extend(core::iter::repeat_n(' ', indent));
}
fn ensure_line_break(out: &mut String) {
if !out.is_empty() && !out.ends_with('\n') {
out.push('\n');
}
}
fn ensure_blank_line(out: &mut String) {
if !out.is_empty() && !out.ends_with("\n\n") {
ensure_line_break(out);
out.push('\n');
}
}
fn trim_trailing_whitespace(out: &mut String) {
while out.ends_with(char::is_whitespace) {
out.pop();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn formats_fields_and_preserves_comments() {
let source =
"Server={\n# host\nhost=String default \"localhost\"; # trailing\n\nport=Int&>0;\n};";
let formatted = format_source(source).unwrap();
assert_eq!(
formatted,
"Server = {\n # host\n host = String default \"localhost\"; # trailing\n\n port = Int & > 0;\n};\n"
);
}
#[test]
fn formats_arrays_and_match_comments() {
let source = "result=match x{# c\n_: [1,{a=2;}];};";
let formatted = format_source(source).unwrap();
assert_eq!(
formatted,
"result = match x {\n # c\n _: [\n 1,\n {\n a = 2;\n },\n ];\n};\n"
);
}
#[test]
fn formats_array_constraints() {
let source = "tags=[...String];ports=[...(Int&>=1)];";
let formatted = format_source(source).unwrap();
assert_eq!(
formatted,
"tags = [...String];\nports = [...(Int & >= 1)];\n"
);
}
#[test]
fn formats_map_constraints_and_range_ascription() {
let source =
"config={service={port=8080;};}as{service={port=Int;};};labels={a=1;b=2;}as{...Int};";
let formatted = format_source(source).unwrap();
assert_eq!(
formatted,
"config = {\n service = {\n port = 8080;\n };\n} as {\n service = {\n port = Int;\n };\n};\nlabels = {\n a = 1;\n b = 2;\n} as {...Int};\n"
);
assert_eq!(format_source(&formatted).unwrap(), formatted);
}
#[test]
fn formats_let_functions_and_precedence_from_the_canonical_ast() {
let source = "value=let x=1+2*3;in(a:Int)=>{result=(x+a)*2;};";
let formatted = format_source(source).unwrap();
assert_eq!(
formatted,
"value = let\n x = 1 + 2 * 3;\nin\n (a: Int) =>\n {\n result = (x + a) * 2;\n };\n"
);
}
#[test]
fn formats_unknown_and_object_rest_constraints() {
let source =
"value={hoge=Int;fuga=String;# remaining fields\n...# unconstrained\nUnknown};";
let formatted = format_source(source).unwrap();
assert_eq!(
formatted,
"value = {\n hoge = Int;\n fuga = String; # remaining fields\n ... # unconstrained\n Unknown\n};\n"
);
}
#[test]
fn preserves_regex_and_escaped_strings() {
let source = r#"value={pattern=/^api\/.+$/;text="a\n\"b";};"#;
let formatted = format_source(source).unwrap();
assert!(formatted.contains(r#"pattern = /^api\/.+$/;"#));
assert!(formatted.contains(r#"text = "a\n\"b";"#));
}
#[test]
fn preserves_comments_inside_grouping_and_between_structural_tokens() {
let source = "value = (# grouped\n 1 + 2); result = match x # before brace\n { _ # before body\n : value; };";
let formatted = format_source(source).unwrap();
assert!(formatted.contains("(# grouped\n 1 + 2)"));
assert!(formatted.contains("match x # before brace\n{"));
assert!(formatted.contains("_ # before body\n : value;"));
assert_eq!(format_source(&formatted).unwrap(), formatted);
}
#[test]
fn wasm_export_returns_the_same_formatter_result() {
let source = "value={a=1;};";
let expected = format_source(source).unwrap();
let output: serde_json::Value = serde_json::from_str(&format_source_json(source)).unwrap();
assert_eq!(output["source"], expected);
}
#[test]
fn formats_all_repository_examples_idempotently() {
let examples = [
(
"advanced/main",
include_str!("../../../examples/advanced/main.dcdl"),
),
(
"advanced/profiles",
include_str!("../../../examples/advanced/profiles.dcdl"),
),
(
"advanced/schema",
include_str!("../../../examples/advanced/schema.dcdl"),
),
(
"arithmetic",
include_str!("../../../examples/arithmetic.dcdl"),
),
(
"array-concat",
include_str!("../../../examples/array-concat.dcdl"),
),
(
"array-constraint",
include_str!("../../../examples/array-constraint.dcdl"),
),
("basic", include_str!("../../../examples/basic.dcdl")),
(
"import/main",
include_str!("../../../examples/import/main.dcdl"),
),
(
"import/schema",
include_str!("../../../examples/import/schema.dcdl"),
),
("logical", include_str!("../../../examples/logical.dcdl")),
(
"map-ascription",
include_str!("../../../examples/map-ascription.dcdl"),
),
(
"regex/main",
include_str!("../../../examples/regex/main.dcdl"),
),
];
for (name, source) in examples {
let once = format_source(source).unwrap_or_else(|error| panic!("{name}: {error}"));
let twice = format_source(&once).unwrap_or_else(|error| {
panic!("{name} reformatted to invalid source: {error}\n{once}")
});
assert_eq!(once, twice, "{name}");
}
}
}