Compare commits

51 Commits
Author SHA1 Message Date
Hare e4b9247c45 Extract CodeMirror language package 2026-07-09 03:49:44 +09:00
Hare 375bc80a57 Add package links to landing page 2026-07-09 03:31:20 +09:00
Hare 335ecde816 Fix JSR wasm package license 2026-07-09 03:29:27 +09:00
Hare 4142fcd46a Prepare 0.1.2 release 2026-07-09 03:12:09 +09:00
Hare 2d06dc3277 Add source link to header 2026-07-09 03:00:30 +09:00
Hare 37237dc1f7 Add source link to footer 2026-07-09 02:59:32 +09:00
Hare 2966d5b979 Add site footer 2026-07-09 02:58:53 +09:00
Hare ea858ad28e Remove landing page eyebrow 2026-07-09 02:57:55 +09:00
Hare 871d342e15 Add site landing page 2026-07-09 02:55:50 +09:00
Hare 6d80c66144 Trim open issues to materialization 2026-07-09 02:46:28 +09:00
Hare 8e1cb01f52 Document Decodal components 2026-07-09 01:47:28 +09:00
Hare 26d8495631 Remove public tokenizer API 2026-07-09 01:09:40 +09:00
Hare c4332ffb9d Restore CodeMirror syntax highlighting 2026-07-09 00:38:31 +09:00
Hare 33ea8b94f9 Fix CodeMirror folding ranges 2026-07-09 00:04:34 +09:00
Hare 720c71157e Fold Decodal delimited blocks from opener 2026-07-08 23:55:04 +09:00
Hare ee031bd2e8 Add canonical grammar and CodeMirror playground 2026-07-08 23:41:26 +09:00
Hare a07f4c48aa Expose tokenizer for web highlighting 2026-07-08 19:50:04 +09:00
Hare fe127428f9 Update generated WASM package README 2026-06-26 01:26:23 +09:00
Hare 0b76a600b1 Make Pages deploy script portable 2026-06-26 01:25:22 +09:00
Hare ada3578a1d Expose derive macro behind feature 2026-06-26 00:59:39 +09:00
Hare b8404df047 Add typed Decodal derive support 2026-06-26 00:39:33 +09:00
Hare 87fef44c68 Make decodal the published library crate 2026-06-25 22:51:16 +09:00
Hare 6e2363632b Prepare crates.io release metadata 2026-06-25 22:25:01 +09:00
Hare c26f9ca50a Add playground example loader 2026-06-25 03:39:40 +09:00
Hare f928028007 Deploy Pages production branch by default 2026-06-25 03:12:37 +09:00
Hare d8af980f7d Add Cloudflare Pages deploy script 2026-06-25 02:49:15 +09:00
Hare a2a6dee025 Show source names in diagnostics 2026-06-22 18:05:26 +09:00
Hare cc5ab63922 Improve composition diagnostics 2026-06-22 17:37:51 +09:00
Hare 8b3df21cfa Document lightweight runtime boundaries 2026-06-19 22:48:33 +09:00
Hare 01ad6dca52 Add array concat operator 2026-06-19 01:01:04 +09:00
Hare 6da0ec4c77 Fix inline code escaping 2026-06-18 23:55:11 +09:00
Hare 683151f6bd Remove unimplemented operator note 2026-06-18 23:54:10 +09:00
Hare 9e728fb634 Document operator reference 2026-06-18 23:42:53 +09:00
Hare 2fe54bda62 Add logical and comparison expressions 2026-06-17 23:38:44 +09:00
Hare 3f7dd7c692 Add arithmetic expressions 2026-06-17 23:00:14 +09:00
Hare dc28cddbff Add dark theme support 2026-06-17 14:39:44 +09:00
Hare aa6e5c51c1 Number manual sidebar navigation 2026-06-17 13:37:53 +09:00
Hare 4ab47e5719 Add multi-file playground imports 2026-06-17 10:17:00 +09:00
Hare 19c9de1601 Add site syntax highlighting 2026-06-17 07:39:00 +09:00
Hare 0e873fbd51 Migrate documentation site to Astro 2026-06-17 00:22:32 +09:00
Hare 4020b7c2d5 Add Svelte docs site and WASM playground 2026-06-17 00:07:11 +09:00
Hare 58d2c9b423 Add Tree-sitter grammar for Decodal 2026-06-16 12:01:29 +09:00
Hare f1a5836247 Add optional regex constraints 2026-06-16 11:27:27 +09:00
Hare 84680e2652 Normalize primitive and numeric constraints 2026-06-16 11:22:09 +09:00
Hare e23b31da46 Add host prelude embedding API 2026-06-16 11:06:56 +09:00
Hare 6316939438 Add advanced Decodal example 2026-06-16 10:04:44 +09:00
Hare bd3da1aeee Implement module loading and imports 2026-06-16 10:01:31 +09:00
Hare fead194ba6 Add Decodal CLI smoke example 2026-06-16 02:11:44 +09:00
Hare c33c484bac Implement evaluator materializer and CLI 2026-06-16 02:10:54 +09:00
Hare ddcee75c8d Implement core lexer and parser 2026-06-16 02:02:17 +09:00
Hare 12a6e9a84c Split core library and CLI workspace 2026-06-16 01:58:17 +09:00
147 changed files with 32269 additions and 253 deletions
+6
View File
@@ -3,3 +3,9 @@
/.env
/.yoi
/result
node_modules
/dist
site/decodal-site/dist
# Astro
site/decodal-site/.astro
Generated
+233 -1
View File
@@ -2,6 +2,238 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "aho-corasick"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
dependencies = [
"memchr",
]
[[package]]
name = "bumpalo"
version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "decodal"
version = "0.1.0"
version = "0.1.2"
dependencies = [
"decodal-derive",
"regex",
]
[[package]]
name = "decodal-cli"
version = "0.1.2"
dependencies = [
"decodal",
]
[[package]]
name = "decodal-derive"
version = "0.1.2"
dependencies = [
"decodal",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "decodal-wasm"
version = "0.1.2"
dependencies = [
"decodal",
"serde_json",
"wasm-bindgen",
]
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "memchr"
version = "2.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
[[package]]
name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
dependencies = [
"proc-macro2",
]
[[package]]
name = "regex"
version = "1.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "rustversion"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "syn"
version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "wasm-bindgen"
version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.125"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
dependencies = [
"unicode-ident",
]
[[package]]
name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+19 -5
View File
@@ -1,6 +1,20 @@
[package]
name = "decodal"
version = "0.1.0"
edition = "2024"
[workspace]
members = [
"crates/decodal-core",
"crates/decodal-cli",
"crates/decodal-wasm",
"crates/decodal-derive",
]
resolver = "2"
[dependencies]
[workspace.package]
version = "0.1.2"
edition = "2024"
rust-version = "1.85"
license = "MIT OR Apache-2.0"
repository = "https://gitea.hareworks.net/Hare/Decodal"
readme = "README.md"
[workspace.dependencies]
serde_json = "1"
wasm-bindgen = "0.2"
+173
View File
@@ -0,0 +1,173 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Decodal contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+82
View File
@@ -0,0 +1,82 @@
# Decodal
Decodal is a small deterministic DSL for describing, composing, validating, and materializing structured data.
It is designed around a lightweight Rust library:
- host-supplied imports through `SourceLoader`
- no filesystem access in the library core
- concrete and abstract values with constraints and defaults
- deterministic expression evaluation
- optional regex support behind a Cargo feature
- browser playground support through WebAssembly
## Library crate
Embedded hosts should depend on `decodal` and provide imports with a `SourceLoader`.
```toml
[dependencies]
decodal = "0.1"
```
## Derive support
For embedded Rust applications, Decodal can generate a schema and typed decoder from a Rust struct with the `derive` feature.
```toml
[dependencies]
decodal = { version = "0.1", features = ["derive"] }
```
```rust
use decodal::{Decodal, DecodalDecode, DecodalSchema, Engine};
#[derive(Decodal)]
struct Service {
name: String,
#[decodal(gt = 443, default = 8443)]
port: i64,
#[decodal(rename = "feature.enable", default = true)]
feature_enabled: bool,
}
```
The derive implements:
- `DecodalSchema`, which produces a host schema for `Engine::bind_global`
- `DecodalDecode`, which converts materialized `Data` into the Rust struct
## CLI
A standalone CLI is kept in this repository as the `decodal-cli` workspace package.
It builds a `decodal` binary, but it is not the primary crates.io package.
Run a Decodal file from the repository:
```sh
cargo run -q -p decodal-cli -- examples/advanced/main.dcdl
```
Enable optional regex support when needed:
```sh
cargo run -q -p decodal-cli --features regex -- examples/regex/main.dcdl
```
## Web playground
The static documentation site and browser playground live under:
```text
site/decodal-site/
```
## License
Licensed under either of:
- Apache License, Version 2.0
- MIT license
at your option.
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "decodal-cli"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
readme.workspace = true
description = "Command-line interface for the Decodal data description language."
keywords = ["decodal", "dsl", "config", "cli"]
categories = ["command-line-utilities", "config"]
publish = false
[[bin]]
name = "decodal"
path = "src/main.rs"
[features]
default = []
regex = ["decodal/regex"]
[dependencies]
decodal = { version = "0.1.2", path = "../decodal-core" }
+222
View File
@@ -0,0 +1,222 @@
use std::{
env, fs,
path::{Path, PathBuf},
process::ExitCode,
};
use decodal::{
Data, Diagnostic, DiagnosticKind, Engine, LoadedSource, SourceId, SourceLoader, Span,
format_diagnostic_with,
};
fn main() -> ExitCode {
match run() {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("{error}");
ExitCode::FAILURE
}
}
}
fn run() -> Result<(), String> {
let mut args = env::args().skip(1);
let first = args.next().unwrap_or_else(|| String::from("--help"));
match first.as_str() {
"--help" | "-h" => {
print_help();
Ok(())
}
"--version" | "-V" => {
println!("Decodal {}", decodal::version());
Ok(())
}
"check" => {
let path = args.next().unwrap_or_else(|| String::from("-"));
let data = materialize_path(&path)?;
drop(data);
println!("ok");
Ok(())
}
"eval" | "materialize" => {
let path = args.next().unwrap_or_else(|| String::from("-"));
let data = materialize_path(&path)?;
print_data(&data, 0);
println!();
Ok(())
}
path => {
let data = materialize_path(path)?;
print_data(&data, 0);
println!();
Ok(())
}
}
}
fn materialize_path(path: &str) -> Result<Data, String> {
let root = read_root_source(path).map_err(format_raw_diagnostic)?;
let root_name = root.name.clone();
let mut engine = Engine::new(FsLoader);
let module = engine
.add_root_source(root.key, root.name, &root.source)
.map_err(|error| format_diagnostic_with_root(&error, &root_name))?;
let value = engine
.eval_module(module)
.map_err(|error| engine.format_diagnostic(&error))?;
engine
.materialize(&value)
.map_err(|error| engine.format_diagnostic(&error))
}
fn read_root_source(path: &str) -> Result<LoadedSource, Diagnostic> {
if path == "-" {
use std::io::Read;
let mut source = String::new();
std::io::stdin()
.read_to_string(&mut source)
.map_err(|error| {
Diagnostic::new(
DiagnosticKind::Import,
Span::default(),
format!("failed to read stdin: {error}"),
)
})?;
Ok(LoadedSource {
key: String::from("<stdin>"),
name: String::from("<stdin>"),
source,
})
} else {
load_path(Path::new(path))
}
}
#[derive(Debug, Clone, Copy)]
struct FsLoader;
impl SourceLoader for FsLoader {
fn load(
&mut self,
current_key: Option<&str>,
specifier: &str,
) -> Result<LoadedSource, Diagnostic> {
let path = Path::new(specifier);
let path = if path.is_absolute() {
PathBuf::from(path)
} else if let Some(current_key) = current_key.filter(|key| *key != "<stdin>") {
Path::new(current_key)
.parent()
.unwrap_or_else(|| Path::new("."))
.join(path)
} else {
PathBuf::from(path)
};
load_path(&path)
}
}
fn load_path(path: &Path) -> Result<LoadedSource, Diagnostic> {
let canonical = path.canonicalize().map_err(|error| {
Diagnostic::new(
DiagnosticKind::Import,
Span::default(),
format!("failed to resolve `{}`: {error}", path.display()),
)
})?;
let source = fs::read_to_string(&canonical).map_err(|error| {
Diagnostic::new(
DiagnosticKind::Import,
Span::default(),
format!("failed to read `{}`: {error}", canonical.display()),
)
})?;
let key = canonical.to_string_lossy().into_owned();
Ok(LoadedSource {
name: key.clone(),
key,
source,
})
}
fn print_help() {
println!("Decodal - Deferred Constraint Data Language");
println!();
println!("Usage:");
println!(" decodal <file.dcdl> Materialize a DCDL file");
println!(" decodal eval <file.dcdl> Materialize a DCDL file");
println!(" decodal check <file.dcdl> Evaluate and materialize without printing data");
println!(" decodal - Read DCDL source from stdin");
}
fn format_diagnostic_with_root(error: &Diagnostic, root_name: &str) -> String {
format_diagnostic_with(error, |source| (source == SourceId(0)).then_some(root_name))
}
fn format_raw_diagnostic(error: Diagnostic) -> String {
format_diagnostic_with(&error, |_| None)
}
fn print_data(data: &Data, indent: usize) {
match data {
Data::String(value) => print_string(value),
Data::Int(value) => print!("{value}"),
Data::Float(value) => print!("{value}"),
Data::Bool(value) => print!("{value}"),
Data::Array(items) => {
print!("[");
if !items.is_empty() {
println!();
for (index, item) in items.iter().enumerate() {
print_indent(indent + 2);
print_data(item, indent + 2);
if index + 1 != items.len() {
print!(",");
}
println!();
}
print_indent(indent);
}
print!("]");
}
Data::Object(fields) => {
print!("{{");
if !fields.is_empty() {
println!();
for (index, field) in fields.iter().enumerate() {
print_indent(indent + 2);
print_string(&field.name);
print!(": ");
print_data(&field.value, indent + 2);
if index + 1 != fields.len() {
print!(",");
}
println!();
}
print_indent(indent);
}
print!("}}");
}
}
}
fn print_indent(indent: usize) {
for _ in 0..indent {
print!(" ");
}
}
fn print_string(value: &str) {
print!("\"");
for ch in value.chars() {
match ch {
'"' => print!("\\\""),
'\\' => print!("\\\\"),
'\n' => print!("\\n"),
'\r' => print!("\\r"),
'\t' => print!("\\t"),
ch => print!("{ch}"),
}
}
print!("\"");
}
+21
View File
@@ -0,0 +1,21 @@
[package]
name = "decodal"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
readme.workspace = true
description = "Parser, evaluator, and embedding API for the Decodal data description language."
keywords = ["decodal", "dsl", "config", "schema"]
categories = ["config", "parser-implementations"]
[features]
default = ["std"]
std = []
derive = ["dep:decodal-derive"]
regex = ["std", "dep:regex"]
[dependencies]
decodal-derive = { version = "0.1.1", path = "../decodal-derive", optional = true }
regex = { version = "1.10", default-features = false, features = ["std", "unicode-perl"], optional = true }
@@ -0,0 +1,38 @@
use decodal::{Data, EmptyLoader, Engine, HostValue};
fn main() -> decodal::Result<()> {
let mut engine = Engine::new(EmptyLoader);
engine.bind_global(
"Service",
HostValue::object([
("name", HostValue::string_type()),
("port", HostValue::int_type().gt(443).default_int(8443)?),
("enabled", HostValue::bool_type().default_bool(true)?),
]),
)?;
let module = engine.add_root_source(
"embedded-main",
"embedded-main",
r#"
Service & {
name = "api";
port = 9443;
}
"#,
)?;
let value = engine.eval_module(module)?;
let data = engine.materialize(&value)?;
if let Data::Object(fields) = data {
assert_eq!(fields[0].value, Data::String(String::from("api")));
assert_eq!(fields[1].value, Data::Int(9443));
assert_eq!(fields[2].value, Data::Bool(true));
} else {
panic!("expected object");
}
Ok(())
}
+156
View File
@@ -0,0 +1,156 @@
use alloc::{string::String, vec::Vec};
use crate::span::Span;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ExprId(pub u32);
#[derive(Debug, Default, Clone)]
pub struct Ast {
exprs: Vec<SpannedExpr>,
}
impl Ast {
pub fn new() -> Self {
Self { exprs: Vec::new() }
}
pub fn push(&mut self, expr: Expr, span: Span) -> ExprId {
let id = ExprId(self.exprs.len() as u32);
self.exprs.push(SpannedExpr { expr, span });
id
}
pub fn get(&self, id: ExprId) -> &SpannedExpr {
&self.exprs[id.0 as usize]
}
pub fn span(&self, id: ExprId) -> Span {
self.get(id).span
}
pub fn len(&self) -> usize {
self.exprs.len()
}
pub fn is_empty(&self) -> bool {
self.exprs.is_empty()
}
}
#[derive(Debug, Clone)]
pub struct SpannedExpr {
pub expr: Expr,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Expr {
Literal(Literal),
Ident(String),
Object(Vec<Field>),
Array(Vec<ExprId>),
Let {
bindings: Vec<Field>,
body: ExprId,
},
Import(String),
Path {
base: ExprId,
field: String,
},
Call {
callee: ExprId,
args: Vec<ExprId>,
},
Function {
params: Vec<Param>,
body: ExprId,
},
Match {
scrutinee: ExprId,
arms: Vec<MatchArm>,
},
Unary {
op: UnaryOp,
expr: ExprId,
},
Binary {
op: BinaryOp,
lhs: ExprId,
rhs: ExprId,
},
Default {
base: ExprId,
fallback: ExprId,
},
CompareConstraint {
op: CompareOp,
value: ExprId,
},
RegexConstraint(String),
Wildcard,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Field {
pub path: Vec<String>,
pub value: ExprId,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Param {
pub name: String,
pub constraint: Option<ExprId>,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq)]
pub struct MatchArm {
pub pattern: ExprId,
pub body: ExprId,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Literal {
String(String),
Int(i64),
Float(f64),
Bool(bool),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UnaryOp {
Neg,
Not,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BinaryOp {
Add,
Sub,
Mul,
Div,
Concat,
Equal,
NotEqual,
Greater,
GreaterEqual,
Less,
LessEqual,
LogicalAnd,
LogicalOr,
And,
Patch,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CompareOp {
Gt,
Gte,
Lt,
Lte,
Eq,
}
+365
View File
@@ -0,0 +1,365 @@
use alloc::{string::String, vec::Vec};
use crate::{
Diagnostic, DiagnosticKind, Span,
ast::CompareOp,
runtime::{Constraint, ConstraintEntry, LiteralValue, PrimitiveType},
};
pub fn normalize_constraints(
constraints: Vec<ConstraintEntry>,
span: Span,
) -> crate::Result<Vec<ConstraintEntry>> {
let mut primitive: Option<(PrimitiveType, Span)> = None;
let mut lower: Option<(Bound, Span)> = None;
let mut upper: Option<(Bound, Span)> = None;
let mut rest = Vec::new();
for entry in constraints {
match entry.constraint {
Constraint::Type(next) => match primitive {
Some((current, current_span)) if current != next => {
return Err(Diagnostic::new(
DiagnosticKind::Conflict,
span,
"primitive type constraints conflict",
)
.with_label(current_span, "first primitive constraint")
.with_label(entry.span, "conflicting primitive constraint"));
}
Some(_) => {}
None => primitive = Some((next, entry.span)),
},
Constraint::Compare(op, value) => {
let number = Number::from_literal(&value).ok_or_else(|| {
Diagnostic::new(
DiagnosticKind::TypeMismatch,
entry.span,
"comparison constraints require numeric literals",
)
})?;
match op {
CompareOp::Gt => {
merge_lower(&mut lower, (Bound::new(number, false), entry.span))
}
CompareOp::Gte => {
merge_lower(&mut lower, (Bound::new(number, true), entry.span))
}
CompareOp::Lt => {
merge_upper(&mut upper, (Bound::new(number, false), entry.span))
}
CompareOp::Lte => {
merge_upper(&mut upper, (Bound::new(number, true), entry.span))
}
CompareOp::Eq => {
merge_lower(&mut lower, (Bound::new(number, true), entry.span));
merge_upper(&mut upper, (Bound::new(number, true), entry.span));
}
}
}
Constraint::Regex(pattern) => rest.push(ConstraintEntry {
constraint: Constraint::Regex(pattern),
span: entry.span,
}),
Constraint::BuiltinPredicate(name) => rest.push(ConstraintEntry {
constraint: Constraint::BuiltinPredicate(name),
span: entry.span,
}),
}
}
if matches!(
primitive,
Some((
PrimitiveType::String | PrimitiveType::Bool | PrimitiveType::Array,
_
))
) && (lower.is_some() || upper.is_some())
{
let mut diagnostic = Diagnostic::new(
DiagnosticKind::Conflict,
span,
"numeric comparison constraints conflict with non-numeric primitive type",
);
if let Some((_, primitive_span)) = primitive {
diagnostic = diagnostic.with_label(primitive_span, "non-numeric primitive constraint");
}
if let Some((_, lower_span)) = lower {
diagnostic = diagnostic.with_label(lower_span, "numeric comparison constraint");
}
if let Some((_, upper_span)) = upper {
diagnostic = diagnostic.with_label(upper_span, "numeric comparison constraint");
}
return Err(diagnostic);
}
if matches!(primitive, Some((PrimitiveType::Int, _)))
&& lower
.iter()
.chain(upper.iter())
.any(|(bound, _)| !matches!(bound.number, Number::Int(_)))
{
let mut diagnostic = Diagnostic::new(
DiagnosticKind::Conflict,
span,
"Int comparison constraints must use integer literals",
);
if let Some((_, primitive_span)) = primitive {
diagnostic = diagnostic.with_label(primitive_span, "Int primitive constraint");
}
if let Some((bound, bound_span)) = lower {
if !matches!(bound.number, Number::Int(_)) {
diagnostic = diagnostic.with_label(bound_span, "non-integer comparison constraint");
}
}
if let Some((bound, bound_span)) = upper {
if !matches!(bound.number, Number::Int(_)) {
diagnostic = diagnostic.with_label(bound_span, "non-integer comparison constraint");
}
}
return Err(diagnostic);
}
ensure_bounds_non_empty(primitive, lower, upper, span)?;
let mut normalized = Vec::new();
if let Some((primitive, primitive_span)) = primitive {
normalized.push(ConstraintEntry {
constraint: Constraint::Type(primitive),
span: primitive_span,
});
}
if let Some((lower, lower_span)) = lower {
normalized.push(ConstraintEntry {
constraint: Constraint::Compare(
if lower.inclusive {
CompareOp::Gte
} else {
CompareOp::Gt
},
lower.number.into_literal(),
),
span: lower_span,
});
}
if let Some((upper, upper_span)) = upper {
normalized.push(ConstraintEntry {
constraint: Constraint::Compare(
if upper.inclusive {
CompareOp::Lte
} else {
CompareOp::Lt
},
upper.number.into_literal(),
),
span: upper_span,
});
}
normalized.extend(rest);
Ok(normalized)
}
#[derive(Debug, Clone, Copy, PartialEq)]
struct Bound {
number: Number,
inclusive: bool,
}
impl Bound {
fn new(number: Number, inclusive: bool) -> Self {
Self { number, inclusive }
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum Number {
Int(i64),
Float(f64),
}
impl Number {
fn from_literal(value: &LiteralValue) -> Option<Self> {
match value {
LiteralValue::Int(value) => Some(Self::Int(*value)),
LiteralValue::Float(value) => Some(Self::Float(*value)),
LiteralValue::String(_) | LiteralValue::Bool(_) => None,
}
}
fn into_literal(self) -> LiteralValue {
match self {
Self::Int(value) => LiteralValue::Int(value),
Self::Float(value) => LiteralValue::Float(value),
}
}
fn as_f64(self) -> f64 {
match self {
Self::Int(value) => value as f64,
Self::Float(value) => value,
}
}
}
fn merge_lower(current: &mut Option<(Bound, Span)>, next: (Bound, Span)) {
match current {
None => *current = Some(next),
Some((current_bound, _)) if is_stricter_lower(next.0, *current_bound) => {
*current = Some(next)
}
Some(_) => {}
}
}
fn merge_upper(current: &mut Option<(Bound, Span)>, next: (Bound, Span)) {
match current {
None => *current = Some(next),
Some((current_bound, _)) if is_stricter_upper(next.0, *current_bound) => {
*current = Some(next)
}
Some(_) => {}
}
}
fn is_stricter_lower(next: Bound, current: Bound) -> bool {
let next_value = next.number.as_f64();
let current_value = current.number.as_f64();
next_value > current_value
|| (next_value == current_value && !next.inclusive && current.inclusive)
}
fn is_stricter_upper(next: Bound, current: Bound) -> bool {
let next_value = next.number.as_f64();
let current_value = current.number.as_f64();
next_value < current_value
|| (next_value == current_value && !next.inclusive && current.inclusive)
}
fn ensure_bounds_non_empty(
primitive: Option<(PrimitiveType, Span)>,
lower: Option<(Bound, Span)>,
upper: Option<(Bound, Span)>,
span: Span,
) -> crate::Result<()> {
if matches!(primitive, Some((PrimitiveType::Int, _))) {
let min = lower
.map(|(bound, _)| int_lower_bound(bound))
.unwrap_or(i128::from(i64::MIN));
let max = upper
.map(|(bound, _)| int_upper_bound(bound))
.unwrap_or(i128::from(i64::MAX));
if min > max {
return Err(empty_numeric_bounds(span, lower, upper));
}
return Ok(());
}
if let (Some((lower_bound, _)), Some((upper_bound, _))) = (lower, upper) {
let lower_value = lower_bound.number.as_f64();
let upper_value = upper_bound.number.as_f64();
if lower_value > upper_value {
return Err(empty_numeric_bounds(span, lower, upper));
}
if lower_value == upper_value && !(lower_bound.inclusive && upper_bound.inclusive) {
return Err(empty_numeric_bounds(span, lower, upper));
}
}
Ok(())
}
fn int_lower_bound(bound: Bound) -> i128 {
let Number::Int(value) = bound.number else {
unreachable!()
};
if bound.inclusive {
i128::from(value)
} else {
i128::from(value) + 1
}
}
fn int_upper_bound(bound: Bound) -> i128 {
let Number::Int(value) = bound.number else {
unreachable!()
};
if bound.inclusive {
i128::from(value)
} else {
i128::from(value) - 1
}
}
fn empty_numeric_bounds(
span: Span,
lower: Option<(Bound, Span)>,
upper: Option<(Bound, Span)>,
) -> Diagnostic {
let mut diagnostic = Diagnostic::new(
DiagnosticKind::Conflict,
span,
String::from("numeric comparison constraints have an empty intersection"),
);
if let Some((_, lower_span)) = lower {
diagnostic = diagnostic.with_label(lower_span, "lower bound constraint");
}
if let Some((_, upper_span)) = upper {
diagnostic = diagnostic.with_label(upper_span, "upper bound constraint");
}
diagnostic
}
#[cfg(test)]
mod tests {
use super::*;
use crate::runtime::LiteralValue;
fn entry(constraint: Constraint) -> ConstraintEntry {
ConstraintEntry {
constraint,
span: Span::default(),
}
}
#[test]
fn detects_primitive_conflict() {
assert!(
normalize_constraints(
alloc::vec![
entry(Constraint::Type(PrimitiveType::Int)),
entry(Constraint::Type(PrimitiveType::String)),
],
Span::default(),
)
.is_err()
);
}
#[test]
fn detects_empty_int_range() {
assert!(
normalize_constraints(
alloc::vec![
entry(Constraint::Type(PrimitiveType::Int)),
entry(Constraint::Compare(CompareOp::Gt, LiteralValue::Int(10))),
entry(Constraint::Compare(CompareOp::Lt, LiteralValue::Int(5))),
],
Span::default(),
)
.is_err()
);
}
#[test]
fn detects_non_integer_int_bound() {
assert!(
normalize_constraints(
alloc::vec![
entry(Constraint::Type(PrimitiveType::Int)),
entry(Constraint::Compare(CompareOp::Gt, LiteralValue::Float(1.5))),
],
Span::default(),
)
.is_err()
);
}
}
+57
View File
@@ -0,0 +1,57 @@
use alloc::{string::String, vec::Vec};
use crate::span::Span;
pub type Result<T> = core::result::Result<T, Diagnostic>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Diagnostic {
pub kind: DiagnosticKind,
pub span: Span,
pub message: String,
pub labels: Vec<DiagnosticLabel>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DiagnosticLabel {
pub span: Span,
pub message: String,
}
impl Diagnostic {
pub fn new(kind: DiagnosticKind, span: Span, message: impl Into<String>) -> Self {
Self {
kind,
span,
message: message.into(),
labels: Vec::new(),
}
}
pub fn syntax(span: Span, message: impl Into<String>) -> Self {
Self::new(DiagnosticKind::Syntax, span, message)
}
pub fn with_label(mut self, span: Span, message: impl Into<String>) -> Self {
self.labels.push(DiagnosticLabel {
span,
message: message.into(),
});
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DiagnosticKind {
Syntax,
UnresolvedIdentifier,
TypeMismatch,
ConstraintViolation,
Conflict,
DefaultConflict,
Cycle,
Import,
MatchFailure,
Materialize,
UnsupportedFeature,
}
+215
View File
@@ -0,0 +1,215 @@
use alloc::{boxed::Box, string::String, vec::Vec};
use crate::runtime::{Constraint, LiteralValue, PrimitiveType};
use crate::{CompareOp, Diagnostic, DiagnosticKind, Result, Span};
#[derive(Debug, Clone, PartialEq)]
pub enum HostValue {
String(String),
Int(i64),
Float(f64),
Bool(bool),
Array(Vec<HostValue>),
Object(Vec<HostField>),
Abstract {
constraints: Vec<Constraint>,
default: Option<Box<HostValue>>,
},
}
#[derive(Debug, Clone, PartialEq)]
pub struct HostField {
pub name: String,
pub value: HostValue,
}
impl HostValue {
pub fn string(value: impl Into<String>) -> Self {
Self::String(value.into())
}
pub fn int(value: i64) -> Self {
Self::Int(value)
}
pub fn float(value: f64) -> Self {
Self::Float(value)
}
pub fn bool(value: bool) -> Self {
Self::Bool(value)
}
pub fn array<I>(items: I) -> Self
where
I: IntoIterator<Item = HostValue>,
{
Self::Array(items.into_iter().collect())
}
pub fn object<I, N>(fields: I) -> Self
where
I: IntoIterator<Item = (N, HostValue)>,
N: Into<String>,
{
Self::Object(
fields
.into_iter()
.map(|(name, value)| HostField {
name: name.into(),
value,
})
.collect(),
)
}
pub fn object_from_paths<I, N>(fields: I) -> Self
where
I: IntoIterator<Item = (N, HostValue)>,
N: Into<String>,
{
let mut root = Vec::new();
for (path, value) in fields {
insert_path(&mut root, &path.into(), value);
}
Self::Object(root)
}
pub fn string_type() -> Self {
Self::abstract_with_constraint(Constraint::Type(PrimitiveType::String))
}
pub fn int_type() -> Self {
Self::abstract_with_constraint(Constraint::Type(PrimitiveType::Int))
}
pub fn float_type() -> Self {
Self::abstract_with_constraint(Constraint::Type(PrimitiveType::Float))
}
pub fn bool_type() -> Self {
Self::abstract_with_constraint(Constraint::Type(PrimitiveType::Bool))
}
pub fn array_type() -> Self {
Self::abstract_with_constraint(Constraint::Type(PrimitiveType::Array))
}
pub fn builtin_predicate(name: impl Into<String>) -> Self {
Self::abstract_with_constraint(Constraint::BuiltinPredicate(name.into()))
}
pub fn abstract_with_constraint(constraint: Constraint) -> Self {
Self::Abstract {
constraints: alloc::vec![constraint],
default: None,
}
}
pub fn with_constraint(mut self, constraint: Constraint) -> Self {
match &mut self {
Self::Abstract { constraints, .. } => constraints.push(constraint),
_ => {
self = Self::Abstract {
constraints: alloc::vec![constraint],
default: Some(Box::new(self)),
};
}
}
self
}
pub fn gt(self, value: i64) -> Self {
self.with_constraint(Constraint::Compare(CompareOp::Gt, LiteralValue::Int(value)))
}
pub fn gte(self, value: i64) -> Self {
self.with_constraint(Constraint::Compare(
CompareOp::Gte,
LiteralValue::Int(value),
))
}
pub fn lt(self, value: i64) -> Self {
self.with_constraint(Constraint::Compare(CompareOp::Lt, LiteralValue::Int(value)))
}
pub fn lte(self, value: i64) -> Self {
self.with_constraint(Constraint::Compare(
CompareOp::Lte,
LiteralValue::Int(value),
))
}
pub fn default(self, value: HostValue) -> Result<Self> {
match self {
Self::Abstract {
constraints,
default: None,
} => Ok(Self::Abstract {
constraints,
default: Some(Box::new(value)),
}),
Self::Abstract { .. } => Err(Diagnostic::new(
DiagnosticKind::DefaultConflict,
Span::default(),
"host value already has a default",
)),
concrete => Ok(Self::Abstract {
constraints: Vec::new(),
default: Some(Box::new(concrete)),
}),
}
}
pub fn default_string(self, value: impl Into<String>) -> Result<Self> {
self.default(Self::string(value))
}
pub fn default_int(self, value: i64) -> Result<Self> {
self.default(Self::int(value))
}
pub fn default_float(self, value: f64) -> Result<Self> {
self.default(Self::float(value))
}
pub fn default_bool(self, value: bool) -> Result<Self> {
self.default(Self::bool(value))
}
}
impl HostField {
pub fn new(name: impl Into<String>, value: HostValue) -> Self {
Self {
name: name.into(),
value,
}
}
}
fn insert_path(fields: &mut Vec<HostField>, path: &str, value: HostValue) {
let mut parts = path.splitn(2, '.');
let Some(head) = parts.next().filter(|part| !part.is_empty()) else {
return;
};
if let Some(tail) = parts.next() {
if let Some(field) = fields.iter_mut().find(|field| field.name == head) {
if let HostValue::Object(children) = &mut field.value {
insert_path(children, tail, value);
} else {
let mut children = Vec::new();
insert_path(&mut children, tail, value);
field.value = HostValue::Object(children);
}
} else {
let mut children = Vec::new();
insert_path(&mut children, tail, value);
fields.push(HostField::new(head, HostValue::Object(children)));
}
} else if let Some(field) = fields.iter_mut().find(|field| field.name == head) {
field.value = value;
} else {
fields.push(HostField::new(head, value));
}
}
File diff suppressed because it is too large Load Diff
+435
View File
@@ -0,0 +1,435 @@
use alloc::{string::String, vec::Vec};
use crate::{Diagnostic, SourceId, Span, diagnostic::Result};
#[derive(Debug, Clone, PartialEq)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq)]
pub enum TokenKind {
Ident(String),
Int(i64),
Float(f64),
String(String),
Regex(String),
True,
False,
Let,
In,
Match,
Import,
Default,
Underscore,
LBrace,
RBrace,
LBracket,
RBracket,
LParen,
RParen,
Semicolon,
Comma,
Dot,
Colon,
Equal,
EqualEqual,
Bang,
BangEqual,
Arrow,
Amp,
AmpAmp,
PipePipe,
Plus,
PlusPlus,
Minus,
Star,
Slash,
SlashSlash,
Gt,
Gte,
Lt,
Lte,
Eof,
}
pub struct Lexer<'a> {
source_id: SourceId,
source: &'a str,
bytes: &'a [u8],
pos: usize,
}
impl<'a> Lexer<'a> {
#[cfg(test)]
pub fn new(source: &'a str) -> Self {
Self::with_source_id(SourceId(0), source)
}
pub fn with_source_id(source_id: SourceId, source: &'a str) -> Self {
Self {
source_id,
source,
bytes: source.as_bytes(),
pos: 0,
}
}
pub fn tokenize(mut self) -> Result<Vec<Token>> {
let mut tokens = Vec::new();
let mut previous = None;
loop {
let token = self.next_token(previous.as_ref())?;
let is_eof = token.kind == TokenKind::Eof;
if !is_eof {
previous = Some(token.kind.clone());
}
tokens.push(token);
if is_eof {
return Ok(tokens);
}
}
}
fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
self.skip_ws_and_comments();
self.next_non_ws_token(previous)
}
fn next_non_ws_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
let start = self.pos;
let Some(ch) = self.peek() else {
return Ok(Token {
kind: TokenKind::Eof,
span: self.empty_span(self.pos),
});
};
let kind = match ch {
b'{' => {
self.pos += 1;
TokenKind::LBrace
}
b'}' => {
self.pos += 1;
TokenKind::RBrace
}
b'[' => {
self.pos += 1;
TokenKind::LBracket
}
b']' => {
self.pos += 1;
TokenKind::RBracket
}
b'(' => {
self.pos += 1;
TokenKind::LParen
}
b')' => {
self.pos += 1;
TokenKind::RParen
}
b';' => {
self.pos += 1;
TokenKind::Semicolon
}
b',' => {
self.pos += 1;
TokenKind::Comma
}
b'.' => {
self.pos += 1;
TokenKind::Dot
}
b':' => {
self.pos += 1;
TokenKind::Colon
}
b'_' => {
self.pos += 1;
TokenKind::Underscore
}
b'&' => {
self.pos += 1;
if self.consume(b'&') {
TokenKind::AmpAmp
} else {
TokenKind::Amp
}
}
b'|' => {
self.pos += 1;
if self.consume(b'|') {
TokenKind::PipePipe
} else {
return Err(Diagnostic::syntax(
self.span(start, self.pos),
"expected '|' after '|'",
));
}
}
b'+' => {
self.pos += 1;
if self.consume(b'+') {
TokenKind::PlusPlus
} else {
TokenKind::Plus
}
}
b'-' => {
self.pos += 1;
TokenKind::Minus
}
b'*' => {
self.pos += 1;
TokenKind::Star
}
b'=' => {
self.pos += 1;
if self.consume(b'>') {
TokenKind::Arrow
} else if self.consume(b'=') {
TokenKind::EqualEqual
} else {
TokenKind::Equal
}
}
b'!' => {
self.pos += 1;
if self.consume(b'=') {
TokenKind::BangEqual
} else {
TokenKind::Bang
}
}
b'>' => {
self.pos += 1;
if self.consume(b'=') {
TokenKind::Gte
} else {
TokenKind::Gt
}
}
b'<' => {
self.pos += 1;
if self.consume(b'=') {
TokenKind::Lte
} else {
TokenKind::Lt
}
}
b'/' => {
self.pos += 1;
if self.consume(b'/') {
TokenKind::SlashSlash
} else if previous.is_some_and(token_can_end_expr) {
TokenKind::Slash
} else {
self.lex_regex(start)?
}
}
b'"' => self.lex_string()?,
b'0'..=b'9' => self.lex_number()?,
c if is_ident_start(c) => self.lex_ident_or_keyword(),
_ => {
return Err(Diagnostic::syntax(
self.span(start, start + 1),
"unexpected character",
));
}
};
Ok(Token {
kind,
span: self.span(start, self.pos),
})
}
fn skip_ws_and_comments(&mut self) {
loop {
while matches!(self.peek(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
self.pos += 1;
}
if self.peek() == Some(b'#') {
while let Some(c) = self.peek() {
self.pos += 1;
if c == b'\n' {
break;
}
}
continue;
}
break;
}
}
fn lex_string(&mut self) -> Result<TokenKind> {
let start = self.pos;
self.pos += 1;
let mut value = String::new();
while let Some(c) = self.peek() {
self.pos += 1;
match c {
b'"' => return Ok(TokenKind::String(value)),
b'\\' => {
let Some(escaped) = self.peek() else {
return Err(Diagnostic::syntax(
self.span(start, self.pos),
"unterminated escape",
));
};
self.pos += 1;
let ch = match escaped {
b'"' => '"',
b'\\' => '\\',
b'n' => '\n',
b'r' => '\r',
b't' => '\t',
other => other as char,
};
value.push(ch);
}
other => value.push(other as char),
}
}
Err(Diagnostic::syntax(
self.span(start, self.pos),
"unterminated string",
))
}
fn lex_regex(&mut self, start: usize) -> Result<TokenKind> {
let mut pattern = String::new();
let mut escaped = false;
while let Some(c) = self.peek() {
self.pos += 1;
if escaped {
pattern.push(c as char);
escaped = false;
continue;
}
match c {
b'\\' => {
pattern.push('\\');
escaped = true;
}
b'/' => return Ok(TokenKind::Regex(pattern)),
other => pattern.push(other as char),
}
}
Err(Diagnostic::syntax(
self.span(start, self.pos),
"unterminated regex",
))
}
fn lex_number(&mut self) -> Result<TokenKind> {
let start = self.pos;
while matches!(self.peek(), Some(b'0'..=b'9')) {
self.pos += 1;
}
let mut is_float = false;
if self.peek() == Some(b'.') && matches!(self.peek_n(1), Some(b'0'..=b'9')) {
is_float = true;
self.pos += 1;
while matches!(self.peek(), Some(b'0'..=b'9')) {
self.pos += 1;
}
}
let text = &self.source[start..self.pos];
if is_float {
text.parse::<f64>().map(TokenKind::Float).map_err(|_| {
Diagnostic::syntax(self.span(start, self.pos), "invalid float literal")
})
} else {
text.parse::<i64>()
.map(TokenKind::Int)
.map_err(|_| Diagnostic::syntax(self.span(start, self.pos), "invalid int literal"))
}
}
fn lex_ident_or_keyword(&mut self) -> TokenKind {
let start = self.pos;
self.pos += 1;
while matches!(self.peek(), Some(c) if is_ident_continue(c)) {
self.pos += 1;
}
let text = &self.source[start..self.pos];
match text {
"true" => TokenKind::True,
"false" => TokenKind::False,
"let" => TokenKind::Let,
"in" => TokenKind::In,
"match" => TokenKind::Match,
"import" => TokenKind::Import,
"default" => TokenKind::Default,
_ => TokenKind::Ident(String::from(text)),
}
}
fn span(&self, start: usize, end: usize) -> Span {
Span::new(self.source_id, start, end)
}
fn empty_span(&self, offset: usize) -> Span {
Span::empty(self.source_id, offset)
}
fn peek(&self) -> Option<u8> {
self.bytes.get(self.pos).copied()
}
fn peek_n(&self, n: usize) -> Option<u8> {
self.bytes.get(self.pos + n).copied()
}
fn consume(&mut self, expected: u8) -> bool {
if self.peek() == Some(expected) {
self.pos += 1;
true
} else {
false
}
}
}
fn is_ident_start(c: u8) -> bool {
c.is_ascii_alphabetic()
}
fn is_ident_continue(c: u8) -> bool {
c.is_ascii_alphanumeric() || c == b'_'
}
fn token_can_end_expr(kind: &TokenKind) -> bool {
matches!(
kind,
TokenKind::Ident(_)
| TokenKind::Int(_)
| TokenKind::Float(_)
| TokenKind::String(_)
| TokenKind::Regex(_)
| TokenKind::True
| TokenKind::False
| TokenKind::Underscore
| TokenKind::RBrace
| TokenKind::RBracket
| TokenKind::RParen
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tokenizes_basic_source() {
let tokens = Lexer::new("port = Int & >= 1;").tokenize().unwrap();
assert!(matches!(tokens[0].kind, TokenKind::Ident(_)));
assert_eq!(tokens[1].kind, TokenKind::Equal);
assert_eq!(tokens[3].kind, TokenKind::Amp);
assert_eq!(tokens[4].kind, TokenKind::Gte);
}
}
+35
View File
@@ -0,0 +1,35 @@
#![cfg_attr(not(feature = "std"), no_std)]
extern crate alloc;
pub mod ast;
pub mod constraints;
pub mod diagnostic;
pub mod embedding;
pub mod eval;
mod lexer;
pub mod module;
pub mod parser;
pub mod runtime;
pub mod span;
pub mod typed;
pub use ast::{Ast, BinaryOp, CompareOp, Expr, ExprId, Field, Literal, Param};
pub use constraints::normalize_constraints;
#[cfg(feature = "derive")]
pub use decodal_derive::Decodal;
pub use diagnostic::{Diagnostic, DiagnosticKind, Result};
pub use embedding::{HostField, HostValue};
pub use eval::{Engine, format_diagnostic_with};
pub use module::{EmptyLoader, LoadedSource, Module, SourceLoader};
pub use parser::{ParseOutput, Parser, SourceForm, parse_source, parse_source_with_source_id};
pub use runtime::{Constraint, Data, ExprRef, LiteralValue, ModuleId, PrimitiveType, RuntimeValue};
pub use span::{SourceId, Span};
pub use typed::{
DecodalDecode, DecodalSchema, DecodeError, DecodeResult, IntoHostValue, data_at_path,
decode_path, prefix_decode_error,
};
pub fn version() -> &'static str {
env!("CARGO_PKG_VERSION")
}
+42
View File
@@ -0,0 +1,42 @@
use alloc::string::String;
use crate::{
Ast, ExprId, SourceForm, SourceId,
runtime::{EnvId, ThunkId},
};
#[derive(Debug, Clone)]
pub struct Module {
pub key: String,
pub name: String,
pub source: SourceId,
pub ast: Ast,
pub root: ExprId,
pub source_form: SourceForm,
pub root_env: EnvId,
pub root_thunk: ThunkId,
}
#[derive(Debug, Clone)]
pub struct LoadedSource {
pub key: String,
pub name: String,
pub source: String,
}
pub trait SourceLoader {
fn load(&mut self, current_key: Option<&str>, specifier: &str) -> crate::Result<LoadedSource>;
}
#[derive(Debug, Clone, Copy, Default)]
pub struct EmptyLoader;
impl SourceLoader for EmptyLoader {
fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> crate::Result<LoadedSource> {
Err(crate::Diagnostic::new(
crate::DiagnosticKind::Import,
crate::Span::default(),
alloc::format!("no source loader is configured for import `{specifier}`"),
))
}
}
+677
View File
@@ -0,0 +1,677 @@
use alloc::{string::String, vec::Vec};
use crate::{
SourceId, Span,
ast::{Ast, BinaryOp, CompareOp, Expr, ExprId, Field, Literal, MatchArm, Param, UnaryOp},
diagnostic::{Diagnostic, Result},
lexer::{Lexer, Token, TokenKind},
};
#[derive(Debug, Clone)]
pub struct ParseOutput {
pub ast: Ast,
pub root: ExprId,
pub source_form: SourceForm,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SourceForm {
Expr,
Fields,
}
pub fn parse_source(source: &str) -> Result<ParseOutput> {
parse_source_with_source_id(SourceId(0), source)
}
pub fn parse_source_with_source_id(source_id: SourceId, source: &str) -> Result<ParseOutput> {
let tokens = Lexer::with_source_id(source_id, source).tokenize()?;
Parser::new(tokens).parse()
}
pub struct Parser {
tokens: Vec<Token>,
pos: usize,
ast: Ast,
}
impl Parser {
pub fn new(tokens: Vec<Token>) -> Self {
Self {
tokens,
pos: 0,
ast: Ast::new(),
}
}
pub fn parse(mut self) -> Result<ParseOutput> {
let (root, source_form) = if self.starts_field() {
let fields = self.parse_fields_until_eof()?;
let span = fields
.first()
.map(|f| {
fields
.iter()
.fold(f.span, |acc, field| acc.join(field.span))
})
.unwrap_or_else(|| self.peek().span);
(
self.ast.push(Expr::Object(fields), span),
SourceForm::Fields,
)
} else {
let expr = self.parse_expr(0)?;
self.expect_eof()?;
(expr, SourceForm::Expr)
};
Ok(ParseOutput {
ast: self.ast,
root,
source_form,
})
}
fn parse_expr(&mut self, min_bp: u8) -> Result<ExprId> {
let mut lhs = self.parse_prefix()?;
loop {
if self.at_eof() || self.is_expr_stop() {
break;
}
// postfix: path reference
if self.consume_kind(&TokenKind::Dot).is_some() {
let (field, field_span) = self.expect_ident()?;
let span = self.ast.span(lhs).join(field_span);
lhs = self.ast.push(Expr::Path { base: lhs, field }, span);
continue;
}
// postfix: call
if self.consume_kind(&TokenKind::LParen).is_some() {
let mut args = Vec::new();
if self.consume_kind(&TokenKind::RParen).is_none() {
loop {
args.push(self.parse_expr(0)?);
if self.consume_kind(&TokenKind::Comma).is_some() {
continue;
}
self.expect_kind(&TokenKind::RParen, "expected ')' after call arguments")?;
break;
}
}
let span = self.ast.span(lhs).join(self.previous_span());
lhs = self.ast.push(Expr::Call { callee: lhs, args }, span);
continue;
}
let Some((kind, l_bp, r_bp)) = self.peek_infix() else {
break;
};
if l_bp < min_bp {
break;
}
let op_span = self.advance().span;
let rhs = self.parse_expr(r_bp)?;
let span = self.ast.span(lhs).join(self.ast.span(rhs)).join(op_span);
lhs = match kind {
InfixKind::Add => self.ast.push(
Expr::Binary {
op: BinaryOp::Add,
lhs,
rhs,
},
span,
),
InfixKind::Sub => self.ast.push(
Expr::Binary {
op: BinaryOp::Sub,
lhs,
rhs,
},
span,
),
InfixKind::Mul => self.ast.push(
Expr::Binary {
op: BinaryOp::Mul,
lhs,
rhs,
},
span,
),
InfixKind::Div => self.ast.push(
Expr::Binary {
op: BinaryOp::Div,
lhs,
rhs,
},
span,
),
InfixKind::Concat => self.ast.push(
Expr::Binary {
op: BinaryOp::Concat,
lhs,
rhs,
},
span,
),
InfixKind::Equal => self.ast.push(
Expr::Binary {
op: BinaryOp::Equal,
lhs,
rhs,
},
span,
),
InfixKind::NotEqual => self.ast.push(
Expr::Binary {
op: BinaryOp::NotEqual,
lhs,
rhs,
},
span,
),
InfixKind::Greater => self.ast.push(
Expr::Binary {
op: BinaryOp::Greater,
lhs,
rhs,
},
span,
),
InfixKind::GreaterEqual => self.ast.push(
Expr::Binary {
op: BinaryOp::GreaterEqual,
lhs,
rhs,
},
span,
),
InfixKind::Less => self.ast.push(
Expr::Binary {
op: BinaryOp::Less,
lhs,
rhs,
},
span,
),
InfixKind::LessEqual => self.ast.push(
Expr::Binary {
op: BinaryOp::LessEqual,
lhs,
rhs,
},
span,
),
InfixKind::LogicalAnd => self.ast.push(
Expr::Binary {
op: BinaryOp::LogicalAnd,
lhs,
rhs,
},
span,
),
InfixKind::LogicalOr => self.ast.push(
Expr::Binary {
op: BinaryOp::LogicalOr,
lhs,
rhs,
},
span,
),
InfixKind::And => self.ast.push(
Expr::Binary {
op: BinaryOp::And,
lhs,
rhs,
},
span,
),
InfixKind::Patch => self.ast.push(
Expr::Binary {
op: BinaryOp::Patch,
lhs,
rhs,
},
span,
),
InfixKind::Default => self.ast.push(
Expr::Default {
base: lhs,
fallback: rhs,
},
span,
),
};
}
Ok(lhs)
}
fn parse_prefix(&mut self) -> Result<ExprId> {
let token = self.advance().clone();
match token.kind {
TokenKind::String(value) => Ok(self
.ast
.push(Expr::Literal(Literal::String(value)), token.span)),
TokenKind::Int(value) => Ok(self
.ast
.push(Expr::Literal(Literal::Int(value)), token.span)),
TokenKind::Float(value) => Ok(self
.ast
.push(Expr::Literal(Literal::Float(value)), token.span)),
TokenKind::True => Ok(self
.ast
.push(Expr::Literal(Literal::Bool(true)), token.span)),
TokenKind::False => Ok(self
.ast
.push(Expr::Literal(Literal::Bool(false)), token.span)),
TokenKind::Ident(name) => Ok(self.ast.push(Expr::Ident(name), token.span)),
TokenKind::Regex(pattern) => {
Ok(self.ast.push(Expr::RegexConstraint(pattern), token.span))
}
TokenKind::Underscore => Ok(self.ast.push(Expr::Wildcard, token.span)),
TokenKind::LBrace => self.parse_object_after_lbrace(token.span),
TokenKind::LBracket => self.parse_array_after_lbracket(token.span),
TokenKind::LParen => self.parse_group_or_function(token.span),
TokenKind::Let => self.parse_let(token.span),
TokenKind::Match => self.parse_match(token.span),
TokenKind::Import => self.parse_import(token.span),
TokenKind::Minus => {
let expr = self.parse_expr(17)?;
let span = token.span.join(self.ast.span(expr));
Ok(self.ast.push(
Expr::Unary {
op: UnaryOp::Neg,
expr,
},
span,
))
}
TokenKind::Bang => {
let expr = self.parse_expr(17)?;
let span = token.span.join(self.ast.span(expr));
Ok(self.ast.push(
Expr::Unary {
op: UnaryOp::Not,
expr,
},
span,
))
}
TokenKind::Gt | TokenKind::Gte | TokenKind::Lt | TokenKind::Lte => {
let op = match token.kind {
TokenKind::Gt => CompareOp::Gt,
TokenKind::Gte => CompareOp::Gte,
TokenKind::Lt => CompareOp::Lt,
TokenKind::Lte => CompareOp::Lte,
_ => unreachable!(),
};
let value = self.parse_expr(12)?;
let span = token.span.join(self.ast.span(value));
Ok(self.ast.push(Expr::CompareConstraint { op, value }, span))
}
_ => Err(Diagnostic::syntax(token.span, "expected expression")),
}
}
fn parse_object_after_lbrace(&mut self, start_span: Span) -> Result<ExprId> {
let mut fields = Vec::new();
if self.consume_kind(&TokenKind::RBrace).is_some() {
return Ok(self
.ast
.push(Expr::Object(fields), start_span.join(self.previous_span())));
}
loop {
fields.push(self.parse_field()?);
if self.consume_kind(&TokenKind::Semicolon).is_some() {
if self.consume_kind(&TokenKind::RBrace).is_some() {
break;
}
continue;
}
self.expect_kind(&TokenKind::RBrace, "expected ';' or '}' after object field")?;
break;
}
let span = start_span.join(self.previous_span());
Ok(self.ast.push(Expr::Object(fields), span))
}
fn parse_array_after_lbracket(&mut self, start_span: Span) -> Result<ExprId> {
let mut items = Vec::new();
if self.consume_kind(&TokenKind::RBracket).is_some() {
return Ok(self
.ast
.push(Expr::Array(items), start_span.join(self.previous_span())));
}
loop {
items.push(self.parse_expr(0)?);
if self.consume_kind(&TokenKind::Comma).is_some() {
if self.consume_kind(&TokenKind::RBracket).is_some() {
break;
}
continue;
}
self.expect_kind(&TokenKind::RBracket, "expected ',' or ']' after array item")?;
break;
}
let span = start_span.join(self.previous_span());
Ok(self.ast.push(Expr::Array(items), span))
}
fn parse_group_or_function(&mut self, start_span: Span) -> Result<ExprId> {
if self.looks_like_params() {
let params = self.parse_params_after_lparen()?;
self.expect_kind(&TokenKind::Arrow, "expected '=>' after function parameters")?;
let body = self.parse_expr(0)?;
let span = start_span.join(self.ast.span(body));
return Ok(self.ast.push(Expr::Function { params, body }, span));
}
let expr = self.parse_expr(0)?;
self.expect_kind(&TokenKind::RParen, "expected ')' after expression")?;
Ok(expr)
}
fn parse_params_after_lparen(&mut self) -> Result<Vec<Param>> {
let mut params = Vec::new();
if self.consume_kind(&TokenKind::RParen).is_some() {
return Ok(params);
}
loop {
let (name, name_span) = self.expect_ident()?;
let constraint = if self.consume_kind(&TokenKind::Colon).is_some() {
Some(self.parse_expr(0)?)
} else {
None
};
let span = constraint
.map(|id| name_span.join(self.ast.span(id)))
.unwrap_or(name_span);
params.push(Param {
name,
constraint,
span,
});
if self.consume_kind(&TokenKind::Comma).is_some() {
continue;
}
self.expect_kind(&TokenKind::RParen, "expected ',' or ')' after parameter")?;
break;
}
Ok(params)
}
fn parse_let(&mut self, start_span: Span) -> Result<ExprId> {
let mut bindings = Vec::new();
while !self.check_kind(&TokenKind::In) && !self.at_eof() {
bindings.push(self.parse_field()?);
self.expect_kind(&TokenKind::Semicolon, "expected ';' after let binding")?;
}
self.expect_kind(&TokenKind::In, "expected 'in' after let bindings")?;
let body = self.parse_expr(0)?;
let span = start_span.join(self.ast.span(body));
Ok(self.ast.push(Expr::Let { bindings, body }, span))
}
fn parse_match(&mut self, start_span: Span) -> Result<ExprId> {
let scrutinee = self.parse_expr(0)?;
self.expect_kind(&TokenKind::LBrace, "expected '{' after match scrutinee")?;
let mut arms = Vec::new();
if self.consume_kind(&TokenKind::RBrace).is_none() {
loop {
let pattern = self.parse_expr(0)?;
self.expect_kind(&TokenKind::Colon, "expected ':' after match pattern")?;
let body = self.parse_expr(0)?;
let span = self.ast.span(pattern).join(self.ast.span(body));
arms.push(MatchArm {
pattern,
body,
span,
});
if self.consume_kind(&TokenKind::Semicolon).is_some() {
if self.consume_kind(&TokenKind::RBrace).is_some() {
break;
}
continue;
}
self.expect_kind(&TokenKind::RBrace, "expected ';' or '}' after match arm")?;
break;
}
}
let span = start_span.join(self.previous_span());
Ok(self.ast.push(Expr::Match { scrutinee, arms }, span))
}
fn parse_import(&mut self, start_span: Span) -> Result<ExprId> {
let token = self.advance().clone();
let path = match token.kind {
TokenKind::String(path) | TokenKind::Ident(path) => path,
_ => return Err(Diagnostic::syntax(token.span, "expected import path")),
};
Ok(self
.ast
.push(Expr::Import(path), start_span.join(token.span)))
}
fn parse_fields_until_eof(&mut self) -> Result<Vec<Field>> {
let mut fields = Vec::new();
while !self.at_eof() {
fields.push(self.parse_field()?);
self.consume_kind(&TokenKind::Semicolon);
}
Ok(fields)
}
fn parse_field(&mut self) -> Result<Field> {
let (first, first_span) = self.expect_ident()?;
let mut path = Vec::new();
path.push(first);
let mut span = first_span;
while self.consume_kind(&TokenKind::Dot).is_some() {
let (name, name_span) = self.expect_ident()?;
span = span.join(name_span);
path.push(name);
}
self.expect_kind(&TokenKind::Equal, "expected '=' after field name")?;
let value = self.parse_expr(0)?;
span = span.join(self.ast.span(value));
Ok(Field { path, value, span })
}
fn starts_field(&self) -> bool {
matches!(self.peek_kind(), TokenKind::Ident(_))
&& matches!(self.peek_kind_n(1), TokenKind::Equal | TokenKind::Dot)
}
fn looks_like_params(&self) -> bool {
match self.peek_kind() {
TokenKind::RParen => matches!(self.peek_kind_n(1), TokenKind::Arrow),
TokenKind::Ident(_) => {
let mut i = self.pos;
loop {
if !matches!(self.kind_at(i), TokenKind::Ident(_)) {
return false;
}
i += 1;
if matches!(self.kind_at(i), TokenKind::Colon) {
// Skip a simple constraint expression approximately until comma/rparen.
i += 1;
let mut depth = 0usize;
while !matches!(self.kind_at(i), TokenKind::Eof) {
match self.kind_at(i) {
TokenKind::LParen | TokenKind::LBrace | TokenKind::LBracket => {
depth += 1
}
TokenKind::RParen if depth == 0 => break,
TokenKind::RParen | TokenKind::RBrace | TokenKind::RBracket => {
depth = depth.saturating_sub(1)
}
TokenKind::Comma if depth == 0 => break,
_ => {}
}
i += 1;
}
}
if matches!(self.kind_at(i), TokenKind::Comma) {
i += 1;
continue;
}
if matches!(self.kind_at(i), TokenKind::RParen) {
return matches!(self.kind_at(i + 1), TokenKind::Arrow);
}
return false;
}
}
_ => false,
}
}
fn peek_infix(&self) -> Option<(InfixKind, u8, u8)> {
match self.peek_kind() {
TokenKind::Default => Some((InfixKind::Default, 1, 2)),
TokenKind::SlashSlash => Some((InfixKind::Patch, 3, 4)),
TokenKind::Amp => Some((InfixKind::And, 5, 6)),
TokenKind::PipePipe => Some((InfixKind::LogicalOr, 7, 8)),
TokenKind::AmpAmp => Some((InfixKind::LogicalAnd, 9, 10)),
TokenKind::EqualEqual => Some((InfixKind::Equal, 11, 12)),
TokenKind::BangEqual => Some((InfixKind::NotEqual, 11, 12)),
TokenKind::Gt => Some((InfixKind::Greater, 11, 12)),
TokenKind::Gte => Some((InfixKind::GreaterEqual, 11, 12)),
TokenKind::Lt => Some((InfixKind::Less, 11, 12)),
TokenKind::Lte => Some((InfixKind::LessEqual, 11, 12)),
TokenKind::PlusPlus => Some((InfixKind::Concat, 12, 13)),
TokenKind::Plus => Some((InfixKind::Add, 13, 14)),
TokenKind::Minus => Some((InfixKind::Sub, 13, 14)),
TokenKind::Star => Some((InfixKind::Mul, 15, 16)),
TokenKind::Slash => Some((InfixKind::Div, 15, 16)),
_ => None,
}
}
fn is_expr_stop(&self) -> bool {
matches!(
self.peek_kind(),
TokenKind::Semicolon
| TokenKind::Comma
| TokenKind::RParen
| TokenKind::RBracket
| TokenKind::RBrace
| TokenKind::Colon
| TokenKind::In
| TokenKind::Arrow
)
}
fn expect_ident(&mut self) -> Result<(String, Span)> {
let token = self.advance().clone();
match token.kind {
TokenKind::Ident(name) => Ok((name, token.span)),
_ => Err(Diagnostic::syntax(token.span, "expected identifier")),
}
}
fn expect_kind(&mut self, expected: &TokenKind, message: &'static str) -> Result<Span> {
if let Some(span) = self.consume_kind(expected) {
Ok(span)
} else {
Err(Diagnostic::syntax(self.peek().span, message))
}
}
fn expect_eof(&mut self) -> Result<()> {
if self.at_eof() {
Ok(())
} else {
Err(Diagnostic::syntax(self.peek().span, "expected end of file"))
}
}
fn consume_kind(&mut self, expected: &TokenKind) -> Option<Span> {
if core::mem::discriminant(self.peek_kind()) == core::mem::discriminant(expected) {
Some(self.advance().span)
} else {
None
}
}
fn check_kind(&self, expected: &TokenKind) -> bool {
core::mem::discriminant(self.peek_kind()) == core::mem::discriminant(expected)
}
fn at_eof(&self) -> bool {
matches!(self.peek_kind(), TokenKind::Eof)
}
fn advance(&mut self) -> &Token {
let index = self.pos;
if !self.at_eof() {
self.pos += 1;
}
&self.tokens[index]
}
fn previous_span(&self) -> Span {
self.tokens[self.pos.saturating_sub(1)].span
}
fn peek(&self) -> &Token {
&self.tokens[self.pos]
}
fn peek_kind(&self) -> &TokenKind {
&self.peek().kind
}
fn peek_kind_n(&self, n: usize) -> &TokenKind {
self.kind_at(self.pos + n)
}
fn kind_at(&self, index: usize) -> &TokenKind {
self.tokens
.get(index)
.map(|token| &token.kind)
.unwrap_or(&TokenKind::Eof)
}
}
#[derive(Debug, Clone, Copy)]
enum InfixKind {
Add,
Sub,
Mul,
Div,
Concat,
Equal,
NotEqual,
Greater,
GreaterEqual,
Less,
LessEqual,
LogicalAnd,
LogicalOr,
And,
Patch,
Default,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ast::Expr;
#[test]
fn parses_top_level_fields_as_object() {
let parsed = parse_source("port = Int & >= 1 default 8080;").unwrap();
assert!(matches!(parsed.ast.get(parsed.root).expr, Expr::Object(_)));
}
#[test]
fn parses_object_dot_field() {
let parsed = parse_source("{ feature.enable = false; }").unwrap();
let Expr::Object(fields) = &parsed.ast.get(parsed.root).expr else {
panic!()
};
assert_eq!(fields[0].path, ["feature", "enable"]);
}
}
+155
View File
@@ -0,0 +1,155 @@
use alloc::{string::String, vec::Vec};
use crate::{ExprId, Span, ast::CompareOp};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ModuleId(pub u32);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ExprRef {
pub module: ModuleId,
pub expr: ExprId,
}
#[derive(Debug, Clone, PartialEq)]
pub enum RuntimeValue {
Concrete(ConcreteValue),
Abstract(AbstractValue),
}
#[derive(Debug, Clone, PartialEq)]
pub enum ConcreteValue {
String(String),
Int(i64),
Float(f64),
Bool(bool),
Array(Vec<ThunkId>),
Object(ObjectValue),
Function(FunctionValue),
}
#[derive(Debug, Clone, PartialEq)]
pub struct ObjectValue {
pub fields: Vec<ObjectField>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ObjectField {
pub name: String,
pub value: ThunkId,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq)]
pub struct FunctionValue {
pub params: Vec<FunctionParam>,
pub body: ExprRef,
pub env: EnvId,
}
#[derive(Debug, Clone, PartialEq)]
pub struct FunctionParam {
pub name: String,
pub constraint: Option<ExprRef>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct AbstractValue {
pub constraints: Vec<ConstraintEntry>,
pub default: Option<ThunkId>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ConstraintEntry {
pub constraint: Constraint,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Constraint {
Type(PrimitiveType),
Compare(CompareOp, LiteralValue),
Regex(String),
BuiltinPredicate(String),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PrimitiveType {
String,
Int,
Float,
Bool,
Array,
}
#[derive(Debug, Clone, PartialEq)]
pub enum LiteralValue {
String(String),
Int(i64),
Float(f64),
Bool(bool),
}
#[derive(Debug, Clone, PartialEq)]
pub enum Data {
String(String),
Int(i64),
Float(f64),
Bool(bool),
Array(Vec<Data>),
Object(Vec<DataField>),
}
#[derive(Debug, Clone, PartialEq)]
pub struct DataField {
pub name: String,
pub value: Data,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ThunkId(pub u32);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct EnvId(pub u32);
#[derive(Debug, Clone)]
pub struct Thunk {
pub kind: ThunkKind,
pub state: ThunkState,
pub span: Span,
}
#[derive(Debug, Clone)]
pub enum ThunkKind {
Expr {
expr: ExprRef,
env: EnvId,
},
Constrained {
constraint: ExprRef,
constraint_env: EnvId,
value: ExprRef,
value_env: EnvId,
},
Value(RuntimeValue),
}
#[derive(Debug, Clone)]
pub enum ThunkState {
Unevaluated,
Evaluating,
Evaluated(RuntimeValue),
Error,
}
#[derive(Debug, Clone)]
pub struct Env {
pub parent: Option<EnvId>,
pub bindings: Vec<Binding>,
}
#[derive(Debug, Clone)]
pub struct Binding {
pub name: String,
pub value: ThunkId,
}
+31
View File
@@ -0,0 +1,31 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub struct SourceId(pub u32);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct Span {
pub source: SourceId,
pub start: u32,
pub end: u32,
}
impl Span {
pub const fn new(source: SourceId, start: usize, end: usize) -> Self {
Self {
source,
start: start as u32,
end: end as u32,
}
}
pub const fn empty(source: SourceId, offset: usize) -> Self {
Self::new(source, offset, offset)
}
pub fn join(self, other: Self) -> Self {
Self {
source: self.source,
start: self.start.min(other.start),
end: self.end.max(other.end),
}
}
}
+227
View File
@@ -0,0 +1,227 @@
use alloc::{format, string::String, vec::Vec};
use crate::{Data, HostValue};
pub trait DecodalSchema {
fn decodal_schema() -> HostValue;
}
pub trait DecodalDecode: Sized {
fn decodal_decode(data: &Data) -> DecodeResult<Self>;
}
pub trait IntoHostValue {
fn into_host_value(self) -> HostValue;
}
pub type DecodeResult<T> = core::result::Result<T, DecodeError>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DecodeError {
pub path: String,
pub message: String,
}
impl DecodeError {
pub fn new(path: impl Into<String>, message: impl Into<String>) -> Self {
Self {
path: path.into(),
message: message.into(),
}
}
pub fn at_type(path: &str, expected: &'static str) -> Self {
Self::new(path, format!("expected {expected}"))
}
}
impl core::fmt::Display for DecodeError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
if self.path.is_empty() {
write!(f, "{}", self.message)
} else {
write!(f, "{}: {}", self.path, self.message)
}
}
}
#[cfg(feature = "std")]
impl std::error::Error for DecodeError {}
pub fn decode_path<T: DecodalDecode>(data: &Data, path: &str) -> DecodeResult<T> {
let value = data_at_path(data, path).ok_or_else(|| DecodeError::new(path, "missing field"))?;
T::decodal_decode(value).map_err(|error| prefix_error(path, error))
}
pub fn data_at_path<'a>(data: &'a Data, path: &str) -> Option<&'a Data> {
let mut current = data;
for part in path.split('.') {
if part.is_empty() {
return None;
}
let Data::Object(fields) = current else {
return None;
};
current = &fields.iter().find(|field| field.name == part)?.value;
}
Some(current)
}
pub fn prefix_decode_error(path: &str, error: DecodeError) -> DecodeError {
prefix_error(path, error)
}
fn prefix_error(path: &str, mut error: DecodeError) -> DecodeError {
if error.path.is_empty() {
error.path = String::from(path);
} else if !path.is_empty() {
error.path = format!("{path}.{}", error.path);
}
error
}
impl DecodalSchema for String {
fn decodal_schema() -> HostValue {
HostValue::string_type()
}
}
impl DecodalDecode for String {
fn decodal_decode(data: &Data) -> DecodeResult<Self> {
match data {
Data::String(value) => Ok(value.clone()),
_ => Err(DecodeError::at_type("", "String")),
}
}
}
impl IntoHostValue for String {
fn into_host_value(self) -> HostValue {
HostValue::string(self)
}
}
impl IntoHostValue for &str {
fn into_host_value(self) -> HostValue {
HostValue::string(self)
}
}
macro_rules! impl_int_decode {
($($ty:ty),* $(,)?) => {
$(
impl DecodalSchema for $ty {
fn decodal_schema() -> HostValue {
HostValue::int_type()
}
}
impl DecodalDecode for $ty {
fn decodal_decode(data: &Data) -> DecodeResult<Self> {
match data {
Data::Int(value) => <$ty>::try_from(*value)
.map_err(|_| DecodeError::new("", "integer value is out of range")),
_ => Err(DecodeError::at_type("", "Int")),
}
}
}
impl IntoHostValue for $ty {
fn into_host_value(self) -> HostValue {
HostValue::int(self as i64)
}
}
)*
};
}
impl_int_decode!(i8, i16, i32, i64, u8, u16, u32);
impl DecodalSchema for f64 {
fn decodal_schema() -> HostValue {
HostValue::float_type()
}
}
impl DecodalDecode for f64 {
fn decodal_decode(data: &Data) -> DecodeResult<Self> {
match data {
Data::Float(value) => Ok(*value),
Data::Int(value) => Ok(*value as f64),
_ => Err(DecodeError::at_type("", "Float")),
}
}
}
impl IntoHostValue for f64 {
fn into_host_value(self) -> HostValue {
HostValue::float(self)
}
}
impl DecodalSchema for f32 {
fn decodal_schema() -> HostValue {
HostValue::float_type()
}
}
impl DecodalDecode for f32 {
fn decodal_decode(data: &Data) -> DecodeResult<Self> {
f64::decodal_decode(data).map(|value| value as f32)
}
}
impl IntoHostValue for f32 {
fn into_host_value(self) -> HostValue {
HostValue::float(f64::from(self))
}
}
impl DecodalSchema for bool {
fn decodal_schema() -> HostValue {
HostValue::bool_type()
}
}
impl DecodalDecode for bool {
fn decodal_decode(data: &Data) -> DecodeResult<Self> {
match data {
Data::Bool(value) => Ok(*value),
_ => Err(DecodeError::at_type("", "Bool")),
}
}
}
impl IntoHostValue for bool {
fn into_host_value(self) -> HostValue {
HostValue::bool(self)
}
}
impl<T> DecodalSchema for Vec<T> {
fn decodal_schema() -> HostValue {
HostValue::array_type()
}
}
impl<T: DecodalDecode> DecodalDecode for Vec<T> {
fn decodal_decode(data: &Data) -> DecodeResult<Self> {
match data {
Data::Array(items) => items
.iter()
.enumerate()
.map(|(index, item)| {
T::decodal_decode(item)
.map_err(|error| prefix_error(&format!("[{index}]"), error))
})
.collect(),
_ => Err(DecodeError::at_type("", "Array")),
}
}
}
impl<T: IntoHostValue> IntoHostValue for Vec<T> {
fn into_host_value(self) -> HostValue {
HostValue::array(self.into_iter().map(IntoHostValue::into_host_value))
}
}
+22
View File
@@ -0,0 +1,22 @@
[package]
name = "decodal-derive"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
readme.workspace = true
description = "Derive macro for generating Decodal schemas and typed decoders from Rust structs."
keywords = ["decodal", "derive", "proc-macro", "schema"]
categories = ["config", "development-tools::procedural-macro-helpers"]
[lib]
proc-macro = true
[dependencies]
proc-macro2 = "1"
quote = "1"
syn = { version = "2", features = ["full"] }
[dev-dependencies]
decodal = { version = "0.1", path = "../decodal-core" }
+212
View File
@@ -0,0 +1,212 @@
use proc_macro::TokenStream;
use proc_macro2::TokenStream as TokenStream2;
use quote::{format_ident, quote};
use syn::{
Data, DeriveInput, Expr, Fields, LitStr, Result, Type, parse_macro_input, spanned::Spanned,
};
#[proc_macro_derive(Decodal, attributes(decodal))]
pub fn derive_decodal(input: TokenStream) -> TokenStream {
match expand_decodal(parse_macro_input!(input as DeriveInput)) {
Ok(tokens) => tokens.into(),
Err(error) => error.to_compile_error().into(),
}
}
fn expand_decodal(input: DeriveInput) -> Result<TokenStream2> {
let name = input.ident;
let fields = match input.data {
Data::Struct(data) => match data.fields {
Fields::Named(fields) => fields.named,
_ => {
return Err(syn::Error::new(
data.struct_token.span(),
"Decodal derive supports structs with named fields only",
));
}
},
_ => {
return Err(syn::Error::new(
name.span(),
"Decodal derive supports structs only",
));
}
};
let mut schema_fields = Vec::new();
let mut decode_fields = Vec::new();
for field in fields {
let ident = field.ident.expect("named field");
let ty = field.ty;
let attrs = FieldAttrs::from_attrs(&field.attrs, &ident.to_string())?;
let path = attrs.rename.clone();
let schema_value = schema_expr(&ty, &attrs)?;
let decode_value = decode_expr(&ty, &path, &attrs);
schema_fields.push(quote! {
(#path, { #schema_value })
});
decode_fields.push(quote! {
#ident: #decode_value
});
}
let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
Ok(quote! {
impl #impl_generics ::decodal::DecodalSchema for #name #ty_generics #where_clause {
fn decodal_schema() -> ::decodal::HostValue {
::decodal::HostValue::object_from_paths([
#(#schema_fields),*
])
}
}
impl #impl_generics ::decodal::DecodalDecode for #name #ty_generics #where_clause {
fn decodal_decode(data: &::decodal::Data) -> ::decodal::DecodeResult<Self> {
Ok(Self {
#(#decode_fields),*
})
}
}
})
}
#[derive(Default)]
struct FieldAttrs {
rename: String,
default: Option<DefaultAttr>,
constraints: Vec<ConstraintAttr>,
}
impl FieldAttrs {
fn from_attrs(attrs: &[syn::Attribute], fallback_name: &str) -> Result<Self> {
let mut output = Self {
rename: fallback_name.to_string(),
..Self::default()
};
for attr in attrs {
if !attr.path().is_ident("decodal") {
continue;
}
attr.parse_nested_meta(|meta| {
if meta.path.is_ident("rename") {
let value: LitStr = meta.value()?.parse()?;
output.rename = value.value();
return Ok(());
}
if meta.path.is_ident("default") {
if meta.input.peek(syn::Token![=]) {
let expr: Expr = meta.value()?.parse()?;
output.default = Some(DefaultAttr::Expr(expr));
} else {
output.default = Some(DefaultAttr::Default);
}
return Ok(());
}
if let Some(kind) = ConstraintKind::from_path(&meta.path) {
let expr: Expr = meta.value()?.parse()?;
output.constraints.push(ConstraintAttr { kind, expr });
return Ok(());
}
Err(meta.error("unsupported decodal attribute"))
})?;
}
Ok(output)
}
}
enum DefaultAttr {
Default,
Expr(Expr),
}
struct ConstraintAttr {
kind: ConstraintKind,
expr: Expr,
}
#[derive(Clone, Copy)]
enum ConstraintKind {
Gt,
Gte,
Lt,
Lte,
}
impl ConstraintKind {
fn from_path(path: &syn::Path) -> Option<Self> {
if path.is_ident("gt") {
Some(Self::Gt)
} else if path.is_ident("gte") {
Some(Self::Gte)
} else if path.is_ident("lt") {
Some(Self::Lt)
} else if path.is_ident("lte") {
Some(Self::Lte)
} else {
None
}
}
fn method(self) -> syn::Ident {
match self {
Self::Gt => format_ident!("gt"),
Self::Gte => format_ident!("gte"),
Self::Lt => format_ident!("lt"),
Self::Lte => format_ident!("lte"),
}
}
}
fn schema_expr(ty: &Type, attrs: &FieldAttrs) -> Result<TokenStream2> {
let mut tokens = quote! {
let mut schema = <#ty as ::decodal::DecodalSchema>::decodal_schema();
};
for constraint in &attrs.constraints {
let method = constraint.kind.method();
let expr = &constraint.expr;
tokens.extend(quote! {
schema = schema.#method(#expr);
});
}
if let Some(default) = &attrs.default {
let default_expr = match default {
DefaultAttr::Default => quote! { <#ty as ::core::default::Default>::default() },
DefaultAttr::Expr(expr) => quote! { (#expr) },
};
tokens.extend(quote! {
schema = schema
.default(::decodal::IntoHostValue::into_host_value(#default_expr))
.expect("Decodal derive generated a valid default");
});
}
tokens.extend(quote! { schema });
Ok(tokens)
}
fn decode_expr(ty: &Type, path: &str, attrs: &FieldAttrs) -> TokenStream2 {
let missing = if let Some(default) = &attrs.default {
match default {
DefaultAttr::Default => quote! { <#ty as ::core::default::Default>::default() },
DefaultAttr::Expr(expr) => quote! { ::core::convert::Into::into(#expr) },
}
} else {
quote! { return Err(::decodal::DecodeError::new(#path, "missing field")); }
};
quote! {
if let Some(value) = ::decodal::data_at_path(data, #path) {
<#ty as ::decodal::DecodalDecode>::decodal_decode(value)
.map_err(|error| ::decodal::prefix_decode_error(#path, error))?
} else {
#missing
}
}
}
+79
View File
@@ -0,0 +1,79 @@
use decodal::{Data, DecodalDecode, DecodalSchema, EmptyLoader, Engine};
use decodal_derive::Decodal;
#[derive(Debug, PartialEq, Decodal)]
struct Service {
name: String,
#[decodal(gt = 443, default = 8443)]
port: i64,
#[decodal(default = true, rename = "feature.enable")]
feature_enabled: bool,
#[decodal(default)]
tags: Vec<String>,
}
#[test]
fn derives_schema_and_decode() {
let mut engine = Engine::new(EmptyLoader);
engine
.bind_global("Service", Service::decodal_schema())
.unwrap();
let module = engine
.add_root_source(
"test",
"test",
r#"
Service & {
name = "api";
port = 9443;
feature.enable = false;
tags = ["web", "prod"];
}
"#,
)
.unwrap();
let value = engine.eval_module(module).unwrap();
let data = engine.materialize(&value).unwrap();
let service = Service::decodal_decode(&data).unwrap();
assert_eq!(
service,
Service {
name: "api".into(),
port: 9443,
feature_enabled: false,
tags: vec!["web".into(), "prod".into()],
}
);
}
#[test]
fn defaults_are_available_in_schema() {
let mut engine = Engine::new(EmptyLoader);
engine
.bind_global("Service", Service::decodal_schema())
.unwrap();
let module = engine
.add_root_source(
"test",
"test",
r#"
Service & {
name = "api";
}
"#,
)
.unwrap();
let value = engine.eval_module(module).unwrap();
let data = engine.materialize(&value).unwrap();
let service = Service::decodal_decode(&data).unwrap();
assert_eq!(service.port, 8443);
assert!(service.feature_enabled);
assert!(service.tags.is_empty());
}
#[test]
fn decode_reports_field_path() {
let data = Data::Object(vec![]);
let error = Service::decodal_decode(&data).unwrap_err();
assert_eq!(error.path, "name");
}
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "decodal-wasm"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
readme.workspace = true
description = "WebAssembly wrapper for evaluating Decodal in browser playgrounds."
keywords = ["decodal", "wasm", "dsl", "config"]
categories = ["wasm", "config"]
publish = false
[lib]
crate-type = ["cdylib", "rlib"]
[dependencies]
decodal = { version = "0.1.2", path = "../decodal-core" }
serde_json.workspace = true
wasm-bindgen.workspace = true
[package.metadata.wasm-pack.profile.release]
wasm-opt = false
+256
View File
@@ -0,0 +1,256 @@
use std::collections::BTreeMap;
use decodal::{
Data, Diagnostic, DiagnosticKind, EmptyLoader, Engine, LoadedSource, SourceId, SourceLoader,
Span, format_diagnostic_with,
};
use wasm_bindgen::prelude::*;
#[wasm_bindgen]
pub fn evaluate(source: &str) -> String {
encode_result(evaluate_inner(source))
}
#[wasm_bindgen(js_name = evaluateProject)]
pub fn evaluate_project(entry: &str, files_json: &str) -> String {
encode_result(evaluate_project_inner(entry, files_json))
}
fn encode_result(result: Result<String, String>) -> String {
match result {
Ok(output) => format!("{{\"ok\":true,\"output\":{}}}", json_string(&output)),
Err(error) => format!("{{\"ok\":false,\"error\":{}}}", json_string(&error)),
}
}
fn evaluate_inner(source: &str) -> Result<String, String> {
let mut engine = Engine::new(EmptyLoader);
let module = match engine.add_root_source("playground", "playground", source) {
Ok(module) => module,
Err(error) => return Err(format_diagnostic_with_root(&error, "playground")),
};
let value = match engine.eval_module(module) {
Ok(value) => value,
Err(error) => return Err(engine.format_diagnostic(&error)),
};
let data = match engine.materialize(&value) {
Ok(data) => data,
Err(error) => return Err(engine.format_diagnostic(&error)),
};
Ok(format_data(&data, 0))
}
fn evaluate_project_inner(entry: &str, files_json: &str) -> Result<String, String> {
let raw_files: BTreeMap<String, String> = serde_json::from_str(files_json)
.map_err(|error| format!("failed to read playground files: {error}"))?;
let mut files = BTreeMap::new();
for (path, source) in raw_files {
let path = normalize_path(&path).ok_or_else(|| format!("invalid file path `{path}`"))?;
files.insert(path, source);
}
let entry = normalize_path(entry).ok_or_else(|| format!("invalid entry path `{entry}`"))?;
let source = files
.get(&entry)
.cloned()
.ok_or_else(|| format!("entry file `{entry}` was not found"))?;
let mut engine = Engine::new(VirtualLoader { files });
let module = match engine.add_root_source(entry.clone(), entry.clone(), &source) {
Ok(module) => module,
Err(error) => return Err(format_diagnostic_with_root(&error, &entry)),
};
let value = match engine.eval_module(module) {
Ok(value) => value,
Err(error) => return Err(engine.format_diagnostic(&error)),
};
let data = match engine.materialize(&value) {
Ok(data) => data,
Err(error) => return Err(engine.format_diagnostic(&error)),
};
Ok(format_data(&data, 0))
}
#[derive(Debug, Clone)]
struct VirtualLoader {
files: BTreeMap<String, String>,
}
impl SourceLoader for VirtualLoader {
fn load(
&mut self,
current_key: Option<&str>,
specifier: &str,
) -> decodal::Result<LoadedSource> {
let key = resolve_import(current_key, specifier).ok_or_else(|| {
Diagnostic::new(
DiagnosticKind::Import,
Span::default(),
format!("invalid import path `{specifier}`"),
)
})?;
let source = self.files.get(&key).cloned().ok_or_else(|| {
Diagnostic::new(
DiagnosticKind::Import,
Span::default(),
format!("import `{specifier}` resolved to `{key}`, but that file does not exist"),
)
})?;
Ok(LoadedSource {
key: key.clone(),
name: key,
source,
})
}
}
fn resolve_import(current_key: Option<&str>, specifier: &str) -> Option<String> {
if specifier.starts_with('/') {
return normalize_path(specifier);
}
let mut base = String::new();
if let Some(current_key) = current_key {
if let Some((parent, _file)) = current_key.rsplit_once('/') {
base.push_str(parent);
base.push('/');
}
}
base.push_str(specifier);
normalize_path(&base)
}
fn normalize_path(path: &str) -> Option<String> {
let mut parts = Vec::new();
let normalized = path.replace('\\', "/");
for part in normalized.split('/') {
match part {
"" | "." => {}
".." => {
parts.pop()?;
}
part => parts.push(part),
}
}
if parts.is_empty() {
None
} else {
Some(parts.join("/"))
}
}
fn format_diagnostic_with_root(diagnostic: &decodal::Diagnostic, root_name: &str) -> String {
format_diagnostic_with(diagnostic, |source| {
(source == SourceId(0)).then_some(root_name)
})
}
fn format_data(data: &Data, indent: usize) -> String {
match data {
Data::String(value) => json_string(value),
Data::Int(value) => value.to_string(),
Data::Float(value) => value.to_string(),
Data::Bool(value) => value.to_string(),
Data::Array(items) => {
if items.is_empty() {
return String::from("[]");
}
let mut out = String::from("[\n");
for (index, item) in items.iter().enumerate() {
out.push_str(&" ".repeat(indent + 2));
out.push_str(&format_data(item, indent + 2));
if index + 1 != items.len() {
out.push(',');
}
out.push('\n');
}
out.push_str(&" ".repeat(indent));
out.push(']');
out
}
Data::Object(fields) => {
if fields.is_empty() {
return String::from("{}");
}
let mut out = String::from("{\n");
for (index, field) in fields.iter().enumerate() {
out.push_str(&" ".repeat(indent + 2));
out.push_str(&json_string(&field.name));
out.push_str(": ");
out.push_str(&format_data(&field.value, indent + 2));
if index + 1 != fields.len() {
out.push(',');
}
out.push('\n');
}
out.push_str(&" ".repeat(indent));
out.push('}');
out
}
}
}
fn json_string(value: &str) -> String {
let mut out = String::from("\"");
for ch in value.chars() {
match ch {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
ch if ch.is_control() => {
use core::fmt::Write;
let _ = write!(out, "\\u{:04x}", ch as u32);
}
ch => out.push(ch),
}
}
out.push('"');
out
}
#[cfg(test)]
mod tests {
use super::{evaluate_project_inner, normalize_path, resolve_import};
#[test]
fn normalizes_virtual_paths() {
assert_eq!(
normalize_path("/schemas/../main.dcdl"),
Some("main.dcdl".into())
);
assert_eq!(normalize_path("../main.dcdl"), None);
}
#[test]
fn resolves_imports_relative_to_current_file() {
assert_eq!(
resolve_import(Some("schemas/service.dcdl"), "./types.dcdl"),
Some("schemas/types.dcdl".into())
);
}
#[test]
fn evaluates_project_imports() {
let files = r#"{
"main.dcdl":"let dep = import \"./schemas/service.dcdl\"; in dep.Service & { port = 9443; }",
"schemas/service.dcdl":"Service = { name = String default \"api\"; port = Int & > 443 default 8443; }"
}"#;
let output = evaluate_project_inner("main.dcdl", files).unwrap();
assert!(output.contains("\"port\": 9443"));
}
#[test]
fn project_diagnostics_use_virtual_file_names() {
let files = r#"{
"main.dcdl":"let dep = import \"./schemas/service.dcdl\"; in dep.Service & { port = 80; }",
"schemas/service.dcdl":"Service = { port = Int & > 443 default 8443; }"
}"#;
let error = evaluate_project_inner("main.dcdl", files).unwrap_err();
assert!(error.contains("main.dcdl:"));
assert!(error.contains("schemas/service.dcdl:"));
assert!(!error.contains("source 0:"));
assert!(!error.contains("source 1:"));
}
}
+4
View File
@@ -4,9 +4,13 @@ pkgs.mkShell {
cargo
clippy
git
lld
nodejs
nixfmt
rustc
rustfmt
tree-sitter
wasm-pack
];
shellHook = ''
+97
View File
@@ -0,0 +1,97 @@
# Components
Decodal is split into a small runtime core and separate syntax tooling components.
The public surface is intentionally organized by use case: execute Decodal with Rust or WebAssembly, edit Decodal on the Web with Lezer and CodeMirror, and integrate Decodal into general-purpose editors with Tree-sitter.
## Runtime components
### Rust crate
The `decodal` crate is the primary Rust runtime and embedding API.
It owns parsing, evaluation, materialization, diagnostics, host-provided values, and schema/decode traits.
Rust applications should use this crate when they want to load Decodal source, evaluate it, or embed Decodal into a host program.
Important paths:
```text
crates/decodal-core/
crates/decodal-derive/
```
`decodal-derive` provides optional derive macros for Rust struct integration.
It is a companion to the runtime crate rather than an editor or syntax-highlighting component.
### WebAssembly package
`decodal-wasm` exposes the runtime to browsers.
The documentation site playground uses it to evaluate Decodal entirely in the browser.
Important paths:
```text
crates/decodal-wasm/
site/decodal-site/src/wasm/
```
The generated files in `site/decodal-site/src/wasm/` are committed so the site can build without requiring every consumer to run `wasm-pack` first.
The WebAssembly package is for execution, not syntax highlighting.
## Web editor components
The Web playground editor uses CodeMirror 6 with a generated Lezer parser.
This is the source of syntax highlighting, folding, indentation, and editor syntax tree behavior in the browser UI.
Important paths:
```text
editors/lezer-decodal/decodal.grammar
packages/decodal-codemirror/src/decodal.js
packages/decodal-codemirror/src/decodal-parser.js
packages/decodal-codemirror/src/decodal-parser.terms.js
```
The npm package is `decodal-codemirror`.
The JSR package is `@hare/decodal-codemirror`.
The Lezer grammar is derived from the canonical grammar documentation, but it is not a literal copy of the EBNF.
Precedence and token conflict handling are represented in the Lezer grammar in the form CodeMirror needs.
## General editor components
Tree-sitter is the portable editor-integration grammar.
Editors such as Zed, Neovim, Helix, and Emacs should consume this component when they need Decodal parsing or highlighting outside the Web playground.
Important paths:
```text
editors/tree-sitter-decodal/grammar.js
editors/tree-sitter-decodal/queries/highlights.scm
editors/tree-sitter-decodal/queries/locals.scm
editors/tree-sitter-decodal/src/
```
The generated parser sources under `editors/tree-sitter-decodal/src/` are committed so editor integrations can consume the grammar without regenerating it first.
## Canonical grammar
The human-readable grammar lives in:
```text
doc/manual/souce/language/grammar.md
```
This EBNF is the language-level reference.
The Rust parser, Lezer grammar, and Tree-sitter grammar should be kept aligned with it, but each implementation may encode precedence and recovery behavior in the form required by its parser generator or runtime.
## What is not a public component
The Rust lexer is an implementation detail of the Rust parser.
Decodal does not expose a standalone public tokenizer API for editor tooling.
Consumers that need syntax information should use the component that matches their environment:
- Rust execution and embedding: `decodal`
- Browser execution: `decodal-wasm`
- Web editor syntax: Lezer / CodeMirror
- General editor syntax: Tree-sitter
This avoids having a separate token stream API whose behavior would have to be kept compatible with both runtime parsing and editor grammars.
@@ -59,8 +59,9 @@ Config = MyConfig & {
## default の合成
初期方針では、`&` による異なる default 同士の合成は conflict とする。
同じ default は同一候補として扱ってよい
`&` で片方だけが default を持つ場合、その default を保持する。
同じ thunk 由来の default 同士は同一 default として保持する
異なる default 同士は conflict とする。
```text
merge_default(None, None) -> None
@@ -117,3 +118,9 @@ Abstract { constraints, default: None }
materialize は default を採用する唯一の段階である。
通常評価中に明示値が得られた場合、default は採用されない。
## Diagnostic context
Composition and materialization keep source spans for object fields, constraints, defaults, and thunks where possible.
When a conflict occurs, the diagnostic should identify the operation span and related participant spans, such as the left field, right field, constraint, or default value.
For object materialization errors, diagnostics also include the field path being processed when known.
@@ -10,10 +10,20 @@ Diagnostic {
kind: DiagnosticKind
span: Span
message: String
notes: Vec<Note>
labels: Vec<DiagnosticLabel>
}
DiagnosticLabel {
span: Span
message: String
}
```
`span` は primary location を示す。
表示時には `Span.source` を source id のまま出すのではなく、可能な限り file path や virtual file name に解決する。
`labels` は同じ error に関係する追加 location を示す。
合成や materialize の失敗では、衝突した constraint、value、default、または処理中 field path を label に含める。
代表的な diagnostic kind:
- syntax error
@@ -38,6 +48,18 @@ Result<RuntimeValue, Diagnostic>
これにより、制約違反、未定義識別子、循環依存、import 失敗などが通常値として流れることを避ける。
## 合成と materialize の diagnostic
合成や materialize の失敗は、以下を示す。
1. どの段階で失敗したか: composition、patch、materialization
2. どの field path を処理中だったか
3. どの constraint、value、default が衝突したか
4. なぜ合成または materialize できないか
例えば default が constraint を満たさない場合は、constraint の位置と default value の位置の両方を label として持つ。
object field の合成で concrete value が衝突する場合は、左辺 field と右辺 field の位置を label として持つ。
## `try / catch` は core に入れない
汎用 `try / catch` は core に入れない。
@@ -47,21 +69,6 @@ fallback は有限で明示的な仕組みに限定する。
- `default`: 未指定値の fallback。
- `match`: 有限 pattern に基づく分岐。
- optional import: ファイル不存在など、限定された失敗だけを fallback 可能にする候補。
- optional field access: field 不在だけを fallback 可能にする候補。
- union / tagged schema: 複数 schema の選択を明示的に表す将来候補。
## optional fallback の扱い
optional import や optional field access を導入する場合も、捕捉できる失敗は限定する。
例として optional import は、ファイル不存在だけを fallback 可能にし、parse error や import 先の制約違反は diagnostic として報告する方がよい。
```text
optional import:
file not found -> fallback
parse error -> diagnostic
eval error -> diagnostic
```
この方針により、fallback は通常の値選択として扱い、エラー内容に依存した実行時分岐は避ける。
Decodal は `unknown` / `any` を持たないため、field 不在や未解決 identifier を fallback 可能な通常値として扱わない。
それらは diagnostic として報告する。
+137
View File
@@ -0,0 +1,137 @@
# Embedding API
Decodal core can be embedded without giving the core crate access to a filesystem.
The host supplies imported sources through `SourceLoader` and may also provide global bindings through the host prelude API.
## Host prelude
`Engine` owns a prelude environment.
Bindings in this environment are visible from every module loaded by the engine.
```text
prelude env
module root env
let / function env
```
Module top-level bindings shadow prelude bindings.
Primitive type names such as `String`, `Int`, `Float`, `Bool`, and `Array` are handled before environment lookup, so they are reserved and cannot be shadowed by host bindings.
## Global bindings
The host can bind values before adding or evaluating user sources.
```rust
use decodal::{EmptyLoader, Engine, HostValue};
let mut engine = Engine::new(EmptyLoader);
engine.bind_global(
"Service",
HostValue::object([
("name", HostValue::string_type()),
("port", HostValue::int_type().gt(443).default_int(8443)?),
("enabled", HostValue::bool_type().default_bool(true)?),
]),
)?;
```
A user source can then refer to `Service` without importing it.
```dcdl
Service & {
name = "api";
port = 9443;
}
```
## HostValue
`HostValue` is the public builder-facing value representation for embedding.
It keeps host code from constructing internal `ThunkId` or `ObjectValue` values directly.
```text
HostValue =
String
Int
Float
Bool
Array(Vec<HostValue>)
Object(Vec<HostField>)
Abstract { constraints, default }
```
When a host value is bound, the engine internalizes it into `RuntimeValue` and allocates value thunks for object fields, array items, and defaults.
## Abstract host objects
A host-provided schema object is represented as a concrete object structure whose fields may contain abstract values.
```rust
HostValue::object([
("name", HostValue::string_type()),
("port", HostValue::int_type().gt(443).default_int(8443)?),
])
```
Conceptually this becomes:
```text
Concrete(Object {
name -> Thunk(Abstract { constraints: [String], default: none })
port -> Thunk(Abstract { constraints: [Int, > 443], default: 8443 })
})
```
This matches the runtime model used for Decodal source-defined schema objects.
## Typed Rust integration
Hosts can enable the `derive` feature on `decodal` to keep a Rust struct, the Decodal schema, and the decoded result in sync.
The derive implements two traits from the `decodal` crate:
- `DecodalSchema`: builds a `HostValue` schema that can be passed to `Engine::bind_global`.
- `DecodalDecode`: decodes materialized `Data` back into the Rust type.
```rust
use decodal::{Decodal, DecodalDecode, DecodalSchema, EmptyLoader, Engine};
#[derive(Decodal)]
struct Service {
name: String,
#[decodal(gt = 443, default = 8443)]
port: i64,
#[decodal(rename = "feature.enable", default = true)]
feature_enabled: bool,
}
let mut engine = Engine::new(EmptyLoader);
engine.bind_global("Service", Service::decodal_schema())?;
let value = engine.eval_module(module)?;
let data = engine.materialize(&value)?;
let service = Service::decodal_decode(&data)?;
```
Supported field attributes are intentionally small:
- `rename = "path.to.field"`
- `default`
- `default = value`
- numeric constraints: `gt`, `gte`, `lt`, `lte`
The derive does not add host callbacks or reflection.
It only generates schema construction and typed decoding code.
## SourceLoader and prelude together
`SourceLoader` and host prelude bindings are independent mechanisms.
- Use `SourceLoader` when user sources should explicitly import host-provided modules.
- Use prelude bindings when host-provided schemas or constants should be globally available.
Both mechanisms share the same runtime evaluator, thunk model, and materialization rules.
+10 -3
View File
@@ -54,23 +54,30 @@ desugar は、意味論を単純にするための表層構文変換を行う。
## module registry
module registry は、読み込んだ module を canonical path で管理する。
module registry は、読み込んだ module を loader が返す安定 key で管理する。
CLI では canonical path を key とする。
組み込み利用では、resource name や static source table の key を使える。
```text
ModuleRegistry:
CanonicalPath -> ModuleId
ModuleKey -> ModuleId
```
処理系は、まず root module を parse / desugar して registry に登録する。
import 先 module は、この段階で全て読み込む必要はない。
import expression が評価されたとき、module registry は path を解決し、未登録なら対象 module を parse / desugar して登録する
import expression が評価されたとき、処理系は `SourceLoader` に現在の module key と import specifier を渡す
loader は module key、表示名、source text を返す。
module registry は key が未登録なら対象 module を parse / desugar して登録する。
登録された module は module root thunk を持つ。
同じ module が複数回 import された場合は、同じ `ModuleId` を返す。
つまり import は module を即時評価しない。
module を読み込み、module root を thunk として登録するだけにする。
AST の `ExprId` は module-local である。
そのため runtime が保持する式参照は `ExprRef { module, expr }` として module-qualified にする。
## demand-driven evaluation
評価器は、必要になった thunk だけを force する。
+67
View File
@@ -0,0 +1,67 @@
# Features
Decodal は embedded use と小さい runtime を優先する。
言語機能を追加するときは、値の合成・検証・materialization に直接必要なものを core に残し、重い依存や高度な推論は optional feature または外部 tooling に分ける。
## Core feature boundary
Core に入れる機能は、基本的に deterministic な value transformation に限る。
- arithmetic / logical / comparison operators
- array concat
- object / constraint composition
- default materialization
- pure function evaluation
- host supplied import evaluation
Core に入れないものは以下である。
- filesystem / network / environment access
- time / random
- mutation
- reflection or existence probing
- arbitrary host function calls
- symbolic constraint solving beyond simple normalization
未解決 identifier や missing field は `unknown` として流れず、diagnostic になる。
この方針により、存在チェックや optional chaining のような dynamic object inspection は core language の対象外とする。
## Constraint reasoning
Constraint normalization は軽量な範囲に留める。
primitive type conflict や明らかな numeric bound conflict は合成時に検出してよい。
一方で、symbolic arithmetic、boolean algebra、regex intersection、array length dependent typing のような重い推論は行わない。
評価済みの concrete value に対する検証は runtime / materialization で行う。
静的に完全な型検査フェーズを増やすのではなく、parse、evaluate、compose、materialize の各段階で自然に分かる error を diagnostic として返す。
## Core defaults
`decodal` defaults to `std` only.
```toml
[features]
default = ["std"]
std = []
regex = ["std", "dep:regex"]
```
Building `decodal` with `--no-default-features` keeps the core in `no_std + alloc` mode and avoids optional dependencies.
## Regex
Regex constraints are implemented behind the `regex` feature.
When the feature is disabled, regex constraints parse and compose, but validating a concrete value against them returns an unsupported feature diagnostic.
```sh
cargo run -q -p decodal-cli --features regex -- examples/regex/main.dcdl
```
Regex constraints are accumulated during `&` composition.
The implementation does not try to prove whether the intersection of two regex constraints is empty.
Concrete strings must match every regex constraint attached to the abstract value.
## CLI features
`decodal-cli` exposes a matching `regex` feature that enables `decodal/regex`.
The feature is not enabled by default so the default CLI binary remains small.
+2
View File
@@ -15,3 +15,5 @@ bytecode VM や JIT ではなく、AST を demand-driven に評価すること
3. [Thunk and Lazy Evaluation](./thunk-and-lazy-evaluation.md)
4. [Composition and Materialization](./composition-and-materialization.md)
5. [Diagnostics and Fallback](./diagnostics-and-fallback.md)
6. [Embedding API](./embedding-api.md)
7. [Features](./features.md)
+30 -2
View File
@@ -14,6 +14,10 @@ RuntimeValue =
`Concrete` は明示的な値である。
`Abstract` は、まだ具体値に確定していない制約付きの値である。
Decodal は `unknown``any``null` のような「存在するが意味が未確定な値」を runtime value として持たない。
識別子や field が解決できない場合は、その場で diagnostic になる。
未解決値を後続の演算へ流して推論することはしない。
## ConcreteValue
```text
@@ -32,7 +36,11 @@ object は concrete structure として扱う。
```text
ObjectValue:
fields: Map<Symbol, ThunkId>
fields: Map<Symbol, ObjectField>
ObjectField:
value: ThunkId
span: Span
```
例えば以下の schema object は、object 自体は concrete だが、field の値は abstract value になる。
@@ -57,9 +65,14 @@ Concrete(Object {
```text
AbstractValue {
constraints: Vec<Constraint>
constraints: Vec<ConstraintEntry>
default: Option<ThunkId>
}
ConstraintEntry {
constraint: Constraint
span: Span
}
```
`default``AbstractValue` にだけ存在する。
@@ -88,6 +101,21 @@ port = 8000;
Concrete(Int(8000))
```
## Runtime scope
Decodal runtime は application runtime ではなく、pure value evaluator である。
同じ source、同じ import sources、同じ host globals が与えられた場合、評価結果は決定的である。
runtime が扱う責務は以下に限る。
- expression を評価する。
- thunk を必要に応じて force する。
- concrete / abstract value を合成する。
- materialize 時に constraint を検証する。
runtime は filesystem、network、environment variable、time、random、mutation を扱わない。
core における import は host supplied source を受け取る境界であり、filesystem access ではない。
## Constraint
constraint は concrete value とは別の型として扱う。
@@ -7,11 +7,16 @@ thunk は、まだ評価していない式をあとで評価できるように
```text
Thunk {
expr: ExprId
expr: ExprRef
env: EnvId
state: ThunkState
}
ExprRef {
module: ModuleId
expr: ExprId
}
ThunkState =
Unevaluated
Evaluating
@@ -19,7 +24,8 @@ ThunkState =
Error(Diagnostic)
```
`expr` は評価対象の AST node を指す。
`expr` は評価対象の AST node を module-qualified に指す。
`ExprId` は module-local な ID なので、runtime では `ModuleId` と組み合わせた `ExprRef` を保持する。
`env` は、その式を評価するときに使う lexical environment を指す。
式だけではなく environment も保持するのは、遅延評価された式が定義時の名前解決文脈を必要とするためである。
@@ -78,5 +84,5 @@ module root や field は thunk として保持され、参照されたときだ
関数引数は thunk として関数の environment に束縛する。
関数本体で引数が参照されたときだけ force する。
任意の関数呼び出し結果グローバルに memoize する必要はない。
関数呼び出し結果そのものはグローバルに memoize ない。
field に束縛された関数呼び出し結果は、その field thunk の評価結果として memoize される。
+244
View File
@@ -0,0 +1,244 @@
# Development
This document describes the development workflow for Decodal itself.
## Rust checks
Run the normal Rust checks from the repository root.
```sh
cargo fmt --check
cargo test
cargo check -p decodal --no-default-features
nix flake check
```
Regex support is optional and should be tested explicitly when touched.
```sh
cargo test -p decodal --features regex
cargo run -q -p decodal-cli --features regex -- examples/regex/main.dcdl
```
## crates.io release
The primary crates.io package is `decodal`, which contains the embeddable library.
`decodal-derive` provides optional derive macros for Rust struct integration and is published only when the derive crate changes.
Workspace support crates such as `decodal-cli` and the Rust source crate `decodal-wasm` are not published to crates.io.
The generated WebAssembly package under `site/decodal-site/src/wasm/` is published to npm and JSR.
The project is dual licensed as `MIT OR Apache-2.0`.
```sh
cargo publish -p decodal
```
Publish `decodal-derive` first only when that crate has a new version:
```sh
cargo publish -p decodal-derive
```
Before publishing, run:
```sh
cargo fmt --check
cargo test
cargo check -p decodal --no-default-features
cargo publish -p decodal --dry-run
```
If `decodal-derive` changed, also run `cargo publish -p decodal-derive --dry-run`.
## Web site and playground
The Astro documentation site and browser playground are kept in:
```text
site/decodal-site/
```
The site imports Markdown files from `doc/manual/souce/` and renders them as mdBook-style pages.
The playground loads `decodal-wasm` and evaluates DCDL entirely in the browser.
Important files:
```text
site/decodal-site/src/pages/docs/[...slug].astro
site/decodal-site/src/pages/playground.astro
site/decodal-site/src/scripts/playground.js
site/decodal-site/src/scripts/playground-examples.js
site/decodal-site/src/layouts/ManualLayout.astro
site/decodal-site/src/lib/docs.js
site/decodal-site/src/lib/highlight.js
packages/decodal-codemirror/src/decodal.js
packages/decodal-codemirror/src/decodal-parser.js
crates/decodal-wasm/src/lib.rs
```
Build the WebAssembly package before building the site:
```sh
cd site/decodal-site
npm install
npm run build:wasm
npm run build
```
`npm run build:wasm` writes generated files into:
```text
site/decodal-site/src/wasm/
```
These generated files are committed so the site can be built without requiring every consumer to regenerate the wasm package first.
Publish the generated WebAssembly package from its package directory:
```sh
cd site/decodal-site/src/wasm
npm publish
npx jsr publish
```
Run dry-runs first when preparing a release:
```sh
npm pack --dry-run
npx jsr publish --dry-run
```
The npm package name is `decodal-wasm`.
The JSR package name is `@hare/decodal-wasm`.
JSR currently expects a single SPDX license identifier in `jsr.json`; the WebAssembly package metadata uses `MIT` while the Rust workspace remains dual licensed as `MIT OR Apache-2.0`.
The playground editor uses the `decodal-codemirror` package with the generated Lezer parser in `packages/decodal-codemirror/src/decodal-parser.js`.
The canonical grammar is documented in `doc/manual/souce/language/grammar.md`; regenerate the Lezer parser when that grammar or `editors/lezer-decodal/decodal.grammar` changes.
Publish the CodeMirror package from its package directory:
```sh
cd packages/decodal-codemirror
npm install
npm publish
npx jsr publish
```
Run dry-runs first when preparing a release:
```sh
npm install
npm pack --dry-run
npx jsr publish --dry-run
```
The npm package name is `decodal-codemirror`.
The JSR package name is `@hare/decodal-codemirror`.
JSR currently expects a single SPDX license identifier in `jsr.json`; the CodeMirror package metadata uses `MIT` while the Rust workspace remains dual licensed as `MIT OR Apache-2.0`.
The documentation build still uses the lightweight JavaScript fallback highlighter so Astro can render Markdown without initializing WASM at build time.
To run the site locally:
```sh
cd site/decodal-site
npm run dev
```
Deploy the static site to Cloudflare Pages with Wrangler direct upload:
```sh
cd site/decodal-site
npm run deploy
```
The deploy script runs `npm run build` and then uploads `dist/` to the Pages project named `decodal-site` on branch `master`.
Cloudflare Pages treats this as production when the project production branch is `master`.
Use `CLOUDFLARE_PROJECT_NAME` when deploying to a differently named Pages project.
```sh
CLOUDFLARE_PROJECT_NAME=my-pages-project npm run deploy
```
If the committed WASM package must be regenerated before deploy, use:
```sh
npm run deploy:wasm
```
## Tree-sitter grammar
The Tree-sitter grammar is kept in:
```text
editors/tree-sitter-decodal/
```
Important files:
```text
editors/tree-sitter-decodal/grammar.js
editors/tree-sitter-decodal/queries/highlights.scm
editors/tree-sitter-decodal/queries/locals.scm
editors/tree-sitter-decodal/corpus/basic.txt
```
The grammar is intended to be portable across editors such as Zed, Neovim, Helix, and Emacs.
Zed support should consume this grammar rather than relying on a TextMate grammar.
## Tree-sitter commands
From the grammar directory:
```sh
cd editors/tree-sitter-decodal
npm install
npx tree-sitter generate
npx tree-sitter test
```
To inspect a parse tree:
```sh
npx tree-sitter parse ../../examples/advanced/main.dcdl
```
The generated parser files under `editors/tree-sitter-decodal/src/` are committed so editor integrations can consume the grammar without regenerating it first.
`node_modules/` is ignored and must not be committed.
## Updating the grammar
When the Decodal syntax changes:
1. Update the canonical EBNF in `doc/manual/souce/language/grammar.md`.
2. Update the Rust parser/lexer as needed.
3. Update `editors/tree-sitter-decodal/grammar.js` and run `npx tree-sitter generate` / `npx tree-sitter test`.
4. Update `editors/lezer-decodal/decodal.grammar` and regenerate the CodeMirror parser:
```sh
cd site/decodal-site
npx lezer-generator ../../editors/lezer-decodal/decodal.grammar -o ../../packages/decodal-codemirror/src/decodal-parser.js
```
5. Add or update corpus/tests/examples.
6. Run Rust and site checks from the repository root.
## Development shell
The Nix development shell includes Rust tooling, Node.js, Tree-sitter CLI tooling, and wasm-pack tooling.
```sh
nix develop
```
The shell provides:
- `cargo`
- `rustc`
- `rustfmt`
- `clippy`
- `node`
- `npm`
- `wasm-pack`
- `lld`
- `tree-sitter`
- `nixfmt`
+7 -3
View File
@@ -6,7 +6,8 @@ Decodal は Deferred Constraint Data Language、略称 DCDL のプロジェク
## 目次
1. [Introduction](./introduction.md)
2. [Language Specification](./language/index.md)
2. [Components](./components.md)
3. [Language Specification](./language/index.md)
1. [Lexical Structure and Syntax](./language/syntax.md)
2. [Value](./language/value/index.md)
1. [String](./language/value/string.md)
@@ -35,10 +36,13 @@ Decodal は Deferred Constraint Data Language、略称 DCDL のプロジェク
9. [Materialization and Errors](./language/materialization-and-errors.md)
10. [Naming Conventions](./language/naming.md)
11. [Examples](./language/examples.md)
3. [Implementation Design](./design/index.md)
4. [Implementation Design](./design/index.md)
1. [Execution Pipeline](./design/execution-pipeline.md)
2. [Runtime Model](./design/runtime-model.md)
3. [Thunk and Lazy Evaluation](./design/thunk-and-lazy-evaluation.md)
4. [Composition and Materialization](./design/composition-and-materialization.md)
5. [Diagnostics and Fallback](./design/diagnostics-and-fallback.md)
4. [Open Issues](./open-issues.md)
6. [Embedding API](./design/embedding-api.md)
7. [Features](./design/features.md)
5. [Development](./development.md)
6. [Open Issues](./open-issues.md)
@@ -31,9 +31,52 @@ A & B = A と B の両方を満たす値または制約
```dcdl
Int & String # エラー
>= 10 & <= 5 # エラーになりうる
> 10 & < 5 # エラー
Int & > 10 & < 11 # エラー。整数値の候補が存在しない
```
## 制約の正規化
`&` によって abstract value 同士を合成した場合、処理系は軽量に判定できる制約を正規化する。
正規化対象:
- primitive type 制約。
- 数値比較制約。
primitive type 制約は、異なる型が同時に要求された場合 conflict になる。
```dcdl
Int & Float
Int & String
```
数値比較制約は上下限として正規化される。
```dcdl
Int & >= 1 & <= 65535 & > 443
```
これは概念的に以下へ正規化される。
```text
Type(Int)
> 443
<= 65535
```
上下限の交差が空であれば conflict になる。
`Int` 制約がある場合は、整数候補が存在するかも判定する。
```dcdl
> 10 & < 5 # conflict
Int & > 10 & < 11 # conflict
Int & >= 10 & <= 10 # OK
```
`Int` の比較制約は整数リテラルを使う。
`Float` の比較制約は整数リテラルまたは浮動小数リテラルを使える。
## 組み込み制約
最小の組み込み制約は以下である。
@@ -53,13 +96,28 @@ IPv4Address
## 正規表現制約
正規表現リテラルは文字列制約として使える候補である
正規表現リテラルは文字列制約として使える。
```dcdl
Host = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/;
```
ただし、組み込み向け実装では正規表現エンジンを optional feature にできる。
正規表現制約は積み重ね可能である。
複数の正規表現制約が同じ abstract value に付与された場合、具体文字列はすべての正規表現制約に一致しなければならない。
```dcdl
String & /^a/ & /z$/
```
処理系は、正規表現制約同士の交差が空であるかを合成時に判定する必要はない。
つまり、以下は合成時には conflict にならず、具体値検証時に失敗する。
```dcdl
String & /^a$/ & /^b$/
```
正規表現エンジンは optional feature にできる。
正規表現 feature が無効な処理系では、正規表現制約の検証は unsupported feature diagnostic になる。
軽量実装では代表的な制約を組み込み述語として提供してもよい。
```dcdl
@@ -130,11 +188,12 @@ RuntimeValue =
## default の合成
同じフィールドに複数の `default` 合成された場合の詳細規則は未確定である。
現時点の単純な方針は以下である。
`default` 合成規則は以下である。
- `&` による default 同士の衝突はエラーにする。
- 同一 default 値は許可してよい
- `//` による patch では右辺 default が左辺 default を置き換える。
- `&` で片方だけが default を持つ場合、その default を保持する。
- `&` で両方が異なる default を持つ場合、conflict になる
- `Abstract & Concrete` が成功した場合、結果は concrete value になり default は消える。
- `//` では右辺が左辺を置き換える。object/object の場合は field ごとに再帰 patch されるため、右辺 field の default が左辺 field の default を置き換える。
この方針により、`&` は制約を保った合成、`//` は上書き操作として説明できる。
`default` thunk は materialize 時に必要になった時点で評価する。
評価された default value は、同じ abstract value に残っている制約を満たす必要がある。
+1 -1
View File
@@ -74,5 +74,5 @@ Error 評価失敗
関数引数は thunk として渡せる。
関数本体内で引数が参照されたときに評価する。
任意の関数呼び出し結果グローバルに memoize することは必須ではない。
関数呼び出し結果そのものはグローバルに memoize ない。
ただし、フィールドに束縛された呼び出し結果は、そのフィールド thunk の評価結果として memoize される。
+15 -25
View File
@@ -5,21 +5,19 @@
## 基本的な設定スキーマ
```dcdl
rec {
Host = IPv4Address;
Host = IPv4Address;
Port = Int & >= 1 & <= 65535;
NarrowedPort = Port & > 443;
Port = Int & >= 1 & <= 65535;
NarrowedPort = Port & > 443;
MyConfig = {
host = Host;
port = NarrowedPort default 8080;
feature_hoge = {
enable = Bool default true;
fuga = Int default 10;
};
MyConfig = {
host = Host;
port = NarrowedPort default 8080;
feature_hoge = {
enable = Bool default true;
fuga = Int default 10;
};
}
};
NewConfig = MyConfig & {
host = "127.0.0.1";
@@ -33,28 +31,20 @@ disabled_config = NewConfig & {
enabled_config = NewConfig;
```
## 関数と文字列生成
## 関数と制約
```dcdl
let
maybe_hw = String & /Hello! .*/;
part = {
greet = String;
target = String;
};
mk_hw = (part: part) =>
maybe_hw & "${part.greet}! ${part.target}";
Port = Int & >= 1 & <= 65535;
add_offset = (base: Port, offset: Int) => base + offset;
in
mk_hw({
greet = "Hello";
target = "World";
})
add_offset(8000, 80)
```
評価結果:
```text
"Hello! World"
8080
```
## match
@@ -0,0 +1,43 @@
# Arithmetic Expression
Decodal supports arithmetic over concrete numeric values.
```dcdl
{
workers = 2 + 2;
timeout = 30.0 / 2;
port = 8000 + 80;
negative = -1;
}
```
## Operators
- `+` addition
- `-` subtraction
- `*` multiplication
- `/` division
- unary `-` negation
`*` and `/` bind tighter than `+` and `-`.
Parentheses can be used to make grouping explicit.
```dcdl
2 + 3 * 4 # 14
(2 + 3) * 4 # 20
```
## Numeric behavior
Arithmetic requires concrete `Int` or `Float` operands.
`Int + Int`, `Int - Int`, and `Int * Int` produce `Int` when no overflow occurs.
Mixed `Int` / `Float` arithmetic produces `Float`.
Division always produces `Float`.
Division by zero and integer overflow are evaluation errors.
Arithmetic expressions can be used anywhere a concrete numeric expression is expected, including defaults and numeric constraints.
```dcdl
port = Int & > 4000 + 42 default 8080;
```
+17 -5
View File
@@ -7,9 +7,21 @@ array expression は、順序付きの値の列を表す。
["a", "b", "c"]
```
## 未確定事項
## Array concat
- 配列要素の制約表現
- 異種配列を許可するか。
- `//` による patch を右辺置換だけにするか。
- append / prepend / remove などの操作を提供するか。
`++` は concrete array 同士を連結する
```dcdl
base = ["read", "write"];
extra = ["admin"];
roles = base ++ extra;
```
`roles` は以下と同じ値になる。
```dcdl
["read", "write", "admin"]
```
`++` は配列要素を変換しない。
左辺の要素の後に右辺の要素が並ぶ。
@@ -3,10 +3,7 @@
function call expression は、関数値を引数に適用する式である。
```dcdl
mk_hw({
greet = "Hello";
target = "World";
})
increment(41)
```
## 評価
@@ -14,4 +11,5 @@ mk_hw({
引数は thunk として渡せる。
関数本体内で引数が参照されたときに評価する。
任意の関数呼び出し結果グローバルに memoize することは必須ではない。
関数呼び出し結果そのものはグローバルに memoize ない。
フィールドに束縛された呼び出し結果は、そのフィールド thunk の評価結果として memoize される。
@@ -3,11 +3,7 @@
function expression は、引数を受け取り式を返す値である。
```dcdl
(part: {
greet = String;
target = String;
}) =>
"${part.greet}! ${part.target}"
(value: Int) => value + 1
```
関数仕様の詳細は [関数](../functions.md) に置く。
+3 -5
View File
@@ -4,12 +4,10 @@ let expression は、ローカル束縛を作る。
```dcdl
let
part = {
greet = "Hello";
target = "World";
};
base = 8000;
offset = 80;
in
"${part.greet}! ${part.target}"
base + offset
```
## 評価
@@ -0,0 +1,41 @@
# Logical and Comparison Expressions
Decodal supports boolean logic over concrete `Bool` values and comparison over concrete scalar values.
```dcdl
{
is_prod = env == "prod";
high_port = port > 9000;
enabled = is_prod && high_port;
disabled = !enabled;
}
```
## Logical operators
- `!expr` negates a concrete `Bool`.
- `lhs && rhs` returns boolean AND.
- `lhs || rhs` returns boolean OR.
`&&` and `||` short-circuit: the right-hand side is evaluated only when needed.
Logical operands must evaluate to concrete `Bool` values.
## Comparison operators
- `==`
- `!=`
- `<`
- `<=`
- `>`
- `>=`
`==` and `!=` compare concrete scalar values: `String`, `Bool`, `Int`, and `Float`.
`Int` and `Float` can be compared to each other numerically.
Ordering operators `<`, `<=`, `>`, and `>=` compare concrete numeric values only.
They are separate from prefix comparison constraints such as `> 443`.
```dcdl
port = Int & > 443 default 9443;
is_high = port > 9000;
```
@@ -12,4 +12,5 @@ config.feature_hoge.enable
左側の式を object として評価し、指定された field を参照する。
参照先 field は必要になるまで評価されない。
存在しない field への参照をエラーにするか、open schema として扱うかは未確定である。
存在しない field への参照は diagnostic になる。
Decodal は unknown / Any のような値を伝播せず、field の有無を曖昧にしない。
@@ -1,12 +1,8 @@
# String Interpolation Expression
# String Interpolation
string interpolation は、文字列内に式を埋め込む候補機能である
文字列補間は初期実装には含めない
```dcdl
"${part.greet}! ${part.target}"
```
Decodal の string literal は、現時点では literal text として扱う。
式を埋め込む構文は定義しない。
## 評価
補間式の評価タイミングは通常の遅延評価に従う。
文字列補間を初期実装に含めるかは未確定である。
必要になった場合は、文字列連結や明示的な formatting function として別途設計する。
+19 -52
View File
@@ -5,54 +5,47 @@
## 構文
```dcdl
(part: {
greet = String;
target = String;
}) =>
"${part.greet}! ${part.target}"
(value: Int) => value + 1
```
関数呼び出しは通常の呼び出し構文で行う。
```dcdl
mk_hw({
greet = "Hello";
target = "World";
})
let
increment = (value: Int) => value + 1;
in
increment(41)
```
複数引数の構文は候補として以下を想定する。
複数引数も指定できる。
```dcdl
(
input_a: {
hoge = Int & >= 0;
},
input_b: {
fuga = 20;
}
) =>
(input_a: { hoge = Int & >= 0; }, input_b: { fuga = Int; }) =>
{
# ...
hoge = input_a.hoge;
fuga = input_b.fuga;
}
```
## 関数の意味
関数は値として扱える
ただし、最終データとして関数値を出力できるかどうかは別途定める。
設定ファイルを materialize する段階では、未適用の関数値は出力不能な値として扱うのが自然である。
関数は runtime value として扱えるが、最終データとして materialize することはできない
未適用の関数値が materialize 対象に残っている場合は diagnostic になる。
関数値は opaque であり、関数値同士の等価性は提供しない。
`&` で関数値同士を合成すると conflict になる。
## 評価方針
関数は以下の方針を基本とする。
関数は以下の方針で評価する。
- 関数は純粋である。
- 関数はレキシカルスコープを持つ。
- 関数は定義時の環境を参照として保持する。
- 引数は必要になるまで評価ない。
- フィールドに束縛された関数呼び出し結果は、そのフィールド評価結果として memoize される
- 任意の関数呼び出しそのものをグローバルに memoize することは必須ではない
- 引数は thunk として渡され、必要になるまで評価されない。
- 関数呼び出し結果そのものはグローバルには memoize しない
- フィールドに束縛された関数呼び出し結果は、そのフィールド thunk の評価結果として memoize される
- 再帰的な依存は thunk cycle として diagnostic になる。
関数値の内部モデル例:
@@ -63,29 +56,3 @@ Function {
env: EnvRef
}
```
## 関数と重さ
関数の文法・パーサー自体は大きくない。
実装上の重さは、主に評価モデルと意味論から発生する。
注意点:
- クロージャが環境を掴む。
- 引数を lazy にするか strict にするかを決める必要がある。
- 関数呼び出し結果をどこまで memoize するかを決める必要がある。
- 再帰関数を許可するかを決める必要がある。
- 関数値同士の `&` をどう扱うかを決める必要がある。
- 関数値を object に入れたとき、materialize 可能かを決める必要がある。
- import 循環と関数適用が絡んだときの cycle detection が必要になる。
軽量に保つため、初期仕様では以下の制限を検討できる。
- 関数は pure。
- lexical closure は許可。
- 引数は thunk として渡す。
- 関数本体は必要になるまで評価しない。
- 関数値は opaque。
- 関数同士の `&` は、同一関数参照以外は conflict。
- 関数値は最終データとして出力できない。
- 再帰は cycle error として扱う、または v1 では禁止する。
+111
View File
@@ -0,0 +1,111 @@
# Grammar
This page is the canonical grammar reference for Decodal syntax.
Parser implementations such as the Rust parser, Tree-sitter grammar, and Lezer grammar should follow this grammar and may add implementation-specific precedence annotations where needed.
## Lexical grammar
```ebnf
source_character = ? any Unicode scalar value ? ;
newline = "\n" | "\r\n" | "\r" ;
space = " " | "\t" | newline ;
comment = "#" , { ? any character except newline ? } ;
digit = "0" "9" ;
letter = "A" "Z" | "a" "z" ;
identifier = letter , { letter | digit | "_" } ;
integer = digit , { digit } ;
float = digit , { digit } , "." , digit , { digit } ;
string = '"' , { string_character | escape } , '"' ;
string_character = ? any character except '"', "\\", or newline ? ;
escape = "\\" , source_character ;
regex = "/" , regex_character , { regex_character } , "/" ;
regex_character = escape | ? any character except "/", "\\", or newline ? ;
```
Whitespace and comments separate tokens and are otherwise ignored by the parser.
## Syntactic grammar
```ebnf
module = { statement } ;
statement = field_definition , [ ";" ]
| expression , [ ";" ] ;
expression = default_expression ;
default_expression = patch_expression , [ "default" , default_expression ] ;
patch_expression = compose_expression , { "//" , compose_expression } ;
compose_expression = logical_or_expression , { "&" , logical_or_expression } ;
logical_or_expression = logical_and_expression , { "||" , logical_and_expression } ;
logical_and_expression = comparison_expression , { "&&" , comparison_expression } ;
comparison_expression = concat_expression , [ comparison_operator , concat_expression ] ;
concat_expression = additive_expression , { "++" , additive_expression } ;
additive_expression = multiplicative_expression , { ( "+" | "-" ) , multiplicative_expression } ;
multiplicative_expression = unary_expression , { ( "*" | "/" ) , unary_expression } ;
unary_expression = [ "!" | "-" ] , postfix_expression ;
postfix_expression = primary_expression , { call_suffix | path_suffix } ;
call_suffix = "(" , [ argument_list ] , ")" ;
path_suffix = "." , identifier ;
primary_expression = literal
| identifier
| comparison_constraint
| object
| array
| let_expression
| function_expression
| match_expression
| import_expression
| "(" , expression , ")" ;
literal = string | integer | float | "true" | "false" | regex ;
comparison_operator = "==" | "!=" | "<" | "<=" | ">" | ">=" ;
comparison_constraint = ( "<" | "<=" | ">" | ">=" ) , expression ;
object = "{" , [ field_definition , { ";" , field_definition } , [ ";" ] ] , "}" ;
field_definition = field_path , "=" , expression ;
field_path = identifier , { "." , identifier } ;
array = "[" , [ expression , { "," , expression } , [ "," ] ] , "]" ;
let_expression = "let" , { field_definition , ";" } , "in" , expression ;
function_expression = "(" , [ parameter_list ] , ")" , "=>" , expression ;
parameter_list = parameter , { "," , parameter } , [ "," ] ;
parameter = identifier , [ ":" , expression ] ;
match_expression = "match" , expression , "{" , [ match_arm , { ";" , match_arm } , [ ";" ] ] , "}" ;
match_arm = pattern , ":" , expression ;
pattern = "_" | expression ;
import_expression = "import" , string ;
argument_list = expression , { "," , expression } , [ "," ] ;
```
## Precedence
Precedence is highest first.
1. function call and field path reference
2. unary `!` and `-`
3. `*` and `/`
4. `+` and `-`
5. `++`
6. `==`, `!=`, `<`, `<=`, `>`, `>=`
7. `&&`
8. `||`
9. `&`
10. `//`
11. `default`
Binary operators are left-associative except `default`, which is right-associative.
## Tooling mapping
Syntax tooling should derive token categories from this grammar rather than making a tool-specific grammar canonical.
Tree-sitter and Lezer grammars are implementation artifacts that follow this page.
@@ -95,4 +95,5 @@ match value {
通常の式はエラー内容に基づいて分岐できない。
汎用 `try / catch` は core には含めない。
fallback は `default``match`、および将来的な optional import / optional field access のような限定された仕組みで表現する。
fallback は `default``match` で表現する。
optional import や optional field access は core には含めない。
@@ -5,12 +5,11 @@
## 構文
```dcdl
import ./config.n
import "./config.n"
import "./config.dcdl"
```
パス表記の詳細は未確定である。
パスリテラルと文字列リテラルの両方を許可するか、どちらかに統一するかは今後決める
import specifier は文字列リテラルとする。
パスリテラル構文は採用しない
## モジュール
@@ -18,6 +17,30 @@ import 先はモジュール単位で読み込まれる。
ただし、モジュール全体を即時評価する必要はない。
各フィールドは thunk として保持され、必要になったときだけ評価される。
top-level に field 定義列を書いた module は、recursive module scope を作る。
つまり、top-level field は同じ module の他の top-level field から識別子として参照できる。
```dcdl
schema = {
hoge = String;
};
result = schema;
```
この場合、`result` の右辺の `schema` は同じ module の top-level field `schema` を参照する。
通常の object literal 内の field は、その object 内の sibling field を暗黙には識別子として参照できない。
object 内の値を参照する場合は、外側で束縛された値や明示的な path reference を使う。
## SourceLoader
`import` specifier の解決は処理系 core ではなく host 側の `SourceLoader` が行う。
CLI では、specifier を現在の module path からの相対 path として解決する。
組み込み利用では、resource table や static source map など、filesystem 以外の loader を使える。
module cache の key は loader が返す安定 key を使う。
CLI では canonical path を key とする。
## 循環 import
モジュール間に循環参照があっても、必要なフィールドの依存関係が循環していなければ評価できる。
@@ -25,25 +48,25 @@ import 先はモジュール単位で読み込まれる。
例:
```dcdl
# main.n
# main.dcdl
{
schema = {
hoge = String;
};
result = (import ./func.n)(schema);
result = (import "./func.dcdl")(schema);
}
```
```dcdl
# func.n
(input: (import ./main.n).schema) =>
# func.dcdl
(input: (import "./main.dcdl").schema) =>
{
# ...
}
```
`func.n``main.n` を import しているが、参照しているのは `main.schema` である。
`func.dcdl``main.dcdl` を import しているが、参照しているのは `main.schema` である。
`main.schema``main.result` に依存していなければ、この循環 import は成立する。
## import の評価単位
+78 -14
View File
@@ -1,6 +1,79 @@
# 合成演算子
# 演算子
この章では、`&``//` の意味を定義する。
この章では、Decodal の演算子の意味を定義する。
## 演算子一覧
| 演算子 | 形 | 種類 | 対象 | 結果 / 意味 |
|---|---|---|---|---|
| `.` | `object.field` | field reference | object / abstract object | field value |
| call | `fn(arg)` | function call | function | function result |
| `!` | `!expr` | unary logical | concrete `Bool` | concrete `Bool` |
| `-` | `-expr` | unary arithmetic | concrete `Int` / `Float` | negated number |
| `*` | `lhs * rhs` | arithmetic | concrete `Int` / `Float` | numeric product |
| `/` | `lhs / rhs` | arithmetic | concrete `Int` / `Float` | `Float` quotient |
| `+` | `lhs + rhs` | arithmetic | concrete `Int` / `Float` | numeric sum |
| `-` | `lhs - rhs` | arithmetic | concrete `Int` / `Float` | numeric difference |
| `++` | `lhs ++ rhs` | array concat | concrete arrays | concatenated array |
| `==` | `lhs == rhs` | equality | concrete scalar | concrete `Bool` |
| `!=` | `lhs != rhs` | equality | concrete scalar | concrete `Bool` |
| `<` | `lhs < rhs` | ordering | concrete `Int` / `Float` | concrete `Bool` |
| `<=` | `lhs <= rhs` | ordering | concrete `Int` / `Float` | concrete `Bool` |
| `>` | `lhs > rhs` | ordering | concrete `Int` / `Float` | concrete `Bool` |
| `>=` | `lhs >= rhs` | ordering | concrete `Int` / `Float` | concrete `Bool` |
| `>` | `> value` | comparison constraint | numeric constraint value | abstract constraint |
| `>=` | `>= value` | comparison constraint | numeric constraint value | abstract constraint |
| `<` | `< value` | comparison constraint | numeric constraint value | abstract constraint |
| `<=` | `<= value` | comparison constraint | numeric constraint value | abstract constraint |
| `&&` | `lhs && rhs` | logical | concrete `Bool` | short-circuit AND |
| `||` | `lhs || rhs` | logical | concrete `Bool` | short-circuit OR |
| `&` | `lhs & rhs` | composition | value / constraint / object | constraint-preserving composition |
| `//` | `lhs // rhs` | patch | object / value | right-biased structural patch |
| `default` | `base default fallback` | default | abstract value | materialization fallback |
`concrete scalar``String``Bool``Int``Float` を指す。
## 優先順位
優先順位は高い順に以下である。
1. 関数呼び出しとフィールド参照
2. unary `!` `-`
3. `*` `/`
4. `+` `-`
5. `++`
6. `==` `!=` `<` `<=` `>` `>=`
7. `&&`
8. `||`
9. `&`
10. `//`
11. `default`
同じ優先順位の二項演算子は左結合である。
`default` は右結合である。
## Arithmetic operators
`+` `-` `*` `/` は具体的な `Int` / `Float` に対する四則演算である。
詳しくは [Arithmetic Expression](./expression/arithmetic.md) を参照する。
## Array concat operator
`++` は concrete array 同士を連結する演算子である。
要素は変換されず、左辺の要素の後に右辺の要素が並ぶ。
```dcdl
["read", "write"] ++ ["admin"]
```
## Logical and comparison operators
`!` `&&` `||` は concrete `Bool` に対する論理演算である。
`&&``||` は短絡評価される。
`==` `!=` は concrete scalar value を比較する。
`<` `<=` `>` `>=` は concrete numeric value を比較する。
詳しくは [Logical and Comparison Expressions](./expression/logical-and-comparison.md) を参照する。
## `&`: 制約合成
@@ -110,19 +183,10 @@ Patched = Base // {
## object 全体の置換
`//` が deep patch である場合、object 全体を置き換えたいときの escape hatch が必要になる。
`//` では object/object は常に deep patch される。
object field 全体を特別に置き換えるための `replace(...)` 構文や組み込み関数は core には含めない。
候補として、`replace(...)` を組み込み関数として提供する
```dcdl
Replaced = Base // {
feature_hoge = replace({
enable = false;
});
};
```
この場合、`feature_hoge` は再帰 patch されず、右辺の object に丸ごと置き換わる。
object 全体を別構造にしたい場合は、patch 対象より外側で値を作り直す
## `&` と `//` の使い分け
+33 -14
View File
@@ -1,7 +1,7 @@
# 構文と字句
この章では、表層構文の方針をまとめる。
厳密な EBNF は未確定であり、今後このファイルに詳細化する。
厳密な構文は [Grammar](./grammar.md) の EBNF を参照する。
## 言語名と拡張子
@@ -15,6 +15,25 @@ schema.dcdl
service.dcdl
```
## Module source
ファイル全体は単一の式として書ける。
また、top-level に field 定義列を書いた場合は、暗黙の object として扱う。
```dcdl
host = String;
port = Int default 8080;
```
上の source は以下と同じ意味である。
```dcdl
{
host = String;
port = Int default 8080;
}
```
## コメント
コメントは `#` から行末までとする。
@@ -38,7 +57,7 @@ host = "127.0.0.1"; # trailing comment
## 識別子
識別子の厳密な字句規則は未確定である。
識別子は ASCII 英字で始まり、ASCII 英数字または `_` を続けられる。
慣習としては `lower_snake``lowerCamel``UpperCamel` を使える想定とする。
```dcdl
@@ -74,9 +93,9 @@ config.feature_hoge.enable
}
```
## 予約語候補
## 予約語
以下は予約語または予約構文として扱う候補である
以下は予約語として扱う
```text
let
@@ -86,22 +105,22 @@ import
default
true
false
rec
```
`rec` の扱いは未確定である。
## 演算子
主要な演算子は以下である。
```text
& 制約合成
// patch 合成
default fallback 指定
=> 関数
. フィールド参照 / ドットパス定義
+ - * / 四則演算
++ 配列結合
! && || 論理演算
== != < <= > >= 比較式
& 制約合成
// patch 合成
default fallback 指定
=> 関数
. フィールド参照 / ドットパス定義
```
演算子の優先順位は未確定である。
詳細は [合成演算子](./operators.md) で定義する。
演算子の優先順位は [合成演算子](./operators.md) で定義する。
+6 -3
View File
@@ -9,7 +9,10 @@ ratio = Float;
threshold = Float default 0.5;
```
## 未確定事項
## Int との関係
`Int``Float` の暗黙変換を許可するかは未確定である。
軽量実装では、両者を明確に分ける方が単純である。
`Int``Float` は primitive type constraint としては別の型である。
`Float` constraint は concrete `Float` を要求し、`Int` constraint は concrete `Int` を要求する。
数値演算や比較式では `Int``Float` を同じ numeric value として扱える。
混在した四則演算の結果は `Float` になる。
+2 -1
View File
@@ -9,9 +9,10 @@ name = String;
retry = Int default 3;
ratio = Float;
enable = Bool default true;
tags = Array;
```
現在の primitive type は `String``Int``Float``Bool` である。
現在の primitive type は `String``Int``Float``Bool``Array` である。
各型の個別仕様へのリンクは [Manual Index](../../index.md) に集約する。
primitive type と制約合成の詳細は [制約と default](../constraints-and-defaults.md) も参照する。
+2 -1
View File
@@ -17,4 +17,5 @@ greeting = String default "hello";
message = String & /Hello! .*/;
```
正規表現制約を必須機能にするかは未確定である。
正規表現制約の検証は Rust crate の `regex` feature で有効化される。
feature が無効な場合、正規表現制約は具体 string に対して検証できず diagnostic になる。
+14 -52
View File
@@ -1,61 +1,23 @@
# 未確定事項
今後決める必要がある事項を管理する。
詳細化するときは、各項目を該当する仕様ファイルへ移動または反映する
実装または仕様方針が固まった項目は、該当する仕様ファイルへ反映してここから外す
## 構文
## Materialize target and host projection
- 正式な字句・構文仕様
- 演算子の優先順位
- `rec` の扱い。
- コメント構文を `#` のみにするか。
- パス import と文字列 import の扱い分け。
Decodal の評価結果は、制約・default・関数値を含む中間値になり得る
そのため、Decodal 単独で常に「最終成果物」を一意に決めるのではなく、host 側が期待する型や出力形式を与えて materialize / decode する経路を明確にする必要がある
## 型・制約
決めること:
- 配列要素の制約表現
- object の open/closed schema の扱い
- 正規表現を必須機能にするか optional feature にするか。
- 代表的な組み込み述語の範囲
- Rust API で評価結果の field/path を選択して decode / materialize できるようにするか
- `decodal-derive` の struct schema と評価結果を合成して decode する経路を、主要な materialize path として位置づけるか
- CLI / WASM では target path を指定して JSON-compatible value へ materialize する形にするか。
- 制約や関数値が残った値を出力したい場合、materialize ではなく inspect/debug API として分けるか
## default
現時点の案:
- `default` 同士の conflict 解決規則
- `&` による default 合成の厳密な規則
- `//` による default 置換の厳密な規則
- default thunk の評価失敗をどの段階で報告するか
## 演算子
- `//` による制約・default の置換詳細。
- `replace(...)` を採用するか、別構文を設けるか。
- 配列に対する patch 操作を右辺置換だけにするか。
- 配列 append / prepend / remove などを提供するか。
## 関数
- 関数値の最終出力可否。
- 再帰関数を許可するか。
- 関数同士の `&` の扱い。
- 関数値の等価性。
- 関数呼び出し結果の memoize 範囲。
## 評価
- thunk のエラー memoize 方針。
- import cache の単位。
- 循環 import の診断メッセージ。
- materialize 対象の範囲指定方法。
## match
- match の網羅性チェックを行うか。
- 到達不能分岐を警告するか。
- パターン構文の範囲。
## エラー処理
- optional import を導入するか。
- optional field access を導入するか。
- optional fallback が捕捉できる失敗の範囲。
- エラー報告に制約由来の説明をどこまで含めるか。
- Rust では `evaluate -> select field/path -> expected schema と合成 -> decode` を主経路にする
- CLI / WASM では、明示された target path または module 全体を JSON-compatible value として materialize する
- materialize できない unresolved abstract value、default のない制約値、関数値は diagnostic にする
- Decodal 言語内には materialize 構文を追加せず、host API / CLI / WASM の責務として扱う
+18
View File
@@ -0,0 +1,18 @@
# lezer-decodal
Lezer implementation of the Decodal grammar for CodeMirror 6.
The canonical grammar is documented in:
```text
../../doc/manual/souce/language/grammar.md
```
Regenerate the CodeMirror parser from the site directory:
```sh
cd ../../site/decodal-site
npx lezer-generator ../../editors/lezer-decodal/decodal.grammar -o ../../packages/decodal-codemirror/src/decodal-parser.js
```
The generated parser is committed with the `decodal-codemirror` package because both the playground and external Web editors consume it.
+172
View File
@@ -0,0 +1,172 @@
@top Source { Statement* }
Statement {
FieldDefinition Semicolon? |
Expression Semicolon?
}
Expression { DefaultExpression }
DefaultExpression {
PatchExpression |
PatchExpression !default Default DefaultExpression
}
PatchExpression {
ComposeExpression |
PatchExpression !patch SlashSlash ComposeExpression
}
ComposeExpression {
LogicalOrExpression |
ComposeExpression !compose Amp LogicalOrExpression
}
LogicalOrExpression {
LogicalAndExpression |
LogicalOrExpression !or PipePipe LogicalAndExpression
}
LogicalAndExpression {
ComparisonExpression |
LogicalAndExpression !and AmpAmp ComparisonExpression
}
ComparisonExpression {
ConcatExpression |
ConcatExpression !compare CompareOperator ConcatExpression
}
ConcatExpression {
AdditiveExpression |
ConcatExpression !concat PlusPlus AdditiveExpression
}
AdditiveExpression {
MultiplicativeExpression |
AdditiveExpression !add (Plus | Minus) MultiplicativeExpression
}
MultiplicativeExpression {
UnaryExpression |
MultiplicativeExpression !multiply (Star | Slash) UnaryExpression
}
UnaryExpression {
PostfixExpression |
(Bang | Minus) !unary UnaryExpression
}
PostfixExpression {
PrimaryExpression |
PostfixExpression !call CallSuffix |
PostfixExpression !path Dot Identifier
}
CallSuffix { LParen ArgumentList? RParen }
ArgumentList { Expression (Comma Expression)* Comma? }
PrimaryExpression {
Literal |
Identifier |
ComparisonConstraint |
Object |
Array |
LetExpression |
FunctionExpression |
MatchExpression |
ImportExpression |
LParen Expression RParen
}
Literal { String | Integer | Float | True | False | Regex }
CompareOperator { EqualEqual | BangEqual | Lt | Lte | Gt | Gte }
ComparisonConstraint { (Lt | Lte | Gt | Gte) Expression }
Object { LBrace (FieldDefinition (Semicolon FieldDefinition)* Semicolon?)? RBrace }
FieldDefinition { FieldPath Equal Expression }
FieldPath { Identifier !fieldPath (Dot Identifier)* }
Array { LBracket (Expression (Comma Expression)* Comma?)? RBracket }
LetExpression { Let FieldDefinitionList In Expression }
FieldDefinitionList { (FieldDefinition Semicolon)* }
FunctionExpression { LParen ParameterList? RParen Arrow Expression }
ParameterList { Parameter (Comma Parameter)* Comma? }
Parameter { Identifier Colon Expression }
MatchExpression { Match Expression LBrace (MatchArm (Semicolon MatchArm)* Semicolon?)? RBrace }
MatchArm { Pattern Colon Expression }
Pattern { Underscore | Expression }
ImportExpression { Import String }
@precedence {
fieldPath @left,
default @right,
patch @left,
compose @left,
or @left,
and @left,
compare @left,
concat @left,
add @left,
multiply @left,
unary,
call @left,
path @left
}
@tokens {
Let { "let" }
In { "in" }
Match { "match" }
Import { "import" }
Default { "default" }
True { "true" }
False { "false" }
Identifier { $[A-Za-z_] $[A-Za-z0-9_]* }
Integer { $[0-9]+ }
Float { $[0-9]+ "." $[0-9]+ }
String { '"' (!["\\\n] | "\\" _)* '"' }
Regex { "/" (![/\\\n] | "\\" _)+ "/" }
Comment { "#" ![\n]* }
LBrace { "{" }
RBrace { "}" }
LBracket { "[" }
RBracket { "]" }
LParen { "(" }
RParen { ")" }
Semicolon { ";" }
Comma { "," }
Dot { "." }
Colon { ":" }
Equal { "=" }
EqualEqual { "==" }
Bang { "!" }
BangEqual { "!=" }
Arrow { "=>" }
Amp { "&" }
AmpAmp { "&&" }
PipePipe { "||" }
Plus { "+" }
PlusPlus { "++" }
Minus { "-" }
Star { "*" }
Slash { "/" }
SlashSlash { "//" }
Gt { ">" }
Gte { ">=" }
Lt { "<" }
Lte { "<=" }
Underscore { "_" }
whitespace { @whitespace+ }
@precedence { Let, In, Match, Import, Default, True, False, Float, Integer, Underscore, Identifier }
}
@skip { whitespace | Comment }
+39
View File
@@ -0,0 +1,39 @@
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[*.{json,toml,yml,gyp}]
indent_style = space
indent_size = 2
[*.js]
indent_style = space
indent_size = 2
[*.rs]
indent_style = space
indent_size = 4
[*.{c,cc,h}]
indent_style = space
indent_size = 4
[*.{py,pyi}]
indent_style = space
indent_size = 4
[*.swift]
indent_style = space
indent_size = 4
[*.go]
indent_style = tab
indent_size = 8
[Makefile]
indent_style = tab
indent_size = 8
@@ -0,0 +1,11 @@
* text eol=lf
src/*.json linguist-generated
src/parser.c linguist-generated
src/tree_sitter/* linguist-generated
bindings/** linguist-generated
binding.gyp linguist-generated
setup.py linguist-generated
Makefile linguist-generated
Package.swift linguist-generated
+38
View File
@@ -0,0 +1,38 @@
# Rust artifacts
Cargo.lock
target/
# Node artifacts
build/
prebuilds/
node_modules/
*.tgz
# Swift artifacts
.build/
# Go artifacts
go.sum
_obj/
# Python artifacts
.venv/
dist/
*.egg-info
*.whl
# C artifacts
*.a
*.so
*.so.*
*.dylib
*.dll
*.pc
# Example dirs
/examples/*/
# Grammar volatiles
*.wasm
*.obj
*.o
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "tree-sitter-decodal"
description = "Decodal grammar for tree-sitter"
version = "0.0.1"
license = "MIT"
readme = "README.md"
keywords = ["incremental", "parsing", "tree-sitter", "decodal"]
categories = ["parsing", "text-editors"]
repository = "https://github.com/tree-sitter/tree-sitter-decodal"
edition = "2021"
autoexamples = false
build = "bindings/rust/build.rs"
include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"]
[lib]
path = "bindings/rust/lib.rs"
[dependencies]
tree-sitter = ">=0.22.6"
[build-dependencies]
cc = "1.0.87"
+112
View File
@@ -0,0 +1,112 @@
VERSION := 0.0.1
LANGUAGE_NAME := tree-sitter-decodal
# repository
SRC_DIR := src
PARSER_REPO_URL := $(shell git -C $(SRC_DIR) remote get-url origin 2>/dev/null)
ifeq ($(PARSER_URL),)
PARSER_URL := $(subst .git,,$(PARSER_REPO_URL))
ifeq ($(shell echo $(PARSER_URL) | grep '^[a-z][-+.0-9a-z]*://'),)
PARSER_URL := $(subst :,/,$(PARSER_URL))
PARSER_URL := $(subst git@,https://,$(PARSER_URL))
endif
endif
TS ?= tree-sitter
# ABI versioning
SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION)))
SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION)))
# install directory layout
PREFIX ?= /usr/local
INCLUDEDIR ?= $(PREFIX)/include
LIBDIR ?= $(PREFIX)/lib
PCLIBDIR ?= $(LIBDIR)/pkgconfig
# source/object files
PARSER := $(SRC_DIR)/parser.c
EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c))
OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS))
# flags
ARFLAGS ?= rcs
override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC
# OS-specific bits
ifeq ($(OS),Windows_NT)
$(error "Windows is not supported")
else ifeq ($(shell uname),Darwin)
SOEXT = dylib
SOEXTVER_MAJOR = $(SONAME_MAJOR).dylib
SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).dylib
LINKSHARED := $(LINKSHARED)-dynamiclib -Wl,
ifneq ($(ADDITIONAL_LIBS),)
LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS),
endif
LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SONAME_MAJOR).dylib,-rpath,@executable_path/../Frameworks
else
SOEXT = so
SOEXTVER_MAJOR = so.$(SONAME_MAJOR)
SOEXTVER = so.$(SONAME_MAJOR).$(SONAME_MINOR)
LINKSHARED := $(LINKSHARED)-shared -Wl,
ifneq ($(ADDITIONAL_LIBS),)
LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS)
endif
LINKSHARED := $(LINKSHARED)-soname,lib$(LANGUAGE_NAME).so.$(SONAME_MAJOR)
endif
ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
PCLIBDIR := $(PREFIX)/libdata/pkgconfig
endif
all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc
lib$(LANGUAGE_NAME).a: $(OBJS)
$(AR) $(ARFLAGS) $@ $^
lib$(LANGUAGE_NAME).$(SOEXT): $(OBJS)
$(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@
ifneq ($(STRIP),)
$(STRIP) $@
endif
$(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in
sed -e 's|@URL@|$(PARSER_URL)|' \
-e 's|@VERSION@|$(VERSION)|' \
-e 's|@LIBDIR@|$(LIBDIR)|' \
-e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
-e 's|@REQUIRES@|$(REQUIRES)|' \
-e 's|@ADDITIONAL_LIBS@|$(ADDITIONAL_LIBS)|' \
-e 's|=$(PREFIX)|=$${prefix}|' \
-e 's|@PREFIX@|$(PREFIX)|' $< > $@
$(PARSER): $(SRC_DIR)/grammar.json
$(TS) generate --no-bindings $^
install: all
install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
install -m644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h
install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc
install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a
install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER)
ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR)
ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT)
uninstall:
$(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \
'$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) \
'$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) \
'$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) \
'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h \
'$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc
clean:
$(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT)
test:
$(TS) test
.PHONY: all install uninstall clean test
+47
View File
@@ -0,0 +1,47 @@
// swift-tools-version:5.3
import PackageDescription
let package = Package(
name: "TreeSitterDecodal",
products: [
.library(name: "TreeSitterDecodal", targets: ["TreeSitterDecodal"]),
],
dependencies: [],
targets: [
.target(name: "TreeSitterDecodal",
path: ".",
exclude: [
"Cargo.toml",
"Makefile",
"binding.gyp",
"bindings/c",
"bindings/go",
"bindings/node",
"bindings/python",
"bindings/rust",
"prebuilds",
"grammar.js",
"package.json",
"package-lock.json",
"pyproject.toml",
"setup.py",
"test",
"examples",
".editorconfig",
".github",
".gitignore",
".gitattributes",
".gitmodules",
],
sources: [
"src/parser.c",
// NOTE: if your language has an external scanner, add it here.
],
resources: [
.copy("queries")
],
publicHeadersPath: "bindings/swift",
cSettings: [.headerSearchPath("src")])
],
cLanguageStandard: .c11
)
+12
View File
@@ -0,0 +1,12 @@
# tree-sitter-decodal
Tree-sitter implementation of the Decodal grammar.
The canonical grammar is documented in:
```text
../../doc/manual/souce/language/grammar.md
```
This grammar is an editor/tooling implementation of that EBNF reference.
Generated parser files under `src/` are committed.
+30
View File
@@ -0,0 +1,30 @@
{
"targets": [
{
"target_name": "tree_sitter_decodal_binding",
"dependencies": [
"<!(node -p \"require('node-addon-api').targets\"):node_addon_api_except",
],
"include_dirs": [
"src",
],
"sources": [
"bindings/node/binding.cc",
"src/parser.c",
# NOTE: if your language has an external scanner, add it here.
],
"conditions": [
["OS!='win'", {
"cflags_c": [
"-std=c11",
],
}, { # OS == "win"
"cflags_c": [
"/std:c11",
"/utf-8",
],
}],
],
}
]
}
@@ -0,0 +1,16 @@
#ifndef TREE_SITTER_DECODAL_H_
#define TREE_SITTER_DECODAL_H_
typedef struct TSLanguage TSLanguage;
#ifdef __cplusplus
extern "C" {
#endif
const TSLanguage *tree_sitter_decodal(void);
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_DECODAL_H_
@@ -0,0 +1,11 @@
prefix=@PREFIX@
libdir=@LIBDIR@
includedir=@INCLUDEDIR@
Name: tree-sitter-decodal
Description: Decodal grammar for tree-sitter
URL: @URL@
Version: @VERSION@
Requires: @REQUIRES@
Libs: -L${libdir} @ADDITIONAL_LIBS@ -ltree-sitter-decodal
Cflags: -I${includedir}
+13
View File
@@ -0,0 +1,13 @@
package tree_sitter_decodal
// #cgo CFLAGS: -std=c11 -fPIC
// #include "../../src/parser.c"
// // NOTE: if your language has an external scanner, add it here.
import "C"
import "unsafe"
// Get the tree-sitter Language for this grammar.
func Language() unsafe.Pointer {
return unsafe.Pointer(C.tree_sitter_decodal())
}
+15
View File
@@ -0,0 +1,15 @@
package tree_sitter_decodal_test
import (
"testing"
tree_sitter "github.com/smacker/go-tree-sitter"
"github.com/tree-sitter/tree-sitter-decodal"
)
func TestCanLoadGrammar(t *testing.T) {
language := tree_sitter.NewLanguage(tree_sitter_decodal.Language())
if language == nil {
t.Errorf("Error loading Decodal grammar")
}
}
+5
View File
@@ -0,0 +1,5 @@
module github.com/tree-sitter/tree-sitter-decodal
go 1.22
require github.com/smacker/go-tree-sitter v0.0.0-20230720070738-0d0a9f78d8f8
+20
View File
@@ -0,0 +1,20 @@
#include <napi.h>
typedef struct TSLanguage TSLanguage;
extern "C" TSLanguage *tree_sitter_decodal();
// "tree-sitter", "language" hashed with BLAKE2
const napi_type_tag LANGUAGE_TYPE_TAG = {
0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16
};
Napi::Object Init(Napi::Env env, Napi::Object exports) {
exports["name"] = Napi::String::New(env, "decodal");
auto language = Napi::External<TSLanguage>::New(env, tree_sitter_decodal());
language.TypeTag(&LANGUAGE_TYPE_TAG);
exports["language"] = language;
return exports;
}
NODE_API_MODULE(tree_sitter_decodal_binding, Init)
+28
View File
@@ -0,0 +1,28 @@
type BaseNode = {
type: string;
named: boolean;
};
type ChildNode = {
multiple: boolean;
required: boolean;
types: BaseNode[];
};
type NodeInfo =
| (BaseNode & {
subtypes: BaseNode[];
})
| (BaseNode & {
fields: { [name: string]: ChildNode };
children: ChildNode[];
});
type Language = {
name: string;
language: unknown;
nodeTypeInfo: NodeInfo[];
};
declare const language: Language;
export = language;
+7
View File
@@ -0,0 +1,7 @@
const root = require("path").join(__dirname, "..", "..");
module.exports = require("node-gyp-build")(root);
try {
module.exports.nodeTypeInfo = require("../../src/node-types.json");
} catch (_) {}
@@ -0,0 +1,5 @@
"Decodal grammar for tree-sitter"
from ._binding import language
__all__ = ["language"]
@@ -0,0 +1 @@
def language() -> int: ...
@@ -0,0 +1,27 @@
#include <Python.h>
typedef struct TSLanguage TSLanguage;
TSLanguage *tree_sitter_decodal(void);
static PyObject* _binding_language(PyObject *self, PyObject *args) {
return PyLong_FromVoidPtr(tree_sitter_decodal());
}
static PyMethodDef methods[] = {
{"language", _binding_language, METH_NOARGS,
"Get the tree-sitter language for this grammar."},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef module = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = "_binding",
.m_doc = NULL,
.m_size = -1,
.m_methods = methods
};
PyMODINIT_FUNC PyInit__binding(void) {
return PyModule_Create(&module);
}
+22
View File
@@ -0,0 +1,22 @@
fn main() {
let src_dir = std::path::Path::new("src");
let mut c_config = cc::Build::new();
c_config.std("c11").include(src_dir);
#[cfg(target_env = "msvc")]
c_config.flag("-utf-8");
let parser_path = src_dir.join("parser.c");
c_config.file(&parser_path);
println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
// NOTE: if your language uses an external scanner, uncomment this block:
/*
let scanner_path = src_dir.join("scanner.c");
c_config.file(&scanner_path);
println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
*/
c_config.compile("tree-sitter-decodal");
}
+54
View File
@@ -0,0 +1,54 @@
//! This crate provides Decodal language support for the [tree-sitter][] parsing library.
//!
//! Typically, you will use the [language][language func] function to add this language to a
//! tree-sitter [Parser][], and then use the parser to parse some code:
//!
//! ```
//! let code = r#"
//! "#;
//! let mut parser = tree_sitter::Parser::new();
//! parser.set_language(&tree_sitter_decodal::language()).expect("Error loading Decodal grammar");
//! let tree = parser.parse(code, None).unwrap();
//! assert!(!tree.root_node().has_error());
//! ```
//!
//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
//! [language func]: fn.language.html
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
//! [tree-sitter]: https://tree-sitter.github.io/
use tree_sitter::Language;
extern "C" {
fn tree_sitter_decodal() -> Language;
}
/// Get the tree-sitter [Language][] for this grammar.
///
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
pub fn language() -> Language {
unsafe { tree_sitter_decodal() }
}
/// The content of the [`node-types.json`][] file for this grammar.
///
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
// Uncomment these to include any queries that this grammar contains
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
#[cfg(test)]
mod tests {
#[test]
fn test_can_load_grammar() {
let mut parser = tree_sitter::Parser::new();
parser
.set_language(&super::language())
.expect("Error loading Decodal grammar");
}
}
@@ -0,0 +1,16 @@
#ifndef TREE_SITTER_DECODAL_H_
#define TREE_SITTER_DECODAL_H_
typedef struct TSLanguage TSLanguage;
#ifdef __cplusplus
extern "C" {
#endif
const TSLanguage *tree_sitter_decodal(void);
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_DECODAL_H_
@@ -0,0 +1,160 @@
==================
Top-level fields
==================
host = String;
port = Int & > 443 default 8080;
---
(source_file
(field_definition
path: (field_path (identifier))
value: (identifier))
(field_definition
path: (field_path (identifier))
value: (default_expression
base: (binary_expression
left: (identifier)
right: (comparison_constraint
value: (integer)))
fallback: (literal (integer)))))
==================
Import and function
==================
let
schema = import "./schema.dcdl";
mk = (cfg: schema.Service) => cfg;
in
mk({ name = "api"; })
---
(source_file
(let_expression
(field_definition
path: (field_path (identifier))
value: (import_expression
specifier: (string)))
(field_definition
path: (field_path (identifier))
value: (function_expression
(parameter
name: (identifier)
constraint: (path_expression
object: (identifier)
field: (identifier)))
body: (identifier)))
body: (call_expression
function: (identifier)
(object
(field_definition
path: (field_path (identifier))
value: (literal (string)))))))
==================
Match and patch
==================
base // {
summary = match env {
"prod": "production";
_: "other";
};
}
---
(source_file
(binary_expression
left: (identifier)
right: (object
(field_definition
path: (field_path (identifier))
value: (match_expression
scrutinee: (identifier)
(match_arm
pattern: (literal (string))
body: (literal (string)))
(match_arm
body: (literal (string))))))))
==================
Arithmetic
==================
{
value = 1 + 2 * 3;
grouped = (1 + 2) / -3;
}
---
(source_file
(object
(field_definition
path: (field_path (identifier))
value: (binary_expression
left: (literal (integer))
right: (binary_expression
left: (literal (integer))
right: (literal (integer)))))
(field_definition
path: (field_path (identifier))
value: (binary_expression
left: (parenthesized_expression
(binary_expression
left: (literal (integer))
right: (literal (integer))))
right: (unary_expression
operand: (literal (integer)))))))
==================
Logical and comparison
==================
{
enabled = env == "prod" && replicas > 1;
disabled = !enabled || false;
}
---
(source_file
(object
(field_definition
path: (field_path (identifier))
value: (binary_expression
left: (binary_expression
left: (identifier)
right: (literal (string)))
right: (binary_expression
left: (identifier)
right: (literal (integer)))))
(field_definition
path: (field_path (identifier))
value: (binary_expression
left: (unary_expression
operand: (identifier))
right: (literal (boolean))))))
==================
Array concat
==================
{
roles = ["read"] ++ ["write", "admin"];
ports = [8000 + 80] ++ [9443];
}
---
(source_file
(object
(field_definition
path: (field_path (identifier))
value: (binary_expression
left: (array
(literal (string)))
right: (array
(literal (string))
(literal (string)))))
(field_definition
path: (field_path (identifier))
value: (binary_expression
left: (array
(binary_expression
left: (literal (integer))
right: (literal (integer))))
right: (array
(literal (integer)))))))
+232
View File
@@ -0,0 +1,232 @@
// Tree-sitter implementation of the canonical EBNF grammar in
// doc/manual/souce/language/grammar.md.
const PREC = {
DEFAULT: 1,
PATCH: 2,
COMPOSE: 3,
OR: 4,
LOGICAL_AND: 5,
COMPARE: 6,
CONCAT: 7,
ADD: 8,
MUL: 9,
UNARY: 10,
CALL: 11,
PATH: 12,
};
function commaSep(rule) {
return optional(seq(rule, repeat(seq(',', rule)), optional(',')));
}
function semiSep(rule) {
return optional(seq(rule, repeat(seq(';', rule)), optional(';')));
}
module.exports = grammar({
name: 'decodal',
extras: $ => [
/[\s\uFEFF\u2060\u200B]/,
$.comment,
],
word: $ => $.identifier,
conflicts: $ => [
[$._expression, $.parameter],
],
rules: {
source_file: $ => repeat($._statement),
_statement: $ => choice(
seq($.field_definition, optional(';')),
seq($._expression, optional(';')),
),
comment: _ => token(seq('#', /.*/)),
_expression: $ => choice(
$.literal,
$.identifier,
$.regex_literal,
$.comparison_constraint,
$.object,
$.array,
$.let_expression,
$.function_expression,
$.match_expression,
$.import_expression,
$.parenthesized_expression,
$.call_expression,
$.path_expression,
$.unary_expression,
$.binary_expression,
$.default_expression,
),
literal: $ => choice(
$.string,
$.integer,
$.float,
$.boolean,
),
boolean: _ => choice('true', 'false'),
string: _ => token(seq(
'"',
repeat(choice(/[^"\\\n]/, /\\./)),
'"',
)),
integer: _ => token(/[0-9]+/),
float: _ => token(/[0-9]+\.[0-9]+/),
identifier: _ => /[A-Za-z][A-Za-z0-9_]*/,
regex_literal: _ => token(seq(
'/',
repeat1(choice(/[^\/\\\n]/, /\\./)),
'/',
)),
object: $ => seq(
'{',
semiSep($.field_definition),
'}',
),
field_definition: $ => seq(
field('path', $.field_path),
'=',
field('value', $._expression),
),
field_path: $ => prec(10, seq(
$.identifier,
repeat(seq('.', $.identifier)),
)),
array: $ => seq(
'[',
commaSep($._expression),
']',
),
let_expression: $ => seq(
'let',
repeat(seq($.field_definition, ';')),
'in',
field('body', $._expression),
),
function_expression: $ => prec.right(seq(
'(',
commaSep($.parameter),
')',
'=>',
field('body', $._expression),
)),
parameter: $ => seq(
field('name', $.identifier),
optional(seq(':', field('constraint', $._expression))),
),
match_expression: $ => seq(
'match',
field('scrutinee', $._expression),
'{',
semiSep($.match_arm),
'}',
),
match_arm: $ => seq(
field('pattern', choice('_', $._expression)),
':',
field('body', $._expression),
),
import_expression: $ => seq(
'import',
field('specifier', $.string),
),
parenthesized_expression: $ => seq('(', $._expression, ')'),
call_expression: $ => prec.left(PREC.CALL, seq(
field('function', $._expression),
'(',
commaSep($._expression),
')',
)),
path_expression: $ => prec.left(PREC.PATH, seq(
field('object', $._expression),
'.',
field('field', $.identifier),
)),
comparison_constraint: $ => prec.right(PREC.UNARY + 1, seq(
field('operator', choice('>', '>=', '<', '<=')),
field('value', $._expression),
)),
unary_expression: $ => prec(PREC.UNARY, seq(
field('operator', choice('-', '!')),
field('operand', $._expression),
)),
binary_expression: $ => choice(
prec.left(PREC.OR, seq(
field('left', $._expression),
field('operator', token(prec(2, '||'))),
field('right', $._expression),
)),
prec.left(PREC.LOGICAL_AND, seq(
field('left', $._expression),
field('operator', token(prec(2, '&&'))),
field('right', $._expression),
)),
prec.left(PREC.COMPARE, seq(
field('left', $._expression),
field('operator', choice('==', '!=', '>', '>=', '<', '<=')),
field('right', $._expression),
)),
prec.left(PREC.CONCAT, seq(
field('left', $._expression),
field('operator', token(prec(2, '++'))),
field('right', $._expression),
)),
prec.left(PREC.ADD, seq(
field('left', $._expression),
field('operator', choice('+', '-')),
field('right', $._expression),
)),
prec.left(PREC.MUL, seq(
field('left', $._expression),
field('operator', choice('*', '/')),
field('right', $._expression),
)),
prec.left(PREC.COMPOSE, seq(
field('left', $._expression),
field('operator', '&'),
field('right', $._expression),
)),
prec.left(PREC.PATCH, seq(
field('left', $._expression),
field('operator', token(prec(2, '//'))),
field('right', $._expression),
)),
),
default_expression: $ => prec.right(PREC.DEFAULT, seq(
field('base', $._expression),
'default',
field('fallback', $._expression),
)),
},
});
+27
View File
@@ -0,0 +1,27 @@
{
"name": "tree-sitter-decodal",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "tree-sitter-decodal",
"version": "0.1.0",
"license": "MIT",
"devDependencies": {
"tree-sitter-cli": "^0.22.6"
}
},
"node_modules/tree-sitter-cli": {
"version": "0.22.6",
"resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.22.6.tgz",
"integrity": "sha512-s7mYOJXi8sIFkt/nLJSqlYZP96VmKTc3BAwIX0rrrlRxWjWuCwixFqwzxWZBQz4R8Hx01iP7z3cT3ih58BUmZQ==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"bin": {
"tree-sitter": "cli.js"
}
}
}
}
+50
View File
@@ -0,0 +1,50 @@
{
"name": "tree-sitter-decodal",
"version": "0.1.0",
"description": "Tree-sitter grammar for Decodal / DCDL",
"main": "grammar.js",
"types": "bindings/node",
"keywords": [
"tree-sitter",
"decodal",
"dcdl"
],
"files": [
"grammar.js",
"binding.gyp",
"prebuilds/**",
"bindings/node/*",
"queries/*",
"src/**"
],
"license": "MIT",
"tree-sitter": [
{
"scope": "source.dcdl",
"file-types": [
"dcdl"
]
}
],
"devDependencies": {
"tree-sitter-cli": "^0.22.6",
"prebuildify": "^6.0.0"
},
"scripts": {
"generate": "tree-sitter generate",
"test": "tree-sitter test",
"install": "node-gyp-build",
"prebuildify": "prebuildify --napi --strip"
},
"dependencies": {
"node-gyp-build": "^4.8.0"
},
"peerDependencies": {
"tree-sitter": "^0.21.0"
},
"peerDependenciesMeta": {
"tree_sitter": {
"optional": true
}
}
}
@@ -0,0 +1,29 @@
[build-system]
requires = ["setuptools>=42", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "tree-sitter-decodal"
description = "Decodal grammar for tree-sitter"
version = "0.0.1"
keywords = ["incremental", "parsing", "tree-sitter", "decodal"]
classifiers = [
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Topic :: Software Development :: Compilers",
"Topic :: Text Processing :: Linguistic",
"Typing :: Typed"
]
requires-python = ">=3.8"
license.text = "MIT"
readme = "README.md"
[project.urls]
Homepage = "https://github.com/tree-sitter/tree-sitter-decodal"
[project.optional-dependencies]
core = ["tree-sitter~=0.21"]
[tool.cibuildwheel]
build = "cp38-*"
build-frontend = "build"
@@ -0,0 +1,68 @@
(comment) @comment
(string) @string
(regex_literal) @string.regexp
(integer) @number
(float) @number
(boolean) @boolean
[
"let"
"in"
"match"
"import"
"default"
] @keyword
[
"&"
"//"
"+"
"-"
"*"
"/"
"++"
"&&"
"||"
"!"
"=="
"!="
"=>"
"="
">"
">="
"<"
"<="
] @operator
[
"{"
"}"
"["
"]"
"("
")"
] @punctuation.bracket
[
";"
","
"."
":"
] @punctuation.delimiter
((identifier) @type.builtin
(#match? @type.builtin "^(String|Int|Float|Bool)$"))
(field_definition
path: (field_path (identifier) @property))
(path_expression
field: (identifier) @property)
(parameter
name: (identifier) @variable.parameter)
(function_expression) @function
(call_expression
function: (identifier) @function.call)
@@ -0,0 +1,3 @@
(parameter name: (identifier) @local.definition)
(field_definition path: (field_path (identifier) @local.definition))
(identifier) @local.reference
+60
View File
@@ -0,0 +1,60 @@
from os.path import isdir, join
from platform import system
from setuptools import Extension, find_packages, setup
from setuptools.command.build import build
from wheel.bdist_wheel import bdist_wheel
class Build(build):
def run(self):
if isdir("queries"):
dest = join(self.build_lib, "tree_sitter_decodal", "queries")
self.copy_tree("queries", dest)
super().run()
class BdistWheel(bdist_wheel):
def get_tag(self):
python, abi, platform = super().get_tag()
if python.startswith("cp"):
python, abi = "cp38", "abi3"
return python, abi, platform
setup(
packages=find_packages("bindings/python"),
package_dir={"": "bindings/python"},
package_data={
"tree_sitter_decodal": ["*.pyi", "py.typed"],
"tree_sitter_decodal.queries": ["*.scm"],
},
ext_package="tree_sitter_decodal",
ext_modules=[
Extension(
name="_binding",
sources=[
"bindings/python/tree_sitter_decodal/binding.c",
"src/parser.c",
# NOTE: if your language uses an external scanner, add it here.
],
extra_compile_args=[
"-std=c11",
] if system() != "Windows" else [
"/std:c11",
"/utf-8",
],
define_macros=[
("Py_LIMITED_API", "0x03080000"),
("PY_SSIZE_T_CLEAN", None)
],
include_dirs=["src"],
py_limited_api=True,
)
],
cmdclass={
"build": Build,
"bdist_wheel": BdistWheel
},
zip_safe=False
)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+54
View File
@@ -0,0 +1,54 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
// Allow clients to override allocation functions
#ifdef TREE_SITTER_REUSE_ALLOCATOR
extern void *(*ts_current_malloc)(size_t);
extern void *(*ts_current_calloc)(size_t, size_t);
extern void *(*ts_current_realloc)(void *, size_t);
extern void (*ts_current_free)(void *);
#ifndef ts_malloc
#define ts_malloc ts_current_malloc
#endif
#ifndef ts_calloc
#define ts_calloc ts_current_calloc
#endif
#ifndef ts_realloc
#define ts_realloc ts_current_realloc
#endif
#ifndef ts_free
#define ts_free ts_current_free
#endif
#else
#ifndef ts_malloc
#define ts_malloc malloc
#endif
#ifndef ts_calloc
#define ts_calloc calloc
#endif
#ifndef ts_realloc
#define ts_realloc realloc
#endif
#ifndef ts_free
#define ts_free free
#endif
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ALLOC_H_
+290
View File
@@ -0,0 +1,290 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./alloc.h"
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array.
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ NULL, 0, 0 }
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), (self)->size, \
0, count, contents \
)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), _index, \
old_count, new_count, new_contents \
)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) \
_array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) {
if (self->contents) {
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = ts_realloc(self->contents, new_capacity * element_size);
} else {
self->contents = ts_malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) {
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
uint32_t new_size = self->size + count;
if (new_size > self->capacity) {
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do { \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) break; \
int comparison; \
while (size > 1) { \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) *(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) *(_exists) = true; \
else if (comparison < 0) *(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
#pragma warning(default : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_
+265
View File
@@ -0,0 +1,265 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
typedef struct {
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value) \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value), \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_
+22
View File
@@ -0,0 +1,22 @@
# Advanced Decodal example
This example exercises multiple current Decodal features together:
- multi-file `import`
- top-level recursive module scope
- schema/value composition with `&`
- deep patch with `//`
- `default` fallback during materialization
- constrained function parameters
- lazy function arguments
- `match` expressions
- nested dot-path fields
- arrays
Run it with:
```sh
cargo run -q -p decodal-cli -- examples/advanced/main.dcdl
```
The entrypoint is `main.dcdl`.

Some files were not shown because too many files have changed in this diff Show More