Share host-aware completion across runtimes

This commit is contained in:
2026-08-13 18:55:03 +09:00
parent e862e53f3e
commit d28edcf041
25 changed files with 2836 additions and 578 deletions
@@ -0,0 +1,905 @@
use std::collections::{BTreeMap, HashMap};
use decodal::{
Engine, HostEnvironment, HostValue, ImportLoader, LoadedImport, Result, RuntimeValue,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CompletionKind {
Keyword,
Constant,
Type,
Variable,
Namespace,
Property,
File,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CompletionItem {
pub label: String,
pub kind: CompletionKind,
pub detail: Option<String>,
pub priority: i32,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CompletionResult {
/// UTF-8 byte offset at which the unfinished token starts.
pub from: usize,
pub items: Vec<CompletionItem>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct FieldTree(BTreeMap<String, FieldTree>);
impl FieldTree {
fn new() -> Self {
Self::default()
}
fn is_empty(&self) -> bool {
self.0.is_empty()
}
fn insert(&mut self, name: String, fields: FieldTree) {
self.0.insert(name, fields);
}
fn get(&self, name: &str) -> Option<&FieldTree> {
self.0.get(name)
}
fn remove(&mut self, name: &str) -> Option<FieldTree> {
self.0.remove(name)
}
}
impl IntoIterator for FieldTree {
type Item = (String, FieldTree);
type IntoIter = std::collections::btree_map::IntoIter<String, FieldTree>;
fn into_iter(self) -> Self::IntoIter {
self.0.into_iter()
}
}
impl FromIterator<(String, FieldTree)> for FieldTree {
fn from_iter<T: IntoIterator<Item = (String, FieldTree)>>(iter: T) -> Self {
Self(iter.into_iter().collect())
}
}
pub(crate) fn complete<E: HostEnvironment>(
environment: &E,
key: &str,
source: &str,
position: usize,
explicit: bool,
) -> Result<Option<CompletionResult>> {
let position = floor_char_boundary(source, position.min(source.len()));
if let Some((from, prefix)) = unfinished_import(source, position) {
let mut loader = environment.create_loader();
let items = loader
.complete_import(Some(key), prefix)?
.into_iter()
.map(|candidate| CompletionItem {
label: candidate.specifier,
kind: CompletionKind::File,
detail: candidate.detail,
priority: 30,
})
.collect();
return Ok(Some(CompletionResult {
from,
items: unique_items(items),
}));
}
if in_string_or_comment(source, position) {
return Ok(None);
}
let tokens = tokenize(source);
let local_fields = collect_fields(&tokens);
let imports = collect_import_bindings(&tokens);
let globals = collect_globals(environment)?;
if let Some((from, path)) = member_path(source, position) {
let mut parts = path.split('.');
let Some(root) = parts.next() else {
return Ok(None);
};
let mut fields;
let detail;
if let Some(specifier) = imports.get(root) {
let mut loader = environment.create_loader();
let loaded = loader.load(Some(key), specifier)?;
detail = match &loaded {
LoadedImport::Source(source) => source.name.clone(),
LoadedImport::Value(value) => value.key.clone(),
};
fields = match loaded {
LoadedImport::Source(source) => collect_fields(&tokenize(&source.source)),
LoadedImport::Value(value) => fields_from_host_value(&value.value),
};
} else if let Some(local) = local_fields.get(root) {
fields = local.clone();
detail = String::from("local value");
} else if let Some(global) = globals.get(root) {
fields = global.clone();
detail = String::from("host global");
} else {
return Ok(None);
}
for part in parts {
let Some(nested) = fields.remove(part) else {
return Ok(None);
};
fields = nested;
}
let items = fields
.into_iter()
.map(|(label, children)| CompletionItem {
label,
kind: if children.is_empty() {
CompletionKind::Property
} else {
CompletionKind::Namespace
},
detail: Some(detail.clone()),
priority: 30,
})
.collect();
return Ok(Some(CompletionResult {
from,
items: unique_items(items),
}));
}
let word_from = word_start(source, position);
if word_from == position && !explicit {
return Ok(None);
}
let mut items = builtin_items();
for (label, children) in local_fields {
items.push(CompletionItem {
label,
kind: if children.is_empty() {
CompletionKind::Variable
} else {
CompletionKind::Namespace
},
detail: Some(if children.is_empty() {
String::from("local value")
} else {
String::from("local object")
}),
priority: 20,
});
}
for label in collect_parameters(&tokens) {
items.push(CompletionItem {
label,
kind: CompletionKind::Variable,
detail: Some(String::from("parameter")),
priority: 20,
});
}
for (label, specifier) in imports {
items.push(CompletionItem {
label,
kind: CompletionKind::Namespace,
detail: Some(specifier),
priority: 30,
});
}
for (label, children) in globals {
items.push(CompletionItem {
label,
kind: if children.is_empty() {
CompletionKind::Variable
} else {
CompletionKind::Namespace
},
detail: Some(String::from("host global")),
priority: 40,
});
}
Ok(Some(CompletionResult {
from: word_from,
items: unique_items(items),
}))
}
fn builtin_items() -> Vec<CompletionItem> {
[
("let", CompletionKind::Keyword, "local bindings", 5),
("in", CompletionKind::Keyword, "let body", 0),
("match", CompletionKind::Keyword, "pattern matching", 5),
("import", CompletionKind::Keyword, "load a module", 5),
("default", CompletionKind::Keyword, "fallback value", 0),
("true", CompletionKind::Constant, "Bool", 0),
("false", CompletionKind::Constant, "Bool", 0),
("String", CompletionKind::Type, "string constraint", 5),
("Int", CompletionKind::Type, "integer constraint", 5),
("Float", CompletionKind::Type, "float constraint", 5),
("Bool", CompletionKind::Type, "boolean constraint", 5),
]
.into_iter()
.map(|(label, kind, detail, priority)| CompletionItem {
label: label.into(),
kind,
detail: Some(detail.into()),
priority,
})
.collect()
}
fn collect_globals<E: HostEnvironment>(environment: &E) -> Result<FieldTree> {
let mut engine = environment.create_engine()?;
let mut fields = FieldTree::new();
for (name, value) in engine.global_values()? {
fields.insert(name, fields_from_runtime(&mut engine, &value, 0)?);
}
Ok(fields)
}
fn fields_from_runtime<L: ImportLoader>(
engine: &mut Engine<L>,
value: &RuntimeValue,
depth: usize,
) -> Result<FieldTree> {
if depth >= 32 {
return Ok(FieldTree::new());
}
let Some(fields) = engine.value_fields(value)? else {
return Ok(FieldTree::new());
};
let mut tree = FieldTree::new();
for (name, value) in fields {
tree.insert(name, fields_from_runtime(engine, &value, depth + 1)?);
}
Ok(tree)
}
fn fields_from_host_value(value: &HostValue) -> FieldTree {
match value {
HostValue::Object(fields) => fields
.iter()
.map(|field| (field.name.clone(), fields_from_host_value(&field.value)))
.collect(),
HostValue::ArrayConstraint {
default: Some(value),
..
}
| HostValue::Abstract {
default: Some(value),
..
} => fields_from_host_value(value),
_ => FieldTree::new(),
}
}
fn unique_items(items: Vec<CompletionItem>) -> Vec<CompletionItem> {
let mut unique = HashMap::<String, CompletionItem>::new();
let mut order = Vec::new();
for item in items {
if !unique.contains_key(&item.label) {
order.push(item.label.clone());
}
let replace = unique
.get(&item.label)
.is_none_or(|previous| item.priority > previous.priority);
if replace {
unique.insert(item.label.clone(), item);
}
}
order
.into_iter()
.filter_map(|label| unique.remove(&label))
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum TokenKind {
Identifier(String),
String(String),
Symbol(char),
Arrow,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Token {
kind: TokenKind,
}
fn tokenize(source: &str) -> Vec<Token> {
let bytes = source.as_bytes();
let mut tokens = Vec::new();
let mut index = 0;
while index < bytes.len() {
match bytes[index] {
b' ' | b'\t' | b'\r' | b'\n' => index += 1,
b'#' => {
while index < bytes.len() && bytes[index] != b'\n' {
index += 1;
}
}
b'"' => {
index += 1;
let start = index;
let mut escaped = false;
while index < bytes.len() {
let byte = bytes[index];
if escaped {
escaped = false;
} else if byte == b'\\' {
escaped = true;
} else if byte == b'"' || byte == b'\n' {
break;
}
index += 1;
}
let value = unescape_string(&source[start..index]);
index = index.saturating_add(1);
tokens.push(Token {
kind: TokenKind::String(value),
});
}
byte if is_identifier_start(byte) => {
let start = index;
index += 1;
while index < bytes.len() && is_identifier_continue(bytes[index]) {
index += 1;
}
tokens.push(Token {
kind: TokenKind::Identifier(source[start..index].into()),
});
}
b'=' if bytes.get(index + 1) == Some(&b'>') => {
tokens.push(Token {
kind: TokenKind::Arrow,
});
index += 2;
}
byte => {
tokens.push(Token {
kind: TokenKind::Symbol(byte as char),
});
index += 1;
}
}
}
tokens
}
fn unescape_string(value: &str) -> String {
let mut chars = value.chars();
let mut unescaped = String::new();
while let Some(ch) = chars.next() {
if ch != '\\' {
unescaped.push(ch);
continue;
}
let Some(escaped) = chars.next() else {
unescaped.push('\\');
break;
};
unescaped.push(match escaped {
'n' => '\n',
'r' => '\r',
't' => '\t',
escaped => escaped,
});
}
unescaped
}
fn collect_fields(tokens: &[Token]) -> FieldTree {
let mut fields = FieldTree::new();
collect_fields_in(tokens, 0, tokens.len(), &mut fields);
fields
}
fn collect_fields_in(tokens: &[Token], start: usize, end: usize, fields: &mut FieldTree) {
let mut index = start;
while index < end {
let Some((path, equals)) = field_definition_at(tokens, index, end) else {
index += 1;
continue;
};
let value_start = equals + 1;
let value_end = definition_end(tokens, value_start, end);
let mut nested = &mut *fields;
for part in path {
nested = nested.0.entry(part).or_default();
}
collect_fields_in(tokens, value_start, value_end, nested);
index = value_end.saturating_add(1);
}
}
fn field_definition_at(tokens: &[Token], start: usize, end: usize) -> Option<(Vec<String>, usize)> {
let TokenKind::Identifier(first) = &tokens.get(start)?.kind else {
return None;
};
let mut path = vec![first.clone()];
let mut index = start + 1;
while index + 1 < end
&& matches!(tokens[index].kind, TokenKind::Symbol('.'))
&& matches!(tokens[index + 1].kind, TokenKind::Identifier(_))
{
let TokenKind::Identifier(part) = &tokens[index + 1].kind else {
unreachable!()
};
path.push(part.clone());
index += 2;
}
matches!(
tokens.get(index).map(|token| &token.kind),
Some(TokenKind::Symbol('='))
)
.then_some((path, index))
}
fn definition_end(tokens: &[Token], start: usize, end: usize) -> usize {
let mut delimiters = Vec::new();
for (index, token) in tokens.iter().enumerate().take(end).skip(start) {
match token.kind {
TokenKind::Symbol('(' | '[' | '{') => delimiters.push(token.kind.clone()),
TokenKind::Symbol(')') if matches!(delimiters.last(), Some(TokenKind::Symbol('('))) => {
delimiters.pop();
}
TokenKind::Symbol(']') if matches!(delimiters.last(), Some(TokenKind::Symbol('['))) => {
delimiters.pop();
}
TokenKind::Symbol('}') if matches!(delimiters.last(), Some(TokenKind::Symbol('{'))) => {
delimiters.pop();
}
TokenKind::Symbol(';') if delimiters.is_empty() => return index,
_ => {}
}
}
end
}
fn collect_import_bindings(tokens: &[Token]) -> BTreeMap<String, String> {
let mut imports = BTreeMap::new();
for window in tokens.windows(4) {
let (
TokenKind::Identifier(binding),
TokenKind::Symbol('='),
TokenKind::Identifier(keyword),
TokenKind::String(specifier),
) = (
&window[0].kind,
&window[1].kind,
&window[2].kind,
&window[3].kind,
)
else {
continue;
};
if keyword == "import" {
imports.insert(binding.clone(), specifier.clone());
}
}
imports
}
fn collect_parameters(tokens: &[Token]) -> Vec<String> {
let mut parameters = Vec::new();
for (close, token) in tokens.iter().enumerate() {
if !matches!(token.kind, TokenKind::Symbol(')'))
|| !matches!(
tokens.get(close + 1).map(|token| &token.kind),
Some(TokenKind::Arrow)
)
{
continue;
}
let Some(open) = matching_open_paren(tokens, close) else {
continue;
};
let mut depth = 0usize;
let mut segment_start = true;
for token in &tokens[open + 1..close] {
match &token.kind {
TokenKind::Symbol('(' | '[' | '{') => depth += 1,
TokenKind::Symbol(')' | ']' | '}') => depth = depth.saturating_sub(1),
TokenKind::Symbol(',') if depth == 0 => segment_start = true,
TokenKind::Identifier(name) if depth == 0 && segment_start => {
parameters.push(name.clone());
segment_start = false;
}
_ => {}
}
}
}
parameters
}
fn matching_open_paren(tokens: &[Token], close: usize) -> Option<usize> {
let mut depth = 0usize;
for index in (0..close).rev() {
match tokens[index].kind {
TokenKind::Symbol(')') => depth += 1,
TokenKind::Symbol('(') if depth == 0 => return Some(index),
TokenKind::Symbol('(') => depth -= 1,
_ => {}
}
}
None
}
fn unfinished_import(source: &str, position: usize) -> Option<(usize, &str)> {
let bytes = source.as_bytes();
let mut index = 0usize;
let mut previous_word = None;
while index < position {
match bytes[index] {
b' ' | b'\t' | b'\r' | b'\n' => index += 1,
b'#' => {
while index < position && bytes[index] != b'\n' {
index += 1;
}
previous_word = None;
}
b'"' => {
let from = index + 1;
index += 1;
let mut escaped = false;
while index < position {
let byte = bytes[index];
if escaped {
escaped = false;
} else if byte == b'\\' {
escaped = true;
} else if byte == b'"' || byte == b'\n' {
break;
}
index += 1;
}
if index == position && previous_word == Some("import") {
return Some((from, &source[from..position]));
}
index = index.saturating_add(1);
previous_word = None;
}
byte if is_identifier_start(byte) => {
let start = index;
index += 1;
while index < position && is_identifier_continue(bytes[index]) {
index += 1;
}
previous_word = Some(&source[start..index]);
}
_ => {
index += 1;
previous_word = None;
}
}
}
None
}
fn in_string_or_comment(source: &str, position: usize) -> bool {
let bytes = source.as_bytes();
let mut index = 0usize;
while index < position {
match bytes[index] {
b'#' => {
while index < position && bytes[index] != b'\n' {
index += 1;
}
if index == position {
return true;
}
}
b'"' => {
index += 1;
let mut escaped = false;
while index < position {
let byte = bytes[index];
if escaped {
escaped = false;
} else if byte == b'\\' {
escaped = true;
} else if byte == b'"' || byte == b'\n' {
break;
}
index += 1;
}
if index == position {
return true;
}
index = index.saturating_add(1);
}
_ => index += 1,
}
}
false
}
fn member_path(source: &str, position: usize) -> Option<(usize, &str)> {
let bytes = source.as_bytes();
let from = word_start(source, position);
if from == 0 || bytes[from - 1] != b'.' {
return None;
}
let mut base_start = from - 1;
while base_start > 0 {
let byte = bytes[base_start - 1];
if is_identifier_continue(byte) || byte == b'.' {
base_start -= 1;
} else {
break;
}
}
let base = &source[base_start..from - 1];
(!base.is_empty()
&& base
.split('.')
.all(|part| !part.is_empty() && is_identifier(part)))
.then_some((from, base))
}
fn word_start(source: &str, position: usize) -> usize {
let bytes = source.as_bytes();
let mut from = position;
while from > 0 && is_identifier_continue(bytes[from - 1]) {
from -= 1;
}
from
}
fn is_identifier(value: &str) -> bool {
let bytes = value.as_bytes();
bytes.first().is_some_and(|byte| is_identifier_start(*byte))
&& bytes[1..].iter().all(|byte| is_identifier_continue(*byte))
}
fn is_identifier_start(byte: u8) -> bool {
byte.is_ascii_alphabetic() || byte == b'_'
}
fn is_identifier_continue(byte: u8) -> bool {
is_identifier_start(byte) || byte.is_ascii_digit()
}
fn floor_char_boundary(source: &str, mut position: usize) -> usize {
while !source.is_char_boundary(position) {
position -= 1;
}
position
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use decodal::{
Diagnostic, DiagnosticKind, EmptyLoader, HostValue, ImportCandidate, LoadedImport, Span,
};
use super::*;
#[derive(Clone, Default)]
struct TestEnvironment {
files: BTreeMap<String, String>,
}
#[derive(Clone, Default)]
struct TestLoader {
files: BTreeMap<String, String>,
}
impl ImportLoader for TestLoader {
fn load(&mut self, _current_key: Option<&str>, specifier: &str) -> Result<LoadedImport> {
if specifier == "./post.md" {
return Ok(LoadedImport::value(
"post.md",
HostValue::object([
(
"frontmatter",
HostValue::object([
("title", HostValue::string("Hello")),
("draft", HostValue::bool(false)),
]),
),
("body", HostValue::string("# Hello")),
]),
));
}
let key = resolve_virtual("main.dcdl", specifier);
let Some(source) = self.files.get(&key) else {
return Err(Diagnostic::new(
DiagnosticKind::Import,
Span::default(),
"unknown import",
));
};
Ok(LoadedImport::source(key.clone(), key, source))
}
fn complete_import(
&mut self,
_current_key: Option<&str>,
prefix: &str,
) -> Result<Vec<ImportCandidate>> {
Ok(self
.files
.keys()
.map(|path| format!("./{path}"))
.filter(|path| path.starts_with(prefix))
.map(ImportCandidate::new)
.collect())
}
}
impl HostEnvironment for TestEnvironment {
type Loader = TestLoader;
fn create_loader(&self) -> Self::Loader {
TestLoader {
files: self.files.clone(),
}
}
fn configure_engine(&self, engine: &mut Engine<Self::Loader>) -> Result<()> {
engine.bind_global(
"App",
HostValue::object([(
"config",
HostValue::object([
("enabled", HostValue::bool_type()),
("port", HostValue::int_type()),
]),
)]),
)?;
Ok(())
}
}
fn environment() -> TestEnvironment {
TestEnvironment {
files: BTreeMap::from([
(
"schemas/service.dcdl".into(),
"Service = { name = String; resources = { cpu = Int; }; };".into(),
),
(
"env/production.dcdl".into(),
"capacity = { cpu = 2000; };".into(),
),
(
"資料/service.dcdl".into(),
"Service = { label = String; };".into(),
),
]),
}
}
fn labels(result: CompletionResult) -> Vec<String> {
result.items.into_iter().map(|item| item.label).collect()
}
#[test]
fn completes_import_paths_from_the_host_loader() {
let source = "schema = import \"./sch";
let result = complete(&environment(), "main.dcdl", source, source.len(), false)
.unwrap()
.unwrap();
assert_eq!(result.from, source.len() - 5);
assert!(labels(result).contains(&"./schemas/service.dcdl".into()));
}
#[test]
fn completes_nested_fields_from_imported_source() {
let source = "let schema = import \"./schemas/service.dcdl\"; in schema.Service.";
let result = complete(&environment(), "main.dcdl", source, source.len(), false)
.unwrap()
.unwrap();
assert_eq!(labels(result), ["name", "resources"]);
}
#[test]
fn completes_structured_host_imports() {
let source = "let post = import \"./post.md\"; in post.frontmatter.";
let result = complete(&environment(), "main.dcdl", source, source.len(), false)
.unwrap()
.unwrap();
assert_eq!(labels(result), ["draft", "title"]);
}
#[test]
fn preserves_unicode_import_specifiers() {
let source = "let schema = import \"./資料/service.dcdl\"; in schema.Service.";
let result = complete(&environment(), "main.dcdl", source, source.len(), false)
.unwrap()
.unwrap();
assert_eq!(labels(result), ["label"]);
}
#[test]
fn completes_host_globals_from_the_evaluation_environment() {
let source = "App.config.";
let result = complete(&environment(), "main.dcdl", source, source.len(), false)
.unwrap()
.unwrap();
assert_eq!(labels(result), ["enabled", "port"]);
}
#[test]
fn completes_language_locals_parameters_and_partial_members() {
let source = "let service = { port = 8080; }; in (value: Int) => service.po";
let result = complete(&environment(), "main.dcdl", source, source.len(), false)
.unwrap()
.unwrap();
assert_eq!(result.from, source.len() - 2);
assert_eq!(labels(result), ["port"]);
let source = "let service = { port = 8080; }; in (value: Int) => val";
let result = complete(&environment(), "main.dcdl", source, source.len(), false)
.unwrap()
.unwrap();
let labels = labels(result);
assert!(labels.contains(&"String".into()));
assert!(labels.contains(&"service".into()));
assert!(labels.contains(&"value".into()));
assert!(labels.contains(&"App".into()));
}
#[test]
fn suppresses_completions_in_strings_and_comments() {
let string = "value = \"hello world\"";
assert!(
complete(&environment(), "main.dcdl", string, string.len() - 1, false)
.unwrap()
.is_none()
);
let comment = "value = true; # hello";
assert!(
complete(&environment(), "main.dcdl", comment, comment.len(), false)
.unwrap()
.is_none()
);
}
#[test]
fn empty_environment_remains_supported() {
let result = complete(&EmptyEnvironment, "main.dcdl", "Str", 3, false)
.unwrap()
.unwrap();
assert!(labels(result).contains(&"String".into()));
}
struct EmptyEnvironment;
impl HostEnvironment for EmptyEnvironment {
type Loader = EmptyLoader;
fn create_loader(&self) -> Self::Loader {
EmptyLoader
}
}
fn resolve_virtual(current: &str, specifier: &str) -> String {
let mut parts = current.split('/').collect::<Vec<_>>();
parts.pop();
for part in specifier.split('/') {
match part {
"" | "." => {}
".." => {
parts.pop();
}
part => parts.push(part),
}
}
parts.join("/")
}
}
@@ -1,5 +1,9 @@
use decodal::{Data, Diagnostic, HostEnvironment, Result};
mod completion;
pub use completion::{CompletionItem, CompletionKind, CompletionResult};
/// Semantic tooling backed by the same host environment as production.
pub struct LanguageService<E> {
environment: E,
@@ -59,6 +63,18 @@ impl<E: HostEnvironment> LanguageService<E> {
},
}
}
/// Completes the document using the same host globals and import loader as
/// production evaluation.
pub fn complete(
&self,
key: impl AsRef<str>,
source: &str,
position: usize,
explicit: bool,
) -> Result<Option<CompletionResult>> {
completion::complete(&self.environment, key.as_ref(), source, position, explicit)
}
}
#[derive(Debug, Clone, PartialEq)]