From 1cbe03b49e48dd7e0be28b76fbc3c932f8920a36 Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 8 Oct 2026 18:27:04 +0900 Subject: [PATCH] feat: expose typed Text View tokens and plain-text conversion T-28: generate owned semantic tokens directly from Object/Interface structure, including all escaped data and layout. Preserve render APIs through the same token generation and delimiter-free concatenation; preserve input limits, validation order and errors. Document splitting, semantic roles and destination escaping boundaries. Validation: cargo fmt --all -- --check; cargo check --workspace --all-targets; cargo test --workspace (312 passing tests/doctests, including 16 Text View consumer tests and 4 Text View/README doctests); nix develop --command cargo clippy --workspace --all-targets -- -D warnings; RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps; RUSTDOCFLAGS="-D warnings" cargo test -p wip-text-view -p wip-protocol --doc; git diff --check. All passed. Existing golden files unchanged; every prior Text View case now verifies token/plain/direct equality or exact error equality. No existing crates depend on wip-text-view; full workspace validation covers adjacent consumers. --- crates/wip-text-view/README.md | 90 +++++++- crates/wip-text-view/src/lib.rs | 55 +++++ crates/wip-text-view/src/render.rs | 213 +++++++++++++------ crates/wip-text-view/src/token.rs | 84 ++++++++ crates/wip-text-view/tests/signatures.rs | 253 ++++++++++++++++++++++- 5 files changed, 627 insertions(+), 68 deletions(-) create mode 100644 crates/wip-text-view/src/token.rs diff --git a/crates/wip-text-view/README.md b/crates/wip-text-view/README.md index ad7c037..17d0a68 100644 --- a/crates/wip-text-view/README.md +++ b/crates/wip-text-view/README.md @@ -42,6 +42,84 @@ requests a path and never infers one from name or ref. It requires no descriptor An Interface needs no Object, separate name/scope/context arguments or lookup. There is no document aggregator, configuration or details selector. +## Typed display tokens + +```rust +use wip_protocol::{Object, InterfaceReference, InterfaceDescriptor, INTERFACE_FORMAT_V1}; +use wip_text_view::{ + Interface, TokenKind, tokenize_object, tokenize_interface, to_plain_text, + render_object, render_interface, +}; + +let reference = InterfaceReference { scope: "/".into(), name: "example".into() }; +let object = Object { + name: "item".into(), description: None, interfaces: vec![reference.clone()], + r#ref: None, validator: None, +}; +let object_tokens = tokenize_object(&object)?; +assert_eq!(object_tokens[0].kind(), TokenKind::Keyword); +assert_eq!(object_tokens[0].text(), "object"); +assert_eq!(to_plain_text(&object_tokens), render_object(&object)?); + +let descriptor = InterfaceDescriptor { + format: INTERFACE_FORMAT_V1.into(), documentation: None, + types: vec![], operations: vec![], +}; +let input = Interface { reference: &reference, descriptor: &descriptor }; +let tokens = tokenize_interface(&input)?; +for token in &tokens { + // Custom renderers can select presentation by role without reparsing text. + if token.kind() == TokenKind::Reference { + assert_eq!(token.text(), "\"/::example\""); + } +} +let joined: String = tokens.iter().map(|token| token.text()).collect(); +assert_eq!(to_plain_text(&tokens), joined); +assert_eq!(to_plain_text(&tokens), render_interface(&input)?); +# Ok::<(), wip_text_view::RenderError>(()) +``` + +`tokenize_object` / `tokenize_interface` return `Result, RenderError>`. +They generate **directly from the input structure**, not by lexing a completed +Text View string. Each token owns its text (and can outlive the input) and exposes +`kind()` / `text()`. Existing render APIs use the same token generation followed +by `to_plain_text(&[Token])`; there is no second grammar/escape/layout engine. + +### Kind and splitting contract + +| Kind | One token contains | +| --- | --- | +| `Keyword` | One grammar keyword, Object property label, or primitive type keyword | +| `TypeName` | One type declaration name or Named type reference | +| `OperationName` | One Operation declaration name | +| `ParameterName` / `FieldName` | One parameter / record field name | +| `CaseName` | One enum or union case name | +| `String` | One complete quoted Object name or description | +| `Reference` | One complete quoted absolute Interface reference, with both quoting layers | +| `Documentation` | Entire `/// "summary"` content, including marker and internal space, excluding indentation/LF | +| `Symbol` | One punctuation mark, except the indivisible two-character `->` arrow | +| `Whitespace` | One inter-token ASCII space | +| `Newline` | One LF, including the final LF | +| `Indentation` | All leading spaces of a non-root line, two spaces per depth level | + +Quoted identifiers retain their semantic role even when named like keywords. +Quotes, backticks and all WIP escapes are part of the data token, never separate +symbols. Host spaces inside quoted data remain in that token. Adjacent symbols +are separate; no empty tokens or zero-depth indentation tokens are emitted. +`TokenKind` is non-exhaustive so consumers should handle future roles with a +fallback. The API does not distinguish type declaration names from Named uses, +or enum cases from union cases; both share the documented role. + +All layout is included. Concatenate token text **without separators**, not with +`join(" ")`. `to_plain_text` is only that concatenation: no interpretation, +validation, quoting, escaping or layout completion. An empty slice yields `""`; +a filtered/reordered slice is allowed but need not be a complete signature. + +Token text is escaped for **WIP Text View**, not for HTML or other destinations. +Custom renderers must escape for their destination and treat Host text as data, +not markup or instructions. This crate supplies no HTML, ANSI, color or theme +renderer/adapter and no external text lexer/parser or AST. + ## Output and validation - UTF-8 plain text with LF, two-space indentation and a final LF, without ANSI, @@ -63,8 +141,8 @@ There is no document aggregator, configuration or details selector. - Object ref/validator, children, domain data and Interface scope_ref/validator are not signature content. No annotations are inferred from operation names. -Both functions return `Result` and never return partial -signatures. Resource checks happen first, followed by existing Protocol validators: +The render functions return `Result`; token generation +returns `Result, RenderError>`. Neither returns partial signatures. Resource checks happen first, followed by existing Protocol validators: Object shape and reference uniqueness, or Interface reference shape plus descriptor structure, duplicate names, unknown Named references and cycles. Descriptor format is checked after semantics: valid but unsupported formats return @@ -94,7 +172,13 @@ the spec's Object, issue.read, CloseResult, search/return documentation and identifier examples (with permitted multiline layout), plus nested documentation. The suite checks all TypeExpr variants/positions, ordering and empty distinctions, keyword and Unicode names, both reference quoting layers, all required control -escapes, input immutability, metadata exclusion and typed rejection cases. +escapes, input immutability, metadata exclusion and typed rejection cases. Every +existing case compares the render result (including exact errors) with token +text concatenation and `to_plain_text`. Additional tests assert complete Object +token splitting, all semantic roles, keyword collisions, atomic escaped data, +all documentation positions, layout tokens and ownership beyond input lifetime. +Input budgets bound token counts per node, indentation depth and escape expansion; +token generation checks them before any rendering/validation traversal. ```console cargo test -p wip-text-view diff --git a/crates/wip-text-view/src/lib.rs b/crates/wip-text-view/src/lib.rs index 64c184a..ae8571d 100644 --- a/crates/wip-text-view/src/lib.rs +++ b/crates/wip-text-view/src/lib.rs @@ -45,11 +45,42 @@ //! assert_eq!(render_interface(&interface)?, "interface \"/::example\" {\n}\n"); //! # Ok::<(), wip_text_view::RenderError>(()) //! ``` +//! +//! # Typed display tokens +//! +//! [`tokenize_object`] and [`tokenize_interface`] traverse the input structure +//! directly and return owned [`Token`] sequences, including all layout. Existing +//! render APIs use this same generation path followed by [`to_plain_text`]. The +//! input limits above also bound token allocation: tokens per node, indentation +//! depth and escape expansion are bounded; no partial signature is returned. +//! Token text is WIP-escaped, **not HTML-escaped** or escaped for other destinations. +//! See [`Token`] for the semantic splitting contract; no renderer/theme is supplied. +//! +//! ``` +//! use wip_protocol::{InterfaceReference, InterfaceDescriptor, INTERFACE_FORMAT_V1}; +//! use wip_text_view::{Interface, TokenKind, tokenize_interface, to_plain_text, render_interface}; +//! let reference = InterfaceReference { scope: "/".into(), name: "example".into() }; +//! let descriptor = InterfaceDescriptor { +//! format: INTERFACE_FORMAT_V1.into(), documentation: None, +//! types: vec![], operations: vec![], +//! }; +//! let input = Interface { reference: &reference, descriptor: &descriptor }; +//! let tokens = tokenize_interface(&input)?; +//! assert_eq!(tokens[0].kind(), TokenKind::Keyword); +//! assert_eq!(tokens[0].text(), "interface"); +//! let concatenated: String = tokens.iter().map(|token| token.text()).collect(); +//! assert_eq!(to_plain_text(&tokens), concatenated); +//! assert_eq!(to_plain_text(&tokens), render_interface(&input)?); +//! # Ok::<(), wip_text_view::RenderError>(()) +//! ``` #![deny(missing_docs)] mod limits; mod render; +mod token; + +pub use token::{Token, TokenKind, to_plain_text}; use std::{error::Error, fmt}; use wip_protocol::{ @@ -135,6 +166,15 @@ impl Error for RenderError { /// distinct; ref and validator bytes are ignored for display. Input is unmodified. /// See [crate documentation](crate) for limits and caller responsibilities. pub fn render_object(object: &Object) -> Result { + tokenize_object(object).map(|tokens| to_plain_text(&tokens)) +} + +/// Generates the complete owned display token sequence for one Object signature. +/// +/// Uses the same validation, limits, metadata exclusion and caller responsibilities +/// as [`render_object`]. No partial sequence is returned on error. Input is not +/// modified. See [`Token`] and [`TokenKind`] for semantic roles and splitting. +pub fn tokenize_object(object: &Object) -> Result, RenderError> { limits::object(object)?; object.validate().map_err(RenderError::InvalidObject)?; Ok(render::object(object)) @@ -148,6 +188,16 @@ pub fn render_object(object: &Object) -> Result { /// documentation summaries are shown, without interpreting Host text. /// Input is unmodified. See [crate documentation](crate) for safety limits. pub fn render_interface(interface: &Interface<'_>) -> Result { + tokenize_interface(interface).map(|tokens| to_plain_text(&tokens)) +} + +/// Generates the complete owned display token sequence for one Interface signature. +/// +/// Uses the same validation, limits, format support and summary-only documentation +/// as [`render_interface`]. No partial sequence is returned on error. Input is not +/// modified. See [`Token`] and [`TokenKind`] for semantic roles and splitting. +/// This traverses structured declarations directly, not a rendered-text lexer. +pub fn tokenize_interface(interface: &Interface<'_>) -> Result, RenderError> { limits::interface(interface)?; interface .reference @@ -164,3 +214,8 @@ pub fn render_interface(interface: &Interface<'_>) -> Result String { @@ -90,159 +94,240 @@ fn quote(value: &str, delimiter: char) -> String { output } -pub(crate) fn object(object: &Object) -> String { - let mut output = format!("object {{\n name: {};\n", quote(&object.name, '"')); +pub(crate) fn object(object: &Object) -> Vec { + let mut writer = Writer(Vec::new()); + writer.word(Keyword, "object"); + writer.space(); + writer.symbol("{"); + writer.newline(); + writer.indent(1); + writer.word(Keyword, "name"); + writer.symbol(":"); + writer.space(); + writer.word(TokenKind::String, quote(&object.name, '"')); + writer.symbol(";"); + writer.newline(); if let Some(description) = &object.description { - writeln!(output, " description: {};", quote(description, '"')).expect("String write"); + writer.indent(1); + writer.word(Keyword, "description"); + writer.symbol(":"); + writer.space(); + writer.word(TokenKind::String, quote(description, '"')); + writer.symbol(";"); + writer.newline(); } - output.push_str(" interfaces: ["); + writer.indent(1); + writer.word(Keyword, "interfaces"); + writer.symbol(":"); + writer.space(); + writer.symbol("["); for (index, interface) in object.interfaces.iter().enumerate() { if index > 0 { - output.push_str(", "); + writer.symbol(","); + writer.space(); } - output.push_str("e(&reference(interface), '"')); + writer.word(Reference, quote(&reference(interface), '"')); } - output.push_str("];\n}\n"); - output + writer.symbol("]"); + writer.symbol(";"); + writer.newline(); + writer.symbol("}"); + writer.newline(); + writer.0 } -pub(crate) fn interface(interface: &Interface<'_>) -> String { - let mut writer = Writer(String::new()); +pub(crate) fn interface(interface: &Interface<'_>) -> Vec { + let mut writer = Writer(Vec::new()); writer.doc(&interface.descriptor.documentation, 0); - writeln!( - writer.0, - "interface {} {{", - quote(&reference(interface.reference), '"') - ) - .expect("String write"); + writer.word(Keyword, "interface"); + writer.space(); + writer.word(Reference, quote(&reference(interface.reference), '"')); + writer.space(); + writer.symbol("{"); + writer.newline(); for declaration in &interface.descriptor.types { writer.doc(&declaration.documentation, 1); writer.indent(1); - write!(writer.0, "type {} = ", identifier(&declaration.name)).expect("String write"); + writer.word(Keyword, "type"); + writer.space(); + writer.word(TypeName, identifier(&declaration.name)); + writer.space(); + writer.symbol("="); + writer.space(); writer.expr(&declaration.definition, 1); - writer.0.push_str(";\n"); + writer.symbol(";"); + writer.newline(); } for operation in &interface.descriptor.operations { writer.operation(operation); } - writer.0.push_str("}\n"); + writer.symbol("}"); + writer.newline(); writer.0 } -struct Writer(String); +struct Writer(Vec); impl Writer { + fn word(&mut self, kind: TokenKind, text: impl Into) { + self.0.push(Token { + kind, + text: text.into(), + }); + } + + fn symbol(&mut self, text: &str) { + self.word(Symbol, text); + } + + fn space(&mut self) { + self.word(Whitespace, " "); + } + + fn newline(&mut self) { + self.word(Newline, "\n"); + } + fn indent(&mut self, depth: usize) { - for _ in 0..depth { - self.0.push_str(" "); + if depth > 0 { + self.word(Indentation, " ".repeat(depth)); } } fn doc(&mut self, doc: &Option, depth: usize) { if let Some(doc) = doc { self.indent(depth); - writeln!(self.0, "/// {}", quote(&doc.summary, '"')).expect("String write"); + self.word( + TokenKind::Documentation, + format!("/// {}", quote(&doc.summary, '"')), + ); + self.newline(); } } - fn member(&mut self, name: &str, required: bool, ty: &TypeExpr, depth: usize) { + fn member(&mut self, kind: TokenKind, name: &str, required: bool, ty: &TypeExpr, depth: usize) { self.indent(depth); - self.0.push_str(&identifier(name)); + self.word(kind, identifier(name)); if !required { - self.0.push('?'); + self.symbol("?"); } - self.0.push_str(": "); + self.symbol(":"); + self.space(); self.expr(ty, depth); - self.0.push_str(",\n"); + self.symbol(","); + self.newline(); } fn operation(&mut self, operation: &OperationDeclaration) { self.doc(&operation.documentation, 1); self.indent(1); - write!(self.0, "operation {}(", identifier(&operation.name)).expect("String write"); + self.word(Keyword, "operation"); + self.space(); + self.word(OperationName, identifier(&operation.name)); + self.symbol("("); if !operation.parameters.is_empty() { - self.0.push('\n'); + self.newline(); for parameter in &operation.parameters { self.doc(¶meter.documentation, 2); - self.member(¶meter.name, parameter.required, ¶meter.r#type, 2); + self.member( + ParameterName, + ¶meter.name, + parameter.required, + ¶meter.r#type, + 2, + ); } self.indent(1); } - self.0.push(')'); + self.symbol(")"); if operation.returns.documentation.is_some() { - self.0.push('\n'); + self.newline(); self.doc(&operation.returns.documentation, 1); self.indent(1); - self.0.push_str("-> "); } else { - self.0.push_str(" -> "); + self.space(); } + self.symbol("->"); + self.space(); self.expr(&operation.returns.r#type, 1); - self.0.push_str(";\n"); + self.symbol(";"); + self.newline(); } // Starts at the current cursor. Composite closing braces align with the // declaration/field/parameter containing this expression, not its width. fn expr(&mut self, expr: &TypeExpr, depth: usize) { match expr { - TypeExpr::Unit => self.0.push_str("unit"), - TypeExpr::Boolean => self.0.push_str("boolean"), - TypeExpr::Integer => self.0.push_str("integer"), - TypeExpr::Number => self.0.push_str("number"), - TypeExpr::String => self.0.push_str("string"), - TypeExpr::Bytes => self.0.push_str("bytes"), - TypeExpr::Json => self.0.push_str("json"), - TypeExpr::Entry => self.0.push_str("entry"), - TypeExpr::Named { name } => self.0.push_str(&identifier(name)), + TypeExpr::Unit => self.word(Keyword, "unit"), + TypeExpr::Boolean => self.word(Keyword, "boolean"), + TypeExpr::Integer => self.word(Keyword, "integer"), + TypeExpr::Number => self.word(Keyword, "number"), + TypeExpr::String => self.word(Keyword, "string"), + TypeExpr::Bytes => self.word(Keyword, "bytes"), + TypeExpr::Json => self.word(Keyword, "json"), + TypeExpr::Entry => self.word(Keyword, "entry"), + TypeExpr::Named { name } => self.word(TypeName, identifier(name)), TypeExpr::List { items } => { - self.0.push('['); + self.symbol("["); self.expr(items, depth); - self.0.push(']'); + self.symbol("]"); } TypeExpr::Record { fields } => { - self.0.push('{'); + self.symbol("{"); if !fields.is_empty() { - self.0.push('\n'); + self.newline(); for field in fields { self.doc(&field.documentation, depth + 1); - self.member(&field.name, field.required, &field.r#type, depth + 1); + self.member( + FieldName, + &field.name, + field.required, + &field.r#type, + depth + 1, + ); } self.indent(depth); } - self.0.push('}'); + self.symbol("}"); } TypeExpr::Enum { cases } => { - self.0.push_str("enum {"); + self.word(Keyword, "enum"); + self.space(); + self.symbol("{"); if !cases.is_empty() { - self.0.push('\n'); + self.newline(); for case in cases { self.doc(&case.documentation, depth + 1); self.indent(depth + 1); - self.0.push_str(&identifier(&case.name)); - self.0.push_str(",\n"); + self.word(CaseName, identifier(&case.name)); + self.symbol(","); + self.newline(); } self.indent(depth); } - self.0.push('}'); + self.symbol("}"); } TypeExpr::Union { cases } => { - self.0.push_str("union {"); + self.word(Keyword, "union"); + self.space(); + self.symbol("{"); if !cases.is_empty() { - self.0.push('\n'); + self.newline(); for case in cases { self.doc(&case.documentation, depth + 1); self.indent(depth + 1); - self.0.push_str(&identifier(&case.name)); + self.word(CaseName, identifier(&case.name)); if let Some(payload) = &case.payload { - self.0.push('('); + self.symbol("("); self.expr(payload, depth + 1); - self.0.push(')'); + self.symbol(")"); } - self.0.push_str(",\n"); + self.symbol(","); + self.newline(); } self.indent(depth); } - self.0.push('}'); + self.symbol("}"); } } } diff --git a/crates/wip-text-view/src/token.rs b/crates/wip-text-view/src/token.rs new file mode 100644 index 0000000..48d4004 --- /dev/null +++ b/crates/wip-text-view/src/token.rs @@ -0,0 +1,84 @@ +//! Owned display data, independent of the Protocol wire model. + +/// Semantic display role assigned from the input structure, not from text parsing. +/// +/// Identifiers retain their role even when quoted or identical to a keyword. +/// Type declarations and Named expressions both use [`Self::TypeName`]. Enum and +/// union cases both use [`Self::CaseName`]. See [`Token`] for splitting rules. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum TokenKind { + /// Grammar keyword, including Object property labels and primitive types. + Keyword, + /// Named type declaration or reference within the same Interface. + TypeName, + /// Operation declaration name (not an invocation target). + OperationName, + /// Operation parameter name. + ParameterName, + /// Record field name. + FieldName, + /// Enum or union case name. + CaseName, + /// Quoted Object name or description. + String, + /// Entire quoted absolute Interface reference, including both quoting layers. + Reference, + /// Entire `/// "summary"` line content, excluding indentation and LF. + Documentation, + /// One punctuation mark, or the complete two-character return arrow `->`. + Symbol, + /// One inter-token ASCII space, never indentation or text inside quotes. + Whitespace, + /// One LF (`\n`), including the final signature LF. + Newline, + /// All leading spaces on a non-root line: two ASCII spaces per depth level. + Indentation, +} + +/// One owned, already escaped fragment of a Text View signature. +/// +/// Each keyword, identifier, quoted string and complete quoted reference is one +/// token, including its delimiters and escapes. Documentation is one token for +/// the `///` marker, space and quoted summary together. Symbols are individual +/// punctuation marks except `->`, which is indivisible. Spaces between tokens, +/// each LF, and each nonzero line indentation have separate tokens. No empty +/// tokens are emitted; an empty value still has its quotes. Adjacent symbols are +/// not merged. Host whitespace inside quoted data stays inside that data token. +/// +/// The generation APIs emit all layout: concatenating [`Self::text`] in order, +/// without separators, produces exactly the existing plain-text signature. +/// Tokens own their text and can outlive the input; they are display data, not +/// an AST, lexer output, Protocol serialization or identity/call-target data. +/// +/// Text is escaped for **WIP Text View**, not for HTML or any other destination. +/// Custom renderers must apply destination-specific escaping and must not treat +/// Host documentation as markup or instructions. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Token { + pub(crate) kind: TokenKind, + pub(crate) text: String, +} + +impl Token { + /// Returns the semantic display role, without reparsing the output text. + pub fn kind(&self) -> TokenKind { + self.kind + } + + /// Returns the exact output fragment, with WIP quoting and escapes applied. + pub fn text(&self) -> &str { + &self.text + } +} + +/// Concatenates token text in order, with no separators or further processing. +/// +/// This performs no grammar interpretation, validation, quoting, escaping or +/// layout completion. An empty slice yields an empty string. Filtering or +/// reordering tokens is allowed, but need not produce a complete signature. +/// Object/Interface generation limits are checked by [`crate::tokenize_object`] +/// and [`crate::tokenize_interface`], not by this concatenation function. +pub fn to_plain_text(tokens: &[Token]) -> String { + tokens.iter().map(Token::text).collect() +} diff --git a/crates/wip-text-view/tests/signatures.rs b/crates/wip-text-view/tests/signatures.rs index 9c85ca3..3cae3c5 100644 --- a/crates/wip-text-view/tests/signatures.rs +++ b/crates/wip-text-view/tests/signatures.rs @@ -1,6 +1,32 @@ use std::error::Error; use wip_protocol::*; -use wip_text_view::{Interface, RenderError, ResourceLimit, render_interface, render_object}; +use wip_text_view::{ + Interface, RenderError, ResourceLimit, Token, TokenKind, to_plain_text, tokenize_interface, + tokenize_object, +}; + +// Run every existing golden/escape/error/limit case through both public paths. +fn equivalent(tokens: Result, RenderError>, direct: &Result) { + let plain = tokens.map(|tokens| { + assert!(tokens.iter().all(|token| !token.text().is_empty())); + let joined: String = tokens.iter().map(Token::text).collect(); + assert_eq!(to_plain_text(&tokens), joined); + joined + }); + assert_eq!(&plain, direct); +} + +fn render_object(object: &Object) -> Result { + let direct = wip_text_view::render_object(object); + equivalent(tokenize_object(object), &direct); + direct +} + +fn render_interface(input: &Interface<'_>) -> Result { + let direct = wip_text_view::render_interface(input); + equivalent(tokenize_interface(input), &direct); + direct +} fn reference(scope: &str, name: &str) -> InterfaceReference { InterfaceReference { @@ -803,3 +829,228 @@ fn input_limits_reject_without_truncation_and_allow_boundary() { }) ); } + +fn parts(tokens: &[Token]) -> Vec<(TokenKind, &str)> { + tokens + .iter() + .map(|token| (token.kind(), token.text())) + .collect() +} + +#[test] +fn object_token_splitting_is_explicit_and_owned() { + use TokenKind::*; + let tokens = { + let object = Object { + name: "name".into(), + description: Some("".into()), + interfaces: vec![reference("/開発::x", "issue#read")], + r#ref: Some("SECRET".into()), + validator: Some(b"SECRET".to_vec()), + }; + tokenize_object(&object).unwrap() + }; // Tokens are usable after the input has been dropped. + assert_eq!( + parts(&tokens), + vec![ + (Keyword, "object"), + (Whitespace, " "), + (Symbol, "{"), + (Newline, "\n"), + (Indentation, " "), + (Keyword, "name"), + (Symbol, ":"), + (Whitespace, " "), + (String, "\"name\""), + (Symbol, ";"), + (Newline, "\n"), + (Indentation, " "), + (Keyword, "description"), + (Symbol, ":"), + (Whitespace, " "), + (String, "\"\""), + (Symbol, ";"), + (Newline, "\n"), + (Indentation, " "), + (Keyword, "interfaces"), + (Symbol, ":"), + (Whitespace, " "), + (Symbol, "["), + (Reference, "\"\\\"/開発::x\\\"::\\\"issue#read\\\"\""), + (Symbol, "]"), + (Symbol, ";"), + (Newline, "\n"), + (Symbol, "}"), + (Newline, "\n"), + ] + ); + assert_eq!(to_plain_text(&[]), ""); + assert_eq!(to_plain_text(&tokens[0..1]), "object"); // No layout completion. + assert!(!to_plain_text(&tokens).contains("SECRET")); +} + +#[test] +fn semantic_roles_do_not_depend_on_lexical_spelling() { + use TokenKind::*; + let mut d = descriptor(); + d.documentation = doc("interface\n 日本語"); + d.types = vec![ + TypeDeclaration { + documentation: doc("type"), + ..ty( + "string", + TypeExpr::Record { + fields: vec![FieldDeclaration { + documentation: doc("field"), + ..field( + "string", + false, + TypeExpr::Enum { + cases: vec![EnumCase { + name: "string".into(), + documentation: doc("enum case"), + }], + }, + ) + }], + }, + ) + }, + ty( + "U", + TypeExpr::Union { + cases: vec![UnionCase { + name: "string".into(), + documentation: doc("union case"), + payload: Some(TypeExpr::Unit), + }], + }, + ), + ]; + d.operations = vec![OperationDeclaration { + documentation: doc("operation"), + returns: ReturnDeclaration { + documentation: doc("return"), + r#type: named("string"), + }, + ..operation( + "string", + vec![ParameterDeclaration { + documentation: doc("parameter"), + ..parameter("string", false, TypeExpr::String) + }], + TypeExpr::Unit, + ) + }]; + let r = reference("/", "example"); + let input = Interface { + reference: &r, + descriptor: &d, + }; + let before = d.clone(); + let tokens = tokenize_interface(&input).unwrap(); + let p = parts(&tokens); + for kind in [TypeName, OperationName, ParameterName, FieldName, CaseName] { + assert!(p.contains(&(kind, "`string`")), "{kind:?}"); + } + assert!(p.contains(&(Keyword, "string"))); + assert_eq!( + p.iter() + .filter(|(k, t)| *k == TypeName && *t == "`string`") + .count(), + 2 + ); + assert_eq!(p.iter().filter(|(k, _)| *k == CaseName).count(), 2); + let docs: Vec<_> = p + .iter() + .filter(|(k, _)| *k == Documentation) + .map(|(_, t)| *t) + .collect(); + assert_eq!( + docs, + vec![ + "/// \"interface\\n 日本語\"", + "/// \"type\"", + "/// \"field\"", + "/// \"enum case\"", + "/// \"union case\"", + "/// \"operation\"", + "/// \"parameter\"", + "/// \"return\"", + ] + ); + assert!( + p.windows(2) + .any(|w| w == [(Indentation, " "), (Symbol, "->")]) + ); + assert!( + p.windows(3) + .any(|w| w == [(FieldName, "`string`"), (Symbol, "?"), (Symbol, ":")]) + ); + assert!(p.contains(&(Indentation, " "))); + for token in &tokens { + match token.kind() { + Whitespace => assert_eq!(token.text(), " "), + Newline => assert_eq!(token.text(), "\n"), + Indentation => { + assert!(token.text().bytes().all(|b| b == b' ')); + assert_eq!(token.text().len() % 2, 0); + } + Symbol => assert!( + ["{", "}", "[", "]", "(", ")", ":", ";", ",", "?", "=", "->"] + .contains(&token.text()) + ), + _ => assert!(!token.text().contains('\n')), + } + } + assert_eq!(to_plain_text(&tokens), render_interface(&input).unwrap()); + assert_eq!(d, before); +} + +#[test] +fn escaped_identifiers_strings_docs_and_references_stay_atomic() { + use TokenKind::*; + let name = "日本語::#`\\\"\n\t\u{001b}\u{202e}"; + let expected = "`日本語::#\\`\\\\\\\"\\n\\t\\u001b\\u202e`"; + let mut d = descriptor(); + d.documentation = doc(name); + d.types = vec![ty(name, TypeExpr::Unit)]; + d.operations = vec![operation( + name, + vec![parameter(name, true, named(name))], + named(name), + )]; + let r = reference("/a\"\\`::#", "日本語\"\\`::#\n"); + let tokens = tokenize_interface(&Interface { + reference: &r, + descriptor: &d, + }) + .unwrap(); + let p = parts(&tokens); + for kind in [TypeName, OperationName, ParameterName] { + assert!(p.contains(&(kind, expected))); + } + assert!(p.contains(&( + Documentation, + "/// \"日本語::#`\\\\\\\"\\n\\t\\u001b\\u202e\"" + ))); + let reference_text = r#""\"/a\\\"\\\\`::#\"::\"日本語\\\"\\\\`::#\\n\"""#; + assert!(p.contains(&(Reference, reference_text)), "{p:?}"); + let object = Object { + name: name.into(), + description: Some(name.into()), + interfaces: vec![r], + r#ref: None, + validator: None, + }; + let tokens = tokenize_object(&object).unwrap(); + assert_eq!( + tokens + .iter() + .filter(|t| t.kind() == String) + .map(Token::text) + .collect::>(), + vec!["\"日本語::#`\\\\\\\"\\n\\t\\u001b\\u202e\""; 2] + ); + assert!(parts(&tokens).contains(&(Reference, reference_text))); +}