feat: expose typed Text View tokens and plain-text conversion
T-28: generate owned semantic tokens directly from Object/Interface structure, including all escaped data and layout. Preserve render APIs through the same token generation and delimiter-free concatenation; preserve input limits, validation order and errors. Document splitting, semantic roles and destination escaping boundaries. Validation: cargo fmt --all -- --check; cargo check --workspace --all-targets; cargo test --workspace (312 passing tests/doctests, including 16 Text View consumer tests and 4 Text View/README doctests); nix develop --command cargo clippy --workspace --all-targets -- -D warnings; RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps; RUSTDOCFLAGS="-D warnings" cargo test -p wip-text-view -p wip-protocol --doc; git diff --check. All passed. Existing golden files unchanged; every prior Text View case now verifies token/plain/direct equality or exact error equality. No existing crates depend on wip-text-view; full workspace validation covers adjacent consumers.
This commit is contained in:
@@ -1,6 +1,32 @@
|
||||
use std::error::Error;
|
||||
use wip_protocol::*;
|
||||
use wip_text_view::{Interface, RenderError, ResourceLimit, render_interface, render_object};
|
||||
use wip_text_view::{
|
||||
Interface, RenderError, ResourceLimit, Token, TokenKind, to_plain_text, tokenize_interface,
|
||||
tokenize_object,
|
||||
};
|
||||
|
||||
// Run every existing golden/escape/error/limit case through both public paths.
|
||||
fn equivalent(tokens: Result<Vec<Token>, RenderError>, direct: &Result<String, RenderError>) {
|
||||
let plain = tokens.map(|tokens| {
|
||||
assert!(tokens.iter().all(|token| !token.text().is_empty()));
|
||||
let joined: String = tokens.iter().map(Token::text).collect();
|
||||
assert_eq!(to_plain_text(&tokens), joined);
|
||||
joined
|
||||
});
|
||||
assert_eq!(&plain, direct);
|
||||
}
|
||||
|
||||
fn render_object(object: &Object) -> Result<String, RenderError> {
|
||||
let direct = wip_text_view::render_object(object);
|
||||
equivalent(tokenize_object(object), &direct);
|
||||
direct
|
||||
}
|
||||
|
||||
fn render_interface(input: &Interface<'_>) -> Result<String, RenderError> {
|
||||
let direct = wip_text_view::render_interface(input);
|
||||
equivalent(tokenize_interface(input), &direct);
|
||||
direct
|
||||
}
|
||||
|
||||
fn reference(scope: &str, name: &str) -> InterfaceReference {
|
||||
InterfaceReference {
|
||||
@@ -803,3 +829,228 @@ fn input_limits_reject_without_truncation_and_allow_boundary() {
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
fn parts(tokens: &[Token]) -> Vec<(TokenKind, &str)> {
|
||||
tokens
|
||||
.iter()
|
||||
.map(|token| (token.kind(), token.text()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_token_splitting_is_explicit_and_owned() {
|
||||
use TokenKind::*;
|
||||
let tokens = {
|
||||
let object = Object {
|
||||
name: "name".into(),
|
||||
description: Some("".into()),
|
||||
interfaces: vec![reference("/開発::x", "issue#read")],
|
||||
r#ref: Some("SECRET".into()),
|
||||
validator: Some(b"SECRET".to_vec()),
|
||||
};
|
||||
tokenize_object(&object).unwrap()
|
||||
}; // Tokens are usable after the input has been dropped.
|
||||
assert_eq!(
|
||||
parts(&tokens),
|
||||
vec![
|
||||
(Keyword, "object"),
|
||||
(Whitespace, " "),
|
||||
(Symbol, "{"),
|
||||
(Newline, "\n"),
|
||||
(Indentation, " "),
|
||||
(Keyword, "name"),
|
||||
(Symbol, ":"),
|
||||
(Whitespace, " "),
|
||||
(String, "\"name\""),
|
||||
(Symbol, ";"),
|
||||
(Newline, "\n"),
|
||||
(Indentation, " "),
|
||||
(Keyword, "description"),
|
||||
(Symbol, ":"),
|
||||
(Whitespace, " "),
|
||||
(String, "\"\""),
|
||||
(Symbol, ";"),
|
||||
(Newline, "\n"),
|
||||
(Indentation, " "),
|
||||
(Keyword, "interfaces"),
|
||||
(Symbol, ":"),
|
||||
(Whitespace, " "),
|
||||
(Symbol, "["),
|
||||
(Reference, "\"\\\"/開発::x\\\"::\\\"issue#read\\\"\""),
|
||||
(Symbol, "]"),
|
||||
(Symbol, ";"),
|
||||
(Newline, "\n"),
|
||||
(Symbol, "}"),
|
||||
(Newline, "\n"),
|
||||
]
|
||||
);
|
||||
assert_eq!(to_plain_text(&[]), "");
|
||||
assert_eq!(to_plain_text(&tokens[0..1]), "object"); // No layout completion.
|
||||
assert!(!to_plain_text(&tokens).contains("SECRET"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_roles_do_not_depend_on_lexical_spelling() {
|
||||
use TokenKind::*;
|
||||
let mut d = descriptor();
|
||||
d.documentation = doc("interface\n<untrusted> 日本語");
|
||||
d.types = vec![
|
||||
TypeDeclaration {
|
||||
documentation: doc("type"),
|
||||
..ty(
|
||||
"string",
|
||||
TypeExpr::Record {
|
||||
fields: vec![FieldDeclaration {
|
||||
documentation: doc("field"),
|
||||
..field(
|
||||
"string",
|
||||
false,
|
||||
TypeExpr::Enum {
|
||||
cases: vec![EnumCase {
|
||||
name: "string".into(),
|
||||
documentation: doc("enum case"),
|
||||
}],
|
||||
},
|
||||
)
|
||||
}],
|
||||
},
|
||||
)
|
||||
},
|
||||
ty(
|
||||
"U",
|
||||
TypeExpr::Union {
|
||||
cases: vec![UnionCase {
|
||||
name: "string".into(),
|
||||
documentation: doc("union case"),
|
||||
payload: Some(TypeExpr::Unit),
|
||||
}],
|
||||
},
|
||||
),
|
||||
];
|
||||
d.operations = vec![OperationDeclaration {
|
||||
documentation: doc("operation"),
|
||||
returns: ReturnDeclaration {
|
||||
documentation: doc("return"),
|
||||
r#type: named("string"),
|
||||
},
|
||||
..operation(
|
||||
"string",
|
||||
vec![ParameterDeclaration {
|
||||
documentation: doc("parameter"),
|
||||
..parameter("string", false, TypeExpr::String)
|
||||
}],
|
||||
TypeExpr::Unit,
|
||||
)
|
||||
}];
|
||||
let r = reference("/", "example");
|
||||
let input = Interface {
|
||||
reference: &r,
|
||||
descriptor: &d,
|
||||
};
|
||||
let before = d.clone();
|
||||
let tokens = tokenize_interface(&input).unwrap();
|
||||
let p = parts(&tokens);
|
||||
for kind in [TypeName, OperationName, ParameterName, FieldName, CaseName] {
|
||||
assert!(p.contains(&(kind, "`string`")), "{kind:?}");
|
||||
}
|
||||
assert!(p.contains(&(Keyword, "string")));
|
||||
assert_eq!(
|
||||
p.iter()
|
||||
.filter(|(k, t)| *k == TypeName && *t == "`string`")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
assert_eq!(p.iter().filter(|(k, _)| *k == CaseName).count(), 2);
|
||||
let docs: Vec<_> = p
|
||||
.iter()
|
||||
.filter(|(k, _)| *k == Documentation)
|
||||
.map(|(_, t)| *t)
|
||||
.collect();
|
||||
assert_eq!(
|
||||
docs,
|
||||
vec![
|
||||
"/// \"interface\\n<untrusted> 日本語\"",
|
||||
"/// \"type\"",
|
||||
"/// \"field\"",
|
||||
"/// \"enum case\"",
|
||||
"/// \"union case\"",
|
||||
"/// \"operation\"",
|
||||
"/// \"parameter\"",
|
||||
"/// \"return\"",
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
p.windows(2)
|
||||
.any(|w| w == [(Indentation, " "), (Symbol, "->")])
|
||||
);
|
||||
assert!(
|
||||
p.windows(3)
|
||||
.any(|w| w == [(FieldName, "`string`"), (Symbol, "?"), (Symbol, ":")])
|
||||
);
|
||||
assert!(p.contains(&(Indentation, " ")));
|
||||
for token in &tokens {
|
||||
match token.kind() {
|
||||
Whitespace => assert_eq!(token.text(), " "),
|
||||
Newline => assert_eq!(token.text(), "\n"),
|
||||
Indentation => {
|
||||
assert!(token.text().bytes().all(|b| b == b' '));
|
||||
assert_eq!(token.text().len() % 2, 0);
|
||||
}
|
||||
Symbol => assert!(
|
||||
["{", "}", "[", "]", "(", ")", ":", ";", ",", "?", "=", "->"]
|
||||
.contains(&token.text())
|
||||
),
|
||||
_ => assert!(!token.text().contains('\n')),
|
||||
}
|
||||
}
|
||||
assert_eq!(to_plain_text(&tokens), render_interface(&input).unwrap());
|
||||
assert_eq!(d, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escaped_identifiers_strings_docs_and_references_stay_atomic() {
|
||||
use TokenKind::*;
|
||||
let name = "日本語::#`\\\"\n\t\u{001b}\u{202e}";
|
||||
let expected = "`日本語::#\\`\\\\\\\"\\n\\t\\u001b\\u202e`";
|
||||
let mut d = descriptor();
|
||||
d.documentation = doc(name);
|
||||
d.types = vec![ty(name, TypeExpr::Unit)];
|
||||
d.operations = vec![operation(
|
||||
name,
|
||||
vec![parameter(name, true, named(name))],
|
||||
named(name),
|
||||
)];
|
||||
let r = reference("/a\"\\`::#", "日本語\"\\`::#\n");
|
||||
let tokens = tokenize_interface(&Interface {
|
||||
reference: &r,
|
||||
descriptor: &d,
|
||||
})
|
||||
.unwrap();
|
||||
let p = parts(&tokens);
|
||||
for kind in [TypeName, OperationName, ParameterName] {
|
||||
assert!(p.contains(&(kind, expected)));
|
||||
}
|
||||
assert!(p.contains(&(
|
||||
Documentation,
|
||||
"/// \"日本語::#`\\\\\\\"\\n\\t\\u001b\\u202e\""
|
||||
)));
|
||||
let reference_text = r#""\"/a\\\"\\\\`::#\"::\"日本語\\\"\\\\`::#\\n\"""#;
|
||||
assert!(p.contains(&(Reference, reference_text)), "{p:?}");
|
||||
let object = Object {
|
||||
name: name.into(),
|
||||
description: Some(name.into()),
|
||||
interfaces: vec![r],
|
||||
r#ref: None,
|
||||
validator: None,
|
||||
};
|
||||
let tokens = tokenize_object(&object).unwrap();
|
||||
assert_eq!(
|
||||
tokens
|
||||
.iter()
|
||||
.filter(|t| t.kind() == String)
|
||||
.map(Token::text)
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["\"日本語::#`\\\\\\\"\\n\\t\\u001b\\u202e\""; 2]
|
||||
);
|
||||
assert!(parts(&tokens).contains(&(Reference, reference_text)));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user