add tokenizer states for text handling; create new token files for quotes and text

This commit is contained in:
Keisuke Hirata 2025-03-26 08:01:50 +09:00
parent 9959e23ab2
commit 8beaa8510d
5 changed files with 7 additions and 2 deletions

View File

@ -5,8 +5,8 @@
(main [] (main []
(std<Image> [alt="hoge" ref="./upload/hoge.jpg"]) (std<Image> [alt="hoge" ref="./upload/hoge.jpg"])
(section [lang=md ] |\ (section [lang=md ] |\
# Hello, world!\ # Hello, world!\
This is a simple example of SIL.\ This is a simple example of SIL.\
|) |)
) )
(footer [] (footer []

View File

@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
} }
} }
}, },
State::InText => match c {
'"' => todo!(),
_ => todo!(),
},
State::InQuote => match c { State::InQuote => match c {
'"' => todo!(), '"' => todo!(),
_ => todo!(), _ => todo!(),
@ -232,6 +236,7 @@ enum State {
Behavior, Behavior,
BehaviorSeparator, BehaviorSeparator,
InText,
InQuote, InQuote,
InDoubleQuote, InDoubleQuote,
} }

View File

View File

0
src/om/tokens/in_text.rs Normal file
View File