add tokenizer states for text handling; create new token files for quotes and text

This commit is contained in:
2025-03-26 08:01:50 +09:00
parent 9959e23ab2
commit 8beaa8510d
5 changed files with 7 additions and 2 deletions
+2 -2
View File
@@ -5,8 +5,8 @@
(main [] (main []
(std<Image> [alt="hoge" ref="./upload/hoge.jpg"]) (std<Image> [alt="hoge" ref="./upload/hoge.jpg"])
(section [lang=md ] |\ (section [lang=md ] |\
# Hello, world!\ # Hello, world!\
This is a simple example of SIL.\ This is a simple example of SIL.\
|) |)
) )
(footer [] (footer []
+5
View File
@@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
} }
} }
}, },
State::InText => match c {
'"' => todo!(),
_ => todo!(),
},
State::InQuote => match c { State::InQuote => match c {
'"' => todo!(), '"' => todo!(),
_ => todo!(), _ => todo!(),
@@ -232,6 +236,7 @@ enum State {
Behavior, Behavior,
BehaviorSeparator, BehaviorSeparator,
InText,
InQuote, InQuote,
InDoubleQuote, InDoubleQuote,
} }
View File
View File
View File