add tokenizer states for text handling; create new token files for quotes and text

This commit is contained in:
Keisuke Hirata 2025-03-26 08:01:50 +09:00
parent 9959e23ab2
commit 8beaa8510d
5 changed files with 7 additions and 2 deletions

View File

@ -5,8 +5,8 @@
(main []
(std<Image> [alt="hoge" ref="./upload/hoge.jpg"])
(section [lang=md ] |\
# Hello, world!\
This is a simple example of SIL.\
# Hello, world!\
This is a simple example of SIL.\
|)
)
(footer []

View File

@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
}
}
},
State::InText => match c {
'"' => todo!(),
_ => todo!(),
},
State::InQuote => match c {
'"' => todo!(),
_ => todo!(),
@ -232,6 +236,7 @@ enum State {
Behavior,
BehaviorSeparator,
InText,
InQuote,
InDoubleQuote,
}

View File

View File

0
src/om/tokens/in_text.rs Normal file
View File