add tokenizer states for text handling; create new token files for quotes and text
This commit is contained in:
parent
9959e23ab2
commit
8beaa8510d
|
@ -5,8 +5,8 @@
|
|||
(main []
|
||||
(std<Image> [alt="hoge" ref="./upload/hoge.jpg"])
|
||||
(section [lang=md ] |\
|
||||
# Hello, world!\
|
||||
This is a simple example of SIL.\
|
||||
# Hello, world!\
|
||||
This is a simple example of SIL.\
|
||||
|)
|
||||
)
|
||||
(footer []
|
||||
|
|
|
@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
|
|||
}
|
||||
}
|
||||
},
|
||||
State::InText => match c {
|
||||
'"' => todo!(),
|
||||
_ => todo!(),
|
||||
},
|
||||
State::InQuote => match c {
|
||||
'"' => todo!(),
|
||||
_ => todo!(),
|
||||
|
@ -232,6 +236,7 @@ enum State {
|
|||
Behavior,
|
||||
BehaviorSeparator,
|
||||
|
||||
InText,
|
||||
InQuote,
|
||||
InDoubleQuote,
|
||||
}
|
||||
|
|
0
src/om/tokens/in_double_quote.rs
Normal file
0
src/om/tokens/in_double_quote.rs
Normal file
0
src/om/tokens/in_quote.rs
Normal file
0
src/om/tokens/in_quote.rs
Normal file
0
src/om/tokens/in_text.rs
Normal file
0
src/om/tokens/in_text.rs
Normal file
Loading…
Reference in New Issue
Block a user