add tokenizer states for text handling; create new token files for quotes and text
This commit is contained in:
parent
9959e23ab2
commit
8beaa8510d
|
@ -5,8 +5,8 @@
|
||||||
(main []
|
(main []
|
||||||
(std<Image> [alt="hoge" ref="./upload/hoge.jpg"])
|
(std<Image> [alt="hoge" ref="./upload/hoge.jpg"])
|
||||||
(section [lang=md ] |\
|
(section [lang=md ] |\
|
||||||
# Hello, world!\
|
# Hello, world!\
|
||||||
This is a simple example of SIL.\
|
This is a simple example of SIL.\
|
||||||
|)
|
|)
|
||||||
)
|
)
|
||||||
(footer []
|
(footer []
|
||||||
|
|
|
@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
State::InText => match c {
|
||||||
|
'"' => todo!(),
|
||||||
|
_ => todo!(),
|
||||||
|
},
|
||||||
State::InQuote => match c {
|
State::InQuote => match c {
|
||||||
'"' => todo!(),
|
'"' => todo!(),
|
||||||
_ => todo!(),
|
_ => todo!(),
|
||||||
|
@ -232,6 +236,7 @@ enum State {
|
||||||
Behavior,
|
Behavior,
|
||||||
BehaviorSeparator,
|
BehaviorSeparator,
|
||||||
|
|
||||||
|
InText,
|
||||||
InQuote,
|
InQuote,
|
||||||
InDoubleQuote,
|
InDoubleQuote,
|
||||||
}
|
}
|
||||||
|
|
0
src/om/tokens/in_double_quote.rs
Normal file
0
src/om/tokens/in_double_quote.rs
Normal file
0
src/om/tokens/in_quote.rs
Normal file
0
src/om/tokens/in_quote.rs
Normal file
0
src/om/tokens/in_text.rs
Normal file
0
src/om/tokens/in_text.rs
Normal file
Loading…
Reference in New Issue
Block a user