add tokenizer states for text handling; create new token files for quotes and text
This commit is contained in:
parent
9959e23ab2
commit
8beaa8510d
|
@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
|
|||
}
|
||||
}
|
||||
},
|
||||
State::InText => match c {
|
||||
'"' => todo!(),
|
||||
_ => todo!(),
|
||||
},
|
||||
State::InQuote => match c {
|
||||
'"' => todo!(),
|
||||
_ => todo!(),
|
||||
|
@ -232,6 +236,7 @@ enum State {
|
|||
Behavior,
|
||||
BehaviorSeparator,
|
||||
|
||||
InText,
|
||||
InQuote,
|
||||
InDoubleQuote,
|
||||
}
|
||||
|
|
0
src/om/tokens/in_double_quote.rs
Normal file
0
src/om/tokens/in_double_quote.rs
Normal file
0
src/om/tokens/in_quote.rs
Normal file
0
src/om/tokens/in_quote.rs
Normal file
0
src/om/tokens/in_text.rs
Normal file
0
src/om/tokens/in_text.rs
Normal file
Loading…
Reference in New Issue
Block a user