add tokenizer states for text handling; create new token files for quotes and text
This commit is contained in:
parent
9959e23ab2
commit
8beaa8510d
|
@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
State::InText => match c {
|
||||||
|
'"' => todo!(),
|
||||||
|
_ => todo!(),
|
||||||
|
},
|
||||||
State::InQuote => match c {
|
State::InQuote => match c {
|
||||||
'"' => todo!(),
|
'"' => todo!(),
|
||||||
_ => todo!(),
|
_ => todo!(),
|
||||||
|
@ -232,6 +236,7 @@ enum State {
|
||||||
Behavior,
|
Behavior,
|
||||||
BehaviorSeparator,
|
BehaviorSeparator,
|
||||||
|
|
||||||
|
InText,
|
||||||
InQuote,
|
InQuote,
|
||||||
InDoubleQuote,
|
InDoubleQuote,
|
||||||
}
|
}
|
||||||
|
|
0
src/om/tokens/in_double_quote.rs
Normal file
0
src/om/tokens/in_double_quote.rs
Normal file
0
src/om/tokens/in_quote.rs
Normal file
0
src/om/tokens/in_quote.rs
Normal file
0
src/om/tokens/in_text.rs
Normal file
0
src/om/tokens/in_text.rs
Normal file
Loading…
Reference in New Issue
Block a user