add tokenizer states for text handling; create new token files for quotes and text

This commit is contained in:
Keisuke Hirata 2025-03-26 08:01:50 +09:00
parent 9959e23ab2
commit 8beaa8510d
5 changed files with 7 additions and 2 deletions

View File

@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
} }
} }
}, },
State::InText => match c {
'"' => todo!(),
_ => todo!(),
},
State::InQuote => match c { State::InQuote => match c {
'"' => todo!(), '"' => todo!(),
_ => todo!(), _ => todo!(),
@ -232,6 +236,7 @@ enum State {
Behavior, Behavior,
BehaviorSeparator, BehaviorSeparator,
InText,
InQuote, InQuote,
InDoubleQuote, InDoubleQuote,
} }

View File

View File

0
src/om/tokens/in_text.rs Normal file
View File