add tokenizer states for text handling; create new token files for quotes and text

This commit is contained in:
Keisuke Hirata 2025-03-26 08:01:50 +09:00
parent 9959e23ab2
commit 8beaa8510d
5 changed files with 7 additions and 2 deletions

View File

@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
}
}
},
State::InText => match c {
'"' => todo!(),
_ => todo!(),
},
State::InQuote => match c {
'"' => todo!(),
_ => todo!(),
@ -232,6 +236,7 @@ enum State {
Behavior,
BehaviorSeparator,
InText,
InQuote,
InDoubleQuote,
}

View File

View File

0
src/om/tokens/in_text.rs Normal file
View File