Archived
add tokenizer states for text handling; create new token files for quotes and text
This commit is contained in:
@@ -212,6 +212,10 @@ impl Iterator for Tokenizer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
State::InText => match c {
|
||||||
|
'"' => todo!(),
|
||||||
|
_ => todo!(),
|
||||||
|
},
|
||||||
State::InQuote => match c {
|
State::InQuote => match c {
|
||||||
'"' => todo!(),
|
'"' => todo!(),
|
||||||
_ => todo!(),
|
_ => todo!(),
|
||||||
@@ -232,6 +236,7 @@ enum State {
|
|||||||
Behavior,
|
Behavior,
|
||||||
BehaviorSeparator,
|
BehaviorSeparator,
|
||||||
|
|
||||||
|
InText,
|
||||||
InQuote,
|
InQuote,
|
||||||
InDoubleQuote,
|
InDoubleQuote,
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user