Add NodeEnd token and multiple nodes support

src/om/tokenizer.rs

@@ -49,6 +49,10 @@ impl Iterator for Tokenizer {
                         self.state = State::NodeOpen;
                     }
                     ' ' | '\n' | '\t' => {}
+                    ')' => {
+                        self.state = State::Data;
+                        return Some(Token::NodeEnd);
+                    }
                     _ if self.is_eof() => {
                         return Some(Token::EOF);
                     }
@@ -80,9 +84,6 @@ impl Iterator for Tokenizer {
                         });
                         self.buffer.clear();
                     }
-                    ')' => {
-                        self.state = State::Data;
-                    }
                     _ if self.is_eof() => {
                         return Some(Token::EOF);
                     }
@@ -171,7 +172,7 @@ impl Iterator for Tokenizer {
                     x if x.is_ascii_alphanumeric() => {
                         self.buffer.push(c);
                     }
-                    ']' => {
+                    ' ' | '\n' | '\t' | ']' => {
                         if let Some(t) = self.latest.as_mut() {
                             match t {
                                 Token::Define {
@@ -181,13 +182,13 @@ impl Iterator for Tokenizer {
                                     behavior.last_mut().unwrap().set_value(self.buffer.clone());
                                     self.buffer.clear();
                                     self.state = State::Behavior;
+                                    self.reconsume = true;
                                 }
                                 _ => {}
                             }
                         }
-                        return self.latest.take();
                     }
-                    ' ' | '\n' => {}
+
                     _ if self.is_eof() => {
                         return Some(Token::EOF);
                     }
@@ -218,6 +219,7 @@ pub enum Token {
         behavior: Vec<BehaviorItem>,
     },
     Character(char),
+    NodeEnd,
     EOF,
 }
 
@@ -251,4 +253,32 @@ mod tests {
             assert_eq!(Some(e), tokenizer.next());
         }
     }
+
+    #[test]
+    fn test_tokenizer_multiple_nodes() {
+        let input = "(foo [prefix:key=value]) (bar [prefix:key=value])".to_string();
+        let mut tokenizer = Tokenizer::new(input);
+        let expected = [
+            Token::Define {
+                name: "foo".to_string(),
+                behavior: vec![BehaviorItem {
+                    prefix: Some("prefix".to_string()),
+                    key: "key".to_string(),
+                    value: "value".to_string(),
+                }],
+            },
+            Token::Define {
+                name: "bar".to_string(),
+                behavior: vec![BehaviorItem {
+                    prefix: Some("prefix".to_string()),
+                    key: "key".to_string(),
+                    value: "value".to_string(),
+                }],
+            },
+            Token::EOF,
+        ];
+        for e in expected {
+            assert_eq!(Some(e), tokenizer.next());
+        }
+    }
 }