Add arithmetic expressions

crates/decodal-core/src/lexer.rs

@@ -36,6 +36,10 @@ pub enum TokenKind {
     Equal,
     Arrow,
     Amp,
+    Plus,
+    Minus,
+    Star,
+    Slash,
     SlashSlash,
     Gt,
     Gte,
@@ -67,9 +71,13 @@ impl<'a> Lexer<'a> {
 
     pub fn tokenize(mut self) -> Result<Vec<Token>> {
         let mut tokens = Vec::new();
+        let mut previous = None;
         loop {
-            let token = self.next_token()?;
+            let token = self.next_token(previous.as_ref())?;
             let is_eof = token.kind == TokenKind::Eof;
+            if !is_eof {
+                previous = Some(token.kind.clone());
+            }
             tokens.push(token);
             if is_eof {
                 return Ok(tokens);
@@ -77,7 +85,7 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    fn next_token(&mut self) -> Result<Token> {
+    fn next_token(&mut self, previous: Option<&TokenKind>) -> Result<Token> {
         self.skip_ws_and_comments();
         let start = self.pos;
         let Some(ch) = self.peek() else {
@@ -136,6 +144,18 @@ impl<'a> Lexer<'a> {
                 self.pos += 1;
                 TokenKind::Amp
             }
+            b'+' => {
+                self.pos += 1;
+                TokenKind::Plus
+            }
+            b'-' => {
+                self.pos += 1;
+                TokenKind::Minus
+            }
+            b'*' => {
+                self.pos += 1;
+                TokenKind::Star
+            }
             b'=' => {
                 self.pos += 1;
                 if self.consume(b'>') {
@@ -164,6 +184,8 @@ impl<'a> Lexer<'a> {
                 self.pos += 1;
                 if self.consume(b'/') {
                     TokenKind::SlashSlash
+                } else if previous.is_some_and(token_can_end_expr) {
+                    TokenKind::Slash
                 } else {
                     self.lex_regex(start)?
                 }
@@ -341,6 +363,23 @@ fn is_ident_continue(c: u8) -> bool {
     c.is_ascii_alphanumeric() || c == b'_'
 }
 
+fn token_can_end_expr(kind: &TokenKind) -> bool {
+    matches!(
+        kind,
+        TokenKind::Ident(_)
+            | TokenKind::Int(_)
+            | TokenKind::Float(_)
+            | TokenKind::String(_)
+            | TokenKind::Regex(_)
+            | TokenKind::True
+            | TokenKind::False
+            | TokenKind::Underscore
+            | TokenKind::RBrace
+            | TokenKind::RBracket
+            | TokenKind::RParen
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;