From 8beaa8510d76c2b5e0715718ce975d1c249f8d39 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 26 Mar 2025 08:01:50 +0900 Subject: [PATCH] add tokenizer states for text handling; create new token files for quotes and text --- resources/test/example_1.sil | 4 ++-- src/om/tokenizer.rs | 5 +++++ src/om/tokens/in_double_quote.rs | 0 src/om/tokens/in_quote.rs | 0 src/om/tokens/in_text.rs | 0 5 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 src/om/tokens/in_double_quote.rs create mode 100644 src/om/tokens/in_quote.rs create mode 100644 src/om/tokens/in_text.rs diff --git a/resources/test/example_1.sil b/resources/test/example_1.sil index f7f6f51..909a990 100644 --- a/resources/test/example_1.sil +++ b/resources/test/example_1.sil @@ -5,8 +5,8 @@ (main [] (std [alt="hoge" ref="./upload/hoge.jpg"]) (section [lang=md ] |\ - # Hello, world!\ - This is a simple example of SIL.\ + # Hello, world!\ + This is a simple example of SIL.\ |) ) (footer [] diff --git a/src/om/tokenizer.rs b/src/om/tokenizer.rs index 52ed73e..a3f71be 100644 --- a/src/om/tokenizer.rs +++ b/src/om/tokenizer.rs @@ -212,6 +212,10 @@ impl Iterator for Tokenizer { } } }, + State::InText => match c { + '"' => todo!(), + _ => todo!(), + }, State::InQuote => match c { '"' => todo!(), _ => todo!(), @@ -232,6 +236,7 @@ enum State { Behavior, BehaviorSeparator, + InText, InQuote, InDoubleQuote, } diff --git a/src/om/tokens/in_double_quote.rs b/src/om/tokens/in_double_quote.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/om/tokens/in_quote.rs b/src/om/tokens/in_quote.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/om/tokens/in_text.rs b/src/om/tokens/in_text.rs new file mode 100644 index 0000000..e69de29