From d09ee2a62b4f86feec1136b0393426355264f0b0 Mon Sep 17 00:00:00 2001
From: James Westman <james@jwestman.net>
Date: Sat, 30 Apr 2022 15:28:56 -0500
Subject: [PATCH] tokenizer: Fix operator tokenization

---
 blueprintcompiler/tokenizer.py | 2 +-
 tests/test_tokenizer.py        | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/blueprintcompiler/tokenizer.py b/blueprintcompiler/tokenizer.py
index bc5c72b..be59e2e 100644
--- a/blueprintcompiler/tokenizer.py
+++ b/blueprintcompiler/tokenizer.py
@@ -46,7 +46,7 @@ _tokens = [
     (TokenType.WHITESPACE,      r"\s+"),
     (TokenType.COMMENT,         r"\/\*[\s\S]*?\*\/"),
     (TokenType.COMMENT,         r"\/\/[^\n]*"),
-    (TokenType.OP,              r"[:=\.=\|<>\+\-/\*]+"),
+    (TokenType.OP,              r"<<|>>|=>|::|<|>|:=|\.|\|\||\||\+|\-|\*|=|:|/"),
     (TokenType.PUNCTUATION,     r"\(|\)|\{|\}|;|\[|\]|\,"),
 ]
 _TOKENS = [(type, re.compile(regex)) for (type, regex) in _tokens]
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
index ec39069..1c87e50 100644
--- a/tests/test_tokenizer.py
+++ b/tests/test_tokenizer.py
@@ -46,7 +46,12 @@ class TestTokenizer(unittest.TestCase):
             (TokenType.PUNCTUATION, "}"),
             (TokenType.PUNCTUATION, ";"),
             (TokenType.WHITESPACE, " \n "),
-            (TokenType.OP, "<<+>>*/="),
+            (TokenType.OP, "<<"),
+            (TokenType.OP, "+"),
+            (TokenType.OP, ">>"),
+            (TokenType.OP, "*"),
+            (TokenType.OP, "/"),
+            (TokenType.OP, "="),
             (TokenType.EOF, ""),
         ])