Siklos/blueprint-compiler
1
0
Fork 0
mirror of https://gitlab.gnome.org/jwestman/blueprint-compiler.git synced 2025-05-04 15:59:08 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

tokenizer: Fix operator tokenization

Browse source
This commit is contained in:
James Westman 2022-04-30 15:28:56 -05:00
parent 0da0c9399f
commit d09ee2a62b
No known key found for this signature in database
GPG key ID: CE2DBA0ADB654EA6
2 changed files with 7 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
blueprintcompiler/tokenizer.py
View file

@ -46,7 +46,7 @@ _tokens = [
(TokenType.WHITESPACE, r"\s+"), (TokenType.WHITESPACE, r"\s+"),
(TokenType.COMMENT, r"\/\*[\s\S]*?\*\/"), (TokenType.COMMENT, r"\/\*[\s\S]*?\*\/"),
(TokenType.COMMENT, r"\/\/[^\n]*"), (TokenType.COMMENT, r"\/\/[^\n]*"),
(TokenType.OP, r"[:=\.=\|<>\+\-/\*]+"), (TokenType.OP, r"<<|>>|=>|::|<|>|:=|\.|\|\||\||\+|\-|\*|=|:|/"),
(TokenType.PUNCTUATION, r"\(|\)|\{|\}|;|\[|\]|\,"), (TokenType.PUNCTUATION, r"\(|\)|\{|\}|;|\[|\]|\,"),
] ]
_TOKENS = [(type, re.compile(regex)) for (type, regex) in _tokens] _TOKENS = [(type, re.compile(regex)) for (type, regex) in _tokens]

7
tests/test_tokenizer.py
View file

@ -46,7 +46,12 @@ class TestTokenizer(unittest.TestCase):
(TokenType.PUNCTUATION, "}"), (TokenType.PUNCTUATION, "}"),
(TokenType.PUNCTUATION, ";"), (TokenType.PUNCTUATION, ";"),
(TokenType.WHITESPACE, " \n "), (TokenType.WHITESPACE, " \n "),
(TokenType.OP, "<<+>>*/="), (TokenType.OP, "<<"),
(TokenType.OP, "+"),
(TokenType.OP, ">>"),
(TokenType.OP, "*"),
(TokenType.OP, "/"),
(TokenType.OP, "="),
(TokenType.EOF, ""), (TokenType.EOF, ""),
]) ])