Siklos/blueprint-compiler
1
0
Fork 0
mirror of https://gitlab.gnome.org/jwestman/blueprint-compiler.git synced 2025-05-04 15:59:08 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add tokenizer tests

Browse source
This commit is contained in:
James Westman 2021-10-18 12:37:05 -05:00
parent cf2f5215c8
commit d98b7d5891
No known key found for this signature in database
GPG key ID: CE2DBA0ADB654EA6
4 changed files with 79 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
gtkblueprinttool/tokenizer.py
View file

@ -55,12 +55,12 @@ _TOKENS = [
(TokenType.OPEN_BLOCK, r"\{"),
(TokenType.CLOSE_BLOCK, r"\}"),
(TokenType.STMT_END, r";"),
(TokenType.OP, r"[:=\.=\|<>]+"),
(TokenType.WHITESPACE, r"\s+"),
(TokenType.COMMENT, r"\/\*.*?\*\/"),
(TokenType.COMMENT, r"/\*[\s\S]*\*/"),
(TokenType.COMMENT, r"\/\/[^\n]*"),
(TokenType.OPEN_BRACKET, r"\["),
(TokenType.CLOSE_BRACKET, r"\]"),
(TokenType.OP, r"[:=\.=\|<>\+\-/\*]+"),
(TokenType.COMMA, r"\,"),
]
_TOKENS = [(type, re.compile(regex)) for (type, regex) in _TOKENS]