Siklos/blueprint-compiler
1
0
Fork 0
mirror of https://gitlab.gnome.org/jwestman/blueprint-compiler.git synced 2025-05-04 15:59:08 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

tokenizer: Fix number parsing (again)

Browse source
This commit is contained in:
James Westman 2022-05-05 13:55:36 -05:00
parent 4fefa0bd73
commit 75475d1a45
No known key found for this signature in database
GPG key ID: CE2DBA0ADB654EA6
1 changed files with 10 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

17
blueprintcompiler/tokenizer.py
View file

@ -40,9 +40,9 @@ _tokens = [
(TokenType.IDENT, r"[A-Za-z_][\d\w\-_]*"), (TokenType.IDENT, r"[A-Za-z_][\d\w\-_]*"),
(TokenType.QUOTED, r'"(\\"|[^"\n])*"'), (TokenType.QUOTED, r'"(\\"|[^"\n])*"'),
(TokenType.QUOTED, r"'(\\'|[^'\n])*'"), (TokenType.QUOTED, r"'(\\'|[^'\n])*'"),
(TokenType.NUMBER, r"0x[A-Fa-f0-9_]+"), (TokenType.NUMBER, r"0x[A-Za-z0-9_]+"),
(TokenType.NUMBER, r"[-+]?[\d_]*\d(\.[\d_]*\d)?"), (TokenType.NUMBER, r"[-+]?[\d_]+(\.[\d_]+)?"),
(TokenType.NUMBER, r"[-+]?\.[\d_]*\d"), (TokenType.NUMBER, r"[-+]?\.[\d_]+"),
(TokenType.WHITESPACE, r"\s+"), (TokenType.WHITESPACE, r"\s+"),
(TokenType.COMMENT, r"\/\*[\s\S]*?\*\/"), (TokenType.COMMENT, r"\/\*[\s\S]*?\*\/"),
(TokenType.COMMENT, r"\/\/[^\n]*"), (TokenType.COMMENT, r"\/\/[^\n]*"),
@ -67,10 +67,13 @@ class Token:
return None return None
string = str(self).replace("_", "") string = str(self).replace("_", "")
if string.startswith("0x"): try:
return int(string, 16) if string.startswith("0x"):
else: return int(string, 16)
return float(string.replace("_", "")) else:
return float(string.replace("_", ""))
except:
raise CompileError(f"{str(self)} is not a valid number literal", self.start, self.end)
def _tokenize(ui_ml: str): def _tokenize(ui_ml: str):