tokenizer: Fix QUOTED regex

unescape_quote() assumed that a QUOTED token wouldn't end in the middle
of an escape sequence, but that assumption could fail (a bug found by
the fuzzer).
This commit is contained in:
James Westman 2023-09-28 18:17:43 -05:00
parent 7c072c0a32
commit 09bed9a9f5

View file

@ -39,8 +39,8 @@ class TokenType(Enum):
_tokens = [
(TokenType.IDENT, r"[A-Za-z_][\d\w\-_]*"),
(TokenType.QUOTED, r'"(\\"|[^"\n])*"'),
(TokenType.QUOTED, r"'(\\'|[^'\n])*'"),
(TokenType.QUOTED, r'"(\\.|[^\\"\n])*"'),
(TokenType.QUOTED, r"'(\\.|[^\\'\n])*'"),
(TokenType.NUMBER, r"0x[A-Za-z0-9_]+"),
(TokenType.NUMBER, r"[\d_]+(\.[\d_]+)?"),
(TokenType.NUMBER, r"\.[\d_]+"),