tokenizer: Allow escaped newlines

The docs said multi-line strings were possible by escaping the newline
character, but this was not actually implemented.

Fixes #132.
This commit is contained in:
James Westman 2023-10-26 18:50:07 -05:00
parent 9543b78138
commit 2faa9207de
4 changed files with 22 additions and 2 deletions

View file

@ -39,8 +39,8 @@ class TokenType(Enum):
_tokens = [ _tokens = [
(TokenType.IDENT, r"[A-Za-z_][\d\w\-_]*"), (TokenType.IDENT, r"[A-Za-z_][\d\w\-_]*"),
(TokenType.QUOTED, r'"(\\.|[^\\"\n])*"'), (TokenType.QUOTED, r'"(\\(.|\n)|[^\\"\n])*"'),
(TokenType.QUOTED, r"'(\\.|[^\\'\n])*'"), (TokenType.QUOTED, r"'(\\(.|\n)|[^\\'\n])*'"),
(TokenType.NUMBER, r"0x[A-Za-z0-9_]+"), (TokenType.NUMBER, r"0x[A-Za-z0-9_]+"),
(TokenType.NUMBER, r"[\d_]+(\.[\d_]+)?"), (TokenType.NUMBER, r"[\d_]+(\.[\d_]+)?"),
(TokenType.NUMBER, r"\.[\d_]+"), (TokenType.NUMBER, r"\.[\d_]+"),

View file

@ -124,6 +124,7 @@ def unescape_quote(string: str) -> str:
string = string[1:-1] string = string[1:-1]
REPLACEMENTS = { REPLACEMENTS = {
"\n": "\n",
"\\": "\\", "\\": "\\",
"n": "\n", "n": "\n",
"t": "\t", "t": "\t",

View file

@ -0,0 +1,6 @@
using Gtk 4.0;
Label {
label: "Hello,\
world!";
}

View file

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
DO NOT EDIT!
This file was @generated by blueprint-compiler. Instead, edit the
corresponding .blp file and regenerate this file with blueprint-compiler.
-->
<interface>
<requires lib="gtk" version="4.0"/>
<object class="GtkLabel">
<property name="label">Hello,
world!</property>
</object>
</interface>