Improve value parsing

Parse values as different AST nodes rather than just strings. This
allows for better validation and will eventually make expressions
possible.
This commit is contained in:
James Westman 2021-11-01 21:51:25 -05:00
parent 5f0eef5f2e
commit 80b5698533
No known key found for this signature in database
GPG key ID: CE2DBA0ADB654EA6
16 changed files with 352 additions and 138 deletions

View file

@ -205,12 +205,6 @@ class ObjectContent(AstNode):
class Property(AstNode):
@property
def gir_property(self):
if self.gir_class is not None:
return self.gir_class.properties.get(self.tokens["name"])
@property
def gir_class(self):
parent = self.parent.parent
@ -222,6 +216,18 @@ class Property(AstNode):
raise CompilerBugError()
@property
def gir_property(self):
if self.gir_class is not None:
return self.gir_class.properties.get(self.tokens["name"])
@property
def value_type(self):
if self.gir_property is not None:
return self.gir_property.type
@validate("name")
def property_exists(self):
if self.gir_class is None:
@ -248,6 +254,10 @@ class Property(AstNode):
def emit_xml(self, xml: XmlEmitter):
values = self.children[Value]
value = values[0] if len(values) == 1 else None
translatable = isinstance(value, TranslatedStringValue)
bind_flags = []
if self.tokens["sync_create"]:
bind_flags.append("sync-create")
@ -257,7 +267,7 @@ class Property(AstNode):
props = {
"name": self.tokens["name"],
"translatable": "yes" if self.tokens["translatable"] else None,
"translatable": "true" if translatable else None,
"bind-source": self.tokens["bind_source"],
"bind-property": self.tokens["bind_property"],
"bind-flags": bind_flags_str,
@ -267,11 +277,14 @@ class Property(AstNode):
xml.start_tag("property", **props)
self.children[Object][0].emit_xml(xml)
xml.end_tag()
elif self.tokens["value"] is None:
elif value is None:
xml.put_self_closing("property", **props)
else:
xml.start_tag("property", **props)
xml.put_text(str(self.tokens["value"]))
if translatable:
xml.put_text(value.string)
else:
value.emit_xml(xml)
xml.end_tag()
@ -323,3 +336,82 @@ class Signal(AstNode):
if self.tokens["detail_name"]:
name += "::" + self.tokens["detail_name"]
xml.put_self_closing("signal", name=name, handler=self.tokens["handler"], swapped="true" if self.tokens["swapped"] else None)
class Value(ast.AstNode):
pass
class TranslatedStringValue(Value):
@property
def string(self):
return self.tokens["value"]
def emit_xml(self, xml):
raise CompilerBugError("TranslatedStringValues must be handled by the parent AST node")
class LiteralValue(Value):
def emit_xml(self, xml: XmlEmitter):
xml.put_text(self.tokens["value"])
class Flag(AstNode):
pass
class FlagsValue(Value):
def emit_xml(self, xml: XmlEmitter):
xml.put_text("|".join([flag.tokens["value"] for flag in self.children[Flag]]))
class IdentValue(Value):
def emit_xml(self, xml: XmlEmitter):
xml.put_text(self.tokens["value"])
@validate()
def validate_for_type(self):
type = self.parent.value_type
if isinstance(type, gir.Enumeration):
if self.tokens["value"] not in type.members:
raise CompileError(
f"{self.tokens['value']} is not a member of {type.full_name}",
did_you_mean=type.members.keys(),
)
elif isinstance(type, gir.BoolType):
# would have been parsed as a LiteralValue if it was correct
raise CompileError(
f"Expected 'true' or 'false' for boolean value",
did_you_mean=["true", "false"],
)
@docs()
def docs(self):
type = self.parent.value_type
if isinstance(type, gir.Enumeration):
if member := type.members.get(self.tokens["value"]):
return member.doc
else:
return type.doc
elif isinstance(type, gir.GirNode):
return type.doc
class BaseAttribute(AstNode):
""" A helper class for attribute syntax of the form `name: literal_value;`"""
tag_name: str = ""
def emit_xml(self, xml: XmlEmitter):
value = self.children[Value][0]
translatable = isinstance(value, TranslatedStringValue)
xml.start_tag(
self.tag_name,
name=self.tokens["name"],
translatable="true" if translatable else None,
)
if translatable:
xml.put_text(value.string)
else:
value.emit_xml(xml)
xml.end_tag()

View file

@ -20,6 +20,7 @@
import typing as T
from collections import ChainMap, defaultdict
from . import ast
from .errors import *
from .utils import lazy_prop
from .xml_emitter import XmlEmitter
@ -133,13 +134,19 @@ def validate(token_name=None, end_token_name=None, skip_incomplete=False):
# This mess of code sets the error's start and end positions
# from the tokens passed to the decorator, if they have not
# already been set
if token_name is not None and e.start is None:
group = self.group.tokens.get(token_name)
if end_token_name is not None and group is None:
group = self.group.tokens[end_token_name]
e.start = group.start
if (token_name is not None or end_token_name is not None) and e.end is None:
e.end = self.group.tokens[end_token_name or token_name].end
if e.start is None:
if token := self.group.tokens.get(token_name):
e.start = token.start
else:
e.start = self.group.start
if e.end is None:
if token := self.group.tokens.get(token_name):
e.end = token.end
elif token := self.group.tokens.get(end_token_name):
e.end = token.end
else:
e.end = self.group.end
# Re-raise the exception
raise e
@ -168,18 +175,3 @@ def docs(*args, **kwargs):
return Docs(func, *args, **kwargs)
return decorator
class BaseAttribute(AstNode):
""" A helper class for attribute syntax of the form `name: literal_value;`"""
tag_name: str = ""
def emit_xml(self, xml: XmlEmitter):
xml.start_tag(
self.tag_name,
name=self.tokens["name"],
translatable="yes" if self.tokens["translatable"] else None,
)
xml.put_text(str(self.tokens["value"]))
xml.end_tag()

View file

@ -20,6 +20,7 @@
import typing as T
from . import ast
from . import gir
from .completions_utils import *
from .lsp_utils import Completion, CompletionItemKind
from .parser import SKIP_TOKENS
@ -28,23 +29,13 @@ from .tokenizer import TokenType, Token
Pattern = T.List[T.Tuple[TokenType, T.Optional[str]]]
def complete(ast_node: ast.AstNode, tokens: T.List[Token], idx: int) -> T.Iterator[Completion]:
def _complete(ast_node: ast.AstNode, tokens: T.List[Token], idx: int, token_idx: int) -> T.Iterator[Completion]:
for child in ast_node.children:
if child.group.start <= idx <= child.group.end:
yield from complete(child, tokens, idx)
if child.group.start <= idx and (idx < child.group.end or (idx == child.group.end and child.incomplete)):
yield from _complete(child, tokens, idx, token_idx)
return
prev_tokens: T.List[Token] = []
token_idx = 0
# find the current token
for i, token in enumerate(tokens):
if token.start < idx <= token.end:
token_idx = i
# if the current token is an identifier, move to the token before it
if tokens[token_idx].type == TokenType.IDENT:
token_idx -= 1
# collect the 5 previous non-skipped tokens
while len(prev_tokens) < 5 and token_idx >= 0:
@ -57,6 +48,21 @@ def complete(ast_node: ast.AstNode, tokens: T.List[Token], idx: int) -> T.Iterat
yield from completer(prev_tokens, ast_node)
def complete(ast_node: ast.AstNode, tokens: T.List[Token], idx: int) -> T.Iterator[Completion]:
token_idx = 0
# find the current token
for i, token in enumerate(tokens):
if token.start < idx <= token.end:
token_idx = i
# if the current token is an identifier or whitespace, move to the token before it
while tokens[token_idx].type in [TokenType.IDENT, TokenType.WHITESPACE]:
idx = tokens[token_idx].start
token_idx -= 1
yield from _complete(ast_node, tokens, idx, token_idx)
@completer([ast.GtkDirective])
def using_gtk(ast_node, match_variables):
yield Completion("using Gtk 4.0;", CompletionItemKind.Keyword)
@ -96,6 +102,22 @@ def property_completer(ast_node, match_variables):
yield Completion(prop, CompletionItemKind.Property, snippet=f"{prop}: $0;")
@completer(
applies_in=[ast.Property],
matches=[
[(TokenType.IDENT, None), (TokenType.OP, ":")]
],
)
def prop_value_completer(ast_node, match_variables):
if isinstance(ast_node.value_type, gir.Enumeration):
for name, member in ast_node.value_type.members.items():
yield Completion(name, CompletionItemKind.EnumMember, docs=member.doc)
elif isinstance(ast_node.value_type, gir.BoolType):
yield Completion("true", CompletionItemKind.Constant)
yield Completion("false", CompletionItemKind.Constant)
@completer(
applies_in=[ast.ObjectContent],
matches=new_statement_patterns,
@ -103,8 +125,10 @@ def property_completer(ast_node, match_variables):
def signal_completer(ast_node, match_variables):
if ast_node.gir_class:
for signal in ast_node.gir_class.signals:
name = ("on" if not isinstance(ast_node.parent, ast.Object)
else "on_" + (ast_node.parent.id or ast_node.parent.class_name.lower()))
if not isinstance(ast_node.parent, ast.Object):
name = "on"
else:
name = "on_" + (ast_node.parent.tokens["id"] or ast_node.parent.tokens["class_name"].lower())
yield Completion(signal, CompletionItemKind.Property, snippet=f"{signal} => ${{1:{name}_{signal.replace('-', '_')}}}()$0;")

View file

@ -18,7 +18,8 @@
# SPDX-License-Identifier: LGPL-3.0-or-later
from ..ast_utils import AstNode, BaseAttribute
from ..ast import BaseAttribute
from ..ast_utils import AstNode
from ..completions_utils import *
from ..parse_tree import *
from ..parser_utils import *

View file

@ -18,7 +18,8 @@
# SPDX-License-Identifier: LGPL-3.0-or-later
from ..ast_utils import AstNode, BaseAttribute
from ..ast import BaseAttribute
from ..ast_utils import AstNode
from ..completions_utils import *
from ..lsp_utils import Completion, CompletionItemKind
from ..parse_tree import *

View file

@ -19,7 +19,7 @@
from .. import ast
from ..ast_utils import AstNode, BaseAttribute
from ..ast_utils import AstNode
from ..completions_utils import *
from ..lsp_utils import Completion, CompletionItemKind
from ..parse_tree import *

View file

@ -55,14 +55,57 @@ def get_namespace(namespace, version):
return _namespace_cache[filename]
class BasicType:
pass
class BoolType(BasicType):
pass
class IntType(BasicType):
pass
class UIntType(BasicType):
pass
class FloatType(BasicType):
pass
_BASIC_TYPES = {
"gboolean": BoolType,
"gint": IntType,
"gint64": IntType,
"guint": UIntType,
"guint64": UIntType,
"gfloat": FloatType,
"gdouble": FloatType,
"float": FloatType,
"double": FloatType,
}
class GirNode:
def __init__(self, xml):
def __init__(self, container, xml):
self.container = container
self.xml = xml
def get_containing(self, container_type):
if self.container is None:
return None
elif isinstance(self.container, container_type):
return self.container
else:
return self.container.get_containing(container_type)
@lazy_prop
def glib_type_name(self):
return self.xml["glib:type-name"]
@lazy_prop
def full_name(self):
if self.container is None:
return self.name
else:
return f"{self.container.name}.{self.name}"
@lazy_prop
def name(self) -> str:
return self.xml["name"]
@ -88,11 +131,18 @@ class GirNode:
def signature(self) -> T.Optional[str]:
return None
@property
def type_name(self):
return self.xml.get_elements('type')[0]['name']
@property
def type(self):
return self.get_containing(Namespace).lookup_type(self.type_name)
class Property(GirNode):
def __init__(self, klass, xml: xml_reader.Element):
super().__init__(xml)
self.klass = klass
super().__init__(klass, xml)
@property
def type_name(self):
@ -100,45 +150,38 @@ class Property(GirNode):
@property
def signature(self):
return f"{self.type_name} {self.klass.name}.{self.name}"
return f"{self.type_name} {self.container.name}.{self.name}"
class Parameter(GirNode):
def __init__(self, xml: xml_reader.Element):
super().__init__(xml)
@property
def type_name(self):
return self.xml.get_elements('type')[0]['name']
def __init__(self, container: GirNode, xml: xml_reader.Element):
super().__init__(container, xml)
class Signal(GirNode):
def __init__(self, klass, xml: xml_reader.Element):
super().__init__(xml)
self.klass = klass
super().__init__(klass, xml)
if parameters := xml.get_elements('parameters'):
self.params = [Parameter(child) for child in parameters[0].get_elements('parameter')]
self.params = [Parameter(self, child) for child in parameters[0].get_elements('parameter')]
else:
self.params = []
@property
def signature(self):
args = ", ".join([f"{p.type_name} {p.name}" for p in self.params])
return f"signal {self.klass.name}.{self.name} ({args})"
return f"signal {self.container.name}.{self.name} ({args})"
class Interface(GirNode):
def __init__(self, ns, xml: xml_reader.Element):
super().__init__(xml)
self.ns = ns
super().__init__(ns, xml)
self.properties = {child["name"]: Property(self, child) for child in xml.get_elements("property")}
self.signals = {child["name"]: Signal(self, child) for child in xml.get_elements("glib:signal")}
class Class(GirNode):
def __init__(self, ns, xml: xml_reader.Element):
super().__init__(xml)
self.ns = ns
super().__init__(ns, xml)
self._parent = xml["parent"]
self.implements = [impl["name"] for impl in xml.get_elements("implements")]
self.own_properties = {child["name"]: Property(self, child) for child in xml.get_elements("property")}
@ -146,9 +189,9 @@ class Class(GirNode):
@property
def signature(self):
result = f"class {self.ns.name}.{self.name}"
result = f"class {self.container.name}.{self.name}"
if self.parent is not None:
result += f" : {self.parent.ns.name}.{self.parent.name}"
result += f" : {self.parent.container.name}.{self.parent.name}"
if len(self.implements):
result += " implements " + ", ".join(self.implements)
return result
@ -161,15 +204,11 @@ class Class(GirNode):
def signals(self):
return { s.name: s for s in self._enum_signals() }
@lazy_prop
def full_name(self):
return f"{self.ns.name}.{self.name}"
@lazy_prop
def parent(self):
if self._parent is None:
return None
return self.ns.lookup_class(self._parent)
return self.get_containing(Namespace).lookup_type(self._parent)
def _enum_properties(self):
@ -179,7 +218,7 @@ class Class(GirNode):
yield from self.parent.properties.values()
for impl in self.implements:
yield from self.ns.lookup_interface(impl).properties.values()
yield from self.get_containing(Namespace).lookup_type(impl).properties.values()
def _enum_signals(self):
yield from self.own_signals.values()
@ -188,15 +227,39 @@ class Class(GirNode):
yield from self.parent.signals.values()
for impl in self.implements:
yield from self.ns.lookup_interface(impl).signals.values()
yield from self.get_containing(Namespace).lookup_type(impl).signals.values()
class EnumMember(GirNode):
def __init__(self, ns, xml: xml_reader.Element):
super().__init__(ns, xml)
self._value = xml["value"]
@property
def value(self):
return self._value
@property
def signature(self):
return f"enum member {self.full_name} = {self.value}"
class Enumeration(GirNode):
def __init__(self, ns, xml: xml_reader.Element):
super().__init__(ns, xml)
self.members = { child["name"]: EnumMember(self, child) for child in xml.get_elements("member") }
@property
def signature(self):
return f"enum {self.full_name}"
class Namespace(GirNode):
def __init__(self, repo, xml: xml_reader.Element):
super().__init__(xml)
self.repo = repo
super().__init__(repo, xml)
self.classes = { child["name"]: Class(self, child) for child in xml.get_elements("class") }
self.interfaces = { child["name"]: Interface(self, child) for child in xml.get_elements("interface") }
self.enumerations = { child["name"]: Enumeration(self, child) for child in xml.get_elements("enumeration") }
self.version = xml["version"]
@property
@ -206,30 +269,25 @@ class Namespace(GirNode):
def get_type(self, name):
""" Gets a type (class, interface, enum, etc.) from this namespace. """
return self.classes.get(name) or self.interfaces.get(name)
return self.classes.get(name) or self.interfaces.get(name) or self.enumerations.get(name)
def lookup_class(self, name: str):
if "." in name:
ns, cls = name.split(".")
return self.repo.lookup_namespace(ns).lookup_class(cls)
def lookup_type(self, type_name: str):
""" Looks up a type in the scope of this namespace (including in the
namespace's dependencies). """
if type_name in _BASIC_TYPES:
return _BASIC_TYPES[type_name]()
elif "." in type_name:
ns, name = type_name.split(".", 1)
return self.get_containing(Repository).get_type(name, ns)
else:
return self.classes.get(name)
def lookup_interface(self, name: str):
if "." in name:
ns, iface = name.split(".")
return self.repo.lookup_namespace(ns).lookup_interface(iface)
else:
return self.interfaces.get(name)
def lookup_namespace(self, ns: str):
return self.repo.lookup_namespace(ns)
return self.get_type(type_name)
class Repository(GirNode):
def __init__(self, xml: xml_reader.Element):
super().__init__(xml)
super().__init__(None, xml)
self.namespaces = { child["name"]: Namespace(self, child) for child in xml.get_elements("namespace") }
try:
@ -237,14 +295,22 @@ class Repository(GirNode):
except:
raise CompilerBugError(f"Failed to load dependencies.")
def lookup_namespace(self, name: str):
ns = self.namespaces.get(name)
if ns is not None:
return ns
for include in self.includes.values():
ns = include.lookup_namespace(name)
if ns is not None:
return ns
def get_type(self, name: str, ns: str) -> T.Optional[GirNode]:
if namespace := self.namespaces.get(ns):
return namespace.get_type(name)
else:
return self.lookup_namespace(ns).get_type(name)
def lookup_namespace(self, ns: str):
""" Finds a namespace among this namespace's dependencies. """
if namespace := self.namespaces.get(ns):
return namespace
else:
for include in self.includes.values():
if namespace := include.get_containing(Repository).lookup_namespace(ns):
return namespace
class GirContext:
@ -260,7 +326,7 @@ class GirContext:
self.namespaces[namespace.name] = namespace
def get_type(self, name: str, ns: str) -> GirNode:
def get_type(self, name: str, ns: str) -> T.Optional[GirNode]:
ns = ns or "Gtk"
if ns not in self.namespaces:
@ -273,9 +339,11 @@ class GirContext:
type = self.get_type(name, ns)
if isinstance(type, Class):
return type
else:
return None
def validate_class(self, name: str, ns: str) -> Class:
def validate_class(self, name: str, ns: str):
""" Raises an exception if there is a problem looking up the given
class (it doesn't exist, it isn't a class, etc.) """

View file

@ -294,7 +294,7 @@ class Statement(ParseNode):
return False
except CompileError as e:
ctx.errors.append(e)
ctx.set_group_incomplete(True)
ctx.set_group_incomplete()
return True
token = ctx.peek_token()

View file

@ -18,6 +18,7 @@
# SPDX-License-Identifier: LGPL-3.0-or-later
from . import ast
from .parse_tree import *
@ -35,23 +36,38 @@ class_name = AnyOf(
UseIdent("class_name"),
)
value = AnyOf(
literal = Group(
ast.LiteralValue,
AnyOf(
Sequence(Keyword("true"), UseLiteral("value", True)),
Sequence(Keyword("false"), UseLiteral("value", False)),
UseNumber("value"),
UseQuoted("value"),
)
)
ident_value = Group(
ast.IdentValue,
UseIdent("value"),
)
flags_value = Group(
ast.FlagsValue,
Sequence(
Group(ast.Flag, UseIdent("value")),
Op("|"),
Delimited(Group(ast.Flag, UseIdent("value")), Op("|")),
),
)
translated_string = Group(
ast.TranslatedStringValue,
Sequence(
Keyword("_"),
OpenParen(),
UseQuoted("value").expected("a quoted string"),
CloseParen().expected("`)`"),
UseLiteral("translatable", True),
),
Sequence(Keyword("True"), UseLiteral("value", True)),
Sequence(Keyword("true"), UseLiteral("value", True)),
Sequence(Keyword("Yes"), UseLiteral("value", True)),
Sequence(Keyword("yes"), UseLiteral("value", True)),
Sequence(Keyword("False"), UseLiteral("value", False)),
Sequence(Keyword("false"), UseLiteral("value", False)),
Sequence(Keyword("No"), UseLiteral("value", False)),
Sequence(Keyword("no"), UseLiteral("value", False)),
UseIdent("value"),
UseNumber("value"),
UseQuoted("value"),
)
value = AnyOf(translated_string, literal, flags_value, ident_value)

View file

@ -27,7 +27,8 @@ from .utils import lazy_prop
# To speed up parsing, we ignore all tags except these
PARSE_GIR = set([
"repository", "namespace", "class", "interface", "property", "glib:signal",
"include", "implements", "type", "parameter", "parameters",
"include", "implements", "type", "parameter", "parameters", "enumeration",
"member",
])

6
tests/samples/flags.blp Normal file
View file

@ -0,0 +1,6 @@
using Gtk 4.0;
using Gio 2.0;
Gio.Application {
flags: is_service | handles_open;
}

7
tests/samples/flags.ui Normal file
View file

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<interface>
<requires lib="gtk" version="4.0"/>
<object class="GApplication">
<property name="flags">is_service|handles_open</property>
</object>
</interface>

View file

@ -2,7 +2,7 @@
<interface>
<requires lib="gtk" version="4.0"/>
<menu>
<attribute name="label" translatable="yes">menu label</attribute>
<attribute name="label" translatable="true">menu label</attribute>
<attribute name="test-custom-attribute">3.1415</attribute>
<submenu>
<section>

View file

@ -1,5 +1,5 @@
using Gtk 4.0;
Box {
orientation: VERTICAL;
orientation: vertical;
}

View file

@ -2,6 +2,6 @@
<interface>
<requires lib="gtk" version="4.0"/>
<object class="GtkBox">
<property name="orientation">VERTICAL</property>
<property name="orientation">vertical</property>
</object>
</interface>

View file

@ -20,38 +20,44 @@
import difflib # I love Python
from pathlib import Path
import traceback
import unittest
from gtkblueprinttool import tokenizer, parser
from gtkblueprinttool.errors import PrintableError
from gtkblueprinttool.errors import PrintableError, MultipleErrors
from gtkblueprinttool.tokenizer import Token, TokenType, tokenize
class TestSamples(unittest.TestCase):
def assert_sample(self, name):
with open((Path(__file__).parent / f"samples/{name}.blp").resolve()) as f:
blueprint = f.read()
with open((Path(__file__).parent / f"samples/{name}.ui").resolve()) as f:
expected = f.read()
try:
with open((Path(__file__).parent / f"samples/{name}.blp").resolve()) as f:
blueprint = f.read()
with open((Path(__file__).parent / f"samples/{name}.ui").resolve()) as f:
expected = f.read()
tokens = tokenizer.tokenize(blueprint)
ast, errors = parser.parse(tokens)
tokens = tokenizer.tokenize(blueprint)
ast, errors = parser.parse(tokens)
if errors:
raise errors
if len(ast.errors):
raise MultipleErrors(ast.errors)
if errors:
raise errors
if len(ast.errors):
raise MultipleErrors(ast.errors)
actual = ast.generate()
if actual.strip() != expected.strip():
diff = difflib.unified_diff(expected.splitlines(), actual.splitlines())
print("\n".join(diff))
actual = ast.generate()
if actual.strip() != expected.strip():
diff = difflib.unified_diff(expected.splitlines(), actual.splitlines())
print("\n".join(diff))
raise AssertionError()
except PrintableError as e:
e.pretty_print(name + ".blp", blueprint)
raise AssertionError()
def test_samples(self):
self.assert_sample("binding")
self.assert_sample("child_type")
self.assert_sample("flags")
self.assert_sample("layout")
self.assert_sample("menu")
self.assert_sample("property")