Clean up AST code

This commit is contained in:
James Westman 2021-10-31 16:44:34 -05:00
parent d7a8a21b8e
commit dc7c0cabd8
No known key found for this signature in database
GPG key ID: CE2DBA0ADB654EA6
5 changed files with 171 additions and 412 deletions

View file

@ -28,123 +28,37 @@ from .utils import lazy_prop
from .xml_emitter import XmlEmitter
class AstNode:
""" Base class for nodes in the abstract syntax tree. """
completers: T.List = []
def __init__(self):
self.group = None
self.parent = None
self.child_nodes = None
self.incomplete = False
def __init_subclass__(cls):
cls.completers = []
@lazy_prop
def root(self):
if self.parent is None:
return self
else:
return self.parent.root
@lazy_prop
def errors(self):
return list(self._get_errors())
def _get_errors(self):
for name, attr in self._attrs_by_type(Validator):
try:
getattr(self, name)
except AlreadyCaughtError:
pass
except CompileError as e:
yield e
for child in self.child_nodes:
yield from child._get_errors()
def _attrs_by_type(self, attr_type):
for name in dir(type(self)):
item = getattr(type(self), name)
if isinstance(item, attr_type):
yield name, item
def generate(self) -> str:
""" Generates an XML string from the node. """
xml = XmlEmitter()
self.emit_xml(xml)
return xml.result
def emit_xml(self, xml: XmlEmitter):
""" Emits the XML representation of this AST node to the XmlEmitter. """
raise NotImplementedError()
def get_docs(self, idx: int) -> T.Optional[str]:
for name, attr in self._attrs_by_type(Docs):
if attr.token_name:
token = self.group.tokens.get(attr.token_name)
if token and token.start <= idx < token.end:
return getattr(self, name)
else:
return getattr(self, name)
for child in self.child_nodes:
if child.group.start <= idx < child.group.end:
docs = child.get_docs(idx)
if docs is not None:
return docs
return None
class UI(AstNode):
""" The AST node for the entire file """
def __init__(self, gtk_directives=[], imports=[], objects=[], templates=[], menus=[]):
super().__init__()
assert_true(len(gtk_directives) == 1)
self.gtk_directive = gtk_directives[0]
self.imports = imports
self.objects = objects
self.templates = templates
self.menus = menus
@validate()
def gir(self):
gir = GirContext()
gir.add_namespace(self.gtk_directive.gir_namespace)
for i in self.imports:
gir.add_namespace(self.children[GtkDirective][0].gir_namespace)
for i in self.children[Import]:
gir.add_namespace(i.gir_namespace)
return gir
@validate()
def at_most_one_template(self):
if len(self.templates) > 1:
if len(self.children[Template]) > 1:
raise CompileError(f"Only one template may be defined per file, but this file contains {len(self.templates)}",
self.templates[1].group.start)
self.children[Template][1].group.start)
def emit_xml(self, xml: XmlEmitter):
xml.start_tag("interface")
for x in self.child_nodes:
for x in self.children:
x.emit_xml(xml)
xml.end_tag()
class GtkDirective(AstNode):
child_type = "gtk_directives"
def __init__(self, version=None):
super().__init__()
self.version = version
@validate("version")
def gir_namespace(self):
if self.version in ["4.0"]:
return get_namespace("Gtk", self.version)
if self.tokens["version"] in ["4.0"]:
return get_namespace("Gtk", self.tokens["version"])
else:
err = CompileError("Only GTK 4 is supported")
if self.version.startswith("4"):
@ -154,46 +68,25 @@ class GtkDirective(AstNode):
raise err
def emit_xml(self, xml: XmlEmitter):
xml.put_self_closing("requires", lib="gtk", version=self.version)
xml.put_self_closing("requires", lib="gtk", version=self.tokens["version"])
class Import(AstNode):
child_type = "imports"
def __init__(self, namespace=None, version=None):
super().__init__()
self.namespace = namespace
self.version = version
@validate("namespace", "version")
def gir_namespace(self):
return get_namespace(self.namespace, self.version)
def emit_xml(self, xml: XmlEmitter):
pass
return get_namespace(self.tokens["namespace"], self.tokens["version"])
class Template(AstNode):
child_type = "templates"
def __init__(self, name=None, class_name=None, object_content=None, namespace=None, ignore_gir=False):
super().__init__()
assert_true(len(object_content) == 1)
self.name = name
self.parent_namespace = namespace
self.parent_class = class_name
self.object_content = object_content[0]
self.ignore_gir = ignore_gir
@validate("namespace", "class_name")
def gir_parent(self):
if not self.ignore_gir:
return self.root.gir.get_class(self.parent_class, self.parent_namespace)
if not self.tokens["ignore_gir"]:
return self.root.gir.get_class(self.tokens["class_name"], self.tokens["namespace"])
@docs("namespace")
def namespace_docs(self):
return self.root.gir.namespaces[self.parent_namespace].doc
return self.root.gir.namespaces[self.tokens["namespace"]].doc
@docs("class_name")
def class_docs(self):
@ -203,34 +96,25 @@ class Template(AstNode):
def emit_xml(self, xml: XmlEmitter):
xml.start_tag("template", **{
"class": self.name,
"parent": self.gir_parent.glib_type_name if self.gir_parent else self.parent_class,
"class": self.tokens["name"],
"parent": self.gir_parent.glib_type_name if self.gir_parent else self.tokens["class_name"],
})
self.object_content.emit_xml(xml)
for child in self.children:
child.emit_xml(xml)
xml.end_tag()
class Object(AstNode):
child_type = "objects"
def __init__(self, class_name=None, object_content=None, namespace=None, id=None, ignore_gir=False):
super().__init__()
assert_true(len(object_content) == 1)
self.namespace = namespace
self.class_name = class_name
self.id = id
self.object_content = object_content[0]
self.ignore_gir = ignore_gir
@validate("namespace", "class_name")
def gir_class(self):
if not self.ignore_gir:
return self.root.gir.get_class(self.class_name, self.namespace)
if not self.tokens["ignore_gir"]:
return self.root.gir.get_class(self.tokens["class_name"], self.tokens["namespace"])
@docs("namespace")
def namespace_docs(self):
return self.root.gir.namespaces[self.namespace].doc
return self.root.gir.namespaces[self.tokens["namespace"]].doc
@docs("class_name")
def class_docs(self):
@ -239,85 +123,52 @@ class Object(AstNode):
def emit_xml(self, xml: XmlEmitter):
print("Emitting object XML! ", self.gir_class)
xml.start_tag("object", **{
"class": self.gir_class.glib_type_name if self.gir_class else self.class_name,
"id": self.id,
"class": self.gir_class.glib_type_name if self.gir_class else self.tokens["class_name"],
"id": self.tokens["id"],
})
self.object_content.emit_xml(xml)
for child in self.children:
child.emit_xml(xml)
xml.end_tag()
class Child(AstNode):
child_type = "children"
def __init__(self, objects=None, child_type=None):
super().__init__()
assert_true(len(objects) == 1)
self.object = objects[0]
self.child_type = child_type
def emit_xml(self, xml: XmlEmitter):
xml.start_tag("child", type=self.child_type)
self.object.emit_xml(xml)
xml.start_tag("child", type=self.tokens["child_type"])
for child in self.children:
child.emit_xml(xml)
xml.end_tag()
class ObjectContent(AstNode):
child_type = "object_content"
def __init__(self, properties=[], signals=[], children=[], style=[], layout=None):
super().__init__()
self.properties = properties
self.signals = signals
self.children = children
self.style = style
self.layout = layout or []
@validate()
def gir_class(self):
parent = self.parent
if isinstance(parent, Template):
return parent.gir_parent
elif isinstance(parent, Object):
return parent.gir_class
if isinstance(self.parent, Template):
return self.parent.gir_parent
elif isinstance(self.parent, Object):
return self.parent.gir_class
else:
raise CompilerBugError()
@validate()
def only_one_style_class(self):
if len(self.style) > 1:
if len(self.children[Style]) > 1:
raise CompileError(
f"Only one style directive allowed per object, but this object contains {len(self.style)}",
start=self.style[1].group.start,
f"Only one style directive allowed per object, but this object contains {len(self.children[Style])}",
start=self.children[Style][1].group.start,
)
def emit_xml(self, xml: XmlEmitter):
for x in self.child_nodes:
for x in self.children:
x.emit_xml(xml)
class Property(AstNode):
child_type = "properties"
def __init__(self, name=None, value=None, translatable=False, bind_source=None, bind_property=None, objects=None, sync_create=None, after=None):
super().__init__()
self.name = name
self.value = value
self.translatable = translatable
self.bind_source = bind_source
self.bind_property = bind_property
self.objects = objects
bind_flags = []
if sync_create:
bind_flags.append("sync-create")
if after:
bind_flags.append("after")
self.bind_flags = "|".join(bind_flags) or None
@validate()
def gir_property(self):
if self.gir_class is not None:
return self.gir_class.properties.get(self.name)
return self.gir_class.properties.get(self.tokens["name"])
@validate()
def gir_class(self):
@ -343,8 +194,8 @@ class Property(AstNode):
if self.gir_property is None:
raise CompileError(
f"Class {self.gir_class.full_name} does not contain a property called {self.name}",
did_you_mean=(self.name, self.gir_class.properties.keys())
f"Class {self.gir_class.full_name} does not contain a property called {self.tokens['name']}",
did_you_mean=(self.tokens["name"], self.gir_class.properties.keys())
)
@ -355,37 +206,34 @@ class Property(AstNode):
def emit_xml(self, xml: XmlEmitter):
bind_flags = []
if self.tokens["sync_create"]:
bind_flags.append("sync-create")
if self.tokens["after"]:
bind_flags.append("after")
bind_flags_str = "|".join(bind_flags) or None
props = {
"name": self.name,
"translatable": "yes" if self.translatable else None,
"bind-source": self.bind_source,
"bind-property": self.bind_property,
"bind-flags": self.bind_flags,
"name": self.tokens["name"],
"translatable": "yes" if self.tokens["translatable"] else None,
"bind-source": self.tokens["bind_source"],
"bind-property": self.tokens["bind_property"],
"bind-flags": bind_flags_str,
}
if self.objects is not None:
if len(self.children[Object]) == 1:
xml.start_tag("property", **props)
self.objects[0].emit_xml(xml)
self.children[Object][0].emit_xml(xml)
xml.end_tag()
elif self.value is None:
elif self.tokens["value"] is None:
xml.put_self_closing("property", **props)
else:
xml.start_tag("property", **props)
xml.put_text(str(self.value))
xml.put_text(str(self.tokens["value"]))
xml.end_tag()
class Signal(AstNode):
child_type = "signals"
def __init__(self, name=None, handler=None, swapped=False, after=False, object=False, detail_name=None):
super().__init__()
self.name = name
self.handler = handler
self.swapped = swapped
self.after = after
self.object = object
self.detail_name = detail_name
@validate()
def gir_signal(self):
if self.gir_class is not None:
@ -416,7 +264,7 @@ class Signal(AstNode):
if self.gir_signal is None:
raise CompileError(
f"Class {self.gir_class.full_name} does not contain a signal called {self.name}",
did_you_mean=(self.name, self.gir_class.signals.keys())
did_you_mean=(self.tokens["name"], self.gir_class.signals.keys())
)
@ -427,95 +275,57 @@ class Signal(AstNode):
def emit_xml(self, xml: XmlEmitter):
name = self.name
if self.detail_name:
name += "::" + self.detail_name
xml.put_self_closing("signal", name=name, handler=self.handler, swapped="true" if self.swapped else None)
name = self.tokens["name"]
if self.tokens["detail_name"]:
name += "::" + self.tokens["detail_name"]
xml.put_self_closing("signal", name=name, handler=self.tokens["handler"], swapped="true" if self.tokens["swapped"] else None)
class Style(AstNode):
child_type = "style"
def __init__(self, style_classes=None):
super().__init__()
self.style_classes = style_classes or []
def emit_xml(self, xml: XmlEmitter):
xml.start_tag("style")
for style in self.style_classes:
style.emit_xml(xml)
for child in self.children:
child.emit_xml(xml)
xml.end_tag()
class StyleClass(AstNode):
child_type = "style_classes"
def __init__(self, name=None):
super().__init__()
self.name = name
def emit_xml(self, xml):
xml.put_self_closing("class", name=self.name)
xml.put_self_closing("class", name=self.tokens["name"])
class Menu(AstNode):
child_type = "menus"
def __init__(self, tag=None, id=None, menus=None, attributes=None):
super().__init__()
self.tag = tag
self.id = id
self.menus = menus or []
self.attributes = attributes or []
def emit_xml(self, xml: XmlEmitter):
xml.start_tag(self.tag, id=self.id)
for attr in self.attributes:
attr.emit_xml(xml)
for menu in self.menus:
menu.emit_xml(xml)
xml.start_tag(self.tokens["tag"], id=self.tokens["id"])
for child in self.children:
child.emit_xml()
xml.end_tag()
class BaseAttribute(AstNode):
child_type = "attributes"
tag_name: str = ""
def __init__(self, name=None, value=None, translatable=False):
super().__init__()
self.name = name
self.value = value
self.translatable = translatable
def emit_xml(self, xml: XmlEmitter):
xml.start_tag(
self.tag_name,
name=self.name,
translatable="yes" if self.translatable else None,
name=self.tokens["name"],
translatable="yes" if self.tokens["translatable"] else None,
)
xml.put_text(str(self.value))
xml.put_text(str(self.tokens["value"]))
xml.end_tag()
class MenuAttribute(BaseAttribute):
child_type = "attributes"
tag_name = "attribute"
class Layout(AstNode):
child_type = "layout"
def __init__(self, layout_props=None):
super().__init__()
self.layout_props = layout_props or []
def emit_xml(self, xml: XmlEmitter):
xml.start_tag("layout")
for prop in self.layout_props:
prop.emit_xml(xml)
for child in self.children:
child.emit_xml(xml)
xml.end_tag()
class LayoutProperty(BaseAttribute):
child_type = "layout_props"
tag_name = "property"

View file

@ -17,7 +17,99 @@
#
# SPDX-License-Identifier: LGPL-3.0-or-later
import typing as T
from collections import ChainMap, defaultdict
from .errors import *
from .utils import lazy_prop
from .xml_emitter import XmlEmitter
class Children:
""" Allows accessing children by type using array syntax. """
def __init__(self, children):
self._children = children
def __iter__(self):
return iter(self._children)
def __getitem__(self, key):
return [child for child in self._children if isinstance(child, key)]
class AstNode:
""" Base class for nodes in the abstract syntax tree. """
completers: T.List = []
def __init__(self, group, children, tokens, incomplete=False):
self.group = group
self.children = Children(children)
self.tokens = ChainMap(tokens, defaultdict(lambda: None))
self.incomplete = incomplete
self.parent = None
for child in self.children:
child.parent = self
def __init_subclass__(cls):
cls.completers = []
@property
def root(self):
if self.parent is None:
return self
else:
return self.parent.root
@lazy_prop
def errors(self):
return list(self._get_errors())
def _get_errors(self):
for name, attr in self._attrs_by_type(Validator):
try:
getattr(self, name)
except AlreadyCaughtError:
pass
except CompileError as e:
yield e
for child in self.children:
yield from child._get_errors()
def _attrs_by_type(self, attr_type):
for name in dir(type(self)):
item = getattr(type(self), name)
if isinstance(item, attr_type):
yield name, item
def generate(self) -> str:
""" Generates an XML string from the node. """
xml = XmlEmitter()
self.emit_xml(xml)
return xml.result
def emit_xml(self, xml: XmlEmitter):
""" Emits the XML representation of this AST node to the XmlEmitter. """
raise NotImplementedError()
def get_docs(self, idx: int) -> T.Optional[str]:
for name, attr in self._attrs_by_type(Docs):
if attr.token_name:
token = self.group.tokens.get(attr.token_name)
if token and token.start <= idx < token.end:
return getattr(self, name)
else:
return getattr(self, name)
for child in self.children:
if child.group.start <= idx < child.group.end:
docs = child.get_docs(idx)
if docs is not None:
return docs
return None
class Validator:
def __init__(self, func, token_name=None, end_token_name=None):

View file

@ -28,7 +28,7 @@ Pattern = T.List[T.Tuple[TokenType, T.Optional[str]]]
def complete(ast_node: ast.AstNode, tokens: T.List[Token], idx: int) -> T.Iterator[Completion]:
for child in ast_node.child_nodes:
for child in ast_node.children:
if child.group.start <= idx <= child.group.end:
yield from complete(child, tokens, idx)
return

View file

@ -61,7 +61,7 @@ class ParseGroup:
def __init__(self, ast_type, start: int):
self.ast_type = ast_type
self.children: T.Dict[str, T.List[ParseGroup]] = defaultdict()
self.children: T.List[ParseGroup] = []
self.keys: T.Dict[str, T.Any] = {}
self.tokens: T.Dict[str, Token] = {}
self.start = start
@ -69,10 +69,7 @@ class ParseGroup:
self.incomplete = False
def add_child(self, child):
child_type = child.ast_type.child_type
if child_type not in self.children:
self.children[child_type] = []
self.children[child_type].append(child)
self.children.append(child)
def set_val(self, key, val, token):
assert_true(key not in self.keys)
@ -82,21 +79,10 @@ class ParseGroup:
def to_ast(self) -> AstNode:
""" Creates an AST node from the match group. """
children = {
child_type: [child.to_ast() for child in children]
for child_type, children in self.children.items()
}
children = [child.to_ast() for child in self.children]
try:
ast = self.ast_type(**children, **self.keys)
ast.incomplete = self.incomplete
ast.group = self
ast.child_nodes = [c for child_type in children.values() for c in child_type]
for child in ast.child_nodes:
child.parent = ast
return ast
return self.ast_type(self, children, self.keys, incomplete=self.incomplete)
except TypeError as e:
raise CompilerBugError(f"Failed to construct ast.{self.ast_type.__name__} from ParseGroup. See the previous stacktrace.")
@ -119,7 +105,7 @@ class ParseContext:
self.group_keys = {}
self.group_children = []
self.last_group = None
self.group_incomplete = True
self.group_incomplete = False
self.errors = []
self.warnings = []
@ -177,7 +163,6 @@ class ParseContext:
def set_group_incomplete(self):
""" Marks the current match group as incomplete (it could not be fully
parsed, but the parser recovered). """
assert_true(key not in self.group_keys)
self.group_incomplete = True
@ -309,7 +294,7 @@ class Statement(ParseNode):
return False
except CompileError as e:
ctx.errors.append(e)
ctx.group
ctx.set_group_incomplete(True)
return True
token = ctx.peek_token()

View file

@ -1,128 +0,0 @@
# test_parser.py
#
# Copyright 2021 James Westman <james@jwestman.net>
#
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This file is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# SPDX-License-Identifier: LGPL-3.0-or-later
import unittest
from gtkblueprinttool.ast import *
from gtkblueprinttool.errors import PrintableError
from gtkblueprinttool.tokenizer import tokenize
from gtkblueprinttool.parser import parse
class TestParser(unittest.TestCase):
def test_parser(self):
f = """
using Gtk 4.0;
template MyAppWindow : Gtk.ApplicationWindow {
title: _("Hello, world!");
app-defined-prop: 10.5;
[titlebar]
Gtk.HeaderBar header_bar {
}
Gtk.Button {
clicked => on_clicked() swapped;
}
}
menu my_menu {
section {
item {
label: "Run";
target: "run";
}
}
submenu {}
item _("Copy") "copy-symbolic" "app.copy";
}
Label {
style "dim-label", "my-class";
label: "Text";
notify::visible => on_notify_visible();
}
Box {}
"""
tokens = tokenize(f)
ui, errors = parse(tokens)
self.assertIsInstance(ui, UI)
self.assertIsNone(errors)
self.assertEqual(len(ui.errors), 0)
self.assertIsInstance(ui.gtk_directive, GtkDirective)
self.assertEqual(ui.gtk_directive.version, "4.0")
self.assertEqual(len(ui.templates), 1)
template = ui.templates[0]
self.assertEqual(template.name, "MyAppWindow")
self.assertEqual(template.parent_namespace, "Gtk")
self.assertEqual(template.parent_class, "ApplicationWindow")
self.assertEqual(len(template.object_content.properties), 2)
prop = template.object_content.properties[0]
self.assertEqual(prop.name, "title")
self.assertEqual(prop.value, "Hello, world!")
self.assertTrue(prop.translatable)
prop = template.object_content.properties[1]
self.assertEqual(prop.name, "app-defined-prop")
self.assertEqual(prop.value, 10.5)
self.assertFalse(prop.translatable)
self.assertEqual(len(template.object_content.children), 2)
child = template.object_content.children[0]
self.assertEqual(child.child_type, "titlebar")
self.assertEqual(child.object.id, "header_bar")
self.assertEqual(child.object.namespace, "Gtk")
self.assertEqual(child.object.class_name, "HeaderBar")
child = template.object_content.children[1]
self.assertIsNone(child.child_type)
self.assertIsNone(child.object.id)
self.assertEqual(child.object.namespace, "Gtk")
self.assertEqual(child.object.class_name, "Button")
self.assertEqual(len(child.object.object_content.signals), 1)
signal = child.object.object_content.signals[0]
self.assertEqual(signal.name, "clicked")
self.assertEqual(signal.handler, "on_clicked")
self.assertTrue(signal.swapped)
self.assertIsNone(signal.detail_name)
self.assertEqual(len(ui.objects), 2)
obj = ui.objects[0]
self.assertIsNone(obj.namespace)
self.assertEqual(obj.class_name, "Label")
self.assertEqual(len(obj.object_content.properties), 1)
prop = obj.object_content.properties[0]
self.assertEqual(prop.name, "label")
self.assertEqual(prop.value, "Text")
self.assertFalse(prop.translatable)
self.assertEqual(len(obj.object_content.signals), 1)
signal = obj.object_content.signals[0]
self.assertEqual(signal.name, "notify")
self.assertEqual(signal.handler, "on_notify_visible")
self.assertEqual(signal.detail_name, "visible")
self.assertFalse(signal.swapped)
self.assertEqual(len(obj.object_content.style), 1)
style = obj.object_content.style[0]
self.assertEqual(len(style.style_classes), 2)
self.assertEqual([s.name for s in style.style_classes], ["dim-label", "my-class"])