diff --git a/blueprintcompiler/ast_utils.py b/blueprintcompiler/ast_utils.py index 7eebe45..59331f9 100644 --- a/blueprintcompiler/ast_utils.py +++ b/blueprintcompiler/ast_utils.py @@ -22,7 +22,8 @@ from collections import ChainMap, defaultdict from functools import cached_property from .errors import * -from .lsp_utils import SemanticToken +from .lsp_utils import DocumentSymbol, SemanticToken +from .tokenizer import Range TType = T.TypeVar("TType") @@ -54,6 +55,18 @@ class Children: return [child for child in self._children if isinstance(child, key)] +class Ranges: + def __init__(self, ranges: T.Dict[str, Range]): + self._ranges = ranges + + def __getitem__(self, key: T.Union[str, tuple[str, str]]) -> T.Optional[Range]: + if isinstance(key, str): + return self._ranges.get(key) + elif isinstance(key, tuple): + start, end = key + return Range.join(self._ranges.get(start), self._ranges.get(end)) + + TCtx = T.TypeVar("TCtx") TAttr = T.TypeVar("TAttr") @@ -102,6 +115,10 @@ class AstNode: def context(self): return Ctx(self) + @cached_property + def ranges(self): + return Ranges(self.group.ranges) + @cached_property def root(self): if self.parent is None: @@ -109,6 +126,10 @@ class AstNode: else: return self.parent.root + @property + def range(self): + return Range(self.group.start, self.group.end, self.group.text) + def parent_by_type(self, type: T.Type[TType]) -> TType: if self.parent is None: raise CompilerBugError() @@ -175,6 +196,20 @@ class AstNode: for child in self.children: yield from child.get_semantic_tokens() + @property + def document_symbol(self) -> T.Optional[DocumentSymbol]: + return None + + def get_document_symbols(self) -> T.List[DocumentSymbol]: + result = [] + for child in self.children: + if s := child.document_symbol: + s.children = child.get_document_symbols() + result.append(s) + else: + result.extend(child.get_document_symbols()) + return result + def validate_unique_in_parent( self, error: str, check: T.Optional[T.Callable[["AstNode"], bool]] = None ): diff --git a/blueprintcompiler/language/adw_breakpoint.py b/blueprintcompiler/language/adw_breakpoint.py index e71b29a..aec4ab5 100644 --- a/blueprintcompiler/language/adw_breakpoint.py +++ b/blueprintcompiler/language/adw_breakpoint.py @@ -35,6 +35,16 @@ class AdwBreakpointCondition(AstNode): def condition(self) -> str: return self.tokens["condition"] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "condition", + SymbolKind.Property, + self.range, + self.group.tokens["kw"].range, + self.condition, + ) + @docs("kw") def keyword_docs(self): klass = self.root.gir.get_type("Breakpoint", "Adw") @@ -93,6 +103,16 @@ class AdwBreakpointSetter(AstNode): else: return None + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + f"{self.object_id}.{self.property_name}", + SymbolKind.Property, + self.range, + self.group.tokens["object"].range, + self.value.range.text, + ) + @context(ValueTypeCtx) def value_type(self) -> ValueTypeCtx: if self.gir_property is not None: @@ -147,12 +167,25 @@ class AdwBreakpointSetter(AstNode): class AdwBreakpointSetters(AstNode): - grammar = ["setters", Match("{").expected(), Until(AdwBreakpointSetter, "}")] + grammar = [ + Keyword("setters"), + Match("{").expected(), + Until(AdwBreakpointSetter, "}"), + ] @property def setters(self) -> T.List[AdwBreakpointSetter]: return self.children[AdwBreakpointSetter] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "setters", + SymbolKind.Struct, + self.range, + self.group.tokens["setters"].range, + ) + @validate() def container_is_breakpoint(self): validate_parent_type(self, "Adw", "Breakpoint", "breakpoint setters") diff --git a/blueprintcompiler/language/adw_message_dialog.py b/blueprintcompiler/language/adw_message_dialog.py index 98c40cd..fb2be66 100644 --- a/blueprintcompiler/language/adw_message_dialog.py +++ b/blueprintcompiler/language/adw_message_dialog.py @@ -84,6 +84,16 @@ class ExtAdwMessageDialogResponse(AstNode): def value(self) -> StringValue: return self.children[0] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.id, + SymbolKind.Field, + self.range, + self.group.tokens["id"].range, + self.value.range.text, + ) + @context(ValueTypeCtx) def value_type(self) -> ValueTypeCtx: return ValueTypeCtx(StringType()) @@ -108,6 +118,15 @@ class ExtAdwMessageDialog(AstNode): def responses(self) -> T.List[ExtAdwMessageDialogResponse]: return self.children + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "responses", + SymbolKind.Array, + self.range, + self.group.tokens["responses"].range, + ) + @validate("responses") def container_is_message_dialog(self): validate_parent_type(self, "Adw", "MessageDialog", "responses") diff --git a/blueprintcompiler/language/common.py b/blueprintcompiler/language/common.py index e45cddc..e1bfa36 100644 --- a/blueprintcompiler/language/common.py +++ b/blueprintcompiler/language/common.py @@ -46,7 +46,14 @@ from ..gir import ( IntType, StringType, ) -from ..lsp_utils import Completion, CompletionItemKind, SemanticToken, SemanticTokenType +from ..lsp_utils import ( + Completion, + CompletionItemKind, + DocumentSymbol, + SemanticToken, + SemanticTokenType, + SymbolKind, +) from ..parse_tree import * OBJECT_CONTENT_HOOKS = AnyOf() diff --git a/blueprintcompiler/language/gobject_object.py b/blueprintcompiler/language/gobject_object.py index 1a42c0a..16c92e7 100644 --- a/blueprintcompiler/language/gobject_object.py +++ b/blueprintcompiler/language/gobject_object.py @@ -21,6 +21,9 @@ import typing as T from functools import cached_property +from blueprintcompiler.errors import T +from blueprintcompiler.lsp_utils import DocumentSymbol + from .common import * from .response_id import ExtResponse from .types import ClassName, ConcreteClassName @@ -59,8 +62,20 @@ class Object(AstNode): def signature(self) -> str: if self.id: return f"{self.class_name.gir_type.full_name} {self.id}" + elif t := self.class_name.gir_type: + return f"{t.full_name}" else: - return f"{self.class_name.gir_type.full_name}" + return f"{self.class_name.as_string}" + + @property + def document_symbol(self) -> T.Optional[DocumentSymbol]: + return DocumentSymbol( + self.class_name.as_string, + SymbolKind.Object, + self.range, + self.children[ClassName][0].range, + self.id, + ) @property def gir_class(self) -> GirType: diff --git a/blueprintcompiler/language/gobject_property.py b/blueprintcompiler/language/gobject_property.py index 68f8c6d..3d2ac5b 100644 --- a/blueprintcompiler/language/gobject_property.py +++ b/blueprintcompiler/language/gobject_property.py @@ -47,6 +47,21 @@ class Property(AstNode): else: return None + @property + def document_symbol(self) -> DocumentSymbol: + if isinstance(self.value, ObjectValue): + detail = None + else: + detail = self.value.range.text + + return DocumentSymbol( + self.name, + SymbolKind.Property, + self.range, + self.group.tokens["name"].range, + detail, + ) + @validate() def binding_valid(self): if ( diff --git a/blueprintcompiler/language/gobject_signal.py b/blueprintcompiler/language/gobject_signal.py index 721c443..063b2e8 100644 --- a/blueprintcompiler/language/gobject_signal.py +++ b/blueprintcompiler/language/gobject_signal.py @@ -51,12 +51,14 @@ class Signal(AstNode): ] ), "=>", + Mark("detail_start"), Optional(["$", UseLiteral("extern", True)]), UseIdent("handler").expected("the name of a function to handle the signal"), Match("(").expected("argument list"), Optional(UseIdent("object")).expected("object identifier"), Match(")").expected(), ZeroOrMore(SignalFlag), + Mark("detail_end"), ) @property @@ -105,6 +107,16 @@ class Signal(AstNode): def gir_class(self): return self.parent.parent.gir_class + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.full_name, + SymbolKind.Event, + self.range, + self.group.tokens["name"].range, + self.ranges["detail_start", "detail_end"].text, + ) + @validate("handler") def old_extern(self): if not self.tokens["extern"]: diff --git a/blueprintcompiler/language/gtk_a11y.py b/blueprintcompiler/language/gtk_a11y.py index 6a346ec..e9850f5 100644 --- a/blueprintcompiler/language/gtk_a11y.py +++ b/blueprintcompiler/language/gtk_a11y.py @@ -139,6 +139,16 @@ class A11yProperty(BaseAttribute): def value_type(self) -> ValueTypeCtx: return ValueTypeCtx(get_types(self.root.gir).get(self.tokens["name"])) + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.name, + SymbolKind.Field, + self.range, + self.group.tokens["name"].range, + self.value.range.text, + ) + @validate("name") def is_valid_property(self): types = get_types(self.root.gir) @@ -172,6 +182,15 @@ class ExtAccessibility(AstNode): def properties(self) -> T.List[A11yProperty]: return self.children[A11yProperty] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "accessibility", + SymbolKind.Struct, + self.range, + self.group.tokens["accessibility"].range, + ) + @validate("accessibility") def container_is_widget(self): validate_parent_type(self, "Gtk", "Widget", "accessibility properties") diff --git a/blueprintcompiler/language/gtk_combo_box_text.py b/blueprintcompiler/language/gtk_combo_box_text.py index e6c804e..4c6fda9 100644 --- a/blueprintcompiler/language/gtk_combo_box_text.py +++ b/blueprintcompiler/language/gtk_combo_box_text.py @@ -31,13 +31,23 @@ class Item(AstNode): ] @property - def name(self) -> str: + def name(self) -> T.Optional[str]: return self.tokens["name"] @property def value(self) -> StringValue: return self.children[StringValue][0] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.value.range.text, + SymbolKind.String, + self.range, + self.value.range, + self.name, + ) + @validate("name") def unique_in_parent(self): if self.name is not None: @@ -54,6 +64,15 @@ class ExtComboBoxItems(AstNode): "]", ] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "items", + SymbolKind.Array, + self.range, + self.group.tokens["items"].range, + ) + @validate("items") def container_is_combo_box_text(self): validate_parent_type(self, "Gtk", "ComboBoxText", "combo box items") diff --git a/blueprintcompiler/language/gtk_file_filter.py b/blueprintcompiler/language/gtk_file_filter.py index 53cd102..1be83e8 100644 --- a/blueprintcompiler/language/gtk_file_filter.py +++ b/blueprintcompiler/language/gtk_file_filter.py @@ -23,6 +23,15 @@ from .gobject_object import ObjectContent, validate_parent_type class Filters(AstNode): + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.tokens["tag_name"], + SymbolKind.Array, + self.range, + self.group.tokens[self.tokens["tag_name"]].range, + ) + @validate() def container_is_file_filter(self): validate_parent_type(self, "Gtk", "FileFilter", "file filter properties") @@ -46,6 +55,15 @@ class FilterString(AstNode): def item(self) -> str: return self.tokens["name"] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.item, + SymbolKind.String, + self.range, + self.group.tokens["name"].range, + ) + @validate() def unique_in_parent(self): self.validate_unique_in_parent( diff --git a/blueprintcompiler/language/gtk_layout.py b/blueprintcompiler/language/gtk_layout.py index 7632c7a..892e0c6 100644 --- a/blueprintcompiler/language/gtk_layout.py +++ b/blueprintcompiler/language/gtk_layout.py @@ -36,6 +36,16 @@ class LayoutProperty(AstNode): def value(self) -> Value: return self.children[Value][0] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.name, + SymbolKind.Field, + self.range, + self.group.tokens["name"].range, + self.value.range.text, + ) + @context(ValueTypeCtx) def value_type(self) -> ValueTypeCtx: # there isn't really a way to validate these @@ -56,6 +66,15 @@ class ExtLayout(AstNode): Until(LayoutProperty, "}"), ) + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "layout", + SymbolKind.Struct, + self.range, + self.group.tokens["layout"].range, + ) + @validate("layout") def container_is_widget(self): validate_parent_type(self, "Gtk", "Widget", "layout properties") diff --git a/blueprintcompiler/language/gtk_list_item_factory.py b/blueprintcompiler/language/gtk_list_item_factory.py index ba2a27f..7d05422 100644 --- a/blueprintcompiler/language/gtk_list_item_factory.py +++ b/blueprintcompiler/language/gtk_list_item_factory.py @@ -1,5 +1,9 @@ +import typing as T + +from blueprintcompiler.errors import T +from blueprintcompiler.lsp_utils import DocumentSymbol + from ..ast_utils import AstNode, validate -from ..parse_tree import Keyword from .common import * from .contexts import ScopeCtx from .gobject_object import ObjectContent, validate_parent_type @@ -17,6 +21,15 @@ class ExtListItemFactory(AstNode): def signature(self) -> str: return f"template {self.gir_class.full_name}" + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.signature, + SymbolKind.Object, + self.range, + self.group.tokens["id"].range, + ) + @property def type_name(self) -> T.Optional[TypeName]: if len(self.children[TypeName]) == 1: diff --git a/blueprintcompiler/language/gtk_menu.py b/blueprintcompiler/language/gtk_menu.py index 824ec5c..e1bbd57 100644 --- a/blueprintcompiler/language/gtk_menu.py +++ b/blueprintcompiler/language/gtk_menu.py @@ -42,6 +42,16 @@ class Menu(AstNode): else: return "Gio.Menu" + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.tokens["tag"], + SymbolKind.Object, + self.range, + self.group.tokens[self.tokens["tag"]].range, + self.id, + ) + @property def tag(self) -> str: return self.tokens["tag"] @@ -72,6 +82,18 @@ class MenuAttribute(AstNode): def value(self) -> StringValue: return self.children[StringValue][0] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.name, + SymbolKind.Field, + self.range, + self.group.tokens["name"].range + if self.group.tokens["name"] + else self.range, + self.value.range.text, + ) + @context(ValueTypeCtx) def value_type(self) -> ValueTypeCtx: return ValueTypeCtx(None) @@ -98,7 +120,7 @@ menu_attribute = Group( menu_section = Group( Menu, [ - "section", + Keyword("section"), UseLiteral("tag", "section"), Optional(UseIdent("id")), Match("{").expected(), @@ -109,7 +131,7 @@ menu_section = Group( menu_submenu = Group( Menu, [ - "submenu", + Keyword("submenu"), UseLiteral("tag", "submenu"), Optional(UseIdent("id")), Match("{").expected(), @@ -120,7 +142,7 @@ menu_submenu = Group( menu_item = Group( Menu, [ - "item", + Keyword("item"), UseLiteral("tag", "item"), Match("{").expected(), Until(menu_attribute, "}"), @@ -130,7 +152,7 @@ menu_item = Group( menu_item_shorthand = Group( Menu, [ - "item", + Keyword("item"), UseLiteral("tag", "item"), "(", Group( diff --git a/blueprintcompiler/language/gtk_scale.py b/blueprintcompiler/language/gtk_scale.py index 18452e1..504e290 100644 --- a/blueprintcompiler/language/gtk_scale.py +++ b/blueprintcompiler/language/gtk_scale.py @@ -58,6 +58,16 @@ class ExtScaleMark(AstNode): else: return None + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + str(self.value), + SymbolKind.Field, + self.range, + self.group.tokens["mark"].range, + self.label.string if self.label else None, + ) + @docs("position") def position_docs(self) -> T.Optional[str]: if member := self.root.gir.get_type("PositionType", "Gtk").members.get( @@ -88,6 +98,15 @@ class ExtScaleMarks(AstNode): def marks(self) -> T.List[ExtScaleMark]: return self.children + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "marks", + SymbolKind.Array, + self.range, + self.group.tokens["marks"].range, + ) + @validate("marks") def container_is_size_group(self): validate_parent_type(self, "Gtk", "Scale", "scale marks") diff --git a/blueprintcompiler/language/gtk_size_group.py b/blueprintcompiler/language/gtk_size_group.py index 2a10a35..60af861 100644 --- a/blueprintcompiler/language/gtk_size_group.py +++ b/blueprintcompiler/language/gtk_size_group.py @@ -30,6 +30,15 @@ class Widget(AstNode): def name(self) -> str: return self.tokens["name"] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.name, + SymbolKind.Field, + self.range, + self.group.tokens["name"].range, + ) + @validate("name") def obj_widget(self): object = self.context[ScopeCtx].objects.get(self.tokens["name"]) @@ -62,6 +71,15 @@ class ExtSizeGroupWidgets(AstNode): "]", ] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "widgets", + SymbolKind.Array, + self.range, + self.group.tokens["widgets"].range, + ) + @validate("widgets") def container_is_size_group(self): validate_parent_type(self, "Gtk", "SizeGroup", "size group properties") diff --git a/blueprintcompiler/language/gtk_string_list.py b/blueprintcompiler/language/gtk_string_list.py index 455960e..37a46ec 100644 --- a/blueprintcompiler/language/gtk_string_list.py +++ b/blueprintcompiler/language/gtk_string_list.py @@ -30,6 +30,15 @@ class Item(AstNode): def child(self) -> StringValue: return self.children[StringValue][0] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.child.range.text, + SymbolKind.String, + self.range, + self.range, + ) + class ExtStringListStrings(AstNode): grammar = [ @@ -39,6 +48,15 @@ class ExtStringListStrings(AstNode): "]", ] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "strings", + SymbolKind.Array, + self.range, + self.group.tokens["strings"].range, + ) + @validate("items") def container_is_string_list(self): validate_parent_type(self, "Gtk", "StringList", "StringList items") diff --git a/blueprintcompiler/language/gtk_styles.py b/blueprintcompiler/language/gtk_styles.py index 8152b82..26c0e74 100644 --- a/blueprintcompiler/language/gtk_styles.py +++ b/blueprintcompiler/language/gtk_styles.py @@ -29,6 +29,15 @@ class StyleClass(AstNode): def name(self) -> str: return self.tokens["name"] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.name, + SymbolKind.String, + self.range, + self.range, + ) + @validate("name") def unique_in_parent(self): self.validate_unique_in_parent( @@ -44,6 +53,15 @@ class ExtStyles(AstNode): "]", ] + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + "styles", + SymbolKind.Array, + self.range, + self.group.tokens["styles"].range, + ) + @validate("styles") def container_is_widget(self): validate_parent_type(self, "Gtk", "Widget", "style classes") diff --git a/blueprintcompiler/language/gtkbuilder_template.py b/blueprintcompiler/language/gtkbuilder_template.py index 7af5cc8..e1e2131 100644 --- a/blueprintcompiler/language/gtkbuilder_template.py +++ b/blueprintcompiler/language/gtkbuilder_template.py @@ -46,10 +46,19 @@ class Template(Object): @property def signature(self) -> str: - if self.parent_type: - return f"template {self.gir_class.full_name} : {self.parent_type.gir_type.full_name}" + if self.parent_type and self.parent_type.gir_type: + return f"template {self.class_name.as_string} : {self.parent_type.gir_type.full_name}" else: - return f"template {self.gir_class.full_name}" + return f"template {self.class_name.as_string}" + + @property + def document_symbol(self) -> DocumentSymbol: + return DocumentSymbol( + self.signature, + SymbolKind.Object, + self.range, + self.group.tokens["id"].range, + ) @property def gir_class(self) -> GirType: diff --git a/blueprintcompiler/lsp.py b/blueprintcompiler/lsp.py index 54a36c8..ec161a8 100644 --- a/blueprintcompiler/lsp.py +++ b/blueprintcompiler/lsp.py @@ -203,6 +203,7 @@ class LanguageServer: "completionProvider": {}, "codeActionProvider": {}, "hoverProvider": True, + "documentSymbolProvider": True, }, "serverInfo": { "name": "Blueprint", @@ -363,6 +364,31 @@ class LanguageServer: self._send_response(id, actions) + @command("textDocument/documentSymbol") + def document_symbols(self, id, params): + open_file = self._open_files[params["textDocument"]["uri"]] + symbols = open_file.ast.get_document_symbols() + + def to_json(symbol: DocumentSymbol): + result = { + "name": symbol.name, + "kind": symbol.kind, + "range": utils.idxs_to_range( + symbol.range.start, symbol.range.end, open_file.text + ), + "selectionRange": utils.idxs_to_range( + symbol.selection_range.start, + symbol.selection_range.end, + open_file.text, + ), + "children": [to_json(child) for child in symbol.children], + } + if symbol.detail is not None: + result["detail"] = symbol.detail + return result + + self._send_response(id, [to_json(symbol) for symbol in symbols]) + def _send_file_updates(self, open_file: OpenFile): self._send_notification( "textDocument/publishDiagnostics", diff --git a/blueprintcompiler/lsp_utils.py b/blueprintcompiler/lsp_utils.py index 2d1072e..2225ba2 100644 --- a/blueprintcompiler/lsp_utils.py +++ b/blueprintcompiler/lsp_utils.py @@ -20,9 +20,10 @@ import enum import typing as T -from dataclasses import dataclass +from dataclasses import dataclass, field from .errors import * +from .tokenizer import Range from .utils import * @@ -129,3 +130,42 @@ class SemanticToken: start: int end: int type: SemanticTokenType + + +class SymbolKind(enum.IntEnum): + File = 1 + Module = 2 + Namespace = 3 + Package = 4 + Class = 5 + Method = 6 + Property = 7 + Field = 8 + Constructor = 9 + Enum = 10 + Interface = 11 + Function = 12 + Variable = 13 + Constant = 14 + String = 15 + Number = 16 + Boolean = 17 + Array = 18 + Object = 19 + Key = 20 + Null = 21 + EnumMember = 22 + Struct = 23 + Event = 24 + Operator = 25 + TypeParameter = 26 + + +@dataclass +class DocumentSymbol: + name: str + kind: SymbolKind + range: Range + selection_range: Range + detail: T.Optional[str] = None + children: T.List["DocumentSymbol"] = field(default_factory=list) diff --git a/blueprintcompiler/parse_tree.py b/blueprintcompiler/parse_tree.py index 3ddac69..b29bd96 100644 --- a/blueprintcompiler/parse_tree.py +++ b/blueprintcompiler/parse_tree.py @@ -20,7 +20,6 @@ """ Utilities for parsing an AST from a token stream. """ import typing as T -from collections import defaultdict from enum import Enum from .ast_utils import AstNode @@ -31,7 +30,7 @@ from .errors import ( UnexpectedTokenError, assert_true, ) -from .tokenizer import Token, TokenType +from .tokenizer import Range, Token, TokenType SKIP_TOKENS = [TokenType.COMMENT, TokenType.WHITESPACE] @@ -63,14 +62,16 @@ class ParseGroup: be converted to AST nodes by passing the children and key=value pairs to the AST node constructor.""" - def __init__(self, ast_type: T.Type[AstNode], start: int): + def __init__(self, ast_type: T.Type[AstNode], start: int, text: str): self.ast_type = ast_type self.children: T.List[ParseGroup] = [] self.keys: T.Dict[str, T.Any] = {} self.tokens: T.Dict[str, T.Optional[Token]] = {} + self.ranges: T.Dict[str, Range] = {} self.start = start self.end: T.Optional[int] = None self.incomplete = False + self.text = text def add_child(self, child: "ParseGroup"): self.children.append(child) @@ -81,6 +82,10 @@ class ParseGroup: self.keys[key] = val self.tokens[key] = token + def set_range(self, key: str, range: Range): + assert_true(key not in self.ranges) + self.ranges[key] = range + def to_ast(self): """Creates an AST node from the match group.""" children = [child.to_ast() for child in self.children] @@ -104,8 +109,9 @@ class ParseGroup: class ParseContext: """Contains the state of the parser.""" - def __init__(self, tokens: T.List[Token], index=0): + def __init__(self, tokens: T.List[Token], text: str, index=0): self.tokens = tokens + self.text = text self.binding_power = 0 self.index = index @@ -113,6 +119,7 @@ class ParseContext: self.group: T.Optional[ParseGroup] = None self.group_keys: T.Dict[str, T.Tuple[T.Any, T.Optional[Token]]] = {} self.group_children: T.List[ParseGroup] = [] + self.group_ranges: T.Dict[str, Range] = {} self.last_group: T.Optional[ParseGroup] = None self.group_incomplete = False @@ -124,7 +131,7 @@ class ParseContext: context will be used to parse one node. If parsing is successful, the new context will be applied to "self". If parsing fails, the new context will be discarded.""" - ctx = ParseContext(self.tokens, self.index) + ctx = ParseContext(self.tokens, self.text, self.index) ctx.errors = self.errors ctx.warnings = self.warnings ctx.binding_power = self.binding_power @@ -140,6 +147,8 @@ class ParseContext: other.group.set_val(key, val, token) for child in other.group_children: other.group.add_child(child) + for key, range in other.group_ranges.items(): + other.group.set_range(key, range) other.group.end = other.tokens[other.index - 1].end other.group.incomplete = other.group_incomplete self.group_children.append(other.group) @@ -148,6 +157,7 @@ class ParseContext: # its matched values self.group_keys = {**self.group_keys, **other.group_keys} self.group_children += other.group_children + self.group_ranges = {**self.group_ranges, **other.group_ranges} self.group_incomplete |= other.group_incomplete self.index = other.index @@ -161,13 +171,19 @@ class ParseContext: def start_group(self, ast_type: T.Type[AstNode]): """Sets this context to have its own match group.""" assert_true(self.group is None) - self.group = ParseGroup(ast_type, self.tokens[self.index].start) + self.group = ParseGroup(ast_type, self.tokens[self.index].start, self.text) def set_group_val(self, key: str, value: T.Any, token: T.Optional[Token]): """Sets a matched key=value pair on the current match group.""" assert_true(key not in self.group_keys) self.group_keys[key] = (value, token) + def set_mark(self, key: str): + """Sets a zero-length range on the current match group at the current position.""" + self.group_ranges[key] = Range( + self.tokens[self.index].start, self.tokens[self.index].start, self.text + ) + def set_group_incomplete(self): """Marks the current match group as incomplete (it could not be fully parsed, but the parser recovered).""" @@ -604,6 +620,15 @@ class Keyword(ParseNode): return str(token) == self.kw +class Mark(ParseNode): + def __init__(self, key: str): + self.key = key + + def _parse(self, ctx: ParseContext): + ctx.set_mark(self.key) + return True + + def to_parse_node(value) -> ParseNode: if isinstance(value, str): return Match(value) diff --git a/blueprintcompiler/parser.py b/blueprintcompiler/parser.py index 8b50de5..a9cc0ae 100644 --- a/blueprintcompiler/parser.py +++ b/blueprintcompiler/parser.py @@ -30,7 +30,8 @@ def parse( """Parses a list of tokens into an abstract syntax tree.""" try: - ctx = ParseContext(tokens) + original_text = tokens[0].string if len(tokens) else "" + ctx = ParseContext(tokens, original_text) AnyOf(UI).parse(ctx) ast_node = ctx.last_group.to_ast() if ctx.last_group else None diff --git a/blueprintcompiler/tokenizer.py b/blueprintcompiler/tokenizer.py index bde8dd1..64abf0f 100644 --- a/blueprintcompiler/tokenizer.py +++ b/blueprintcompiler/tokenizer.py @@ -20,6 +20,7 @@ import re import typing as T +from dataclasses import dataclass from enum import Enum from .errors import CompileError, CompilerBugError @@ -62,6 +63,10 @@ class Token: def __str__(self) -> str: return self.string[self.start : self.end] + @property + def range(self) -> "Range": + return Range(self.start, self.end, self.string) + def get_number(self) -> T.Union[int, float]: if self.type != TokenType.NUMBER: raise CompilerBugError() @@ -103,3 +108,22 @@ def _tokenize(ui_ml: str): def tokenize(data: str) -> T.List[Token]: return list(_tokenize(data)) + + +@dataclass +class Range: + start: int + end: int + original_text: str + + @property + def text(self) -> str: + return self.original_text[self.start : self.end] + + @staticmethod + def join(a: T.Optional["Range"], b: T.Optional["Range"]) -> T.Optional["Range"]: + if a is None: + return b + if b is None: + return a + return Range(min(a.start, b.start), max(a.end, b.end), a.original_text) diff --git a/tests/test_samples.py b/tests/test_samples.py index bacca80..9ad8cc0 100644 --- a/tests/test_samples.py +++ b/tests/test_samples.py @@ -40,13 +40,12 @@ from blueprintcompiler.tokenizer import Token, TokenType, tokenize class TestSamples(unittest.TestCase): - def assert_docs_dont_crash(self, text, ast): + def assert_ast_doesnt_crash(self, text, tokens, ast): for i in range(len(text)): ast.get_docs(i) - - def assert_completions_dont_crash(self, text, ast, tokens): for i in range(len(text)): list(complete(ast, tokens, i)) + ast.get_document_symbols() def assert_sample(self, name, skip_run=False): print(f'assert_sample("{name}", skip_run={skip_run})') @@ -79,8 +78,7 @@ class TestSamples(unittest.TestCase): print("\n".join(diff)) raise AssertionError() - self.assert_docs_dont_crash(blueprint, ast) - self.assert_completions_dont_crash(blueprint, ast, tokens) + self.assert_ast_doesnt_crash(blueprint, tokens, ast) except PrintableError as e: # pragma: no cover e.pretty_print(name + ".blp", blueprint) raise AssertionError() @@ -105,8 +103,7 @@ class TestSamples(unittest.TestCase): ast, errors, warnings = parser.parse(tokens) if ast is not None: - self.assert_docs_dont_crash(blueprint, ast) - self.assert_completions_dont_crash(blueprint, ast, tokens) + self.assert_ast_doesnt_crash(blueprint, tokens, ast) if errors: raise errors