lsp: Add document outline

blueprintcompiler/ast_utils.py

@@ -22,7 +22,8 @@ from collections import ChainMap, defaultdict
 from functools import cached_property
 
 from .errors import *
-from .lsp_utils import SemanticToken
+from .lsp_utils import DocumentSymbol, SemanticToken
+from .tokenizer import Range
 
 TType = T.TypeVar("TType")
 
@@ -54,6 +55,18 @@ class Children:
             return [child for child in self._children if isinstance(child, key)]
 
 
+class Ranges:
+    def __init__(self, ranges: T.Dict[str, Range]):
+        self._ranges = ranges
+
+    def __getitem__(self, key: T.Union[str, tuple[str, str]]) -> T.Optional[Range]:
+        if isinstance(key, str):
+            return self._ranges.get(key)
+        elif isinstance(key, tuple):
+            start, end = key
+            return Range.join(self._ranges.get(start), self._ranges.get(end))
+
+
 TCtx = T.TypeVar("TCtx")
 TAttr = T.TypeVar("TAttr")
 
@@ -102,6 +115,10 @@ class AstNode:
     def context(self):
         return Ctx(self)
 
+    @cached_property
+    def ranges(self):
+        return Ranges(self.group.ranges)
+
     @cached_property
     def root(self):
         if self.parent is None:
@@ -109,6 +126,10 @@ class AstNode:
         else:
             return self.parent.root
 
+    @property
+    def range(self):
+        return Range(self.group.start, self.group.end, self.group.text)
+
     def parent_by_type(self, type: T.Type[TType]) -> TType:
         if self.parent is None:
             raise CompilerBugError()
@@ -175,6 +196,20 @@ class AstNode:
         for child in self.children:
             yield from child.get_semantic_tokens()
 
+    @property
+    def document_symbol(self) -> T.Optional[DocumentSymbol]:
+        return None
+
+    def get_document_symbols(self) -> T.List[DocumentSymbol]:
+        result = []
+        for child in self.children:
+            if s := child.document_symbol:
+                s.children = child.get_document_symbols()
+                result.append(s)
+            else:
+                result.extend(child.get_document_symbols())
+        return result
+
     def validate_unique_in_parent(
         self, error: str, check: T.Optional[T.Callable[["AstNode"], bool]] = None
     ):

blueprintcompiler/language/adw_breakpoint.py

@@ -35,6 +35,16 @@ class AdwBreakpointCondition(AstNode):
     def condition(self) -> str:
         return self.tokens["condition"]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "condition",
+            SymbolKind.Property,
+            self.range,
+            self.group.tokens["kw"].range,
+            self.condition,
+        )
+
     @docs("kw")
     def keyword_docs(self):
         klass = self.root.gir.get_type("Breakpoint", "Adw")
@@ -93,6 +103,16 @@ class AdwBreakpointSetter(AstNode):
         else:
             return None
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            f"{self.object_id}.{self.property_name}",
+            SymbolKind.Property,
+            self.range,
+            self.group.tokens["object"].range,
+            self.value.range.text,
+        )
+
     @context(ValueTypeCtx)
     def value_type(self) -> ValueTypeCtx:
         if self.gir_property is not None:
@@ -147,12 +167,25 @@ class AdwBreakpointSetter(AstNode):
 
 
 class AdwBreakpointSetters(AstNode):
-    grammar = ["setters", Match("{").expected(), Until(AdwBreakpointSetter, "}")]
+    grammar = [
+        Keyword("setters"),
+        Match("{").expected(),
+        Until(AdwBreakpointSetter, "}"),
+    ]
 
     @property
     def setters(self) -> T.List[AdwBreakpointSetter]:
         return self.children[AdwBreakpointSetter]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "setters",
+            SymbolKind.Struct,
+            self.range,
+            self.group.tokens["setters"].range,
+        )
+
     @validate()
     def container_is_breakpoint(self):
         validate_parent_type(self, "Adw", "Breakpoint", "breakpoint setters")

blueprintcompiler/language/adw_message_dialog.py

@@ -84,6 +84,16 @@ class ExtAdwMessageDialogResponse(AstNode):
     def value(self) -> StringValue:
         return self.children[0]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.id,
+            SymbolKind.Field,
+            self.range,
+            self.group.tokens["id"].range,
+            self.value.range.text,
+        )
+
     @context(ValueTypeCtx)
     def value_type(self) -> ValueTypeCtx:
         return ValueTypeCtx(StringType())
@@ -108,6 +118,15 @@ class ExtAdwMessageDialog(AstNode):
     def responses(self) -> T.List[ExtAdwMessageDialogResponse]:
         return self.children
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "responses",
+            SymbolKind.Array,
+            self.range,
+            self.group.tokens["responses"].range,
+        )
+
     @validate("responses")
     def container_is_message_dialog(self):
         validate_parent_type(self, "Adw", "MessageDialog", "responses")

blueprintcompiler/language/common.py

@@ -46,7 +46,14 @@ from ..gir import (
     IntType,
     StringType,
 )
-from ..lsp_utils import Completion, CompletionItemKind, SemanticToken, SemanticTokenType
+from ..lsp_utils import (
+    Completion,
+    CompletionItemKind,
+    DocumentSymbol,
+    SemanticToken,
+    SemanticTokenType,
+    SymbolKind,
+)
 from ..parse_tree import *
 
 OBJECT_CONTENT_HOOKS = AnyOf()

blueprintcompiler/language/gobject_object.py

@@ -21,6 +21,9 @@
 import typing as T
 from functools import cached_property
 
+from blueprintcompiler.errors import T
+from blueprintcompiler.lsp_utils import DocumentSymbol
+
 from .common import *
 from .response_id import ExtResponse
 from .types import ClassName, ConcreteClassName
@@ -59,8 +62,20 @@ class Object(AstNode):
     def signature(self) -> str:
         if self.id:
             return f"{self.class_name.gir_type.full_name} {self.id}"
+        elif t := self.class_name.gir_type:
+            return f"{t.full_name}"
         else:
-            return f"{self.class_name.gir_type.full_name}"
+            return f"{self.class_name.as_string}"
+
+    @property
+    def document_symbol(self) -> T.Optional[DocumentSymbol]:
+        return DocumentSymbol(
+            self.class_name.as_string,
+            SymbolKind.Object,
+            self.range,
+            self.children[ClassName][0].range,
+            self.id,
+        )
 
     @property
     def gir_class(self) -> GirType:

blueprintcompiler/language/gobject_property.py

@@ -47,6 +47,21 @@ class Property(AstNode):
         else:
             return None
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        if isinstance(self.value, ObjectValue):
+            detail = None
+        else:
+            detail = self.value.range.text
+
+        return DocumentSymbol(
+            self.name,
+            SymbolKind.Property,
+            self.range,
+            self.group.tokens["name"].range,
+            detail,
+        )
+
     @validate()
     def binding_valid(self):
         if (

blueprintcompiler/language/gobject_signal.py

@@ -51,12 +51,14 @@ class Signal(AstNode):
             ]
         ),
         "=>",
+        Mark("detail_start"),
         Optional(["$", UseLiteral("extern", True)]),
         UseIdent("handler").expected("the name of a function to handle the signal"),
         Match("(").expected("argument list"),
         Optional(UseIdent("object")).expected("object identifier"),
         Match(")").expected(),
         ZeroOrMore(SignalFlag),
+        Mark("detail_end"),
     )
 
     @property
@@ -105,6 +107,16 @@ class Signal(AstNode):
     def gir_class(self):
         return self.parent.parent.gir_class
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.full_name,
+            SymbolKind.Event,
+            self.range,
+            self.group.tokens["name"].range,
+            self.ranges["detail_start", "detail_end"].text,
+        )
+
     @validate("handler")
     def old_extern(self):
         if not self.tokens["extern"]:

blueprintcompiler/language/gtk_a11y.py

@@ -139,6 +139,16 @@ class A11yProperty(BaseAttribute):
     def value_type(self) -> ValueTypeCtx:
         return ValueTypeCtx(get_types(self.root.gir).get(self.tokens["name"]))
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.name,
+            SymbolKind.Field,
+            self.range,
+            self.group.tokens["name"].range,
+            self.value.range.text,
+        )
+
     @validate("name")
     def is_valid_property(self):
         types = get_types(self.root.gir)
@@ -172,6 +182,15 @@ class ExtAccessibility(AstNode):
     def properties(self) -> T.List[A11yProperty]:
         return self.children[A11yProperty]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "accessibility",
+            SymbolKind.Struct,
+            self.range,
+            self.group.tokens["accessibility"].range,
+        )
+
     @validate("accessibility")
     def container_is_widget(self):
         validate_parent_type(self, "Gtk", "Widget", "accessibility properties")

blueprintcompiler/language/gtk_combo_box_text.py

@@ -31,13 +31,23 @@ class Item(AstNode):
     ]
 
     @property
-    def name(self) -> str:
+    def name(self) -> T.Optional[str]:
         return self.tokens["name"]
 
     @property
     def value(self) -> StringValue:
         return self.children[StringValue][0]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.value.range.text,
+            SymbolKind.String,
+            self.range,
+            self.value.range,
+            self.name,
+        )
+
     @validate("name")
     def unique_in_parent(self):
         if self.name is not None:
@@ -54,6 +64,15 @@ class ExtComboBoxItems(AstNode):
         "]",
     ]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "items",
+            SymbolKind.Array,
+            self.range,
+            self.group.tokens["items"].range,
+        )
+
     @validate("items")
     def container_is_combo_box_text(self):
         validate_parent_type(self, "Gtk", "ComboBoxText", "combo box items")

blueprintcompiler/language/gtk_file_filter.py

@@ -23,6 +23,15 @@ from .gobject_object import ObjectContent, validate_parent_type
 
 
 class Filters(AstNode):
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.tokens["tag_name"],
+            SymbolKind.Array,
+            self.range,
+            self.group.tokens[self.tokens["tag_name"]].range,
+        )
+
     @validate()
     def container_is_file_filter(self):
         validate_parent_type(self, "Gtk", "FileFilter", "file filter properties")
@@ -46,6 +55,15 @@ class FilterString(AstNode):
     def item(self) -> str:
         return self.tokens["name"]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.item,
+            SymbolKind.String,
+            self.range,
+            self.group.tokens["name"].range,
+        )
+
     @validate()
     def unique_in_parent(self):
         self.validate_unique_in_parent(

blueprintcompiler/language/gtk_layout.py

@@ -36,6 +36,16 @@ class LayoutProperty(AstNode):
     def value(self) -> Value:
         return self.children[Value][0]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.name,
+            SymbolKind.Field,
+            self.range,
+            self.group.tokens["name"].range,
+            self.value.range.text,
+        )
+
     @context(ValueTypeCtx)
     def value_type(self) -> ValueTypeCtx:
         # there isn't really a way to validate these
@@ -56,6 +66,15 @@ class ExtLayout(AstNode):
         Until(LayoutProperty, "}"),
     )
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "layout",
+            SymbolKind.Struct,
+            self.range,
+            self.group.tokens["layout"].range,
+        )
+
     @validate("layout")
     def container_is_widget(self):
         validate_parent_type(self, "Gtk", "Widget", "layout properties")

blueprintcompiler/language/gtk_list_item_factory.py

@@ -1,5 +1,9 @@
+import typing as T
+
+from blueprintcompiler.errors import T
+from blueprintcompiler.lsp_utils import DocumentSymbol
+
 from ..ast_utils import AstNode, validate
-from ..parse_tree import Keyword
 from .common import *
 from .contexts import ScopeCtx
 from .gobject_object import ObjectContent, validate_parent_type
@@ -17,6 +21,15 @@ class ExtListItemFactory(AstNode):
     def signature(self) -> str:
         return f"template {self.gir_class.full_name}"
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.signature,
+            SymbolKind.Object,
+            self.range,
+            self.group.tokens["id"].range,
+        )
+
     @property
     def type_name(self) -> T.Optional[TypeName]:
         if len(self.children[TypeName]) == 1:

blueprintcompiler/language/gtk_menu.py

@@ -42,6 +42,16 @@ class Menu(AstNode):
         else:
             return "Gio.Menu"
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.tokens["tag"],
+            SymbolKind.Object,
+            self.range,
+            self.group.tokens[self.tokens["tag"]].range,
+            self.id,
+        )
+
     @property
     def tag(self) -> str:
         return self.tokens["tag"]
@@ -72,6 +82,18 @@ class MenuAttribute(AstNode):
     def value(self) -> StringValue:
         return self.children[StringValue][0]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.name,
+            SymbolKind.Field,
+            self.range,
+            self.group.tokens["name"].range
+            if self.group.tokens["name"]
+            else self.range,
+            self.value.range.text,
+        )
+
     @context(ValueTypeCtx)
     def value_type(self) -> ValueTypeCtx:
         return ValueTypeCtx(None)
@@ -98,7 +120,7 @@ menu_attribute = Group(
 menu_section = Group(
     Menu,
     [
-        "section",
+        Keyword("section"),
         UseLiteral("tag", "section"),
         Optional(UseIdent("id")),
         Match("{").expected(),
@@ -109,7 +131,7 @@ menu_section = Group(
 menu_submenu = Group(
     Menu,
     [
-        "submenu",
+        Keyword("submenu"),
         UseLiteral("tag", "submenu"),
         Optional(UseIdent("id")),
         Match("{").expected(),
@@ -120,7 +142,7 @@ menu_submenu = Group(
 menu_item = Group(
     Menu,
     [
-        "item",
+        Keyword("item"),
         UseLiteral("tag", "item"),
         Match("{").expected(),
         Until(menu_attribute, "}"),
@@ -130,7 +152,7 @@ menu_item = Group(
 menu_item_shorthand = Group(
     Menu,
     [
-        "item",
+        Keyword("item"),
         UseLiteral("tag", "item"),
         "(",
         Group(

blueprintcompiler/language/gtk_scale.py

@@ -58,6 +58,16 @@ class ExtScaleMark(AstNode):
         else:
             return None
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            str(self.value),
+            SymbolKind.Field,
+            self.range,
+            self.group.tokens["mark"].range,
+            self.label.string if self.label else None,
+        )
+
     @docs("position")
     def position_docs(self) -> T.Optional[str]:
         if member := self.root.gir.get_type("PositionType", "Gtk").members.get(
@@ -88,6 +98,15 @@ class ExtScaleMarks(AstNode):
     def marks(self) -> T.List[ExtScaleMark]:
         return self.children
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "marks",
+            SymbolKind.Array,
+            self.range,
+            self.group.tokens["marks"].range,
+        )
+
     @validate("marks")
     def container_is_size_group(self):
         validate_parent_type(self, "Gtk", "Scale", "scale marks")

blueprintcompiler/language/gtk_size_group.py

@@ -30,6 +30,15 @@ class Widget(AstNode):
     def name(self) -> str:
         return self.tokens["name"]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.name,
+            SymbolKind.Field,
+            self.range,
+            self.group.tokens["name"].range,
+        )
+
     @validate("name")
     def obj_widget(self):
         object = self.context[ScopeCtx].objects.get(self.tokens["name"])
@@ -62,6 +71,15 @@ class ExtSizeGroupWidgets(AstNode):
         "]",
     ]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "widgets",
+            SymbolKind.Array,
+            self.range,
+            self.group.tokens["widgets"].range,
+        )
+
     @validate("widgets")
     def container_is_size_group(self):
         validate_parent_type(self, "Gtk", "SizeGroup", "size group properties")

blueprintcompiler/language/gtk_string_list.py

@@ -30,6 +30,15 @@ class Item(AstNode):
     def child(self) -> StringValue:
         return self.children[StringValue][0]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.child.range.text,
+            SymbolKind.String,
+            self.range,
+            self.range,
+        )
+
 
 class ExtStringListStrings(AstNode):
     grammar = [
@@ -39,6 +48,15 @@ class ExtStringListStrings(AstNode):
         "]",
     ]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "strings",
+            SymbolKind.Array,
+            self.range,
+            self.group.tokens["strings"].range,
+        )
+
     @validate("items")
     def container_is_string_list(self):
         validate_parent_type(self, "Gtk", "StringList", "StringList items")

blueprintcompiler/language/gtk_styles.py

@@ -29,6 +29,15 @@ class StyleClass(AstNode):
     def name(self) -> str:
         return self.tokens["name"]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.name,
+            SymbolKind.String,
+            self.range,
+            self.range,
+        )
+
     @validate("name")
     def unique_in_parent(self):
         self.validate_unique_in_parent(
@@ -44,6 +53,15 @@ class ExtStyles(AstNode):
         "]",
     ]
 
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            "styles",
+            SymbolKind.Array,
+            self.range,
+            self.group.tokens["styles"].range,
+        )
+
     @validate("styles")
     def container_is_widget(self):
         validate_parent_type(self, "Gtk", "Widget", "style classes")

blueprintcompiler/language/gtkbuilder_template.py

@@ -46,10 +46,19 @@ class Template(Object):
 
     @property
     def signature(self) -> str:
-        if self.parent_type:
+        if self.parent_type and self.parent_type.gir_type:
-            return f"template {self.gir_class.full_name} : {self.parent_type.gir_type.full_name}"
+            return f"template {self.class_name.as_string} : {self.parent_type.gir_type.full_name}"
         else:
-            return f"template {self.gir_class.full_name}"
+            return f"template {self.class_name.as_string}"
+
+    @property
+    def document_symbol(self) -> DocumentSymbol:
+        return DocumentSymbol(
+            self.signature,
+            SymbolKind.Object,
+            self.range,
+            self.group.tokens["id"].range,
+        )
 
     @property
     def gir_class(self) -> GirType:

blueprintcompiler/lsp.py

@@ -203,6 +203,7 @@ class LanguageServer:
                     "completionProvider": {},
                     "codeActionProvider": {},
                     "hoverProvider": True,
+                    "documentSymbolProvider": True,
                 },
                 "serverInfo": {
                     "name": "Blueprint",
@@ -363,6 +364,31 @@ class LanguageServer:
 
         self._send_response(id, actions)
 
+    @command("textDocument/documentSymbol")
+    def document_symbols(self, id, params):
+        open_file = self._open_files[params["textDocument"]["uri"]]
+        symbols = open_file.ast.get_document_symbols()
+
+        def to_json(symbol: DocumentSymbol):
+            result = {
+                "name": symbol.name,
+                "kind": symbol.kind,
+                "range": utils.idxs_to_range(
+                    symbol.range.start, symbol.range.end, open_file.text
+                ),
+                "selectionRange": utils.idxs_to_range(
+                    symbol.selection_range.start,
+                    symbol.selection_range.end,
+                    open_file.text,
+                ),
+                "children": [to_json(child) for child in symbol.children],
+            }
+            if symbol.detail is not None:
+                result["detail"] = symbol.detail
+            return result
+
+        self._send_response(id, [to_json(symbol) for symbol in symbols])
+
     def _send_file_updates(self, open_file: OpenFile):
         self._send_notification(
             "textDocument/publishDiagnostics",

blueprintcompiler/lsp_utils.py

@@ -20,9 +20,10 @@
 
 import enum
 import typing as T
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from .errors import *
+from .tokenizer import Range
 from .utils import *
 
 
@@ -129,3 +130,42 @@ class SemanticToken:
     start: int
     end: int
     type: SemanticTokenType
+
+
+class SymbolKind(enum.IntEnum):
+    File = 1
+    Module = 2
+    Namespace = 3
+    Package = 4
+    Class = 5
+    Method = 6
+    Property = 7
+    Field = 8
+    Constructor = 9
+    Enum = 10
+    Interface = 11
+    Function = 12
+    Variable = 13
+    Constant = 14
+    String = 15
+    Number = 16
+    Boolean = 17
+    Array = 18
+    Object = 19
+    Key = 20
+    Null = 21
+    EnumMember = 22
+    Struct = 23
+    Event = 24
+    Operator = 25
+    TypeParameter = 26
+
+
+@dataclass
+class DocumentSymbol:
+    name: str
+    kind: SymbolKind
+    range: Range
+    selection_range: Range
+    detail: T.Optional[str] = None
+    children: T.List["DocumentSymbol"] = field(default_factory=list)

blueprintcompiler/parse_tree.py

@@ -20,7 +20,6 @@
 """ Utilities for parsing an AST from a token stream. """
 
 import typing as T
-from collections import defaultdict
 from enum import Enum
 
 from .ast_utils import AstNode
@@ -31,7 +30,7 @@ from .errors import (
     UnexpectedTokenError,
     assert_true,
 )
-from .tokenizer import Token, TokenType
+from .tokenizer import Range, Token, TokenType
 
 SKIP_TOKENS = [TokenType.COMMENT, TokenType.WHITESPACE]
 
@@ -63,14 +62,16 @@ class ParseGroup:
     be converted to AST nodes by passing the children and key=value pairs to
     the AST node constructor."""
 
-    def __init__(self, ast_type: T.Type[AstNode], start: int):
+    def __init__(self, ast_type: T.Type[AstNode], start: int, text: str):
         self.ast_type = ast_type
         self.children: T.List[ParseGroup] = []
         self.keys: T.Dict[str, T.Any] = {}
         self.tokens: T.Dict[str, T.Optional[Token]] = {}
+        self.ranges: T.Dict[str, Range] = {}
         self.start = start
         self.end: T.Optional[int] = None
         self.incomplete = False
+        self.text = text
 
     def add_child(self, child: "ParseGroup"):
         self.children.append(child)
@@ -81,6 +82,10 @@ class ParseGroup:
         self.keys[key] = val
         self.tokens[key] = token
 
+    def set_range(self, key: str, range: Range):
+        assert_true(key not in self.ranges)
+        self.ranges[key] = range
+
     def to_ast(self):
         """Creates an AST node from the match group."""
         children = [child.to_ast() for child in self.children]
@@ -104,8 +109,9 @@ class ParseGroup:
 class ParseContext:
     """Contains the state of the parser."""
 
-    def __init__(self, tokens: T.List[Token], index=0):
+    def __init__(self, tokens: T.List[Token], text: str, index=0):
         self.tokens = tokens
+        self.text = text
 
         self.binding_power = 0
         self.index = index
@@ -113,6 +119,7 @@ class ParseContext:
         self.group: T.Optional[ParseGroup] = None
         self.group_keys: T.Dict[str, T.Tuple[T.Any, T.Optional[Token]]] = {}
         self.group_children: T.List[ParseGroup] = []
+        self.group_ranges: T.Dict[str, Range] = {}
         self.last_group: T.Optional[ParseGroup] = None
         self.group_incomplete = False
 
@@ -124,7 +131,7 @@ class ParseContext:
         context will be used to parse one node. If parsing is successful, the
         new context will be applied to "self". If parsing fails, the new
         context will be discarded."""
-        ctx = ParseContext(self.tokens, self.index)
+        ctx = ParseContext(self.tokens, self.text, self.index)
         ctx.errors = self.errors
         ctx.warnings = self.warnings
         ctx.binding_power = self.binding_power
@@ -140,6 +147,8 @@ class ParseContext:
                 other.group.set_val(key, val, token)
             for child in other.group_children:
                 other.group.add_child(child)
+            for key, range in other.group_ranges.items():
+                other.group.set_range(key, range)
             other.group.end = other.tokens[other.index - 1].end
             other.group.incomplete = other.group_incomplete
             self.group_children.append(other.group)
@@ -148,6 +157,7 @@ class ParseContext:
             # its matched values
             self.group_keys = {**self.group_keys, **other.group_keys}
             self.group_children += other.group_children
+            self.group_ranges = {**self.group_ranges, **other.group_ranges}
             self.group_incomplete |= other.group_incomplete
 
         self.index = other.index
@@ -161,13 +171,19 @@ class ParseContext:
     def start_group(self, ast_type: T.Type[AstNode]):
         """Sets this context to have its own match group."""
         assert_true(self.group is None)
-        self.group = ParseGroup(ast_type, self.tokens[self.index].start)
+        self.group = ParseGroup(ast_type, self.tokens[self.index].start, self.text)
 
     def set_group_val(self, key: str, value: T.Any, token: T.Optional[Token]):
         """Sets a matched key=value pair on the current match group."""
         assert_true(key not in self.group_keys)
         self.group_keys[key] = (value, token)
 
+    def set_mark(self, key: str):
+        """Sets a zero-length range on the current match group at the current position."""
+        self.group_ranges[key] = Range(
+            self.tokens[self.index].start, self.tokens[self.index].start, self.text
+        )
+
     def set_group_incomplete(self):
         """Marks the current match group as incomplete (it could not be fully
         parsed, but the parser recovered)."""
@@ -604,6 +620,15 @@ class Keyword(ParseNode):
         return str(token) == self.kw
 
 
+class Mark(ParseNode):
+    def __init__(self, key: str):
+        self.key = key
+
+    def _parse(self, ctx: ParseContext):
+        ctx.set_mark(self.key)
+        return True
+
+
 def to_parse_node(value) -> ParseNode:
     if isinstance(value, str):
         return Match(value)

blueprintcompiler/parser.py

@@ -30,7 +30,8 @@ def parse(
     """Parses a list of tokens into an abstract syntax tree."""
 
     try:
-        ctx = ParseContext(tokens)
+        original_text = tokens[0].string if len(tokens) else ""
+        ctx = ParseContext(tokens, original_text)
         AnyOf(UI).parse(ctx)
         ast_node = ctx.last_group.to_ast() if ctx.last_group else None
 

blueprintcompiler/tokenizer.py

@@ -20,6 +20,7 @@
 
 import re
 import typing as T
+from dataclasses import dataclass
 from enum import Enum
 
 from .errors import CompileError, CompilerBugError
@@ -62,6 +63,10 @@ class Token:
     def __str__(self) -> str:
         return self.string[self.start : self.end]
 
+    @property
+    def range(self) -> "Range":
+        return Range(self.start, self.end, self.string)
+
     def get_number(self) -> T.Union[int, float]:
         if self.type != TokenType.NUMBER:
             raise CompilerBugError()
@@ -103,3 +108,22 @@ def _tokenize(ui_ml: str):
 
 def tokenize(data: str) -> T.List[Token]:
     return list(_tokenize(data))
+
+
+@dataclass
+class Range:
+    start: int
+    end: int
+    original_text: str
+
+    @property
+    def text(self) -> str:
+        return self.original_text[self.start : self.end]
+
+    @staticmethod
+    def join(a: T.Optional["Range"], b: T.Optional["Range"]) -> T.Optional["Range"]:
+        if a is None:
+            return b
+        if b is None:
+            return a
+        return Range(min(a.start, b.start), max(a.end, b.end), a.original_text)

tests/test_samples.py

@@ -40,13 +40,12 @@ from blueprintcompiler.tokenizer import Token, TokenType, tokenize
 
 
 class TestSamples(unittest.TestCase):
-    def assert_docs_dont_crash(self, text, ast):
+    def assert_ast_doesnt_crash(self, text, tokens, ast):
         for i in range(len(text)):
             ast.get_docs(i)
-
-    def assert_completions_dont_crash(self, text, ast, tokens):
         for i in range(len(text)):
             list(complete(ast, tokens, i))
+        ast.get_document_symbols()
 
     def assert_sample(self, name, skip_run=False):
         print(f'assert_sample("{name}", skip_run={skip_run})')
@@ -79,8 +78,7 @@ class TestSamples(unittest.TestCase):
                 print("\n".join(diff))
                 raise AssertionError()
 
-            self.assert_docs_dont_crash(blueprint, ast)
+            self.assert_ast_doesnt_crash(blueprint, tokens, ast)
-            self.assert_completions_dont_crash(blueprint, ast, tokens)
         except PrintableError as e:  # pragma: no cover
             e.pretty_print(name + ".blp", blueprint)
             raise AssertionError()
@@ -105,8 +103,7 @@ class TestSamples(unittest.TestCase):
             ast, errors, warnings = parser.parse(tokens)
 
             if ast is not None:
-                self.assert_docs_dont_crash(blueprint, ast)
+                self.assert_ast_doesnt_crash(blueprint, tokens, ast)
-                self.assert_completions_dont_crash(blueprint, ast, tokens)
 
             if errors:
                 raise errors