lexer+parser: add BEGIN token + add delimiting pseudo_nodes to anonymous blocks (including root)

compiler/lexer.py

@@ -48,6 +48,7 @@ class Tokens(enum.Enum):
     Comment = re.compile(r"//.*")
 
     Newline = re.compile(r"\n", flags=re.MULTILINE)
+    BEGIN = re.compile(r"\A")
     EOF = re.compile(r"\Z")
     Blank = re.compile(r"[ \t]+")
     Unknown = re.compile(r".*", flags=re.DOTALL)
@@ -87,6 +88,13 @@ class Lexer(collections.abc.Sequence):
 
     def _next_token(self) -> Token:
         actual_result: Token
+        if len(self.tokens) == 0:
+            self.tokens += [Token(Tokens.BEGIN,
+                                  loc=SourceLocation(
+                                      Location(line=0, character=0),
+                                      source=self.data
+                                  ),
+                                  value="")]
         if self.begin < len(self.data):
             best_result: Token = Token(Tokens.Unknown,
                                        loc=SourceLocation(

compiler/parser.py

@@ -200,22 +200,27 @@ class Parser:
         if lbrace := self.accept(Tokens.Brace_Left):
             block = self.block(name="anon")
             rbrace = self.expect(Tokens.Brace_Right)
-            return Statement(block, pseudo_nodes=[PseudoNode(lbrace), PseudoNode(rbrace)])
+            block.pseudo_nodes = [PseudoNode(lbrace), PseudoNode(rbrace)]
+            return Statement(block)
         elif expr := self.expression(mandatory):
             semicolon = PseudoNode(self.expect(Tokens.Semicolon))
             return Statement(expr, pseudo_nodes=[semicolon])
         elif mandatory:
             raise UnexpectedTokenError(expr, wanted="expression")
 
-    def block(self, name: str) -> Block:
+    def block(self, name: str, pseudo_nodes: list[PseudoNode] | None = None) -> Block:
         nodes: list[Statement] = []
         while stmt := self.statement(mandatory=False):
             nodes += [stmt]
-        return Block(name, *nodes)
+        return Block(name, *nodes, pseudo_nodes=pseudo_nodes)
 
     @tracer.trace_method
     def root(self) -> Node:
-        return self.block(name="root")
+        begin = self.expect(Tokens.BEGIN)
+        root_block = self.block(name="root")
+        end = self.expect(Tokens.EOF)
+        root_block.pseudo_nodes = [begin, end]
+        return root_block
 
     def parse(self) -> Node:
         try: