diff --git a/compiler/lexer.py b/compiler/lexer.py index 77187dd..912d839 100644 --- a/compiler/lexer.py +++ b/compiler/lexer.py @@ -48,6 +48,7 @@ class Tokens(enum.Enum): Comment = re.compile(r"//.*") Newline = re.compile(r"\n", flags=re.MULTILINE) + BEGIN = re.compile(r"\A") EOF = re.compile(r"\Z") Blank = re.compile(r"[ \t]+") Unknown = re.compile(r".*", flags=re.DOTALL) @@ -87,6 +88,13 @@ class Lexer(collections.abc.Sequence): def _next_token(self) -> Token: actual_result: Token + if len(self.tokens) == 0: + self.tokens += [Token(Tokens.BEGIN, + loc=SourceLocation( + Location(line=0, character=0), + source=self.data + ), + value="")] if self.begin < len(self.data): best_result: Token = Token(Tokens.Unknown, loc=SourceLocation( diff --git a/compiler/parser.py b/compiler/parser.py index d574956..f195dbb 100644 --- a/compiler/parser.py +++ b/compiler/parser.py @@ -200,22 +200,27 @@ class Parser: if lbrace := self.accept(Tokens.Brace_Left): block = self.block(name="anon") rbrace = self.expect(Tokens.Brace_Right) - return Statement(block, pseudo_nodes=[PseudoNode(lbrace), PseudoNode(rbrace)]) + block.pseudo_nodes = [PseudoNode(lbrace), PseudoNode(rbrace)] + return Statement(block) elif expr := self.expression(mandatory): semicolon = PseudoNode(self.expect(Tokens.Semicolon)) return Statement(expr, pseudo_nodes=[semicolon]) elif mandatory: raise UnexpectedTokenError(expr, wanted="expression") - def block(self, name: str) -> Block: + def block(self, name: str, pseudo_nodes: list[PseudoNode] | None = None) -> Block: nodes: list[Statement] = [] while stmt := self.statement(mandatory=False): nodes += [stmt] - return Block(name, *nodes) + return Block(name, *nodes, pseudo_nodes=pseudo_nodes) @tracer.trace_method def root(self) -> Node: - return self.block(name="root") + begin = self.expect(Tokens.BEGIN) + root_block = self.block(name="root") + end = self.expect(Tokens.EOF) + root_block.pseudo_nodes = [begin, end] + return root_block def parse(self) -> Node: try: