aviallon/compiler
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lexer+parser: add BEGIN token + add delimiting pseudo_nodes to anonymous blocks (including root)

Browse source
This commit is contained in:
Antoine Viallon 2024-01-11 00:02:08 +01:00
parent 6d4c999be7
commit d2d519acf4
Signed by: aviallon
GPG key ID: 186FC35EDEB25716
2 changed files with 17 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
compiler/lexer.py
View file

@ -48,6 +48,7 @@ class Tokens(enum.Enum):
Comment = re.compile(r"//.*") Comment = re.compile(r"//.*")
Newline = re.compile(r"\n", flags=re.MULTILINE) Newline = re.compile(r"\n", flags=re.MULTILINE)
BEGIN = re.compile(r"\A")
EOF = re.compile(r"\Z") EOF = re.compile(r"\Z")
Blank = re.compile(r"[ \t]+") Blank = re.compile(r"[ \t]+")
Unknown = re.compile(r".*", flags=re.DOTALL) Unknown = re.compile(r".*", flags=re.DOTALL)
@ -87,6 +88,13 @@ class Lexer(collections.abc.Sequence):
def _next_token(self) -> Token: def _next_token(self) -> Token:
actual_result: Token actual_result: Token
if len(self.tokens) == 0:
self.tokens += [Token(Tokens.BEGIN,
loc=SourceLocation(
Location(line=0, character=0),
source=self.data
),
value="")]
if self.begin < len(self.data): if self.begin < len(self.data):
best_result: Token = Token(Tokens.Unknown, best_result: Token = Token(Tokens.Unknown,
loc=SourceLocation( loc=SourceLocation(

13
compiler/parser.py
View file

@ -200,22 +200,27 @@ class Parser:
if lbrace := self.accept(Tokens.Brace_Left): if lbrace := self.accept(Tokens.Brace_Left):
block = self.block(name="anon") block = self.block(name="anon")
rbrace = self.expect(Tokens.Brace_Right) rbrace = self.expect(Tokens.Brace_Right)
return Statement(block, pseudo_nodes=[PseudoNode(lbrace), PseudoNode(rbrace)]) block.pseudo_nodes = [PseudoNode(lbrace), PseudoNode(rbrace)]
return Statement(block)
elif expr := self.expression(mandatory): elif expr := self.expression(mandatory):
semicolon = PseudoNode(self.expect(Tokens.Semicolon)) semicolon = PseudoNode(self.expect(Tokens.Semicolon))
return Statement(expr, pseudo_nodes=[semicolon]) return Statement(expr, pseudo_nodes=[semicolon])
elif mandatory: elif mandatory:
raise UnexpectedTokenError(expr, wanted="expression") raise UnexpectedTokenError(expr, wanted="expression")
def block(self, name: str) -> Block: def block(self, name: str, pseudo_nodes: list[PseudoNode] | None = None) -> Block:
nodes: list[Statement] = [] nodes: list[Statement] = []
while stmt := self.statement(mandatory=False): while stmt := self.statement(mandatory=False):
nodes += [stmt] nodes += [stmt]
return Block(name, *nodes) return Block(name, *nodes, pseudo_nodes=pseudo_nodes)
@tracer.trace_method @tracer.trace_method
def root(self) -> Node: def root(self) -> Node:
return self.block(name="root") begin = self.expect(Tokens.BEGIN)
root_block = self.block(name="root")
end = self.expect(Tokens.EOF)
root_block.pseudo_nodes = [begin, end]
return root_block
def parse(self) -> Node: def parse(self) -> Node:
try: try: