aviallon/compiler
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lexer: fix non-stopping list() issue

Browse source 
Convert a SystemExit to a CompilationError
This commit is contained in:
Antoine Viallon 2024-07-26 01:33:00 +02:00
parent 7655b2a6a8
commit d7613c8ed2
Signed by: aviallon
GPG key ID: 186FC35EDEB25716
1 changed files with 9 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

14
compiler/lexer.py
View file

@ -7,6 +7,7 @@ import typing
from dataclasses import dataclass, field
from typing import cast
from .errors import CompilationError
from .logger import Logger
from .source import SourceLocation, Location, TextIOWithMemory
from .typechecking import typecheck
@ -68,7 +69,7 @@ class Tokens(enum.Enum):
class Lexer(collections.abc.Iterator):
def __init__(self, input_stream: TextIOWithMemory, token_filter: typing.Callable[[Token], bool] | None = None):
self.input = input_stream
self.tokens = []
self.tokens: list[Token] = []
self.length: int | None = None
self.begin: int = 0
self.end: int = 0
@ -77,6 +78,8 @@ class Lexer(collections.abc.Iterator):
self.token_filter = token_filter
def __next__(self):
if len(self.tokens) and self.tokens[-1].kind == Tokens.EOF:
raise StopIteration("EOF already reached")
return self._filtered_next_token()
def _filtered_next_token(self) -> Token:
@ -103,6 +106,9 @@ class Lexer(collections.abc.Iterator):
if self.tokens[-1].kind in [Tokens.BEGIN, Tokens.Newline]:
self.input.readline()
if self.tokens[-1].kind == Tokens.EOF:
raise StopIteration("EOF already reached")
if self.begin == len(self.input.stream_cache):
eof_token = Token(Tokens.EOF, value=None, loc=SourceLocation(
Location(line=self.line, character=0),
@ -144,8 +150,8 @@ class Lexer(collections.abc.Iterator):
self.character += len(best_result.value)
if best_result.kind == Tokens.Unknown:
source_hint = best_result.loc.show_in_source()
logger.error(f"{best_result.loc}: Unknown token '{best_result.loc.source_substring}'\n{source_hint}")
exit(1)
raise CompilationError(best_result.loc,
message=f"Unknown token '{best_result.loc.source_substring}'\n{source_hint}")
elif best_result.kind == Tokens.Newline:
self.line += 1
self.character = 0
@ -155,5 +161,3 @@ class Lexer(collections.abc.Iterator):
self.tokens += [best_result]
return best_result
else:
raise IndexError("EOF already reached")