aviallon/compiler
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests: lexer: simplify code, add blocks and whitespaces test-cases

Browse source
This commit is contained in:
Antoine Viallon 2024-07-26 00:12:37 +02:00
parent 89a9bb7901
commit 032a6d9109
Signed by: aviallon
GPG key ID: 186FC35EDEB25716
1 changed files with 31 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

45
compiler/tests/test_lexer.py
View file

@ -3,30 +3,47 @@ import io
from compiler.lexer import Tokens, TextIOWithMemory, Lexer, Token from compiler.lexer import Tokens, TextIOWithMemory, Lexer, Token
def _check_token_type(lexer: Lexer, expected_tokens: list[Tokens]):
tokens = []
tok: Token = Token(Tokens.Unknown)
while tok.kind != Tokens.EOF:
tok = next(lexer)
tokens += [tok.kind]
assert tuple(tokens) == tuple(expected_tokens)
def test_lexer_empty(): def test_lexer_empty():
data_raw = io.StringIO("") data_raw = io.StringIO("")
data = TextIOWithMemory(data_raw) data = TextIOWithMemory(data_raw)
my_lexer = Lexer(input_stream=data) my_lexer = Lexer(input_stream=data)
assert next(my_lexer).kind == Tokens.BEGIN _check_token_type(my_lexer, [Tokens.BEGIN, Tokens.EOF])
assert next(my_lexer).kind == Tokens.EOF
def test_lexer_arithmetic(): def test_lexer_arithmetic():
data_raw = io.StringIO("1+5-4*2/1") data_raw = io.StringIO("1+5-41*2/1")
data = TextIOWithMemory(data_raw) data = TextIOWithMemory(data_raw)
my_lexer = Lexer(input_stream=data) my_lexer = Lexer(input_stream=data)
tokens = [] _check_token_type(my_lexer, [Tokens.BEGIN, Tokens.Integer, Tokens.Op_Plus, Tokens.Integer,
expected_tokens = [Tokens.BEGIN, Tokens.Integer, Tokens.Op_Plus, Tokens.Integer,
Tokens.Op_Minus, Tokens.Integer, Tokens.Op_Multiply, Tokens.Integer, Tokens.Op_Minus, Tokens.Integer, Tokens.Op_Multiply, Tokens.Integer,
Tokens.Op_Divide, Tokens.Integer, Tokens.EOF] Tokens.Op_Divide, Tokens.Integer, Tokens.EOF])
tok: Token = Token(Tokens.Unknown)
while tok.kind != Tokens.EOF:
tok = next(my_lexer)
tokens += [tok]
assert len(tokens) == len(expected_tokens) def test_lexer_blocks():
for tok, expected in zip(tokens, expected_tokens): data = TextIOWithMemory(io.StringIO("""
assert tok.kind == expected {
{ 1+1; {}; }
}
"""))
my_lexer = Lexer(input_stream=data)
_check_token_type(my_lexer, [
Tokens.BEGIN, Tokens.Newline,
Tokens.Blank, Tokens.Brace_Left, Tokens.Newline,
Tokens.Blank, Tokens.Brace_Left, Tokens.Blank, Tokens.Integer, Tokens.Op_Plus, Tokens.Integer, Tokens.Semicolon,
Tokens.Blank, Tokens.Brace_Left, Tokens.Brace_Right, Tokens.Semicolon, Tokens.Blank, Tokens.Brace_Right,
Tokens.Newline,
Tokens.Blank, Tokens.Brace_Right, Tokens.Newline,
Tokens.Blank, Tokens.EOF])