tests: add basic lexer tests

compiler/tests/test_lexer.py

@@ -0,0 +1,32 @@
+import io
+
+from compiler.lexer import Tokens, TextIOWithMemory, Lexer, Token
+
+
+def test_lexer_empty():
+    data_raw = io.StringIO("")
+    data = TextIOWithMemory(data_raw)
+
+    my_lexer = Lexer(input_stream=data)
+    assert next(my_lexer).kind == Tokens.BEGIN
+    assert next(my_lexer).kind == Tokens.EOF
+
+
+def test_lexer_arithmetic():
+    data_raw = io.StringIO("1+5-4*2/1")
+    data = TextIOWithMemory(data_raw)
+
+    my_lexer = Lexer(input_stream=data)
+    tokens = []
+    expected_tokens = [Tokens.BEGIN, Tokens.Integer, Tokens.Op_Plus, Tokens.Integer,
+                       Tokens.Op_Minus, Tokens.Integer, Tokens.Op_Multiply, Tokens.Integer,
+                       Tokens.Op_Divide, Tokens.Integer, Tokens.EOF]
+
+    tok: Token = Token(Tokens.Unknown)
+    while tok.kind != Tokens.EOF:
+        tok = next(my_lexer)
+        tokens += [tok]
+
+    assert len(tokens) == len(expected_tokens)
+    for tok, expected in zip(tokens, expected_tokens):
+        assert tok.kind == expected