main: convert to use new iterator Lexer

compiler/__main__.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
 import argparse
+import collections.abc
 import sys
-from pprint import pprint
 
 from . import semantic, ir, optimizations
 from .errors import CompilationError, CompilationWarning
-from .lexer import Lexer, Tokens
+from .lexer import Lexer, Tokens, Token
 from .logger import rootLogger, LogLevel
 from .parser import Parser
 
@@ -36,11 +36,9 @@ def main():
         data = sys.stdin.read().strip()
 
     print("Source:\n", data)
-    tokens = Lexer(data)
-
-    tokens = [token for token in tokens if token.kind not in [Tokens.Blank, Tokens.Newline, Tokens.Comment]]
-    if rootLogger.level <= LogLevel.Debug:
-        pprint(tokens)
+    tokens: collections.abc.Iterator[Token] = \
+        Lexer(data,
+              token_filter=lambda token: token.kind not in [Tokens.Blank, Tokens.Newline, Tokens.Comment])
 
     parser = Parser(tokens)
     try: