parser: fully rewrite parsing

Use a simpler and more direct recursive descent method.

compiler/tokenizer.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from dataclasses import dataclass, field
 from beartype import beartype
-from beartype.typing import Optional
+from beartype.typing import Optional, List
 
 import enum
 import re
@@ -31,8 +31,12 @@ class Tokens(enum.Enum):
     Parens_Left = re.compile(r"\(")
     Parens_Right = re.compile(r"\)")
     Blank = re.compile(r"\s+")
+    EOF = re.compile(r"\Z")
     Unknown = re.compile(r".*")
 
+    def __bool__(self):
+        return True
+
 
 class Tokenizer:
     def __init__(self):
@@ -67,4 +71,7 @@ class Tokenizer:
 
             results += [best_result]
             begin += len(best_result.value)
+        results += [Token(Tokens.EOF, value=None, loc=SourceLocation(
+            Location(line=0, character=len(data)), source=data
+        ))]
         return results