feat: Add documentation for parser

FallenDeity · Apr 29, 2023 · 80dcbcc · 80dcbcc
1 parent 6b933f9
commit 80dcbcc
Show file tree

Hide file tree

Showing 9 changed files with 659 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -18,6 +18,7 @@
     * [JIT compilers](#jit-compilers)
   * [Compiler vs Interpreter](#compiler-vs-interpreter)
   * [References](#references)
+  * [Progress](#progress)
 
 # Design of Programming Languages and Interpreters
 
@@ -202,6 +203,10 @@ A JIT compiler is a compiler that generates target code at runtime. This is less
 - [Crafting Interpreters](https://craftinginterpreters.com/)
 - [WikiBooks](https://en.wikibooks.org/wiki/Compiler_Construction)
 
+## Progress
+
+You can check out the progress of the project [here](./docs/tasks.html).
+
 <html lang="en">
     <style>
         .btn-blue {

diff --git a/__main__.py b/__main__.py
@@ -1,4 +1,4 @@
 from src import PyLox
 
 if __name__ == "__main__":
-    PyLox("test.lox").run()
+    PyLox().run_prompt()
diff --git a/docs/parser.md b/docs/parser.md
diff --git a/docs/tasks.md b/docs/tasks.md
@@ -0,0 +1,68 @@
+# Progress
+
+Note that this is a list of features that I want to implement, and not necessarily the order in which I will implement them.
+
+What has already been implemented:
+
+- [x] Entire lexer implementation
+- [x] Expression parsing
+- [x] Syntax Errors
+- [x] Expression execution
+- [x] Print statements
+- [x] Variable declaration, and global scope
+- [x] Interactive REPL
+- [x] Variable assignment
+- [x] Proper runtime errors
+- [x] Synchronization and reporting multiple parse errors
+- [x] Local scope, enclosing scope, blocks and nesting
+- [x] `if`-`else` statements
+- [x] `while` loops
+- [x] `for` loops
+- [x] Logical `and` and `or` operators
+- [x] Function declarations, calls, first class functions and callbacks
+- [x] Return values
+- [x] Closures
+- [x] Compile time variable resolution and binding
+- [x] Class declarations, objects, and properties
+- [x] Class properties
+- [x] Methods
+- [x] `this` attribute
+- [x] Constructors
+- [x] Inheritance
+
+## Extra features
+
+Here's the full set of extra features, and their progress:
+
+- [x] _Much_ better error messages
+- [x] Namespace concept for inbuilt libraries
+- [x] Allowing single quotes for strings
+- [x] String escapes: `\n`, `\t`, `\'`, `\"`, `\\` and `\↵`
+- [x] New operators:
+  - [x] Modulo `%`
+  - [x] Integer division `\`
+  - [x] Power `^`
+- [x] Lambda expressions
+- [x] New data types:
+  - [x] string: `"Hello, world!"`
+  - [x] int: `42`
+  - [x] list: `[42, 56]`
+  - [x] dictionary: `{42: "Forty two"}`
+- [x] Indexing support: for lists, dictionaries and strings
+- [x] Comparison operators work on strings
+- [x] `break` and `continue` semantics in loops
+- [x] Exceptions, `try` / `except` blocks and `raise` statements
+- [ ] Default values for function parameters
+- [ ] `*args`
+- [x] Added builtin functions:
+  - [x] `input`
+  - [x] `read` and `write` for files
+  - [x] `min`, `max` and `abs`
+  - [x] `map`, `filter` and `reduce` that take lists and return new lists
+- [x] An `import` system
+- [x] Builtin Callable types
+- [x] A standard library
+  - [x] `random` module
+  - [x] `io` module string and binary reads/writes to files
+  - [x] `http` library to make a web server and client
+  - [x] `math` module
diff --git a/src/interpreter/interpreter.py b/src/interpreter/interpreter.py
@@ -84,9 +84,9 @@ def _load_builtins(self) -> None:
 
     def error(self, token: "Token", message: str, /) -> str:
         """Raise a runtime error."""
-        line = self._lox.lexer._source.splitlines()[token.line - 1]
+        line = self._lox._source.splitlines()[token.line - 1]
         error_ = f"\n{line}\n{'~' * token.column}^\n{message}"
-        return f"RuntimeError at line {token.line}:{token.column}{error_}"
+        return f"RuntimeError at line {token.line  - self._lox._process.lines}:{token.column}{error_}"
 
     @staticmethod
     def is_equal(left: Equals, right: Equals, /) -> bool:
@@ -220,8 +220,8 @@ def visit_try_stmt(self, stmt: "Try") -> t.Any:
 
     def visit_print_stmt(self, stmt: "Print") -> None:
         """Visit a print statement."""
-        value: t.Any = self._evaluate(stmt.expression)
-        print(self.stringify(value))
+        value = self.stringify(self._evaluate(stmt.expression))
+        print(bytes(value, "utf-8").decode("unicode_escape"))
 
     def visit_return_stmt(self, stmt: "Return") -> None:
         """Visit a return statement."""
@@ -398,9 +398,14 @@ def visit_call_expr(self, expr: "Call") -> t.Any:
         try:
             return callee(self, arguments)
         except Exception as e:
-            self._logger.error(f"Error while calling function {expr.paren.line}:{expr.paren.column}: \n{e}")
+            self._logger.error(
+                f"Error calling function {expr.paren.line  - self._lox._process.lines}:{expr.paren.column}:\n{e}"
+            )
             raise PyLoxRuntimeError(
-                self.error(expr.paren, f"Error while calling function {expr.paren.line}:{expr.paren.column}: \n{e}")
+                self.error(
+                    expr.paren,
+                    f"Error calling function {expr.paren.line - self._lox._process.lines}:{expr.paren.column}:\n{e}",
+                )
             )
 
     def visit_get_expr(self, expr: "Get") -> t.Any:

diff --git a/src/interpreter/lox.py b/src/interpreter/lox.py
@@ -18,8 +18,8 @@ def __init__(self, source: str | pathlib.Path = "") -> None:
         self.logger = Logger(name="PyLox")
         self.interpreter = Interpreter(self, self.logger)
         self._source = self._read_file(self._file_path) if self._file_path else ""
-        process = PreProcessor(self._source)
-        self._source = process.source
+        self._process = PreProcessor(self._source)
+        self._source = self._process.source
         self.lexer = Lexer(self._source, self.logger)
 
     @staticmethod
@@ -36,25 +36,29 @@ def run_prompt(self) -> None:
                 self.logger.debug("Exiting PyLox...")
                 raise PyLoxKeyboardInterrupt
             else:
-                self.logger.info("Running PyLox...")
-                self.lexer.source = f"{source}\n"
                 try:
+                    self.logger.info("Running PyLox...")
+                    self._process = PreProcessor(source)
+                    self._source = self._process.source
+                    self.lexer = Lexer(self._source, self.logger)
                     tokens = self.lexer.scan_tokens()
-                    parser = Parser(tokens, self.logger, self._source)
+                    parser = Parser(self, tokens, self.logger, self._source)
                     statements = parser.parse()
+                    print(statements)
                     if parser._has_error:
                         continue
                     resolver = Resolver(self.interpreter)
                     resolver._resolve(statements)
                     self.interpreter.interpret(statements)
                     self.logger.info("Finished running PyLox.")
-                except PyLoxException:
+                except PyLoxException as e:
+                    self.logger.error(e)
                     continue
 
     def run(self) -> None:
         self.logger.info("Running PyLox...")
         tokens = self.lexer.scan_tokens()
-        parser = Parser(tokens, self.logger, self._source)
+        parser = Parser(self, tokens, self.logger, self._source)
         statements = parser.parse()
         if parser._has_error:
             return

diff --git a/src/parser/parser.py b/src/parser/parser.py
@@ -34,6 +34,7 @@
 )
 
 if t.TYPE_CHECKING:
+    from src.interpreter import PyLox
     from src.lexer.tokens import Token, TokenType
     from src.logger import Logger
 
@@ -42,13 +43,14 @@
 
 
 class Parser:
-    def __init__(self, tokens: list["Token"], logger: "Logger", source: str, debug: bool = True) -> None:
+    def __init__(self, lox: "PyLox", tokens: list["Token"], logger: "Logger", source: str, debug: bool = True) -> None:
         self._tokens = tokens
         self._current = 0
         self._logger = logger
         self._debug = debug
         self._source = source
         self._has_error = False
+        self._lox = lox
 
     def parse(self) -> list["Stmt"]:
         """Parse the tokens."""
@@ -101,7 +103,10 @@ def _consume(self, token_type: "TokenType", message: str) -> "Token":  # type: i
         token = self._previous()
         if self._check(token_type):
             return self._advance()
-        error = f"Expected {str(token_type)} but got {str(token.token_type)} in {token.line}:{token.column}"
+        error = (
+            f"Expected {str(token_type)} but got {str(token.token_type)} "
+            f"in {token.line - self._lox._process.lines}:{token.column}"
+        )
         self._error(token, error, message)
 
     def _synchronize(self) -> None:
@@ -430,7 +435,9 @@ def _term(self) -> Expr:
         :return: The parsed data
         """
         expr = self._factor()
-        while self._match(SimpleTokenType.MINUS, SimpleTokenType.PLUS, ComplexTokenType.BACKSLASH):
+        while self._match(
+            SimpleTokenType.MINUS, SimpleTokenType.PLUS
+        ):  # if there is an term operator parse the right hand side
             operator, right = self._previous(), self._factor()
             expr = Binary(expr, operator, right)
         return expr
@@ -441,7 +448,13 @@ def _factor(self) -> Expr:
         :return: The parsed data
         """
         expr = self._unary()
-        while self._match(SimpleTokenType.SLASH, SimpleTokenType.STAR, SimpleTokenType.MODULO, SimpleTokenType.CARAT):
+        while self._match(
+            SimpleTokenType.CARAT,
+            SimpleTokenType.SLASH,
+            ComplexTokenType.BACKSLASH,
+            SimpleTokenType.STAR,
+            SimpleTokenType.MODULO,
+        ):
             operator, right = self._previous(), self._unary()
             expr = Binary(expr, operator, right)
         return expr

diff --git a/src/preprocessor.py b/src/preprocessor.py
@@ -25,6 +25,7 @@ class PreProcessor:
     def __init__(self, source: str) -> None:
         self._includes: dict[str, Import] = {}
         self._source = source
+        self.lines = 0
         self._resolve_imports()
 
     def _resolve_imports(self) -> None:
@@ -45,12 +46,14 @@ def _resolve_imports(self) -> None:
                 self._includes[path.as_posix()] = Import(path, n, match.start(), match.end(), module)
         for module in self._includes.values():
             text = module.path.read_text()
+            self.lines += text.count("\n")
             if (
                 module.module.startswith("<")
                 and ("init(" not in text or "init()" in text)
                 and f"class {module.module[1:-1]}" in text
             ):
                 text += f"\nvar {module.module[1:-1]} = {module.module[1:-1]}();"
+                self.lines += 1
             self._source = self._source.replace(f"import {module.module}", text)
 
     @property

diff --git a/test.lox b/test.lox
@@ -1,8 +1,5 @@
-import <random>
+class Foo {
+}
 
-var arr = array();
-for (var i = 0; i < 10; i = i + 1) {
-    arr.append(i);
+class Bar < Foo {
 }
-random.shuffle(arr);
-print arr;