update Lexer to use Errors and add some support for C23 stuffs (#489)

* fix types related to error formatters

- add variable annotation for 'name' field in '_formatter' class
- fix runtime type checking in '__init__' of '_formatter' class
- fix wrong type in 'json.dumps' calls that use 'separators' parameter
  with a string

* add to 'col' parameter in 'NormWarning' and 'NormError' classes to not be optional

* add to 'Error' and 'Highlight' dataclasses to be sortable

* add to 'level' and 'highlights' field of 'Error' dataclass be optional

* add 'from_name' constructor for 'Error' dataclass

* fix opening bad file paths when source is empty in File class

* add to Token class be a dataclass

* improve 'Errors' and 'Error' classes with QoL

- add 'add_error' method in 'Errors' class
- add to repr of 'Errors' be the same than the inner list
- add 'add_highlight' method in 'Error' dataclass

* fix 'Error.text' field being override to 'Error not found' in HumanizedErrorsFormatter

* fix unexpected errors in Lexer.pop that is expected a raise UnexpectedEOF

* add 'use_escape' parameter in 'Lexer.pop' method

* refactor Lexer to use errors instead of Exceptions to be more descritive

* add to support digraphs and trigraphs when parsing an operator

* add 'Lexer.peek' tests

* add removed preproc tests

* add tests for floats with fractional part

* remove asserts in Lexer

* add complex suffix for float literals

* add to support u and u8 prefixes for chars and strings

* add to parse hexadecimal floats

* add to handle multiple '.' and 'x' in float literals

* fix test_errors

* fix Lexer.get_next_token type of self.__pos

* fix types in rule.py

* move 'lexer_from_source()' and 'dict_to_pytest_param()' to tests/utils.py

norminette/__main__.py

@@ -7,7 +7,7 @@
 import argparse
 from norminette.errors import formatters
 from norminette.file import File
-from norminette.lexer import Lexer, TokenError
+from norminette.lexer import Lexer
 from norminette.exceptions import CParsingError
 from norminette.registry import Registry
 from norminette.context import Context
@@ -127,10 +127,10 @@ def main():
     for file in files:
         try:
             lexer = Lexer(file)
-            tokens = lexer.get_tokens()
+            tokens = list(lexer)
             context = Context(file, tokens, debug, args.R)
             registry.run(context)
-        except (TokenError, CParsingError) as e:
+        except CParsingError as e:
             print(file.path + f": Error!\n\t{colors(e.msg, 'red')}")
             sys.exit(1)
         except KeyboardInterrupt:

norminette/errors.py

@@ -3,54 +3,146 @@
 import os
 import json
 from dataclasses import dataclass, field, asdict
-from functools import cmp_to_key
-from typing import TYPE_CHECKING, Sequence, Union, Literal, Optional, List
+from typing import (
+    TYPE_CHECKING,
+    Sequence,
+    Union,
+    Literal,
+    Optional,
+    List,
+    overload,
+    Any,
+    Type,
+)
 
 from norminette.norm_error import NormError, NormWarning, errors as errors_dict
 
 if TYPE_CHECKING:
     from norminette.file import File
 
 
-def sort_errs(a: Error, b: Error):
-    # TODO Add to Error and Highlight dataclasses be sortable to remove this fn
-    ah: Highlight = a.highlights[0]
-    bh: Highlight = b.highlights[0]
-    if ah.column == bh.column and ah.lineno == bh.lineno:
-        return 1 if a.name > b.name else -1
-    return ah.column - bh.column if ah.lineno == bh.lineno else ah.lineno - bh.lineno
-
-
 @dataclass
 class Highlight:
     lineno: int
     column: int
     length: Optional[int] = field(default=None)
     hint: Optional[str] = field(default=None)
 
+    def __lt__(self, other: Any) -> bool:
+        assert isinstance(other, Highlight)
+        if self.lineno == other.lineno:
+            if self.column == other.column:
+                return len(self.hint or '') > len(other.hint or '')
+            return self.column > other.column
+        return self.lineno > other.lineno
+
 
 @dataclass
 class Error:
     name: str
     text: str
-    level: Literal["Error", "Notice"]
-    highlights: List[Highlight]
+    level: Literal["Error", "Notice"] = field(default="Error")
+    highlights: List[Highlight] = field(default_factory=list)
+
+    @classmethod
+    def from_name(cls: Type[Error], /, name: str, **kwargs) -> Error:
+        return cls(name, errors_dict[name], **kwargs)
+
+    def __lt__(self, other: Any) -> bool:
+        assert isinstance(other, Error)
+        if not self.highlights:
+            return bool(other.highlights) or self.name > other.name
+        if not other.highlights:
+            return bool(self.highlights) or other.name > self.name
+        ah, bh = min(self.highlights), min(other.highlights)
+        if ah.column == bh.column and ah.lineno == bh.lineno:
+            return self.name < other.name
+        return (ah.lineno, ah.column) < (bh.lineno, bh.column)
+
+    @overload
+    def add_highlight(
+        self,
+        lineno: int,
+        column: int,
+        length: Optional[int] = None,
+        hint: Optional[str] = None,
+    ) -> None: ...
+    @overload
+    def add_highlight(self, highlight: Highlight, /) -> None: ...
+
+    def add_highlight(self, *args, **kwargs) -> None:
+        if len(args) == 1:
+            highlight, = args
+        else:
+            highlight = Highlight(*args, **kwargs)
+        self.highlights.append(highlight)
 
 
 class Errors:
     __slots__ = "_inner"
 
     def __init__(self) -> None:
-        self._inner = []
+        self._inner: List[Error] = []
+
+    def __repr__(self) -> str:
+        return repr(self._inner)
 
     def __len__(self) -> int:
         return len(self._inner)
 
     def __iter__(self):
-        self._inner.sort(key=cmp_to_key(sort_errs))
+        self._inner.sort()
         return iter(self._inner)
 
-    # TODO Add `add(...)` method to allow creating `Highlight`s and `Error`s easily
+    @overload
+    def add(self, error: Error) -> None:
+        """Add an `Error` instance to the errors.
+        """
+        ...
+
+    @overload
+    def add(self, name: str, *, level: Literal["Error", "Notice"] = "Error", highlights: List[Highlight] = ...) -> None:
+        """Builds an `Error` instance from a name in `errors_dict` and adds it to the errors.
+
+        ```python
+        >>> errors.add("TOO_MANY_LINES")
+        >>> errors.add("INVALID_HEADER")
+        >>> errors.add("GLOBAL_VAR_DETECTED", level="Notice")
+        ```
+        """
+        ...
+
+    @overload
+    def add(
+        self,
+        /,
+        name: str,
+        text: str,
+        *,
+        level: Literal["Error", "Notice"] = "Error",
+        highlights: List[Highlight] = ...,
+    ) -> None:
+        """Builds an `Error` instance and adds it to the errors.
+
+        ```python
+        >>> errors.add("BAD_IDENTATION", "You forgot an column here")
+        >>> errors.add("CUSTOM_ERROR", f"name {not_defined!r} is not defined. Did you mean: {levenshtein_distance}?")
+        >>> errors.add("NOOP", "Empty if statement", level="Notice")
+        ```
+        """
+        ...
+
+    def add(self, *args, **kwargs) -> None:
+        kwargs.setdefault("level", "Error")
+        error = None
+        if len(args) == 1:
+            error = args[0]
+            if isinstance(error, str):
+                error = Error.from_name(error, **kwargs)
+        if len(args) == 2:
+            error = Error(*args, **kwargs)
+        assert isinstance(error, Error), "bad function call"
+        return self._inner.append(error)
 
     @property
     def status(self) -> Literal["OK", "Error"]:
@@ -60,15 +152,17 @@ def append(self, value: Union[NormError, NormWarning]) -> None:
         # TODO Remove NormError and NormWarning since it does not provide `length` data
         assert isinstance(value, (NormError, NormWarning))
         level = "Error" if isinstance(value, NormError) else "Notice"
-        value = Error(value.errno, value.error_msg, level, highlights=[
+        error = Error(value.errno, value.error_msg, level, highlights=[
             Highlight(value.line, value.col, None),
         ])
-        self._inner.append(value)
+        self._inner.append(error)
 
 
 class _formatter:
+    name: str
+
     def __init__(self, files: Union[File, Sequence[File]]) -> None:
-        if not isinstance(files, list):
+        if not isinstance(files, Sequence):
             files = [files]
         self.files = files
 
@@ -82,10 +176,9 @@ def __str__(self) -> str:
         for file in self.files:
             output += f"{file.basename}: {file.errors.status}!"
             for error in file.errors:
-                brief = errors_dict.get(error.name, "Error not found")
                 highlight = error.highlights[0]
                 output += f"\n{error.level}: {error.name:<20} "
-                output += f"(line: {highlight.lineno:>3}, col: {highlight.column:>3}):\t{brief}"
+                output += f"(line: {highlight.lineno:>3}, col: {highlight.column:>3}):\t{error.text}"
             output += '\n'
         return output
 
@@ -102,7 +195,7 @@ def __str__(self):
         output = {
             "files": files,
         }
-        return json.dumps(output, separators=",:") + '\n'
+        return json.dumps(output, separators=(',', ':')) + '\n'
 
 
 formatters = (

norminette/file.py

@@ -15,7 +15,7 @@ def __init__(self, path: str, source: Optional[str] = None) -> None:
 
     @property
     def source(self) -> str:
-        if not self._source:
+        if self._source is None:
             with open(self.path) as file:
                 self._source = file.read()
         return self._source

norminette/lexer/__init__.py

@@ -1,5 +1,4 @@
 from norminette.lexer.lexer import Lexer
-from norminette.lexer.lexer import TokenError
 from norminette.lexer.tokens import Token
 
-__all__ = ["Lexer", "TokenError", "Token"]
+__all__ = ["Lexer", "Token"]

norminette/lexer/dictionary.py

@@ -108,6 +108,7 @@
     ">>": "RIGHT_SHIFT",
     "<<": "LEFT_SHIFT",
     "?": "TERN_CONDITION",
+    "#": "HASH",
 }
 
 brackets = {