diff --git a/.flake8 b/.flake8 index 16e32d9..69e3e63 100644 --- a/.flake8 +++ b/.flake8 @@ -4,4 +4,4 @@ max-line-length = 88 extend-ignore = # See https://github.com/PyCQA/pycodestyle/issues/373 - E203, E501 + E203, E501, W503, E701, E704 diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1c215dc..33fd641 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -38,7 +38,7 @@ jobs: # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --ignore=E203,E501 --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --ignore=E203,E501,W503,E701,E704 --statistics - name: Test with pytest run: | python -m pytest tests --slow diff --git a/CHANGELOG.md b/CHANGELOG.md index 399beac..9140c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,19 @@ ### 1.1.2 @ 2/14/2024 +#### :mega: New + +1. Provide `.dockerignore` file for removing unexpected files in the built docker image. + #### :wrench: Fix 1. Fix: Remove an unexpected in-release package `version` which should only appear during the package building. +#### :floppy_disk: Change + +1. Improve the quality of codes. +2. Make the `flake8` warning validator omit the issues exempted by `black`. + ### 1.1.1 @ 2/5/2024 #### :wrench: Fix diff --git a/Dockerfile.dockergitignore b/Dockerfile.dockergitignore new file mode 100644 index 0000000..15f7255 --- /dev/null +++ b/Dockerfile.dockergitignore @@ -0,0 +1,104 @@ +# Ignore temp files. +/alpha* +/data* +/legacy* +/docs* +/temp* +/.vscode/* + +# Git/Github folders. +.git/ +.github/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/pyrightconfig.json b/pyrightconfig.json index dd720d8..43b7fdc 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -9,6 +9,9 @@ ".VSCodeCounter", ".github", ".git", + "dist", + "build", + "*.egg-info", "**/node_modules", "**/__pycache__", "typestubs" diff --git a/syncstream/host.py b/syncstream/host.py index ae8938d..0a44135 100644 --- a/syncstream/host.py +++ b/syncstream/host.py @@ -490,7 +490,7 @@ def read_serialized( for val in self.read(size=size) ) - def serve(self, app: flask.Flask) -> None: + def serve(self, app: flask.Flask) -> None: # noqa: C901 """Provide the service of the host buffer. The service would be equipped as an independent thread. Each time the request diff --git a/syncstream/mproc.py b/syncstream/mproc.py index c378a10..06a7d2f 100644 --- a/syncstream/mproc.py +++ b/syncstream/mproc.py @@ -131,6 +131,56 @@ def parse_lines(self, lines: Sequence[Union[str, T]]) -> None: """ self.storage.extend(lines) + def __read_all(self) -> Tuple[Union[T, str], ...]: + """Real all lines. + + Private method. Use it to read all lines stored in this buffer. + """ + has_last_line = self.last_line.tell() > 0 + n_lines = len(self.storage) + if has_last_line: + len_max = self.storage.maxlen + if len_max and n_lines == len_max: + value = self.storage.popleft() + results = (*self.storage, self.last_line.getvalue()) + self.storage.appendleft(value) + elif n_lines > 0: + results = (*self.storage, self.last_line.getvalue()) + else: + results = (self.last_line.getvalue(),) + return results + else: + return tuple(self.storage) + + def __read_n(self, size: int) -> Tuple[Union[T, str], ...]: + """Real given number of lines. + + Private method. Use it to read some lines specified in the argument `size`. + """ + has_last_line = self.last_line.tell() > 0 + n_lines = len(self.storage) + len_max = self.storage.maxlen + if has_last_line and len_max and n_lines == len_max: + preserved_value = self.storage.popleft() + else: + preserved_value = None + n_read = min( + size - 1 if has_last_line else size, + n_lines if preserved_value is None else n_lines - 1, + ) + results = list() + if n_read > 0: + self.storage.rotate(n_read) + for _ in range(n_read): + value = self.storage.popleft() + results.append(value) + self.storage.append(value) + if has_last_line: + results.append(self.last_line.getvalue()) + if preserved_value is not None: + self.storage.appendleft(preserved_value) + return tuple(results) + def read(self, size: Optional[int] = None) -> Tuple[Union[T, str], ...]: """Read the records. @@ -153,44 +203,10 @@ def read(self, size: Optional[int] = None) -> Tuple[Union[T, str], ...]: A sequence of fetched record items. Results are sorted in the FIFO order. """ with self.__last_line_lock: - has_last_line = self.last_line.tell() > 0 - n_lines = len(self.storage) if size is None: - if has_last_line: - len_max = self.storage.maxlen - if len_max and n_lines == len_max: - value = self.storage.popleft() - results = (*self.storage, self.last_line.getvalue()) - self.storage.appendleft(value) - elif n_lines > 0: - results = (*self.storage, self.last_line.getvalue()) - else: - results = (self.last_line.getvalue(),) - return results - else: - return tuple(self.storage) + return self.__read_all() elif size > 0: - len_max = self.storage.maxlen - if has_last_line and len_max and n_lines == len_max: - preserved_value = self.storage.popleft() - else: - preserved_value = None - n_read = min( - size - 1 if has_last_line else size, - n_lines if preserved_value is None else n_lines - 1, - ) - results = list() - if n_read > 0: - self.storage.rotate(n_read) - for _ in range(n_read): - value = self.storage.popleft() - results.append(value) - self.storage.append(value) - if has_last_line: - results.append(self.last_line.getvalue()) - if preserved_value is not None: - self.storage.appendleft(preserved_value) - return tuple(results) + return self.__read_n(size=size) else: return tuple() diff --git a/syncstream/utils.py b/syncstream/utils.py index 33dea19..14a3482 100644 --- a/syncstream/utils.py +++ b/syncstream/utils.py @@ -38,7 +38,7 @@ from builtins import list as List, type as Type from collections.abc import Sequence as _Sequence -from typing_extensions import TypeGuard +from typing_extensions import Literal, Never, TypeGuard, overload try: # Suppos that this `utils` module is placed at the root. from . import __name__ as __pkg_name__ @@ -72,6 +72,7 @@ class _Missing: class ModuleReplaceError(ImportError): """Exception raised when replacing an existing lazy module. + Replacing an existing lazy module is not allowed, because this will cause the previously configured lazy module point to a not-existing "real" module. @@ -105,6 +106,7 @@ class _LazyModule(__LazyModule): def force_load(self) -> None: """Call this method will force the lazy module to be actually loaded. + This module is used by `load_module` when relative dependency is needed to be solved. """ @@ -138,7 +140,9 @@ def __repr__(self) -> str: def __getattribute__(self, attr: str): """Get the attribute of the module. + If the attribute is missing, load the module. + Once the module is actually loaded, since the class of the module will be replaced by the actual module class, this method will not be used any more. @@ -325,59 +329,58 @@ def exec_module(self, module: ModuleType) -> None: mdict["__dep_modules__"] = self.__dep_modules -def lazy_import( - name: str, - package: Optional[str] = __pkg_name__, - required: bool = True, - dependencies: Optional[Union[str, Sequence[str]]] = None, - rel_dependencies: Optional[Union[ModuleType, Sequence[ModuleType]]] = None, -) -> ModuleType: - """Perform the lazy import for a module. - The returned module will not be loaded until it is actually used. - - Modified from: - https://docs.python.org/3/library/importlib.html#implementing-lazy-imports +class _LazyImporter: + """The lazy importer. - Arguments: - name: The name of the module. It does not need to start with the `.` - symbol. - package: The name of the package (anchor). - By default: will use the `__name__` of the pacakge where this - `utils` module is placed. if `utils` is not placed as a sub- - module, and `package` is not specified, will search `name` by - absolute import. - If using `None`: will search `name` by absolute import. - required: Whether to require the existence of the module. If not - specified, will allow to load an empty module when the module - is not found. - dependencies: One or more depdencies for the module to be loaded. - If not specified, it means that the module does not need - dependencies. If specified, the module is only loaded when - all dependencies are detected. Otherwise, returns a module - placeholder. The dependencies are module names following the - abosolute import rules. - Returns: - #1: A lazy loaded module. It will be loaded when actually using it. + This private class is temporarily used during the lazy importing. """ - full_name = ".".join(map(str, filter(bool, (package, name)))) - # Fetch the module directly if it has been loaded. - prev_module = sys.modules.get(full_name, None) - if isinstance(prev_module, ModuleType): - return prev_module - # Fail to hit the module. - # Start the check all absolute module dependencies. - is_deps_missing = False - if dependencies is not None: + + @staticmethod + def check_is_dep_missing(dependencies: Optional[Union[str, Sequence[str]]]) -> bool: + """Check if the provided dependencies are missing or not. + + Arguments + --------- + dependencies: `str | [str] | None` + A list of absolute dependencies' names. + + Returns + ------- + #1: `False` if all dependencies exists. + """ + if dependencies is None: + return False if isinstance(dependencies, str) or (not isinstance(dependencies, _Sequence)): dependencies = (dependencies,) for dep in dependencies: spec = importlib.util.find_spec(str(dep)) if spec is None: - is_deps_missing = True - break - # Gather the relative module dependencies, send them to the module loader. - deps: List[ModuleType] = list() - if (not is_deps_missing) and rel_dependencies is not None: + return True + return False + + @staticmethod + def gather_relative_module_dependencies( + rel_dependencies: Optional[Union[ModuleType, Sequence[ModuleType]]], + is_deps_missing: bool, + ) -> List[ModuleType]: + """Gather the dependencies that are relative packages. + + These modules (maynbe lazy) need to be loaded before this module is loaded. + + Arguments + --------- + rel_dependencies: `ModuleType | [ModuleType] | None` + A list of relative dependencies. Each item is a module (can be lazy). + If using `None`, will return an empty list. + + Returns + ------- + #1: A list of lazy modules that are the members of the provided relative + dependencies. + """ + deps: List[ModuleType] = list() + if is_deps_missing or rel_dependencies is None: + return deps if isinstance(rel_dependencies, str): raise TypeError( 'utils: The argument "rel_dependencies" should be a module or a ' @@ -391,40 +394,168 @@ def lazy_import( for dep in rel_dependencies: if isinstance(dep, _LazyModule): deps.append(dep) - # Start to create the lazy-loaded module. - if package is None: - spec = importlib.util.find_spec(name) - else: - spec = importlib.util.find_spec(".{0}".format(name), package=package) - if is_deps_missing or (spec is None): + return deps + + @overload + @staticmethod + def create_module_placeholder(full_name: str, required: Literal[True]) -> Never: ... + + @overload + @staticmethod + def create_module_placeholder( + full_name: str, required: bool = False + ) -> _ModulePlaceholder: ... + + @staticmethod + def create_module_placeholder(full_name: str, required: bool = False): + """Create a module placeholder. + + Use this method when the lazy-imported module fails to be imported. + + Arguments + --------- + full_name: `str` + The name of the optional module. + + required: `bool` + If `True`, this method will raise `ModuleNotFoundError`. Otherwise, + returns a placeholder module. + + Returns + ------- + #1: `_ModulePlaceholder` + Return the placeholder only when the `required` is `False`. + """ if required: raise ModuleNotFoundError( "utils: The required module to be lazily loaded is not found: " "{0}".format(full_name) ) - else: - if sys.modules.get(full_name, None) is not None: - raise ModuleReplaceError( - "utils: Try to define a new module placeholder. However, a" - " previous module has been found. Replacing an existing " - "(lazy) module or a placeholder with a new placeholder is " - "not allowed: " - "{0}".format(full_name) - ) - module = _ModulePlaceholder(name=full_name) - sys.modules[full_name] = module - return module - if spec.loader is None: - raise TypeError( - "utils: The spec.loader of the required module is None, which cannot" - "be used for establishing the lazily loaded module: {0}".format(spec) + if sys.modules.get(full_name, None) is not None: + raise ModuleReplaceError( + "utils: Try to define a new module placeholder. However, a" + " previous module has been found. Replacing an existing " + "(lazy) module or a placeholder with a new placeholder is " + "not allowed: " + "{0}".format(full_name) + ) + module = _ModulePlaceholder(name=full_name) + sys.modules[full_name] = module + return module + + @classmethod + def lazy_import( + cls, + name: str, + package: Optional[str] = __pkg_name__, + required: bool = True, + dependencies: Optional[Union[str, Sequence[str]]] = None, + rel_dependencies: Optional[Union[ModuleType, Sequence[ModuleType]]] = None, + ) -> ModuleType: + """Perform the lazy import for a module. + The returned module will not be loaded until it is actually used. + + Modified from: + https://docs.python.org/3/library/importlib.html#implementing-lazy-imports + + Arguments + --------- + see the module method `lazy_import()`. + + Returns + ------- + #1: `ModuleType` + A lazy loaded module. It will be loaded when actually using it. + """ + full_name = ".".join(map(str, filter(bool, (package, name)))) + # Fetch the module directly if it has been loaded. + prev_module = sys.modules.get(full_name, None) + if isinstance(prev_module, ModuleType): + return prev_module + # Fail to hit the module. + # Start the check all absolute module dependencies. + is_deps_missing = cls.check_is_dep_missing(dependencies) + # Gather the relative module dependencies, send them to the module loader. + deps = cls.gather_relative_module_dependencies( + rel_dependencies, is_deps_missing=is_deps_missing ) - loader = _LazyLoader(spec.loader, deps) - spec.loader = loader - module = importlib.util.module_from_spec(spec) - sys.modules[full_name] = module - loader.exec_module(module) - return module + # Start to create the lazy-loaded module. + if package is None: + spec = importlib.util.find_spec(name) + else: + spec = importlib.util.find_spec(".{0}".format(name), package=package) + if is_deps_missing or (spec is None): + return cls.create_module_placeholder(full_name=full_name, required=required) + if spec.loader is None: + raise TypeError( + "utils: The spec.loader of the required module is None, which cannot" + "be used for establishing the lazily loaded module: {0}".format(spec) + ) + loader = _LazyLoader(spec.loader, deps) + spec.loader = loader + module = importlib.util.module_from_spec(spec) + sys.modules[full_name] = module + loader.exec_module(module) + return module + + +def lazy_import( + name: str, + package: Optional[str] = __pkg_name__, + required: bool = True, + dependencies: Optional[Union[str, Sequence[str]]] = None, + rel_dependencies: Optional[Union[ModuleType, Sequence[ModuleType]]] = None, +) -> ModuleType: + """Perform the lazy import for a module. + The returned module will not be loaded until it is actually used. + + Modified from: + https://docs.python.org/3/library/importlib.html#implementing-lazy-imports + + Arguments + --------- + name: `str` + The name of the module. It does not need to start with the `.` symbol. + + package: `str | None` + The name of the package (anchor). + + By default: will use the `__name__` of the pacakge where this + `utils` module is placed. if `utils` is not placed as a sub- + module, and `package` is not specified, will search `name` by + absolute import. + + If using `None`: will search `name` by absolute import. + + required: `bool` + Whether to require the existence of the module. If not specified, + will allow to load an empty module when the module is not found. + + dependencies: `str | [str] | None` + One or more depdencies for the module to be loaded. + + If not specified, it means that the module does not need + dependencies. If specified, the module is only loaded when + all dependencies are detected. Otherwise, returns a module + placeholder. The dependencies are module names following the + abosolute import rules. + + rel_dependencies: `ModuleType | [ModuleType] | None` + One or more relative dependencies. Each item is a module (can be lazy). + If using `None`, will return an empty list. + + Returns + ------- + #1: `ModuleType` + A lazy loaded module. It will be loaded when actually using it. + """ + return _LazyImporter().lazy_import( + name=name, + package=package, + required=required, + dependencies=dependencies, + rel_dependencies=rel_dependencies, + ) def get_lazy_attribute( @@ -442,20 +573,28 @@ def get_lazy_attribute( def is_module_invalid(module: ModuleType) -> TypeGuard[_ModulePlaceholder]: """Check whether a lazy module is invalid. - Arguments: - module: Can be a lazy module, a module or a module placeholder. - Returns: - #1: True only when the given module is a module placeholder. + + Arguments + --------- + module: `ModuleType` + Can be a lazy module, a module or a module placeholder. + + Returns + ------- + #1: `bool` + `True` only when the given module is a module placeholder. """ return isinstance(module, _ModulePlaceholder) class cached_property(Generic[K, T], property): """Decorator: Cached Property + Modified from ```python werkzeug.utils.cached_property ``` + A decorator that converts a function into a lazy property. The function wrapped is called the first time to retrieve the result and then that calculated result is used the next time you access @@ -473,6 +612,7 @@ def foo(self, val: int) -> None: def foo(self) -> None: del self.__foo ``` + The class has to have a `__dict__` in order for this property to work. """ @@ -493,6 +633,12 @@ def __init__( doc: Optional[str] = None, name: Optional[str] = None, ) -> None: + """Initialization. + + Arguments + --------- + Similar to the decorator `property`. + """ super().__init__(fget=fget, fset=fset, fdel=fdel, doc=doc) name_ = ( name