From 41c8e602635fc06b3fbab5518715d87ef232e8be Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 15 Nov 2023 20:46:12 +0800 Subject: [PATCH 01/22] + License MIT -> BSD 3 --- LICENSE | 21 --------------------- LICENSE.txt | 28 ++++++++++++++++++++++++++++ README.md | 2 +- pyproject.toml | 4 ++-- 4 files changed, 31 insertions(+), 24 deletions(-) delete mode 100644 LICENSE create mode 100644 LICENSE.txt diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 6ef7ebe..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2023 Jun Xiang - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..afae473 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2023, Jun Xiang + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 7489251..359af48 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Don't forget to give the project a star! Thanks again! ## License -Distributed under the MIT License. See `LICENSE.txt` for more information. +Distributed under the BSD-3-Clause License. See `LICENSE.txt` for more information.

(back to top)

diff --git a/pyproject.toml b/pyproject.toml index 9ed2794..355d7fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "thread" version = "0.1.1" description = "Threading module extension" authors = ["Alex "] -license = "MIT" +license = "BSD-3-Clause" readme = "README.md" packages = [{include = "thread", from = "src"}] include = [{ path = "tests", format = "sdist" }] @@ -14,7 +14,7 @@ keywords = ["threading", "extension", "multiprocessing"] classifiers = [ "Development Status :: 1 - Planning", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License" + "License :: OSI Approved :: BSD License" ] [tool.poetry.urls] From 264dbe193f2146cef43e7a3060c720cc5dea86cf Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 15 Nov 2023 20:48:10 +0800 Subject: [PATCH 02/22] + roadmap update --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 359af48..4c55398 100644 --- a/README.md +++ b/README.md @@ -104,12 +104,12 @@ _Below is an example of how you can instruct your audience on installing and set ## Roadmap -- [x] 0.0.1 Release -- [x] Bug fixes -- [x] 0.1.0 Release -- [x] 0.1.1 Hotfix +- [ ] Bug fixes +- [ ] Set Thread class to inherit from threading.Thread +- [ ] Add kill method - [ ] New Features -- [ ] 0.1.1 Release +- [ ] Bug fixes +- [ ] v0.1.2 Release See the [open issues](https://github.com/caffeine-addictt/thread/issues) for a full list of proposed features (and known issues). From f5b65fcecd20e9dcaa8e369c3cb6e23534f643e6 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 15 Nov 2023 20:51:57 +0800 Subject: [PATCH 03/22] + license shield --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c55398..94cc542 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ [![Forks][forks-shield]][forks-url] [![Stargazers][stars-shield]][stars-url] [![Issues][issues-shield]][issues-url] -[![MIT License][license-shield]][license-url] +[![BSD-3-Clause License][license-shield]][license-url] [![LinkedIn][linkedin-shield]][linkedin-url] From eedca08f128246e3672c0aebb83e0f5f2979b4ab Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 15 Nov 2023 21:09:47 +0800 Subject: [PATCH 04/22] bug fix for #8 --- src/thread/thread.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/thread/thread.py b/src/thread/thread.py index 67d93c6..8e12a4f 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -36,7 +36,7 @@ class Thread: hooks : List[Callable[[Data_Out], Any | None]] returned_value : Data_Out - target : Callable[Concatenate[Data_In, ...], Data_Out] + target : Callable[..., Data_Out] args : Sequence[Data_In] kwargs : Mapping[str, Data_In] @@ -49,7 +49,7 @@ class Thread: def __init__( self, - target: Callable[Concatenate[Data_In, ...], Data_Out], + target: Callable[..., Data_Out], args: Sequence[Data_In] = (), kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), @@ -98,8 +98,8 @@ def __init__( def _wrap_target( self, - target: Callable[Concatenate[Data_In, ...], Data_Out] - ) -> Callable[Concatenate[Data_In, ...], Data_Out]: + target: Callable[..., Data_Out] + ) -> Callable[..., Data_Out]: @wraps(target) def wrapper(*args: Any, **kwargs: Any) -> Any: self.status = 'Running' From f82bc0403b22cd766c36eb447f39be9911f0a3cf Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 10:48:53 +0800 Subject: [PATCH 05/22] + bug fix for #8 + join() to return a bool --- src/thread/thread.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/thread/thread.py b/src/thread/thread.py index 8e12a4f..c89fdb4 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -12,12 +12,6 @@ ) -@dataclass -class JoinTerminatedStatus: - """How the `Thread.join()` method terminated""" - status: Literal['Timeout Exceeded', 'Thread terminated'] - - ThreadStatus = Literal['Idle', 'Running', 'Invoking hooks', 'Completed', 'Errored'] Data_In = Any Data_Out = Any @@ -194,7 +188,7 @@ def add_hook(self, hook: Callable[[Data_Out], Any | None]) -> None: self.hooks.append(hook) - def join(self, timeout: Optional[float] = None) -> 'JoinTerminatedStatus': + def join(self, timeout: Optional[float] = None) -> bool: """ Halts the current thread execution until a thread completes or exceeds the timeout @@ -204,7 +198,7 @@ def join(self, timeout: Optional[float] = None) -> 'JoinTerminatedStatus': Returns ------- - :returns JoinTerminatedStatus: Why the method stoped halting the thread + :returns bool: True if the thread is no-longer alive Raises ------ @@ -219,7 +213,7 @@ def join(self, timeout: Optional[float] = None) -> 'JoinTerminatedStatus': self._thread.join(timeout) self._handle_exceptions() - return JoinTerminatedStatus(self._thread.is_alive() and 'Timeout Exceeded' or 'Thread terminated') + return not self._thread.is_alive() def get_return_value(self) -> Data_Out: @@ -389,13 +383,13 @@ def get_return_values(self) -> List[Data_Out]: return results - def join(self) -> 'JoinTerminatedStatus': + def join(self) -> bool: """ Halts the current thread execution until a thread completes or exceeds the timeout Returns ------- - :returns JoinTerminatedStatus: Why the method stoped halting the thread + :returns bool: True if the thread is no-longer alive Raises ------ @@ -410,7 +404,7 @@ def join(self) -> 'JoinTerminatedStatus': for thread in self._threads: thread.join() - return JoinTerminatedStatus('Thread terminated') + return True def start(self) -> None: From 8cf81b1d6a43c7f9a4d0251f928f433881c842c4 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 13:41:32 +0800 Subject: [PATCH 06/22] + update tests --- tests/test_parallelprocessing.py | 23 ++++++++--------------- tests/test_thread.py | 29 ++++++++++++----------------- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/tests/test_parallelprocessing.py b/tests/test_parallelprocessing.py index 165f3ac..8686b59 100644 --- a/tests/test_parallelprocessing.py +++ b/tests/test_parallelprocessing.py @@ -24,7 +24,8 @@ def test_threadsScaleDown(): function = _dummy_dataProcessor, dataset = dataset, max_threads = 4, - kwargs = { 'delay': 2 } + kwargs = { 'delay': 2 }, + daemon = True ) new.start() assert len(new._threads) == 2 @@ -35,7 +36,8 @@ def test_threadsProcessing(): new = ParallelProcessing( function = _dummy_dataProcessor, dataset = dataset, - args = [0.001] + args = [0.001], + daemon = True ) new.start() assert new.get_return_values() == dataset @@ -44,24 +46,14 @@ def test_threadsProcessing(): # >>>>>>>>>> Raising Exceptions <<<<<<<<<< # -def test_raises_notInitializedError(): - """This test should raise ThreadNotInitializedError""" - dataset = numpy.arange(0, 8).tolist() - new = ParallelProcessing( - function = _dummy_dataProcessor, - dataset = dataset - ) - - with pytest.raises(exceptions.ThreadNotInitializedError): - new.results - def test_raises_StillRunningError(): """This test should raise ThreadStillRunningError""" dataset = numpy.arange(0, 8).tolist() new = ParallelProcessing( function = _dummy_dataProcessor, dataset = dataset, - args = [1] + args = [1], + daemon = True ) new.start() with pytest.raises(exceptions.ThreadStillRunningError): @@ -73,7 +65,8 @@ def test_raises_RunTimeError(): new = ParallelProcessing( function = _dummy_raiseException, dataset = dataset, - args = [0.01] + args = [0.01], + daemon = True ) with pytest.raises(RuntimeError): new.start() diff --git a/tests/test_thread.py b/tests/test_thread.py index c7c0121..1d5b18a 100644 --- a/tests/test_thread.py +++ b/tests/test_thread.py @@ -21,11 +21,11 @@ def test_threadCreation(): new = Thread( target = _dummy_target_raiseToPower, args = [4], - kwargs = { 'power': 2 } + kwargs = { 'power': 2 }, + daemon = True ) new.start() - s = new.join() - assert s.status == 'Thread terminated' + assert new.join() assert new.result == 16 def test_threadingThreadParsing(): @@ -37,7 +37,7 @@ def test_threadingThreadParsing(): daemon = True ) new.start() - assert new._thread and new._thread.name == 'testingThread' + assert new.name == 'testingThread' def test_suppressAll(): """This test is for testing that errors are suppressed properly""" @@ -58,6 +58,7 @@ def test_ignoreSpecificError(): target = _dummy_raiseException, args = [ValueError()], ignore_errors = [ValueError], + daemon = True ) new.start() new.join() @@ -69,6 +70,7 @@ def test_ignoreAll(): target = _dummy_raiseException, args = [ValueError()], ignore_errors = [Exception], + daemon = True ) new.start() new.join() @@ -78,21 +80,12 @@ def test_ignoreAll(): # >>>>>>>>>> Raising Exceptions <<<<<<<<<< # -def test_raises_notInitializedError(): - """This test should raise ThreadNotInitializedError""" - new = Thread( - target = _dummy_target_raiseToPower, - args = [4, 2] - ) - - with pytest.raises(exceptions.ThreadNotInitializedError): - new.result - def test_raises_stillRunningError(): """This test should raise ThreadStillRunningError""" new = Thread( target = _dummy_target_raiseToPower, - args = [4, 2, 5] + args = [4, 2, 5], + daemon = True ) new.start() @@ -105,7 +98,8 @@ def test_raises_ignoreSpecificError(): target = _dummy_raiseException, args = [FileExistsError()], ignore_errors = [ValueError], - suppress_errors = False + suppress_errors = False, + daemon = True ) with pytest.raises(FileExistsError): new.start() @@ -115,7 +109,8 @@ def test_raises_HookError(): """This test should raise """ new = Thread( target = _dummy_target_raiseToPower, - args = [4, 2] + args = [4, 2], + daemon = True ) def newhook(x: int): From 5565367badf70c4484f2b5198803402e3a2420ce Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 13:41:54 +0800 Subject: [PATCH 07/22] + update initalized error msg --- src/thread/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/thread/exceptions.py b/src/thread/exceptions.py index 77a5813..581d340 100644 --- a/src/thread/exceptions.py +++ b/src/thread/exceptions.py @@ -20,7 +20,7 @@ class ThreadNotRunningError(ThreadErrorBase): class ThreadNotInitializedError(ThreadErrorBase): """Exception class for attempting to invoke a method which requires the thread to be initialized, but isn't""" - message: str = 'Thread is not initialized, unable to invoke method. Have you ran `Thread.start()` at least once?' + message: str = 'Thread is not initialized, unable to invoke method.' class HookRuntimeError(ThreadErrorBase): """Exception class for hook runtime errors""" From f9ed52614c84013d2806958152cc671537b02381 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 13:43:46 +0800 Subject: [PATCH 08/22] + inherit from threading.Thread --- src/thread/thread.py | 83 +++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 47 deletions(-) diff --git a/src/thread/thread.py b/src/thread/thread.py index c89fdb4..008c8b8 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -17,7 +17,7 @@ Data_Out = Any Overflow_In = Any -class Thread: +class Thread(threading.Thread): """ Wraps python's `threading.Thread` class --------------------------------------- @@ -25,21 +25,17 @@ class Thread: Type-Safe and provides more functionality on top """ - _thread : Optional[threading.Thread] status : ThreadStatus hooks : List[Callable[[Data_Out], Any | None]] - returned_value : Data_Out - - target : Callable[..., Data_Out] - args : Sequence[Data_In] - kwargs : Mapping[str, Data_In] + returned_value: Data_Out errors : List[Exception] ignore_errors : Sequence[type[Exception]] suppress_errors: bool - overflow_args : Sequence[Overflow_In] - overflow_kwargs: Mapping[str, Overflow_In] + # threading.Thread stuff + _initialized : bool + def __init__( self, @@ -51,6 +47,7 @@ def __init__( name: Optional[str] = None, daemon: bool = False, + group = None, *overflow_args: Overflow_In, **overflow_kwargs: Overflow_In ) -> None: @@ -66,34 +63,33 @@ def __init__( :param suppress_errors: This should be a boolean indicating whether exceptions will be raised, else will only write to internal `errors` property :param name: This is an argument parsed to `threading.Thread` :param daemon: This is an argument parsed to `threading.Thread` + :param group: This does nothing right now, but should be left as None :param *: These are arguments parsed to `threading.Thread` :param **: These are arguments parsed to `thread.Thread` """ - self._thread = None + target = self._wrap_target(target) + self.returned_values = None self.status = 'Idle' self.hooks = [] - self.returned_value = None - - self.target = self._wrap_target(target) - self.args = args - self.kwargs = kwargs self.errors = [] self.ignore_errors = ignore_errors self.suppress_errors = suppress_errors - self.overflow_args = overflow_args - self.overflow_kwargs = { - 'name': name, - 'daemon': daemon, + super().__init__( + target = target, + args = args, + kwargs = kwargs, + name = name, + daemon = daemon, + group = group, + *overflow_args, **overflow_kwargs - } + ) - def _wrap_target( - self, - target: Callable[..., Data_Out] - ) -> Callable[..., Data_Out]: + def _wrap_target(self, target: Callable[..., Data_Out]) -> Callable[..., Data_Out]: + """Wraps the target function""" @wraps(target) def wrapper(*args: Any, **kwargs: Any) -> Any: self.status = 'Running' @@ -110,22 +106,22 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: self._invoke_hooks() self.status = 'Completed' return wrapper - + def _invoke_hooks(self) -> None: - trace = exceptions.HookRuntimeError() + err = exceptions.HookRuntimeError() for hook in self.hooks: try: hook(self.returned_value) except Exception as e: if not any(isinstance(e, ignore) for ignore in self.ignore_errors): - trace.add_exception_case( + err.add_exception_case( hook.__name__, e ) - if trace.count > 0: - self.errors.append(trace) + if err.count > 0: + self.errors.append(err) def _handle_exceptions(self) -> None: @@ -135,7 +131,7 @@ def _handle_exceptions(self) -> None: for e in self.errors: raise e - + @property def result(self) -> Data_Out: @@ -144,13 +140,12 @@ def result(self) -> Data_Out: Raises ------ - ThreadNotInitializedError: If the thread is not initialized + ThreadNotInitializedError: If the thread is not intialized ThreadNotRunningError: If the thread is not running ThreadStillRunningError: If the thread is still running """ - if not self._thread: + if not self._initialized: raise exceptions.ThreadNotInitializedError() - if self.status == 'Idle': raise exceptions.ThreadNotRunningError() @@ -169,9 +164,9 @@ def is_alive(self) -> bool: ------ ThreadNotInitializedError: If the thread is not intialized """ - if not self._thread: + if not self._initialized: raise exceptions.ThreadNotInitializedError() - return self._thread.is_alive() + return super().is_alive() def add_hook(self, hook: Callable[[Data_Out], Any | None]) -> None: @@ -205,15 +200,15 @@ def join(self, timeout: Optional[float] = None) -> bool: ThreadNotInitializedError: If the thread is not initialized ThreadNotRunningError: If the thread is not running """ - if not self._thread: + if not self._initialized: raise exceptions.ThreadNotInitializedError() if self.status == 'Idle': raise exceptions.ThreadNotRunningError() - self._thread.join(timeout) + super().join(timeout) self._handle_exceptions() - return not self._thread.is_alive() + return not self.is_alive() def get_return_value(self) -> Data_Out: @@ -234,19 +229,13 @@ def start(self) -> None: Raises ------ + ThreadNotInitializedError: If the thread is not intialized ThreadStillRunningError: If there already is a running thread """ - if self._thread is not None and self._thread.is_alive(): + if self.is_alive(): raise exceptions.ThreadStillRunningError() - self._thread = threading.Thread( - target = self.target, - args = self.args, - kwargs = self.kwargs, - *self.overflow_args, - **self.overflow_kwargs - ) - self._thread.start() + super().start() From 83c815f036d24da4a4873947bc0b5b0ee8121288 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 13:44:03 +0800 Subject: [PATCH 09/22] + coverage --- .coverage | Bin 53248 -> 53248 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/.coverage b/.coverage index acb4b5460eba0bbd448261c8af688767fadd091d..096fb913bea33adf137f858cd23bd3b84f0bb436 100644 GIT binary patch delta 162 zcmZozz}&Eac>`MmUlRlWPyUzu5BZny&){#`EGUr2UoXPO!pNz`)@S`9_WrN9_f_n% zuXmr5{_dWvH4Bhk!Sb#l{ej=FI);Yw{}=z-{x{BL3(#X!W#D9(Fo%I- zfV4l8iV1vavA(HS;k$@i75uIVQRC3;DIp`(9hKGO}=T N@-cF4W}NWf9smd;G=2a8 delta 162 zcmZozz}&Eac>`MmUp)i=PyXlp_xTs`PvfuOEGUr8UoXnW!pNz^)@S`<@4l~l?|tR7 zdL4DY^c@!~P*{xh+4FbzWUX0%>E+a!m{@>k_2l^WP89dw^%Jws|F#|R8F*^w`0cj;Bx$+D7waxRs#@=US OWn|&x-2AVf$pHZ8PCLB- From 62f8feac32b841fcd4f240fee70496c010047eb3 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 13:44:52 +0800 Subject: [PATCH 10/22] + py3.9, 3.10 --- .github/workflows/test-worker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-worker.yml b/.github/workflows/test-worker.yml index bfb507a..71d033c 100644 --- a/.github/workflows/test-worker.yml +++ b/.github/workflows/test-worker.yml @@ -1,6 +1,6 @@ name: Run Python tests -on: [ push, pull_request ] +on: [ push ] jobs: build: @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11", "3.x"] + python-version: ["3.9", "3.10", "3.11", "3.x"] steps: From 96653f51a750dddb254440d27193569eb5d253ca Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 13:53:37 +0800 Subject: [PATCH 11/22] + change import to typing_ext --- src/thread/thread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/thread/thread.py b/src/thread/thread.py index 008c8b8..42c7e9c 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -3,10 +3,10 @@ from . import exceptions from functools import wraps -from dataclasses import dataclass +from typing_extensions import Concatenate from typing import ( Any, List, - Callable, Concatenate, + Callable, Optional, Literal, Mapping, Sequence ) From c314a8ee0ccedd2583dd9d719f55833339130eb2 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 14:00:11 +0800 Subject: [PATCH 12/22] + add typing_extensions --- poetry.lock | 13 ++++++++++++- pyproject.toml | 1 + requirements.txt | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index ab61971..8173b83 100644 --- a/poetry.lock +++ b/poetry.lock @@ -203,7 +203,18 @@ files = [ {file = "ruff-0.1.5.tar.gz", hash = "sha256:5cbec0ef2ae1748fb194f420fb03fb2c25c3258c86129af7172ff8f198f125ab"}, ] +[[package]] +name = "typing-extensions" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, +] + [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "781a675fea5f2f70ce3fda416982ebf031e9951e266d0ecdafd8960fe52a4d90" +content-hash = "07e6ff0a0176752eab68a233e414e2009bd7b1cd732ca68167dd80f1b4b206aa" diff --git a/pyproject.toml b/pyproject.toml index 355d7fc..c97b5f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.11" numpy = "^1.26.2" +typing-extensions = "^4.8.0" [tool.poetry.group.dev.dependencies] diff --git a/requirements.txt b/requirements.txt index 1357faf..ded088e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ numpy==1.26.2 +typing_extensions==4.8.0 From f772a02f3c4b3d3c14f39fbcedd507a14a7e624f Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 17:29:53 +0800 Subject: [PATCH 13/22] Make compatible with 3.9+ --- src/thread/thread.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/thread/thread.py b/src/thread/thread.py index 42c7e9c..72d2957 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -3,12 +3,10 @@ from . import exceptions from functools import wraps -from typing_extensions import Concatenate from typing import ( Any, List, - Callable, - Optional, Literal, - Mapping, Sequence + Callable, Union, Optional, Literal, + Mapping, Sequence, Tuple ) @@ -17,6 +15,7 @@ Data_Out = Any Overflow_In = Any + class Thread(threading.Thread): """ Wraps python's `threading.Thread` class @@ -26,7 +25,7 @@ class Thread(threading.Thread): """ status : ThreadStatus - hooks : List[Callable[[Data_Out], Any | None]] + hooks : List[Callable[[Data_Out], Union[Any, None]]] returned_value: Data_Out errors : List[Exception] @@ -67,7 +66,7 @@ def __init__( :param *: These are arguments parsed to `threading.Thread` :param **: These are arguments parsed to `thread.Thread` """ - target = self._wrap_target(target) + _target = self._wrap_target(target) self.returned_values = None self.status = 'Idle' self.hooks = [] @@ -77,7 +76,7 @@ def __init__( self.suppress_errors = suppress_errors super().__init__( - target = target, + target = _target, args = args, kwargs = kwargs, name = name, @@ -109,19 +108,21 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: def _invoke_hooks(self) -> None: - err = exceptions.HookRuntimeError() + errors: List[Tuple[Exception, str]] = [] for hook in self.hooks: try: hook(self.returned_value) except Exception as e: if not any(isinstance(e, ignore) for ignore in self.ignore_errors): - err.add_exception_case( - hook.__name__, - e - ) + errors.append(( + e, + hook.__name__ + )) - if err.count > 0: - self.errors.append(err) + if len(errors) > 0: + self.errors.append(exceptions.HookRuntimeError( + None, errors + )) def _handle_exceptions(self) -> None: @@ -169,7 +170,7 @@ def is_alive(self) -> bool: return super().is_alive() - def add_hook(self, hook: Callable[[Data_Out], Any | None]) -> None: + def add_hook(self, hook: Callable[[Data_Out], Union[Any, None]]) -> None: """ Adds a hook to the thread ------------------------- @@ -253,7 +254,7 @@ class ParallelProcessing: _return_vales : Mapping[int, List[Data_Out]] status : ThreadStatus - function : Callable[Concatenate[Sequence[Data_In], ...], List[Data_Out]] + function : Callable[..., List[Data_Out]] dataset : Sequence[Data_In] max_threads : int @@ -262,7 +263,7 @@ class ParallelProcessing: def __init__( self, - function: Callable[Concatenate[Data_In, ...], Data_Out], + function: Callable[..., Data_Out], dataset: Sequence[Data_In], max_threads: int = 8, @@ -277,7 +278,7 @@ def __init__( Parameters ---------- - :param function: This should be the function to validate each data entry in the `dataset` + :param function: This should be the function to validate each data entry in the `dataset`, the first argument parsed will be a value of the dataset :param dataset: This should be an iterable sequence of data entries :param max_threads: This should be an integer value of the max threads allowed :param *: These are arguments parsed to `threading.Thread` and `Thread` @@ -303,8 +304,8 @@ def __init__( def _wrap_function( self, - function: Callable[Concatenate[Data_In, ...], Data_Out] - ) -> Callable[Concatenate[Sequence[Data_In], ...], List[Data_Out]]: + function: Callable[..., Data_Out] + ) -> Callable[..., List[Data_Out]]: @wraps(function) def wrapper(data_chunk: Sequence[Data_In], *args: Any, **kwargs: Any) -> List[Data_Out]: computed: List[Data_Out] = [] From c1f7c4b69127bc32aa1ad990900ad847458835d1 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 17:30:34 +0800 Subject: [PATCH 14/22] Make compatible with 3.9+ --- src/thread/exceptions.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/thread/exceptions.py b/src/thread/exceptions.py index 581d340..bdc6048 100644 --- a/src/thread/exceptions.py +++ b/src/thread/exceptions.py @@ -1,5 +1,5 @@ import traceback -from typing import Any, Optional +from typing import Any, Optional, Sequence, Tuple class ThreadErrorBase(Exception): @@ -27,10 +27,30 @@ class HookRuntimeError(ThreadErrorBase): message: str = 'Encountered runtime errors in hooks' count: int = 0 - def add_exception_case(self, func_name: str, error: Exception): - self.count += 1 - trace = '\n'.join(traceback.format_stack()) - - self.add_note(f'\n{self.count}. {func_name}\n>>>>>>>>>>') - self.add_note(f'{trace}\n{error}') - self.add_note('<<<<<<<<<<') + def __init__(self, message: Optional[str] = '', extra: Sequence[Tuple[Exception, str]] = []) -> None: + """ + Extra for parsing all hooks that errored + + Parameters + ---------- + :param message: The message to be parsed, can be left blank + :param extra: Tuple of (Exception_Raised, function_name) + """ + new_message: str = message or self.message + + for i, v in enumerate(extra): + trace = '\n'.join(traceback.format_stack()) + new_message += f'\n\n{i}. {v[1]}\n>>>>>>>>>>' + new_message += f'{trace}\n{v[0]}' + new_message += '<<<<<<<<<<' + super().__init__(new_message) + + + # Python 3.9 doesn't support Exception.add_note() + # def add_exception_case(self, func_name: str, error: Exception): + # self.count += 1 + # trace = '\n'.join(traceback.format_stack()) + + # self.add_note(f'\n{self.count}. {func_name}\n>>>>>>>>>>') + # self.add_note(f'{trace}\n{error}') + # self.add_note('<<<<<<<<<<') From e93311aa6c7d63420445ef76ce81bd91e8494bd5 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 17:38:32 +0800 Subject: [PATCH 15/22] lower python requirement to 3.9 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c97b5f8..57cf462 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ "Bug Tracker" = "https://github.com/caffeine-addictt/thread/issues" [tool.poetry.dependencies] -python = "^3.11" +python = "^3.9" numpy = "^1.26.2" typing-extensions = "^4.8.0" From 1485edc26159b5e14156579f03df884a03024d40 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 17:45:22 +0800 Subject: [PATCH 16/22] README update --- README.md | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8fde8e3..4d7d71c 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,6 @@ I hope thread will become your threading solution! ♡⸜(˶˃ ᵕ ˂˶)⸝♡ -### Built With - -* [![Python 3.11.6+](https://img.shields.io/badge/python-3.11.6+-blue.svg)](https://www.python.org/downloads/release/python-3116/) - -

(back to top)

- - - ## Getting Started @@ -70,7 +62,7 @@ To get a local copy up and running follow these simple example steps. ### Prerequisites -* Python 3.10+ +* Python 3.9+ ### Installation @@ -102,11 +94,10 @@ Our docs are [here!](/docs/getting-started.md) ## Roadmap -- [ ] Bug fixes -- [ ] Set Thread class to inherit from threading.Thread +- [x] Bug fixes +- [x] Set Thread class to inherit from threading.Thread - [ ] Add kill method -- [ ] New Features -- [ ] Bug fixes +- [ ] Docs Update - [ ] v0.1.2 Release See the [open issues](https://github.com/caffeine-addictt/thread/issues) for a full list of proposed features (and known issues). From f3ec68fb43833d1c9a137f4d0815c6030954bf30 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 20:12:54 +0800 Subject: [PATCH 17/22] + Thread killing\n+ Graceful killing --- src/thread/thread.py | 86 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/src/thread/thread.py b/src/thread/thread.py index 72d2957..c45978d 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -1,5 +1,8 @@ -import numpy +import sys +import signal import threading + +import numpy from . import exceptions from functools import wraps @@ -10,7 +13,7 @@ ) -ThreadStatus = Literal['Idle', 'Running', 'Invoking hooks', 'Completed', 'Errored'] +ThreadStatus = Literal['Idle', 'Running', 'Invoking hooks', 'Completed', 'Errored', 'Killed'] Data_In = Any Data_Out = Any Overflow_In = Any @@ -34,6 +37,7 @@ class Thread(threading.Thread): # threading.Thread stuff _initialized : bool + _run : Callable def __init__( @@ -108,6 +112,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: def _invoke_hooks(self) -> None: + """Invokes hooks in the thread""" errors: List[Tuple[Exception, str]] = [] for hook in self.hooks: try: @@ -126,7 +131,7 @@ def _invoke_hooks(self) -> None: def _handle_exceptions(self) -> None: - """Raises exceptions if not suppressed""" + """Raises exceptions if not suppressed in the main thread""" if self.suppress_errors: return @@ -134,6 +139,25 @@ def _handle_exceptions(self) -> None: raise e + def global_trace(self, frame, event: str, arg) -> Callable | None: + if event == 'call': + return self.local_trace + + def local_trace(self, frame, event, arg): + if self.status == 'Killed' and event == 'line': + print('KILLED ident:%s' % self.ident) + raise SystemExit() + return self.local_trace + + def _run_with_trace(self) -> None: + """This will replace `threading.Thread`'s `run()` method""" + if not self._run: + raise exceptions.ThreadNotInitializedError('Running `_run_with_trace` may cause unintended behaviour, run `start` instead') + + sys.settrace(self.global_trace) + self._run() + + @property def result(self) -> Data_Out: """ @@ -147,7 +171,7 @@ def result(self) -> Data_Out: """ if not self._initialized: raise exceptions.ThreadNotInitializedError() - if self.status == 'Idle': + if self.status in ['Idle', 'Killed']: raise exceptions.ThreadNotRunningError() self._handle_exceptions() @@ -204,7 +228,7 @@ def join(self, timeout: Optional[float] = None) -> bool: if not self._initialized: raise exceptions.ThreadNotInitializedError() - if self.status == 'Idle': + if self.status == ['Idle', 'Killed']: raise exceptions.ThreadNotRunningError() super().join(timeout) @@ -224,6 +248,20 @@ def get_return_value(self) -> Data_Out: return self.result + def kill(self) -> None: + """ + Kills the thread + + Raises + ------ + ThreadNotInitializedError: If the thread is not initialized + ThreadNotRunningError: If the thread is not running + """ + if not self.is_alive(): + raise exceptions.ThreadNotRunningError() + self.status = 'Killed' + + def start(self) -> None: """ Starts the thread @@ -236,6 +274,8 @@ def start(self) -> None: if self.is_alive(): raise exceptions.ThreadStillRunningError() + self._run = self.run + self.run = self._run_with_trace super().start() @@ -338,10 +378,6 @@ def results(self) -> Data_Out: results: List[Data_Out] = [] for thread in self._threads: results += thread.result - if thread.status == 'Idle': - raise exceptions.ThreadNotRunningError() - elif thread.status == 'Running': - raise exceptions.ThreadStillRunningError() return results @@ -397,6 +433,19 @@ def join(self) -> bool: return True + def kill(self) -> None: + """ + Kills the threads + + Raises + ------ + ThreadNotInitializedError: If the thread is not initialized + ThreadNotRunningError: If the thread is not running + """ + for thread in self._threads: + thread.kill() + + def start(self) -> None: """ Starts the threads @@ -424,3 +473,22 @@ def start(self) -> None: ) self._threads.append(chunk_thread) chunk_thread.start() + + + + + +# Handle abrupt exit +def service_shutdown(signum, frame): + print('\nCaught signal %d' % signum) + print('Gracefully killing active threads') + + for thread in threading.enumerate(): + if isinstance(thread, Thread): + thread.kill() + sys.exit(0) + + +# Register the signal handlers +signal.signal(signal.SIGTERM, service_shutdown) +signal.signal(signal.SIGINT, service_shutdown) From 2dd31fd1e59d0d358782e383063f2315b1f20715 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 21:19:48 +0800 Subject: [PATCH 18/22] + yielding on kill method --- src/thread/thread.py | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/thread/thread.py b/src/thread/thread.py index c45978d..697d942 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -1,4 +1,6 @@ +import os import sys +import time import signal import threading @@ -13,7 +15,16 @@ ) -ThreadStatus = Literal['Idle', 'Running', 'Invoking hooks', 'Completed', 'Errored', 'Killed'] +ThreadStatus = Literal[ + 'Idle', + 'Running', + 'Invoking hooks', + 'Completed', + + 'Errored', + 'Kill Scheduled', + 'Killed' +] Data_In = Any Data_Out = Any Overflow_In = Any @@ -143,9 +154,10 @@ def global_trace(self, frame, event: str, arg) -> Callable | None: if event == 'call': return self.local_trace - def local_trace(self, frame, event, arg): - if self.status == 'Killed' and event == 'line': + def local_trace(self, frame, event: str, arg): + if self.status == 'Kill Scheduled' and event == 'line': print('KILLED ident:%s' % self.ident) + self.status = 'Killed' raise SystemExit() return self.local_trace @@ -248,9 +260,18 @@ def get_return_value(self) -> Data_Out: return self.result - def kill(self) -> None: + def kill(self, yielding: bool = False, timeout: float = 5) -> bool: """ - Kills the thread + Schedules a thread to be killed + + Parameters + ---------- + :param yielding: If true, halts the current thread execution until the thread is killed + :param timeout: The maximum number of seconds to wait before exiting + + Returns + ------- + :returns bool: False if the it exceeded the timeout Raises ------ @@ -259,7 +280,18 @@ def kill(self) -> None: """ if not self.is_alive(): raise exceptions.ThreadNotRunningError() - self.status = 'Killed' + + self.status = 'Kill Scheduled' + if not yielding: + return True + + start = time.perf_counter() + while self.status != 'Killed': + time.sleep(0.01) + if (time.perf_counter() - start) >= timeout: + return False + + return True def start(self) -> None: From 41c8678d4ef5b34cb1818c60a643e93b96f70b23 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 21:20:04 +0800 Subject: [PATCH 19/22] + coverage report --- .coverage | Bin 53248 -> 53248 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/.coverage b/.coverage index 096fb913bea33adf137f858cd23bd3b84f0bb436..5a7fa844d141697a2f082bb2aba6fe13c4bb5834 100644 GIT binary patch delta 171 zcmZozz}&Eac>`MmUl9ZUPyQGD5BR6?ck>r*78Ho%uNP-yVdOMm>+4{sy}o_l_43;5 zvR|V1zh24A$qJMZVSV=eU7QCCD`MmUlRlWPyUzu5BZny&){#`EGUr2UoXPO!pNz`)@S`9_WrN9_f_n% zuXmr5{_dWvH4Bhk!Sb#l{ej=FI);Yw{}=z-{x{BL3(#X!W#D9(Fo%I- zfV4l8iV1vavA(HS;k$@i75uIVQRC3;DIp`(9hKGO}=T N@-cF4W}NWf9smd;G=2a8 From 711a0f3f66f7a5e43b40b1b865210b5a3b6c0ac5 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 21:20:25 +0800 Subject: [PATCH 20/22] + added test for kill method --- tests/test_thread.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_thread.py b/tests/test_thread.py index 1d5b18a..99d8cdd 100644 --- a/tests/test_thread.py +++ b/tests/test_thread.py @@ -12,6 +12,11 @@ def _dummy_raiseException(x: Exception, delay: float = 0): time.sleep(delay) raise x +def _dummy_iterative(itemCount: int, pTime: float = 0.1, delay: float = 0): + time.sleep(delay) + for i in range(itemCount): + time.sleep(pTime) + @@ -76,6 +81,16 @@ def test_ignoreAll(): new.join() assert len(new.errors) == 0 +def test_threadKilling(): + """This test is for testing that threads are killed properly""" + new = Thread( + target = _dummy_iterative, + args = [5, 0.1, 0] + ) + new.start() + new.kill(True) + assert not new.is_alive() + From 7b242c7f08df1725ac7bd15161e7b96487a635de Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 21:26:57 +0800 Subject: [PATCH 21/22] + python3.9 test failed fix --- .gitignore | 2 +- src/thread/thread.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 10ef4e2..06b49b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # Python stuff -venv/ +venv*/ *.pytest_cache __pycache__/ diff --git a/src/thread/thread.py b/src/thread/thread.py index 697d942..444e88c 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -150,7 +150,7 @@ def _handle_exceptions(self) -> None: raise e - def global_trace(self, frame, event: str, arg) -> Callable | None: + def global_trace(self, frame, event: str, arg) -> Optional[Callable]: if event == 'call': return self.local_trace From d4a89680ef17b35fc5c2d4fd7f54b3de318f59eb Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Nov 2023 21:29:12 +0800 Subject: [PATCH 22/22] - removed unnecessary import --- src/thread/thread.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/thread/thread.py b/src/thread/thread.py index 444e88c..c8345e8 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -1,4 +1,3 @@ -import os import sys import time import signal