Squash benc-mypy history

.github/workflows/ci.yaml

@@ -10,7 +10,7 @@ jobs:
   main-test-suite:
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.11"]
     runs-on: ubuntu-20.04
     timeout-minutes: 30
 

Makefile

@@ -46,7 +46,7 @@ clean_coverage:
 
 .PHONY: mypy
 mypy: ## run mypy checks
-	MYPYPATH=$(CWD)/mypy-stubs mypy parsl/
+	PYTHONPATH=$(CWD)/mypy-plugins:$(PYTHONPATH) MYPYPATH=$(CWD)/mypy-stubs mypy --no-incremental parsl/ --show-traceback
 
 .PHONY: local_thread_test
 local_thread_test: ## run all tests with local_thread config

mypy-plugins/parsl_mypy.py

@@ -0,0 +1,56 @@
+from mypy.plugin import FunctionContext, Plugin
+from mypy.types import Type
+import mypy.nodes as nodes
+
+def plugin(v):
+    return ParslMypyPlugin
+
+class ParslMypyPlugin(Plugin):
+    def get_type_analyze_hook(self, t):
+        # print("BENC: gtah t={}".format(t))
+        return None
+
+    def get_function_hook(self, f):
+        if f == "parsl.app.app.python_appXXXX":
+            return python_app_function_hook
+        else:
+            return None
+
+def python_app_function_hook(ctx: FunctionContext) -> Type:
+    print("inside python_app function_hook")
+    print("ctx = {}".format(ctx))
+
+    # if python_app is being called with a function parameter (rather than
+    # None, the default) then the return type of the python_app decorator
+    # is a variation (with a Future added on the type of the decorated
+    # function...)
+
+    if ctx.callee_arg_names[0] == "function":  # will this always be at position 0? probably fragile to assume so, but this code does make that assumption
+        print(f"python_app called with a function supplied: {ctx.args[0]}")
+        function_node = ctx.args[0][0]
+        print(f"function node repr is {repr(function_node)} with type {type(function_node)}")
+
+        # return the type of function_node - actually it needs modifying to have the Future wrapper added....
+        if isinstance(function_node, nodes.TempNode):
+            print(f"temporary node has type {function_node.type}")
+            print(f"Python type of tempnode.type is {type(function_node.type)}")
+            print(ctx.api)
+            # return_type = ctx.api.named_type_or_none("concurrent.futures.Future", [function_node.type.ret_type])
+            # return_type = ctx.api.named_generic_type("concurrent.futures.Future", [function_node.type.ret_type])
+            # return_type = ctx.api.named_generic_type("builtins.list", [function_node.type.ret_type])
+            return_type = function_node.type.ret_type
+            # return_type = ctx.default_return_type
+            print(f"plugin chosen return type is {return_type}")
+            return function_node.type.copy_modified(ret_type=return_type)
+        else:
+            print("function node is not specified as something this plugin understands")
+            return_type = ctx.default_return_type
+            return return_type
+    else:
+        print("python_app called without a function supplied")
+        # TODO: this should return a type that is aligned with the implementation:
+        # it's the type of the decorator, assuming that it will definitely be given
+        # a function this time? or something along those lines...
+
+        print("will return ctx.default_return_type = {}".format(ctx.default_return_type))
+        return ctx.default_return_type

mypy.ini

@@ -1,11 +1,12 @@
 [mypy]
-plugins = sqlmypy
+plugins = sqlmypy, parsl_mypy
+enable_error_code = ignore-without-code
 
 # globally disabled error codes:
 #   str-bytes-safe warns that a byte string is formatted into a string.
 #                  which is commonly done with manager IDs in the parsl
 #                  codebase.
-disable_error_code = str-bytes-safe
+# disable_error_code = str-bytes-safe
 no_implicit_reexport = True
 warn_redundant_casts = True
 
@@ -14,6 +15,9 @@ no_implicit_optional = True
 strict_equality = True
 warn_unused_ignores = True
 
+# there are some exceptions to this even in the more-strongly-typed sections
+warn_unreachable = True
+
 [mypy-non_existent.*]
 ignore_missing_imports = True
 
@@ -22,19 +26,138 @@ ignore_missing_imports = True
 # parsl to be checked that way, so these options could be default instead of
 # listed per package; but unless/until that happens, the more-strictly-checked
 # code should be listed here:
+
 [mypy-parsl.providers.base.*]
+disallow_untyped_defs = True
 disallow_untyped_decorators = True
 check_untyped_defs = True
 disallow_subclassing_any = True
-warn_unreachable = True
+
+
+# modules to be checked mostly more strongly than default:
+
+[mypy-parsl.addresses.*]
 disallow_untyped_defs = True
+disallow_any_decorated = True
 
-[mypy-parsl.executors.high_throughput.interchange.*]
-check_untyped_defs = True
+[mypy-parsl.dataflow.dflow.*]
+disallow_untyped_defs = True
+
+warn_unreachable = False
+# this is because of some tangle of types
+# to do with channel script dirs: script dir
+# can be none at the start before initialisation,
+# but must be set later on, and I can't
+# represent that session type in mypy.
+# (or can I?!)
+
+[mypy-parsl.dataflow.error.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.executor_status.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.flow_control.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.futures.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.memoization.*]
+disallow_untyped_defs = True
+
+[mypy-parsl.dataflow.rundirs.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.strategy.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.states.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.taskrecord.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.job_status_poller.*]
+disallow_untyped_defs = True
+
+# merge of #1877 introduced stuff that violates this so disabling pending perhaps further investigation
+# disallow_any_expr = True
+
+disallow_any_decorated = True
+
+[mypy-parsl.config.*]
+disallow_untyped_defs = True
+# Any has to be allowed because TaskRecord now forms part of the type signature of config,
+# and task record has Any from the type of tasks args
+#disallow_any_expr = True
+#disallow_any_decorated = True
+
+[mypy-parsl.channels.base.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+
+[mypy-parsl.channels.ssh.*]
+disallow_untyped_defs = True
+
+[mypy-parsl.launchers.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+
+
+[mypy-parsl.executors.base.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+[mypy-parsl.serialize.*]
+disallow_untyped_defs = True
+
+
+# modules to be checked more weakly than default:
+
+[mypy-parsl.executors.flux.*]
+ignore_errors = True
 
 [mypy-parsl.executors.extreme_scale.*]
 ignore_errors = True
 
+[mypy-parsl.providers.aws.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.pbspro.pbspro.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.lsf.lsf.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.torque.torque.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.grid_engine.grid_engine.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.googlecloud.*]
+check_untyped_defs = False
+
+[mypy-parsl.monitoring.db_manager.*]
+check_untyped_defs = False
+
+[mypy-parsl.executors.high_throughput.interchange.*]
+check_untyped_defs = True
+
 [mypy-parsl.executors.workqueue.*]
 check_untyped_defs = True
 
@@ -48,7 +171,6 @@ ignore_errors = True
 disallow_untyped_decorators = True
 check_untyped_defs = True
 disallow_subclassing_any = True
-warn_unreachable = True
 disallow_untyped_defs = True
 
 # visualization typechecks much less well than the rest of monitoring,
@@ -60,9 +182,10 @@ ignore_errors = True
 ignore_missing_imports = True
 
 [mypy-parsl.utils]
-warn_unreachable = True
 disallow_untyped_defs = True
 
+# imports from elsewhere that there are no stubs for:
+
 [mypy-flask_sqlalchemy.*]
 ignore_missing_imports = True
 
@@ -126,22 +249,6 @@ ignore_missing_imports = True
 [mypy-zmq.*]
 ignore_missing_imports = True
 
-[mypy-mpi4py.*]
-ignore_missing_imports = True
-
-[mypy-flask.*]
-ignore_missing_imports = True
-
-# this is an internal undocumentated package
-# of multiprocessing - trying to get Event
-# to typecheck in monitoring, but it's not
-# a top level class as far as mypy is concerned.
-# but... when commented out seems ok?
-# so lets see when happens when I try to merge
-# in clean CI
-#[mypy-multiprocessing.synchronization.*]
-#ignore_missing_imports = True
-
 [mypy-pandas.*]
 ignore_missing_imports = True
 

parsl/addresses.py

@@ -13,7 +13,7 @@
 try:
     import fcntl
 except ImportError:
-    fcntl = None  # type: ignore
+    fcntl = None  # type: ignore[assignment]
 import struct
 import typeguard
 import psutil

parsl/app/app.py

@@ -11,6 +11,17 @@
 
 from parsl.dataflow.dflow import DataFlowKernel
 
+from typing import TYPE_CHECKING
+from typing import Callable
+
+if TYPE_CHECKING:
+    from typing import Dict
+    from typing import Any
+
+from parsl.dataflow.futures import AppFuture
+from concurrent.futures import Future
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -22,7 +33,12 @@ class AppBase(metaclass=ABCMeta):
 
     """
 
-    def __init__(self, func, data_flow_kernel=None, executors='all', cache=False, ignore_for_cache=None):
+    @typeguard.typechecked
+    def __init__(self, func: Callable,
+                 data_flow_kernel: Optional[DataFlowKernel] = None,
+                 executors: Union[List[str], Literal['all']] = 'all',
+                 cache: bool = False,
+                 ignore_for_cache=None) -> None:
         """Construct the App object.
 
         Args:
@@ -45,13 +61,15 @@ def __init__(self, func, data_flow_kernel=None, executors='all', cache=False, ig
         self.executors = executors
         self.cache = cache
         self.ignore_for_cache = ignore_for_cache
-        if not (isinstance(executors, list) or isinstance(executors, str)):
-            logger.error("App {} specifies invalid executor option, expects string or list".format(
-                func.__name__))
+
+        # unreachable if properly typechecked
+        # if not (isinstance(executors, list) or isinstance(executors, str)):
+        #    logger.error("App {} specifies invalid executor option, expects string or list".format(
+        #        func.__name__))
 
         params = signature(func).parameters
 
-        self.kwargs = {}
+        self.kwargs = {}  # type: Dict[str, Any]
         if 'stdout' in params:
             self.kwargs['stdout'] = params['stdout'].default
         if 'stderr' in params:
@@ -64,7 +82,7 @@ def __init__(self, func, data_flow_kernel=None, executors='all', cache=False, ig
         self.inputs = params['inputs'].default if 'inputs' in params else []
 
     @abstractmethod
-    def __call__(self, *args, **kwargs):
+    def __call__(self, *args, **kwargs) -> AppFuture:
         pass
 
 
@@ -74,7 +92,8 @@ def python_app(function=None,
                cache: bool = False,
                executors: Union[List[str], Literal['all']] = 'all',
                ignore_for_cache: Optional[List[str]] = None,
-               join: bool = False):
+               join: bool = False,
+               do_not_use_F: Optional[Future] = None) -> Callable:
     """Decorator function for making python apps.
 
     Parameters
@@ -113,7 +132,7 @@ def join_app(function=None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
              executors: Union[List[str], Literal['all']] = 'all',
-             ignore_for_cache: Optional[List[str]] = None):
+             ignore_for_cache: Optional[List[str]] = None) -> Callable:
     """Decorator function for making join apps
 
     Parameters
@@ -150,7 +169,7 @@ def bash_app(function=None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
              executors: Union[List[str], Literal['all']] = 'all',
-             ignore_for_cache: Optional[List[str]] = None):
+             ignore_for_cache: Optional[List[str]] = None) -> Callable:
     """Decorator function for making bash apps.
 
     Parameters