From 76291be87d3a73f5889464e4bef185f77d5148f5 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 1 Oct 2024 09:15:13 +0200 Subject: [PATCH] Remove conda support in executorlib Users who want conda support have to first decorate the function and then submit it with executorlib. --- .ci_support/environment-mpich.yml | 1 - .ci_support/environment-old.yml | 1 - .ci_support/environment-openmpi.yml | 1 - .ci_support/environment-win.yml | 1 - .github/workflows/unittest-flux-mpich.yml | 1 - .github/workflows/unittest-flux-openmpi.yml | 1 - .github/workflows/unittest-mpich.yml | 1 - .github/workflows/unittest-openmpi.yml | 1 - .github/workflows/unittest-win.yml | 1 - .github/workflows/unittests-old.yml | 1 - .readthedocs.yml | 2 +- executorlib/__init__.py | 12 ---- executorlib/interactive/__init__.py | 6 -- executorlib/interactive/flux.py | 8 --- executorlib/shared/communication.py | 13 ---- executorlib/shared/executor.py | 6 -- executorlib/shared/spawner.py | 29 ++------ executorlib/shell/executor.py | 35 ++-------- notebooks/examples.ipynb | 4 -- pyproject.toml | 1 - tests/test_executor_conda.py | 76 --------------------- tests/test_flux_executor.py | 10 --- 22 files changed, 11 insertions(+), 201 deletions(-) delete mode 100644 tests/test_executor_conda.py diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 8f67c272..cd962b4e 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -5,7 +5,6 @@ dependencies: - numpy - mpich - cloudpickle =3.0.0 -- conda_subprocess =0.0.4 - mpi4py =4.0.0 - pyzmq =26.2.0 - h5py =3.11.0 diff --git a/.ci_support/environment-old.yml b/.ci_support/environment-old.yml index f2bb44f2..0cbe69c5 100644 --- a/.ci_support/environment-old.yml +++ b/.ci_support/environment-old.yml @@ -5,7 +5,6 @@ dependencies: - numpy - openmpi =4.1.4 - cloudpickle =2.0.0 -- conda_subprocess =0.0.3 - mpi4py =3.1.4 - pyzmq =25.0.0 - h5py =3.6.0 diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index f712a8e4..3c74c613 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -5,7 +5,6 @@ dependencies: - numpy - openmpi - cloudpickle =3.0.0 -- conda_subprocess =0.0.4 - mpi4py =4.0.0 - pyzmq =26.2.0 - h5py =3.11.0 diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index 8c4300f0..fc3cacd9 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -5,7 +5,6 @@ dependencies: - numpy - msmpi - cloudpickle =3.0.0 -- conda_subprocess =0.0.4 - mpi4py =4.0.0 - pyzmq =26.2.0 - h5py =3.11.0 diff --git a/.github/workflows/unittest-flux-mpich.yml b/.github/workflows/unittest-flux-mpich.yml index 49cc1cb7..1ed22ca4 100644 --- a/.github/workflows/unittest-flux-mpich.yml +++ b/.github/workflows/unittest-flux-mpich.yml @@ -27,7 +27,6 @@ jobs: shell: bash -l {0} timeout-minutes: 5 run: | - conda create -y -n py312 python=3.12.1 executorlib pip install . --no-deps --no-build-isolation python -m unittest discover tests - name: Test Flux diff --git a/.github/workflows/unittest-flux-openmpi.yml b/.github/workflows/unittest-flux-openmpi.yml index 90315407..5cf3315f 100644 --- a/.github/workflows/unittest-flux-openmpi.yml +++ b/.github/workflows/unittest-flux-openmpi.yml @@ -27,7 +27,6 @@ jobs: shell: bash -l {0} timeout-minutes: 5 run: | - conda create -y -n py312 python=3.12.1 executorlib pip install . --no-deps --no-build-isolation coverage run -a --omit="executorlib/_version.py,tests/*" -m unittest discover tests - name: Test Flux with OpenMPI diff --git a/.github/workflows/unittest-mpich.yml b/.github/workflows/unittest-mpich.yml index 9c8f5c6b..5c12f105 100644 --- a/.github/workflows/unittest-mpich.yml +++ b/.github/workflows/unittest-mpich.yml @@ -42,7 +42,6 @@ jobs: timeout-minutes: 5 run: | pip install versioneer[toml]==0.29 - conda create -y -n py312 python=3.12.1 executorlib pip install . --no-deps --no-build-isolation cd tests python -m unittest discover . diff --git a/.github/workflows/unittest-openmpi.yml b/.github/workflows/unittest-openmpi.yml index 1e874d45..69f09961 100644 --- a/.github/workflows/unittest-openmpi.yml +++ b/.github/workflows/unittest-openmpi.yml @@ -42,7 +42,6 @@ jobs: timeout-minutes: 5 run: | pip install versioneer[toml]==0.29 - conda create -y -n py312 python=3.12.1 executorlib pip install . --no-deps --no-build-isolation cd tests python -m unittest discover . diff --git a/.github/workflows/unittest-win.yml b/.github/workflows/unittest-win.yml index 23706812..91186e43 100644 --- a/.github/workflows/unittest-win.yml +++ b/.github/workflows/unittest-win.yml @@ -32,7 +32,6 @@ jobs: timeout-minutes: 5 run: | pip install versioneer[toml]==0.29 - conda create -y -n py312 python=3.12.1 executorlib pip install . --no-deps --no-build-isolation cd tests python -m unittest discover . diff --git a/.github/workflows/unittests-old.yml b/.github/workflows/unittests-old.yml index 8bccd5e7..ce5c227b 100644 --- a/.github/workflows/unittests-old.yml +++ b/.github/workflows/unittests-old.yml @@ -26,7 +26,6 @@ jobs: timeout-minutes: 5 run: | pip install versioneer[toml]==0.29 - conda create -y -n py312 python=3.12.1 executorlib pip install . --no-deps --no-build-isolation cd tests python -m unittest discover . diff --git a/.readthedocs.yml b/.readthedocs.yml index bee0cd7a..477fd3ea 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -25,6 +25,6 @@ sphinx: # Optionally build your docs in additional formats such as PDF and ePub formats: [] -# Install pyiron from conda +# Install executorlib from conda conda: environment: .ci_support/environment-docs.yml diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 1e7a1aef..f8e9a588 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -51,8 +51,6 @@ class Executor: flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. - conda_environment_name (str): name of the conda environment to initialize - conda_environment_path (str): path of the conda environment to initialize hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the context of an HPC cluster this essential to be able to communicate to an Executor running on a different compute node within the same allocation. And @@ -105,8 +103,6 @@ def __init__( flux_executor=None, flux_executor_pmi_mode: Optional[str] = None, flux_executor_nesting: bool = False, - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, hostname_localhost: bool = False, block_allocation: bool = True, init_function: Optional[callable] = None, @@ -131,8 +127,6 @@ def __new__( flux_executor=None, flux_executor_pmi_mode: Optional[str] = None, flux_executor_nesting: bool = False, - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, hostname_localhost: bool = False, block_allocation: bool = True, init_function: Optional[callable] = None, @@ -163,8 +157,6 @@ def __new__( flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. - conda_environment_name (str): name of the conda environment to initialize - conda_environment_path (str): path of the conda environment to initialize hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the context of an HPC cluster this essential to be able to communicate to an Executor running on a different compute node within the same allocation. And @@ -197,8 +189,6 @@ def __new__( flux_executor=flux_executor, flux_executor_pmi_mode=flux_executor_pmi_mode, flux_executor_nesting=flux_executor_nesting, - conda_environment_name=conda_environment_name, - conda_environment_path=conda_environment_path, hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, @@ -221,8 +211,6 @@ def __new__( flux_executor=flux_executor, flux_executor_pmi_mode=flux_executor_pmi_mode, flux_executor_nesting=flux_executor_nesting, - conda_environment_name=conda_environment_name, - conda_environment_path=conda_environment_path, hostname_localhost=hostname_localhost, block_allocation=block_allocation, init_function=init_function, diff --git a/executorlib/interactive/__init__.py b/executorlib/interactive/__init__.py index d1203f69..21b1f9fe 100644 --- a/executorlib/interactive/__init__.py +++ b/executorlib/interactive/__init__.py @@ -40,8 +40,6 @@ def create_executor( flux_executor=None, flux_executor_pmi_mode: Optional[str] = None, flux_executor_nesting: bool = False, - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, hostname_localhost: bool = False, block_allocation: bool = False, init_function: Optional[callable] = None, @@ -69,8 +67,6 @@ def create_executor( flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux flux_executor_pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None (Flux only) flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function. - conda_environment_name (str): name of the conda environment to initialize - conda_environment_path (str): path of the conda environment to initialize hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the context of an HPC cluster this essential to be able to communicate to an Executor running on a different compute node within the same allocation. And in principle @@ -92,8 +88,6 @@ def create_executor( "cores": cores_per_worker, "hostname_localhost": hostname_localhost, "cwd": cwd, - "conda_environment_name": conda_environment_name, - "conda_environment_path": conda_environment_path, } if backend == "flux": check_oversubscribe(oversubscribe=openmpi_oversubscribe) diff --git a/executorlib/interactive/flux.py b/executorlib/interactive/flux.py index cfd79c42..bbcbab17 100644 --- a/executorlib/interactive/flux.py +++ b/executorlib/interactive/flux.py @@ -47,16 +47,12 @@ def __init__( def bootup( self, command_lst: list[str], - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, ): """ Boot up the client process to connect to the SocketInterface. Args: command_lst (list[str]): List of strings to start the client process. - conda_environment_name (str, optional): Name of the conda environment to initialize. Defaults to None. - conda_environment_path (str, optional): Path of the conda environment to initialize. Defaults to None. Raises: ValueError: If oversubscribing is not supported for the Flux adapter or if conda environments are not supported. """ @@ -64,10 +60,6 @@ def bootup( raise ValueError( "Oversubscribing is currently not supported for the Flux adapter." ) - if conda_environment_name is not None or conda_environment_path is not None: - raise ValueError( - "Conda environments are currently not supported for the Flux adapter." - ) if self._flux_executor is None: self._flux_executor = flux.job.FluxExecutor() if not self._flux_executor_nesting: diff --git a/executorlib/shared/communication.py b/executorlib/shared/communication.py index 44d5f22c..9245f1a3 100644 --- a/executorlib/shared/communication.py +++ b/executorlib/shared/communication.py @@ -1,5 +1,4 @@ from socket import gethostname -from typing import Optional import cloudpickle import zmq @@ -76,21 +75,15 @@ def bind_to_random_port(self): def bootup( self, command_lst: list[str], - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, ): """ Boot up the client process to connect to the SocketInterface. Args: command_lst (list): list of strings to start the client process - conda_environment_name (str): name of the conda environment to initialize - conda_environment_path (str): path of the conda environment to initialize """ self._spawner.bootup( command_lst=command_lst, - conda_environment_name=conda_environment_name, - conda_environment_path=conda_environment_path, ) def shutdown(self, wait: bool = True): @@ -127,8 +120,6 @@ def interface_bootup( command_lst: list[str], connections, hostname_localhost: bool = False, - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, ): """ Start interface for ZMQ communication @@ -144,8 +135,6 @@ def interface_bootup( points to the same address as localhost. Still MacOS >= 12 seems to disable this look up for security reasons. So on MacOS it is required to set this option to true - conda_environment_name (str): name of the conda environment to initialize - conda_environment_path (str): path of the conda environment to initialize Returns: executorlib.shared.communication.SocketInterface: socket interface for zmq communication @@ -162,8 +151,6 @@ def interface_bootup( ] interface.bootup( command_lst=command_lst, - conda_environment_name=conda_environment_name, - conda_environment_path=conda_environment_path, ) return interface diff --git a/executorlib/shared/executor.py b/executorlib/shared/executor.py index aeb85137..c579d9f8 100644 --- a/executorlib/shared/executor.py +++ b/executorlib/shared/executor.py @@ -304,8 +304,6 @@ def execute_parallel_tasks( spawner: BaseSpawner = MpiExecSpawner, hostname_localhost: bool = False, init_function: Optional[Callable] = None, - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, **kwargs, ) -> None: """ @@ -323,8 +321,6 @@ def execute_parallel_tasks( this look up for security reasons. So on MacOS it is required to set this option to true init_function (callable): optional function to preset arguments for functions which are submitted later - conda_environment_name (str): name of the conda environment to initialize - conda_environment_path (str): path of the conda environment to initialize """ interface = interface_bootup( command_lst=_get_backend_path( @@ -332,8 +328,6 @@ def execute_parallel_tasks( ), connections=spawner(cores=cores, **kwargs), hostname_localhost=hostname_localhost, - conda_environment_path=conda_environment_path, - conda_environment_name=conda_environment_name, ) if init_function is not None: interface.send_dict( diff --git a/executorlib/shared/spawner.py b/executorlib/shared/spawner.py index 45f13ec5..d9456724 100644 --- a/executorlib/shared/spawner.py +++ b/executorlib/shared/spawner.py @@ -23,16 +23,12 @@ def __init__(self, cwd: str, cores: int = 1, openmpi_oversubscribe: bool = False def bootup( self, command_lst: list[str], - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, ): """ Method to start the interface. Args: command_lst (list[str]): The command list to execute. - conda_environment_name (str, optional): The prefix name. Defaults to None. - conda_environment_path (str, optional): The prefix path. Defaults to None. """ raise NotImplementedError @@ -80,33 +76,18 @@ def __init__( def bootup( self, command_lst: list[str], - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, ): """ Method to start the subprocess interface. Args: command_lst (list[str]): The command list to execute. - conda_environment_name (str, optional): The prefix name. Defaults to None. - conda_environment_path (str, optional): The prefix path. Defaults to None. """ - if conda_environment_name is None and conda_environment_path is None: - self._process = subprocess.Popen( - args=self.generate_command(command_lst=command_lst), - cwd=self._cwd, - stdin=subprocess.DEVNULL, - ) - else: - import conda_subprocess - - self._process = conda_subprocess.Popen( - args=self.generate_command(command_lst=command_lst), - cwd=self._cwd, - stdin=subprocess.DEVNULL, - prefix_path=conda_environment_path, - prefix_name=conda_environment_name, - ) + self._process = subprocess.Popen( + args=self.generate_command(command_lst=command_lst), + cwd=self._cwd, + stdin=subprocess.DEVNULL, + ) def generate_command(self, command_lst: list[str]) -> list[str]: """ diff --git a/executorlib/shell/executor.py b/executorlib/shell/executor.py index 5d81518a..480866c3 100644 --- a/executorlib/shell/executor.py +++ b/executorlib/shell/executor.py @@ -1,7 +1,7 @@ import queue import subprocess from concurrent.futures import Future -from typing import Any, Optional +from typing import Any from executorlib.shared.executor import ExecutorBroker from executorlib.shared.thread import RaisingThread @@ -9,16 +9,12 @@ def execute_single_task( future_queue: queue.Queue, - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, ) -> None: """ Process items received via the queue. Args: future_queue (queue.Queue): The queue containing the tasks to be executed. - conda_environment_name (Optional[str]): The name of the conda environment to initialize. - conda_environment_path (Optional[str]): The path of the conda environment to initialize. """ while True: task_dict = future_queue.get() @@ -30,26 +26,11 @@ def execute_single_task( f = task_dict.pop("future") if f.set_running_or_notify_cancel(): try: - if ( - conda_environment_name is None - and conda_environment_path is None - ): - f.set_result( - subprocess.check_output( - *task_dict["args"], **task_dict["kwargs"] - ) - ) - else: - import conda_subprocess - - f.set_result( - conda_subprocess.check_output( - *task_dict["args"], - **task_dict["kwargs"], - prefix_name=conda_environment_name, - prefix_path=conda_environment_path, - ) + f.set_result( + subprocess.check_output( + *task_dict["args"], **task_dict["kwargs"] ) + ) except Exception as thread_exception: future_queue.task_done() f.set_exception(exception=thread_exception) @@ -69,8 +50,6 @@ class SubprocessExecutor(ExecutorBroker): Args: max_workers (int): defines the number workers which can execute functions in parallel - conda_environment_name (str): name of the conda environment to initialize - conda_environment_path (str): path of the conda environment to initialize Examples: @@ -85,8 +64,6 @@ class SubprocessExecutor(ExecutorBroker): def __init__( self, max_workers: int = 1, - conda_environment_name: Optional[str] = None, - conda_environment_path: Optional[str] = None, ): super().__init__() self._set_process( @@ -96,8 +73,6 @@ def __init__( kwargs={ # Executor Arguments "future_queue": self._future_queue, - "conda_environment_name": conda_environment_name, - "conda_environment_path": conda_environment_path, }, ) for _ in range(max_workers) diff --git a/notebooks/examples.ipynb b/notebooks/examples.ipynb index 0df1721a..166139f1 100644 --- a/notebooks/examples.ipynb +++ b/notebooks/examples.ipynb @@ -221,8 +221,6 @@ " flux_executor=flux_exe,\n", " flux_executor_pmi_mode=None,\n", " flux_executor_nesting=False,\n", - " conda_environment_name=None, # execute submitted function in separate conda environment\n", - " conda_environment_path=None,\n", " hostname_localhost=False, # only required on MacOS\n", " block_allocation=False, # reuse existing processes with fixed resources\n", " init_function=None, # only available with block_allocation=True\n", @@ -245,8 +243,6 @@ " \"flux_executor\": flux_exe,\n", " \"flux_executor_pmi_mode\": None,\n", " \"flux_executor_nesting\": False,\n", - " \"conda_environment_name\": None,\n", - " \"conda_environment_path\": None,\n", " \"hostname_localhost\": False, # only required on MacOS\n", " },\n", " )\n", diff --git a/pyproject.toml b/pyproject.toml index 8fd66bfc..622d2d5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,6 @@ Documentation = "https://executorlib.readthedocs.io" Repository = "https://github.com/pyiron/executorlib" [project.optional-dependencies] -conda = ["conda_subprocess==0.0.4"] mpi = ["mpi4py==4.0.0"] hdf = [ "h5py==3.11.0", diff --git a/tests/test_executor_conda.py b/tests/test_executor_conda.py deleted file mode 100644 index 64e104b6..00000000 --- a/tests/test_executor_conda.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -import unittest - -from executorlib import Executor -from executorlib.shared.executor import cloudpickle_register -from executorlib.shell.executor import SubprocessExecutor - -try: - from conda.base.context import context - - skip_conda_test = False -except ImportError: - skip_conda_test = True - - -def get_conda_env_prefix(): - return os.environ["CONDA_PREFIX"] - - -@unittest.skipIf( - skip_conda_test, "conda is not installed, so the conda tests are skipped." -) -class CondaExecutorTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.env_name = "py312" - if "envs" in context.root_prefix: - cls.env_path = os.path.abspath( - os.path.join(context.root_prefix, "..", cls.env_name) - ) - else: - cls.env_path = os.path.abspath( - os.path.join(context.root_prefix, "envs", cls.env_name) - ) - - def test_shell_executor_conda(self): - with SubprocessExecutor( - max_workers=1, conda_environment_path=self.env_path - ) as exe: - future = exe.submit(["python", "--version"], universal_newlines=True) - self.assertFalse(future.done()) - self.assertEqual("Python 3.12.1\n", future.result()) - self.assertTrue(future.done()) - - def test_shell_executor_conda_name(self): - with SubprocessExecutor( - max_workers=1, conda_environment_name=self.env_name - ) as exe: - future = exe.submit(["python", "--version"], universal_newlines=True) - self.assertFalse(future.done()) - self.assertEqual("Python 3.12.1\n", future.result()) - self.assertTrue(future.done()) - - def test_python_executor_conda_path(self): - with Executor( - max_cores=1, - hostname_localhost=True, - backend="local", - conda_environment_path=self.env_path, - ) as exe: - cloudpickle_register(ind=1) - fs = exe.submit(get_conda_env_prefix) - self.assertEqual(fs.result(), self.env_path) - self.assertTrue(fs.done()) - - def test_python_executor_conda_name(self): - with Executor( - max_cores=1, - hostname_localhost=True, - backend="local", - conda_environment_name=self.env_name, - ) as exe: - cloudpickle_register(ind=1) - fs = exe.submit(get_conda_env_prefix) - self.assertEqual(fs.result(), self.env_path) - self.assertTrue(fs.done()) diff --git a/tests/test_flux_executor.py b/tests/test_flux_executor.py index 5753b1c2..56dbc424 100644 --- a/tests/test_flux_executor.py +++ b/tests/test_flux_executor.py @@ -157,13 +157,3 @@ def test_interface_exception(self): flux_executor=self.flux_executor, openmpi_oversubscribe=True ) flux_interface.bootup(command_lst=[]) - with self.assertRaises(ValueError): - flux_interface = FluxPythonSpawner(flux_executor=self.flux_executor) - flux_interface.bootup( - command_lst=[], conda_environment_path="/path/to/conda/env" - ) - with self.assertRaises(ValueError): - flux_interface = FluxPythonSpawner(flux_executor=self.flux_executor) - flux_interface.bootup( - command_lst=[], conda_environment_name="conda_env_name" - )