From 9ec60a60aa5e7913baa5bf5661b819c2f6c3b237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 30 Jul 2023 16:29:31 -0600 Subject: [PATCH] Link python executable as sys.executable rather than python --- pympipool/legacy/shared/interface.py | 5 +++-- pympipool/shared/taskexecutor.py | 3 ++- tests/test_interface.py | 3 ++- tests/test_parse.py | 13 +++++++------ tests/test_parse_legacy.py | 10 ++++++---- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/pympipool/legacy/shared/interface.py b/pympipool/legacy/shared/interface.py index b8afb912..c9cd6b9a 100644 --- a/pympipool/legacy/shared/interface.py +++ b/pympipool/legacy/shared/interface.py @@ -1,5 +1,6 @@ import os import queue +import sys import time from pympipool.shared.communication import interface_bootup @@ -57,12 +58,12 @@ def get_pool_command(cores_total, ranks_per_task=1): os.path.join(__file__, "..", "..", "backend", "mpipool.py") ) if ranks_per_task == 1: - command_lst = ["python", "-m", "mpi4py.futures", executable] + command_lst = [sys.executable, "-m", "mpi4py.futures", executable] cores = cores_total else: # Running MPI parallel tasks within the map() requires mpi4py to use mpi spawn: # https://github.com/mpi4py/mpi4py/issues/324 - command_lst = ["python", executable] + command_lst = [sys.executable, executable] cores = 1 command_lst += [ "--cores-per-task", diff --git a/pympipool/shared/taskexecutor.py b/pympipool/shared/taskexecutor.py index c976bacd..78592a27 100644 --- a/pympipool/shared/taskexecutor.py +++ b/pympipool/shared/taskexecutor.py @@ -1,6 +1,7 @@ import inspect import os import queue +import sys import cloudpickle @@ -71,7 +72,7 @@ def execute_parallel_tasks( queue_adapter_kwargs (dict/None): keyword arguments for the submit_job() function of the queue adapter """ command_lst = [ - "python", + sys.executable, os.path.abspath(os.path.join(__file__, "..", "..", "backend", "mpiexec.py")), ] interface = interface_bootup( diff --git a/tests/test_interface.py b/tests/test_interface.py index abcc3d8d..00ef3d05 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -1,4 +1,5 @@ import os +import sys import numpy as np import unittest @@ -24,7 +25,7 @@ def test_interface(self): ) ) interface.bootup(command_lst=[ - "python", + sys.executable, os.path.abspath(os.path.join(__file__, "..", "..", "pympipool", "backend", "mpiexec.py")), "--zmqport", str(interface.bind_to_random_port()), diff --git a/tests/test_parse.py b/tests/test_parse.py index 52b8fda9..40cc8c3f 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,4 +1,5 @@ import os +import sys import unittest from pympipool.shared.backend import parse_arguments from pympipool.shared.connections import MpiExecInterface, FluxCmdInterface, SlurmSubprocessInterface @@ -14,13 +15,13 @@ def test_command_local(self): 'mpiexec', '-n', '2', '--oversubscribe', - 'python', '/', + sys.executable, '/', '--zmqport', result_dict['zmqport'] ] interface = MpiExecInterface(cwd=None, cores=2, gpus_per_core=0, oversubscribe=True) self.assertEqual( command_lst, - interface.generate_command(command_lst=['python', '/', '--zmqport', result_dict['zmqport']]) + interface.generate_command(command_lst=[sys.executable, '/', '--zmqport', result_dict['zmqport']]) ) self.assertEqual(result_dict, parse_arguments(command_lst)) @@ -33,14 +34,14 @@ def test_command_flux(self): 'flux', 'run', '-n', '2', "--cwd=" + os.path.abspath("."), '--gpus-per-task=1', - 'python', '/', + sys.executable, '/', '--host', result_dict['host'], '--zmqport', result_dict['zmqport'] ] interface = FluxCmdInterface(cwd=os.path.abspath("."), cores=2, gpus_per_core=1, oversubscribe=False) self.assertEqual( command_lst, - interface.generate_command(command_lst=['python', '/', '--host', result_dict['host'], '--zmqport', result_dict['zmqport']]) + interface.generate_command(command_lst=[sys.executable, '/', '--host', result_dict['host'], '--zmqport', result_dict['zmqport']]) ) self.assertEqual(result_dict, parse_arguments(command_lst)) @@ -59,13 +60,13 @@ def test_command_slurm(self): "-D", os.path.abspath("."), '--gpus-per-task=1', '--oversubscribe', - 'python', '/', + sys.executable, '/', '--host', result_dict['host'], '--zmqport', result_dict['zmqport'] ] interface = SlurmSubprocessInterface(cwd=os.path.abspath("."), cores=2, gpus_per_core=1, oversubscribe=True) self.assertEqual( command_lst, - interface.generate_command(command_lst=['python', '/', '--host', result_dict['host'], '--zmqport', result_dict['zmqport']]) + interface.generate_command(command_lst=[sys.executable, '/', '--host', result_dict['host'], '--zmqport', result_dict['zmqport']]) ) self.assertEqual(result_dict, parse_arguments(command_lst)) diff --git a/tests/test_parse_legacy.py b/tests/test_parse_legacy.py index cef871db..2e35e148 100644 --- a/tests/test_parse_legacy.py +++ b/tests/test_parse_legacy.py @@ -1,5 +1,7 @@ import unittest import os +import sys + from pympipool.legacy.shared.backend import parse_arguments from pympipool.shared.connections import MpiExecInterface, FluxCmdInterface @@ -16,7 +18,7 @@ def test_command_local(self): 'mpiexec', '-n', result_dict['total_cores'], '--oversubscribe', - 'python', '-m', 'mpi4py.futures', '/', + sys.executable, '-m', 'mpi4py.futures', '/', '--zmqport', result_dict['zmqport'], '--cores-per-task', result_dict['cores_per_task'], '--cores-total', result_dict['total_cores'] @@ -31,7 +33,7 @@ def test_command_local(self): command_lst, interface.generate_command( command_lst=[ - 'python', '-m', 'mpi4py.futures', '/', + sys.executable, '-m', 'mpi4py.futures', '/', '--zmqport', result_dict['zmqport'], '--cores-per-task', '1', '--cores-total', '2' ] @@ -49,7 +51,7 @@ def test_command_flux(self): command_lst = [ 'flux', 'run', '-n', '1', "--cwd=" + os.path.abspath("."), - 'python', '/', + sys.executable, '/', '--host', result_dict['host'], '--zmqport', result_dict['zmqport'], '--cores-per-task', result_dict['cores_per_task'], @@ -65,7 +67,7 @@ def test_command_flux(self): command_lst, interface.generate_command( command_lst=[ - 'python', '/', '--host', result_dict['host'], + sys.executable, '/', '--host', result_dict['host'], '--zmqport', result_dict['zmqport'], '--cores-per-task', '2', '--cores-total', '2' ]