From b9533056bd884b37e36d24903d65e645696523de Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 31 Aug 2023 22:30:35 +0200 Subject: [PATCH 1/3] Remove parallel launcher from remote detection --- reframe/frontend/autodetect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index 0ad927a765..e1e523f4c7 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -125,10 +125,9 @@ def _is_part_local(part): def _remote_detect(part): def _emit_script(job, env): - launcher_cmd = job.launcher.run_command(job) commands = [ f'./bootstrap.sh', - f'{launcher_cmd} ./bin/reframe --detect-host-topology=topo.json' + f'./bin/reframe --detect-host-topology=topo.json' ] job.prepare(commands, env, trap_errors=True) From 84f95c7c48d10e89f0a135ead777b45518d17a41 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 31 Aug 2023 22:45:05 +0200 Subject: [PATCH 2/3] Fix remote detection for pip installations --- reframe/frontend/autodetect.py | 46 +++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index e1e523f4c7..1adf4c401f 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -48,15 +48,19 @@ def __enter__(self): tempfile.mkdtemp(prefix='rfm.', dir=self._prefix) ) paths = ['bin/', 'reframe/', 'bootstrap.sh', 'requirements.txt'] - for p in paths: - src = os.path.join(rfm.INSTALL_PREFIX, p) - if os.path.isdir(src): - dst = os.path.join(self._workdir, p) - osext.copytree(src, dst, dirs_exist_ok=True) - else: - shutil.copy2(src, self._workdir) - - return self._workdir + use_pip = False + try: + for p in paths: + src = os.path.join(rfm.INSTALL_PREFIX, p) + if os.path.isdir(src): + dst = os.path.join(self._workdir, p) + osext.copytree(src, dst, dirs_exist_ok=True) + else: + shutil.copy2(src, self._workdir) + except OSError: + use_pip = True + + return self._workdir, use_pip def __exit__(self, exc_type, exc_val, exc_tb): osext.rmtree(self._workdir) @@ -124,10 +128,20 @@ def _is_part_local(part): def _remote_detect(part): - def _emit_script(job, env): + def _emit_script_for_source(job, env): + commands = [ + './bootstrap.sh', + './bin/reframe --detect-host-topology=topo.json' + ] + job.prepare(commands, env, trap_errors=True) + + def _emit_script_for_pip(job, env): commands = [ - f'./bootstrap.sh', - f'./bin/reframe --detect-host-topology=topo.json' + 'python3 -m venv venv.reframe', + 'source venv.reframe/bin/activate', + 'pip install reframe-hpc', + 'reframe --detect-host-topology=topo.json', + 'deactivate' ] job.prepare(commands, env, trap_errors=True) @@ -138,13 +152,17 @@ def _emit_script(job, env): topo_info = {} try: prefix = runtime.runtime().get_option('general/0/remote_workdir') - with _copy_reframe(prefix) as dirname: + with _copy_reframe(prefix) as (dirname, use_pip): with osext.change_dir(dirname): job = Job.create(part.scheduler, part.launcher_type(), name='rfm-detect-job', sched_access=part.access) - _emit_script(job, [part.local_env]) + if use_pip: + _emit_script_for_pip(job, [part.local_env]) + else: + _emit_script_for_source(job, [part.local_env]) + getlogger().debug('submitting detection script') _log_contents(job.script_filename) job.submit() From c8983c5f8ce76c46716781bc9170e91343b8d393 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 2 Sep 2023 00:24:39 +0200 Subject: [PATCH 3/3] Fine tune implementation --- reframe/frontend/autodetect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/autodetect.py b/reframe/frontend/autodetect.py index 1adf4c401f..d4c75674f9 100644 --- a/reframe/frontend/autodetect.py +++ b/reframe/frontend/autodetect.py @@ -57,7 +57,7 @@ def __enter__(self): osext.copytree(src, dst, dirs_exist_ok=True) else: shutil.copy2(src, self._workdir) - except OSError: + except FileNotFoundError: use_pip = True return self._workdir, use_pip @@ -139,7 +139,7 @@ def _emit_script_for_pip(job, env): commands = [ 'python3 -m venv venv.reframe', 'source venv.reframe/bin/activate', - 'pip install reframe-hpc', + f'pip install reframe-hpc=={rfm.VERSION}', 'reframe --detect-host-topology=topo.json', 'deactivate' ]