Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[bugfix] Fix remote topology detection for pip installations #2978

Merged
merged 3 commits into from Sep 11, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
47 changes: 32 additions & 15 deletions reframe/frontend/autodetect.py
Expand Up @@ -48,15 +48,19 @@
tempfile.mkdtemp(prefix='rfm.', dir=self._prefix)
)
paths = ['bin/', 'reframe/', 'bootstrap.sh', 'requirements.txt']
for p in paths:
src = os.path.join(rfm.INSTALL_PREFIX, p)
if os.path.isdir(src):
dst = os.path.join(self._workdir, p)
osext.copytree(src, dst, dirs_exist_ok=True)
else:
shutil.copy2(src, self._workdir)

return self._workdir
use_pip = False
try:
for p in paths:
src = os.path.join(rfm.INSTALL_PREFIX, p)
if os.path.isdir(src):
dst = os.path.join(self._workdir, p)
osext.copytree(src, dst, dirs_exist_ok=True)

Check warning on line 57 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L51-L57

Added lines #L51 - L57 were not covered by tests
else:
shutil.copy2(src, self._workdir)
except FileNotFoundError:
use_pip = True

Check warning on line 61 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L59-L61

Added lines #L59 - L61 were not covered by tests

return self._workdir, use_pip

Check warning on line 63 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L63

Added line #L63 was not covered by tests

def __exit__(self, exc_type, exc_val, exc_tb):
osext.rmtree(self._workdir)
Expand Down Expand Up @@ -124,11 +128,20 @@


def _remote_detect(part):
def _emit_script(job, env):
launcher_cmd = job.launcher.run_command(job)
def _emit_script_for_source(job, env):
commands = [

Check warning on line 132 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L131-L132

Added lines #L131 - L132 were not covered by tests
'./bootstrap.sh',
'./bin/reframe --detect-host-topology=topo.json'
]
job.prepare(commands, env, trap_errors=True)

Check warning on line 136 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L136

Added line #L136 was not covered by tests

def _emit_script_for_pip(job, env):

Check warning on line 138 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L138

Added line #L138 was not covered by tests
commands = [
f'./bootstrap.sh',
f'{launcher_cmd} ./bin/reframe --detect-host-topology=topo.json'
'python3 -m venv venv.reframe',
'source venv.reframe/bin/activate',
f'pip install reframe-hpc=={rfm.VERSION}',
'reframe --detect-host-topology=topo.json',
'deactivate'
]
job.prepare(commands, env, trap_errors=True)

Expand All @@ -139,13 +152,17 @@
topo_info = {}
try:
prefix = runtime.runtime().get_option('general/0/remote_workdir')
with _copy_reframe(prefix) as dirname:
with _copy_reframe(prefix) as (dirname, use_pip):

Check warning on line 155 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L155

Added line #L155 was not covered by tests
with osext.change_dir(dirname):
job = Job.create(part.scheduler,
part.launcher_type(),
name='rfm-detect-job',
sched_access=part.access)
_emit_script(job, [part.local_env])
if use_pip:
_emit_script_for_pip(job, [part.local_env])

Check warning on line 162 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L161-L162

Added lines #L161 - L162 were not covered by tests
else:
_emit_script_for_source(job, [part.local_env])

Check warning on line 164 in reframe/frontend/autodetect.py

View check run for this annotation

Codecov / codecov/patch

reframe/frontend/autodetect.py#L164

Added line #L164 was not covered by tests

getlogger().debug('submitting detection script')
_log_contents(job.script_filename)
job.submit()
Expand Down