From 29c5f58df574b17ba4e1607828c3852d57d9fe37 Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Tue, 10 Oct 2023 14:16:52 -0600 Subject: [PATCH 1/3] Allows non-integer minutes_per_sim --- buildstockbatch/eagle.py | 2 +- buildstockbatch/schemas/v0.3.yaml | 2 +- docs/changelog/changelog_dev.rst | 7 +++++++ docs/project_defn.rst | 4 ++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/buildstockbatch/eagle.py b/buildstockbatch/eagle.py index 7b682420..ec71278c 100644 --- a/buildstockbatch/eagle.py +++ b/buildstockbatch/eagle.py @@ -426,7 +426,6 @@ def run_building(cls, output_dir, cfg, n_datapoints, i, upgrade_idx=None): def queue_jobs(self, array_ids=None, hipri=False): eagle_cfg = self.cfg['eagle'] - minutes_per_sim = eagle_cfg.get('minutes_per_sim', 3) with open(os.path.join(self.output_dir, 'job001.json'), 'r') as f: job_json = json.load(f) n_sims_per_job = len(job_json['batch']) @@ -444,6 +443,7 @@ def queue_jobs(self, array_ids=None, hipri=False): # Estimate the wall time in minutes cores_per_node = 36 + minutes_per_sim = eagle_cfg['minutes_per_sim'] walltime = math.ceil(math.ceil(n_sims_per_job / cores_per_node) * minutes_per_sim) # Queue up simulations diff --git a/buildstockbatch/schemas/v0.3.yaml b/buildstockbatch/schemas/v0.3.yaml index b26d6d28..2ddf948c 100644 --- a/buildstockbatch/schemas/v0.3.yaml +++ b/buildstockbatch/schemas/v0.3.yaml @@ -39,7 +39,7 @@ aws-emr-spec: hpc-spec: account: str(required=True) - minutes_per_sim: int(max=480, required=False) + minutes_per_sim: num(min=0.05, max=480, required=True) n_jobs: int(required=False) postprocessing: include('hpc-postprocessing-spec', required=False) sampling: include('sampling-spec', required=False) diff --git a/docs/changelog/changelog_dev.rst b/docs/changelog/changelog_dev.rst index 16361cd7..74ef68ab 100644 --- a/docs/changelog/changelog_dev.rst +++ b/docs/changelog/changelog_dev.rst @@ -51,3 +51,10 @@ Development Changelog :pullreq: 365 Upload buildstock.csv to S3 during postprocessing + + .. change:: + :tags: feature + :pullreq: + + Allow fractional ``eagle.minutes_per_sim`` for simulations that run less + than a minute. Making that it a required input. diff --git a/docs/project_defn.rst b/docs/project_defn.rst index 71644571..35d46e71 100644 --- a/docs/project_defn.rst +++ b/docs/project_defn.rst @@ -146,8 +146,8 @@ Under the ``eagle`` key is a list of configuration for running the batch job on the Eagle supercomputer. * ``n_jobs``: Number of eagle jobs to parallelize the simulation into -* ``minutes_per_sim``: Maximum allocated simulation time in minutes -* ``account``: Eagle allocation account to charge the job to +* ``minutes_per_sim``: Required. Maximum allocated simulation time in minutes. +* ``account``: Required. Eagle allocation account to charge the job to. * ``sampling``: Configuration for the sampling in eagle * ``time``: Maximum time in minutes to allocate to sampling job From e1f03be299f748e61eb2f2a10dce1b58ed9d104f Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Tue, 10 Oct 2023 14:54:45 -0600 Subject: [PATCH 2/3] adding and fixing tests --- buildstockbatch/test/conftest.py | 1 + buildstockbatch/test/test_eagle.py | 27 ++++++++++++++++++++++++++- docs/changelog/changelog_dev.rst | 3 ++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/buildstockbatch/test/conftest.py b/buildstockbatch/test/conftest.py index 5a1240bd..c1e9e460 100644 --- a/buildstockbatch/test/conftest.py +++ b/buildstockbatch/test/conftest.py @@ -104,6 +104,7 @@ def _basic_residential_project_file(update_args={}, raw=False): 'time': 20 }, 'account': 'testaccount', + 'minutes_per_sim': 1 }, 'schema_version': '0.3' } diff --git a/buildstockbatch/test/test_eagle.py b/buildstockbatch/test/test_eagle.py index a138bf94..595a7829 100644 --- a/buildstockbatch/test/test_eagle.py +++ b/buildstockbatch/test/test_eagle.py @@ -3,7 +3,6 @@ import os import pandas as pd import pathlib -import requests import shutil import tarfile from unittest.mock import patch @@ -195,6 +194,32 @@ def test_qos_high_job_submit(mock_subprocess, basic_residential_project_file, mo assert '--qos=high' in mock_subprocess.run.call_args[0][0] +def test_queue_jobs_minutes_per_sim(mocker, basic_residential_project_file): + mock_subprocess = mocker.patch('buildstockbatch.eagle.subprocess') + mocker.patch.object(EagleBatch, 'weather_dir', None) + mock_subprocess.run.return_value.stdout = 'Submitted batch job 1\n' + mock_subprocess.PIPE = None + project_filename, results_dir = basic_residential_project_file(update_args={ + 'eagle': { + 'sampling': { + 'time': 20 + }, + 'account': 'testaccount', + 'minutes_per_sim': 0.5 + } + }) + shutil.rmtree(results_dir) + + batch = EagleBatch(project_filename) + for i in range(1, 11): + pathlib.Path(results_dir, 'job{:03d}.json'.format(i)).touch() + with open(os.path.join(results_dir, 'job001.json'), 'w') as f: + json.dump({'batch': list(range(1000))}, f) + batch.queue_jobs() + mock_subprocess.run.assert_called_once() + assert '--time=14' in mock_subprocess.run.call_args[0][0] + + def test_run_building_process(mocker, basic_residential_project_file): project_filename, results_dir = basic_residential_project_file(raw=True) results_dir = pathlib.Path(results_dir) diff --git a/docs/changelog/changelog_dev.rst b/docs/changelog/changelog_dev.rst index 74ef68ab..3028d4d0 100644 --- a/docs/changelog/changelog_dev.rst +++ b/docs/changelog/changelog_dev.rst @@ -54,7 +54,8 @@ Development Changelog .. change:: :tags: feature - :pullreq: + :pullreq: 396 + :tickets: 377 Allow fractional ``eagle.minutes_per_sim`` for simulations that run less than a minute. Making that it a required input. From 7cee1754519961563008b85d7585536dde450556 Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Tue, 10 Oct 2023 15:17:43 -0600 Subject: [PATCH 3/3] adding fake conda prefix for testing --- buildstockbatch/test/test_eagle.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildstockbatch/test/test_eagle.py b/buildstockbatch/test/test_eagle.py index 595a7829..91a16da3 100644 --- a/buildstockbatch/test/test_eagle.py +++ b/buildstockbatch/test/test_eagle.py @@ -194,7 +194,7 @@ def test_qos_high_job_submit(mock_subprocess, basic_residential_project_file, mo assert '--qos=high' in mock_subprocess.run.call_args[0][0] -def test_queue_jobs_minutes_per_sim(mocker, basic_residential_project_file): +def test_queue_jobs_minutes_per_sim(mocker, basic_residential_project_file, monkeypatch): mock_subprocess = mocker.patch('buildstockbatch.eagle.subprocess') mocker.patch.object(EagleBatch, 'weather_dir', None) mock_subprocess.run.return_value.stdout = 'Submitted batch job 1\n' @@ -209,6 +209,7 @@ def test_queue_jobs_minutes_per_sim(mocker, basic_residential_project_file): } }) shutil.rmtree(results_dir) + monkeypatch.setenv('CONDA_PREFIX', 'something') batch = EagleBatch(project_filename) for i in range(1, 11):