Merge pull request #406 from NREL/eagle_tmp_scratch_cleanup

Removing /tmp/scratch files at end of array job

buildstockbatch/eagle.py

@@ -61,7 +61,6 @@ class EagleBatch(BuildStockBatchBase):
     min_sims_per_job = 36 * 2
 
     local_scratch = pathlib.Path(os.environ.get("LOCAL_SCRATCH", "/tmp/scratch"))
-    local_project_dir = local_scratch / "project"
     local_buildstock_dir = local_scratch / "buildstock"
     local_weather_dir = local_scratch / "weather"
     local_output_dir = local_scratch / "output"
@@ -312,6 +311,24 @@ def run_building_d(i, upgrade_idx):
 
         logger.info("batch complete")
 
+        # Remove local scratch files
+        dirs_to_remove = [
+            self.local_buildstock_dir,
+            self.local_weather_dir,
+            self.local_output_dir,
+            self.local_housing_characteristics_dir,
+        ]
+
+        logger.info(f"Cleaning up {self.local_scratch}")
+        for dir in dirs_to_remove:
+            logger.debug(f"Removing {dir}")
+            if dir.exists():
+                shutil.rmtree(dir)
+            else:
+                logger.warning(f"Directory does not exist {dir}")
+        logger.debug(f"Removing {self.local_singularity_img}")
+        self.local_singularity_img.unlink(missing_ok=True)
+
     @classmethod
     def run_building(cls, output_dir, cfg, n_datapoints, i, upgrade_idx=None):
         fs = LocalFileSystem()

buildstockbatch/test/test_eagle.py

@@ -256,6 +256,7 @@ def sequential_parallel(**kwargs):
         return joblib.Parallel(**kw2)
 
     mocker.patch("buildstockbatch.eagle.shutil.copy2")
+    rmtree_mock = mocker.patch("buildstockbatch.eagle.shutil.rmtree")
     mocker.patch("buildstockbatch.eagle.Parallel", sequential_parallel)
     mocker.patch("buildstockbatch.eagle.subprocess")
 
@@ -287,6 +288,10 @@ def make_sim_dir_mock(building_id, upgrade_idx, base_dir, overwrite_existing=Fal
     b.run_batch(sampling_only=True)  # so the directories can be created
     sampler_mock.run_sampling.assert_called_once()
     b.run_job_batch(1)
+    rmtree_mock.assert_any_call(b.local_buildstock_dir)
+    rmtree_mock.assert_any_call(b.local_weather_dir)
+    rmtree_mock.assert_any_call(b.local_output_dir)
+    rmtree_mock.assert_any_call(b.local_housing_characteristics_dir)
 
     # check results job-json
     refrence_path = pathlib.Path(__file__).resolve().parent / "test_results" / "reference_files"
@@ -341,6 +346,7 @@ def sequential_parallel(**kwargs):
         return joblib.Parallel(**kw2)
 
     mocker.patch("buildstockbatch.eagle.shutil.copy2")
+    mocker.patch("buildstockbatch.eagle.shutil.rmtree")
     mocker.patch("buildstockbatch.eagle.Parallel", sequential_parallel)
     mocker.patch("buildstockbatch.eagle.subprocess")
 

create_eagle_env.sh

@@ -32,7 +32,7 @@ conda activate "$MY_CONDA_PREFIX"
 which pip
 if [ $DEV -eq 1 ]
 then
-    pip install --no-cache-dir -e .
+    pip install --no-cache-dir -e ".[dev]"
 else
     pip install --no-cache-dir .
 fi

docs/changelog/changelog_dev.rst

@@ -14,3 +14,10 @@ Development Changelog
         This is an example change. Please copy and paste it - for valid tags please refer to ``conf.py`` in the docs
         directory. ``pullreq`` should be set to the appropriate pull request number and ``tickets`` to any related
         github issues. These will be automatically linked in the documentation.
+
+    .. change::
+        :tags: eagle, bugfix
+        :pullreq: 406
+        :tickets: 404
+
+        Cleans out the ``/tmp/scratch`` folder on Eagle at the end of each array job.