Merge pull request #35 from PhangsTeam/comp_diag

Compress diagnostic plots

CHANGES.rst

@@ -1,3 +1,8 @@
+0.10.2 (Unreleased)
+===================
+
+- Diagnostic plots are compressed by default in ``release_step``
+
 0.10.1 (2023-11-20)
 ===================
 

pjpipe/release/release_step.py

@@ -24,6 +24,7 @@ def __init__(
         move_individual_fields=False,
         move_psf_matched=False,
         move_diagnostic_plots=False,
+        compress_diagnostic_plots=True,
         lv3_dir="lv3",
         tweakback_dir="lv3",
         tweakback_ext="tweakback",
@@ -59,6 +60,8 @@ def __init__(
                 Defaults to False
             move_diagnostic_plots: Whether to move various diagnostic plots or not.
                 Defaults to False
+            compress_diagnostic_plots: Whether to compress the diagnostic plot folder
+                to limit file number. Defaults to True
             lv3_dir: Where level 3 files are located, relative
                 to the target directory structure. Defaults to "lv3"
             background_dir: Where tweakback files are located, relative
@@ -85,6 +88,7 @@ def __init__(
         self.move_individual_fields = move_individual_fields
         self.move_psf_matched = move_psf_matched
         self.move_diagnostic_plots = move_diagnostic_plots
+        self.compress_diagnostic_plots = compress_diagnostic_plots
         self.overwrite = overwrite
 
         self.hdu_ext_to_delete = [
@@ -190,6 +194,9 @@ def do_step(self):
                     band=band,
                 )
 
+        if self.move_diagnostic_plots and self.compress_diagnostic_plots:
+            self.do_compress_diagnostic_plots()
+
         with open(step_complete_file, "w+") as f:
             f.close()
 
@@ -526,3 +533,27 @@ def do_move_diagnostic_plots(
             os.system(f"cp {file} {out_name}")
 
         return True
+
+    def do_compress_diagnostic_plots(self):
+        """Compress diagnostic plot directories"""
+
+        orig_dir = os.getcwd()
+
+        out_dir = os.path.join(self.out_dir, self.target)
+
+        os.chdir(out_dir)
+
+        plot_dirs = glob.glob("*_diagnostic_plots")
+
+        for plot_dir in tqdm(
+                plot_dirs,
+                ascii=True,
+                desc="Compressing diagnostic plots",
+                leave=False,
+        ):
+            os.system(f"tar -czf {plot_dir}.tar.gz {plot_dir}")
+            os.system(f"rm -rf {plot_dir}")
+
+        os.chdir(orig_dir)
+
+        return True