Compress diagnostic plots

CHANGES.rst

@@ -1,3 +1,8 @@
+0.10.2 (Unreleased)
+===================
+
+- Diagnostic plots are compressed by default in ``release_step``
+
 0.10.1 (2023-11-20)
 ===================
 

pjpipe/release/release_step.py

@@ -24,6 +24,7 @@ def __init__(
         move_individual_fields=False,
         move_psf_matched=False,
         move_diagnostic_plots=False,
+        compress_diagnostic_plots=True,
         lv3_dir="lv3",
         tweakback_dir="lv3",
         tweakback_ext="tweakback",
@@ -59,6 +60,8 @@ def __init__(
                 Defaults to False
             move_diagnostic_plots: Whether to move various diagnostic plots or not.
                 Defaults to False
+            compress_diagnostic_plots: Whether to compress the diagnostic plot folder
+                to limit file number. Defaults to True
             lv3_dir: Where level 3 files are located, relative
                 to the target directory structure. Defaults to "lv3"
             background_dir: Where tweakback files are located, relative
@@ -85,6 +88,7 @@ def __init__(
         self.move_individual_fields = move_individual_fields
         self.move_psf_matched = move_psf_matched
         self.move_diagnostic_plots = move_diagnostic_plots
+        self.compress_diagnostic_plots = compress_diagnostic_plots
         self.overwrite = overwrite
 
         self.hdu_ext_to_delete = [
@@ -190,6 +194,22 @@ def do_step(self):
                     band=band,
                 )
 
+        # Compress the anchoring directory, and remove directories if we're compressing
+        # the diagnostic plots
+        if self.move_diagnostic_plots and self.compress_diagnostic_plots:
+
+            orig_dir = os.getcwd()
+            os.chdir(out_dir)
+
+            anchor_plot_dir = f"anchoring_diagnostic_plots"
+            if os.path.exists(anchor_plot_dir):
+                os.system(f"tar -czf {anchor_plot_dir}.tar.gz {anchor_plot_dir}")
+
+            os.chdir(orig_dir)
+
+            rm_str = os.path.join(out_dir, "*_diagnostic_plots")
+            os.system(f"rm -rf {rm_str}")
+
         with open(step_complete_file, "w+") as f:
             f.close()
 
@@ -525,4 +545,13 @@ def do_move_diagnostic_plots(
             )
             os.system(f"cp {file} {out_name}")
 
+        # Finally, compress
+        if self.compress_diagnostic_plots:
+            orig_dir = os.getcwd()
+            os.chdir(os.path.join(self.out_dir, self.target))
+            out_dir_short = f"{band.lower()}_diagnostic_plots"
+            os.system(f"tar -czf {out_dir_short}.tar.gz {out_dir_short}")
+
+            os.chdir(orig_dir)
+
         return True