Fixed provenance tracking for NCL multipanel PNGs (#3332)

Co-authored-by: Axel Lauer <axel.lauer@dlr.de>
ESMValGroup · Aug 18, 2023 · d062450 · d062450
1 parent fd2bee7
commit d062450
Show file tree

Hide file tree

Showing 9 changed files with 32 additions and 98 deletions.
diff --git a/esmvaltool/diag_scripts/clouds/clouds.ncl b/esmvaltool/diag_scripts/clouds/clouds.ncl
@@ -688,7 +688,6 @@ begin
     ; function in aux_plotting.ncl
 
     if (ii.eq.0) then
-      nframe = 0
       ndframe = 0
       ; note: an array of workspaces (i.e. wks(numseas)) does not work as
       ;       attributes cannot be assigned to each array element
@@ -768,8 +767,6 @@ begin
       maps(imod, 0) = gsn_csm_contour_map(wks0, data1, res)
     end if
 
-    nframe = nframe + 1
-
     ; mandatory netcdf output
 
     data1@var = var0 + "_mean_" + names(imod)
@@ -1310,19 +1307,10 @@ begin
     end if
   end if  ; if embracesetup
 
-  nframe = nframe + 1
-
   do is = 0, numseas - 1
     log_info("Wrote " + outfile(is))
   end do
 
-  do is = 0, numseas - 1
-    suffix = get_file_suffix(outfile(is), 0)
-    if (suffix .eq. ".png") then
-      outfile(is) = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-    end if
-  end do
-
   ; ------------------------------------------------------------------------
   ; write provenance to netcdf output and plot file(s) (mean)
   ; ------------------------------------------------------------------------
@@ -1482,13 +1470,6 @@ begin
       end if
     end if  ; end if embracesetup
 
-    do is = 0, numseas - 1
-      suffix = get_file_suffix(outfile_d(is), 0)
-      if (suffix .eq. ".png") then
-        outfile_d(is) = suffix@fBase + "." + sprinti("%0.6i", ndframe) + suffix
-      end if
-    end do
-
     do is = 0, numseas - 1
       log_info(" Wrote " + outfile(is))
 

diff --git a/esmvaltool/diag_scripts/clouds/clouds_dyn_matrix.ncl b/esmvaltool/diag_scripts/clouds/clouds_dyn_matrix.ncl
@@ -681,8 +681,6 @@ begin
   xyres2@tmXTLabelsOn = True
   xyres2@tmXBLabelsOn = False
 
-  nframe = 0
-
   do ii = 0, nplots - 1
     if (ii .lt. dim_MOD) then
       plotdata = result(ii, :, :)
@@ -775,8 +773,6 @@ begin
 
     delete(plotdata)
     delete(countdata)
-
-    nframe = nframe + 1
   end do
 
 ;  pres                      = True    ; needed to override
@@ -799,11 +795,6 @@ begin
   log_info("Wrote " + doutfile)
   log_info("Wrote " + coutfile)
 
-  suffix = get_file_suffix(outfile, 0)
-  if (suffix .eq. ".png") then
-    outfile = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-  end if
-
   ; ==========================================================================
 
   ; ----------------------------------------------------------------------

diff --git a/esmvaltool/diag_scripts/clouds/clouds_interannual.ncl b/esmvaltool/diag_scripts/clouds/clouds_interannual.ncl
@@ -445,11 +445,9 @@ begin
     if (ii.eq.0) then
       wks = get_wks("dummy_for_wks", DIAG_SCRIPT, "clouds_interannual_" \
                     + var0 + filename_add)
-      nframe = 0
     end if
 
     maps(ii) = gsn_csm_contour_map(wks, data1, res)
-    nframe = nframe + 1
 
     ; #########################################
     ; # output all datasets to common netCDF  #
@@ -478,7 +476,6 @@ begin
   pres@pmLabelBarOrthogonalPosF   = -0.01     ; shift label bar a bit to
                                               ; the bottom
   outfile = panelling(wks, maps, (dim_MOD + 3) / 4, 4, pres)
-  nframe = nframe + 1
 
   ; plot multi-obs and multi-model average (if requested)
 
@@ -535,11 +532,6 @@ begin
 
   log_info(" Wrote " + outfile)
 
-  suffix = get_file_suffix(outfile, 0)
-  if (suffix .eq. ".png") then
-    outfile = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-  end if
-
   ; ------------------------------------------------------------------------
   ; write provenance to common netcdf and plot file
   ; ------------------------------------------------------------------------

diff --git a/esmvaltool/diag_scripts/clouds/clouds_lifrac_scatter.ncl b/esmvaltool/diag_scripts/clouds/clouds_lifrac_scatter.ncl
@@ -652,8 +652,6 @@ begin
     data_l(0, :) = data@_FillValue
   end if
 
-  nframe = 0
-
   do ii = 0, dim_MOD + add_dim_MOD - 1
     if (ii .eq. refidx_ta) then
       continue
@@ -724,8 +722,6 @@ begin
     frame(wks_i)
     draw(plots_l(ii))
     frame(wks_l)
-
-    nframe = nframe + 1
   end do
 
   pres                      = True    ; needed to override
@@ -747,17 +743,6 @@ begin
   log_info("Wrote " + outfile_l)
   delete(idx0)
 
-  nframe = nframe + 1
-
-  suffix = get_file_suffix(outfile_i, 0)
-  if (suffix .eq. ".png") then
-    outfile_i = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-  end if
-  suffix = get_file_suffix(outfile_l, 0)
-  if (suffix .eq. ".png") then
-    outfile_l = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-  end if
-
   ; ==========================================================================
 
   ; ----------------------------------------------------------------------

diff --git a/esmvaltool/diag_scripts/clouds/clouds_pdf.ncl b/esmvaltool/diag_scripts/clouds/clouds_pdf.ncl
@@ -220,7 +220,6 @@ begin
     ; function in aux_plotting.ncl
 
     if (ii.eq.0) then
-      nframe = 0
       wks = get_wks("dummy_for_wks", DIAG_SCRIPT, "clouds_pdf_" + var0 \
                     + filename_add)
       wks_line = get_wks("dummy_for_wks", DIAG_SCRIPT, "clouds_pdf_" + var0 \
@@ -235,7 +234,6 @@ begin
     res@tiMainString             = names(imod)
 
     plot(ii) = gsn_histogram(wks, ndtooned(data1), res)
-    nframe = nframe + 1
 
     ; mandatory netcdf output
 
@@ -364,12 +362,6 @@ begin
 
   gsn_panel(wks, plot(plotind), plotsperline, pres)
   outfile = wks@fullname
-  nframe = nframe + 1
-
-  suffix = get_file_suffix(outfile, 0)
-  if (suffix .eq. ".png") then
-    outfile = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-  end if
 
   ; ========================================================================
 

diff --git a/esmvaltool/diag_scripts/clouds/clouds_scatter.ncl b/esmvaltool/diag_scripts/clouds/clouds_scatter.ncl
@@ -434,8 +434,6 @@ begin
     data(0, :) = data@_FillValue
   end if
 
-  nframe = 0
-
   do ii = 0, dim_MOD - 1
     if (ii .eq. refidx_x) then
       continue
@@ -485,7 +483,6 @@ begin
     res@tiMainString = names(ii)
 
     plots(ii) = gsn_csm_xy(wks, result_avg&bin, data, res)
-    nframe = nframe + 1
 
     if (ii .eq. mm_ind) then
       res_std = True
@@ -554,16 +551,10 @@ begin
   pres@lbLabelFontHeightF               = min((/0.01, 0.01 * 6.0 \
                                               / tofloat((dim_MOD + 1) / 2)/))
   outfile = panelling(wks, plots(idx0), (n + 3) / 4, 4, pres)
-  nframe = nframe + 1
   delete(idx0)
 
   log_info("Wrote " + outfile)
 
-  suffix = get_file_suffix(outfile, 0)
-  if (suffix .eq. ".png") then
-    outfile = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-  end if
-
   ; ==========================================================================
 
   ; ----------------------------------------------------------------------

diff --git a/esmvaltool/diag_scripts/clouds/clouds_seasonal_cycle.ncl b/esmvaltool/diag_scripts/clouds/clouds_seasonal_cycle.ncl
@@ -405,11 +405,9 @@ begin
     if (ii.eq.0) then
       wks = get_wks("dummy_for_wks", DIAG_SCRIPT, "clouds_seasonal_cycle_" \
                     + var0 + filename_add)
-      nframe = 0
     end if
 
     maps(ii) = gsn_csm_contour_map(wks, data1, res)
-    nframe = nframe + 1
 
     ; #########################################
     ; # output all datasets to common netCDF  #
@@ -438,12 +436,6 @@ begin
   pres@pmLabelBarOrthogonalPosF   = -0.01     ; shift label bar a bit to
                                               ; the bottom
   outfile = panelling(wks, maps, (dim_MOD + 3) / 4, 4, pres)
-  nframe = nframe + 1
-
-  suffix = get_file_suffix(outfile, 0)
-  if (suffix .eq. ".png") then
-    outfile = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-  end if
 
   ; plot multi-obs and multi-model average (if requested)
 
@@ -455,7 +447,6 @@ begin
     res@gsnLeftString = "mean = " + sprintf("%6.3f", gavg)
     res@tiMainString = "Multi-obs average"
     map_multiobs = gsn_csm_contour_map(wks, multiobs, res)
-    nframe = nframe + 1
   end if
 
   if (flag_multimod) then
@@ -488,7 +479,6 @@ begin
     res@gsnRightString = "rmsd = " + sprintf("%6.3f", rmsd)
     res@tiMainString = "Multi-model average"
     map_multimod = gsn_csm_contour_map(wks, multimod, res)
-    nframe = nframe + 1
   end if
 
   log_info(" Wrote " + outfile)

diff --git a/esmvaltool/diag_scripts/clouds/clouds_zonal.ncl b/esmvaltool/diag_scripts/clouds/clouds_zonal.ncl
@@ -483,7 +483,6 @@ begin
       ;       individually
       wks0 = get_wks("dummy_for_wks", DIAG_SCRIPT, "clouds_zonal_" + var0 + \
                      "_" + season(0) + filename_add)
-      nframe = 0
       ; difference plots will be saved to a different file
       if (flag_diff) then
         wks0d = get_wks("dummy_for_wks", DIAG_SCRIPT, "clouds_zonal_" + \
@@ -575,7 +574,6 @@ begin
         maps(imod, 0) = gsn_csm_contour(wks0, data1, res)
       end if
     end if
-    nframe = nframe + 1
 
     ; mandatory netcdf output
 
@@ -1121,19 +1119,11 @@ begin
                              4, pres)
     end if
   end if  ; if embracesetup
-  nframe = nframe + 1
 
   do is = 0, numseas - 1
     log_info("Wrote " + outfile(is))
   end do
 
-  do is = 0, numseas - 1
-    suffix = get_file_suffix(outfile(is), 0)
-    if (suffix .eq. ".png") then
-      outfile(is) = suffix@fBase + "." + sprinti("%0.6i", nframe) + suffix
-    end if
-  end do
-
   ; ------------------------------------------------------------------------
   ; write provenance to netcdf output and plot file(s) (mean)
   ; ------------------------------------------------------------------------
@@ -1290,13 +1280,6 @@ begin
     end if  ; end if embracesetup
     ndframe = ndframe + 1
 
-    do is = 0, numseas - 1
-      suffix = get_file_suffix(outfile_d(is), 0)
-      if (suffix .eq. ".png") then
-        outfile_d(is) = suffix@fBase + "." + sprinti("%0.6i", ndframe) + suffix
-      end if
-    end do
-
     do is = 0, numseas - 1
       log_info(" Wrote " + outfile(is))
 

diff --git a/esmvaltool/interface_scripts/logging.ncl b/esmvaltool/interface_scripts/logging.ncl
@@ -334,9 +334,38 @@ begin
 
   ; Save entries for outfile if not "n/a"
   if (outfile .ne. "n/a") then
-    outstring2 = outstring
-    outstring2(0) = "? " + outfile
-    outstring := array_append_record(outstring, outstring2, 0)
+    suffix = get_file_suffix(outfile, 0)
+
+    ; For PNGs, additionally check for existence of files like
+    ; "plot_file.000001.png", "plot_file.000002.png", etc. and save
+    ; provenance record for each of them
+    if ((suffix .eq. ".png") .and. (.not. fileexists(outfile))) then
+      do file_idx = 1, 999999
+        potential_outfile = suffix@fBase + "." + sprinti("%0.6i", file_idx) + \
+                            suffix
+        if (fileexists(potential_outfile)) then
+          if (.not. isvar("all_outfiles")) then
+            all_outfiles = potential_outfile
+          else
+            all_outfiles := array_append_record(all_outfiles, \
+                                                potential_outfile, 0)
+          end if
+        else
+          break
+        end if
+      end do
+    else
+      all_outfiles = outfile
+    end if
+
+    ; Save provenance record of all files
+    original_entry = outstring
+    do outfile_idx = 0, dimsizes(all_outfiles) - 1
+      file_to_add := all_outfiles(outfile_idx)
+      new_entry = (/original_entry/)
+      new_entry(0) = "? " + file_to_add
+      outstring := array_append_record(outstring, new_entry, 0)
+    end do
   end if
 
   ; Save existing information to avoid overwriting