Fixed plot paths in NCL provenance tracking (#3422)

Co-authored-by: Klaus Zimmermann <klaus.zimmermann@smhi.se>

esmvaltool/diag_scripts/austral_jet/asr.ncl

@@ -289,7 +289,7 @@ begin
     work_dir = output@work_dir
     opt = diag_script_info
     log_info(opt@wdiag)
-    plot_path = "missing"
+    plot_path = "n/a"
     plot_type = ""
 
     ; Iterate over all datasets

esmvaltool/diag_scripts/austral_jet/main.ncl

@@ -731,7 +731,7 @@ begin
   if (output) then
     work_dir := output@work_dir
     opt = diag_script_info
-    plot_path = "missing"
+    plot_path = "n/a"
     plot_type = ""
 
     ; Iterate over all desired diagnostics

esmvaltool/diag_scripts/carbon_ec/carbon_beta.ncl

@@ -279,11 +279,7 @@ begin
   create_legend_lines(leg@annots, leg, plot_dir + \
                       DIAG_SCRIPT + "_legend", "markers")
 
-  if (file_type .ne. "png") then
-    plotname = plot_dir + plot_file + "." + file_type
-  else
-    plotname = plot_dir + plot_file + ".000001.png"
-  end if
+  plotname = plot_dir + plot_file + "." + file_type
 
   ; Call provenance logger
   log_provenance(ncdf_outfile, \

esmvaltool/diag_scripts/carbon_ec/carbon_co2_cycle.ncl

@@ -492,11 +492,7 @@ begin
   ; Write NetCDF output
   ncdf_outfile = ncdf_write(CO2var, new_path)
 
-  if (file_type .ne. "png") then
-    plotname = plot_dir + plot_file + "." + file_type
-  else
-    plotname = plot_dir + plot_file + ".000001.png"
-  end if
+  plotname = plot_dir + plot_file + "." + file_type
 
   ; -----------------------------------------------------------
   ; Call provenance logger

esmvaltool/diag_scripts/carbon_ec/carbon_constraint.ncl

@@ -331,7 +331,7 @@ begin
 
   ; Call provenance logger
   log_provenance(ncdf_outfile, \
-                 output_dir + "." + file_type, \
+                 output_dir + plot_file + "." + file_type, \
                  XStg + " vs " + YStg, \
                  (/"anomaly", "corr", "stddev"/), \
                  (/"trop", "global"/),\

esmvaltool/diag_scripts/carbon_ec/carbon_gammaHist.ncl

@@ -336,7 +336,7 @@ begin
 
   ; Call provenance logger
   log_provenance(ncdf_outfile, \
-                 output_dir + "." + file_type, \
+                 output_dir + plot_file + "." + file_type, \
                  XStg + " vs " + YStg, \
                  (/"anomaly", "corr", "stddev"/), \
                  (/"trop", "global"/),\

esmvaltool/diag_scripts/ipcc_ar5/ch12_calc_IAV_for_stippandhatch.ncl

@@ -410,15 +410,14 @@ begin
 
   ; collect meta-data
   nc_file = ncdf_outfile
-  plot_file = work_dir + "/None"
   caption = "Inter-annual variability based on piControl runs."
   statistics = ("var")
   domains = ("global")
   plot_types = ("other")
   authors = (/"lorenz_ruth"/)
   references = (/"collins13ipcc"/)
   infiles = metadata_att_as_array(info_items, "filename")
-  log_provenance(nc_file, plot_file, caption, statistics, domains, \
+  log_provenance(nc_file, "n/a", caption, statistics, domains, \
                  plot_types, authors, references, infiles)
 
   leave_msg(DIAG_SCRIPT, "")

esmvaltool/diag_scripts/mder/select_for_mder.ncl

@@ -97,7 +97,7 @@ begin
   AUTHORS = (/"wenzel_sabrina", "schlund_manuel"/)
   DOMAIN = diag_script_info@domain
   PLOT_TYPE = ""
-  PLOT_FILE = "missing"
+  PLOT_FILE = "n/a"
   REFERENCES = (/"wenzel16jclim"/)
 
   ; Get reference dataset(s)

esmvaltool/diag_scripts/perfmetrics/collect.ncl

@@ -502,8 +502,9 @@ begin
   domains = (/"global"/)
 
   ; Call provenance logger
-  log_provenance(ncdf_outfile, plotpath, caption, statistics, domains, \
-                 plottype, authors, references, data_files)
+  log_provenance(ncdf_outfile, plotpath + "." + file_type, caption, \
+                 statistics, domains, plottype, authors, references, \
+                 data_files)
 
   leave_msg(DIAG_SCRIPT, "")
 

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig2.ncl

@@ -158,7 +158,7 @@ begin
 
   ; Call provenance logger
   log_provenance(ncdf_outfile, \
-                 plotpath, \
+                 plotpath + "." + output_type(), \
                  "Russell et al 2018 figure 2", \
                  "mean", \
                  "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig3b-2.ncl

@@ -274,7 +274,7 @@ begin
     delete(last_index)
     delete(out_var)
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 3b part 2", \
                    "mean", \
                    "sh",   \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig3b.ncl

@@ -264,7 +264,7 @@ begin
     delete(last_index)
     delete(out_var)
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 3b", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig4.ncl

@@ -368,7 +368,7 @@ begin
     delete(totaltransport)
     ; Call provenance logger
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + file_type, \
                    "Russell et al 2018 figure 4", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig5.ncl

@@ -169,7 +169,7 @@ begin
     delete(dataset)
 
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 5 -polar", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig5g.ncl

@@ -226,7 +226,7 @@ begin
 
     ; Call provenance logger
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 5g", \
                    "mean", \
                    "sh",   \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl

@@ -699,7 +699,7 @@ begin
     ncdf_outfile = ncdf_write(out_var, nc_filename)
 
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 6 part a", \
                    "mean", \
                    "sh",   \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl

@@ -678,7 +678,7 @@ begin
     ncdf_outfile = ncdf_write(out_var, nc_filename)
 
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 6b", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7h.ncl

@@ -164,7 +164,7 @@ begin
     delete(var_lon_avg)
     delete(dataset)
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 7h", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl

@@ -202,7 +202,7 @@ begin
     end if
 
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 7i", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9a.ncl

@@ -317,7 +317,7 @@ begin
     ncdf_outfile = ncdf_write(outvar, nc_filename)
 
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 9a", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9b.ncl

@@ -335,7 +335,7 @@ begin
     ncdf_outfile = ncdf_write(outvar, nc_filename)
 
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 9b", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl

@@ -331,7 +331,7 @@ begin
     ncdf_outfile = ncdf_write(outvar, nc_filename)
 
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 figure 9c", \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/russell18jgr/russell18jgr-polar.ncl

@@ -232,7 +232,7 @@ begin
     ncdf_outfile = ncdf_write(dataset, nc_filename)
     delete(dataset)
     log_provenance(ncdf_outfile, \
-                   plotpath, \
+                   plotpath + "." + output_type(), \
                    "Russell et al 2018 polar plot " + var0, \
                    "mean", \
                    "sh", \

esmvaltool/diag_scripts/tebaldi21esd/calc_IAV_hatching.ncl

@@ -399,15 +399,14 @@ begin
 
   ; collect meta-data
   nc_file = ncdf_outfile
-  plot_file = plot_dir + "/None"
   caption = "Inter-annual variability based on piControl runs."
   statistics = ("var")
   domains = ("global")
   plot_types = ("other")
   authors = (/"lorenz_ruth"/)
   references = (/"collins13ipcc"/)
   infiles = metadata_att_as_array(info_items, "filename")
-  log_provenance(nc_file, plot_file, caption, statistics, domains, \
+  log_provenance(nc_file, "n/a", caption, statistics, domains, \
                  plot_types, authors, references, infiles)
 
   leave_msg(DIAG_SCRIPT, "")

esmvaltool/diag_scripts/xco2_analysis/delta_T.ncl

@@ -332,7 +332,7 @@ begin
   ncdf_outfile = ncdf_write(delta_t_detr, outfile_netcdf_dtgr_detr)
 
   log_provenance(ncdf_outfile, \
-                 outfile_netcdf_dtgr_detr, \
+                 outfile_dtgr_detr + "." + file_type, \
                  "Sensitivity of interannual variability of " \
                  + var0 + " growth rate in the " + region + ", " + \
                  start_year + "-" + end_year + "to the interannual " \

esmvaltool/diag_scripts/xco2_analysis/global_maps.ncl

@@ -472,7 +472,7 @@ begin
 
   ; Provenance
   log_provenance(ncdf_outfile, \
-                 outfile, \
+                 outfile + "." + file_type, \
                  "Maps of mean Seasonal Cycle Amplitude of " + var0 + " " \
                  + DATASETS(imod) + " for " + start_year + "-" \
                  + end_year + ". Top: SCA with observational sampling (left)" \
@@ -587,7 +587,7 @@ begin
 
   ; Provenance
   log_provenance(ncdf_outfile, \
-                 outfile_scap, \
+                 outfile_scap + "." + file_type, \
                  "Maps of mean Seasonal Cycle Amplitude of " \
                  + var0 + " for " + start_year + "-" \
                  + end_year + ". Similar to Gier et al 2020, Fig 5.", \

esmvaltool/diag_scripts/xco2_analysis/main.ncl

@@ -438,7 +438,7 @@ begin
 
   ; Provenance
   log_provenance(ncdf_outfile, \
-                 outfile_ts_panels, \
+                 outfile_ts_panels + "." + file_type, \
                  "Timeseries, growth rate and seasonal cycle of " \
                  + var0 + " for " + region + ", " + start_year + "-" \
                  + end_year + ". Using masking: " + opt_mask \
@@ -550,7 +550,7 @@ begin
 
   ; Provenance
   log_provenance(ncdf_outfile, \
-                 outfile_grbp, \
+                 outfile_grbp + "." + file_type, \
                  "Growth Rate histogram of " \
                  + var0 + " for " + region + ", " + start_year + "-" \
                  + end_year + ". Using masking: " + opt_mask \
@@ -653,7 +653,7 @@ begin
 
   ; Provenance
   log_provenance(ncdf_outfile, \
-                 outfile_sabp, \
+                 outfile_sabp + "." + file_type, \
                  "Seasonal Cycle Amplitude histogram of " \
                  + var0 + " for " + region + ", " + start_year + "-" \
                  + end_year + ". Using masking: " + opt_mask, \

esmvaltool/diag_scripts/xco2_analysis/panel_plots.ncl

@@ -360,7 +360,7 @@ begin
 
   ; Provenance
   log_provenance(ncdf_outfile, \
-                 outfile_scaplot, \
+                 outfile_scaplot + "." + file_type, \
                  "Trend of Seasonal Cycle Amplitude with " \
                  + var0 + " for " + region + ", " + start_year + "-" \
                  + end_year + ". Using masking: " + opt_mask \
@@ -391,7 +391,7 @@ begin
 
   ; Provenance
   log_provenance(ncdf_outfile, \
-                 outfile_grplot, \
+                 outfile_grplot + "." + file_type, \
                  "Trend of Growth Rate with " \
                  + var0 + " for " + region + ", " + start_year + "-" \
                  + end_year + ". Using masking: " + opt_mask \

esmvaltool/diag_scripts/xco2_analysis/sat_masks.ncl

@@ -258,7 +258,7 @@ begin
 
   ; Provenance
   log_provenance(ncdf_outfile, \
-                 outfile, \
+                 outfile + "." + file_type, \
                  var0 + "fractional data coverage " + start_year + "-" \
                  + end_year + ". Similar to Gier et al 2020, Fig 1.", \
                  (/"mean"/), \
@@ -377,7 +377,7 @@ begin
     delete(res)
 
     log_provenance(ncdf_outfile, \
-                   outfile_c3s, \
+                   outfile_c3s + "." + file_type, \
                    var0 + "fractional data coverage for different obs " \
                    + "periods. Similar to Gier et al 2020, Fig 8.", \
                    (/"mean"/), \

esmvaltool/diag_scripts/xco2_analysis/station_comparison.ncl

@@ -285,7 +285,7 @@ begin
 
     ; Provenance
     log_provenance(ncdf_outfile, \
-                   outfile_stations_masked, \
+                   outfile_stations_masked + "." + file_type, \
                    "Time series of station " + station_array&station(istat) \
                    + ", dotted denoting areas masked like observations.", \
                    (/""/), \
@@ -334,7 +334,7 @@ begin
 
     ; Provenance
     log_provenance(ncdf_outfile, \
-                   outfile_stations_obs, \
+                   outfile_stations_obs + "." + file_type, \
                    "Time series of station " + station_array&station(istat) \
                    + ", for " + var0 + " and " + var1 + ".", \
                    (/"mean"/), \
@@ -501,7 +501,7 @@ begin
 
         ; Provenance
     log_provenance(ncdf_outfile, \
-                   outfile_map, \
+                   outfile_map + "." + file_type, \
                    "Time series for satellite, multi-model mean and station " \
                    + var0 + " and " + var1 \
                    + ". Similar to Gier et al, 2020 Fig. 2", \

esmvaltool/interface_scripts/logging.ncl

@@ -336,6 +336,12 @@ begin
   if (outfile .ne. "n/a") then
     suffix = get_file_suffix(outfile, 0)
 
+    if (ismissing(suffix)) then
+      error_msg("f", scriptname, funcname, "got invalid value for outfile " + \
+                "(path to figure): '" + outfile + "'; expected path to a " + \
+                "file or 'n/a'")
+    end if
+
     ; For PNGs, additionally check for existence of files like
     ; "plot_file.000001.png", "plot_file.000002.png", etc. and save
     ; provenance record for each of them
@@ -354,14 +360,24 @@ begin
           break
         end if
       end do
-    else
+    end if
+
+    ; For all other cases, use outfile
+    if (.not. isvar("all_outfiles")) then
       all_outfiles = outfile
     end if
 
     ; Save provenance record of all files
     original_entry = outstring
     do outfile_idx = 0, dimsizes(all_outfiles) - 1
       file_to_add := all_outfiles(outfile_idx)
+      if (.not. fileexists(outfile)) then
+        error_msg("f", scriptname, funcname, "outfile (path to figure) '" + \
+                  file_to_add + "' does not exist (for PNGs, this " + \
+                  "function also searches for 'FILE.000001.png', " + \
+                  "'FILE.000002.png', etc.); if no plot file is available " + \
+                  "use 'n/a'")
+      end if
       new_entry = (/original_entry/)
       new_entry(0) = "? " + file_to_add
       outstring := array_append_record(outstring, new_entry, 0)