From ecd8f6a4ea1a5ad03c0a42f1c6d339cbe9f4aa78 Mon Sep 17 00:00:00 2001 From: Steve Coss Date: Fri, 11 Aug 2023 17:13:16 -0400 Subject: [PATCH 1/5] update to stats modified how outputs were constructed to use the input dictionary keys as names for the outputs. --- val/validation.py | 12 +++++++++--- validation_confluence.py | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/val/validation.py b/val/validation.py index 356a57e..850e708 100644 --- a/val/validation.py +++ b/val/validation.py @@ -35,6 +35,7 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): KGEo=[] RMSEo=[] no=[] + offkey=[] if len(Sq)<10: for Grp in Sq: Sq_=Sq[Grp] @@ -72,14 +73,18 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): #RMSE RMSE=np.sqrt((np.sum( Sq_ - Vq_t)**2)/n) RMSEo.append(RMSE) + offkey.append(Grp) else: NSEo.append(EMPTY) Rsqo.append(EMPTY) KGEo.append(EMPTY) no.append(EMPTY) RMSEo.append(EMPTY) + offkey.append(EMPTY) + + validout={ - "algorithm": np.array(['geobam','hivdi','metroman','momma']), + "algorithm": np.array([offkey]), "NSE":NSEo[:], "Rsq":Rsqo[:], "KGE":KGEo[:], @@ -140,15 +145,16 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): #RMSE RMSE=np.sqrt((np.sum( Sq_ - Vq_t)**2)/n) RMSEo.append(RMSE) + offkey.append(Grp) else: NSEo.append(EMPTY) Rsqo.append(EMPTY) KGEo.append(EMPTY) no.append(EMPTY) RMSEo.append(EMPTY) + offkey.append(EMPTY) validout={ - "algorithm": np.array(['geobam_q_c','hivdi_q_c','metroman_q_c','momma_q_c','sad_q_c', - 'geobam_q_uc','hivdi_q_uc','metroman_q_uc','momma_q_uc','sad_q_uc']), + "algorithm": np.array([offkey]), "NSE":NSEo[:], "Rsq":Rsqo[:], "KGE":KGEo[:], diff --git a/validation_confluence.py b/validation_confluence.py index 2fbff09..25d2bbf 100644 --- a/validation_confluence.py +++ b/validation_confluence.py @@ -226,6 +226,8 @@ def read_offline_data(self, offline_dir): offline_data["metroman_q_uc"] = off[convention_dict["metro_q_uc"]][:].filled(np.nan) offline_data["momma_q_uc"] = off[convention_dict["momma_q_uc"]][:].filled(np.nan) offline_data["sad_q_uc"] = off[convention_dict["sads_q_uc"]][:].filled(np.nan) + offline_data["consensus_q_c"] = off[convention_dict["consensus_q_c"]][:].filled(np.nan) + offline_data["consensus_q_uc"] = off[convention_dict["consensus_q_uc"]][:].filled(np.nan) off.close() if self.is_offline_valid(offline_data): From 0e160db7ee5e1b96650a2c251e7fee74846a0776 Mon Sep 17 00:00:00 2001 From: Steve Coss Date: Mon, 14 Aug 2023 12:29:11 -0400 Subject: [PATCH 2/5] Added stats Added nRMSE.nb,rRMSE to stats class. and updated write class to include them. --- val/validation.py | 34 ++++++++++++++++++++++++++++++++++ validation_confluence.py | 12 ++++++++++++ 2 files changed, 46 insertions(+) diff --git a/val/validation.py b/val/validation.py index 850e708..79a0791 100644 --- a/val/validation.py +++ b/val/validation.py @@ -34,6 +34,9 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): Rsqo=[] KGEo=[] RMSEo=[] + nRMSEo=[] + nBIASo=[] + rRMSEo=[] no=[] offkey=[] if len(Sq)<10: @@ -74,6 +77,16 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): RMSE=np.sqrt((np.sum( Sq_ - Vq_t)**2)/n) RMSEo.append(RMSE) offkey.append(Grp) + #nRMSE + NRMSE=RMSE/np.mean(Vq_t) + nRMSEo.append(NRMSE) + #nBIASo + BIAS= np.sum(Sq_ - Vq_t)/len( Vq_t) + nBIAS=BIAS/np.mean(Vq_t) + nBIASo.append(nBIAS) + #rRMSE + rRMSEo.append(np.sqrt(NRMSE**2-nBIAS**2)) + else: NSEo.append(EMPTY) Rsqo.append(EMPTY) @@ -81,6 +94,9 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): no.append(EMPTY) RMSEo.append(EMPTY) offkey.append(EMPTY) + nRMSEo.append(EMPTY) + nBIASo.append(EMPTY) + rRMSEo.append(EMPTY) validout={ @@ -89,6 +105,9 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): "Rsq":Rsqo[:], "KGE":KGEo[:], "RMSE":RMSEo[:], + "nRMSE":nRMSEo[:], + "nBIAS":nBIASo[:], + "rRMSE":rRMSEo[:], "n":no[:], "t":St_ } @@ -146,6 +165,15 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): RMSE=np.sqrt((np.sum( Sq_ - Vq_t)**2)/n) RMSEo.append(RMSE) offkey.append(Grp) + #nRMSE + NRMSE=RMSE/np.mean(Vq_t) + nRMSEo.append(NRMSE) + #nBIASo + BIAS= np.sum(Sq_ - Vq_t)/len( Vq_t) + nBIAS=BIAS/np.mean(Vq_t) + nBIASo.append(nBIAS) + #rRMSE + rRMSEo.append(np.sqrt(NRMSE**2-nBIAS**2)) else: NSEo.append(EMPTY) Rsqo.append(EMPTY) @@ -153,12 +181,18 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): no.append(EMPTY) RMSEo.append(EMPTY) offkey.append(EMPTY) + nRMSEo.append(EMPTY) + nBIASo.append(EMPTY) + rRMSEo.append(EMPTY) validout={ "algorithm": np.array([offkey]), "NSE":NSEo[:], "Rsq":Rsqo[:], "KGE":KGEo[:], "RMSE":RMSEo[:], + "nRMSE":nRMSEo[:], + "nBIAS":nBIASo[:], + "rRMSE":rRMSEo[:], "n":no[:], "t":St_ } diff --git a/validation_confluence.py b/validation_confluence.py index 25d2bbf..897af4f 100644 --- a/validation_confluence.py +++ b/validation_confluence.py @@ -360,6 +360,18 @@ def write(self, stats, reach_id, gage_type): rmse_v = out.createVariable("RMSE", "f8", ("num_algos",), fill_value=fill) rmse_v.units = "m^3/s" rmse_v[:] = np.where(np.isclose(stats["RMSE"], empty), fill, stats["RMSE"]) + + nrmse_v = out.createVariable("nRMSE", "f8", ("num_algos",), fill_value=fill) + nrmse_v.units = "none" + nrmse_v[:] = np.where(np.isclose(stats["nRMSE"], empty), fill, stats["nRMSE"]) + + nb_v = out.createVariable("nBIAS", "f8", ("num_algos",), fill_value=fill) + nb_v.units = "none" + nb_v[:] = np.where(np.isclose(stats["nBIAS"], empty), fill, stats["nBIAS"]) + + rrmse_v = out.createVariable("rRMSE", "f8", ("num_algos",), fill_value=fill) + rrmse_v.units = "none" + rrmse_v[:] = np.where(np.isclose(stats["rRMSE"], empty), fill, stats["rRMSE"]) n_v = out.createVariable("testn", "f8", ("num_algos",), fill_value=fill) n_v[:] = np.where(np.isclose(stats["n"], empty), fill, stats["n"]) From 5322fcb4d08a3ccfeb4eda15558dda41a6c29d37 Mon Sep 17 00:00:00 2001 From: Steve Coss Date: Thu, 17 Aug 2023 11:03:19 -0400 Subject: [PATCH 3/5] Update validation_confluence.py-FillValues Update validation_confluence.py with appropriate filled arrays for the new stats. --- validation_confluence.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validation_confluence.py b/validation_confluence.py index 897af4f..acad3c7 100644 --- a/validation_confluence.py +++ b/validation_confluence.py @@ -294,6 +294,9 @@ def validate(self): "KGE": np.full((self.NUM_ALGOS), fill_value=-9999), "RMSE": np.full((self.NUM_ALGOS), fill_value=-9999), "n": np.full((self.NUM_ALGOS), fill_value=-9999), + "nRMSE":np.full((self.NUM_ALGOS), fill_value=-9999), + "nBIAS":np.full((self.NUM_ALGOS), fill_value=-9999), + "rRMSE":np.full((self.NUM_ALGOS), fill_value=-9999), } no_offline = False # Check if there is data to validate From 3918ea3a7731d91a7544088b0139309e1d0663aa Mon Sep 17 00:00:00 2001 From: Travis-Simmons Date: Thu, 24 Aug 2023 12:35:56 -0400 Subject: [PATCH 4/5] dimension bugfix on stats and naming convention change --- validation_confluence.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/validation_confluence.py b/validation_confluence.py index 897af4f..c0709b3 100644 --- a/validation_confluence.py +++ b/validation_confluence.py @@ -216,18 +216,18 @@ def read_offline_data(self, offline_dir): offline_file = f"{offline_dir}/{self.reach_id}_offline.nc" off = Dataset(offline_file, 'r') offline_data = {} - offline_data["geobam_q_c"] = off[convention_dict["bam_q_c"]][:].filled(np.nan) - offline_data["hivdi_q_c"] = off[convention_dict["hivdi_q_c"]][:].filled(np.nan) - offline_data["metroman_q_c"] = off[convention_dict["metro_q_c"]][:].filled(np.nan) - offline_data["momma_q_c"] = off[convention_dict["momma_q_c"]][:].filled(np.nan) - offline_data["sad_q_c"] = off[convention_dict["sads_q_c"]][:].filled(np.nan) - offline_data["geobam_q_uc"] = off[convention_dict["bam_q_uc"]][:].filled(np.nan) - offline_data["hivdi_q_uc"] = off[convention_dict["hivdi_q_uc"]][:].filled(np.nan) - offline_data["metroman_q_uc"] = off[convention_dict["metro_q_uc"]][:].filled(np.nan) - offline_data["momma_q_uc"] = off[convention_dict["momma_q_uc"]][:].filled(np.nan) - offline_data["sad_q_uc"] = off[convention_dict["sads_q_uc"]][:].filled(np.nan) - offline_data["consensus_q_c"] = off[convention_dict["consensus_q_c"]][:].filled(np.nan) - offline_data["consensus_q_uc"] = off[convention_dict["consensus_q_uc"]][:].filled(np.nan) + offline_data[convention_dict["bam_q_c"]] = off[convention_dict["bam_q_c"]][:].filled(np.nan) + offline_data[convention_dict["hivdi_q_c"]] = off[convention_dict["hivdi_q_c"]][:].filled(np.nan) + offline_data[convention_dict["metro_q_c"]] = off[convention_dict["metro_q_c"]][:].filled(np.nan) + offline_data[convention_dict["momma_q_c"]] = off[convention_dict["momma_q_c"]][:].filled(np.nan) + offline_data[convention_dict["sads_q_c"]] = off[convention_dict["sads_q_c"]][:].filled(np.nan) + offline_data[convention_dict["bam_q_uc"]] = off[convention_dict["bam_q_uc"]][:].filled(np.nan) + offline_data[convention_dict["hivdi_q_uc"]] = off[convention_dict["hivdi_q_uc"]][:].filled(np.nan) + offline_data[convention_dict["metro_q_uc"]] = off[convention_dict["metro_q_uc"]][:].filled(np.nan) + offline_data[convention_dict["momma_q_uc"]] = off[convention_dict["momma_q_uc"]][:].filled(np.nan) + offline_data[convention_dict["sads_q_uc"]] = off[convention_dict["sads_q_uc"]][:].filled(np.nan) + offline_data[convention_dict["consensus_q_c"]] = off[convention_dict["consensus_q_c"]][:].filled(np.nan) + offline_data[convention_dict["consensus_q_uc"]] = off[convention_dict["consensus_q_uc"]][:].filled(np.nan) off.close() if self.is_offline_valid(offline_data): @@ -346,7 +346,7 @@ def write(self, stats, reach_id, gage_type): t_v[:] = stats["t"] a_v = out.createVariable("algorithm", 'S1', ("num_algos", "nchar"),) - a_v[:] = stringtochar(stats["algorithm"].astype("S16")) + a_v[:] = stringtochar(stats["algorithm"][0].astype("S16")) nse_v = out.createVariable("NSE", "f8", ("num_algos",), fill_value=fill) nse_v[:] = np.where(np.isclose(stats["NSE"], empty), fill, stats["NSE"]) From 99b1e4d491ecb879b1f651a249b1eae7c76cd545 Mon Sep 17 00:00:00 2001 From: Steve Coss Date: Thu, 24 Aug 2023 14:56:28 -0400 Subject: [PATCH 5/5] fixed RMSE bug typo in RMSE calculation --- val/validation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/val/validation.py b/val/validation.py index 79a0791..01629f8 100644 --- a/val/validation.py +++ b/val/validation.py @@ -74,7 +74,7 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): n=len(Vq_t) no.append(n) #RMSE - RMSE=np.sqrt((np.sum( Sq_ - Vq_t)**2)/n) + RMSE=np.sqrt((np.sum( (Sq_ - Vq_t)**2))/n) RMSEo.append(RMSE) offkey.append(Grp) #nRMSE @@ -162,7 +162,7 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): n=len(Vq_t) no.append(n) #RMSE - RMSE=np.sqrt((np.sum( Sq_ - Vq_t)**2)/n) + RMSE=np.sqrt((np.sum( (Sq_ - Vq_t)**2))/n) RMSEo.append(RMSE) offkey.append(Grp) #nRMSE @@ -198,4 +198,4 @@ def stats(St,Sq,Vt,Vq,IDstr,figdir): } return validout - \ No newline at end of file +