From 0607d312a502fe72e8bfadb19b53256c0e385d48 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:13:07 +0200 Subject: [PATCH 01/36] update get_l3 to add historical data --- setup.py | 2 +- .../variable_aliases_GC-Net - Kopi.csv | 78 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 src/pypromice/process/variable_aliases_GC-Net - Kopi.csv diff --git a/setup.py b/setup.py index 117e4e84..852a0047 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", package_data={ - "pypromice.process": ["metadata.csv", "variables.csv"], + "pypromice.process": ["metadata.csv", "variables.csv", "variable_aliases_GC-Net.csv"], "pypromice.tx": ["payload_formats.csv", "payload_types.csv"], "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], diff --git a/src/pypromice/process/variable_aliases_GC-Net - Kopi.csv b/src/pypromice/process/variable_aliases_GC-Net - Kopi.csv new file mode 100644 index 00000000..92704ab8 --- /dev/null +++ b/src/pypromice/process/variable_aliases_GC-Net - Kopi.csv @@ -0,0 +1,78 @@ +GEUS_name,old_name +time,timestamp +p_u,P +p_l, +t_u,TA2 +t_l,TA1 +rh_u,RH2 +rh_u_cor,RH2_cor +qh_u,Q2 +rh_l,RH1 +rh_l_cor,RH1_cor +qh_l,Q1 +wspd_u,VW2 +wspd_l,VW1 +wdir_u,DW2 +wdir_l,DW1 +dsr, +dsr_cor,ISWR +usr, +usr_cor,OSWR +albedo,Alb +dlr, +ulr, +cc, +t_surf, +dlhf_u,LHF +dlhf_l, +dshf_u,SHF +dshf_l, +z_boom_u,HW2 +z_boom_l,HW1 +precip_u, +precip_u_cor, +precip_l, +precip_l_cor, +t_i_1,TS1 +t_i_2,TS2 +t_i_3,TS3 +t_i_4,TS4 +t_i_5,TS5 +t_i_6,TS6 +t_i_7,TS7 +t_i_8,TS8 +t_i_9,TS9 +t_i_10,TS10 +t_i_11, +tilt_x, +tilt_y, +rot, +gps_lat,latitude +gps_lon,longitude +gps_alt,elevation +gps_time, +gps_geounit, +gps_hdop, +batt_v,V +fan_dc_u, +fan_dc_l, +t_rad, +msg_lat, +msg_lon, +z_surf_1,HS1 +z_surf_2,HS2 +z_surf_1_adj_flag,HS1_adj_flag +z_surf_2_adj_flag,HS2_adj_flag +z_surf_combined,HS_combined +depth_t_i_1,DTS1 +depth_t_i_2,DTS2 +depth_t_i_3,DTS3 +depth_t_i_4,DTS4 +depth_t_i_5,DTS5 +depth_t_i_6,DTS6 +depth_t_i_7,DTS7 +depth_t_i_8,DTS8 +depth_t_i_9,DTS9 +depth_t_i_10,DTS10 +depth_t_i_11, +t_i_10m,TS_10m From fe7149cc0c4ed1b468175073b33c3a4fe49cc1e7 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:13:07 +0200 Subject: [PATCH 02/36] update get_l3 to add historical data --- src/pypromice/process/get_l3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypromice/process/get_l3.py b/src/pypromice/process/get_l3.py index eccdab1a..5e46460c 100644 --- a/src/pypromice/process/get_l3.py +++ b/src/pypromice/process/get_l3.py @@ -40,7 +40,7 @@ def get_l3(): m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv') else: m = args.metadata - + # Define input path station_name = args.config_file.split('/')[-1].split('.')[0] station_path = os.path.join(args.inpath, station_name) From 8f9e69d0bc12182a375611e0c8c713ca2b3ed2e7 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Wed, 12 Jun 2024 12:49:52 +0200 Subject: [PATCH 03/36] Create get_l3_new.py --- src/pypromice/process/get_l3_new.py | 295 ++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 src/pypromice/process/get_l3_new.py diff --git a/src/pypromice/process/get_l3_new.py b/src/pypromice/process/get_l3_new.py new file mode 100644 index 00000000..8272b51b --- /dev/null +++ b/src/pypromice/process/get_l3_new.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python +import logging, os, sys, unittest +from argparse import ArgumentParser +import pypromice +from pypromice.process.aws import AWS + +old_name = { + 'CEN2': ['GITS'], + 'CP1': ['CrawfordPoint1'], + 'DY2': ['DYE-2'], + 'JAR': ['JAR1'], + 'HUM': ['Humboldt'], + 'NAU': ['NASA-U'], + 'NAE': ['NASA-E'], + 'NEM': ['NEEM'], + 'NSE': ['NASA-SE'], + 'EGP': ['EastGRIP'], + 'SDL': ['Saddle'], + 'SDM': ['SouthDome'], + 'SWC': ['SwissCamp', 'SwissCamp10m'], + 'TUN': ['Tunu-N'], + } + +def parse_arguments_getl3(debug_args=None): + parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") + parser.add_argument('-s', '--file1', type=str, required=True, nargs='+', + help='Path to source L2 file') + parser.add_argument('-g', '--folder_gcnet', type=str, required=True, nargs='+', + help='Path to GC-Net historical L1 folder') + parser.add_argument('-p', '--folder_l2', type=str, required=True, nargs='+', + help='Path to level 2 folder') + parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=True, + help='Path where to write output') + parser.add_argument('-v', '--variables', default=None, type=str, required=False, + help='Path to variables look-up table .csv file for variable name retained'''), + parser.add_argument('-m', '--metadata', default=None, type=str, required=False, + help='Path to metadata table .csv file for metadata information'''), + parser.add_argument('-d', '--datatype', default='raw', type=str, required=False, + help='Data type to output, raw or tx') + args = parser.parse_args(args=debug_args) + args.file1 = ' '.join(args.file1) + args.folder_gcnet = ' '.join(args.folder_gcnet) + args.folder_l2 = ' '.join(args.folder_l2) + return args + +def readNead(infile): + with open(infile) as f: + fmt = f.readline() + assert(fmt[0] == "#") + assert(fmt.split("#")[1].split()[0] == "NEAD") + assert(fmt.split("#")[1].split()[1] == "1.0") + assert(fmt.split("#")[1].split()[2] == "UTF-8") + + line = f.readline() + assert(line[0] == "#") + assert(line.split("#")[1].strip() == '[METADATA]') + + meta = {} + fields = {} + section = 'meta' + while True: + line = f.readline() + if line.strip(' ') == '#': continue + if line == "# [DATA]\n": break # done reading header + if line == "# [FIELDS]\n": + section = 'fields' + continue # done reading header + + if line[0] == "\n": continue # blank line + assert(line[0] == "#") # if not blank, must start with "#" + + key_eq_val = line.split("#")[1].strip() + if key_eq_val == '' or key_eq_val == None: continue # Line is just "#" or "# " or "# #"... + assert("=" in key_eq_val), print(line, key_eq_val) + key = key_eq_val.split("=")[0].strip() + val = key_eq_val.split("=")[1].strip() + + # Convert from string to number if it is a number + if val.strip('-').strip('+').replace('.','').isdigit(): + val = float(val) + if val == int(val): + val = int(val) + + if section == 'meta': meta[key] = val + if section == 'fields': fields[key] = val + # done reading header + + # Find delimiter and fields for reading NEAD as simple CSV + assert("field_delimiter" in meta.keys()) + assert("fields" in fields.keys()) + FD = meta["field_delimiter"] + names = [_.strip() for _ in fields.pop('fields').split(FD)] + + df = pd.read_csv(infile, + comment = "#", + names = names, + sep = FD, + usecols=np.arange(len(names)), + skip_blank_lines = True) + df['timestamp'] = pd.to_datetime(df.timestamp).dt.tz_localize(None) + df = df.set_index('timestamp') + ds = df.to_xarray() + ds.attrs = meta + return ds + + +def loadArr(infile): + if infile.split('.')[-1].lower() in 'csv': + with open(infile) as f: + text_splitted = f.read().splitlines() + first_char = text_splitted[0][0] + + if first_char != '#': + df = pd.read_csv(infile) + df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None) + df = df.set_index('time') + ds = xr.Dataset.from_dataframe(df) + else: + ds = readNead(infile) + f.close() + elif infile.split('.')[-1].lower() in 'nc': + ds = xr.open_dataset(infile) + + try: + name = ds.attrs['station_name'] + except: + name = infile.split('/')[-1].split('.')[0].split('_hour')[0].split('_10min')[0] + + print(f'{name} array loaded from {infile}') + return ds, name + + +def gcnet_postprocessing(ds2): + file_path = pkg_resources.resource_stream('pypromice', 'process/variable_aliases_GC-Net.csv') + + var_name = pd.read_csv(file_path) + var_name = var_name.set_index('old_name').GEUS_name + msk = [v for v in var_name.index if v in ds2.data_vars] + var_name = var_name.loc[msk].to_dict() + + ds2['TA1'] = ds2['TA1'].combine_first(ds2['TA3']) + ds2['TA2'] = ds2['TA2'].combine_first(ds2['TA4']) + + ds2=ds2.rename(var_name) + ds2=ds2.rename({'timestamp':'time'}) + return ds2 + +# will be used in the future +# def aligning_surface_heights(ds1, ds2): + # df_aux['z_surf_combined'] = \ + # df_aux['z_surf_combined'] \ + # - df_aux.loc[df_aux.z_surf_combined.last_valid_index(), 'z_surf_combined'] \ + # + df_v6.loc[df_v6.z_surf_combined.first_valid_index(), 'z_surf_combined'] + + # if s == 'Swiss Camp 10m': + # df.loc[:df.HS_combined.first_valid_index(), 'HS_combined'] = \ + # df2.loc[:df.HS_combined.first_valid_index(), 'HS_combined'] \ + # - df2.loc[df2.HS_combined.last_valid_index(), 'HS_combined'] \ + # + df.loc[df.HS_combined.first_valid_index(), 'HS_combined'] + + + # df.loc[df.HS_combined.diff()==0,'HS_combined'] = np.nan + + # fit = np.polyfit(df.loc[df.HS_combined.notnull(),:].index.astype('int64'), + # df.loc[df.HS_combined.notnull(),'HS_combined'], 1) + # fit_fn = np.poly1d(fit) + + # df['HS_combined'] = df['HS_combined'].values \ + # - fit_fn( + # df_in.loc[[df_in.z_surf_combined.first_valid_index()],:].index.astype('int64')[0] + # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] + # return ds1 + +def get_l3(): + args = parse_arguments_getl3() + + # Check files + if os.path.isfile(args.file1): + # Load L2 data arrays + ds1, n1 = loadArr(args.file1) + + # converts to L3: + # - derives sensible heat fluxes + # - appends historical data + + ds1 = toL3(ds1) + + if n1 in old_name.keys(): + n2 = old_name[n1] + # for all secondary stations + for n in n2: + # loading secondary file, either from GC-Net or PROMICE folders + file2 = args.folder_gcnet+n+'.csv' + is_gcnet = True + if not os.path.isfile(file2): + file2 = args.folder_l2+'/'+n+'/'+n+'_hour.csv' + if not os.path.isfile(file2): + print('could not find',n, 'in',args.folder_gcnet,'nor',args.folder_l2) + continue + else: + is_gcnet = False + + ds2, n2 = loadArr(file2) + + if is_gcnet: + ds2 = gcnet_postprocessing(ds2) + ds2 = ds2[[v for v in ds1.data_vars if v in ds2.data_vars]] + else: + # then it is a GEUS L2 file that needs to be processed into L3 + # converts to L3 + ds2 = toL3(ds2) + + # prepairing for the merging + for v in ds1.data_vars: + if v not in ds2.data_vars: + ds2[v] = ds2.t_u*np.nan + + print('dropping') + for v in ds2.data_vars: + if v not in ds1.data_vars: + print(v) + ds2 = ds2.drop(v) + + # merging by time block + ds1 = xr.concat((ds2.sel( + time=slice(ds2.time.isel(time=0), + ds1.time.isel(time=0)) + ), ds1), dim='time') + + elif os.path.isfile(args.file1): + ds1, name = loadArr(args.file1) + print(f'No historical station for {n1}...') + + else: + print(f'Invalid file {args.file1}') + exit() + all_values = [] + for l in old_name.values(): + for ll in l: + all_values.append(ll) + if n1 in all_values: + print(n1, 'is used as auxilary at another site') + exit() + # Get hourly, daily and monthly datasets + print('Resampling L3 data to hourly, daily and monthly resolutions...') + l3_h = resampleL2(ds1, '60min') + l3_d = resampleL2(ds1, '1D') + l3_m = resampleL2(ds1, 'M') + + print(f'Adding variable information from {args.variables}...') + + # Load variables look-up table + var = getVars(args.variables) + + # Round all values to specified decimals places + l3_h = roundValues(l3_h, var) + l3_d = roundValues(l3_d, var) + l3_m = roundValues(l3_m, var) + + # Get columns to keep + if hasattr(ds1, 'p_l'): + col_names = getColNames(var, 2, args.datatype.lower()) + else: + col_names = getColNames(var, 1, args.datatype.lower()) + + # Assign site id + site_id = n1.replace('v3','').replace('CEN2','CEN') + for l in [l3_h, l3_d, l3_m]: + l.attrs['site_id'] = site_id + l.attrs['station_id'] = site_id + if n1 in old_name.keys(): + l.attrs['list_station_id'] = '('+n1+', '+', '.join(old_name[n1])+')' + else: + l.attrs['list_station_id'] = '('+n1+')' + + # Define input path + station_name = args.config_file.split('/')[-1].split('.')[0] + station_path = os.path.join(args.inpath, station_name) + if os.path.exists(station_path): + aws = AWS(args.config_file, station_path, v, m) + else: + aws = AWS(args.config_file, args.inpath, v, m) + + # Perform level 1 to 3 processing + aws.getL1() + aws.getL2() + aws.getL3() + + # Write out Level 3 + if args.outpath is not None: + aws.writeL3(args.outpath) + +if __name__ == "__main__": + get_l3() + From e9144039d00c610349f1711fc83a47438dfded4e Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:13:07 +0200 Subject: [PATCH 04/36] update get_l3 to add historical data --- setup.py | 2 +- .../variable_aliases_GC-Net - Kopi.csv | 78 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 src/pypromice/process/variable_aliases_GC-Net - Kopi.csv diff --git a/setup.py b/setup.py index 5675d6f6..761d2bee 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", package_data={ - "pypromice.process": ["metadata.csv", "variables.csv"], + "pypromice.process": ["metadata.csv", "variables.csv", "variable_aliases_GC-Net.csv"], "pypromice.tx": ["payload_formats.csv", "payload_types.csv"], "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], diff --git a/src/pypromice/process/variable_aliases_GC-Net - Kopi.csv b/src/pypromice/process/variable_aliases_GC-Net - Kopi.csv new file mode 100644 index 00000000..92704ab8 --- /dev/null +++ b/src/pypromice/process/variable_aliases_GC-Net - Kopi.csv @@ -0,0 +1,78 @@ +GEUS_name,old_name +time,timestamp +p_u,P +p_l, +t_u,TA2 +t_l,TA1 +rh_u,RH2 +rh_u_cor,RH2_cor +qh_u,Q2 +rh_l,RH1 +rh_l_cor,RH1_cor +qh_l,Q1 +wspd_u,VW2 +wspd_l,VW1 +wdir_u,DW2 +wdir_l,DW1 +dsr, +dsr_cor,ISWR +usr, +usr_cor,OSWR +albedo,Alb +dlr, +ulr, +cc, +t_surf, +dlhf_u,LHF +dlhf_l, +dshf_u,SHF +dshf_l, +z_boom_u,HW2 +z_boom_l,HW1 +precip_u, +precip_u_cor, +precip_l, +precip_l_cor, +t_i_1,TS1 +t_i_2,TS2 +t_i_3,TS3 +t_i_4,TS4 +t_i_5,TS5 +t_i_6,TS6 +t_i_7,TS7 +t_i_8,TS8 +t_i_9,TS9 +t_i_10,TS10 +t_i_11, +tilt_x, +tilt_y, +rot, +gps_lat,latitude +gps_lon,longitude +gps_alt,elevation +gps_time, +gps_geounit, +gps_hdop, +batt_v,V +fan_dc_u, +fan_dc_l, +t_rad, +msg_lat, +msg_lon, +z_surf_1,HS1 +z_surf_2,HS2 +z_surf_1_adj_flag,HS1_adj_flag +z_surf_2_adj_flag,HS2_adj_flag +z_surf_combined,HS_combined +depth_t_i_1,DTS1 +depth_t_i_2,DTS2 +depth_t_i_3,DTS3 +depth_t_i_4,DTS4 +depth_t_i_5,DTS5 +depth_t_i_6,DTS6 +depth_t_i_7,DTS7 +depth_t_i_8,DTS8 +depth_t_i_9,DTS9 +depth_t_i_10,DTS10 +depth_t_i_11, +t_i_10m,TS_10m From a8b37ec81dc93aa76c0d52b3330bfb71db1170fe Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:13:07 +0200 Subject: [PATCH 05/36] update get_l3 to add historical data --- src/pypromice/process/get_l3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypromice/process/get_l3.py b/src/pypromice/process/get_l3.py index 688ada59..3c19c10b 100644 --- a/src/pypromice/process/get_l3.py +++ b/src/pypromice/process/get_l3.py @@ -42,7 +42,7 @@ def get_l3(): m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv') else: m = args.metadata - + # Define input path station_name = args.config_file.split('/')[-1].split('.')[0] station_path = os.path.join(args.inpath, station_name) From e2760643665f71e4ab217d000e91efbe2b8abb68 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Wed, 12 Jun 2024 12:49:52 +0200 Subject: [PATCH 06/36] Create get_l3_new.py --- src/pypromice/process/get_l3_new.py | 295 ++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 src/pypromice/process/get_l3_new.py diff --git a/src/pypromice/process/get_l3_new.py b/src/pypromice/process/get_l3_new.py new file mode 100644 index 00000000..8272b51b --- /dev/null +++ b/src/pypromice/process/get_l3_new.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python +import logging, os, sys, unittest +from argparse import ArgumentParser +import pypromice +from pypromice.process.aws import AWS + +old_name = { + 'CEN2': ['GITS'], + 'CP1': ['CrawfordPoint1'], + 'DY2': ['DYE-2'], + 'JAR': ['JAR1'], + 'HUM': ['Humboldt'], + 'NAU': ['NASA-U'], + 'NAE': ['NASA-E'], + 'NEM': ['NEEM'], + 'NSE': ['NASA-SE'], + 'EGP': ['EastGRIP'], + 'SDL': ['Saddle'], + 'SDM': ['SouthDome'], + 'SWC': ['SwissCamp', 'SwissCamp10m'], + 'TUN': ['Tunu-N'], + } + +def parse_arguments_getl3(debug_args=None): + parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") + parser.add_argument('-s', '--file1', type=str, required=True, nargs='+', + help='Path to source L2 file') + parser.add_argument('-g', '--folder_gcnet', type=str, required=True, nargs='+', + help='Path to GC-Net historical L1 folder') + parser.add_argument('-p', '--folder_l2', type=str, required=True, nargs='+', + help='Path to level 2 folder') + parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=True, + help='Path where to write output') + parser.add_argument('-v', '--variables', default=None, type=str, required=False, + help='Path to variables look-up table .csv file for variable name retained'''), + parser.add_argument('-m', '--metadata', default=None, type=str, required=False, + help='Path to metadata table .csv file for metadata information'''), + parser.add_argument('-d', '--datatype', default='raw', type=str, required=False, + help='Data type to output, raw or tx') + args = parser.parse_args(args=debug_args) + args.file1 = ' '.join(args.file1) + args.folder_gcnet = ' '.join(args.folder_gcnet) + args.folder_l2 = ' '.join(args.folder_l2) + return args + +def readNead(infile): + with open(infile) as f: + fmt = f.readline() + assert(fmt[0] == "#") + assert(fmt.split("#")[1].split()[0] == "NEAD") + assert(fmt.split("#")[1].split()[1] == "1.0") + assert(fmt.split("#")[1].split()[2] == "UTF-8") + + line = f.readline() + assert(line[0] == "#") + assert(line.split("#")[1].strip() == '[METADATA]') + + meta = {} + fields = {} + section = 'meta' + while True: + line = f.readline() + if line.strip(' ') == '#': continue + if line == "# [DATA]\n": break # done reading header + if line == "# [FIELDS]\n": + section = 'fields' + continue # done reading header + + if line[0] == "\n": continue # blank line + assert(line[0] == "#") # if not blank, must start with "#" + + key_eq_val = line.split("#")[1].strip() + if key_eq_val == '' or key_eq_val == None: continue # Line is just "#" or "# " or "# #"... + assert("=" in key_eq_val), print(line, key_eq_val) + key = key_eq_val.split("=")[0].strip() + val = key_eq_val.split("=")[1].strip() + + # Convert from string to number if it is a number + if val.strip('-').strip('+').replace('.','').isdigit(): + val = float(val) + if val == int(val): + val = int(val) + + if section == 'meta': meta[key] = val + if section == 'fields': fields[key] = val + # done reading header + + # Find delimiter and fields for reading NEAD as simple CSV + assert("field_delimiter" in meta.keys()) + assert("fields" in fields.keys()) + FD = meta["field_delimiter"] + names = [_.strip() for _ in fields.pop('fields').split(FD)] + + df = pd.read_csv(infile, + comment = "#", + names = names, + sep = FD, + usecols=np.arange(len(names)), + skip_blank_lines = True) + df['timestamp'] = pd.to_datetime(df.timestamp).dt.tz_localize(None) + df = df.set_index('timestamp') + ds = df.to_xarray() + ds.attrs = meta + return ds + + +def loadArr(infile): + if infile.split('.')[-1].lower() in 'csv': + with open(infile) as f: + text_splitted = f.read().splitlines() + first_char = text_splitted[0][0] + + if first_char != '#': + df = pd.read_csv(infile) + df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None) + df = df.set_index('time') + ds = xr.Dataset.from_dataframe(df) + else: + ds = readNead(infile) + f.close() + elif infile.split('.')[-1].lower() in 'nc': + ds = xr.open_dataset(infile) + + try: + name = ds.attrs['station_name'] + except: + name = infile.split('/')[-1].split('.')[0].split('_hour')[0].split('_10min')[0] + + print(f'{name} array loaded from {infile}') + return ds, name + + +def gcnet_postprocessing(ds2): + file_path = pkg_resources.resource_stream('pypromice', 'process/variable_aliases_GC-Net.csv') + + var_name = pd.read_csv(file_path) + var_name = var_name.set_index('old_name').GEUS_name + msk = [v for v in var_name.index if v in ds2.data_vars] + var_name = var_name.loc[msk].to_dict() + + ds2['TA1'] = ds2['TA1'].combine_first(ds2['TA3']) + ds2['TA2'] = ds2['TA2'].combine_first(ds2['TA4']) + + ds2=ds2.rename(var_name) + ds2=ds2.rename({'timestamp':'time'}) + return ds2 + +# will be used in the future +# def aligning_surface_heights(ds1, ds2): + # df_aux['z_surf_combined'] = \ + # df_aux['z_surf_combined'] \ + # - df_aux.loc[df_aux.z_surf_combined.last_valid_index(), 'z_surf_combined'] \ + # + df_v6.loc[df_v6.z_surf_combined.first_valid_index(), 'z_surf_combined'] + + # if s == 'Swiss Camp 10m': + # df.loc[:df.HS_combined.first_valid_index(), 'HS_combined'] = \ + # df2.loc[:df.HS_combined.first_valid_index(), 'HS_combined'] \ + # - df2.loc[df2.HS_combined.last_valid_index(), 'HS_combined'] \ + # + df.loc[df.HS_combined.first_valid_index(), 'HS_combined'] + + + # df.loc[df.HS_combined.diff()==0,'HS_combined'] = np.nan + + # fit = np.polyfit(df.loc[df.HS_combined.notnull(),:].index.astype('int64'), + # df.loc[df.HS_combined.notnull(),'HS_combined'], 1) + # fit_fn = np.poly1d(fit) + + # df['HS_combined'] = df['HS_combined'].values \ + # - fit_fn( + # df_in.loc[[df_in.z_surf_combined.first_valid_index()],:].index.astype('int64')[0] + # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] + # return ds1 + +def get_l3(): + args = parse_arguments_getl3() + + # Check files + if os.path.isfile(args.file1): + # Load L2 data arrays + ds1, n1 = loadArr(args.file1) + + # converts to L3: + # - derives sensible heat fluxes + # - appends historical data + + ds1 = toL3(ds1) + + if n1 in old_name.keys(): + n2 = old_name[n1] + # for all secondary stations + for n in n2: + # loading secondary file, either from GC-Net or PROMICE folders + file2 = args.folder_gcnet+n+'.csv' + is_gcnet = True + if not os.path.isfile(file2): + file2 = args.folder_l2+'/'+n+'/'+n+'_hour.csv' + if not os.path.isfile(file2): + print('could not find',n, 'in',args.folder_gcnet,'nor',args.folder_l2) + continue + else: + is_gcnet = False + + ds2, n2 = loadArr(file2) + + if is_gcnet: + ds2 = gcnet_postprocessing(ds2) + ds2 = ds2[[v for v in ds1.data_vars if v in ds2.data_vars]] + else: + # then it is a GEUS L2 file that needs to be processed into L3 + # converts to L3 + ds2 = toL3(ds2) + + # prepairing for the merging + for v in ds1.data_vars: + if v not in ds2.data_vars: + ds2[v] = ds2.t_u*np.nan + + print('dropping') + for v in ds2.data_vars: + if v not in ds1.data_vars: + print(v) + ds2 = ds2.drop(v) + + # merging by time block + ds1 = xr.concat((ds2.sel( + time=slice(ds2.time.isel(time=0), + ds1.time.isel(time=0)) + ), ds1), dim='time') + + elif os.path.isfile(args.file1): + ds1, name = loadArr(args.file1) + print(f'No historical station for {n1}...') + + else: + print(f'Invalid file {args.file1}') + exit() + all_values = [] + for l in old_name.values(): + for ll in l: + all_values.append(ll) + if n1 in all_values: + print(n1, 'is used as auxilary at another site') + exit() + # Get hourly, daily and monthly datasets + print('Resampling L3 data to hourly, daily and monthly resolutions...') + l3_h = resampleL2(ds1, '60min') + l3_d = resampleL2(ds1, '1D') + l3_m = resampleL2(ds1, 'M') + + print(f'Adding variable information from {args.variables}...') + + # Load variables look-up table + var = getVars(args.variables) + + # Round all values to specified decimals places + l3_h = roundValues(l3_h, var) + l3_d = roundValues(l3_d, var) + l3_m = roundValues(l3_m, var) + + # Get columns to keep + if hasattr(ds1, 'p_l'): + col_names = getColNames(var, 2, args.datatype.lower()) + else: + col_names = getColNames(var, 1, args.datatype.lower()) + + # Assign site id + site_id = n1.replace('v3','').replace('CEN2','CEN') + for l in [l3_h, l3_d, l3_m]: + l.attrs['site_id'] = site_id + l.attrs['station_id'] = site_id + if n1 in old_name.keys(): + l.attrs['list_station_id'] = '('+n1+', '+', '.join(old_name[n1])+')' + else: + l.attrs['list_station_id'] = '('+n1+')' + + # Define input path + station_name = args.config_file.split('/')[-1].split('.')[0] + station_path = os.path.join(args.inpath, station_name) + if os.path.exists(station_path): + aws = AWS(args.config_file, station_path, v, m) + else: + aws = AWS(args.config_file, args.inpath, v, m) + + # Perform level 1 to 3 processing + aws.getL1() + aws.getL2() + aws.getL3() + + # Write out Level 3 + if args.outpath is not None: + aws.writeL3(args.outpath) + +if __name__ == "__main__": + get_l3() + From ef4ae44a0e67d5a2d6081f025d1852deaf666fe3 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:50:41 +0200 Subject: [PATCH 07/36] further work on join_l3, varible_aliases in ressource folder --- setup.py | 3 +- src/pypromice/process/join_l2.py | 2 +- .../process/{get_l3_new.py => join_l3.py} | 264 ++++++++---------- src/pypromice/ressources/__init__.py | 0 .../variable_aliases_GC-Net.csv} | 0 5 files changed, 120 insertions(+), 149 deletions(-) rename src/pypromice/process/{get_l3_new.py => join_l3.py} (53%) create mode 100644 src/pypromice/ressources/__init__.py rename src/pypromice/{process/variable_aliases_GC-Net - Kopi.csv => ressources/variable_aliases_GC-Net.csv} (100%) diff --git a/setup.py b/setup.py index 761d2bee..e3545430 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,8 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", package_data={ - "pypromice.process": ["metadata.csv", "variables.csv", "variable_aliases_GC-Net.csv"], + "pypromice.ressources": ["variable_aliases_GC-Net.csv"], + "pypromice.process": ["metadata.csv", "variables.csv"], "pypromice.tx": ["payload_formats.csv", "payload_types.csv"], "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index 28bf2e0e..4ba4f657 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -101,4 +101,4 @@ def join_l2(): print(f'Files saved to {os.path.join(args.outpath, name)}...') if __name__ == "__main__": - join_levels() + join_l2() diff --git a/src/pypromice/process/get_l3_new.py b/src/pypromice/process/join_l3.py similarity index 53% rename from src/pypromice/process/get_l3_new.py rename to src/pypromice/process/join_l3.py index 8272b51b..70f09dac 100644 --- a/src/pypromice/process/get_l3_new.py +++ b/src/pypromice/process/join_l3.py @@ -1,36 +1,30 @@ #!/usr/bin/env python -import logging, os, sys, unittest +import logging, os, sys, unittest, toml, pkg_resources from argparse import ArgumentParser import pypromice from pypromice.process.aws import AWS +from pypromice.process import load +from pypromice.process.write import prepare_and_write +import numpy as np +import pandas as pd +import xarray as xr +from pypromice.process.load import getVars, getMeta -old_name = { - 'CEN2': ['GITS'], - 'CP1': ['CrawfordPoint1'], - 'DY2': ['DYE-2'], - 'JAR': ['JAR1'], - 'HUM': ['Humboldt'], - 'NAU': ['NASA-U'], - 'NAE': ['NASA-E'], - 'NEM': ['NEEM'], - 'NSE': ['NASA-SE'], - 'EGP': ['EastGRIP'], - 'SDL': ['Saddle'], - 'SDM': ['SouthDome'], - 'SWC': ['SwissCamp', 'SwissCamp10m'], - 'TUN': ['Tunu-N'], - } - -def parse_arguments_getl3(debug_args=None): +def parse_arguments_joinl3(debug_args=None): parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") - parser.add_argument('-s', '--file1', type=str, required=True, nargs='+', - help='Path to source L2 file') - parser.add_argument('-g', '--folder_gcnet', type=str, required=True, nargs='+', - help='Path to GC-Net historical L1 folder') - parser.add_argument('-p', '--folder_l2', type=str, required=True, nargs='+', + parser.add_argument('-c', '--config_folder', type=str, required=True, + help='Path to folder with sites configuration (TOML) files') + parser.add_argument('-s', '--site', default=None, type=str, required=False, + help='Name of site to process (default: all sites are processed)') + + parser.add_argument('-p', '--folder_l2', type=str, required=True, help='Path to level 2 folder') + parser.add_argument('-g', '--folder_gcnet', type=str, required=False, + help='Path to GC-Net historical L1 folder') + parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=True, help='Path where to write output') + parser.add_argument('-v', '--variables', default=None, type=str, required=False, help='Path to variables look-up table .csv file for variable name retained'''), parser.add_argument('-m', '--metadata', default=None, type=str, required=False, @@ -38,9 +32,6 @@ def parse_arguments_getl3(debug_args=None): parser.add_argument('-d', '--datatype', default='raw', type=str, required=False, help='Data type to output, raw or tx') args = parser.parse_args(args=debug_args) - args.file1 = ' '.join(args.file1) - args.folder_gcnet = ' '.join(args.folder_gcnet) - args.folder_l2 = ' '.join(args.folder_l2) return args def readNead(infile): @@ -130,23 +121,23 @@ def loadArr(infile): return ds, name -def gcnet_postprocessing(ds2): - file_path = pkg_resources.resource_stream('pypromice', 'process/variable_aliases_GC-Net.csv') +def gcnet_postprocessing(l3): + file_path = pkg_resources.resource_stream('pypromice', 'ressources/variable_aliases_GC-Net.csv') var_name = pd.read_csv(file_path) var_name = var_name.set_index('old_name').GEUS_name - msk = [v for v in var_name.index if v in ds2.data_vars] + msk = [v for v in var_name.index if v in l3.data_vars] var_name = var_name.loc[msk].to_dict() - ds2['TA1'] = ds2['TA1'].combine_first(ds2['TA3']) - ds2['TA2'] = ds2['TA2'].combine_first(ds2['TA4']) + l3['TA1'] = l3['TA1'].combine_first(l3['TA3']) + l3['TA2'] = l3['TA2'].combine_first(l3['TA4']) - ds2=ds2.rename(var_name) - ds2=ds2.rename({'timestamp':'time'}) - return ds2 + l3=l3.rename(var_name) + l3=l3.rename({'timestamp':'time'}) + return l3 # will be used in the future -# def aligning_surface_heights(ds1, ds2): +# def aligning_surface_heights(l3m, l3): # df_aux['z_surf_combined'] = \ # df_aux['z_surf_combined'] \ # - df_aux.loc[df_aux.z_surf_combined.last_valid_index(), 'z_surf_combined'] \ @@ -169,127 +160,106 @@ def gcnet_postprocessing(ds2): # - fit_fn( # df_in.loc[[df_in.z_surf_combined.first_valid_index()],:].index.astype('int64')[0] # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] - # return ds1 + # return l3m -def get_l3(): - args = parse_arguments_getl3() - - # Check files - if os.path.isfile(args.file1): - # Load L2 data arrays - ds1, n1 = loadArr(args.file1) +def join_l3(): + args = parse_arguments_joinl3() + _join_l3() + +def _join_l3(config_folder='C:/Users/bav/GitHub/PROMICE data/aws-l0/configurations/sites', + site='CEN', + folder_l3='C:/Users/bav/GitHub/PROMICE data/aws-l3-dev/level_3', + folder_gcnet='C:/Users/bav/OneDrive - GEUS/Code/PROMICE/GC-Net-Level-1-data-processing/L1/hourly', + outpath='level_3_merged', variables=None, metadata=None, datatype=None): + # %% + config_file = os.path.join(config_folder,site+'.toml') + conf = toml.load(config_file) + plt.figure() + l3m = xr.Dataset() + for stid in conf['list_station_id']: + print(stid) - # converts to L3: - # - derives sensible heat fluxes - # - appends historical data + is_promice = False + is_gcnet = False + filepath = os.path.join(folder_l3, stid, stid+'_hour.nc') + if os.path.isfile(filepath): + is_promice = True + else: + filepath = os.path.join(folder_gcnet, stid+'.csv') + if os.path.isfile(filepath): + is_gcnet = True + if not is_promice and not is_gcnet: + print(stid, 'not found either in', folder_l3, 'or', folder_gcnet) + continue + + l3, _ = loadArr(filepath) - ds1 = toL3(ds1) + if is_gcnet: + l3 = gcnet_postprocessing(l3) - if n1 in old_name.keys(): - n2 = old_name[n1] - # for all secondary stations - for n in n2: - # loading secondary file, either from GC-Net or PROMICE folders - file2 = args.folder_gcnet+n+'.csv' - is_gcnet = True - if not os.path.isfile(file2): - file2 = args.folder_l2+'/'+n+'/'+n+'_hour.csv' - if not os.path.isfile(file2): - print('could not find',n, 'in',args.folder_gcnet,'nor',args.folder_l2) - continue - else: - is_gcnet = False - - ds2, n2 = loadArr(file2) - - if is_gcnet: - ds2 = gcnet_postprocessing(ds2) - ds2 = ds2[[v for v in ds1.data_vars if v in ds2.data_vars]] - else: - # then it is a GEUS L2 file that needs to be processed into L3 - # converts to L3 - ds2 = toL3(ds2) + # lat, lon and alt should be just variables, not coordinates + if 'lat' in l3.keys(): + l3 = l3.reset_coords(['lat', 'lon', 'alt']) + + if len(l3m)==0: + # saving attributes of station under an attribute called $stid + l3m.attrs[stid] = l3.attrs.copy() + # then stripping attributes + attrs_list = list(l3.attrs.keys()) + for k in attrs_list: + del l3.attrs[k] - # prepairing for the merging - for v in ds1.data_vars: - if v not in ds2.data_vars: - ds2[v] = ds2.t_u*np.nan - - print('dropping') - for v in ds2.data_vars: - if v not in ds1.data_vars: + l3m = l3.copy() + l3.t_u.plot(ax=plt.gca()) + else: + # if l3 (older data) is missing variables compared to l3m (newer data) + # , then we fill them with nan + for v in l3m.data_vars: + if v not in l3.data_vars: + l3[v] = l3.t_u*np.nan + + # if l3 (older data) has variables that does not have l3m (newer data) + # then they are removed from l3 + print('dropping') + for v in l3.data_vars: + if v not in l3m.data_vars: + if v != 'z_stake': print(v) - ds2 = ds2.drop(v) + l3 = l3.drop(v) + - # merging by time block - ds1 = xr.concat((ds2.sel( - time=slice(ds2.time.isel(time=0), - ds1.time.isel(time=0)) - ), ds1), dim='time') + l3.sel( + time=slice(l3.time.isel(time=0), + l3m.time.isel(time=0))).t_u.plot(ax=plt.gca()) - elif os.path.isfile(args.file1): - ds1, name = loadArr(args.file1) - print(f'No historical station for {n1}...') - - else: - print(f'Invalid file {args.file1}') - exit() - all_values = [] - for l in old_name.values(): - for ll in l: - all_values.append(ll) - if n1 in all_values: - print(n1, 'is used as auxilary at another site') - exit() - # Get hourly, daily and monthly datasets - print('Resampling L3 data to hourly, daily and monthly resolutions...') - l3_h = resampleL2(ds1, '60min') - l3_d = resampleL2(ds1, '1D') - l3_m = resampleL2(ds1, 'M') - - print(f'Adding variable information from {args.variables}...') - - # Load variables look-up table - var = getVars(args.variables) - - # Round all values to specified decimals places - l3_h = roundValues(l3_h, var) - l3_d = roundValues(l3_d, var) - l3_m = roundValues(l3_m, var) - - # Get columns to keep - if hasattr(ds1, 'p_l'): - col_names = getColNames(var, 2, args.datatype.lower()) - else: - col_names = getColNames(var, 1, args.datatype.lower()) + # saving attributes of station under an attribute called $stid + l3m = l3m.assign_attrs({stid : l3.attrs.copy()}) + # then stripping attributes + attrs_list = list(l3.attrs.keys()) + for k in attrs_list: + del l3.attrs[k] + + l3m.attrs[stid]['first_timestamp'] = l3.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + l3m.attrs[stid]['last_timestamp'] = l3m.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() - # Assign site id - site_id = n1.replace('v3','').replace('CEN2','CEN') - for l in [l3_h, l3_d, l3_m]: - l.attrs['site_id'] = site_id - l.attrs['station_id'] = site_id - if n1 in old_name.keys(): - l.attrs['list_station_id'] = '('+n1+', '+', '.join(old_name[n1])+')' - else: - l.attrs['list_station_id'] = '('+n1+')' - - # Define input path - station_name = args.config_file.split('/')[-1].split('.')[0] - station_path = os.path.join(args.inpath, station_name) - if os.path.exists(station_path): - aws = AWS(args.config_file, station_path, v, m) - else: - aws = AWS(args.config_file, args.inpath, v, m) + # merging by time block + l3m = xr.concat((l3.sel( + time=slice(l3.time.isel(time=0), + l3m.time.isel(time=0)) + ), l3m), dim='time') + - # Perform level 1 to 3 processing - aws.getL1() - aws.getL2() - aws.getL3() - - # Write out Level 3 - if args.outpath is not None: - aws.writeL3(args.outpath) + # Assign site id + l3m.attrs['site_id'] = site + l3m.attrs['list_station_id'] = conf['list_station_id'] + v = getVars() + m = getMeta() + if outpath is not None: + prepare_and_write(l3m, outpath, v, m, '60min') + prepare_and_write(l3m, outpath, v, m, '1D') + prepare_and_write(l3m, outpath, v, m, 'M') + # %% if __name__ == "__main__": - get_l3() + join_l3() diff --git a/src/pypromice/ressources/__init__.py b/src/pypromice/ressources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pypromice/process/variable_aliases_GC-Net - Kopi.csv b/src/pypromice/ressources/variable_aliases_GC-Net.csv similarity index 100% rename from src/pypromice/process/variable_aliases_GC-Net - Kopi.csv rename to src/pypromice/ressources/variable_aliases_GC-Net.csv From b0ceb916898c200594ee145dd60f115b6c41d0d0 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:53:12 +0200 Subject: [PATCH 08/36] cleaning up debug code in join_l3 --- src/pypromice/process/join_l3.py | 33 ++++++++++---------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 70f09dac..e005892e 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -164,32 +164,25 @@ def gcnet_postprocessing(l3): def join_l3(): args = parse_arguments_joinl3() - _join_l3() -def _join_l3(config_folder='C:/Users/bav/GitHub/PROMICE data/aws-l0/configurations/sites', - site='CEN', - folder_l3='C:/Users/bav/GitHub/PROMICE data/aws-l3-dev/level_3', - folder_gcnet='C:/Users/bav/OneDrive - GEUS/Code/PROMICE/GC-Net-Level-1-data-processing/L1/hourly', - outpath='level_3_merged', variables=None, metadata=None, datatype=None): - # %% - config_file = os.path.join(config_folder,site+'.toml') + config_file = os.path.join(args.config_folder, args.site+'.toml') conf = toml.load(config_file) - plt.figure() + l3m = xr.Dataset() for stid in conf['list_station_id']: print(stid) is_promice = False is_gcnet = False - filepath = os.path.join(folder_l3, stid, stid+'_hour.nc') + filepath = os.path.join(args.folder_l3, stid, stid+'_hour.nc') if os.path.isfile(filepath): is_promice = True else: - filepath = os.path.join(folder_gcnet, stid+'.csv') + filepath = os.path.join(args.folder_gcnet, stid+'.csv') if os.path.isfile(filepath): is_gcnet = True if not is_promice and not is_gcnet: - print(stid, 'not found either in', folder_l3, 'or', folder_gcnet) + print(stid, 'not found either in', args.folder_l3, 'or', args.folder_gcnet) continue l3, _ = loadArr(filepath) @@ -210,7 +203,6 @@ def _join_l3(config_folder='C:/Users/bav/GitHub/PROMICE data/aws-l0/configuratio del l3.attrs[k] l3m = l3.copy() - l3.t_u.plot(ax=plt.gca()) else: # if l3 (older data) is missing variables compared to l3m (newer data) # , then we fill them with nan @@ -227,11 +219,6 @@ def _join_l3(config_folder='C:/Users/bav/GitHub/PROMICE data/aws-l0/configuratio print(v) l3 = l3.drop(v) - - l3.sel( - time=slice(l3.time.isel(time=0), - l3m.time.isel(time=0))).t_u.plot(ax=plt.gca()) - # saving attributes of station under an attribute called $stid l3m = l3m.assign_attrs({stid : l3.attrs.copy()}) # then stripping attributes @@ -250,14 +237,14 @@ def _join_l3(config_folder='C:/Users/bav/GitHub/PROMICE data/aws-l0/configuratio # Assign site id - l3m.attrs['site_id'] = site + l3m.attrs['site_id'] = args.site l3m.attrs['list_station_id'] = conf['list_station_id'] v = getVars() m = getMeta() - if outpath is not None: - prepare_and_write(l3m, outpath, v, m, '60min') - prepare_and_write(l3m, outpath, v, m, '1D') - prepare_and_write(l3m, outpath, v, m, 'M') + if args.outpath is not None: + prepare_and_write(l3m, args.outpath, v, m, '60min') + prepare_and_write(l3m, args.outpath, v, m, '1D') + prepare_and_write(l3m, args.outpath, v, m, 'M') # %% if __name__ == "__main__": From a2fa829345582bb098e206bf722067e8cae02dd3 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Thu, 13 Jun 2024 22:05:00 +0200 Subject: [PATCH 09/36] small fix in join_l3 --- src/pypromice/process/join_l3.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index e005892e..a24189f3 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -17,9 +17,9 @@ def parse_arguments_joinl3(debug_args=None): parser.add_argument('-s', '--site', default=None, type=str, required=False, help='Name of site to process (default: all sites are processed)') - parser.add_argument('-p', '--folder_l2', type=str, required=True, - help='Path to level 2 folder') - parser.add_argument('-g', '--folder_gcnet', type=str, required=False, + parser.add_argument('-l3', '--folder_l3', type=str, required=True, + help='Path to level 3 folder') + parser.add_argument('-gc', '--folder_gcnet', type=str, required=False, help='Path to GC-Net historical L1 folder') parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=True, @@ -167,9 +167,14 @@ def join_l3(): config_file = os.path.join(args.config_folder, args.site+'.toml') conf = toml.load(config_file) + + if not site: + site_list=conf['list_station_id'] + else: + site_list=site l3m = xr.Dataset() - for stid in conf['list_station_id']: + for stid in site_list: print(stid) is_promice = False From 4533b7fffa6ab64382f963e01ef91e676ed7a2bc Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Thu, 13 Jun 2024 23:23:58 +0200 Subject: [PATCH 10/36] working verion --- setup.py | 1 + src/pypromice/process/join_l3.py | 16 +++---- src/pypromice/process/utilities.py | 14 ++++-- src/pypromice/process/write.py | 73 ++++++++++++++++++++++-------- 4 files changed, 72 insertions(+), 32 deletions(-) diff --git a/setup.py b/setup.py index e3545430..cb1310cb 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ 'get_promice_data = pypromice.get.get_promice_data:get_promice_data', 'get_l0tx = pypromice.tx.get_l0tx:get_l0tx', 'join_l2 = pypromice.process.join_l2:join_l2', + 'join_l3 = pypromice.process.join_l3:join_l3', 'get_l2 = pypromice.process.get_l2:get_l2', 'get_l3 = pypromice.process.get_l3:get_l3', 'get_l2tol3 = pypromice.process.get_l2tol3:get_l2tol3', diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index a24189f3..f0e7986c 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -165,16 +165,12 @@ def gcnet_postprocessing(l3): def join_l3(): args = parse_arguments_joinl3() + config_file = os.path.join(args.config_folder, args.site+'.toml') conf = toml.load(config_file) - - if not site: - site_list=conf['list_station_id'] - else: - site_list=site l3m = xr.Dataset() - for stid in site_list: + for stid in conf['stations']: print(stid) is_promice = False @@ -221,8 +217,10 @@ def join_l3(): for v in l3.data_vars: if v not in l3m.data_vars: if v != 'z_stake': - print(v) - l3 = l3.drop(v) + print(v) + l3 = l3.drop(v) + else: + l3m[v] = ('time', l3m.t_u.data*np.nan) # saving attributes of station under an attribute called $stid l3m = l3m.assign_attrs({stid : l3.attrs.copy()}) @@ -243,7 +241,7 @@ def join_l3(): # Assign site id l3m.attrs['site_id'] = args.site - l3m.attrs['list_station_id'] = conf['list_station_id'] + l3m.attrs['stations'] = conf['stations'] v = getVars() m = getMeta() if args.outpath is not None: diff --git a/src/pypromice/process/utilities.py b/src/pypromice/process/utilities.py index ad0d66ca..0bad8861 100644 --- a/src/pypromice/process/utilities.py +++ b/src/pypromice/process/utilities.py @@ -41,7 +41,12 @@ def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']): '''Switch gps_lon to negative values (degrees_east). We do this here, and NOT in addMeta, otherwise we switch back to positive when calling getMeta in joinL2''' - if dataset.attrs['station_id'] not in exempt: + if 'station_id' in dataset.attrs.keys(): + id = dataset.attrs['station_id'] + else: + id = dataset.attrs['site_id'] + + if id not in exempt: dataset['gps_lon'] = dataset['gps_lon'] * -1 return dataset @@ -163,7 +168,10 @@ def addMeta(ds, meta): # ds[k].attrs['units'] = 'degrees_C' # https://wiki.esipfed.org/Attribute_Convention_for_Data_Discovery_1-3#geospatial_bounds - ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['station_id'])) + if 'station_id' in ds.attrs.keys(): + ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['station_id'])) + else: + ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['site_id'])) ds.attrs['history'] = 'Generated on ' + datetime.datetime.utcnow().isoformat() ds.attrs['date_created'] = str(datetime.datetime.now().isoformat()) ds.attrs['date_modified'] = ds.attrs['date_created'] @@ -230,4 +238,4 @@ def _addAttr(ds, key, value): pass # logger.info(f'Unable to add metadata to {key.split(".")[0]}') else: - ds.attrs[key] = value \ No newline at end of file + ds.attrs[key] = value diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index 75c36485..b7436556 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -46,34 +46,36 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp # Round all values to specified decimals places d2 = utilities.roundValues(d2, vars_df) - - # Create out directory - outdir = os.path.join(outpath, d2.attrs['station_id']) - if not os.path.isdir(outdir): - os.mkdir(outdir) - + # Get variable names to write out col_names = write.getColNames( vars_df, - d2.attrs['number_of_booms'], - d2.attrs['format'], - d2.attrs['bedrock'], - ) - + d2) + # Define filename based on resample rate t = int(pd.Timedelta((d2['time'][1] - d2['time'][0]).values).total_seconds()) + if 'station_id' in d2.attrs.keys(): + name = d2.attrs['station_id'] + else: + name = d2.attrs['site_id'] + + # Create out directory + outdir = os.path.join(outpath, name) + if not os.path.isdir(outdir): + os.mkdir(outdir) + if t == 600: - out_csv = os.path.join(outdir, d2.attrs['station_id']+'_10min.csv') - out_nc = os.path.join(outdir, d2.attrs['station_id']+'_10min.nc') + out_csv = os.path.join(outdir, name+'_10min.csv') + out_nc = os.path.join(outdir, name+'_10min.nc') elif t == 3600: - out_csv = os.path.join(outdir, d2.attrs['station_id']+'_hour.csv') - out_nc = os.path.join(outdir, d2.attrs['station_id']+'_hour.nc') + out_csv = os.path.join(outdir, name+'_hour.csv') + out_nc = os.path.join(outdir, name+'_hour.nc') elif t == 86400: - out_csv = os.path.join(outdir, d2.attrs['station_id']+'_day.csv') - out_nc = os.path.join(outdir, d2.attrs['station_id']+'_day.nc') + out_csv = os.path.join(outdir, name+'_day.csv') + out_nc = os.path.join(outdir, name+'_day.nc') else: - out_csv = os.path.join(outdir, d2.attrs['station_id']+'_month.csv') - out_nc = os.path.join(outdir, d2.attrs['station_id']+'_month.nc') + out_csv = os.path.join(outdir, name+'_month.csv') + out_nc = os.path.join(outdir, name+'_month.nc') if not os.path.isdir(outdir): os.mkdir(outdir) # Write to csv file @@ -150,7 +152,38 @@ def writeNC(outfile, Lx, col_names=None): names = list(Lx.keys()) Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True) -def getColNames(vars_df, booms=None, data_type=None, bedrock=False): +def getColNames(vars_df, ds): + '''Get all variable names for a given data type, based on a variables + look-up table. This is mainly for exporting purposes + + Parameters + ---------- + vars_df : pd.DataFrame + Variables look-up table + ds: xr.dataset + Dataset to write + Returns + ------- + list + Variable names + ''' + if 'data_type' in ds.attrs.keys(): + if ds.attrs['data_type']=='TX': + vars_df = vars_df.loc[vars_df['data_type'].isin(['TX','all'])] + elif ds.attrs['data_type']=='STM' or ds.attrs['data_type']=='raw': + vars_df = vars_df.loc[vars_df['data_type'].isin(['raw','all'])] + + var_list = list(vars_df.index) + for v in var_list: + if v not in ds.keys(): + var_list.remove(v) + continue + if ds[v].isnull().all(): + var_list.remove(v) + return var_list + + +def getColNames_old(vars_df, booms=None, data_type=None, bedrock=False): '''Get all variable names for a given data type, based on a variables look-up table. This is mainly for exporting purposes From 320ab52210bc649b0354b656278d63c4bbdb3a53 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Fri, 14 Jun 2024 12:54:38 +0200 Subject: [PATCH 11/36] delete encoding info after reading netcdf, debug of getColNames --- src/pypromice/process/aws.py | 33 +---------- src/pypromice/process/get_l2tol3.py | 5 +- src/pypromice/process/join_l2.py | 3 + src/pypromice/process/join_l3.py | 3 + src/pypromice/process/utilities.py | 3 + src/pypromice/process/variables.csv | 2 +- src/pypromice/process/write.py | 89 +++++++++-------------------- 7 files changed, 42 insertions(+), 96 deletions(-) diff --git a/src/pypromice/process/aws.py b/src/pypromice/process/aws.py index 0d830f34..7314a19e 100644 --- a/src/pypromice/process/aws.py +++ b/src/pypromice/process/aws.py @@ -52,9 +52,7 @@ def __init__(self, config_file, inpath, var_file=None, meta_file=None): L0 = self.loadL0() self.L0=[] for l in L0: - n = write.getColNames(self.vars, - l.attrs['number_of_booms'], - l.attrs['format']) + n = write.getColNames(self.vars, l) self.L0.append(utilities.popCols(l, n)) self.L1 = None @@ -106,18 +104,6 @@ def getL3(self): logger.info('Level 3 processing...') self.L3 = toL3(self.L2) - # def resample(self, dataset): - # '''Resample dataset to specific temporal resolution (based on input - # data type)''' - # f = [l.attrs['format'] for l in self.L0] - # if 'raw' in f or 'STM' in f: - # logger.info('Resampling to 10 minute') - # resampled = resample_dataset(dataset, '10min') - # else: - # resampled = resample_dataset(dataset, '60min') - # logger.info('Resampling to hour') - # return resampled - def writeArr(self, dataset, outpath, t=None): '''Write L3 data to .nc and .csv hourly and daily files @@ -141,23 +127,6 @@ def writeArr(self, dataset, outpath, t=None): else: write.prepare_and_write(dataset, outpath, self.vars, self.meta, '60min') - - # def addAttributes(self, dataset): - # '''Add variable and attribute metadata - - # Parameters - # ---------- - # dataset : xr.Dataset - # Dataset (i.e. L2 or L3) object - - # Returns - # ------- - # d2 : xr.Dataset - # Data object with attributes - # ''' - # d2 = utilities.addVars(dataset, self.vars) - # d2 = utilities.addMeta(d2, self.meta) - # return d2 def loadConfig(self, config_file, inpath): '''Load configuration from .toml file diff --git a/src/pypromice/process/get_l2tol3.py b/src/pypromice/process/get_l2tol3.py index e8ed5a96..7fa3e318 100644 --- a/src/pypromice/process/get_l2tol3.py +++ b/src/pypromice/process/get_l2tol3.py @@ -41,7 +41,10 @@ def get_l2tol3(): # Define Level 2 dataset from file with xr.open_dataset(args.inpath) as l2: - l2.load() + l2.load() + for varname in l2.variables: + if 'encoding' in l2[varname].attrs: + del l2[varname].attrs['encoding'] if 'bedrock' in l2.attrs.keys(): l2.attrs['bedrock'] = l2.attrs['bedrock'] == 'True' if 'number_of_booms' in l2.attrs.keys(): diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index 4ba4f657..a01fe8ca 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -30,6 +30,9 @@ def loadArr(infile): elif infile.split('.')[-1].lower() == 'nc': with xr.open_dataset(infile) as ds: ds.load() + for varname in ds.variables: + if 'encoding' in ds[varname].attrs: + del ds[varname].attrs['encoding'] try: name = ds.attrs['station_id'] diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index f0e7986c..778e8741 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -111,6 +111,9 @@ def loadArr(infile): f.close() elif infile.split('.')[-1].lower() in 'nc': ds = xr.open_dataset(infile) + for varname in ds.variables: + if 'encoding' in ds[varname].attrs: + del ds[varname].attrs['encoding'] try: name = ds.attrs['station_name'] diff --git a/src/pypromice/process/utilities.py b/src/pypromice/process/utilities.py index 0bad8861..d0dad5a5 100644 --- a/src/pypromice/process/utilities.py +++ b/src/pypromice/process/utilities.py @@ -47,6 +47,9 @@ def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']): id = dataset.attrs['site_id'] if id not in exempt: + if 'gps_lon' not in dataset.keys(): + print("?????????", id, "missing gps_lon") + return dataset dataset['gps_lon'] = dataset['gps_lon'] * -1 return dataset diff --git a/src/pypromice/process/variables.csv b/src/pypromice/process/variables.csv index 2960259b..29908293 100644 --- a/src/pypromice/process/variables.csv +++ b/src/pypromice/process/variables.csv @@ -64,7 +64,7 @@ rot,platform_azimuth_angle,Station rotation from true North,degrees,0,360,,all,a gps_lat,gps_latitude,Latitude,degrees_north,50,83,,all,all,6,coordinate,time lat lon alt,True, gps_lon,gps_longitude,Longitude,degrees_east,5,70,,all,all,6,coordinate,time lat lon alt,True, gps_alt,gps_altitude,Altitude,m,0,3000,,all,all,2,coordinate,time lat lon alt,True, -gps_time,gps_time,GPS time,s,0,240000,,all,all,,coordinate,time lat lon alt,True, +gps_time,gps_time,GPS time,s,0,240000,,all,all,,physicalMeasurement,time lat lon alt,True, gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,,,,one-boom,all,,physicalMeasurement,time lat lon alt,True, gps_geounit,gps_geounit,GeoUnit,-,,,,all,,,qualityInformation,time lat lon alt,True,L0 only gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,,,,all,all,2,qualityInformation,time lat lon alt,True,NMEA: Horizontal dilution of precision diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index b7436556..5a0252f6 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) from pypromice.process.resample import resample_dataset -from pypromice.process import utilities, write +from pypromice.process import utilities def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resample=True): '''Prepare data with resampling, formating and metadata population; then @@ -37,9 +37,17 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp # Reformat time d2 = utilities.reformat_time(d2) + # finding station/site name + if 'station_id' in d2.attrs.keys(): + name = d2.attrs['station_id'] + else: + name = d2.attrs['site_id'] + # Reformat longitude (to negative values) - d2 = utilities.reformat_lon(d2) - + if 'gps_lon' in d2.keys(): + d2 = utilities.reformat_lon(d2) + else: + logger.info('%s does not have gpd_lon'%name) # Add variable attributes and metadata d2 = utilities.addVars(d2, vars_df) d2 = utilities.addMeta(d2, meta_dict) @@ -48,16 +56,10 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp d2 = utilities.roundValues(d2, vars_df) # Get variable names to write out - col_names = write.getColNames( - vars_df, - d2) + col_names = getColNames(vars_df, d2, remove_nan_fields=True) # Define filename based on resample rate t = int(pd.Timedelta((d2['time'][1] - d2['time'][0]).values).total_seconds()) - if 'station_id' in d2.attrs.keys(): - name = d2.attrs['station_id'] - else: - name = d2.attrs['site_id'] # Create out directory outdir = os.path.join(outpath, name) @@ -80,11 +82,11 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp os.mkdir(outdir) # Write to csv file logger.info('Writing to files...') - write.writeCSV(out_csv, d2, col_names) + writeCSV(out_csv, d2, col_names) # Write to netcdf file col_names = col_names + ['lat', 'lon', 'alt'] - write.writeNC(out_nc, d2, col_names) + writeNC(out_nc, d2, col_names) logger.info(f'Written to {out_csv}') logger.info(f'Written to {out_nc}') @@ -152,7 +154,7 @@ def writeNC(outfile, Lx, col_names=None): names = list(Lx.keys()) Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True) -def getColNames(vars_df, ds): +def getColNames(vars_df, ds, remove_nan_fields=False): '''Get all variable names for a given data type, based on a variables look-up table. This is mainly for exporting purposes @@ -172,55 +174,18 @@ def getColNames(vars_df, ds): vars_df = vars_df.loc[vars_df['data_type'].isin(['TX','all'])] elif ds.attrs['data_type']=='STM' or ds.attrs['data_type']=='raw': vars_df = vars_df.loc[vars_df['data_type'].isin(['raw','all'])] - + if 'number_of_booms' in ds.attrs.keys(): + if ds.attrs['number_of_booms']==1: + vars_df = vars_df.loc[vars_df['station_type'].isin(['one-boom','all'])] + elif ds.attrs['number_of_booms']==2: + vars_df = vars_df.loc[vars_df['station_type'].isin(['two-boom','all'])] var_list = list(vars_df.index) - for v in var_list: - if v not in ds.keys(): - var_list.remove(v) - continue - if ds[v].isnull().all(): - var_list.remove(v) + if remove_nan_fields: + for v in var_list: + if v not in ds.keys(): + var_list.remove(v) + continue + if ds[v].isnull().all(): + var_list.remove(v) return var_list - -def getColNames_old(vars_df, booms=None, data_type=None, bedrock=False): - '''Get all variable names for a given data type, based on a variables - look-up table. This is mainly for exporting purposes - - Parameters - ---------- - vars_df : pd.DataFrame - Variables look-up table - booms : int, optional - Number of booms. If this parameter is empty then all variables - regardless of boom type will be passed. The default is None. - data_type : str, optional - Data type, "tx", "STM" or "raw". If this parameter is empty then all - variables regardless of data type will be passed. The default is None. - - Returns - ------- - list - Variable names - ''' - if booms==1: - vars_df = vars_df.loc[vars_df['station_type'].isin(['one-boom','all'])] - elif booms==2: - vars_df = vars_df.loc[vars_df['station_type'].isin(['two-boom','all'])] - - if data_type=='TX': - vars_df = vars_df.loc[vars_df['data_type'].isin(['TX','all'])] - elif data_type=='STM' or data_type=='raw': - vars_df = vars_df.loc[vars_df['data_type'].isin(['raw','all'])] - - col_names = list(vars_df.index) - if isinstance(bedrock, str): - bedrock = (bedrock.lower() == 'true') - if bedrock == True: - col_names.remove('cc') - for v in ['dlhf_u', 'dlhf_l', 'dshf_u', 'dshf_l']: - try: - col_names.remove(v) - except: - pass - return col_names From 1499e5d40a30eb19a8b92111ea184251854c005b Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Fri, 14 Jun 2024 12:56:16 +0200 Subject: [PATCH 12/36] delete get_l3.py --- src/pypromice/process/get_l3.py | 67 --------------------------------- 1 file changed, 67 deletions(-) delete mode 100644 src/pypromice/process/get_l3.py diff --git a/src/pypromice/process/get_l3.py b/src/pypromice/process/get_l3.py deleted file mode 100644 index 3c19c10b..00000000 --- a/src/pypromice/process/get_l3.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -import logging, os, sys, unittest -from argparse import ArgumentParser -import pypromice -from pypromice.process.aws import AWS - -def parse_arguments_l3(): - parser = ArgumentParser(description="AWS L3 processor") - - parser.add_argument('-c', '--config_file', type=str, required=True, - help='Path to config (TOML) file') - parser.add_argument('-i', '--inpath', type=str, required=True, - help='Path to input data') - parser.add_argument('-o', '--outpath', default=None, type=str, required=False, - help='Path where to write output') - parser.add_argument('-v', '--variables', default=None, type=str, - required=False, help='File path to variables look-up table') - parser.add_argument('-m', '--metadata', default=None, type=str, - required=False, help='File path to metadata') - parser.add_argument('-t', '--time', default=None, type=str, - required=False, help='Resampling frequency') - args = parser.parse_args() - return args - -def get_l3(): - args = parse_arguments_l3() - - logging.basicConfig( - format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", - level=logging.INFO, - stream=sys.stdout, - ) - - # Define variables (either from file or pypromice package defaults) - if args.variables is None: - v = os.path.join(os.path.dirname(pypromice.__file__),'process/variables.csv') - else: - v = args.variables - - # Define metadata (either from file or pypromice package defaults) - if args.variables is None: - m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv') - else: - m = args.metadata - - # Define input path - station_name = args.config_file.split('/')[-1].split('.')[0] - station_path = os.path.join(args.inpath, station_name) - if os.path.exists(station_path): - aws = AWS(args.config_file, station_path, v, m) - else: - aws = AWS(args.config_file, args.inpath, v, m) - - # Perform level 1 to 3 processing - aws.getL1() - aws.getL2() - aws.getL3() - - # Write out level 3 - if args.outpath is not None: - if not os.path.isdir(args.outpath): - os.mkdir(args.outpath) - aws.writeArr(aws.L3, args.outpath, args.time) - -if __name__ == "__main__": - get_l3() - From efbe35f885cdf3c894a480998b4e0f4274cb4822 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:44:33 +0200 Subject: [PATCH 13/36] removing variables and output files metadata --- setup.py | 2 - src/pypromice/process/metadata.csv | 56 ----------- src/pypromice/process/variables.csv | 92 ------------------- src/pypromice/ressources/__init__.py | 0 .../ressources/variable_aliases_GC-Net.csv | 78 ---------------- 5 files changed, 228 deletions(-) delete mode 100644 src/pypromice/process/metadata.csv delete mode 100644 src/pypromice/process/variables.csv delete mode 100644 src/pypromice/ressources/__init__.py delete mode 100644 src/pypromice/ressources/variable_aliases_GC-Net.csv diff --git a/setup.py b/setup.py index cb1310cb..7dcd1309 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,6 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", package_data={ - "pypromice.ressources": ["variable_aliases_GC-Net.csv"], - "pypromice.process": ["metadata.csv", "variables.csv"], "pypromice.tx": ["payload_formats.csv", "payload_types.csv"], "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], diff --git a/src/pypromice/process/metadata.csv b/src/pypromice/process/metadata.csv deleted file mode 100644 index 01b79ff5..00000000 --- a/src/pypromice/process/metadata.csv +++ /dev/null @@ -1,56 +0,0 @@ -attribute,entry -acknowledgements,The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) -alt.axis,Z -alt.coverage_content_type,coordinate -gps_alt.positive,up -cdm_data_type, -comment,https://doi.org/10.22008/promice/data/aws -contributor_name, -contributor_role, -conventions,ACDD-1.3; CF-1.7 -creater_email,pho@geus.dk -creater_url,https://promice.dk -creator_institution,Geological Survey of Denmark and Greenland (GEUS) -creator_name,Penelope How -creator_type,person -featureType,timeSeries -geospatial_bounds_crs,EPSG:4979 -geospatial_lat_extents_match,gps_lat -geospatial_lat_resolution, -geospatial_lat_units,degrees_north -geospatial_lon_extents_match,gps_lon -geospatial_lon_resolution, -geospatial_lon_units,degrees_east -geospatial_vertical_resolution, -geospatial_vertical_units,EPSG:4979 -institution,Geological Survey of Denmark and Greenland (GEUS) -instrument,See https://doi.org/10.5194/essd-13-3819-2021 -instrument_vocabulary,GCMD:GCMD Keywords -keywords,GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > ICE SHEETS > ICE SHEET MEASUREMENTS; GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > GLACIER MASS BALANCE/ICE SHEET MASS BALANCE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE GROWTH/MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC PRESSURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > ALBEDO; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > INCOMING SOLAR RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > DOWNWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > UPWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > NET RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > OUTGOING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FLUX; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FORCING; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION > DOWNWELLING SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SUNSHINE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC TEMPERATURE > SURFACE TEMPERATURE > AIR TEMPERATURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > ABSOLUTE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > RELATIVE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > LOCAL WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > U/V WIND COMPONENTS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND DIRECTION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND SPEED; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > CLOUDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > PRECIPITATION -keywords_vocabulary,GCMDSK:GCMD Science Keywords:https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords -lat.axis,Y -lat.coverage_content_type,coordinate -lat.long_name,station latitude -license,Creative Commons Attribution 4.0 International (CC-BY-4.0) https://creativecommons.org/licenses/by/4.0 -lon.axis,X -lon.coverage_content_type,coordinate -lon.long_name,station longitude -lon.units,degrees_east -metadata_link, -naming_authority,dk.geus.promice -platform, -platform_vocabulary,GCMD:GCMD Keywords -processing_level,Level 3 -product_status,beta -product_version,4 -program,PROMICE -project,PROMICE -publisher_email,info@promice.dk -publisher_institution,GEUS -publisher_name,GEUS -publisher_type,institution -publisher_url,https://promice.dk -references,"How, P.; Abermann, J.; Ahlstrøm, A.P.; Andersen, S.B.; Box, J. E.; Citterio, M.; Colgan, W.T.; Fausto. R.S.; Karlsson, N.B.; Jakobsen, J.; Langley, K.; Larsen, S.H.; Mankoff, K.D.; Pedersen, A.Ø.; Rutishauser, A.; Shield, C.L.; Solgaard, A.M.; van As, D.; Vandecrux, B.; Wright, P.J., 2022, ""PROMICE and GC-Net automated weather station data in Greenland"", https://doi.org/10.22008/FK2/IW73UU, GEUS Dataverse" -references_bib,@article{How2022; doi = {10.22008/FK2/IW73UU}; url = {https://doi.org/10.22008/FK2/IW73UU}; year = {2022}; month=10; publisher= {GEUS Dataverse}; author = {Penelope How and Jakob Abermann and Andreas P. Ahlstr{\o}m and Signe B. Andersen and Jason E. Box and Michele Citterio and William Colgan and Robert S. Fausto and Nanna B. Karlsson and Jakob Jakobsen and Kirsty Langley and Signe Hillerup Larsen and Kenneth D. Mankoff and Allan {\O}. Pedersen and Anja Rutishauser and Christopher L. Shields and Anne M. Solgaard and Dirk van As and Baptiste Vandecrux}; title = {PROMICE and GC-Net automated weather station data in Greenland}; journal = {GEUS Dataverse}} -standard_name_vocabulary,CF Standard Name Table (v77; 19 January 2021) -summary,"The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) and Greenland Climate Network (GC-Net) have been measuring climate and ice sheet properties since 2007 and 1995, respectively. The PROMICE weather station network monitors glacier mass balance in the melt zone of the Greenland Ice Sheet, providing ground truth data to calibrate mass budget models. GC-Net weather stations measure snowfall and surface properties in the accumulation zone, providing valuable knowledge on the Greenland Ice Sheet’s mass gain and climatology.Accurate measurements of the surface and near-surface atmospheric conditions in a changing climate is important for reliable present and future assessment of changes to the Greenland Ice Sheet. All measurements are handled and processed with pypromice, which is a peer-reviewed and freely available Python package with source code available at https://github.com/GEUS-Glaciology-and-Climate/pypromice. A user-contributable dynamic web-based database of known data quality issues is associated with the data products at https://github.com/GEUS-PROMICE/ PROMICE-AWS-data-issues/." diff --git a/src/pypromice/process/variables.csv b/src/pypromice/process/variables.csv deleted file mode 100644 index 29908293..00000000 --- a/src/pypromice/process/variables.csv +++ /dev/null @@ -1,92 +0,0 @@ -field,standard_name,long_name,units,lo,hi,OOL,station_type,data_type,max_decimals,coverage_content_type,coordinates,instantaneous_hourly,comment -time,time,Time,yyyy-mm-dd HH:MM:SS,,,,all,all,,physicalMeasurement,time lat lon alt,, -rec,record,Record,-,,,,all,,0,referenceInformation,time lat lon alt,,L0 only -p_u,air_pressure,Air pressure (upper boom),hPa,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,False, -p_l,air_pressure,Air pressure (lower boom),hPa,650,1100,dshf_l dlhf_l qh_l,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -t_u,air_temperature,Air temperature (upper boom),degrees_C,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,False, -t_l,air_temperature,Air temperature (lower boom),degrees_C,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l ,two-boom,all,4,physicalMeasurement,time lat lon alt,False,PT100 temperature at boom -rh_u,relative_humidity,Relative humidity (upper boom),%,0,100,rh_u_cor,all,all,4,physicalMeasurement,time lat lon alt,False, -rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,0,150,dshf_u dlhf_u qh_u,all,all,4,modelResult,time lat lon alt,False, -qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,0,100,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -rh_l,relative_humidity,Relative humidity (lower boom),%,0,100,rh_l_cor,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,0,150,dshf_l dlhf_l qh_l,two-boom,all,4,modelResult,time lat lon alt,False, -qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,0,100,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -wspd_u,wind_speed,Wind speed (upper boom),m s-1,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,all,4,physicalMeasurement,time lat lon alt,False, -wspd_l,wind_speed,Wind speed (lower boom),m s-1,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,all,4,physicalMeasurement,time lat lon alt,False, -wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,1,360,wspd_x_u wspd_y_u,all,all,4,physicalMeasurement,time lat lon alt,False, -wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,,,,one-boom,,4,qualityInformation,time lat lon alt,False,L0 only -wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,1,360,wspd_x_l wspd_y_l,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,-100,100,wdir_u wspd_u,all,all,4,modelResult,time lat lon alt,False,L0 only -wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,-100,100,wdir_u wspd_u,all,all,4,modelResult,time lat lon alt,False,L0 only -wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,-100,100,wdir_l wspd_l,two-boom,all,4,modelResult,time lat lon alt,False,L0 only -wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,-100,100,wdir_l wspd_l,two-boom,all,4,modelResult,time lat lon alt,False,L0 only -dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,-10,1500,albedo dsr_cor usr_cor,all,all,4,physicalMeasurement,time lat lon alt,False,"Actually radiation_at_sensor, not flux. Units 1E-5 V. Engineering units." -dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,-10,1000,albedo dsr_cor usr_cor,all,all,4,physicalMeasurement,time lat lon alt,False, -usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,0,1000,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -albedo,surface_albedo,Albedo,-,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,50,500,albedo dsr_cor usr_cor cc t_surf,all,all,4,physicalMeasurement,time lat lon alt,False, -ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,50,500,t_surf,all,all,4,physicalMeasurement,time lat lon alt,False, -cc,cloud_area_fraction,Cloud cover,%,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -t_surf,surface_temperature,Surface temperature,C,-80,40,dshf_u dlhf_u qh_u,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,,,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,,,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -z_boom_u,distance_to_surface_from_boom,Upper boom height,m,0.3,10,dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,True, -z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,,,,all,,4,qualityInformation,time lat lon alt,True,L0 only -z_boom_l,distance_to_surface_from_boom,Lower boom height,m,0.3,5,dshf_l dlhf_l qh_l,two-boom,all,4,physicalMeasurement,time lat lon alt,True, -z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,,,,two-boom,,4,qualityInformation,time lat lon alt,True,L0 only -z_stake,distance_to_surface_from_stake_assembly,Stake height,m,0.3,8,,one-boom,all,4,physicalMeasurement,time lat lon alt,True,HeightStakes(m) -z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,,,,one-boom,,4,qualityInformation,time lat lon alt,True,L0 only -z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,0,30,z_pt_cor,one-boom,all,4,physicalMeasurement,time lat lon alt,False,DepthPressureTransducer(m) -z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,0,30,,one-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,0,,precip_u_cor precip_u_rate,all,all,4,physicalMeasurement,time lat lon alt,True,Without wind/undercatch correction -precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,0,,,all,all,4,modelResult,time lat lon alt,True,With wind/undercatch correction -precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,0,,,all,all,4,modelResult,time lat lon alt,True,L0 only -precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,0,,precip_l_cor precip_l_rate,two-boom,all,4,physicalMeasurement,time lat lon alt,True,Without wind/undercatch correction -precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,0,,,two-boom,all,4,modelResult,time lat lon alt,True,With wind/undercatch correction -precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,0,,,two-boom,all,4,modelResult,time lat lon alt,True,L0 only -t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,t1 is installed @ 1 m depth -t_i_2,ice_temperature_at_t2,Ice temperature at sensor 2,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_3,ice_temperature_at_t3,Ice temperature at sensor 3,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_4,ice_temperature_at_t4,Ice temperature at sensor 4,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_5,ice_temperature_at_t5,Ice temperature at sensor 5,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_6,ice_temperature_at_t6,Ice temperature at sensor 6,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_7,ice_temperature_at_t7,Ice temperature at sensor 7,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_8,ice_temperature_at_t8,Ice temperature at sensor 8,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,t8 is installed @ 10 m depth -t_i_9,ice_temperature_at_t9,Ice temperature at sensor 9,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -t_i_10,ice_temperature_at_t10,Ice temperature at sensor 10,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -t_i_11,ice_temperature_at_t11,Ice temperature at sensor 11,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -tilt_x,platform_view_angle_x,Tilt to east,degrees,-30,30,dsr_cor usr_cor albedo,all,all,4,physicalMeasurement,time lat lon alt,False, -tilt_y,platform_view_angle_y,Tilt to north,degrees,-30,30,dsr_cor usr_cor albedo,all,all,4,physicalMeasurement,time lat lon alt,False, -rot,platform_azimuth_angle,Station rotation from true North,degrees,0,360,,all,all,2,physicalMeasurement,time lat lon alt,False,v4 addition -gps_lat,gps_latitude,Latitude,degrees_north,50,83,,all,all,6,coordinate,time lat lon alt,True, -gps_lon,gps_longitude,Longitude,degrees_east,5,70,,all,all,6,coordinate,time lat lon alt,True, -gps_alt,gps_altitude,Altitude,m,0,3000,,all,all,2,coordinate,time lat lon alt,True, -gps_time,gps_time,GPS time,s,0,240000,,all,all,,physicalMeasurement,time lat lon alt,True, -gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,,,,one-boom,all,,physicalMeasurement,time lat lon alt,True, -gps_geounit,gps_geounit,GeoUnit,-,,,,all,,,qualityInformation,time lat lon alt,True,L0 only -gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,,,,all,all,2,qualityInformation,time lat lon alt,True,NMEA: Horizontal dilution of precision -gps_numsat,gps_numsat,GPS number of satellites,-,,,,,all,0,qualityInformation,time lat lon alt,True,L0 only -gps_q,gps_q,Quality,-,,,,,all,,qualityInformation,time lat lon alt,True,L0 only -lat,gps_mean_latitude,GPS mean latitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True, -lon,gps_mean_longitude,GPS mean longitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True, -alt,gps_mean_altitude,GPS mean altitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True, -batt_v,battery_voltage,Battery voltage,V,0,30,,all,all,2,physicalMeasurement,time lat lon alt,True, -batt_v_ini,,,-,0,30,,,all,2,physicalMeasurement,time lat lon alt,True,L0 only -batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,0,30,,,all,2,physicalMeasurement,time lat lon alt,True,L0 only -fan_dc_u,fan_current,Fan current (upper boom),mA,0,200,,all,all,2,physicalMeasurement,time lat lon alt,True, -fan_dc_l,fan_current,Fan current (lower boom),mA,0,200,,two-boom,all,2,physicalMeasurement,time lat lon alt,True, -freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,0,10000,precip_u,,all,,physicalMeasurement,time lat lon alt,True,L0 only -t_log,temperature_of_logger,Logger temperature,degrees_C,-80,40,,one-boom,all,4,physicalMeasurement,time lat lon alt,True,LoggerTemperature(C) -t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,-80,40,t_surf dlr ulr,all,all,4,physicalMeasurement,time lat lon alt,False, -p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,-350,100,,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations -t_i,air_temperature,Air temperature (instantaneous),degrees_C,-80,40,,all,all,4,physicalMeasurement,time lat lon alt,True,"PT100 temperature at boom, for meteorological observations" -rh_i,relative_humidity,Relative humidity (instantaneous),%,0,150,rh_i_cor,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations -rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,0,100,,all,all,4,modelResult,time lat lon alt,True,For meteorological observations -wspd_i,wind_speed,Wind speed (instantaneous),m s-1,0,100,wdir_i wspd_x_i wspd_y_i,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations -wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,1,360,wspd_x_i wspd_y_i,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations -wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,all,4,modelResult,time lat lon alt,True,For meteorological observations -wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,all,4,modelResult,time lat lon alt,True,For meteorological observations -msg_i,message,Message string (instantaneous),-,,,,all,,,qualityInformation,time lat lon alt,True,L0 only diff --git a/src/pypromice/ressources/__init__.py b/src/pypromice/ressources/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/pypromice/ressources/variable_aliases_GC-Net.csv b/src/pypromice/ressources/variable_aliases_GC-Net.csv deleted file mode 100644 index 92704ab8..00000000 --- a/src/pypromice/ressources/variable_aliases_GC-Net.csv +++ /dev/null @@ -1,78 +0,0 @@ -GEUS_name,old_name -time,timestamp -p_u,P -p_l, -t_u,TA2 -t_l,TA1 -rh_u,RH2 -rh_u_cor,RH2_cor -qh_u,Q2 -rh_l,RH1 -rh_l_cor,RH1_cor -qh_l,Q1 -wspd_u,VW2 -wspd_l,VW1 -wdir_u,DW2 -wdir_l,DW1 -dsr, -dsr_cor,ISWR -usr, -usr_cor,OSWR -albedo,Alb -dlr, -ulr, -cc, -t_surf, -dlhf_u,LHF -dlhf_l, -dshf_u,SHF -dshf_l, -z_boom_u,HW2 -z_boom_l,HW1 -precip_u, -precip_u_cor, -precip_l, -precip_l_cor, -t_i_1,TS1 -t_i_2,TS2 -t_i_3,TS3 -t_i_4,TS4 -t_i_5,TS5 -t_i_6,TS6 -t_i_7,TS7 -t_i_8,TS8 -t_i_9,TS9 -t_i_10,TS10 -t_i_11, -tilt_x, -tilt_y, -rot, -gps_lat,latitude -gps_lon,longitude -gps_alt,elevation -gps_time, -gps_geounit, -gps_hdop, -batt_v,V -fan_dc_u, -fan_dc_l, -t_rad, -msg_lat, -msg_lon, -z_surf_1,HS1 -z_surf_2,HS2 -z_surf_1_adj_flag,HS1_adj_flag -z_surf_2_adj_flag,HS2_adj_flag -z_surf_combined,HS_combined -depth_t_i_1,DTS1 -depth_t_i_2,DTS2 -depth_t_i_3,DTS3 -depth_t_i_4,DTS4 -depth_t_i_5,DTS5 -depth_t_i_6,DTS6 -depth_t_i_7,DTS7 -depth_t_i_8,DTS8 -depth_t_i_9,DTS9 -depth_t_i_10,DTS10 -depth_t_i_11, -t_i_10m,TS_10m From 28160e0b8a63beba2ddf357801d5b6e8722e5126 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:54:42 +0200 Subject: [PATCH 14/36] new variable description files --- setup.py | 1 + src/pypromice/ressources/__init__.py | 0 .../ressources/variable_description.csv | 91 +++++++++++++++++++ .../ressources/variable_processing_info.csv | 91 +++++++++++++++++++ 4 files changed, 183 insertions(+) create mode 100644 src/pypromice/ressources/__init__.py create mode 100644 src/pypromice/ressources/variable_description.csv create mode 100644 src/pypromice/ressources/variable_processing_info.csv diff --git a/setup.py b/setup.py index 7dcd1309..7289e381 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", package_data={ + "pypromice.ressources": ["variable_description.csv", "variable_processing_info.csv"], "pypromice.tx": ["payload_formats.csv", "payload_types.csv"], "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], diff --git a/src/pypromice/ressources/__init__.py b/src/pypromice/ressources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pypromice/ressources/variable_description.csv b/src/pypromice/ressources/variable_description.csv new file mode 100644 index 00000000..c8050114 --- /dev/null +++ b/src/pypromice/ressources/variable_description.csv @@ -0,0 +1,91 @@ +field,standard_name,long_name,units,coverage_content_type,coordinates,instantaneous_hourly,where_to_find,comment +time,time,Time,yyyy-mm-dd HH:MM:SS,physicalMeasurement,time lat lon alt,,, +rec,record,Record,-,referenceInformation,time lat lon alt,,L0 or L2, +p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time lat lon alt,FALSE,, +p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time lat lon alt,FALSE,, +t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time lat lon alt,FALSE,, +rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time lat lon alt,FALSE,L2 or later, +qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time lat lon alt,FALSE,L2 or later, +rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time lat lon alt,FALSE,, +rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time lat lon alt,FALSE,L2 or later, +qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,modelResult,time lat lon alt,FALSE,L2 or later, +wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time lat lon alt,FALSE,, +wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time lat lon alt,FALSE,, +wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,physicalMeasurement,time lat lon alt,FALSE,, +wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,qualityInformation,time lat lon alt,FALSE,L0 or L2, +wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,physicalMeasurement,time lat lon alt,FALSE,, +wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2, +wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2, +wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2, +wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2, +dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,, +dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,modelResult,time lat lon alt,FALSE,L2 or later, +usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,, +usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,modelResult,time lat lon alt,FALSE,L2 or later, +albedo,surface_albedo,Albedo,-,modelResult,time lat lon alt,FALSE,L2 or later, +dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,, +ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,, +cc,cloud_area_fraction,Cloud cover,%,modelResult,time lat lon alt,FALSE,L2 or later, +t_surf,surface_temperature,Surface temperature,C,modelResult,time lat lon alt,FALSE,L2 or later, +dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later, +dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later, +dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later, +dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later, +z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time lat lon alt,TRUE,, +z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2, +z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time lat lon alt,TRUE,, +z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2, +z_stake,distance_to_surface_from_stake_assembly,Stake height,m,physicalMeasurement,time lat lon alt,TRUE,, +z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2, +z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time lat lon alt,FALSE,, +z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,modelResult,time lat lon alt,FALSE,L2 or later, +precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time lat lon alt,TRUE,,Without wind/undercatch correction +precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,With wind/undercatch correction +precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later, +precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time lat lon alt,TRUE,,Without wind/undercatch correction +precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,With wind/undercatch correction +precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later, +t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_2,ice_temperature_at_t2,Ice temperature at sensor 2,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_3,ice_temperature_at_t3,Ice temperature at sensor 3,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_4,ice_temperature_at_t4,Ice temperature at sensor 4,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_5,ice_temperature_at_t5,Ice temperature at sensor 5,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_6,ice_temperature_at_t6,Ice temperature at sensor 6,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_7,ice_temperature_at_t7,Ice temperature at sensor 7,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_8,ice_temperature_at_t8,Ice temperature at sensor 8,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_9,ice_temperature_at_t9,Ice temperature at sensor 9,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_10,ice_temperature_at_t10,Ice temperature at sensor 10,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +t_i_11,ice_temperature_at_t11,Ice temperature at sensor 11,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time lat lon alt,FALSE,, +tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time lat lon alt,FALSE,, +rot,platform_azimuth_angle,Station rotation from true North,degrees,physicalMeasurement,time lat lon alt,FALSE,, +gps_lat,gps_latitude,Latitude,degrees_north,coordinate,time lat lon alt,TRUE,, +gps_lon,gps_longitude,Longitude,degrees_east,coordinate,time lat lon alt,TRUE,, +gps_alt,gps_altitude,Altitude,m,coordinate,time lat lon alt,TRUE,, +gps_time,gps_time,GPS time,s,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, +gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, +gps_geounit,gps_geounit,GeoUnit,-,qualityInformation,time lat lon alt,TRUE,L0 or L2, +gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,qualityInformation,time lat lon alt,TRUE,L0 or L2, +gps_numsat,gps_numsat,GPS number of satellites,-,qualityInformation,time lat lon alt,TRUE,L0 or L2, +gps_q,gps_q,Quality,-,qualityInformation,time lat lon alt,TRUE,L0 or L2, +lat,gps_mean_latitude,GPS mean latitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,, +lon,gps_mean_longitude,GPS mean longitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,, +alt,gps_mean_altitude,GPS mean altitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,, +batt_v,battery_voltage,Battery voltage,V,physicalMeasurement,time lat lon alt,TRUE,, +batt_v_ini,,,-,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, +batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, +fan_dc_u,fan_current,Fan current (upper boom),mA,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, +fan_dc_l,fan_current,Fan current (lower boom),mA,physicalMeasurement,time lat lon alt,TRUE,, +freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, +t_log,temperature_of_logger,Logger temperature,degrees_C,physicalMeasurement,time lat lon alt,TRUE,, +t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, +p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,physicalMeasurement,time lat lon alt,TRUE,, +t_i,air_temperature,Air temperature (instantaneous),degrees_C,physicalMeasurement,time lat lon alt,TRUE,, +rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time lat lon alt,TRUE,, +rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,modelResult,time lat lon alt,TRUE,L2 or later, +wspd_i,wind_speed,Wind speed (instantaneous),m s-1,physicalMeasurement,time lat lon alt,TRUE,, +wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,physicalMeasurement,time lat lon alt,TRUE,, +wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time lat lon alt,TRUE,L2 or later, +wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time lat lon alt,TRUE,L2 or later, \ No newline at end of file diff --git a/src/pypromice/ressources/variable_processing_info.csv b/src/pypromice/ressources/variable_processing_info.csv new file mode 100644 index 00000000..0c00932e --- /dev/null +++ b/src/pypromice/ressources/variable_processing_info.csv @@ -0,0 +1,91 @@ +field,lo,hi,OOL,station_type,write_in_l0,write_in_l2,write_in_l3,max_decimals,coverage_content_type,coordinates,instantaneous_hourly,comment +time,,,,all,1,1,1,,physicalMeasurement,time lat lon alt,, +rec,,,,all,1,1,0,0,referenceInformation,time lat lon alt,,L0 only +p_u,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +p_l,650,1100,dshf_l dlhf_l qh_l,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_u,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_l,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,PT100 temperature at boom +rh_u,0,100,rh_u_cor,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +rh_u_cor,0,150,dshf_u dlhf_u qh_u,all,0,1,1,4,modelResult,time lat lon alt,FALSE, +qh_u,0,100,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +rh_l,0,100,rh_l_cor,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +rh_l_cor,0,150,dshf_l dlhf_l qh_l,two-boom,0,1,1,4,modelResult,time lat lon alt,FALSE, +qh_l,0,100,,two-boom,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +wspd_u,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +wspd_l,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +wdir_u,1,360,wspd_x_u wspd_y_u,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +wdir_std_u,,,,one-boom,1,1,0,4,qualityInformation,time lat lon alt,FALSE,L0 only +wdir_l,1,360,wspd_x_l wspd_y_l,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +wspd_x_u,-100,100,wdir_u wspd_u,all,0,1,1,4,modelResult,time lat lon alt,FALSE,L0 only +wspd_y_u,-100,100,wdir_u wspd_u,all,0,1,1,4,modelResult,time lat lon alt,FALSE,L0 only +wspd_x_l,-100,100,wdir_l wspd_l,two-boom,0,1,1,4,modelResult,time lat lon alt,FALSE,L0 only +wspd_y_l,-100,100,wdir_l wspd_l,two-boom,0,1,1,4,modelResult,time lat lon alt,FALSE,L0 only +dsr,-10,1500,albedo dsr_cor usr_cor,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,"Actually radiation_at_sensor, not flux. Units 1E-5 V. Engineering units." +dsr_cor,,,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +usr,-10,1000,albedo dsr_cor usr_cor,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +usr_cor,0,1000,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +albedo,,,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +dlr,50,500,albedo dsr_cor usr_cor cc t_surf,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +ulr,50,500,t_surf,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +cc,,,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +t_surf,-80,40,dshf_u dlhf_u qh_u,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +dlhf_u,,,,all,0,0,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +dlhf_l,,,,two-boom,0,0,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +dshf_u,,,,all,0,0,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +dshf_l,,,,two-boom,0,0,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +z_boom_u,0.3,10,dshf_u dlhf_u qh_u,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE, +z_boom_q_u,,,,all,1,1,0,4,qualityInformation,time lat lon alt,TRUE,L0 only +z_boom_l,0.3,5,dshf_l dlhf_l qh_l,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE, +z_boom_q_l,,,,two-boom,1,1,0,4,qualityInformation,time lat lon alt,TRUE,L0 only +z_stake,0.3,8,,one-boom,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,HeightStakes(m) +z_stake_q,,,,one-boom,1,1,0,4,qualityInformation,time lat lon alt,TRUE,L0 only +z_pt,0,30,z_pt_cor,one-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,DepthPressureTransducer(m) +z_pt_cor,0,30,,one-boom,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) +precip_u,0,,precip_u_cor precip_u_rate,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,Without wind/undercatch correction +precip_u_cor,0,,,all,0,1,1,4,modelResult,time lat lon alt,TRUE,With wind/undercatch correction +precip_u_rate,0,,,all,0,1,1,4,modelResult,time lat lon alt,TRUE,L0 only +precip_l,0,,precip_l_cor precip_l_rate,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,Without wind/undercatch correction +precip_l_cor,0,,,two-boom,0,1,1,4,modelResult,time lat lon alt,TRUE,With wind/undercatch correction +precip_l_rate,0,,,two-boom,0,1,1,4,modelResult,time lat lon alt,TRUE,L0 only +t_i_1,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,t1 is installed @ 1 m depth +t_i_2,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_i_3,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_i_4,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_i_5,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_i_6,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_i_7,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_i_8,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,t8 is installed @ 10 m depth +t_i_9,-80,1,,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_i_10,-80,1,,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +t_i_11,-80,1,,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +tilt_x,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +tilt_y,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +rot,0,360,,all,1,1,1,2,physicalMeasurement,time lat lon alt,FALSE,v4 addition +gps_lat,50,83,,all,1,1,1,6,coordinate,time lat lon alt,TRUE, +gps_lon,5,70,,all,1,1,1,6,coordinate,time lat lon alt,TRUE, +gps_alt,0,3000,,all,1,1,1,2,coordinate,time lat lon alt,TRUE, +gps_time,0,240000,,all,1,1,0,,physicalMeasurement,time lat lon alt,TRUE, +gps_geoid,,,,one-boom,1,1,0,,physicalMeasurement,time lat lon alt,TRUE, +gps_geounit,,,,all,1,1,0,,qualityInformation,time lat lon alt,TRUE,L0 only +gps_hdop,,,,all,1,1,0,2,qualityInformation,time lat lon alt,TRUE,NMEA: Horizontal dilution of precision +gps_numsat,,,,,1,1,0,0,qualityInformation,time lat lon alt,TRUE,L0 only +gps_q,,,,,1,1,0,,qualityInformation,time lat lon alt,TRUE,L0 only +lat,,,,all,1,1,1,6,modelResult,time lat lon alt,TRUE, +lon,,,,all,1,1,1,6,modelResult,time lat lon alt,TRUE, +alt,,,,all,1,1,1,6,modelResult,time lat lon alt,TRUE, +batt_v,0,30,,all,1,1,1,2,physicalMeasurement,time lat lon alt,TRUE, +batt_v_ini,0,30,,,1,1,0,2,physicalMeasurement,time lat lon alt,TRUE,L0 only +batt_v_ss,0,30,,,1,1,0,2,physicalMeasurement,time lat lon alt,TRUE,L0 only +fan_dc_u,0,200,,all,1,1,0,2,physicalMeasurement,time lat lon alt,TRUE, +fan_dc_l,0,200,,two-boom,1,1,0,2,physicalMeasurement,time lat lon alt,TRUE, +freq_vw,0,10000,precip_u,,1,1,0,,physicalMeasurement,time lat lon alt,TRUE,L0 only +t_log,-80,40,,one-boom,1,1,0,4,physicalMeasurement,time lat lon alt,TRUE,LoggerTemperature(C) +t_rad,-80,40,t_surf dlr ulr,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, +p_i,-350,100,,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,For meteorological observations +t_i,-80,40,,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,"PT100 temperature at boom, for meteorological observations" +rh_i,0,150,rh_i_cor,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,For meteorological observations +rh_i_cor,0,100,,all,0,1,1,4,modelResult,time lat lon alt,TRUE,For meteorological observations +wspd_i,0,100,wdir_i wspd_x_i wspd_y_i,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,For meteorological observations +wdir_i,1,360,wspd_x_i wspd_y_i,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,For meteorological observations +wspd_x_i,-100,100,wdir_i wspd_i,all,0,1,1,4,modelResult,time lat lon alt,TRUE,For meteorological observations +wspd_y_i,-100,100,wdir_i wspd_i,all,0,1,1,4,modelResult,time lat lon alt,TRUE,For meteorological observations \ No newline at end of file From a8479584ee86e9fdbfa5d67660446014f25f4248 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Fri, 14 Jun 2024 17:31:14 +0200 Subject: [PATCH 15/36] added back ressource files, use level attributes for output definition --- setup.py | 2 +- src/pypromice/process/L0toL1.py | 1 + src/pypromice/process/L1toL2.py | 1 + src/pypromice/process/L2toL3.py | 1 + src/pypromice/process/get_l2.py | 21 +- src/pypromice/process/get_l2tol3.py | 11 +- src/pypromice/process/join_l2.py | 7 +- src/pypromice/process/join_l3.py | 15 +- src/pypromice/process/load.py | 7 +- src/pypromice/process/utilities.py | 4 +- src/pypromice/process/write.py | 24 ++- src/pypromice/ressources/file_attributes.csv | 56 ++++++ .../ressources/variable_aliases_GC-Net.csv | 78 ++++++++ .../ressources/variable_processing_info.csv | 91 --------- ...variable_description.csv => variables.csv} | 182 +++++++++--------- 15 files changed, 263 insertions(+), 238 deletions(-) create mode 100644 src/pypromice/ressources/file_attributes.csv create mode 100644 src/pypromice/ressources/variable_aliases_GC-Net.csv delete mode 100644 src/pypromice/ressources/variable_processing_info.csv rename src/pypromice/ressources/{variable_description.csv => variables.csv} (57%) diff --git a/setup.py b/setup.py index 7289e381..f5635c25 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", package_data={ - "pypromice.ressources": ["variable_description.csv", "variable_processing_info.csv"], + "pypromice.ressources": ["variables.csv", "file_attributes.csv", "variable_aliases_GC-Net.csv"], "pypromice.tx": ["payload_formats.csv", "payload_types.csv"], "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], diff --git a/src/pypromice/process/L0toL1.py b/src/pypromice/process/L0toL1.py index a056a08b..41850044 100644 --- a/src/pypromice/process/L0toL1.py +++ b/src/pypromice/process/L0toL1.py @@ -31,6 +31,7 @@ def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100): ''' assert(type(L0) == xr.Dataset) ds = L0 + ds.attrs['level'] = 'L1' for l in list(ds.keys()): if l not in ['time', 'msg_i', 'gps_lat', 'gps_lon', 'gps_alt', 'gps_time']: diff --git a/src/pypromice/process/L1toL2.py b/src/pypromice/process/L1toL2.py index d9ee1e8e..1a0087cb 100644 --- a/src/pypromice/process/L1toL2.py +++ b/src/pypromice/process/L1toL2.py @@ -70,6 +70,7 @@ def toL2( Level 2 dataset ''' ds = L1.copy(deep=True) # Reassign dataset + ds.attrs['level'] = 'L2' try: ds = adjustTime(ds) # Adjust time after a user-defined csv files ds = flagNAN(ds) # Flag NaNs after a user-defined csv files diff --git a/src/pypromice/process/L2toL3.py b/src/pypromice/process/L2toL3.py index 9751122e..2e05ff04 100755 --- a/src/pypromice/process/L2toL3.py +++ b/src/pypromice/process/L2toL3.py @@ -32,6 +32,7 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071, 1013.246. ''' ds = L2 + ds.attrs['level'] = 'L3' T_100 = _getTempK(T_0) # Get steam point temperature as K diff --git a/src/pypromice/process/get_l2.py b/src/pypromice/process/get_l2.py index 0122b53d..734aa86e 100644 --- a/src/pypromice/process/get_l2.py +++ b/src/pypromice/process/get_l2.py @@ -4,7 +4,6 @@ import pypromice from pypromice.process.aws import AWS from pypromice.process.write import prepare_and_write -from pypromice.process.load import getVars, getMeta def parse_arguments_l2(): parser = ArgumentParser(description="AWS L2 processor") @@ -30,26 +29,14 @@ def get_l2(): level=logging.INFO, stream=sys.stdout, ) - - # Define variables (either from file or pypromice package defaults) - if args.variables is None: - v = os.path.join(os.path.dirname(pypromice.__file__),'process/variables.csv') - else: - v = args.variables - - # Define metadata (either from file or pypromice package defaults) - if args.variables is None: - m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv') - else: - m = args.metadata # Define input path station_name = args.config_file.split('/')[-1].split('.')[0] station_path = os.path.join(args.inpath, station_name) if os.path.exists(station_path): - aws = AWS(args.config_file, station_path, v, m) + aws = AWS(args.config_file, station_path, args.variables, args.metadata) else: - aws = AWS(args.config_file, args.inpath, v, m) + aws = AWS(args.config_file, args.inpath, args.variables, args.metadata) # Perform level 1 and 2 processing aws.getL1() @@ -60,8 +47,8 @@ def get_l2(): if not os.path.isdir(args.outpath): os.mkdir(args.outpath) if aws.L2.attrs['format'] == 'raw': - prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '10min') - prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '60min') + prepare_and_write(aws.L2, args.outpath, args.variables, args.metadata, '10min') + prepare_and_write(aws.L2, args.outpath, args.variables, args.metadata, '60min') if __name__ == "__main__": diff --git a/src/pypromice/process/get_l2tol3.py b/src/pypromice/process/get_l2tol3.py index 7fa3e318..daa3e2c6 100644 --- a/src/pypromice/process/get_l2tol3.py +++ b/src/pypromice/process/get_l2tol3.py @@ -3,7 +3,6 @@ import xarray as xr from argparse import ArgumentParser import pypromice -from pypromice.process.load import getVars, getMeta from pypromice.process.L2toL3 import toL3 from pypromice.process.write import prepare_and_write @@ -34,10 +33,6 @@ def get_l2tol3(): level=logging.INFO, stream=sys.stdout, ) - - # Define variables and metadata (either from file or pypromice package defaults) - v = getVars(args.variables) - m = getMeta(args.metadata) # Define Level 2 dataset from file with xr.open_dataset(args.inpath) as l2: @@ -55,9 +50,9 @@ def get_l2tol3(): # Write Level 3 dataset to file if output directory given if args.outpath is not None: - prepare_and_write(l3, args.outpath, v, m, '60min') - prepare_and_write(l3, args.outpath, v, m, '1D') - prepare_and_write(l3, args.outpath, v, m, 'M') + prepare_and_write(l3, args.outpath, args.variables, args.metadata, '60min') + prepare_and_write(l3, args.outpath, args.variables, args.metadata, '1D') + prepare_and_write(l3, args.outpath, args.variables, args.metadata, 'M') if __name__ == "__main__": get_l2tol3() diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index a01fe8ca..514b1e80 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -3,7 +3,6 @@ import pandas as pd import xarray as xr from argparse import ArgumentParser -from pypromice.process.load import getVars, getMeta from pypromice.process.utilities import addMeta, roundValues from pypromice.process.write import prepare_and_write from pypromice.process.L1toL2 import correctPrecip @@ -51,10 +50,6 @@ def loadArr(infile): def join_l2(): args = parse_arguments_join() - # Define variables and metadata (either from file or pypromice package defaults) - v = getVars(args.variables) - m = getMeta(args.metadata) - # Check files if os.path.isfile(args.file1) and os.path.isfile(args.file2): @@ -99,7 +94,7 @@ def join_l2(): # Resample to hourly, daily and monthly datasets and write to file - prepare_and_write(all_ds, args.outpath, v, m, resample = False) + prepare_and_write(all_ds, args.outpath, args.variables, args.metadata, resample = False) print(f'Files saved to {os.path.join(args.outpath, name)}...') diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 778e8741..e10f3c56 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -1,14 +1,10 @@ #!/usr/bin/env python import logging, os, sys, unittest, toml, pkg_resources from argparse import ArgumentParser -import pypromice -from pypromice.process.aws import AWS -from pypromice.process import load from pypromice.process.write import prepare_and_write import numpy as np import pandas as pd import xarray as xr -from pypromice.process.load import getVars, getMeta def parse_arguments_joinl3(debug_args=None): parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") @@ -173,6 +169,7 @@ def join_l3(): conf = toml.load(config_file) l3m = xr.Dataset() + l3m.attrs['level'] = 'L3' for stid in conf['stations']: print(stid) @@ -245,13 +242,11 @@ def join_l3(): # Assign site id l3m.attrs['site_id'] = args.site l3m.attrs['stations'] = conf['stations'] - v = getVars() - m = getMeta() + if args.outpath is not None: - prepare_and_write(l3m, args.outpath, v, m, '60min') - prepare_and_write(l3m, args.outpath, v, m, '1D') - prepare_and_write(l3m, args.outpath, v, m, 'M') - # %% + prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '60min') + prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '1D') + prepare_and_write(l3m, args.outpath, args.variables, args.metadata, 'M') if __name__ == "__main__": join_l3() diff --git a/src/pypromice/process/load.py b/src/pypromice/process/load.py index 186784c6..4ee15148 100644 --- a/src/pypromice/process/load.py +++ b/src/pypromice/process/load.py @@ -120,6 +120,7 @@ def getL0(infile, nodata, cols, skiprows, file_version, # Carry relevant metadata with ds ds = xr.Dataset.from_dataframe(df) + ds.attrs['level'] = 'L0' return ds def getVars(v_file=None): @@ -136,11 +137,11 @@ def getVars(v_file=None): Variables dataframe ''' if v_file is None: - with pkg_resources.resource_stream('pypromice', 'process/variables.csv') as stream: + with pkg_resources.resource_stream('pypromice', 'ressources/variables.csv') as stream: return pd.read_csv(stream, index_col=0, comment="#", encoding='utf-8') else: return pd.read_csv(v_file, index_col=0, comment="#") - + def getMeta(m_file=None, delimiter=','): #TODO change to DataFrame output to match variables.csv '''Load metadata table @@ -159,7 +160,7 @@ def getMeta(m_file=None, delimiter=','): ''' meta={} if m_file is None: - with pkg_resources.resource_stream('pypromice', 'process/metadata.csv') as stream: + with pkg_resources.resource_stream('pypromice', 'ressources/file_attributes.csv') as stream: lines = stream.read().decode("utf-8") lines = lines.split("\n") else: diff --git a/src/pypromice/process/utilities.py b/src/pypromice/process/utilities.py index d0dad5a5..0fa251ce 100644 --- a/src/pypromice/process/utilities.py +++ b/src/pypromice/process/utilities.py @@ -109,11 +109,9 @@ def populateMeta(ds, conf, skip): ds : xarray.Dataset L0 dataset with metadata populated as Dataset attributes ''' - meta = {} # skip = ["columns", "skiprows"] for k in conf.keys(): - if k not in skip: meta[k] = conf[k] - ds.attrs = meta + if k not in skip: ds.attrs[k] = conf[k] return ds diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index 5a0252f6..89059df9 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -8,9 +8,9 @@ logger = logging.getLogger(__name__) from pypromice.process.resample import resample_dataset -from pypromice.process import utilities +from pypromice.process import utilities, load -def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resample=True): +def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60min', resample=True): '''Prepare data with resampling, formating and metadata population; then write data to .nc and .csv hourly and daily files @@ -48,7 +48,13 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp d2 = utilities.reformat_lon(d2) else: logger.info('%s does not have gpd_lon'%name) + # Add variable attributes and metadata + if vars_df is None: + vars_df = load.getVars() + if meta_dict is None: + meta_dict = load.getMeta() + d2 = utilities.addVars(d2, vars_df) d2 = utilities.addMeta(d2, meta_dict) @@ -90,6 +96,8 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp logger.info(f'Written to {out_csv}') logger.info(f'Written to {out_nc}') + + def writeAll(outpath, station_id, l3_h, l3_d, l3_m, csv_order=None): '''Write L3 hourly, daily and monthly datasets to .nc and .csv files @@ -169,16 +177,16 @@ def getColNames(vars_df, ds, remove_nan_fields=False): list Variable names ''' - if 'data_type' in ds.attrs.keys(): - if ds.attrs['data_type']=='TX': - vars_df = vars_df.loc[vars_df['data_type'].isin(['TX','all'])] - elif ds.attrs['data_type']=='STM' or ds.attrs['data_type']=='raw': - vars_df = vars_df.loc[vars_df['data_type'].isin(['raw','all'])] - if 'number_of_booms' in ds.attrs.keys(): + # selecting variable list based on level + vars_df = vars_df.loc[vars_df[ds.attrs['level']] == 1] + + # selecting variable list based on geometry + if ds.attrs['level'] in ['L0', 'L1', 'L2']: if ds.attrs['number_of_booms']==1: vars_df = vars_df.loc[vars_df['station_type'].isin(['one-boom','all'])] elif ds.attrs['number_of_booms']==2: vars_df = vars_df.loc[vars_df['station_type'].isin(['two-boom','all'])] + var_list = list(vars_df.index) if remove_nan_fields: for v in var_list: diff --git a/src/pypromice/ressources/file_attributes.csv b/src/pypromice/ressources/file_attributes.csv new file mode 100644 index 00000000..9fa56bc4 --- /dev/null +++ b/src/pypromice/ressources/file_attributes.csv @@ -0,0 +1,56 @@ +attribute,entry +acknowledgements,The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) +alt.axis,Z +alt.coverage_content_type,coordinate +gps_alt.positive,up +cdm_data_type, +comment,https://doi.org/10.22008/promice/data/aws +contributor_name, +contributor_role, +conventions,ACDD-1.3; CF-1.7 +creater_email,pho@geus.dk +creater_url,https://promice.dk +creator_institution,Geological Survey of Denmark and Greenland (GEUS) +creator_name,Penelope How +creator_type,person +featureType,timeSeries +geospatial_bounds_crs,EPSG:4979 +geospatial_lat_extents_match,gps_lat +geospatial_lat_resolution, +geospatial_lat_units,degrees_north +geospatial_lon_extents_match,gps_lon +geospatial_lon_resolution, +geospatial_lon_units,degrees_east +geospatial_vertical_resolution, +geospatial_vertical_units,EPSG:4979 +institution,Geological Survey of Denmark and Greenland (GEUS) +instrument,See https://doi.org/10.5194/essd-13-3819-2021 +instrument_vocabulary,GCMD:GCMD Keywords +keywords,GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > ICE SHEETS > ICE SHEET MEASUREMENTS; GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > GLACIER MASS BALANCE/ICE SHEET MASS BALANCE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE GROWTH/MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC PRESSURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > ALBEDO; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > INCOMING SOLAR RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > DOWNWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > UPWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > NET RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > OUTGOING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FLUX; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FORCING; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION > DOWNWELLING SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SUNSHINE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC TEMPERATURE > SURFACE TEMPERATURE > AIR TEMPERATURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > ABSOLUTE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > RELATIVE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > LOCAL WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > U/V WIND COMPONENTS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND DIRECTION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND SPEED; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > CLOUDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > PRECIPITATION +keywords_vocabulary,GCMDSK:GCMD Science Keywords:https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords +lat.axis,Y +lat.coverage_content_type,coordinate +lat.long_name,station latitude +license,Creative Commons Attribution 4.0 International (CC-BY-4.0) https://creativecommons.org/licenses/by/4.0 +lon.axis,X +lon.coverage_content_type,coordinate +lon.long_name,station longitude +lon.units,degrees_east +metadata_link, +naming_authority,dk.geus.promice +platform, +platform_vocabulary,GCMD:GCMD Keywords +processing_level,Level 3 +product_status,beta +product_version,4 +program,PROMICE +project,PROMICE +publisher_email,info@promice.dk +publisher_institution,GEUS +publisher_name,GEUS +publisher_type,institution +publisher_url,https://promice.dk +references,"How, P.; Abermann, J.; Ahlstrøm, A.P.; Andersen, S.B.; Box, J. E.; Citterio, M.; Colgan, W.T.; Fausto. R.S.; Karlsson, N.B.; Jakobsen, J.; Langley, K.; Larsen, S.H.; Mankoff, K.D.; Pedersen, A.Ø.; Rutishauser, A.; Shield, C.L.; Solgaard, A.M.; van As, D.; Vandecrux, B.; Wright, P.J., 2022, ""PROMICE and GC-Net automated weather station data in Greenland"", https://doi.org/10.22008/FK2/IW73UU, GEUS Dataverse" +references_bib,@article{How2022; doi = {10.22008/FK2/IW73UU}; url = {https://doi.org/10.22008/FK2/IW73UU}; year = {2022}; month=10; publisher= {GEUS Dataverse}; author = {Penelope How and Jakob Abermann and Andreas P. Ahlstr{\o}m and Signe B. Andersen and Jason E. Box and Michele Citterio and William Colgan and Robert S. Fausto and Nanna B. Karlsson and Jakob Jakobsen and Kirsty Langley and Signe Hillerup Larsen and Kenneth D. Mankoff and Allan {\O}. Pedersen and Anja Rutishauser and Christopher L. Shields and Anne M. Solgaard and Dirk van As and Baptiste Vandecrux}; title = {PROMICE and GC-Net automated weather station data in Greenland}; journal = {GEUS Dataverse}} +standard_name_vocabulary,CF Standard Name Table (v77; 19 January 2021) +summary,"The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) and Greenland Climate Network (GC-Net) have been measuring climate and ice sheet properties since 2007 and 1995, respectively. The PROMICE weather station network monitors glacier mass balance in the melt zone of the Greenland Ice Sheet, providing ground truth data to calibrate mass budget models. GC-Net weather stations measure snowfall and surface properties in the accumulation zone, providing valuable knowledge on the Greenland Ice Sheet’s mass gain and climatology.Accurate measurements of the surface and near-surface atmospheric conditions in a changing climate is important for reliable present and future assessment of changes to the Greenland Ice Sheet. All measurements are handled and processed with pypromice, which is a peer-reviewed and freely available Python package with source code available at https://github.com/GEUS-Glaciology-and-Climate/pypromice. A user-contributable dynamic web-based database of known data quality issues is associated with the data products at https://github.com/GEUS-PROMICE/ PROMICE-AWS-data-issues/." diff --git a/src/pypromice/ressources/variable_aliases_GC-Net.csv b/src/pypromice/ressources/variable_aliases_GC-Net.csv new file mode 100644 index 00000000..a0b4fe14 --- /dev/null +++ b/src/pypromice/ressources/variable_aliases_GC-Net.csv @@ -0,0 +1,78 @@ +GEUS_name,old_name +time,timestamp +p_u,P +p_l, +t_u,TA2 +t_l,TA1 +rh_u,RH2 +rh_u_cor,RH2_cor +qh_u,Q2 +rh_l,RH1 +rh_l_cor,RH1_cor +qh_l,Q1 +wspd_u,VW2 +wspd_l,VW1 +wdir_u,DW2 +wdir_l,DW1 +dsr, +dsr_cor,ISWR +usr, +usr_cor,OSWR +albedo,Alb +dlr, +ulr, +cc, +t_surf, +dlhf_u,LHF +dlhf_l, +dshf_u,SHF +dshf_l, +z_boom_u,HW2 +z_boom_l,HW1 +precip_u, +precip_u_cor, +precip_l, +precip_l_cor, +t_i_1,TS1 +t_i_2,TS2 +t_i_3,TS3 +t_i_4,TS4 +t_i_5,TS5 +t_i_6,TS6 +t_i_7,TS7 +t_i_8,TS8 +t_i_9,TS9 +t_i_10,TS10 +t_i_11, +tilt_x, +tilt_y, +rot, +gps_lat,latitude +gps_lon,longitude +gps_alt,elevation +gps_time, +gps_geounit, +gps_hdop, +batt_v,V +fan_dc_u, +fan_dc_l, +t_rad, +msg_lat, +msg_lon, +z_surf_1,HS1 +z_surf_2,HS2 +z_surf_1_adj_flag,HS1_adj_flag +z_surf_2_adj_flag,HS2_adj_flag +z_surf_combined,HS_combined +depth_t_i_1,DTS1 +depth_t_i_2,DTS2 +depth_t_i_3,DTS3 +depth_t_i_4,DTS4 +depth_t_i_5,DTS5 +depth_t_i_6,DTS6 +depth_t_i_7,DTS7 +depth_t_i_8,DTS8 +depth_t_i_9,DTS9 +depth_t_i_10,DTS10 +depth_t_i_11, +t_i_10m,TS_10m diff --git a/src/pypromice/ressources/variable_processing_info.csv b/src/pypromice/ressources/variable_processing_info.csv deleted file mode 100644 index 0c00932e..00000000 --- a/src/pypromice/ressources/variable_processing_info.csv +++ /dev/null @@ -1,91 +0,0 @@ -field,lo,hi,OOL,station_type,write_in_l0,write_in_l2,write_in_l3,max_decimals,coverage_content_type,coordinates,instantaneous_hourly,comment -time,,,,all,1,1,1,,physicalMeasurement,time lat lon alt,, -rec,,,,all,1,1,0,0,referenceInformation,time lat lon alt,,L0 only -p_u,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -p_l,650,1100,dshf_l dlhf_l qh_l,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_u,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_l,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,PT100 temperature at boom -rh_u,0,100,rh_u_cor,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -rh_u_cor,0,150,dshf_u dlhf_u qh_u,all,0,1,1,4,modelResult,time lat lon alt,FALSE, -qh_u,0,100,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -rh_l,0,100,rh_l_cor,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -rh_l_cor,0,150,dshf_l dlhf_l qh_l,two-boom,0,1,1,4,modelResult,time lat lon alt,FALSE, -qh_l,0,100,,two-boom,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -wspd_u,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -wspd_l,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -wdir_u,1,360,wspd_x_u wspd_y_u,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -wdir_std_u,,,,one-boom,1,1,0,4,qualityInformation,time lat lon alt,FALSE,L0 only -wdir_l,1,360,wspd_x_l wspd_y_l,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -wspd_x_u,-100,100,wdir_u wspd_u,all,0,1,1,4,modelResult,time lat lon alt,FALSE,L0 only -wspd_y_u,-100,100,wdir_u wspd_u,all,0,1,1,4,modelResult,time lat lon alt,FALSE,L0 only -wspd_x_l,-100,100,wdir_l wspd_l,two-boom,0,1,1,4,modelResult,time lat lon alt,FALSE,L0 only -wspd_y_l,-100,100,wdir_l wspd_l,two-boom,0,1,1,4,modelResult,time lat lon alt,FALSE,L0 only -dsr,-10,1500,albedo dsr_cor usr_cor,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,"Actually radiation_at_sensor, not flux. Units 1E-5 V. Engineering units." -dsr_cor,,,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -usr,-10,1000,albedo dsr_cor usr_cor,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -usr_cor,0,1000,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -albedo,,,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -dlr,50,500,albedo dsr_cor usr_cor cc t_surf,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -ulr,50,500,t_surf,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -cc,,,,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -t_surf,-80,40,dshf_u dlhf_u qh_u,all,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -dlhf_u,,,,all,0,0,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -dlhf_l,,,,two-boom,0,0,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -dshf_u,,,,all,0,0,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -dshf_l,,,,two-boom,0,0,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -z_boom_u,0.3,10,dshf_u dlhf_u qh_u,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE, -z_boom_q_u,,,,all,1,1,0,4,qualityInformation,time lat lon alt,TRUE,L0 only -z_boom_l,0.3,5,dshf_l dlhf_l qh_l,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE, -z_boom_q_l,,,,two-boom,1,1,0,4,qualityInformation,time lat lon alt,TRUE,L0 only -z_stake,0.3,8,,one-boom,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,HeightStakes(m) -z_stake_q,,,,one-boom,1,1,0,4,qualityInformation,time lat lon alt,TRUE,L0 only -z_pt,0,30,z_pt_cor,one-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,DepthPressureTransducer(m) -z_pt_cor,0,30,,one-boom,0,1,1,4,modelResult,time lat lon alt,FALSE,Derived value (L2 or later) -precip_u,0,,precip_u_cor precip_u_rate,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,Without wind/undercatch correction -precip_u_cor,0,,,all,0,1,1,4,modelResult,time lat lon alt,TRUE,With wind/undercatch correction -precip_u_rate,0,,,all,0,1,1,4,modelResult,time lat lon alt,TRUE,L0 only -precip_l,0,,precip_l_cor precip_l_rate,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,Without wind/undercatch correction -precip_l_cor,0,,,two-boom,0,1,1,4,modelResult,time lat lon alt,TRUE,With wind/undercatch correction -precip_l_rate,0,,,two-boom,0,1,1,4,modelResult,time lat lon alt,TRUE,L0 only -t_i_1,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,t1 is installed @ 1 m depth -t_i_2,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_i_3,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_i_4,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_i_5,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_i_6,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_i_7,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_i_8,-80,1,,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE,t8 is installed @ 10 m depth -t_i_9,-80,1,,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_i_10,-80,1,,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -t_i_11,-80,1,,two-boom,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -tilt_x,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -tilt_y,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -rot,0,360,,all,1,1,1,2,physicalMeasurement,time lat lon alt,FALSE,v4 addition -gps_lat,50,83,,all,1,1,1,6,coordinate,time lat lon alt,TRUE, -gps_lon,5,70,,all,1,1,1,6,coordinate,time lat lon alt,TRUE, -gps_alt,0,3000,,all,1,1,1,2,coordinate,time lat lon alt,TRUE, -gps_time,0,240000,,all,1,1,0,,physicalMeasurement,time lat lon alt,TRUE, -gps_geoid,,,,one-boom,1,1,0,,physicalMeasurement,time lat lon alt,TRUE, -gps_geounit,,,,all,1,1,0,,qualityInformation,time lat lon alt,TRUE,L0 only -gps_hdop,,,,all,1,1,0,2,qualityInformation,time lat lon alt,TRUE,NMEA: Horizontal dilution of precision -gps_numsat,,,,,1,1,0,0,qualityInformation,time lat lon alt,TRUE,L0 only -gps_q,,,,,1,1,0,,qualityInformation,time lat lon alt,TRUE,L0 only -lat,,,,all,1,1,1,6,modelResult,time lat lon alt,TRUE, -lon,,,,all,1,1,1,6,modelResult,time lat lon alt,TRUE, -alt,,,,all,1,1,1,6,modelResult,time lat lon alt,TRUE, -batt_v,0,30,,all,1,1,1,2,physicalMeasurement,time lat lon alt,TRUE, -batt_v_ini,0,30,,,1,1,0,2,physicalMeasurement,time lat lon alt,TRUE,L0 only -batt_v_ss,0,30,,,1,1,0,2,physicalMeasurement,time lat lon alt,TRUE,L0 only -fan_dc_u,0,200,,all,1,1,0,2,physicalMeasurement,time lat lon alt,TRUE, -fan_dc_l,0,200,,two-boom,1,1,0,2,physicalMeasurement,time lat lon alt,TRUE, -freq_vw,0,10000,precip_u,,1,1,0,,physicalMeasurement,time lat lon alt,TRUE,L0 only -t_log,-80,40,,one-boom,1,1,0,4,physicalMeasurement,time lat lon alt,TRUE,LoggerTemperature(C) -t_rad,-80,40,t_surf dlr ulr,all,1,1,1,4,physicalMeasurement,time lat lon alt,FALSE, -p_i,-350,100,,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,For meteorological observations -t_i,-80,40,,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,"PT100 temperature at boom, for meteorological observations" -rh_i,0,150,rh_i_cor,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,For meteorological observations -rh_i_cor,0,100,,all,0,1,1,4,modelResult,time lat lon alt,TRUE,For meteorological observations -wspd_i,0,100,wdir_i wspd_x_i wspd_y_i,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,For meteorological observations -wdir_i,1,360,wspd_x_i wspd_y_i,all,1,1,1,4,physicalMeasurement,time lat lon alt,TRUE,For meteorological observations -wspd_x_i,-100,100,wdir_i wspd_i,all,0,1,1,4,modelResult,time lat lon alt,TRUE,For meteorological observations -wspd_y_i,-100,100,wdir_i wspd_i,all,0,1,1,4,modelResult,time lat lon alt,TRUE,For meteorological observations \ No newline at end of file diff --git a/src/pypromice/ressources/variable_description.csv b/src/pypromice/ressources/variables.csv similarity index 57% rename from src/pypromice/ressources/variable_description.csv rename to src/pypromice/ressources/variables.csv index c8050114..ab8e4317 100644 --- a/src/pypromice/ressources/variable_description.csv +++ b/src/pypromice/ressources/variables.csv @@ -1,91 +1,91 @@ -field,standard_name,long_name,units,coverage_content_type,coordinates,instantaneous_hourly,where_to_find,comment -time,time,Time,yyyy-mm-dd HH:MM:SS,physicalMeasurement,time lat lon alt,,, -rec,record,Record,-,referenceInformation,time lat lon alt,,L0 or L2, -p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time lat lon alt,FALSE,, -p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time lat lon alt,FALSE,, -t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time lat lon alt,FALSE,, -rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time lat lon alt,FALSE,L2 or later, -qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time lat lon alt,FALSE,L2 or later, -rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time lat lon alt,FALSE,, -rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time lat lon alt,FALSE,L2 or later, -qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,modelResult,time lat lon alt,FALSE,L2 or later, -wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time lat lon alt,FALSE,, -wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time lat lon alt,FALSE,, -wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,physicalMeasurement,time lat lon alt,FALSE,, -wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,qualityInformation,time lat lon alt,FALSE,L0 or L2, -wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,physicalMeasurement,time lat lon alt,FALSE,, -wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2, -wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2, -wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2, -wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2, -dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,, -dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,modelResult,time lat lon alt,FALSE,L2 or later, -usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,, -usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,modelResult,time lat lon alt,FALSE,L2 or later, -albedo,surface_albedo,Albedo,-,modelResult,time lat lon alt,FALSE,L2 or later, -dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,, -ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,, -cc,cloud_area_fraction,Cloud cover,%,modelResult,time lat lon alt,FALSE,L2 or later, -t_surf,surface_temperature,Surface temperature,C,modelResult,time lat lon alt,FALSE,L2 or later, -dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later, -dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later, -dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later, -dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later, -z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time lat lon alt,TRUE,, -z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2, -z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time lat lon alt,TRUE,, -z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2, -z_stake,distance_to_surface_from_stake_assembly,Stake height,m,physicalMeasurement,time lat lon alt,TRUE,, -z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2, -z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time lat lon alt,FALSE,, -z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,modelResult,time lat lon alt,FALSE,L2 or later, -precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time lat lon alt,TRUE,,Without wind/undercatch correction -precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,With wind/undercatch correction -precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later, -precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time lat lon alt,TRUE,,Without wind/undercatch correction -precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,With wind/undercatch correction -precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later, -t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_2,ice_temperature_at_t2,Ice temperature at sensor 2,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_3,ice_temperature_at_t3,Ice temperature at sensor 3,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_4,ice_temperature_at_t4,Ice temperature at sensor 4,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_5,ice_temperature_at_t5,Ice temperature at sensor 5,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_6,ice_temperature_at_t6,Ice temperature at sensor 6,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_7,ice_temperature_at_t7,Ice temperature at sensor 7,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_8,ice_temperature_at_t8,Ice temperature at sensor 8,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_9,ice_temperature_at_t9,Ice temperature at sensor 9,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_10,ice_temperature_at_t10,Ice temperature at sensor 10,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -t_i_11,ice_temperature_at_t11,Ice temperature at sensor 11,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time lat lon alt,FALSE,, -tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time lat lon alt,FALSE,, -rot,platform_azimuth_angle,Station rotation from true North,degrees,physicalMeasurement,time lat lon alt,FALSE,, -gps_lat,gps_latitude,Latitude,degrees_north,coordinate,time lat lon alt,TRUE,, -gps_lon,gps_longitude,Longitude,degrees_east,coordinate,time lat lon alt,TRUE,, -gps_alt,gps_altitude,Altitude,m,coordinate,time lat lon alt,TRUE,, -gps_time,gps_time,GPS time,s,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, -gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, -gps_geounit,gps_geounit,GeoUnit,-,qualityInformation,time lat lon alt,TRUE,L0 or L2, -gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,qualityInformation,time lat lon alt,TRUE,L0 or L2, -gps_numsat,gps_numsat,GPS number of satellites,-,qualityInformation,time lat lon alt,TRUE,L0 or L2, -gps_q,gps_q,Quality,-,qualityInformation,time lat lon alt,TRUE,L0 or L2, -lat,gps_mean_latitude,GPS mean latitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,, -lon,gps_mean_longitude,GPS mean longitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,, -alt,gps_mean_altitude,GPS mean altitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,, -batt_v,battery_voltage,Battery voltage,V,physicalMeasurement,time lat lon alt,TRUE,, -batt_v_ini,,,-,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, -batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, -fan_dc_u,fan_current,Fan current (upper boom),mA,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, -fan_dc_l,fan_current,Fan current (lower boom),mA,physicalMeasurement,time lat lon alt,TRUE,, -freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time lat lon alt,TRUE,L0 or L2, -t_log,temperature_of_logger,Logger temperature,degrees_C,physicalMeasurement,time lat lon alt,TRUE,, -t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time lat lon alt,FALSE,, -p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,physicalMeasurement,time lat lon alt,TRUE,, -t_i,air_temperature,Air temperature (instantaneous),degrees_C,physicalMeasurement,time lat lon alt,TRUE,, -rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time lat lon alt,TRUE,, -rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,modelResult,time lat lon alt,TRUE,L2 or later, -wspd_i,wind_speed,Wind speed (instantaneous),m s-1,physicalMeasurement,time lat lon alt,TRUE,, -wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,physicalMeasurement,time lat lon alt,TRUE,, -wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time lat lon alt,TRUE,L2 or later, -wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time lat lon alt,TRUE,L2 or later, \ No newline at end of file +field,standard_name,long_name,units,coverage_content_type,coordinates,instantaneous_hourly,where_to_find,lo,hi,OOL,station_type,L0,L2,L3,max_decimals +time,time,Time,yyyy-mm-dd HH:MM:SS,physicalMeasurement,time lat lon alt,,,,,,all,1,1,1, +rec,record,Record,-,referenceInformation,time lat lon alt,,L0 or L2,,,,all,1,1,0,0 +p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time lat lon alt,FALSE,,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,1,1,1,4 +p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time lat lon alt,FALSE,,650,1100,dshf_l dlhf_l qh_l,two-boom,1,1,1,4 +t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,1,1,1,4 +t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l,two-boom,1,1,1,4 +rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time lat lon alt,FALSE,,0,100,rh_u_cor,all,1,1,1,4 +rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time lat lon alt,FALSE,L2 or later,0,150,dshf_u dlhf_u qh_u,all,0,1,1,4 +qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time lat lon alt,FALSE,L2 or later,0,100,,all,0,1,1,4 +rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time lat lon alt,FALSE,,0,100,rh_l_cor,two-boom,1,1,1,4 +rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time lat lon alt,FALSE,L2 or later,0,150,dshf_l dlhf_l qh_l,two-boom,0,1,1,4 +qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,modelResult,time lat lon alt,FALSE,L2 or later,0,100,,two-boom,0,1,1,4 +wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time lat lon alt,FALSE,,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,1,1,1,4 +wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time lat lon alt,FALSE,,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,1,1,1,4 +wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,physicalMeasurement,time lat lon alt,FALSE,,1,360,wspd_x_u wspd_y_u,all,1,1,1,4 +wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,qualityInformation,time lat lon alt,FALSE,L0 or L2,,,,one-boom,1,1,0,4 +wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,physicalMeasurement,time lat lon alt,FALSE,,1,360,wspd_x_l wspd_y_l,two-boom,1,1,1,4 +wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4 +wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4 +wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4 +wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4 +dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,,-10,1500,albedo dsr_cor usr_cor,all,1,1,1,4 +dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,modelResult,time lat lon alt,FALSE,L2 or later,,,,all,0,1,1,4 +usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,,-10,1000,albedo dsr_cor usr_cor,all,1,1,1,4 +usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,modelResult,time lat lon alt,FALSE,L2 or later,0,1000,,all,0,1,1,4 +albedo,surface_albedo,Albedo,-,modelResult,time lat lon alt,FALSE,L2 or later,,,,all,0,1,1,4 +dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,,50,500,albedo dsr_cor usr_cor cc t_surf,all,1,1,1,4 +ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,,50,500,t_surf,all,1,1,1,4 +cc,cloud_area_fraction,Cloud cover,%,modelResult,time lat lon alt,FALSE,L2 or later,,,,all,0,1,1,4 +t_surf,surface_temperature,Surface temperature,C,modelResult,time lat lon alt,FALSE,L2 or later,-80,40,dshf_u dlhf_u qh_u,all,0,1,1,4 +dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later,,,,all,0,0,1,4 +dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later,,,,two-boom,0,0,1,4 +dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later,,,,all,0,0,1,4 +dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later,,,,two-boom,0,0,1,4 +z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time lat lon alt,TRUE,,0.3,10,dshf_u dlhf_u qh_u,all,1,1,1,4 +z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,all,1,1,0,4 +z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time lat lon alt,TRUE,,0.3,5,dshf_l dlhf_l qh_l,two-boom,1,1,1,4 +z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,two-boom,1,1,0,4 +z_stake,distance_to_surface_from_stake_assembly,Stake height,m,physicalMeasurement,time lat lon alt,TRUE,,0.3,8,,one-boom,1,1,1,4 +z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,one-boom,1,1,0,4 +z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time lat lon alt,FALSE,,0,30,z_pt_cor,one-boom,1,1,1,4 +z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,modelResult,time lat lon alt,FALSE,L2 or later,0,30,,one-boom,0,1,1,4 +precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time lat lon alt,TRUE,,0,,precip_u_cor precip_u_rate,all,1,1,1,4 +precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,0,,,all,0,1,1,4 +precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,0,,,all,0,1,1,4 +precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time lat lon alt,TRUE,,0,,precip_l_cor precip_l_rate,two-boom,1,1,1,4 +precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,0,,,two-boom,0,1,1,4 +precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,0,,,two-boom,0,1,1,4 +t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_2,ice_temperature_at_t2,Ice temperature at sensor 2,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_3,ice_temperature_at_t3,Ice temperature at sensor 3,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_4,ice_temperature_at_t4,Ice temperature at sensor 4,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_5,ice_temperature_at_t5,Ice temperature at sensor 5,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_6,ice_temperature_at_t6,Ice temperature at sensor 6,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_7,ice_temperature_at_t7,Ice temperature at sensor 7,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_8,ice_temperature_at_t8,Ice temperature at sensor 8,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_9,ice_temperature_at_t9,Ice temperature at sensor 9,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,two-boom,1,1,1,4 +t_i_10,ice_temperature_at_t10,Ice temperature at sensor 10,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,two-boom,1,1,1,4 +t_i_11,ice_temperature_at_t11,Ice temperature at sensor 11,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,two-boom,1,1,1,4 +tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time lat lon alt,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4 +tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time lat lon alt,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4 +rot,platform_azimuth_angle,Station rotation from true North,degrees,physicalMeasurement,time lat lon alt,FALSE,,0,360,,all,1,1,1,2 +gps_lat,gps_latitude,Latitude,degrees_north,coordinate,time lat lon alt,TRUE,,50,83,,all,1,1,1,6 +gps_lon,gps_longitude,Longitude,degrees_east,coordinate,time lat lon alt,TRUE,,5,70,,all,1,1,1,6 +gps_alt,gps_altitude,Altitude,m,coordinate,time lat lon alt,TRUE,,0,3000,,all,1,1,1,2 +gps_time,gps_time,GPS time,s,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,240000,,all,1,1,0, +gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,,,,one-boom,1,1,0, +gps_geounit,gps_geounit,GeoUnit,-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,all,1,1,0, +gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,all,1,1,0,2 +gps_numsat,gps_numsat,GPS number of satellites,-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,,1,1,0,0 +gps_q,gps_q,Quality,-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,,1,1,0, +lat,gps_mean_latitude,GPS mean latitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,,,,,all,1,1,1,6 +lon,gps_mean_longitude,GPS mean longitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,,,,,all,1,1,1,6 +alt,gps_mean_altitude,GPS mean altitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,,,,,all,1,1,1,6 +batt_v,battery_voltage,Battery voltage,V,physicalMeasurement,time lat lon alt,TRUE,,0,30,,all,1,1,1,2 +batt_v_ini,,,-,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,30,,,1,1,0,2 +batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,30,,,1,1,0,2 +fan_dc_u,fan_current,Fan current (upper boom),mA,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,200,,all,1,1,0,2 +fan_dc_l,fan_current,Fan current (lower boom),mA,physicalMeasurement,time lat lon alt,TRUE,,0,200,,two-boom,1,1,0,2 +freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,10000,precip_u,,1,1,0, +t_log,temperature_of_logger,Logger temperature,degrees_C,physicalMeasurement,time lat lon alt,TRUE,,-80,40,,one-boom,1,1,0,4 +t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,40,t_surf dlr ulr,all,1,1,1,4 +p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,physicalMeasurement,time lat lon alt,TRUE,,-350,100,,all,1,1,1,4 +t_i,air_temperature,Air temperature (instantaneous),degrees_C,physicalMeasurement,time lat lon alt,TRUE,,-80,40,,all,1,1,1,4 +rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time lat lon alt,TRUE,,0,150,rh_i_cor,all,1,1,1,4 +rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,modelResult,time lat lon alt,TRUE,L2 or later,0,100,,all,0,1,1,4 +wspd_i,wind_speed,Wind speed (instantaneous),m s-1,physicalMeasurement,time lat lon alt,TRUE,,0,100,wdir_i wspd_x_i wspd_y_i,all,1,1,1,4 +wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,physicalMeasurement,time lat lon alt,TRUE,,1,360,wspd_x_i wspd_y_i,all,1,1,1,4 +wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time lat lon alt,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4 +wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time lat lon alt,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4 \ No newline at end of file From 569b8bd70f39cf0cb6aa66ee126e5dc9c33a8654 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Sat, 15 Jun 2024 14:50:00 +0200 Subject: [PATCH 16/36] make list of sites from station_config, switched print to logger.info --- src/pypromice/process/L0toL1.py | 2 +- src/pypromice/process/join_l2.py | 20 +++++++------ src/pypromice/process/join_l3.py | 48 ++++++++++++++++++++++++-------- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/src/pypromice/process/L0toL1.py b/src/pypromice/process/L0toL1.py index 41850044..3386e82e 100644 --- a/src/pypromice/process/L0toL1.py +++ b/src/pypromice/process/L0toL1.py @@ -248,7 +248,7 @@ def getPressDepth(z_pt, p, pt_antifreeze, pt_z_factor, pt_z_coef, pt_z_p_coef): rho_af = 1145 else: rho_af = np.nan - print('ERROR: Incorrect metadata: "pt_antifreeze" = ' + + logger.info('ERROR: Incorrect metadata: "pt_antifreeze" = ' + f'{pt_antifreeze}. Antifreeze mix only supported at 50% or 100%') # assert(False) diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index 514b1e80..43f29edc 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -43,13 +43,17 @@ def loadArr(infile): if 'number_of_booms' in ds.attrs.keys(): ds.attrs['number_of_booms'] = int(ds.attrs['number_of_booms']) - print(f'{name} array loaded from {infile}') + logger.info(f'{name} array loaded from {infile}') return ds, name def join_l2(): args = parse_arguments_join() - + logging.basicConfig( + format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", + level=logging.INFO, + stream=sys.stdout, + ) # Check files if os.path.isfile(args.file1) and os.path.isfile(args.file2): @@ -61,7 +65,7 @@ def join_l2(): if n1.lower() == n2.lower(): # Merge arrays - print(f'Combining {args.file1} with {args.file2}...') + logger.info(f'Combining {args.file1} with {args.file2}...') name = n1 all_ds = ds1.combine_first(ds2) @@ -75,28 +79,28 @@ def join_l2(): all_ds['precip_l_cor'], _ = correctPrecip(all_ds['precip_l'], all_ds['wspd_l']) else: - print(f'Mismatched station names {n1}, {n2}') + logger.info(f'Mismatched station names {n1}, {n2}') exit() elif os.path.isfile(args.file1): ds1, name = loadArr(args.file1) - print(f'Only one file found {args.file1}...') + logger.info(f'Only one file found {args.file1}...') all_ds = ds1 elif os.path.isfile(args.file2): ds2, name = loadArr(args.file2) - print(f'Only one file found {args.file2}...') + logger.info(f'Only one file found {args.file2}...') all_ds = ds2 else: - print(f'Invalid files {args.file1}, {args.file2}') + logger.info(f'Invalid files {args.file1}, {args.file2}') exit() # Resample to hourly, daily and monthly datasets and write to file prepare_and_write(all_ds, args.outpath, args.variables, args.metadata, resample = False) - print(f'Files saved to {os.path.join(args.outpath, name)}...') + logger.info(f'Files saved to {os.path.join(args.outpath, name)}...') if __name__ == "__main__": join_l2() diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index e10f3c56..a0f75f05 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -160,18 +160,43 @@ def gcnet_postprocessing(l3): # df_in.loc[[df_in.z_surf_combined.first_valid_index()],:].index.astype('int64')[0] # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] # return l3m - + +def build_station_dict(config_folder): + station_dict = {} + + # Iterate through all files in the given folder + for filename in os.listdir(config_folder): + if filename.endswith(".toml"): + # Construct full file path + file_path = os.path.join(config_folder, filename) + + # Load TOML file + with open(file_path, 'r') as file: + data = toml.load(file) + station_key = next(iter(data)) + station_data = data[station_key] + + if station_data["station_site"] in station_dict: + station_dict[station_data["station_site"]].append(station_key) + else: + station_dict[station_data["station_site"]] = [station_key] + + return station_dict + def join_l3(): args = parse_arguments_joinl3() - - - config_file = os.path.join(args.config_folder, args.site+'.toml') - conf = toml.load(config_file) + logging.basicConfig( + format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", + level=logging.INFO, + stream=sys.stdout, + ) + + station_dict = build_station_dict(args.config_folder) l3m = xr.Dataset() l3m.attrs['level'] = 'L3' - for stid in conf['stations']: - print(stid) + for stid in station_dict[args.site]: + logger.info(stid) is_promice = False is_gcnet = False @@ -183,7 +208,7 @@ def join_l3(): if os.path.isfile(filepath): is_gcnet = True if not is_promice and not is_gcnet: - print(stid, 'not found either in', args.folder_l3, 'or', args.folder_gcnet) + logger.info(stid, 'not found either in', args.folder_l3, 'or', args.folder_gcnet) continue l3, _ = loadArr(filepath) @@ -213,14 +238,15 @@ def join_l3(): # if l3 (older data) has variables that does not have l3m (newer data) # then they are removed from l3 - print('dropping') + list_dropped = [] for v in l3.data_vars: if v not in l3m.data_vars: if v != 'z_stake': - print(v) + list_dropped.append(v) l3 = l3.drop(v) else: l3m[v] = ('time', l3m.t_u.data*np.nan) + logger.info('Unused variables in older dataset:',list_dropped) # saving attributes of station under an attribute called $stid l3m = l3m.assign_attrs({stid : l3.attrs.copy()}) @@ -241,7 +267,7 @@ def join_l3(): # Assign site id l3m.attrs['site_id'] = args.site - l3m.attrs['stations'] = conf['stations'] + l3m.attrs['stations'] = station_dict[args.site] if args.outpath is not None: prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '60min') From 22b88ddc77b003a5c264d34a93acb086e36e7f82 Mon Sep 17 00:00:00 2001 From: BaptisteVandecrux Date: Sat, 15 Jun 2024 14:55:31 +0200 Subject: [PATCH 17/36] removing get_l3, remove inst. values from averaged files, fixes on logging, attributes and tests, --- setup.py | 1 - src/pypromice/process/L0toL1.py | 6 +++--- src/pypromice/process/get_l2tol3.py | 1 + src/pypromice/process/join_l2.py | 4 +++- src/pypromice/process/join_l3.py | 17 ++++++++++------- src/pypromice/process/load.py | 2 +- src/pypromice/process/resample.py | 9 +++++---- src/pypromice/process/test.py | 14 +++++++------- src/pypromice/process/utilities.py | 13 +++++++------ src/pypromice/process/write.py | 10 ++++++++++ 10 files changed, 47 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index f5635c25..194d358e 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,6 @@ 'join_l2 = pypromice.process.join_l2:join_l2', 'join_l3 = pypromice.process.join_l3:join_l3', 'get_l2 = pypromice.process.get_l2:get_l2', - 'get_l3 = pypromice.process.get_l3:get_l3', 'get_l2tol3 = pypromice.process.get_l2tol3:get_l2tol3', 'get_watsontx = pypromice.tx.get_watsontx:get_watsontx', 'get_bufr = pypromice.postprocess.get_bufr:main', diff --git a/src/pypromice/process/L0toL1.py b/src/pypromice/process/L0toL1.py index 3386e82e..0d55c730 100644 --- a/src/pypromice/process/L0toL1.py +++ b/src/pypromice/process/L0toL1.py @@ -5,9 +5,9 @@ import numpy as np import pandas as pd import xarray as xr -import re - +import re, logging from pypromice.process.value_clipping import clip_values +logger = logging.getLogger(__name__) def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100): @@ -28,7 +28,7 @@ def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100): ------- ds : xarray.Dataset Level 1 dataset - ''' + ''' assert(type(L0) == xr.Dataset) ds = L0 ds.attrs['level'] = 'L1' diff --git a/src/pypromice/process/get_l2tol3.py b/src/pypromice/process/get_l2tol3.py index daa3e2c6..8204bfc6 100644 --- a/src/pypromice/process/get_l2tol3.py +++ b/src/pypromice/process/get_l2tol3.py @@ -5,6 +5,7 @@ import pypromice from pypromice.process.L2toL3 import toL3 from pypromice.process.write import prepare_and_write +logger = logging.getLogger(__name__) def parse_arguments_l2tol3(debug_args=None): parser = ArgumentParser(description="AWS L3 script for the processing L3 "+ diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index 43f29edc..37abff8d 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -1,11 +1,12 @@ #!/usr/bin/env python -import os, unittest +import logging, sys, os, unittest import pandas as pd import xarray as xr from argparse import ArgumentParser from pypromice.process.utilities import addMeta, roundValues from pypromice.process.write import prepare_and_write from pypromice.process.L1toL2 import correctPrecip +logger = logging.getLogger(__name__) def parse_arguments_join(): parser = ArgumentParser(description="AWS L2 joiner for merging together two L2 products, for example an L2 RAW and L2 TX data product. An hourly, daily and monthly L2 data product is outputted to the defined output path") @@ -96,6 +97,7 @@ def join_l2(): logger.info(f'Invalid files {args.file1}, {args.file2}') exit() + all_ds.attrs['format'] = 'merged RAW and TX' # Resample to hourly, daily and monthly datasets and write to file prepare_and_write(all_ds, args.outpath, args.variables, args.metadata, resample = False) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index a0f75f05..669126c5 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd import xarray as xr +logger = logging.getLogger(__name__) def parse_arguments_joinl3(debug_args=None): parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") @@ -194,7 +195,6 @@ def join_l3(): station_dict = build_station_dict(args.config_folder) l3m = xr.Dataset() - l3m.attrs['level'] = 'L3' for stid in station_dict[args.site]: logger.info(stid) @@ -208,7 +208,7 @@ def join_l3(): if os.path.isfile(filepath): is_gcnet = True if not is_promice and not is_gcnet: - logger.info(stid, 'not found either in', args.folder_l3, 'or', args.folder_gcnet) + logger.info(stid+' not found either in '+args.folder_l3+' or '+args.folder_gcnet) continue l3, _ = loadArr(filepath) @@ -246,17 +246,20 @@ def join_l3(): l3 = l3.drop(v) else: l3m[v] = ('time', l3m.t_u.data*np.nan) - logger.info('Unused variables in older dataset:',list_dropped) + logger.info('Unused variables in older dataset: '+' '.join(list_dropped)) # saving attributes of station under an attribute called $stid - l3m = l3m.assign_attrs({stid : l3.attrs.copy()}) + st_attrs = l3m.attrs.get('stations_attributes', {}) + st_attrs[stid] = l3.attrs.copy() + l3m.attrs["stations_attributes"] = st_attrs + # then stripping attributes attrs_list = list(l3.attrs.keys()) for k in attrs_list: del l3.attrs[k] - l3m.attrs[stid]['first_timestamp'] = l3.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() - l3m.attrs[stid]['last_timestamp'] = l3m.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + l3m.attrs['stations_attributes'][stid]['first_timestamp'] = l3.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + l3m.attrs['stations_attributes'][stid]['last_timestamp'] = l3m.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() # merging by time block l3m = xr.concat((l3.sel( @@ -268,7 +271,7 @@ def join_l3(): # Assign site id l3m.attrs['site_id'] = args.site l3m.attrs['stations'] = station_dict[args.site] - + l3m.attrs['level'] = 'L3' if args.outpath is not None: prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '60min') prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '1D') diff --git a/src/pypromice/process/load.py b/src/pypromice/process/load.py index 4ee15148..a2334096 100644 --- a/src/pypromice/process/load.py +++ b/src/pypromice/process/load.py @@ -98,7 +98,7 @@ def getL0(infile, nodata, cols, skiprows, file_version, try: df.index = pd.to_datetime(df.index) except ValueError as e: - logger.info("\n", infile) + logger.info("\n"+ infile) logger.info("\nValueError:") logger.info(e) logger.info('\t\t> Trying pd.to_datetime with format=mixed') diff --git a/src/pypromice/process/resample.py b/src/pypromice/process/resample.py index b4af07e2..daef8624 100644 --- a/src/pypromice/process/resample.py +++ b/src/pypromice/process/resample.py @@ -34,13 +34,13 @@ def resample_dataset(ds_h, t): df_d = ds_h.to_dataframe().resample(t).mean() # recalculating wind direction from averaged directional wind speeds - for var in ['wdir_u','wdir_l','wdir_i']: + for var in ['wdir_u','wdir_l']: if var in df_d.columns: if ('wspd_x_'+var.split('_')[1] in df_d.columns) & ('wspd_x_'+var.split('_')[1] in df_d.columns): df_d[var] = _calcWindDir(df_d['wspd_x_'+var.split('_')[1]], df_d['wspd_y_'+var.split('_')[1]]) else: - logger.info(var,'in dataframe but not','wspd_x_'+var.split('_')[1],'wspd_x_'+var.split('_')[1]) + logger.info(var+' in dataframe but not wspd_x_'+var.split('_')[1]+' nor wspd_x_'+var.split('_')[1]) # recalculating relative humidity from average vapour pressure and average # saturation vapor pressure @@ -53,8 +53,9 @@ def resample_dataset(ds_h, t): df_d[var] = (p_vap.to_series().resample(t).mean() \ / es_wtr.to_series().resample(t).mean())*100 - df_d[var+'_cor'] = (p_vap.to_series().resample(t).mean() \ - / es_cor.to_series().resample(t).mean())*100 + if var+'_cor' in df_d.keys(): + df_d[var+'_cor'] = (p_vap.to_series().resample(t).mean() \ + / es_cor.to_series().resample(t).mean())*100 vals = [xr.DataArray(data=df_d[c], dims=['time'], coords={'time':df_d.index}, attrs=ds_h[c].attrs) for c in df_d.columns] diff --git a/src/pypromice/process/test.py b/src/pypromice/process/test.py index df81e88d..b9e15807 100644 --- a/src/pypromice/process/test.py +++ b/src/pypromice/process/test.py @@ -61,21 +61,21 @@ def testCLIgetl2(self): '''Test get_l2 CLI''' exit_status = os.system('get_l2 -h') self.assertEqual(exit_status, 0) - - def testCLIgetl3(self): - '''Test get_l3 CLI''' - exit_status = os.system('get_l3 -h') - self.assertEqual(exit_status, 0) def testCLIjoinl2(self): '''Test join_l2 CLI''' exit_status = os.system('join_l2 -h') self.assertEqual(exit_status, 0) - def testCLIjoinl3(self): - '''Test join_l2 CLI''' + def testCLIgetl2tol3(self): + '''Test get_l2tol3 CLI''' exit_status = os.system('get_l2tol3 -h') self.assertEqual(exit_status, 0) + + def testCLIjoinl3(self): + '''Test join_l3 CLI''' + exit_status = os.system('join_l3 -h') + self.assertEqual(exit_status, 0) if __name__ == "__main__": diff --git a/src/pypromice/process/utilities.py b/src/pypromice/process/utilities.py index 0fa251ce..4bcca216 100644 --- a/src/pypromice/process/utilities.py +++ b/src/pypromice/process/utilities.py @@ -154,14 +154,15 @@ def addMeta(ds, meta): ds : xarray.Dataset Dataset with metadata ''' - ds['lon'] = ds['gps_lon'].mean() - ds['lon'].attrs = ds['gps_lon'].attrs + if 'gps_lon' in ds.keys(): + ds['lon'] = ds['gps_lon'].mean() + ds['lon'].attrs = ds['gps_lon'].attrs - ds['lat'] = ds['gps_lat'].mean() - ds['lat'].attrs = ds['gps_lat'].attrs + ds['lat'] = ds['gps_lat'].mean() + ds['lat'].attrs = ds['gps_lat'].attrs - ds['alt'] = ds['gps_alt'].mean() - ds['alt'].attrs = ds['gps_alt'].attrs + ds['alt'] = ds['gps_alt'].mean() + ds['alt'].attrs = ds['gps_alt'].attrs # for k in ds.keys(): # for each var # if 'units' in ds[k].attrs: diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index 89059df9..732ec12e 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -79,9 +79,17 @@ def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60mi out_csv = os.path.join(outdir, name+'_hour.csv') out_nc = os.path.join(outdir, name+'_hour.nc') elif t == 86400: + # removing instantaneous values from daily and monthly files + for v in col_names: + if ('_i' in v) and ('_i_' not in v): + col_names.remove(v) out_csv = os.path.join(outdir, name+'_day.csv') out_nc = os.path.join(outdir, name+'_day.nc') else: + # removing instantaneous values from daily and monthly files + for v in col_names: + if ('_i' in v) and ('_i_' not in v): + col_names.remove(v) out_csv = os.path.join(outdir, name+'_month.csv') out_nc = os.path.join(outdir, name+'_month.nc') if not os.path.isdir(outdir): @@ -160,6 +168,8 @@ def writeNC(outfile, Lx, col_names=None): names = [c for c in col_names if c in list(Lx.keys())] else: names = list(Lx.keys()) + for var in names: + Lx[var].encoding = {} Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True) def getColNames(vars_df, ds, remove_nan_fields=False): From a42c814b59501d9b821786fae14a1c07f478cf6a Mon Sep 17 00:00:00 2001 From: Penny How Date: Mon, 17 Jun 2024 12:13:36 -0100 Subject: [PATCH 18/36] Updates to numpy dependency version and pandas deprecation warnings (#258) * numpy dependency <2.0 * resample rules updated (deprecation warning) * fillna replaced with ffill (deprecation warning) * get_l3 called directly rather than from file * process action restructured --- .github/workflows/process_l3_test.yml | 47 ------------------- .../{process_l2_test.yml => process_test.yml} | 24 ++++++++-- setup.py | 2 +- src/pypromice/process/L1toL2.py | 6 +-- src/pypromice/qc/persistence.py | 2 +- 5 files changed, 24 insertions(+), 57 deletions(-) delete mode 100644 .github/workflows/process_l3_test.yml rename .github/workflows/{process_l2_test.yml => process_test.yml} (69%) diff --git a/.github/workflows/process_l3_test.yml b/.github/workflows/process_l3_test.yml deleted file mode 100644 index 97f8a034..00000000 --- a/.github/workflows/process_l3_test.yml +++ /dev/null @@ -1,47 +0,0 @@ -on: - pull_request: - types: [opened, reopened, synchronize, edited] - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - name: process_l3_test - steps: - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Checkout repo - uses: actions/checkout@v3 - with: - path: "main" - token: ${{ secrets.GITHUB_TOKEN }} - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - pip install wheel - python3 -m pip install --upgrade setuptools - cd $GITHUB_WORKSPACE/main - pip install . - - name: Clone AWS Level 0 data repo for testing - env: - GITLAB_TOKEN : ${{ secrets.GITLAB_TOKEN }} - run: | - cd $GITHUB_WORKSPACE - git clone --depth 1 https://oauth2:${{ env.GITLAB_TOKEN }}@geusgitlab.geus.dk/glaciology-and-climate/promice/aws-l0.git - - name: Run data processing - env: - TEST_STATION: KPC_U CEN2 - shell: bash - run: | - mkdir $GITHUB_WORKSPACE/out/ - for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do - python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/ - done - - name: Upload test output - uses: actions/upload-artifact@v3 - with: - name: result - path: out diff --git a/.github/workflows/process_l2_test.yml b/.github/workflows/process_test.yml similarity index 69% rename from .github/workflows/process_l2_test.yml rename to .github/workflows/process_test.yml index 810086cb..2e938e2a 100644 --- a/.github/workflows/process_l2_test.yml +++ b/.github/workflows/process_test.yml @@ -31,20 +31,34 @@ jobs: run: | cd $GITHUB_WORKSPACE git clone --depth 1 https://oauth2:${{ env.GITLAB_TOKEN }}@geusgitlab.geus.dk/glaciology-and-climate/promice/aws-l0.git - - name: Run L0 to L3 processing + - name: Run L0 to L2 processing env: TEST_STATION: KAN_U HUM shell: bash run: | mkdir $GITHUB_WORKSPACE/out/ - mkdir $GITHUB_WORKSPACE/out/L2/ + mkdir $GITHUB_WORKSPACE/out/L0toL2/ for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do - python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx -o $GITHUB_WORKSPACE/out/L2/ + python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx -o $GITHUB_WORKSPACE/out/L0toL2/ done -# mkdir $GITHUB_WORKSPACE/out/L3/ +# - name: Run L0 to L2 processing +# env: +# TEST_STATION: KAN_U HUM +# shell: bash +# run: | +# mkdir $GITHUB_WORKSPACE/out/L2toL3/ # for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do -# python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2tol3.py -i $GITHUB_WORKSPACE/out/L2/$i/$i_hour.nc -o $GITHUB_WORKSPACE/out/ -t 60min +# python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2tol3.py -i $GITHUB_WORKSPACE/out/L0toL2/$i/$i_hour.nc -o $GITHUB_WORKSPACE/out/L2toL3 -t 60min # done + - name: Run L0 to L3 processing + env: + TEST_STATION: KAN_U HUM + shell: bash + run: | + mkdir $GITHUB_WORKSPACE/out/L0toL3/ + for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do + python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx -o $GITHUB_WORKSPACE/out/L2/ + done - name: Upload test output uses: actions/upload-artifact@v3 with: diff --git a/setup.py b/setup.py index 194d358e..552a7da6 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], }, - install_requires=['numpy>=1.23.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'], + install_requires=['numpy>=1.23.0,<2.0.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'], # extras_require={'postprocess': ['eccodes','scikit-learn>=1.1.0']}, entry_points={ 'console_scripts': [ diff --git a/src/pypromice/process/L1toL2.py b/src/pypromice/process/L1toL2.py index 1a0087cb..c8b32bbc 100644 --- a/src/pypromice/process/L1toL2.py +++ b/src/pypromice/process/L1toL2.py @@ -86,7 +86,7 @@ def toL2( # filtering gps_lat, gps_lon and gps_alt based on the difference to a baseline elevation # right now baseline elevation is gapfilled monthly median elevation - baseline_elevation = (ds.gps_alt.to_series().resample('M').median() + baseline_elevation = (ds.gps_alt.to_series().resample('MS').median() .reindex(ds.time.to_series().index, method='nearest') .ffill().bfill()) mask = (np.abs(ds.gps_alt - baseline_elevation) < 100) & ds.gps_alt.notnull() @@ -327,7 +327,7 @@ def smoothTilt(da: xr.DataArray, threshold=0.2): # we calculate the moving standard deviation over a 3-day sliding window # hourly resampling is necessary to make sure the same threshold can be used # for 10 min and hourly data - moving_std_gap_filled = da.to_series().resample('H').median().rolling( + moving_std_gap_filled = da.to_series().resample('h').median().rolling( 3*24, center=True, min_periods=2 ).std().reindex(da.time, method='bfill').values # we select the good timestamps and gapfill assuming that @@ -354,7 +354,7 @@ def smoothRot(da: xr.DataArray, threshold=4): xarray.DataArray smoothed rotation measurements from inclinometer ''' - moving_std_gap_filled = da.to_series().resample('H').median().rolling( + moving_std_gap_filled = da.to_series().resample('h').median().rolling( 3*24, center=True, min_periods=2 ).std().reindex(da.time, method='bfill').values # same as for tilt with, in addition: diff --git a/src/pypromice/qc/persistence.py b/src/pypromice/qc/persistence.py index f59bf45c..5f5d55f4 100644 --- a/src/pypromice/qc/persistence.py +++ b/src/pypromice/qc/persistence.py @@ -161,6 +161,6 @@ def duration_consecutive_true( # assert series.dtype == bool cumsum = ((series.index - series.index[0]).total_seconds()/3600).to_series(index=series.index) is_first = series.astype("int").diff() == 1 - offset = (is_first * cumsum).replace(0, np.nan).fillna(method="ffill").fillna(0) + offset = (is_first * cumsum).replace(0, np.nan).ffill().fillna(0) return (cumsum - offset) * series From 96e667413649bb81ca8480081ccf08fcbddf15fb Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:43:33 +0200 Subject: [PATCH 19/36] small changes following review, restored variable.csv history renamed new variable.csv moved old variable.csv renamed new variables.csv recreate variables.csv --- src/pypromice/process/get_l2.py | 8 +++++--- src/pypromice/process/get_l2tol3.py | 11 +++++++---- src/pypromice/process/utilities.py | 1 - 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/pypromice/process/get_l2.py b/src/pypromice/process/get_l2.py index 734aa86e..01747501 100644 --- a/src/pypromice/process/get_l2.py +++ b/src/pypromice/process/get_l2.py @@ -3,6 +3,7 @@ from argparse import ArgumentParser import pypromice from pypromice.process.aws import AWS +from pypromice.process.load import getVars, getMeta from pypromice.process.write import prepare_and_write def parse_arguments_l2(): @@ -41,14 +42,15 @@ def get_l2(): # Perform level 1 and 2 processing aws.getL1() aws.getL2() - + v = getVars(args.variables) + m = getMeta(args.metadata) # Write out level 2 if args.outpath is not None: if not os.path.isdir(args.outpath): os.mkdir(args.outpath) if aws.L2.attrs['format'] == 'raw': - prepare_and_write(aws.L2, args.outpath, args.variables, args.metadata, '10min') - prepare_and_write(aws.L2, args.outpath, args.variables, args.metadata, '60min') + prepare_and_write(aws.L2, args.outpath, v, m, '10min') + prepare_and_write(aws.L2, args.outpath, v, m, '60min') if __name__ == "__main__": diff --git a/src/pypromice/process/get_l2tol3.py b/src/pypromice/process/get_l2tol3.py index 8204bfc6..b79a7227 100644 --- a/src/pypromice/process/get_l2tol3.py +++ b/src/pypromice/process/get_l2tol3.py @@ -4,6 +4,7 @@ from argparse import ArgumentParser import pypromice from pypromice.process.L2toL3 import toL3 +from pypromice.process.load import getVars, getMeta from pypromice.process.write import prepare_and_write logger = logging.getLogger(__name__) @@ -48,12 +49,14 @@ def get_l2tol3(): # Perform Level 3 processing l3 = toL3(l2) - + # Write Level 3 dataset to file if output directory given + v = getVars(args.variables) + m = getMeta(args.metadata) if args.outpath is not None: - prepare_and_write(l3, args.outpath, args.variables, args.metadata, '60min') - prepare_and_write(l3, args.outpath, args.variables, args.metadata, '1D') - prepare_and_write(l3, args.outpath, args.variables, args.metadata, 'M') + prepare_and_write(l3, args.outpath, v, m, '60min') + prepare_and_write(l3, args.outpath, v, m, '1D') + prepare_and_write(l3, args.outpath, v, m, 'M') if __name__ == "__main__": get_l2tol3() diff --git a/src/pypromice/process/utilities.py b/src/pypromice/process/utilities.py index 4bcca216..4a7973af 100644 --- a/src/pypromice/process/utilities.py +++ b/src/pypromice/process/utilities.py @@ -48,7 +48,6 @@ def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']): if id not in exempt: if 'gps_lon' not in dataset.keys(): - print("?????????", id, "missing gps_lon") return dataset dataset['gps_lon'] = dataset['gps_lon'] * -1 return dataset From 34643df3fb2f977f4b66c54b2e2eef1ea4a2f67c Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:41:29 +0200 Subject: [PATCH 20/36] buiding a station list instead of a station_dict --- src/pypromice/process/join_l3.py | 44 ++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 669126c5..980d71e6 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -162,27 +162,39 @@ def gcnet_postprocessing(l3): # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] # return l3m -def build_station_dict(config_folder): - station_dict = {} +def build_station_list(config_folder: str, target_station_site: str): + """ + Get a list of unique station IDs (stid) for a given station site. + + Parameters + ---------- + config_folder : str + Path to the folder containing the station configuration TOML files. + target_station_site : str + The station site to filter the station IDs by. + + Returns + ------- + list + A list of unique station IDs that have the specified station site. + """ + unique_stids = [] # Initialize an empty list to store unique station IDs - # Iterate through all files in the given folder for filename in os.listdir(config_folder): if filename.endswith(".toml"): - # Construct full file path file_path = os.path.join(config_folder, filename) - # Load TOML file with open(file_path, 'r') as file: - data = toml.load(file) - station_key = next(iter(data)) - station_data = data[station_key] - - if station_data["station_site"] in station_dict: - station_dict[station_data["station_site"]].append(station_key) - else: - station_dict[station_data["station_site"]] = [station_key] + data = toml.load(file) # Load the TOML file + station_site = data.get("station_site") # Get the station site + stid = data.get("stid") # Get the station ID + + # Check if the station site matches the target and stid is unique + if station_site == target_station_site and stid and stid not in unique_stids: + unique_stids.append(stid) # Add the stid to the list if unique - return station_dict + return unique_stids + def join_l3(): args = parse_arguments_joinl3() @@ -192,10 +204,10 @@ def join_l3(): stream=sys.stdout, ) - station_dict = build_station_dict(args.config_folder) + station_list = build_station_list(args.config_folder, args.site) l3m = xr.Dataset() - for stid in station_dict[args.site]: + for stid in station_list: logger.info(stid) is_promice = False From 35ea17426f3d515d2d31f8e26eb99e22864c28c5 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:49:26 +0200 Subject: [PATCH 21/36] renamed l3m to l3_merged, reintroduced getVars and getMeta --- src/pypromice/process/join_l3.py | 52 +++++++++++++++++--------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 980d71e6..2ad2846e 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import logging, os, sys, unittest, toml, pkg_resources from argparse import ArgumentParser +from pypromice.process.load import getVars, getMeta from pypromice.process.write import prepare_and_write import numpy as np import pandas as pd @@ -137,7 +138,7 @@ def gcnet_postprocessing(l3): return l3 # will be used in the future -# def aligning_surface_heights(l3m, l3): +# def aligning_surface_heights(l3_merged, l3): # df_aux['z_surf_combined'] = \ # df_aux['z_surf_combined'] \ # - df_aux.loc[df_aux.z_surf_combined.last_valid_index(), 'z_surf_combined'] \ @@ -160,7 +161,7 @@ def gcnet_postprocessing(l3): # - fit_fn( # df_in.loc[[df_in.z_surf_combined.first_valid_index()],:].index.astype('int64')[0] # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] - # return l3m + # return l3_merged def build_station_list(config_folder: str, target_station_site: str): """ @@ -206,7 +207,7 @@ def join_l3(): station_list = build_station_list(args.config_folder, args.site) - l3m = xr.Dataset() + l3_merged = xr.Dataset() for stid in station_list: logger.info(stid) @@ -232,62 +233,65 @@ def join_l3(): if 'lat' in l3.keys(): l3 = l3.reset_coords(['lat', 'lon', 'alt']) - if len(l3m)==0: + if len(l3_merged)==0: # saving attributes of station under an attribute called $stid - l3m.attrs[stid] = l3.attrs.copy() + l3_merged.attrs[stid] = l3.attrs.copy() # then stripping attributes attrs_list = list(l3.attrs.keys()) for k in attrs_list: del l3.attrs[k] - l3m = l3.copy() + l3_merged = l3.copy() else: - # if l3 (older data) is missing variables compared to l3m (newer data) + # if l3 (older data) is missing variables compared to l3_merged (newer data) # , then we fill them with nan - for v in l3m.data_vars: + for v in l3_merged.data_vars: if v not in l3.data_vars: l3[v] = l3.t_u*np.nan - # if l3 (older data) has variables that does not have l3m (newer data) + # if l3 (older data) has variables that does not have l3_merged (newer data) # then they are removed from l3 list_dropped = [] for v in l3.data_vars: - if v not in l3m.data_vars: + if v not in l3_merged.data_vars: if v != 'z_stake': list_dropped.append(v) l3 = l3.drop(v) else: - l3m[v] = ('time', l3m.t_u.data*np.nan) + l3_merged[v] = ('time', l3_merged.t_u.data*np.nan) logger.info('Unused variables in older dataset: '+' '.join(list_dropped)) # saving attributes of station under an attribute called $stid - st_attrs = l3m.attrs.get('stations_attributes', {}) + st_attrs = l3_merged.attrs.get('stations_attributes', {}) st_attrs[stid] = l3.attrs.copy() - l3m.attrs["stations_attributes"] = st_attrs + l3_merged.attrs["stations_attributes"] = st_attrs # then stripping attributes attrs_list = list(l3.attrs.keys()) for k in attrs_list: del l3.attrs[k] - l3m.attrs['stations_attributes'][stid]['first_timestamp'] = l3.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() - l3m.attrs['stations_attributes'][stid]['last_timestamp'] = l3m.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + l3_merged.attrs['stations_attributes'][stid]['first_timestamp'] = l3.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + l3_merged.attrs['stations_attributes'][stid]['last_timestamp'] = l3_merged.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() # merging by time block - l3m = xr.concat((l3.sel( + l3_merged = xr.concat((l3.sel( time=slice(l3.time.isel(time=0), - l3m.time.isel(time=0)) - ), l3m), dim='time') + l3_merged.time.isel(time=0)) + ), l3_merged), dim='time') # Assign site id - l3m.attrs['site_id'] = args.site - l3m.attrs['stations'] = station_dict[args.site] - l3m.attrs['level'] = 'L3' + l3_merged.attrs['site_id'] = args.site + l3_merged.attrs['stations'] = ' '.join(station_list) + l3_merged.attrs['level'] = 'L3' + + v = getVars(args.variables) + m = getMeta(args.metadata) if args.outpath is not None: - prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '60min') - prepare_and_write(l3m, args.outpath, args.variables, args.metadata, '1D') - prepare_and_write(l3m, args.outpath, args.variables, args.metadata, 'M') + prepare_and_write(l3_merged, args.outpath, v, m, '60min') + prepare_and_write(l3_merged, args.outpath, v, m, '1D') + prepare_and_write(l3_merged, args.outpath, v, m, 'M') if __name__ == "__main__": join_l3() From ce802a77e279c8cc6928735532963f0033537dc5 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:17:39 +0200 Subject: [PATCH 22/36] moving gcnet_postprocessing as part of readNead --- src/pypromice/process/join_l3.py | 35 ++++++++++++++------------------ 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 2ad2846e..0d52d1ac 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -89,7 +89,21 @@ def readNead(infile): df['timestamp'] = pd.to_datetime(df.timestamp).dt.tz_localize(None) df = df.set_index('timestamp') ds = df.to_xarray() - ds.attrs = meta + ds.attrs = meta + + # renaming variables + file_path = pkg_resources.resource_stream('pypromice', 'ressources/variable_aliases_GC-Net.csv') + var_name = pd.read_csv(file_path) + var_name = var_name.set_index('old_name').GEUS_name + msk = [v for v in var_name.index if v in ds.data_vars] + var_name = var_name.loc[msk].to_dict() + + # combining thermocouple and CS100 temperatures + ds['TA1'] = ds['TA1'].combine_first(ds['TA3']) + ds['TA2'] = ds['TA2'].combine_first(ds['TA4']) + + ds=ds.rename(var_name) + ds=ds.rename({'timestamp':'time'}) return ds @@ -121,22 +135,6 @@ def loadArr(infile): print(f'{name} array loaded from {infile}') return ds, name - -def gcnet_postprocessing(l3): - file_path = pkg_resources.resource_stream('pypromice', 'ressources/variable_aliases_GC-Net.csv') - - var_name = pd.read_csv(file_path) - var_name = var_name.set_index('old_name').GEUS_name - msk = [v for v in var_name.index if v in l3.data_vars] - var_name = var_name.loc[msk].to_dict() - - l3['TA1'] = l3['TA1'].combine_first(l3['TA3']) - l3['TA2'] = l3['TA2'].combine_first(l3['TA4']) - - l3=l3.rename(var_name) - l3=l3.rename({'timestamp':'time'}) - return l3 - # will be used in the future # def aligning_surface_heights(l3_merged, l3): # df_aux['z_surf_combined'] = \ @@ -226,9 +224,6 @@ def join_l3(): l3, _ = loadArr(filepath) - if is_gcnet: - l3 = gcnet_postprocessing(l3) - # lat, lon and alt should be just variables, not coordinates if 'lat' in l3.keys(): l3 = l3.reset_coords(['lat', 'lon', 'alt']) From 5e7673ba2290fb2dcaa20a70abc13eabe1021d96 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:38:59 +0200 Subject: [PATCH 23/36] sorting out the station_list in reverse chronological order --- src/pypromice/process/join_l3.py | 33 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 0d52d1ac..702ff7f4 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -160,8 +160,8 @@ def loadArr(infile): # df_in.loc[[df_in.z_surf_combined.first_valid_index()],:].index.astype('int64')[0] # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] # return l3_merged - -def build_station_list(config_folder: str, target_station_site: str): + +def build_station_list(config_folder: str, target_station_site: str) -> list: """ Get a list of unique station IDs (stid) for a given station site. @@ -194,7 +194,6 @@ def build_station_list(config_folder: str, target_station_site: str): return unique_stids - def join_l3(): args = parse_arguments_joinl3() logging.basicConfig( @@ -203,12 +202,12 @@ def join_l3(): stream=sys.stdout, ) - station_list = build_station_list(args.config_folder, args.site) - - l3_merged = xr.Dataset() - for stid in station_list: - logger.info(stid) - + # Get the list of stations associated with the given site + list_stations = build_station_list(args.config_folder, args.site) + + # Read the datasets and store them into a list along with their latest timestamp + list_station_data = [] + for stid in list_stations: is_promice = False is_gcnet = False filepath = os.path.join(args.folder_l3, stid, stid+'_hour.nc') @@ -222,12 +221,16 @@ def join_l3(): logger.info(stid+' not found either in '+args.folder_l3+' or '+args.folder_gcnet) continue - l3, _ = loadArr(filepath) - - # lat, lon and alt should be just variables, not coordinates - if 'lat' in l3.keys(): - l3 = l3.reset_coords(['lat', 'lon', 'alt']) - + l3, _ = loadArr(filepath) + list_station_data.append((l3, l3.time.max().values, stid)) + + # Sort the list in reverse chronological order so that we start with the latest data + sorted_list_station_data = sorted(list_station_data, key=lambda x: x[1], reverse=True) + sorted_stids = [stid for _, _, stid in sorted_list_station_data] + logger.info('joining %s' % ' '.join(sorted_stids)) + + l3_merged = xr.Dataset() + for l3, _, stid in sorted_list_station_data: if len(l3_merged)==0: # saving attributes of station under an attribute called $stid l3_merged.attrs[stid] = l3.attrs.copy() From 1272a7bd1007c87744ddc06b06da049aed87e5eb Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 18 Jun 2024 17:04:22 +0200 Subject: [PATCH 24/36] using tilde notation in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 552a7da6..a7c3fc3d 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], }, - install_requires=['numpy>=1.23.0,<2.0.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'], + install_requires=['numpy~=1.23', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'], # extras_require={'postprocess': ['eccodes','scikit-learn>=1.1.0']}, entry_points={ 'console_scripts': [ From d445a4519668fee93f083481d1ed0bd4e7378fb0 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Tue, 18 Jun 2024 17:17:59 +0200 Subject: [PATCH 25/36] better initialisation of station_attributes attribute --- src/pypromice/process/join_l3.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 702ff7f4..f6ce011c 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -229,17 +229,27 @@ def join_l3(): sorted_stids = [stid for _, _, stid in sorted_list_station_data] logger.info('joining %s' % ' '.join(sorted_stids)) - l3_merged = xr.Dataset() + l3_merged = None for l3, _, stid in sorted_list_station_data: - if len(l3_merged)==0: - # saving attributes of station under an attribute called $stid - l3_merged.attrs[stid] = l3.attrs.copy() + if l3_merged is None: + # saving attributes of stid + st_attrs = {} + st_attrs[stid] = l3.attrs.copy() + # adding timestamps info + st_attrs[stid]['first_timestamp'] = l3.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + st_attrs[stid]['last_timestamp'] = l3.time.isel(time=-1).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + # then stripping attributes attrs_list = list(l3.attrs.keys()) for k in attrs_list: del l3.attrs[k] - + + # initializing l3_merged with l3 l3_merged = l3.copy() + + # creating the station_attributes attribute in l3_merged + l3_merged.attrs["stations_attributes"] = st_attrs + else: # if l3 (older data) is missing variables compared to l3_merged (newer data) # , then we fill them with nan From 7756f3062ba3606ec7ae965ff8f0932165616abb Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:13:40 +0200 Subject: [PATCH 26/36] moved addMeta, addVars, roundValues, reformatTime, reformatLon to write.py --- src/pypromice/process/utilities.py | 174 ---------------------- src/pypromice/process/write.py | 223 ++++++++++++++++++++++++++--- 2 files changed, 202 insertions(+), 195 deletions(-) diff --git a/src/pypromice/process/utilities.py b/src/pypromice/process/utilities.py index 4a7973af..60e3bab1 100644 --- a/src/pypromice/process/utilities.py +++ b/src/pypromice/process/utilities.py @@ -3,55 +3,8 @@ """ Utilities module for data formatting, populating and metadata handling """ -import datetime, uuid -from importlib import metadata -import pandas as pd import numpy as np -def roundValues(ds, df, col='max_decimals'): - '''Round all variable values in data array based on pre-defined rounding - value in variables look-up table DataFrame - - Parameters - ---------- - ds : xr.Dataset - Dataset to round values in - df : pd.Dataframe - Variable look-up table with rounding values - col : str - Column in variable look-up table that contains rounding values. The - default is "max_decimals" - ''' - df = df[col] - df = df.dropna(how='all') - for var in df.index: - if var not in list(ds.variables): - continue - if df[var] is not np.nan: - ds[var] = ds[var].round(decimals=int(df[var])) - return ds - -def reformat_time(dataset): - '''Re-format time''' - t = dataset['time'].values - dataset['time'] = list(t) - return dataset - -def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']): - '''Switch gps_lon to negative values (degrees_east). We do this here, and - NOT in addMeta, otherwise we switch back to positive when calling getMeta - in joinL2''' - if 'station_id' in dataset.attrs.keys(): - id = dataset.attrs['station_id'] - else: - id = dataset.attrs['site_id'] - - if id not in exempt: - if 'gps_lon' not in dataset.keys(): - return dataset - dataset['gps_lon'] = dataset['gps_lon'] * -1 - return dataset - def popCols(ds, names): '''Populate dataset with all given variable names @@ -113,130 +66,3 @@ def populateMeta(ds, conf, skip): if k not in skip: ds.attrs[k] = conf[k] return ds - -def addVars(ds, variables): - '''Add variable attributes from file to dataset - - Parameters - ---------- - ds : xarray.Dataset - Dataset to add variable attributes to - variables : pandas.DataFrame - Variables lookup table file - - Returns - ------- - ds : xarray.Dataset - Dataset with metadata - ''' - for k in ds.keys(): - if k not in variables.index: continue - ds[k].attrs['standard_name'] = variables.loc[k]['standard_name'] - ds[k].attrs['long_name'] = variables.loc[k]['long_name'] - ds[k].attrs['units'] = variables.loc[k]['units'] - ds[k].attrs['coverage_content_type'] = variables.loc[k]['coverage_content_type'] - ds[k].attrs['coordinates'] = variables.loc[k]['coordinates'] - return ds - -def addMeta(ds, meta): - '''Add metadata attributes from file to dataset - - Parameters - ---------- - ds : xarray.Dataset - Dataset to add metadata attributes to - meta : dict - Metadata file - - Returns - ------- - ds : xarray.Dataset - Dataset with metadata - ''' - if 'gps_lon' in ds.keys(): - ds['lon'] = ds['gps_lon'].mean() - ds['lon'].attrs = ds['gps_lon'].attrs - - ds['lat'] = ds['gps_lat'].mean() - ds['lat'].attrs = ds['gps_lat'].attrs - - ds['alt'] = ds['gps_alt'].mean() - ds['alt'].attrs = ds['gps_alt'].attrs - - # for k in ds.keys(): # for each var - # if 'units' in ds[k].attrs: - # if ds[k].attrs['units'] == 'C': - # ds[k].attrs['units'] = 'degrees_C' - - # https://wiki.esipfed.org/Attribute_Convention_for_Data_Discovery_1-3#geospatial_bounds - if 'station_id' in ds.attrs.keys(): - ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['station_id'])) - else: - ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['site_id'])) - ds.attrs['history'] = 'Generated on ' + datetime.datetime.utcnow().isoformat() - ds.attrs['date_created'] = str(datetime.datetime.now().isoformat()) - ds.attrs['date_modified'] = ds.attrs['date_created'] - ds.attrs['date_issued'] = ds.attrs['date_created'] - ds.attrs['date_metadata_modified'] = ds.attrs['date_created'] - - ds.attrs['geospatial_bounds'] = "POLYGON((" + \ - f"{ds['lat'].min().values} {ds['lon'].min().values}, " + \ - f"{ds['lat'].min().values} {ds['lon'].max().values}, " + \ - f"{ds['lat'].max().values} {ds['lon'].max().values}, " + \ - f"{ds['lat'].max().values} {ds['lon'].min().values}, " + \ - f"{ds['lat'].min().values} {ds['lon'].min().values}))" - - ds.attrs['geospatial_lat_min'] = str(ds['lat'].min().values) - ds.attrs['geospatial_lat_max'] = str(ds['lat'].max().values) - ds.attrs['geospatial_lon_min'] = str(ds['lon'].min().values) - ds.attrs['geospatial_lon_max'] = str(ds['lon'].max().values) - ds.attrs['geospatial_vertical_min'] = str(ds['alt'].min().values) - ds.attrs['geospatial_vertical_max'] = str(ds['alt'].max().values) - ds.attrs['geospatial_vertical_positive'] = 'up' - ds.attrs['time_coverage_start'] = str(ds['time'][0].values) - ds.attrs['time_coverage_end'] = str(ds['time'][-1].values) - - try: - ds.attrs['source']= 'pypromice v' + str(metadata.version('pypromice')) - except: - ds.attrs['source'] = 'pypromice' - - # https://www.digi.com/resources/documentation/digidocs/90001437-13/reference/r_iso_8601_duration_format.htm - try: - ds.attrs['time_coverage_duration'] = str(pd.Timedelta((ds['time'][-1] - ds['time'][0]).values).isoformat()) - ds.attrs['time_coverage_resolution'] = str(pd.Timedelta((ds['time'][1] - ds['time'][0]).values).isoformat()) - except: - ds.attrs['time_coverage_duration'] = str(pd.Timedelta(0).isoformat()) - ds.attrs['time_coverage_resolution'] = str(pd.Timedelta(0).isoformat()) - - # Note: int64 dtype (long int) is incompatible with OPeNDAP access via THREDDS for NetCDF files - # See https://stackoverflow.com/questions/48895227/output-int32-time-dimension-in-netcdf-using-xarray - ds.time.encoding["dtype"] = "i4" # 32-bit signed integer - #ds.time.encoding["calendar"] = 'proleptic_gregorian' # this is default - - # Load metadata attributes and add to Dataset - [_addAttr(ds, key, value) for key,value in meta.items()] - - # Check attribute formating - for k,v in ds.attrs.items(): - if not isinstance(v, str) or not isinstance(v, int): - ds.attrs[k]=str(v) - return ds - -def _addAttr(ds, key, value): - '''Add attribute to xarray dataset - - ds : xr.Dataset - Dataset to add attribute to - key : str - Attribute name, with "." denoting variable attributes - value : str/int - Value for attribute''' - if len(key.split('.')) == 2: - try: - ds[key.split('.')[0]].attrs[key.split('.')[1]] = str(value) - except: - pass - # logger.info(f'Unable to add metadata to {key.split(".")[0]}') - else: - ds.attrs[key] = value diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index 732ec12e..79cab613 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -Write dataset module +Module containing all the functions needed to prepare and AWS data """ -import os, logging +import os, logging, datetime, uuid import pandas as pd -logger = logging.getLogger(__name__) - +from importlib import metadata from pypromice.process.resample import resample_dataset -from pypromice.process import utilities, load +from pypromice.process import load +logger = logging.getLogger(__name__) def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60min', resample=True): '''Prepare data with resampling, formating and metadata population; then @@ -35,7 +35,7 @@ def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60mi d2 = dataset.copy() # Reformat time - d2 = utilities.reformat_time(d2) + d2 = reformat_time(d2) # finding station/site name if 'station_id' in d2.attrs.keys(): @@ -45,7 +45,7 @@ def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60mi # Reformat longitude (to negative values) if 'gps_lon' in d2.keys(): - d2 = utilities.reformat_lon(d2) + d2 = reformat_lon(d2) else: logger.info('%s does not have gpd_lon'%name) @@ -55,11 +55,11 @@ def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60mi if meta_dict is None: meta_dict = load.getMeta() - d2 = utilities.addVars(d2, vars_df) - d2 = utilities.addMeta(d2, meta_dict) + d2 = addVars(d2, vars_df) + d2 = addMeta(d2, meta_dict) # Round all values to specified decimals places - d2 = utilities.roundValues(d2, vars_df) + d2 = roundValues(d2, vars_df) # Get variable names to write out col_names = getColNames(vars_df, d2, remove_nan_fields=True) @@ -105,7 +105,6 @@ def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60mi logger.info(f'Written to {out_nc}') - def writeAll(outpath, station_id, l3_h, l3_d, l3_m, csv_order=None): '''Write L3 hourly, daily and monthly datasets to .nc and .csv files @@ -173,16 +172,16 @@ def writeNC(outfile, Lx, col_names=None): Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True) def getColNames(vars_df, ds, remove_nan_fields=False): - '''Get all variable names for a given data type, based on a variables - look-up table. This is mainly for exporting purposes - - Parameters - ---------- - vars_df : pd.DataFrame - Variables look-up table - ds: xr.dataset - Dataset to write - Returns + ''' + Get variable names for a given dataset with respect to its type and processing level + + The dataset must have the the following attributes: + * level + * number_of_booms when the processing level is <= 2 + + This is mainly for exporting purposes. + + Parameters ------- list Variable names @@ -207,3 +206,185 @@ def getColNames(vars_df, ds, remove_nan_fields=False): var_list.remove(v) return var_list +def addVars(ds, variables): + '''Add variable attributes from file to dataset + + Parameters + ---------- + ds : xarray.Dataset + Dataset to add variable attributes to + variables : pandas.DataFrame + Variables lookup table file + + Returns + ------- + ds : xarray.Dataset + Dataset with metadata + ''' + for k in ds.keys(): + if k not in variables.index: continue + ds[k].attrs['standard_name'] = variables.loc[k]['standard_name'] + ds[k].attrs['long_name'] = variables.loc[k]['long_name'] + ds[k].attrs['units'] = variables.loc[k]['units'] + ds[k].attrs['coverage_content_type'] = variables.loc[k]['coverage_content_type'] + ds[k].attrs['coordinates'] = variables.loc[k]['coordinates'] + return ds + +def addMeta(ds, meta): + '''Add metadata attributes from file to dataset + + Parameters + ---------- + ds : xarray.Dataset + Dataset to add metadata attributes to + meta : dict + Metadata file + + Returns + ------- + ds : xarray.Dataset + Dataset with metadata + ''' + if 'gps_lon' in ds.keys(): + ds['lon'] = ds['gps_lon'].mean() + ds['lon'].attrs = ds['gps_lon'].attrs + + ds['lat'] = ds['gps_lat'].mean() + ds['lat'].attrs = ds['gps_lat'].attrs + + ds['alt'] = ds['gps_alt'].mean() + ds['alt'].attrs = ds['gps_alt'].attrs + + # Attribute convention for data discovery + # https://wiki.esipfed.org/Attribute_Convention_for_Data_Discovery_1-3 + + # Determine the temporal resolution + time_diff = pd.Timedelta((ds['time'][1] - ds['time'][0]).values) + if time_diff == pd.Timedelta('10min'): + sample_rate = "10min" + elif time_diff == pd.Timedelta('1H'): + sample_rate = "hourly" + elif time_diff == pd.Timedelta('1D'): + sample_rate = "daily" + elif time_diff == pd.Timedelta('1M'): + sample_rate = "monthly" + else: + sample_rate = "unknown_sample_rate" + + if 'station_id' in ds.attrs.keys(): + ds.attrs['id'] = 'dk.geus.promice.station.' + ds.attrs['station_id']+'.'+sample_rate + else: + ds.attrs['id'] = 'dk.geus.promice.site.' + ds.attrs['site_id'] +'.'+sample_rate + + ds.attrs['history'] = 'Generated on ' + datetime.datetime.utcnow().isoformat() + ds.attrs['date_created'] = str(datetime.datetime.now().isoformat()) + ds.attrs['date_modified'] = ds.attrs['date_created'] + ds.attrs['date_issued'] = ds.attrs['date_created'] + ds.attrs['date_metadata_modified'] = ds.attrs['date_created'] + ds.attrs['processing_level'] = ds.attrs['level'].replace('L','level ') + + ds.attrs['geospatial_bounds'] = "POLYGON((" + \ + f"{ds['lat'].min().values} {ds['lon'].min().values}, " + \ + f"{ds['lat'].min().values} {ds['lon'].max().values}, " + \ + f"{ds['lat'].max().values} {ds['lon'].max().values}, " + \ + f"{ds['lat'].max().values} {ds['lon'].min().values}, " + \ + f"{ds['lat'].min().values} {ds['lon'].min().values}))" + + ds.attrs['geospatial_lat_min'] = str(ds['lat'].min().values) + ds.attrs['geospatial_lat_max'] = str(ds['lat'].max().values) + ds.attrs['geospatial_lon_min'] = str(ds['lon'].min().values) + ds.attrs['geospatial_lon_max'] = str(ds['lon'].max().values) + ds.attrs['geospatial_vertical_min'] = str(ds['alt'].min().values) + ds.attrs['geospatial_vertical_max'] = str(ds['alt'].max().values) + ds.attrs['geospatial_vertical_positive'] = 'up' + ds.attrs['time_coverage_start'] = str(ds['time'][0].values) + ds.attrs['time_coverage_end'] = str(ds['time'][-1].values) + + try: + ds.attrs['source']= 'pypromice v' + str(metadata.version('pypromice')) + except: + ds.attrs['source'] = 'pypromice' + + # https://www.digi.com/resources/documentation/digidocs/90001437-13/reference/r_iso_8601_duration_format.htm + try: + ds.attrs['time_coverage_duration'] = str(pd.Timedelta((ds['time'][-1] - ds['time'][0]).values).isoformat()) + ds.attrs['time_coverage_resolution'] = str(pd.Timedelta((ds['time'][1] - ds['time'][0]).values).isoformat()) + except: + ds.attrs['time_coverage_duration'] = str(pd.Timedelta(0).isoformat()) + ds.attrs['time_coverage_resolution'] = str(pd.Timedelta(0).isoformat()) + + # Note: int64 dtype (long int) is incompatible with OPeNDAP access via THREDDS for NetCDF files + # See https://stackoverflow.com/questions/48895227/output-int32-time-dimension-in-netcdf-using-xarray + ds.time.encoding["dtype"] = "i4" # 32-bit signed integer + #ds.time.encoding["calendar"] = 'proleptic_gregorian' # this is default + + # Load metadata attributes and add to Dataset + [_addAttr(ds, key, value) for key,value in meta.items()] + + # Check attribute formating + for k,v in ds.attrs.items(): + if not isinstance(v, str) or not isinstance(v, int): + ds.attrs[k]=str(v) + return ds + +def _addAttr(ds, key, value): + '''Add attribute to xarray dataset + + ds : xr.Dataset + Dataset to add attribute to + key : str + Attribute name, with "." denoting variable attributes + value : str/int + Value for attribute''' + if len(key.split('.')) == 2: + try: + ds[key.split('.')[0]].attrs[key.split('.')[1]] = str(value) + except: + pass + # logger.info(f'Unable to add metadata to {key.split(".")[0]}') + else: + ds.attrs[key] = value + +def roundValues(ds, df, col='max_decimals'): + '''Round all variable values in data array based on pre-defined rounding + value in variables look-up table DataFrame + + Parameters + ---------- + ds : xr.Dataset + Dataset to round values in + df : pd.Dataframe + Variable look-up table with rounding values + col : str + Column in variable look-up table that contains rounding values. The + default is "max_decimals" + ''' + df = df[col] + df = df.dropna(how='all') + for var in df.index: + if var not in list(ds.variables): + continue + if df[var] is not np.nan: + ds[var] = ds[var].round(decimals=int(df[var])) + return ds + +def reformat_time(dataset): + '''Re-format time''' + t = dataset['time'].values + dataset['time'] = list(t) + return dataset + +def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']): + '''Switch gps_lon to negative values (degrees_east). We do this here, and + NOT in addMeta, otherwise we switch back to positive when calling getMeta + in joinL2''' + if 'station_id' in dataset.attrs.keys(): + id = dataset.attrs['station_id'] + else: + id = dataset.attrs['site_id'] + + if id not in exempt: + if 'gps_lon' not in dataset.keys(): + return dataset + dataset['gps_lon'] = dataset['gps_lon'] * -1 + return dataset \ No newline at end of file From 48817e65788a1fb13e77e39d78a3c0986cb829d3 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:19:42 +0200 Subject: [PATCH 27/36] Inline comment describing enocding attribute removal when reading a netcdf --- src/pypromice/process/get_l2tol3.py | 5 ++++- src/pypromice/process/join_l2.py | 1 + src/pypromice/process/join_l3.py | 1 + src/pypromice/process/write.py | 3 +-- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pypromice/process/get_l2tol3.py b/src/pypromice/process/get_l2tol3.py index b79a7227..e9ac24ce 100644 --- a/src/pypromice/process/get_l2tol3.py +++ b/src/pypromice/process/get_l2tol3.py @@ -39,9 +39,12 @@ def get_l2tol3(): # Define Level 2 dataset from file with xr.open_dataset(args.inpath) as l2: l2.load() + + # Remove encoding attributes from NetCDF for varname in l2.variables: if 'encoding' in l2[varname].attrs: - del l2[varname].attrs['encoding'] + del l2[varname].attrs['encoding'] + if 'bedrock' in l2.attrs.keys(): l2.attrs['bedrock'] = l2.attrs['bedrock'] == 'True' if 'number_of_booms' in l2.attrs.keys(): diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index 37abff8d..0294feaa 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -30,6 +30,7 @@ def loadArr(infile): elif infile.split('.')[-1].lower() == 'nc': with xr.open_dataset(infile) as ds: ds.load() + # Remove encoding attributes from NetCDF for varname in ds.variables: if 'encoding' in ds[varname].attrs: del ds[varname].attrs['encoding'] diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index f6ce011c..5c10606d 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -123,6 +123,7 @@ def loadArr(infile): f.close() elif infile.split('.')[-1].lower() in 'nc': ds = xr.open_dataset(infile) + # Remove encoding attributes from NetCDF for varname in ds.variables: if 'encoding' in ds[varname].attrs: del ds[varname].attrs['encoding'] diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index 79cab613..e8efee2d 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -167,8 +167,7 @@ def writeNC(outfile, Lx, col_names=None): names = [c for c in col_names if c in list(Lx.keys())] else: names = list(Lx.keys()) - for var in names: - Lx[var].encoding = {} + Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True) def getColNames(vars_df, ds, remove_nan_fields=False): From a49de43739ddb5bfe49b4930e6d7f903f390f9ee Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:37:52 +0200 Subject: [PATCH 28/36] loading toml file as dictionary within join_l3 instead of just reading the stid to join --- src/pypromice/process/join_l3.py | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 5c10606d..84af7485 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -161,24 +161,24 @@ def loadArr(infile): # df_in.loc[[df_in.z_surf_combined.first_valid_index()],:].index.astype('int64')[0] # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] # return l3_merged - + def build_station_list(config_folder: str, target_station_site: str) -> list: """ - Get a list of unique station IDs (stid) for a given station site. + Get a list of unique station information dictionaries for a given station site. Parameters ---------- config_folder : str Path to the folder containing the station configuration TOML files. target_station_site : str - The station site to filter the station IDs by. + The station site to filter the station information by. Returns ------- list - A list of unique station IDs that have the specified station site. + A list of dictionaries containing station information that have the specified station site. """ - unique_stids = [] # Initialize an empty list to store unique station IDs + station_info_list = [] # Initialize an empty list to store station information for filename in os.listdir(config_folder): if filename.endswith(".toml"): @@ -190,25 +190,22 @@ def build_station_list(config_folder: str, target_station_site: str) -> list: stid = data.get("stid") # Get the station ID # Check if the station site matches the target and stid is unique - if station_site == target_station_site and stid and stid not in unique_stids: - unique_stids.append(stid) # Add the stid to the list if unique + if station_site == target_station_site and stid: + station_info = data.copy() # Copy all attributes from the TOML file + station_info_list.append(station_info) # Add the station info to the list - return unique_stids + return station_info_list def join_l3(): args = parse_arguments_joinl3() - logging.basicConfig( - format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", - level=logging.INFO, - stream=sys.stdout, - ) - # Get the list of stations associated with the given site - list_stations = build_station_list(args.config_folder, args.site) + # Get the list of station information dictionaries associated with the given site + list_station_info = build_station_list(args.config_folder, args.site) - # Read the datasets and store them into a list along with their latest timestamp + # Read the datasets and store them into a list along with their latest timestamp and station info list_station_data = [] - for stid in list_stations: + for station_info in list_station_info: + stid = station_info["stid"] is_promice = False is_gcnet = False filepath = os.path.join(args.folder_l3, stid, stid+'_hour.nc') @@ -223,15 +220,16 @@ def join_l3(): continue l3, _ = loadArr(filepath) - list_station_data.append((l3, l3.time.max().values, stid)) + list_station_data.append((l3, l3.time.max().values, station_info)) # Sort the list in reverse chronological order so that we start with the latest data sorted_list_station_data = sorted(list_station_data, key=lambda x: x[1], reverse=True) - sorted_stids = [stid for _, _, stid in sorted_list_station_data] + sorted_stids = [info["stid"] for _, _, info in sorted_list_station_data] logger.info('joining %s' % ' '.join(sorted_stids)) l3_merged = None - for l3, _, stid in sorted_list_station_data: + for l3, _, station_info in sorted_list_station_data: + stid = station_info["stid"] if l3_merged is None: # saving attributes of stid st_attrs = {} From 48f682312a8ddf70d8612964b2e86689beee0260 Mon Sep 17 00:00:00 2001 From: Penny How Date: Wed, 19 Jun 2024 10:41:27 -0100 Subject: [PATCH 29/36] ressources renamed to resources (#261) --- MANIFEST.in | 1 + setup.py | 1 - src/pypromice/process/join_l3.py | 2 +- src/pypromice/process/load.py | 6 +++--- src/pypromice/{ressources => resources}/__init__.py | 0 src/pypromice/{ressources => resources}/file_attributes.csv | 0 .../{ressources => resources}/variable_aliases_GC-Net.csv | 0 src/pypromice/{ressources => resources}/variables.csv | 0 8 files changed, 5 insertions(+), 5 deletions(-) rename src/pypromice/{ressources => resources}/__init__.py (100%) rename src/pypromice/{ressources => resources}/file_attributes.csv (100%) rename src/pypromice/{ressources => resources}/variable_aliases_GC-Net.csv (100%) rename src/pypromice/{ressources => resources}/variables.csv (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 4c48bf4e..08bdee57 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include src/pypromice/test/* +include src/pypromice/resources/* diff --git a/setup.py b/setup.py index a7c3fc3d..835a6b84 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", package_data={ - "pypromice.ressources": ["variables.csv", "file_attributes.csv", "variable_aliases_GC-Net.csv"], "pypromice.tx": ["payload_formats.csv", "payload_types.csv"], "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 5c10606d..1f603621 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -92,7 +92,7 @@ def readNead(infile): ds.attrs = meta # renaming variables - file_path = pkg_resources.resource_stream('pypromice', 'ressources/variable_aliases_GC-Net.csv') + file_path = pkg_resources.resource_stream('pypromice', 'resources/variable_aliases_GC-Net.csv') var_name = pd.read_csv(file_path) var_name = var_name.set_index('old_name').GEUS_name msk = [v for v in var_name.index if v in ds.data_vars] diff --git a/src/pypromice/process/load.py b/src/pypromice/process/load.py index a2334096..97b52375 100644 --- a/src/pypromice/process/load.py +++ b/src/pypromice/process/load.py @@ -137,7 +137,7 @@ def getVars(v_file=None): Variables dataframe ''' if v_file is None: - with pkg_resources.resource_stream('pypromice', 'ressources/variables.csv') as stream: + with pkg_resources.resource_stream('pypromice', 'resources/variables.csv') as stream: return pd.read_csv(stream, index_col=0, comment="#", encoding='utf-8') else: return pd.read_csv(v_file, index_col=0, comment="#") @@ -160,7 +160,7 @@ def getMeta(m_file=None, delimiter=','): ''' meta={} if m_file is None: - with pkg_resources.resource_stream('pypromice', 'ressources/file_attributes.csv') as stream: + with pkg_resources.resource_stream('pypromice', 'resources/file_attributes.csv') as stream: lines = stream.read().decode("utf-8") lines = lines.split("\n") else: @@ -171,4 +171,4 @@ def getMeta(m_file=None, delimiter=','): meta[l.split(',')[0]] = l.split(delimiter)[1].split('\n')[0].replace(';',',') except IndexError: pass - return meta \ No newline at end of file + return meta diff --git a/src/pypromice/ressources/__init__.py b/src/pypromice/resources/__init__.py similarity index 100% rename from src/pypromice/ressources/__init__.py rename to src/pypromice/resources/__init__.py diff --git a/src/pypromice/ressources/file_attributes.csv b/src/pypromice/resources/file_attributes.csv similarity index 100% rename from src/pypromice/ressources/file_attributes.csv rename to src/pypromice/resources/file_attributes.csv diff --git a/src/pypromice/ressources/variable_aliases_GC-Net.csv b/src/pypromice/resources/variable_aliases_GC-Net.csv similarity index 100% rename from src/pypromice/ressources/variable_aliases_GC-Net.csv rename to src/pypromice/resources/variable_aliases_GC-Net.csv diff --git a/src/pypromice/ressources/variables.csv b/src/pypromice/resources/variables.csv similarity index 100% rename from src/pypromice/ressources/variables.csv rename to src/pypromice/resources/variables.csv From 79a597234d8dfe239c225a0039eaddb1d3a942e7 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:04:17 +0200 Subject: [PATCH 30/36] using project attribute of a station locate AWS file and specify whether it's a Nead file --- src/pypromice/process/join_l3.py | 32 +++++++++++++------------------- src/pypromice/process/write.py | 1 + 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 84af7485..8c286ab5 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -107,20 +107,16 @@ def readNead(infile): return ds -def loadArr(infile): +def loadArr(infile, isNead): if infile.split('.')[-1].lower() in 'csv': - with open(infile) as f: - text_splitted = f.read().splitlines() - first_char = text_splitted[0][0] - - if first_char != '#': + if isNead: + ds = readNead(infile) + else: df = pd.read_csv(infile) df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None) df = df.set_index('time') ds = xr.Dataset.from_dataframe(df) - else: - ds = readNead(infile) - f.close() + elif infile.split('.')[-1].lower() in 'nc': ds = xr.open_dataset(infile) # Remove encoding attributes from NetCDF @@ -206,20 +202,17 @@ def join_l3(): list_station_data = [] for station_info in list_station_info: stid = station_info["stid"] - is_promice = False - is_gcnet = False + filepath = os.path.join(args.folder_l3, stid, stid+'_hour.nc') - if os.path.isfile(filepath): - is_promice = True - else: + isNead = False + if station_info["project"].lower() in ["historical gc-net", "glaciobasis"]: filepath = os.path.join(args.folder_gcnet, stid+'.csv') - if os.path.isfile(filepath): - is_gcnet = True - if not is_promice and not is_gcnet: - logger.info(stid+' not found either in '+args.folder_l3+' or '+args.folder_gcnet) + isNead = True + if not os.path.isfile(filepath): + logger.info(stid+' is from an project '+args.folder_l3+' or '+args.folder_gcnet) continue - l3, _ = loadArr(filepath) + l3, _ = loadArr(filepath, isNead) list_station_data.append((l3, l3.time.max().values, station_info)) # Sort the list in reverse chronological order so that we start with the latest data @@ -230,6 +223,7 @@ def join_l3(): l3_merged = None for l3, _, station_info in sorted_list_station_data: stid = station_info["stid"] + if l3_merged is None: # saving attributes of stid st_attrs = {} diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index e8efee2d..540f8f2d 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -5,6 +5,7 @@ """ import os, logging, datetime, uuid import pandas as pd +import numpy as np from importlib import metadata from pypromice.process.resample import resample_dataset from pypromice.process import load From a184277594ed20188bf470e20caa9a2f0e31724a Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:08:21 +0200 Subject: [PATCH 31/36] update test after moving addVars and addMeta --- src/pypromice/process/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypromice/process/test.py b/src/pypromice/process/test.py index b9e15807..3a0e5d9d 100644 --- a/src/pypromice/process/test.py +++ b/src/pypromice/process/test.py @@ -5,7 +5,7 @@ """ from pypromice.process.aws import AWS from pypromice.process.load import getVars, getMeta -from pypromice.process.utilities import addVars, addMeta +from pypromice.process.write import addVars, addMeta import xarray as xr import pandas as pd import unittest, datetime, os From a2fe85835f8a6e8a684ab72cf59254d51387feeb Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:08:42 +0200 Subject: [PATCH 32/36] fixed logger message in resample --- src/pypromice/process/resample.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pypromice/process/resample.py b/src/pypromice/process/resample.py index daef8624..7c7e2ed7 100644 --- a/src/pypromice/process/resample.py +++ b/src/pypromice/process/resample.py @@ -36,11 +36,11 @@ def resample_dataset(ds_h, t): # recalculating wind direction from averaged directional wind speeds for var in ['wdir_u','wdir_l']: if var in df_d.columns: - if ('wspd_x_'+var.split('_')[1] in df_d.columns) & ('wspd_x_'+var.split('_')[1] in df_d.columns): + if ('wspd_x_'+var.split('_')[1] in df_d.columns) & ('wspd_y_'+var.split('_')[1] in df_d.columns): df_d[var] = _calcWindDir(df_d['wspd_x_'+var.split('_')[1]], df_d['wspd_y_'+var.split('_')[1]]) else: - logger.info(var+' in dataframe but not wspd_x_'+var.split('_')[1]+' nor wspd_x_'+var.split('_')[1]) + logger.info(var+' in dataframe but not wspd_x_'+var.split('_')[1]+' nor wspd_y_'+var.split('_')[1]) # recalculating relative humidity from average vapour pressure and average # saturation vapor pressure From bd1983c793868b24bdb9c10646d02e3f9be1ea8d Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:16:24 +0200 Subject: [PATCH 33/36] better definition of monthly sample rates in addMeta --- src/pypromice/process/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index 540f8f2d..cc4d8fe5 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -266,7 +266,7 @@ def addMeta(ds, meta): sample_rate = "hourly" elif time_diff == pd.Timedelta('1D'): sample_rate = "daily" - elif time_diff == pd.Timedelta('1M'): + elif 28 <= time_diff.days <= 31: sample_rate = "monthly" else: sample_rate = "unknown_sample_rate" From 036ba7885852cc9fa90d0b2e7765860c2e2e9785 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:22:02 +0200 Subject: [PATCH 34/36] dummy datasaet built in unit test now has 'level' attribute --- src/pypromice/process/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pypromice/process/test.py b/src/pypromice/process/test.py index 3a0e5d9d..f5e26601 100644 --- a/src/pypromice/process/test.py +++ b/src/pypromice/process/test.py @@ -39,6 +39,7 @@ def testAddAll(self): d['time'] = [datetime.datetime.now(), datetime.datetime.now()-timedelta(days=365)] d.attrs['station_id']='TEST' + d.attrs['level']='L2_test' meta = getMeta() d = addVars(d, v) d = addMeta(d, meta) From 9d56d84070bb2de042412f46b265907416496680 Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:53:20 +0200 Subject: [PATCH 35/36] not storing timestamp_max for each station but pulling the info directly from the dataset when sorting --- src/pypromice/process/join_l3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py index 7f8b4835..905ee81f 100644 --- a/src/pypromice/process/join_l3.py +++ b/src/pypromice/process/join_l3.py @@ -213,15 +213,15 @@ def join_l3(): continue l3, _ = loadArr(filepath, isNead) - list_station_data.append((l3, l3.time.max().values, station_info)) + list_station_data.append((l3, station_info)) # Sort the list in reverse chronological order so that we start with the latest data - sorted_list_station_data = sorted(list_station_data, key=lambda x: x[1], reverse=True) - sorted_stids = [info["stid"] for _, _, info in sorted_list_station_data] + sorted_list_station_data = sorted(list_station_data, key=lambda x: x[0].time.max(), reverse=True) + sorted_stids = [info["stid"] for _, info in sorted_list_station_data] logger.info('joining %s' % ' '.join(sorted_stids)) l3_merged = None - for l3, _, station_info in sorted_list_station_data: + for l3, station_info in sorted_list_station_data: stid = station_info["stid"] if l3_merged is None: From 1263d7fdc4783079e8b574d724743fa6bc5dbeab Mon Sep 17 00:00:00 2001 From: Baptiste Vandecrux <35140661+BaptisteVandecrux@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:55:16 +0200 Subject: [PATCH 36/36] removing unecessary import of addMeta, roundValues --- src/pypromice/process/join_l2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index 0294feaa..0c31db48 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -3,8 +3,6 @@ import pandas as pd import xarray as xr from argparse import ArgumentParser -from pypromice.process.utilities import addMeta, roundValues -from pypromice.process.write import prepare_and_write from pypromice.process.L1toL2 import correctPrecip logger = logging.getLogger(__name__)