diff --git a/.github/workflows/process_l3_test.yml b/.github/workflows/process_l3_test.yml deleted file mode 100644 index 97f8a034..00000000 --- a/.github/workflows/process_l3_test.yml +++ /dev/null @@ -1,47 +0,0 @@ -on: - pull_request: - types: [opened, reopened, synchronize, edited] - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - name: process_l3_test - steps: - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Checkout repo - uses: actions/checkout@v3 - with: - path: "main" - token: ${{ secrets.GITHUB_TOKEN }} - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - pip install wheel - python3 -m pip install --upgrade setuptools - cd $GITHUB_WORKSPACE/main - pip install . - - name: Clone AWS Level 0 data repo for testing - env: - GITLAB_TOKEN : ${{ secrets.GITLAB_TOKEN }} - run: | - cd $GITHUB_WORKSPACE - git clone --depth 1 https://oauth2:${{ env.GITLAB_TOKEN }}@geusgitlab.geus.dk/glaciology-and-climate/promice/aws-l0.git - - name: Run data processing - env: - TEST_STATION: KPC_U CEN2 - shell: bash - run: | - mkdir $GITHUB_WORKSPACE/out/ - for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do - python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l3.py -c $GITHUB_WORKSPACE/aws-l0/raw/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/raw -o $GITHUB_WORKSPACE/out/ - done - - name: Upload test output - uses: actions/upload-artifact@v3 - with: - name: result - path: out diff --git a/.github/workflows/process_l2_test.yml b/.github/workflows/process_test.yml similarity index 69% rename from .github/workflows/process_l2_test.yml rename to .github/workflows/process_test.yml index 810086cb..2e938e2a 100644 --- a/.github/workflows/process_l2_test.yml +++ b/.github/workflows/process_test.yml @@ -31,20 +31,34 @@ jobs: run: | cd $GITHUB_WORKSPACE git clone --depth 1 https://oauth2:${{ env.GITLAB_TOKEN }}@geusgitlab.geus.dk/glaciology-and-climate/promice/aws-l0.git - - name: Run L0 to L3 processing + - name: Run L0 to L2 processing env: TEST_STATION: KAN_U HUM shell: bash run: | mkdir $GITHUB_WORKSPACE/out/ - mkdir $GITHUB_WORKSPACE/out/L2/ + mkdir $GITHUB_WORKSPACE/out/L0toL2/ for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do - python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx -o $GITHUB_WORKSPACE/out/L2/ + python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx -o $GITHUB_WORKSPACE/out/L0toL2/ done -# mkdir $GITHUB_WORKSPACE/out/L3/ +# - name: Run L0 to L2 processing +# env: +# TEST_STATION: KAN_U HUM +# shell: bash +# run: | +# mkdir $GITHUB_WORKSPACE/out/L2toL3/ # for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do -# python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2tol3.py -i $GITHUB_WORKSPACE/out/L2/$i/$i_hour.nc -o $GITHUB_WORKSPACE/out/ -t 60min +# python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2tol3.py -i $GITHUB_WORKSPACE/out/L0toL2/$i/$i_hour.nc -o $GITHUB_WORKSPACE/out/L2toL3 -t 60min # done + - name: Run L0 to L3 processing + env: + TEST_STATION: KAN_U HUM + shell: bash + run: | + mkdir $GITHUB_WORKSPACE/out/L0toL3/ + for i in $(echo ${{ env.TEST_STATION }} | tr ' ' '\n'); do + python3 $GITHUB_WORKSPACE/main/src/pypromice/process/get_l2.py -c $GITHUB_WORKSPACE/aws-l0/tx/config/$i.toml -i $GITHUB_WORKSPACE/aws-l0/tx -o $GITHUB_WORKSPACE/out/L2/ + done - name: Upload test output uses: actions/upload-artifact@v3 with: diff --git a/MANIFEST.in b/MANIFEST.in index 4c48bf4e..08bdee57 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include src/pypromice/test/* +include src/pypromice/resources/* diff --git a/setup.py b/setup.py index 5675d6f6..835a6b84 100644 --- a/setup.py +++ b/setup.py @@ -31,20 +31,19 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.8", package_data={ - "pypromice.process": ["metadata.csv", "variables.csv"], "pypromice.tx": ["payload_formats.csv", "payload_types.csv"], "pypromice.qc.percentiles": ["thresholds.csv"], "pypromice.postprocess": ["station_configurations.toml", "positions_seed.csv"], }, - install_requires=['numpy>=1.23.0', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'], + install_requires=['numpy~=1.23', 'pandas>=1.5.0', 'xarray>=2022.6.0', 'toml', 'scipy>=1.9.0', 'Bottleneck', 'netcdf4', 'pyDataverse==0.3.1', 'eccodes', 'scikit-learn>=1.1.0'], # extras_require={'postprocess': ['eccodes','scikit-learn>=1.1.0']}, entry_points={ 'console_scripts': [ 'get_promice_data = pypromice.get.get_promice_data:get_promice_data', 'get_l0tx = pypromice.tx.get_l0tx:get_l0tx', 'join_l2 = pypromice.process.join_l2:join_l2', + 'join_l3 = pypromice.process.join_l3:join_l3', 'get_l2 = pypromice.process.get_l2:get_l2', - 'get_l3 = pypromice.process.get_l3:get_l3', 'get_l2tol3 = pypromice.process.get_l2tol3:get_l2tol3', 'get_watsontx = pypromice.tx.get_watsontx:get_watsontx', 'get_bufr = pypromice.postprocess.get_bufr:main', diff --git a/src/pypromice/process/L0toL1.py b/src/pypromice/process/L0toL1.py index a056a08b..0d55c730 100644 --- a/src/pypromice/process/L0toL1.py +++ b/src/pypromice/process/L0toL1.py @@ -5,9 +5,9 @@ import numpy as np import pandas as pd import xarray as xr -import re - +import re, logging from pypromice.process.value_clipping import clip_values +logger = logging.getLogger(__name__) def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100): @@ -28,9 +28,10 @@ def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100): ------- ds : xarray.Dataset Level 1 dataset - ''' + ''' assert(type(L0) == xr.Dataset) ds = L0 + ds.attrs['level'] = 'L1' for l in list(ds.keys()): if l not in ['time', 'msg_i', 'gps_lat', 'gps_lon', 'gps_alt', 'gps_time']: @@ -247,7 +248,7 @@ def getPressDepth(z_pt, p, pt_antifreeze, pt_z_factor, pt_z_coef, pt_z_p_coef): rho_af = 1145 else: rho_af = np.nan - print('ERROR: Incorrect metadata: "pt_antifreeze" = ' + + logger.info('ERROR: Incorrect metadata: "pt_antifreeze" = ' + f'{pt_antifreeze}. Antifreeze mix only supported at 50% or 100%') # assert(False) diff --git a/src/pypromice/process/L1toL2.py b/src/pypromice/process/L1toL2.py index d9ee1e8e..c8b32bbc 100644 --- a/src/pypromice/process/L1toL2.py +++ b/src/pypromice/process/L1toL2.py @@ -70,6 +70,7 @@ def toL2( Level 2 dataset ''' ds = L1.copy(deep=True) # Reassign dataset + ds.attrs['level'] = 'L2' try: ds = adjustTime(ds) # Adjust time after a user-defined csv files ds = flagNAN(ds) # Flag NaNs after a user-defined csv files @@ -85,7 +86,7 @@ def toL2( # filtering gps_lat, gps_lon and gps_alt based on the difference to a baseline elevation # right now baseline elevation is gapfilled monthly median elevation - baseline_elevation = (ds.gps_alt.to_series().resample('M').median() + baseline_elevation = (ds.gps_alt.to_series().resample('MS').median() .reindex(ds.time.to_series().index, method='nearest') .ffill().bfill()) mask = (np.abs(ds.gps_alt - baseline_elevation) < 100) & ds.gps_alt.notnull() @@ -326,7 +327,7 @@ def smoothTilt(da: xr.DataArray, threshold=0.2): # we calculate the moving standard deviation over a 3-day sliding window # hourly resampling is necessary to make sure the same threshold can be used # for 10 min and hourly data - moving_std_gap_filled = da.to_series().resample('H').median().rolling( + moving_std_gap_filled = da.to_series().resample('h').median().rolling( 3*24, center=True, min_periods=2 ).std().reindex(da.time, method='bfill').values # we select the good timestamps and gapfill assuming that @@ -353,7 +354,7 @@ def smoothRot(da: xr.DataArray, threshold=4): xarray.DataArray smoothed rotation measurements from inclinometer ''' - moving_std_gap_filled = da.to_series().resample('H').median().rolling( + moving_std_gap_filled = da.to_series().resample('h').median().rolling( 3*24, center=True, min_periods=2 ).std().reindex(da.time, method='bfill').values # same as for tilt with, in addition: diff --git a/src/pypromice/process/L2toL3.py b/src/pypromice/process/L2toL3.py index 9751122e..2e05ff04 100755 --- a/src/pypromice/process/L2toL3.py +++ b/src/pypromice/process/L2toL3.py @@ -32,6 +32,7 @@ def toL3(L2, T_0=273.15, z_0=0.001, R_d=287.05, eps=0.622, es_0=6.1071, 1013.246. ''' ds = L2 + ds.attrs['level'] = 'L3' T_100 = _getTempK(T_0) # Get steam point temperature as K diff --git a/src/pypromice/process/aws.py b/src/pypromice/process/aws.py index 0d830f34..7314a19e 100644 --- a/src/pypromice/process/aws.py +++ b/src/pypromice/process/aws.py @@ -52,9 +52,7 @@ def __init__(self, config_file, inpath, var_file=None, meta_file=None): L0 = self.loadL0() self.L0=[] for l in L0: - n = write.getColNames(self.vars, - l.attrs['number_of_booms'], - l.attrs['format']) + n = write.getColNames(self.vars, l) self.L0.append(utilities.popCols(l, n)) self.L1 = None @@ -106,18 +104,6 @@ def getL3(self): logger.info('Level 3 processing...') self.L3 = toL3(self.L2) - # def resample(self, dataset): - # '''Resample dataset to specific temporal resolution (based on input - # data type)''' - # f = [l.attrs['format'] for l in self.L0] - # if 'raw' in f or 'STM' in f: - # logger.info('Resampling to 10 minute') - # resampled = resample_dataset(dataset, '10min') - # else: - # resampled = resample_dataset(dataset, '60min') - # logger.info('Resampling to hour') - # return resampled - def writeArr(self, dataset, outpath, t=None): '''Write L3 data to .nc and .csv hourly and daily files @@ -141,23 +127,6 @@ def writeArr(self, dataset, outpath, t=None): else: write.prepare_and_write(dataset, outpath, self.vars, self.meta, '60min') - - # def addAttributes(self, dataset): - # '''Add variable and attribute metadata - - # Parameters - # ---------- - # dataset : xr.Dataset - # Dataset (i.e. L2 or L3) object - - # Returns - # ------- - # d2 : xr.Dataset - # Data object with attributes - # ''' - # d2 = utilities.addVars(dataset, self.vars) - # d2 = utilities.addMeta(d2, self.meta) - # return d2 def loadConfig(self, config_file, inpath): '''Load configuration from .toml file diff --git a/src/pypromice/process/get_l2.py b/src/pypromice/process/get_l2.py index 0122b53d..01747501 100644 --- a/src/pypromice/process/get_l2.py +++ b/src/pypromice/process/get_l2.py @@ -3,8 +3,8 @@ from argparse import ArgumentParser import pypromice from pypromice.process.aws import AWS -from pypromice.process.write import prepare_and_write from pypromice.process.load import getVars, getMeta +from pypromice.process.write import prepare_and_write def parse_arguments_l2(): parser = ArgumentParser(description="AWS L2 processor") @@ -30,38 +30,27 @@ def get_l2(): level=logging.INFO, stream=sys.stdout, ) - - # Define variables (either from file or pypromice package defaults) - if args.variables is None: - v = os.path.join(os.path.dirname(pypromice.__file__),'process/variables.csv') - else: - v = args.variables - - # Define metadata (either from file or pypromice package defaults) - if args.variables is None: - m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv') - else: - m = args.metadata # Define input path station_name = args.config_file.split('/')[-1].split('.')[0] station_path = os.path.join(args.inpath, station_name) if os.path.exists(station_path): - aws = AWS(args.config_file, station_path, v, m) + aws = AWS(args.config_file, station_path, args.variables, args.metadata) else: - aws = AWS(args.config_file, args.inpath, v, m) + aws = AWS(args.config_file, args.inpath, args.variables, args.metadata) # Perform level 1 and 2 processing aws.getL1() aws.getL2() - + v = getVars(args.variables) + m = getMeta(args.metadata) # Write out level 2 if args.outpath is not None: if not os.path.isdir(args.outpath): os.mkdir(args.outpath) if aws.L2.attrs['format'] == 'raw': - prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '10min') - prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '60min') + prepare_and_write(aws.L2, args.outpath, v, m, '10min') + prepare_and_write(aws.L2, args.outpath, v, m, '60min') if __name__ == "__main__": diff --git a/src/pypromice/process/get_l2tol3.py b/src/pypromice/process/get_l2tol3.py index e8ed5a96..e9ac24ce 100644 --- a/src/pypromice/process/get_l2tol3.py +++ b/src/pypromice/process/get_l2tol3.py @@ -3,9 +3,10 @@ import xarray as xr from argparse import ArgumentParser import pypromice -from pypromice.process.load import getVars, getMeta from pypromice.process.L2toL3 import toL3 +from pypromice.process.load import getVars, getMeta from pypromice.process.write import prepare_and_write +logger = logging.getLogger(__name__) def parse_arguments_l2tol3(debug_args=None): parser = ArgumentParser(description="AWS L3 script for the processing L3 "+ @@ -34,14 +35,16 @@ def get_l2tol3(): level=logging.INFO, stream=sys.stdout, ) - - # Define variables and metadata (either from file or pypromice package defaults) - v = getVars(args.variables) - m = getMeta(args.metadata) # Define Level 2 dataset from file with xr.open_dataset(args.inpath) as l2: - l2.load() + l2.load() + + # Remove encoding attributes from NetCDF + for varname in l2.variables: + if 'encoding' in l2[varname].attrs: + del l2[varname].attrs['encoding'] + if 'bedrock' in l2.attrs.keys(): l2.attrs['bedrock'] = l2.attrs['bedrock'] == 'True' if 'number_of_booms' in l2.attrs.keys(): @@ -49,8 +52,10 @@ def get_l2tol3(): # Perform Level 3 processing l3 = toL3(l2) - + # Write Level 3 dataset to file if output directory given + v = getVars(args.variables) + m = getMeta(args.metadata) if args.outpath is not None: prepare_and_write(l3, args.outpath, v, m, '60min') prepare_and_write(l3, args.outpath, v, m, '1D') diff --git a/src/pypromice/process/get_l3.py b/src/pypromice/process/get_l3.py deleted file mode 100644 index 688ada59..00000000 --- a/src/pypromice/process/get_l3.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -import logging, os, sys, unittest -from argparse import ArgumentParser -import pypromice -from pypromice.process.aws import AWS - -def parse_arguments_l3(): - parser = ArgumentParser(description="AWS L3 processor") - - parser.add_argument('-c', '--config_file', type=str, required=True, - help='Path to config (TOML) file') - parser.add_argument('-i', '--inpath', type=str, required=True, - help='Path to input data') - parser.add_argument('-o', '--outpath', default=None, type=str, required=False, - help='Path where to write output') - parser.add_argument('-v', '--variables', default=None, type=str, - required=False, help='File path to variables look-up table') - parser.add_argument('-m', '--metadata', default=None, type=str, - required=False, help='File path to metadata') - parser.add_argument('-t', '--time', default=None, type=str, - required=False, help='Resampling frequency') - args = parser.parse_args() - return args - -def get_l3(): - args = parse_arguments_l3() - - logging.basicConfig( - format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", - level=logging.INFO, - stream=sys.stdout, - ) - - # Define variables (either from file or pypromice package defaults) - if args.variables is None: - v = os.path.join(os.path.dirname(pypromice.__file__),'process/variables.csv') - else: - v = args.variables - - # Define metadata (either from file or pypromice package defaults) - if args.variables is None: - m = os.path.join(os.path.dirname(pypromice.__file__),'process/metadata.csv') - else: - m = args.metadata - - # Define input path - station_name = args.config_file.split('/')[-1].split('.')[0] - station_path = os.path.join(args.inpath, station_name) - if os.path.exists(station_path): - aws = AWS(args.config_file, station_path, v, m) - else: - aws = AWS(args.config_file, args.inpath, v, m) - - # Perform level 1 to 3 processing - aws.getL1() - aws.getL2() - aws.getL3() - - # Write out level 3 - if args.outpath is not None: - if not os.path.isdir(args.outpath): - os.mkdir(args.outpath) - aws.writeArr(aws.L3, args.outpath, args.time) - -if __name__ == "__main__": - get_l3() - diff --git a/src/pypromice/process/join_l2.py b/src/pypromice/process/join_l2.py index 28bf2e0e..0c31db48 100644 --- a/src/pypromice/process/join_l2.py +++ b/src/pypromice/process/join_l2.py @@ -1,12 +1,10 @@ #!/usr/bin/env python -import os, unittest +import logging, sys, os, unittest import pandas as pd import xarray as xr from argparse import ArgumentParser -from pypromice.process.load import getVars, getMeta -from pypromice.process.utilities import addMeta, roundValues -from pypromice.process.write import prepare_and_write from pypromice.process.L1toL2 import correctPrecip +logger = logging.getLogger(__name__) def parse_arguments_join(): parser = ArgumentParser(description="AWS L2 joiner for merging together two L2 products, for example an L2 RAW and L2 TX data product. An hourly, daily and monthly L2 data product is outputted to the defined output path") @@ -30,6 +28,10 @@ def loadArr(infile): elif infile.split('.')[-1].lower() == 'nc': with xr.open_dataset(infile) as ds: ds.load() + # Remove encoding attributes from NetCDF + for varname in ds.variables: + if 'encoding' in ds[varname].attrs: + del ds[varname].attrs['encoding'] try: name = ds.attrs['station_id'] @@ -41,17 +43,17 @@ def loadArr(infile): if 'number_of_booms' in ds.attrs.keys(): ds.attrs['number_of_booms'] = int(ds.attrs['number_of_booms']) - print(f'{name} array loaded from {infile}') + logger.info(f'{name} array loaded from {infile}') return ds, name def join_l2(): args = parse_arguments_join() - - # Define variables and metadata (either from file or pypromice package defaults) - v = getVars(args.variables) - m = getMeta(args.metadata) - + logging.basicConfig( + format="%(asctime)s; %(levelname)s; %(name)s; %(message)s", + level=logging.INFO, + stream=sys.stdout, + ) # Check files if os.path.isfile(args.file1) and os.path.isfile(args.file2): @@ -63,7 +65,7 @@ def join_l2(): if n1.lower() == n2.lower(): # Merge arrays - print(f'Combining {args.file1} with {args.file2}...') + logger.info(f'Combining {args.file1} with {args.file2}...') name = n1 all_ds = ds1.combine_first(ds2) @@ -77,28 +79,29 @@ def join_l2(): all_ds['precip_l_cor'], _ = correctPrecip(all_ds['precip_l'], all_ds['wspd_l']) else: - print(f'Mismatched station names {n1}, {n2}') + logger.info(f'Mismatched station names {n1}, {n2}') exit() elif os.path.isfile(args.file1): ds1, name = loadArr(args.file1) - print(f'Only one file found {args.file1}...') + logger.info(f'Only one file found {args.file1}...') all_ds = ds1 elif os.path.isfile(args.file2): ds2, name = loadArr(args.file2) - print(f'Only one file found {args.file2}...') + logger.info(f'Only one file found {args.file2}...') all_ds = ds2 else: - print(f'Invalid files {args.file1}, {args.file2}') + logger.info(f'Invalid files {args.file1}, {args.file2}') exit() + all_ds.attrs['format'] = 'merged RAW and TX' # Resample to hourly, daily and monthly datasets and write to file - prepare_and_write(all_ds, args.outpath, v, m, resample = False) + prepare_and_write(all_ds, args.outpath, args.variables, args.metadata, resample = False) - print(f'Files saved to {os.path.join(args.outpath, name)}...') + logger.info(f'Files saved to {os.path.join(args.outpath, name)}...') if __name__ == "__main__": - join_levels() + join_l2() diff --git a/src/pypromice/process/join_l3.py b/src/pypromice/process/join_l3.py new file mode 100644 index 00000000..905ee81f --- /dev/null +++ b/src/pypromice/process/join_l3.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python +import logging, os, sys, unittest, toml, pkg_resources +from argparse import ArgumentParser +from pypromice.process.load import getVars, getMeta +from pypromice.process.write import prepare_and_write +import numpy as np +import pandas as pd +import xarray as xr +logger = logging.getLogger(__name__) + +def parse_arguments_joinl3(debug_args=None): + parser = ArgumentParser(description="AWS L3 script for the processing L3 data from L2 and merging the L3 data with its historical site. An hourly, daily and monthly L3 data product is outputted to the defined output path") + parser.add_argument('-c', '--config_folder', type=str, required=True, + help='Path to folder with sites configuration (TOML) files') + parser.add_argument('-s', '--site', default=None, type=str, required=False, + help='Name of site to process (default: all sites are processed)') + + parser.add_argument('-l3', '--folder_l3', type=str, required=True, + help='Path to level 3 folder') + parser.add_argument('-gc', '--folder_gcnet', type=str, required=False, + help='Path to GC-Net historical L1 folder') + + parser.add_argument('-o', '--outpath', default=os.getcwd(), type=str, required=True, + help='Path where to write output') + + parser.add_argument('-v', '--variables', default=None, type=str, required=False, + help='Path to variables look-up table .csv file for variable name retained'''), + parser.add_argument('-m', '--metadata', default=None, type=str, required=False, + help='Path to metadata table .csv file for metadata information'''), + parser.add_argument('-d', '--datatype', default='raw', type=str, required=False, + help='Data type to output, raw or tx') + args = parser.parse_args(args=debug_args) + return args + +def readNead(infile): + with open(infile) as f: + fmt = f.readline() + assert(fmt[0] == "#") + assert(fmt.split("#")[1].split()[0] == "NEAD") + assert(fmt.split("#")[1].split()[1] == "1.0") + assert(fmt.split("#")[1].split()[2] == "UTF-8") + + line = f.readline() + assert(line[0] == "#") + assert(line.split("#")[1].strip() == '[METADATA]') + + meta = {} + fields = {} + section = 'meta' + while True: + line = f.readline() + if line.strip(' ') == '#': continue + if line == "# [DATA]\n": break # done reading header + if line == "# [FIELDS]\n": + section = 'fields' + continue # done reading header + + if line[0] == "\n": continue # blank line + assert(line[0] == "#") # if not blank, must start with "#" + + key_eq_val = line.split("#")[1].strip() + if key_eq_val == '' or key_eq_val == None: continue # Line is just "#" or "# " or "# #"... + assert("=" in key_eq_val), print(line, key_eq_val) + key = key_eq_val.split("=")[0].strip() + val = key_eq_val.split("=")[1].strip() + + # Convert from string to number if it is a number + if val.strip('-').strip('+').replace('.','').isdigit(): + val = float(val) + if val == int(val): + val = int(val) + + if section == 'meta': meta[key] = val + if section == 'fields': fields[key] = val + # done reading header + + # Find delimiter and fields for reading NEAD as simple CSV + assert("field_delimiter" in meta.keys()) + assert("fields" in fields.keys()) + FD = meta["field_delimiter"] + names = [_.strip() for _ in fields.pop('fields').split(FD)] + + df = pd.read_csv(infile, + comment = "#", + names = names, + sep = FD, + usecols=np.arange(len(names)), + skip_blank_lines = True) + df['timestamp'] = pd.to_datetime(df.timestamp).dt.tz_localize(None) + df = df.set_index('timestamp') + ds = df.to_xarray() + ds.attrs = meta + + # renaming variables + file_path = pkg_resources.resource_stream('pypromice', 'resources/variable_aliases_GC-Net.csv') + var_name = pd.read_csv(file_path) + var_name = var_name.set_index('old_name').GEUS_name + msk = [v for v in var_name.index if v in ds.data_vars] + var_name = var_name.loc[msk].to_dict() + + # combining thermocouple and CS100 temperatures + ds['TA1'] = ds['TA1'].combine_first(ds['TA3']) + ds['TA2'] = ds['TA2'].combine_first(ds['TA4']) + + ds=ds.rename(var_name) + ds=ds.rename({'timestamp':'time'}) + return ds + + +def loadArr(infile, isNead): + if infile.split('.')[-1].lower() in 'csv': + if isNead: + ds = readNead(infile) + else: + df = pd.read_csv(infile) + df['time'] = pd.to_datetime(df['time']).dt.tz_localize(None) + df = df.set_index('time') + ds = xr.Dataset.from_dataframe(df) + + elif infile.split('.')[-1].lower() in 'nc': + ds = xr.open_dataset(infile) + # Remove encoding attributes from NetCDF + for varname in ds.variables: + if 'encoding' in ds[varname].attrs: + del ds[varname].attrs['encoding'] + + try: + name = ds.attrs['station_name'] + except: + name = infile.split('/')[-1].split('.')[0].split('_hour')[0].split('_10min')[0] + + print(f'{name} array loaded from {infile}') + return ds, name + +# will be used in the future +# def aligning_surface_heights(l3_merged, l3): + # df_aux['z_surf_combined'] = \ + # df_aux['z_surf_combined'] \ + # - df_aux.loc[df_aux.z_surf_combined.last_valid_index(), 'z_surf_combined'] \ + # + df_v6.loc[df_v6.z_surf_combined.first_valid_index(), 'z_surf_combined'] + + # if s == 'Swiss Camp 10m': + # df.loc[:df.HS_combined.first_valid_index(), 'HS_combined'] = \ + # df2.loc[:df.HS_combined.first_valid_index(), 'HS_combined'] \ + # - df2.loc[df2.HS_combined.last_valid_index(), 'HS_combined'] \ + # + df.loc[df.HS_combined.first_valid_index(), 'HS_combined'] + + + # df.loc[df.HS_combined.diff()==0,'HS_combined'] = np.nan + + # fit = np.polyfit(df.loc[df.HS_combined.notnull(),:].index.astype('int64'), + # df.loc[df.HS_combined.notnull(),'HS_combined'], 1) + # fit_fn = np.poly1d(fit) + + # df['HS_combined'] = df['HS_combined'].values \ + # - fit_fn( + # df_in.loc[[df_in.z_surf_combined.first_valid_index()],:].index.astype('int64')[0] + # ) + df_in.loc[df_in.z_surf_combined.first_valid_index(), 'z_surf_combined'] + # return l3_merged + +def build_station_list(config_folder: str, target_station_site: str) -> list: + """ + Get a list of unique station information dictionaries for a given station site. + + Parameters + ---------- + config_folder : str + Path to the folder containing the station configuration TOML files. + target_station_site : str + The station site to filter the station information by. + + Returns + ------- + list + A list of dictionaries containing station information that have the specified station site. + """ + station_info_list = [] # Initialize an empty list to store station information + + for filename in os.listdir(config_folder): + if filename.endswith(".toml"): + file_path = os.path.join(config_folder, filename) + + with open(file_path, 'r') as file: + data = toml.load(file) # Load the TOML file + station_site = data.get("station_site") # Get the station site + stid = data.get("stid") # Get the station ID + + # Check if the station site matches the target and stid is unique + if station_site == target_station_site and stid: + station_info = data.copy() # Copy all attributes from the TOML file + station_info_list.append(station_info) # Add the station info to the list + + return station_info_list + +def join_l3(): + args = parse_arguments_joinl3() + + # Get the list of station information dictionaries associated with the given site + list_station_info = build_station_list(args.config_folder, args.site) + + # Read the datasets and store them into a list along with their latest timestamp and station info + list_station_data = [] + for station_info in list_station_info: + stid = station_info["stid"] + + filepath = os.path.join(args.folder_l3, stid, stid+'_hour.nc') + isNead = False + if station_info["project"].lower() in ["historical gc-net", "glaciobasis"]: + filepath = os.path.join(args.folder_gcnet, stid+'.csv') + isNead = True + if not os.path.isfile(filepath): + logger.info(stid+' is from an project '+args.folder_l3+' or '+args.folder_gcnet) + continue + + l3, _ = loadArr(filepath, isNead) + list_station_data.append((l3, station_info)) + + # Sort the list in reverse chronological order so that we start with the latest data + sorted_list_station_data = sorted(list_station_data, key=lambda x: x[0].time.max(), reverse=True) + sorted_stids = [info["stid"] for _, info in sorted_list_station_data] + logger.info('joining %s' % ' '.join(sorted_stids)) + + l3_merged = None + for l3, station_info in sorted_list_station_data: + stid = station_info["stid"] + + if l3_merged is None: + # saving attributes of stid + st_attrs = {} + st_attrs[stid] = l3.attrs.copy() + # adding timestamps info + st_attrs[stid]['first_timestamp'] = l3.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + st_attrs[stid]['last_timestamp'] = l3.time.isel(time=-1).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + + # then stripping attributes + attrs_list = list(l3.attrs.keys()) + for k in attrs_list: + del l3.attrs[k] + + # initializing l3_merged with l3 + l3_merged = l3.copy() + + # creating the station_attributes attribute in l3_merged + l3_merged.attrs["stations_attributes"] = st_attrs + + else: + # if l3 (older data) is missing variables compared to l3_merged (newer data) + # , then we fill them with nan + for v in l3_merged.data_vars: + if v not in l3.data_vars: + l3[v] = l3.t_u*np.nan + + # if l3 (older data) has variables that does not have l3_merged (newer data) + # then they are removed from l3 + list_dropped = [] + for v in l3.data_vars: + if v not in l3_merged.data_vars: + if v != 'z_stake': + list_dropped.append(v) + l3 = l3.drop(v) + else: + l3_merged[v] = ('time', l3_merged.t_u.data*np.nan) + logger.info('Unused variables in older dataset: '+' '.join(list_dropped)) + + # saving attributes of station under an attribute called $stid + st_attrs = l3_merged.attrs.get('stations_attributes', {}) + st_attrs[stid] = l3.attrs.copy() + l3_merged.attrs["stations_attributes"] = st_attrs + + # then stripping attributes + attrs_list = list(l3.attrs.keys()) + for k in attrs_list: + del l3.attrs[k] + + l3_merged.attrs['stations_attributes'][stid]['first_timestamp'] = l3.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + l3_merged.attrs['stations_attributes'][stid]['last_timestamp'] = l3_merged.time.isel(time=0).dt.strftime( date_format='%Y-%m-%d %H:%M:%S').item() + + # merging by time block + l3_merged = xr.concat((l3.sel( + time=slice(l3.time.isel(time=0), + l3_merged.time.isel(time=0)) + ), l3_merged), dim='time') + + + # Assign site id + l3_merged.attrs['site_id'] = args.site + l3_merged.attrs['stations'] = ' '.join(station_list) + l3_merged.attrs['level'] = 'L3' + + v = getVars(args.variables) + m = getMeta(args.metadata) + if args.outpath is not None: + prepare_and_write(l3_merged, args.outpath, v, m, '60min') + prepare_and_write(l3_merged, args.outpath, v, m, '1D') + prepare_and_write(l3_merged, args.outpath, v, m, 'M') + +if __name__ == "__main__": + join_l3() + diff --git a/src/pypromice/process/load.py b/src/pypromice/process/load.py index 186784c6..97b52375 100644 --- a/src/pypromice/process/load.py +++ b/src/pypromice/process/load.py @@ -98,7 +98,7 @@ def getL0(infile, nodata, cols, skiprows, file_version, try: df.index = pd.to_datetime(df.index) except ValueError as e: - logger.info("\n", infile) + logger.info("\n"+ infile) logger.info("\nValueError:") logger.info(e) logger.info('\t\t> Trying pd.to_datetime with format=mixed') @@ -120,6 +120,7 @@ def getL0(infile, nodata, cols, skiprows, file_version, # Carry relevant metadata with ds ds = xr.Dataset.from_dataframe(df) + ds.attrs['level'] = 'L0' return ds def getVars(v_file=None): @@ -136,11 +137,11 @@ def getVars(v_file=None): Variables dataframe ''' if v_file is None: - with pkg_resources.resource_stream('pypromice', 'process/variables.csv') as stream: + with pkg_resources.resource_stream('pypromice', 'resources/variables.csv') as stream: return pd.read_csv(stream, index_col=0, comment="#", encoding='utf-8') else: return pd.read_csv(v_file, index_col=0, comment="#") - + def getMeta(m_file=None, delimiter=','): #TODO change to DataFrame output to match variables.csv '''Load metadata table @@ -159,7 +160,7 @@ def getMeta(m_file=None, delimiter=','): ''' meta={} if m_file is None: - with pkg_resources.resource_stream('pypromice', 'process/metadata.csv') as stream: + with pkg_resources.resource_stream('pypromice', 'resources/file_attributes.csv') as stream: lines = stream.read().decode("utf-8") lines = lines.split("\n") else: @@ -170,4 +171,4 @@ def getMeta(m_file=None, delimiter=','): meta[l.split(',')[0]] = l.split(delimiter)[1].split('\n')[0].replace(';',',') except IndexError: pass - return meta \ No newline at end of file + return meta diff --git a/src/pypromice/process/resample.py b/src/pypromice/process/resample.py index b4af07e2..7c7e2ed7 100644 --- a/src/pypromice/process/resample.py +++ b/src/pypromice/process/resample.py @@ -34,13 +34,13 @@ def resample_dataset(ds_h, t): df_d = ds_h.to_dataframe().resample(t).mean() # recalculating wind direction from averaged directional wind speeds - for var in ['wdir_u','wdir_l','wdir_i']: + for var in ['wdir_u','wdir_l']: if var in df_d.columns: - if ('wspd_x_'+var.split('_')[1] in df_d.columns) & ('wspd_x_'+var.split('_')[1] in df_d.columns): + if ('wspd_x_'+var.split('_')[1] in df_d.columns) & ('wspd_y_'+var.split('_')[1] in df_d.columns): df_d[var] = _calcWindDir(df_d['wspd_x_'+var.split('_')[1]], df_d['wspd_y_'+var.split('_')[1]]) else: - logger.info(var,'in dataframe but not','wspd_x_'+var.split('_')[1],'wspd_x_'+var.split('_')[1]) + logger.info(var+' in dataframe but not wspd_x_'+var.split('_')[1]+' nor wspd_y_'+var.split('_')[1]) # recalculating relative humidity from average vapour pressure and average # saturation vapor pressure @@ -53,8 +53,9 @@ def resample_dataset(ds_h, t): df_d[var] = (p_vap.to_series().resample(t).mean() \ / es_wtr.to_series().resample(t).mean())*100 - df_d[var+'_cor'] = (p_vap.to_series().resample(t).mean() \ - / es_cor.to_series().resample(t).mean())*100 + if var+'_cor' in df_d.keys(): + df_d[var+'_cor'] = (p_vap.to_series().resample(t).mean() \ + / es_cor.to_series().resample(t).mean())*100 vals = [xr.DataArray(data=df_d[c], dims=['time'], coords={'time':df_d.index}, attrs=ds_h[c].attrs) for c in df_d.columns] diff --git a/src/pypromice/process/test.py b/src/pypromice/process/test.py index df81e88d..f5e26601 100644 --- a/src/pypromice/process/test.py +++ b/src/pypromice/process/test.py @@ -5,7 +5,7 @@ """ from pypromice.process.aws import AWS from pypromice.process.load import getVars, getMeta -from pypromice.process.utilities import addVars, addMeta +from pypromice.process.write import addVars, addMeta import xarray as xr import pandas as pd import unittest, datetime, os @@ -39,6 +39,7 @@ def testAddAll(self): d['time'] = [datetime.datetime.now(), datetime.datetime.now()-timedelta(days=365)] d.attrs['station_id']='TEST' + d.attrs['level']='L2_test' meta = getMeta() d = addVars(d, v) d = addMeta(d, meta) @@ -61,21 +62,21 @@ def testCLIgetl2(self): '''Test get_l2 CLI''' exit_status = os.system('get_l2 -h') self.assertEqual(exit_status, 0) - - def testCLIgetl3(self): - '''Test get_l3 CLI''' - exit_status = os.system('get_l3 -h') - self.assertEqual(exit_status, 0) def testCLIjoinl2(self): '''Test join_l2 CLI''' exit_status = os.system('join_l2 -h') self.assertEqual(exit_status, 0) - def testCLIjoinl3(self): - '''Test join_l2 CLI''' + def testCLIgetl2tol3(self): + '''Test get_l2tol3 CLI''' exit_status = os.system('get_l2tol3 -h') self.assertEqual(exit_status, 0) + + def testCLIjoinl3(self): + '''Test join_l3 CLI''' + exit_status = os.system('join_l3 -h') + self.assertEqual(exit_status, 0) if __name__ == "__main__": diff --git a/src/pypromice/process/utilities.py b/src/pypromice/process/utilities.py index ad0d66ca..60e3bab1 100644 --- a/src/pypromice/process/utilities.py +++ b/src/pypromice/process/utilities.py @@ -3,48 +3,8 @@ """ Utilities module for data formatting, populating and metadata handling """ -import datetime, uuid -from importlib import metadata -import pandas as pd import numpy as np -def roundValues(ds, df, col='max_decimals'): - '''Round all variable values in data array based on pre-defined rounding - value in variables look-up table DataFrame - - Parameters - ---------- - ds : xr.Dataset - Dataset to round values in - df : pd.Dataframe - Variable look-up table with rounding values - col : str - Column in variable look-up table that contains rounding values. The - default is "max_decimals" - ''' - df = df[col] - df = df.dropna(how='all') - for var in df.index: - if var not in list(ds.variables): - continue - if df[var] is not np.nan: - ds[var] = ds[var].round(decimals=int(df[var])) - return ds - -def reformat_time(dataset): - '''Re-format time''' - t = dataset['time'].values - dataset['time'] = list(t) - return dataset - -def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']): - '''Switch gps_lon to negative values (degrees_east). We do this here, and - NOT in addMeta, otherwise we switch back to positive when calling getMeta - in joinL2''' - if dataset.attrs['station_id'] not in exempt: - dataset['gps_lon'] = dataset['gps_lon'] * -1 - return dataset - def popCols(ds, names): '''Populate dataset with all given variable names @@ -101,133 +61,8 @@ def populateMeta(ds, conf, skip): ds : xarray.Dataset L0 dataset with metadata populated as Dataset attributes ''' - meta = {} # skip = ["columns", "skiprows"] for k in conf.keys(): - if k not in skip: meta[k] = conf[k] - ds.attrs = meta - return ds - - -def addVars(ds, variables): - '''Add variable attributes from file to dataset - - Parameters - ---------- - ds : xarray.Dataset - Dataset to add variable attributes to - variables : pandas.DataFrame - Variables lookup table file - - Returns - ------- - ds : xarray.Dataset - Dataset with metadata - ''' - for k in ds.keys(): - if k not in variables.index: continue - ds[k].attrs['standard_name'] = variables.loc[k]['standard_name'] - ds[k].attrs['long_name'] = variables.loc[k]['long_name'] - ds[k].attrs['units'] = variables.loc[k]['units'] - ds[k].attrs['coverage_content_type'] = variables.loc[k]['coverage_content_type'] - ds[k].attrs['coordinates'] = variables.loc[k]['coordinates'] + if k not in skip: ds.attrs[k] = conf[k] return ds -def addMeta(ds, meta): - '''Add metadata attributes from file to dataset - - Parameters - ---------- - ds : xarray.Dataset - Dataset to add metadata attributes to - meta : dict - Metadata file - - Returns - ------- - ds : xarray.Dataset - Dataset with metadata - ''' - ds['lon'] = ds['gps_lon'].mean() - ds['lon'].attrs = ds['gps_lon'].attrs - - ds['lat'] = ds['gps_lat'].mean() - ds['lat'].attrs = ds['gps_lat'].attrs - - ds['alt'] = ds['gps_alt'].mean() - ds['alt'].attrs = ds['gps_alt'].attrs - - # for k in ds.keys(): # for each var - # if 'units' in ds[k].attrs: - # if ds[k].attrs['units'] == 'C': - # ds[k].attrs['units'] = 'degrees_C' - - # https://wiki.esipfed.org/Attribute_Convention_for_Data_Discovery_1-3#geospatial_bounds - ds.attrs['id'] = 'dk.geus.promice:' + str(uuid.uuid3(uuid.NAMESPACE_DNS, ds.attrs['station_id'])) - ds.attrs['history'] = 'Generated on ' + datetime.datetime.utcnow().isoformat() - ds.attrs['date_created'] = str(datetime.datetime.now().isoformat()) - ds.attrs['date_modified'] = ds.attrs['date_created'] - ds.attrs['date_issued'] = ds.attrs['date_created'] - ds.attrs['date_metadata_modified'] = ds.attrs['date_created'] - - ds.attrs['geospatial_bounds'] = "POLYGON((" + \ - f"{ds['lat'].min().values} {ds['lon'].min().values}, " + \ - f"{ds['lat'].min().values} {ds['lon'].max().values}, " + \ - f"{ds['lat'].max().values} {ds['lon'].max().values}, " + \ - f"{ds['lat'].max().values} {ds['lon'].min().values}, " + \ - f"{ds['lat'].min().values} {ds['lon'].min().values}))" - - ds.attrs['geospatial_lat_min'] = str(ds['lat'].min().values) - ds.attrs['geospatial_lat_max'] = str(ds['lat'].max().values) - ds.attrs['geospatial_lon_min'] = str(ds['lon'].min().values) - ds.attrs['geospatial_lon_max'] = str(ds['lon'].max().values) - ds.attrs['geospatial_vertical_min'] = str(ds['alt'].min().values) - ds.attrs['geospatial_vertical_max'] = str(ds['alt'].max().values) - ds.attrs['geospatial_vertical_positive'] = 'up' - ds.attrs['time_coverage_start'] = str(ds['time'][0].values) - ds.attrs['time_coverage_end'] = str(ds['time'][-1].values) - - try: - ds.attrs['source']= 'pypromice v' + str(metadata.version('pypromice')) - except: - ds.attrs['source'] = 'pypromice' - - # https://www.digi.com/resources/documentation/digidocs/90001437-13/reference/r_iso_8601_duration_format.htm - try: - ds.attrs['time_coverage_duration'] = str(pd.Timedelta((ds['time'][-1] - ds['time'][0]).values).isoformat()) - ds.attrs['time_coverage_resolution'] = str(pd.Timedelta((ds['time'][1] - ds['time'][0]).values).isoformat()) - except: - ds.attrs['time_coverage_duration'] = str(pd.Timedelta(0).isoformat()) - ds.attrs['time_coverage_resolution'] = str(pd.Timedelta(0).isoformat()) - - # Note: int64 dtype (long int) is incompatible with OPeNDAP access via THREDDS for NetCDF files - # See https://stackoverflow.com/questions/48895227/output-int32-time-dimension-in-netcdf-using-xarray - ds.time.encoding["dtype"] = "i4" # 32-bit signed integer - #ds.time.encoding["calendar"] = 'proleptic_gregorian' # this is default - - # Load metadata attributes and add to Dataset - [_addAttr(ds, key, value) for key,value in meta.items()] - - # Check attribute formating - for k,v in ds.attrs.items(): - if not isinstance(v, str) or not isinstance(v, int): - ds.attrs[k]=str(v) - return ds - -def _addAttr(ds, key, value): - '''Add attribute to xarray dataset - - ds : xr.Dataset - Dataset to add attribute to - key : str - Attribute name, with "." denoting variable attributes - value : str/int - Value for attribute''' - if len(key.split('.')) == 2: - try: - ds[key.split('.')[0]].attrs[key.split('.')[1]] = str(value) - except: - pass - # logger.info(f'Unable to add metadata to {key.split(".")[0]}') - else: - ds.attrs[key] = value \ No newline at end of file diff --git a/src/pypromice/process/variables.csv b/src/pypromice/process/variables.csv deleted file mode 100644 index 2960259b..00000000 --- a/src/pypromice/process/variables.csv +++ /dev/null @@ -1,92 +0,0 @@ -field,standard_name,long_name,units,lo,hi,OOL,station_type,data_type,max_decimals,coverage_content_type,coordinates,instantaneous_hourly,comment -time,time,Time,yyyy-mm-dd HH:MM:SS,,,,all,all,,physicalMeasurement,time lat lon alt,, -rec,record,Record,-,,,,all,,0,referenceInformation,time lat lon alt,,L0 only -p_u,air_pressure,Air pressure (upper boom),hPa,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,False, -p_l,air_pressure,Air pressure (lower boom),hPa,650,1100,dshf_l dlhf_l qh_l,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -t_u,air_temperature,Air temperature (upper boom),degrees_C,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,False, -t_l,air_temperature,Air temperature (lower boom),degrees_C,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l ,two-boom,all,4,physicalMeasurement,time lat lon alt,False,PT100 temperature at boom -rh_u,relative_humidity,Relative humidity (upper boom),%,0,100,rh_u_cor,all,all,4,physicalMeasurement,time lat lon alt,False, -rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,0,150,dshf_u dlhf_u qh_u,all,all,4,modelResult,time lat lon alt,False, -qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,0,100,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -rh_l,relative_humidity,Relative humidity (lower boom),%,0,100,rh_l_cor,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,0,150,dshf_l dlhf_l qh_l,two-boom,all,4,modelResult,time lat lon alt,False, -qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,0,100,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -wspd_u,wind_speed,Wind speed (upper boom),m s-1,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,all,4,physicalMeasurement,time lat lon alt,False, -wspd_l,wind_speed,Wind speed (lower boom),m s-1,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,all,4,physicalMeasurement,time lat lon alt,False, -wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,1,360,wspd_x_u wspd_y_u,all,all,4,physicalMeasurement,time lat lon alt,False, -wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,,,,one-boom,,4,qualityInformation,time lat lon alt,False,L0 only -wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,1,360,wspd_x_l wspd_y_l,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,-100,100,wdir_u wspd_u,all,all,4,modelResult,time lat lon alt,False,L0 only -wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,-100,100,wdir_u wspd_u,all,all,4,modelResult,time lat lon alt,False,L0 only -wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,-100,100,wdir_l wspd_l,two-boom,all,4,modelResult,time lat lon alt,False,L0 only -wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,-100,100,wdir_l wspd_l,two-boom,all,4,modelResult,time lat lon alt,False,L0 only -dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,-10,1500,albedo dsr_cor usr_cor,all,all,4,physicalMeasurement,time lat lon alt,False,"Actually radiation_at_sensor, not flux. Units 1E-5 V. Engineering units." -dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,-10,1000,albedo dsr_cor usr_cor,all,all,4,physicalMeasurement,time lat lon alt,False, -usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,0,1000,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -albedo,surface_albedo,Albedo,-,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,50,500,albedo dsr_cor usr_cor cc t_surf,all,all,4,physicalMeasurement,time lat lon alt,False, -ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,50,500,t_surf,all,all,4,physicalMeasurement,time lat lon alt,False, -cc,cloud_area_fraction,Cloud cover,%,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -t_surf,surface_temperature,Surface temperature,C,-80,40,dshf_u dlhf_u qh_u,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,,,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,,,,all,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,,,,two-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -z_boom_u,distance_to_surface_from_boom,Upper boom height,m,0.3,10,dshf_u dlhf_u qh_u,all,all,4,physicalMeasurement,time lat lon alt,True, -z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,,,,all,,4,qualityInformation,time lat lon alt,True,L0 only -z_boom_l,distance_to_surface_from_boom,Lower boom height,m,0.3,5,dshf_l dlhf_l qh_l,two-boom,all,4,physicalMeasurement,time lat lon alt,True, -z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,,,,two-boom,,4,qualityInformation,time lat lon alt,True,L0 only -z_stake,distance_to_surface_from_stake_assembly,Stake height,m,0.3,8,,one-boom,all,4,physicalMeasurement,time lat lon alt,True,HeightStakes(m) -z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,,,,one-boom,,4,qualityInformation,time lat lon alt,True,L0 only -z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,0,30,z_pt_cor,one-boom,all,4,physicalMeasurement,time lat lon alt,False,DepthPressureTransducer(m) -z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,0,30,,one-boom,all,4,modelResult,time lat lon alt,False,Derived value (L2 or later) -precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,0,,precip_u_cor precip_u_rate,all,all,4,physicalMeasurement,time lat lon alt,True,Without wind/undercatch correction -precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,0,,,all,all,4,modelResult,time lat lon alt,True,With wind/undercatch correction -precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,0,,,all,all,4,modelResult,time lat lon alt,True,L0 only -precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,0,,precip_l_cor precip_l_rate,two-boom,all,4,physicalMeasurement,time lat lon alt,True,Without wind/undercatch correction -precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,0,,,two-boom,all,4,modelResult,time lat lon alt,True,With wind/undercatch correction -precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,0,,,two-boom,all,4,modelResult,time lat lon alt,True,L0 only -t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,t1 is installed @ 1 m depth -t_i_2,ice_temperature_at_t2,Ice temperature at sensor 2,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_3,ice_temperature_at_t3,Ice temperature at sensor 3,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_4,ice_temperature_at_t4,Ice temperature at sensor 4,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_5,ice_temperature_at_t5,Ice temperature at sensor 5,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_6,ice_temperature_at_t6,Ice temperature at sensor 6,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_7,ice_temperature_at_t7,Ice temperature at sensor 7,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False, -t_i_8,ice_temperature_at_t8,Ice temperature at sensor 8,degrees_C,-80,1,,all,all,4,physicalMeasurement,time lat lon alt,False,t8 is installed @ 10 m depth -t_i_9,ice_temperature_at_t9,Ice temperature at sensor 9,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -t_i_10,ice_temperature_at_t10,Ice temperature at sensor 10,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -t_i_11,ice_temperature_at_t11,Ice temperature at sensor 11,degrees_C,-80,1,,two-boom,all,4,physicalMeasurement,time lat lon alt,False, -tilt_x,platform_view_angle_x,Tilt to east,degrees,-30,30,dsr_cor usr_cor albedo,all,all,4,physicalMeasurement,time lat lon alt,False, -tilt_y,platform_view_angle_y,Tilt to north,degrees,-30,30,dsr_cor usr_cor albedo,all,all,4,physicalMeasurement,time lat lon alt,False, -rot,platform_azimuth_angle,Station rotation from true North,degrees,0,360,,all,all,2,physicalMeasurement,time lat lon alt,False,v4 addition -gps_lat,gps_latitude,Latitude,degrees_north,50,83,,all,all,6,coordinate,time lat lon alt,True, -gps_lon,gps_longitude,Longitude,degrees_east,5,70,,all,all,6,coordinate,time lat lon alt,True, -gps_alt,gps_altitude,Altitude,m,0,3000,,all,all,2,coordinate,time lat lon alt,True, -gps_time,gps_time,GPS time,s,0,240000,,all,all,,coordinate,time lat lon alt,True, -gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,,,,one-boom,all,,physicalMeasurement,time lat lon alt,True, -gps_geounit,gps_geounit,GeoUnit,-,,,,all,,,qualityInformation,time lat lon alt,True,L0 only -gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,,,,all,all,2,qualityInformation,time lat lon alt,True,NMEA: Horizontal dilution of precision -gps_numsat,gps_numsat,GPS number of satellites,-,,,,,all,0,qualityInformation,time lat lon alt,True,L0 only -gps_q,gps_q,Quality,-,,,,,all,,qualityInformation,time lat lon alt,True,L0 only -lat,gps_mean_latitude,GPS mean latitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True, -lon,gps_mean_longitude,GPS mean longitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True, -alt,gps_mean_altitude,GPS mean altitude (from all time-series),degrees,,,,all,,6,modelResult,time lat lon alt,True, -batt_v,battery_voltage,Battery voltage,V,0,30,,all,all,2,physicalMeasurement,time lat lon alt,True, -batt_v_ini,,,-,0,30,,,all,2,physicalMeasurement,time lat lon alt,True,L0 only -batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,0,30,,,all,2,physicalMeasurement,time lat lon alt,True,L0 only -fan_dc_u,fan_current,Fan current (upper boom),mA,0,200,,all,all,2,physicalMeasurement,time lat lon alt,True, -fan_dc_l,fan_current,Fan current (lower boom),mA,0,200,,two-boom,all,2,physicalMeasurement,time lat lon alt,True, -freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,0,10000,precip_u,,all,,physicalMeasurement,time lat lon alt,True,L0 only -t_log,temperature_of_logger,Logger temperature,degrees_C,-80,40,,one-boom,all,4,physicalMeasurement,time lat lon alt,True,LoggerTemperature(C) -t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,-80,40,t_surf dlr ulr,all,all,4,physicalMeasurement,time lat lon alt,False, -p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,-350,100,,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations -t_i,air_temperature,Air temperature (instantaneous),degrees_C,-80,40,,all,all,4,physicalMeasurement,time lat lon alt,True,"PT100 temperature at boom, for meteorological observations" -rh_i,relative_humidity,Relative humidity (instantaneous),%,0,150,rh_i_cor,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations -rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,0,100,,all,all,4,modelResult,time lat lon alt,True,For meteorological observations -wspd_i,wind_speed,Wind speed (instantaneous),m s-1,0,100,wdir_i wspd_x_i wspd_y_i,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations -wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,1,360,wspd_x_i wspd_y_i,all,all,4,physicalMeasurement,time lat lon alt,True,For meteorological observations -wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,all,4,modelResult,time lat lon alt,True,For meteorological observations -wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,-100,100,wdir_i wspd_i,all,all,4,modelResult,time lat lon alt,True,For meteorological observations -msg_i,message,Message string (instantaneous),-,,,,all,,,qualityInformation,time lat lon alt,True,L0 only diff --git a/src/pypromice/process/write.py b/src/pypromice/process/write.py index 75c36485..cc4d8fe5 100644 --- a/src/pypromice/process/write.py +++ b/src/pypromice/process/write.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -Write dataset module +Module containing all the functions needed to prepare and AWS data """ -import os, logging +import os, logging, datetime, uuid import pandas as pd -logger = logging.getLogger(__name__) - +import numpy as np +from importlib import metadata from pypromice.process.resample import resample_dataset -from pypromice.process import utilities, write +from pypromice.process import load +logger = logging.getLogger(__name__) -def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resample=True): +def prepare_and_write(dataset, outpath, vars_df=None, meta_dict=None, time='60min', resample=True): '''Prepare data with resampling, formating and metadata population; then write data to .nc and .csv hourly and daily files @@ -35,57 +36,76 @@ def prepare_and_write(dataset, outpath, vars_df, meta_dict, time='60min', resamp d2 = dataset.copy() # Reformat time - d2 = utilities.reformat_time(d2) + d2 = reformat_time(d2) + # finding station/site name + if 'station_id' in d2.attrs.keys(): + name = d2.attrs['station_id'] + else: + name = d2.attrs['site_id'] + # Reformat longitude (to negative values) - d2 = utilities.reformat_lon(d2) - + if 'gps_lon' in d2.keys(): + d2 = reformat_lon(d2) + else: + logger.info('%s does not have gpd_lon'%name) + # Add variable attributes and metadata - d2 = utilities.addVars(d2, vars_df) - d2 = utilities.addMeta(d2, meta_dict) + if vars_df is None: + vars_df = load.getVars() + if meta_dict is None: + meta_dict = load.getMeta() + + d2 = addVars(d2, vars_df) + d2 = addMeta(d2, meta_dict) # Round all values to specified decimals places - d2 = utilities.roundValues(d2, vars_df) - - # Create out directory - outdir = os.path.join(outpath, d2.attrs['station_id']) - if not os.path.isdir(outdir): - os.mkdir(outdir) - + d2 = roundValues(d2, vars_df) + # Get variable names to write out - col_names = write.getColNames( - vars_df, - d2.attrs['number_of_booms'], - d2.attrs['format'], - d2.attrs['bedrock'], - ) - + col_names = getColNames(vars_df, d2, remove_nan_fields=True) + # Define filename based on resample rate t = int(pd.Timedelta((d2['time'][1] - d2['time'][0]).values).total_seconds()) + + # Create out directory + outdir = os.path.join(outpath, name) + if not os.path.isdir(outdir): + os.mkdir(outdir) + if t == 600: - out_csv = os.path.join(outdir, d2.attrs['station_id']+'_10min.csv') - out_nc = os.path.join(outdir, d2.attrs['station_id']+'_10min.nc') + out_csv = os.path.join(outdir, name+'_10min.csv') + out_nc = os.path.join(outdir, name+'_10min.nc') elif t == 3600: - out_csv = os.path.join(outdir, d2.attrs['station_id']+'_hour.csv') - out_nc = os.path.join(outdir, d2.attrs['station_id']+'_hour.nc') + out_csv = os.path.join(outdir, name+'_hour.csv') + out_nc = os.path.join(outdir, name+'_hour.nc') elif t == 86400: - out_csv = os.path.join(outdir, d2.attrs['station_id']+'_day.csv') - out_nc = os.path.join(outdir, d2.attrs['station_id']+'_day.nc') + # removing instantaneous values from daily and monthly files + for v in col_names: + if ('_i' in v) and ('_i_' not in v): + col_names.remove(v) + out_csv = os.path.join(outdir, name+'_day.csv') + out_nc = os.path.join(outdir, name+'_day.nc') else: - out_csv = os.path.join(outdir, d2.attrs['station_id']+'_month.csv') - out_nc = os.path.join(outdir, d2.attrs['station_id']+'_month.nc') + # removing instantaneous values from daily and monthly files + for v in col_names: + if ('_i' in v) and ('_i_' not in v): + col_names.remove(v) + out_csv = os.path.join(outdir, name+'_month.csv') + out_nc = os.path.join(outdir, name+'_month.nc') if not os.path.isdir(outdir): os.mkdir(outdir) # Write to csv file logger.info('Writing to files...') - write.writeCSV(out_csv, d2, col_names) + writeCSV(out_csv, d2, col_names) # Write to netcdf file col_names = col_names + ['lat', 'lon', 'alt'] - write.writeNC(out_nc, d2, col_names) + writeNC(out_nc, d2, col_names) logger.info(f'Written to {out_csv}') logger.info(f'Written to {out_nc}') + def writeAll(outpath, station_id, l3_h, l3_d, l3_m, csv_order=None): '''Write L3 hourly, daily and monthly datasets to .nc and .csv files @@ -148,46 +168,223 @@ def writeNC(outfile, Lx, col_names=None): names = [c for c in col_names if c in list(Lx.keys())] else: names = list(Lx.keys()) + Lx[names].to_netcdf(outfile, mode='w', format='NETCDF4', compute=True) -def getColNames(vars_df, booms=None, data_type=None, bedrock=False): - '''Get all variable names for a given data type, based on a variables - look-up table. This is mainly for exporting purposes +def getColNames(vars_df, ds, remove_nan_fields=False): + ''' + Get variable names for a given dataset with respect to its type and processing level + + The dataset must have the the following attributes: + * level + * number_of_booms when the processing level is <= 2 + + This is mainly for exporting purposes. + + Parameters + ------- + list + Variable names + ''' + # selecting variable list based on level + vars_df = vars_df.loc[vars_df[ds.attrs['level']] == 1] + + # selecting variable list based on geometry + if ds.attrs['level'] in ['L0', 'L1', 'L2']: + if ds.attrs['number_of_booms']==1: + vars_df = vars_df.loc[vars_df['station_type'].isin(['one-boom','all'])] + elif ds.attrs['number_of_booms']==2: + vars_df = vars_df.loc[vars_df['station_type'].isin(['two-boom','all'])] + + var_list = list(vars_df.index) + if remove_nan_fields: + for v in var_list: + if v not in ds.keys(): + var_list.remove(v) + continue + if ds[v].isnull().all(): + var_list.remove(v) + return var_list + +def addVars(ds, variables): + '''Add variable attributes from file to dataset Parameters ---------- - vars_df : pd.DataFrame - Variables look-up table - booms : int, optional - Number of booms. If this parameter is empty then all variables - regardless of boom type will be passed. The default is None. - data_type : str, optional - Data type, "tx", "STM" or "raw". If this parameter is empty then all - variables regardless of data type will be passed. The default is None. + ds : xarray.Dataset + Dataset to add variable attributes to + variables : pandas.DataFrame + Variables lookup table file Returns ------- - list - Variable names + ds : xarray.Dataset + Dataset with metadata + ''' + for k in ds.keys(): + if k not in variables.index: continue + ds[k].attrs['standard_name'] = variables.loc[k]['standard_name'] + ds[k].attrs['long_name'] = variables.loc[k]['long_name'] + ds[k].attrs['units'] = variables.loc[k]['units'] + ds[k].attrs['coverage_content_type'] = variables.loc[k]['coverage_content_type'] + ds[k].attrs['coordinates'] = variables.loc[k]['coordinates'] + return ds + +def addMeta(ds, meta): + '''Add metadata attributes from file to dataset + + Parameters + ---------- + ds : xarray.Dataset + Dataset to add metadata attributes to + meta : dict + Metadata file + + Returns + ------- + ds : xarray.Dataset + Dataset with metadata + ''' + if 'gps_lon' in ds.keys(): + ds['lon'] = ds['gps_lon'].mean() + ds['lon'].attrs = ds['gps_lon'].attrs + + ds['lat'] = ds['gps_lat'].mean() + ds['lat'].attrs = ds['gps_lat'].attrs + + ds['alt'] = ds['gps_alt'].mean() + ds['alt'].attrs = ds['gps_alt'].attrs + + # Attribute convention for data discovery + # https://wiki.esipfed.org/Attribute_Convention_for_Data_Discovery_1-3 + + # Determine the temporal resolution + time_diff = pd.Timedelta((ds['time'][1] - ds['time'][0]).values) + if time_diff == pd.Timedelta('10min'): + sample_rate = "10min" + elif time_diff == pd.Timedelta('1H'): + sample_rate = "hourly" + elif time_diff == pd.Timedelta('1D'): + sample_rate = "daily" + elif 28 <= time_diff.days <= 31: + sample_rate = "monthly" + else: + sample_rate = "unknown_sample_rate" + + if 'station_id' in ds.attrs.keys(): + ds.attrs['id'] = 'dk.geus.promice.station.' + ds.attrs['station_id']+'.'+sample_rate + else: + ds.attrs['id'] = 'dk.geus.promice.site.' + ds.attrs['site_id'] +'.'+sample_rate + + ds.attrs['history'] = 'Generated on ' + datetime.datetime.utcnow().isoformat() + ds.attrs['date_created'] = str(datetime.datetime.now().isoformat()) + ds.attrs['date_modified'] = ds.attrs['date_created'] + ds.attrs['date_issued'] = ds.attrs['date_created'] + ds.attrs['date_metadata_modified'] = ds.attrs['date_created'] + ds.attrs['processing_level'] = ds.attrs['level'].replace('L','level ') + + ds.attrs['geospatial_bounds'] = "POLYGON((" + \ + f"{ds['lat'].min().values} {ds['lon'].min().values}, " + \ + f"{ds['lat'].min().values} {ds['lon'].max().values}, " + \ + f"{ds['lat'].max().values} {ds['lon'].max().values}, " + \ + f"{ds['lat'].max().values} {ds['lon'].min().values}, " + \ + f"{ds['lat'].min().values} {ds['lon'].min().values}))" + + ds.attrs['geospatial_lat_min'] = str(ds['lat'].min().values) + ds.attrs['geospatial_lat_max'] = str(ds['lat'].max().values) + ds.attrs['geospatial_lon_min'] = str(ds['lon'].min().values) + ds.attrs['geospatial_lon_max'] = str(ds['lon'].max().values) + ds.attrs['geospatial_vertical_min'] = str(ds['alt'].min().values) + ds.attrs['geospatial_vertical_max'] = str(ds['alt'].max().values) + ds.attrs['geospatial_vertical_positive'] = 'up' + ds.attrs['time_coverage_start'] = str(ds['time'][0].values) + ds.attrs['time_coverage_end'] = str(ds['time'][-1].values) + + try: + ds.attrs['source']= 'pypromice v' + str(metadata.version('pypromice')) + except: + ds.attrs['source'] = 'pypromice' + + # https://www.digi.com/resources/documentation/digidocs/90001437-13/reference/r_iso_8601_duration_format.htm + try: + ds.attrs['time_coverage_duration'] = str(pd.Timedelta((ds['time'][-1] - ds['time'][0]).values).isoformat()) + ds.attrs['time_coverage_resolution'] = str(pd.Timedelta((ds['time'][1] - ds['time'][0]).values).isoformat()) + except: + ds.attrs['time_coverage_duration'] = str(pd.Timedelta(0).isoformat()) + ds.attrs['time_coverage_resolution'] = str(pd.Timedelta(0).isoformat()) + + # Note: int64 dtype (long int) is incompatible with OPeNDAP access via THREDDS for NetCDF files + # See https://stackoverflow.com/questions/48895227/output-int32-time-dimension-in-netcdf-using-xarray + ds.time.encoding["dtype"] = "i4" # 32-bit signed integer + #ds.time.encoding["calendar"] = 'proleptic_gregorian' # this is default + + # Load metadata attributes and add to Dataset + [_addAttr(ds, key, value) for key,value in meta.items()] + + # Check attribute formating + for k,v in ds.attrs.items(): + if not isinstance(v, str) or not isinstance(v, int): + ds.attrs[k]=str(v) + return ds + +def _addAttr(ds, key, value): + '''Add attribute to xarray dataset + + ds : xr.Dataset + Dataset to add attribute to + key : str + Attribute name, with "." denoting variable attributes + value : str/int + Value for attribute''' + if len(key.split('.')) == 2: + try: + ds[key.split('.')[0]].attrs[key.split('.')[1]] = str(value) + except: + pass + # logger.info(f'Unable to add metadata to {key.split(".")[0]}') + else: + ds.attrs[key] = value + +def roundValues(ds, df, col='max_decimals'): + '''Round all variable values in data array based on pre-defined rounding + value in variables look-up table DataFrame + + Parameters + ---------- + ds : xr.Dataset + Dataset to round values in + df : pd.Dataframe + Variable look-up table with rounding values + col : str + Column in variable look-up table that contains rounding values. The + default is "max_decimals" ''' - if booms==1: - vars_df = vars_df.loc[vars_df['station_type'].isin(['one-boom','all'])] - elif booms==2: - vars_df = vars_df.loc[vars_df['station_type'].isin(['two-boom','all'])] - - if data_type=='TX': - vars_df = vars_df.loc[vars_df['data_type'].isin(['TX','all'])] - elif data_type=='STM' or data_type=='raw': - vars_df = vars_df.loc[vars_df['data_type'].isin(['raw','all'])] - - col_names = list(vars_df.index) - if isinstance(bedrock, str): - bedrock = (bedrock.lower() == 'true') - if bedrock == True: - col_names.remove('cc') - for v in ['dlhf_u', 'dlhf_l', 'dshf_u', 'dshf_l']: - try: - col_names.remove(v) - except: - pass - return col_names + df = df[col] + df = df.dropna(how='all') + for var in df.index: + if var not in list(ds.variables): + continue + if df[var] is not np.nan: + ds[var] = ds[var].round(decimals=int(df[var])) + return ds + +def reformat_time(dataset): + '''Re-format time''' + t = dataset['time'].values + dataset['time'] = list(t) + return dataset + +def reformat_lon(dataset, exempt=['UWN', 'Roof_GEUS', 'Roof_PROMICE']): + '''Switch gps_lon to negative values (degrees_east). We do this here, and + NOT in addMeta, otherwise we switch back to positive when calling getMeta + in joinL2''' + if 'station_id' in dataset.attrs.keys(): + id = dataset.attrs['station_id'] + else: + id = dataset.attrs['site_id'] + + if id not in exempt: + if 'gps_lon' not in dataset.keys(): + return dataset + dataset['gps_lon'] = dataset['gps_lon'] * -1 + return dataset \ No newline at end of file diff --git a/src/pypromice/qc/persistence.py b/src/pypromice/qc/persistence.py index f59bf45c..5f5d55f4 100644 --- a/src/pypromice/qc/persistence.py +++ b/src/pypromice/qc/persistence.py @@ -161,6 +161,6 @@ def duration_consecutive_true( # assert series.dtype == bool cumsum = ((series.index - series.index[0]).total_seconds()/3600).to_series(index=series.index) is_first = series.astype("int").diff() == 1 - offset = (is_first * cumsum).replace(0, np.nan).fillna(method="ffill").fillna(0) + offset = (is_first * cumsum).replace(0, np.nan).ffill().fillna(0) return (cumsum - offset) * series diff --git a/src/pypromice/resources/__init__.py b/src/pypromice/resources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pypromice/process/metadata.csv b/src/pypromice/resources/file_attributes.csv similarity index 99% rename from src/pypromice/process/metadata.csv rename to src/pypromice/resources/file_attributes.csv index 01b79ff5..9fa56bc4 100644 --- a/src/pypromice/process/metadata.csv +++ b/src/pypromice/resources/file_attributes.csv @@ -1,56 +1,56 @@ -attribute,entry -acknowledgements,The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) -alt.axis,Z -alt.coverage_content_type,coordinate -gps_alt.positive,up -cdm_data_type, -comment,https://doi.org/10.22008/promice/data/aws -contributor_name, -contributor_role, -conventions,ACDD-1.3; CF-1.7 -creater_email,pho@geus.dk -creater_url,https://promice.dk -creator_institution,Geological Survey of Denmark and Greenland (GEUS) -creator_name,Penelope How -creator_type,person -featureType,timeSeries -geospatial_bounds_crs,EPSG:4979 -geospatial_lat_extents_match,gps_lat -geospatial_lat_resolution, -geospatial_lat_units,degrees_north -geospatial_lon_extents_match,gps_lon -geospatial_lon_resolution, -geospatial_lon_units,degrees_east -geospatial_vertical_resolution, -geospatial_vertical_units,EPSG:4979 -institution,Geological Survey of Denmark and Greenland (GEUS) -instrument,See https://doi.org/10.5194/essd-13-3819-2021 -instrument_vocabulary,GCMD:GCMD Keywords -keywords,GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > ICE SHEETS > ICE SHEET MEASUREMENTS; GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > GLACIER MASS BALANCE/ICE SHEET MASS BALANCE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE GROWTH/MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC PRESSURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > ALBEDO; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > INCOMING SOLAR RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > DOWNWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > UPWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > NET RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > OUTGOING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FLUX; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FORCING; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION > DOWNWELLING SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SUNSHINE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC TEMPERATURE > SURFACE TEMPERATURE > AIR TEMPERATURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > ABSOLUTE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > RELATIVE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > LOCAL WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > U/V WIND COMPONENTS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND DIRECTION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND SPEED; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > CLOUDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > PRECIPITATION -keywords_vocabulary,GCMDSK:GCMD Science Keywords:https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords -lat.axis,Y -lat.coverage_content_type,coordinate -lat.long_name,station latitude -license,Creative Commons Attribution 4.0 International (CC-BY-4.0) https://creativecommons.org/licenses/by/4.0 -lon.axis,X -lon.coverage_content_type,coordinate -lon.long_name,station longitude -lon.units,degrees_east -metadata_link, -naming_authority,dk.geus.promice -platform, -platform_vocabulary,GCMD:GCMD Keywords -processing_level,Level 3 -product_status,beta -product_version,4 -program,PROMICE -project,PROMICE -publisher_email,info@promice.dk -publisher_institution,GEUS -publisher_name,GEUS -publisher_type,institution -publisher_url,https://promice.dk -references,"How, P.; Abermann, J.; Ahlstrøm, A.P.; Andersen, S.B.; Box, J. E.; Citterio, M.; Colgan, W.T.; Fausto. R.S.; Karlsson, N.B.; Jakobsen, J.; Langley, K.; Larsen, S.H.; Mankoff, K.D.; Pedersen, A.Ø.; Rutishauser, A.; Shield, C.L.; Solgaard, A.M.; van As, D.; Vandecrux, B.; Wright, P.J., 2022, ""PROMICE and GC-Net automated weather station data in Greenland"", https://doi.org/10.22008/FK2/IW73UU, GEUS Dataverse" -references_bib,@article{How2022; doi = {10.22008/FK2/IW73UU}; url = {https://doi.org/10.22008/FK2/IW73UU}; year = {2022}; month=10; publisher= {GEUS Dataverse}; author = {Penelope How and Jakob Abermann and Andreas P. Ahlstr{\o}m and Signe B. Andersen and Jason E. Box and Michele Citterio and William Colgan and Robert S. Fausto and Nanna B. Karlsson and Jakob Jakobsen and Kirsty Langley and Signe Hillerup Larsen and Kenneth D. Mankoff and Allan {\O}. Pedersen and Anja Rutishauser and Christopher L. Shields and Anne M. Solgaard and Dirk van As and Baptiste Vandecrux}; title = {PROMICE and GC-Net automated weather station data in Greenland}; journal = {GEUS Dataverse}} -standard_name_vocabulary,CF Standard Name Table (v77; 19 January 2021) -summary,"The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) and Greenland Climate Network (GC-Net) have been measuring climate and ice sheet properties since 2007 and 1995, respectively. The PROMICE weather station network monitors glacier mass balance in the melt zone of the Greenland Ice Sheet, providing ground truth data to calibrate mass budget models. GC-Net weather stations measure snowfall and surface properties in the accumulation zone, providing valuable knowledge on the Greenland Ice Sheet’s mass gain and climatology.Accurate measurements of the surface and near-surface atmospheric conditions in a changing climate is important for reliable present and future assessment of changes to the Greenland Ice Sheet. All measurements are handled and processed with pypromice, which is a peer-reviewed and freely available Python package with source code available at https://github.com/GEUS-Glaciology-and-Climate/pypromice. A user-contributable dynamic web-based database of known data quality issues is associated with the data products at https://github.com/GEUS-PROMICE/ PROMICE-AWS-data-issues/." +attribute,entry +acknowledgements,The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) +alt.axis,Z +alt.coverage_content_type,coordinate +gps_alt.positive,up +cdm_data_type, +comment,https://doi.org/10.22008/promice/data/aws +contributor_name, +contributor_role, +conventions,ACDD-1.3; CF-1.7 +creater_email,pho@geus.dk +creater_url,https://promice.dk +creator_institution,Geological Survey of Denmark and Greenland (GEUS) +creator_name,Penelope How +creator_type,person +featureType,timeSeries +geospatial_bounds_crs,EPSG:4979 +geospatial_lat_extents_match,gps_lat +geospatial_lat_resolution, +geospatial_lat_units,degrees_north +geospatial_lon_extents_match,gps_lon +geospatial_lon_resolution, +geospatial_lon_units,degrees_east +geospatial_vertical_resolution, +geospatial_vertical_units,EPSG:4979 +institution,Geological Survey of Denmark and Greenland (GEUS) +instrument,See https://doi.org/10.5194/essd-13-3819-2021 +instrument_vocabulary,GCMD:GCMD Keywords +keywords,GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > ICE SHEETS > ICE SHEET MEASUREMENTS; GCMDSK:EARTH SCIENCE > CRYOSPHERE > GLACIERS/ICE SHEETS > GLACIER MASS BALANCE/ICE SHEET MASS BALANCE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > CRYOSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ALBEDO; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE GROWTH/MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > ICE VELOCITY; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW DEPTH; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW MELT; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE > SNOW/ICE TEMPERATURE; GCMDSK:EARTH SCIENCE > TERRESTRIAL HYDROSPHERE > SNOW/ICE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC PRESSURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > ALBEDO; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > INCOMING SOLAR RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > DOWNWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION > UPWELLING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > NET RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > OUTGOING LONGWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FLUX; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > RADIATIVE FORCING; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION > DOWNWELLING SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SHORTWAVE RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION > SUNSHINE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC RADIATION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC TEMPERATURE > SURFACE TEMPERATURE > AIR TEMPERATURE; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > ABSOLUTE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WATER VAPOR > WATER VAPOR INDICATORS > HUMIDITY > RELATIVE HUMIDITY; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > LOCAL WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > U/V WIND COMPONENTS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND DIRECTION; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS > WIND SPEED; GCMDSK:EARTH SCIENCE > ATMOSPHERE > ATMOSPHERIC WINDS > SURFACE WINDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > CLOUDS; GCMDSK:EARTH SCIENCE > ATMOSPHERE > PRECIPITATION +keywords_vocabulary,GCMDSK:GCMD Science Keywords:https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords +lat.axis,Y +lat.coverage_content_type,coordinate +lat.long_name,station latitude +license,Creative Commons Attribution 4.0 International (CC-BY-4.0) https://creativecommons.org/licenses/by/4.0 +lon.axis,X +lon.coverage_content_type,coordinate +lon.long_name,station longitude +lon.units,degrees_east +metadata_link, +naming_authority,dk.geus.promice +platform, +platform_vocabulary,GCMD:GCMD Keywords +processing_level,Level 3 +product_status,beta +product_version,4 +program,PROMICE +project,PROMICE +publisher_email,info@promice.dk +publisher_institution,GEUS +publisher_name,GEUS +publisher_type,institution +publisher_url,https://promice.dk +references,"How, P.; Abermann, J.; Ahlstrøm, A.P.; Andersen, S.B.; Box, J. E.; Citterio, M.; Colgan, W.T.; Fausto. R.S.; Karlsson, N.B.; Jakobsen, J.; Langley, K.; Larsen, S.H.; Mankoff, K.D.; Pedersen, A.Ø.; Rutishauser, A.; Shield, C.L.; Solgaard, A.M.; van As, D.; Vandecrux, B.; Wright, P.J., 2022, ""PROMICE and GC-Net automated weather station data in Greenland"", https://doi.org/10.22008/FK2/IW73UU, GEUS Dataverse" +references_bib,@article{How2022; doi = {10.22008/FK2/IW73UU}; url = {https://doi.org/10.22008/FK2/IW73UU}; year = {2022}; month=10; publisher= {GEUS Dataverse}; author = {Penelope How and Jakob Abermann and Andreas P. Ahlstr{\o}m and Signe B. Andersen and Jason E. Box and Michele Citterio and William Colgan and Robert S. Fausto and Nanna B. Karlsson and Jakob Jakobsen and Kirsty Langley and Signe Hillerup Larsen and Kenneth D. Mankoff and Allan {\O}. Pedersen and Anja Rutishauser and Christopher L. Shields and Anne M. Solgaard and Dirk van As and Baptiste Vandecrux}; title = {PROMICE and GC-Net automated weather station data in Greenland}; journal = {GEUS Dataverse}} +standard_name_vocabulary,CF Standard Name Table (v77; 19 January 2021) +summary,"The Programme for Monitoring of the Greenland Ice Sheet (PROMICE) and Greenland Climate Network (GC-Net) have been measuring climate and ice sheet properties since 2007 and 1995, respectively. The PROMICE weather station network monitors glacier mass balance in the melt zone of the Greenland Ice Sheet, providing ground truth data to calibrate mass budget models. GC-Net weather stations measure snowfall and surface properties in the accumulation zone, providing valuable knowledge on the Greenland Ice Sheet’s mass gain and climatology.Accurate measurements of the surface and near-surface atmospheric conditions in a changing climate is important for reliable present and future assessment of changes to the Greenland Ice Sheet. All measurements are handled and processed with pypromice, which is a peer-reviewed and freely available Python package with source code available at https://github.com/GEUS-Glaciology-and-Climate/pypromice. A user-contributable dynamic web-based database of known data quality issues is associated with the data products at https://github.com/GEUS-PROMICE/ PROMICE-AWS-data-issues/." diff --git a/src/pypromice/resources/variable_aliases_GC-Net.csv b/src/pypromice/resources/variable_aliases_GC-Net.csv new file mode 100644 index 00000000..a0b4fe14 --- /dev/null +++ b/src/pypromice/resources/variable_aliases_GC-Net.csv @@ -0,0 +1,78 @@ +GEUS_name,old_name +time,timestamp +p_u,P +p_l, +t_u,TA2 +t_l,TA1 +rh_u,RH2 +rh_u_cor,RH2_cor +qh_u,Q2 +rh_l,RH1 +rh_l_cor,RH1_cor +qh_l,Q1 +wspd_u,VW2 +wspd_l,VW1 +wdir_u,DW2 +wdir_l,DW1 +dsr, +dsr_cor,ISWR +usr, +usr_cor,OSWR +albedo,Alb +dlr, +ulr, +cc, +t_surf, +dlhf_u,LHF +dlhf_l, +dshf_u,SHF +dshf_l, +z_boom_u,HW2 +z_boom_l,HW1 +precip_u, +precip_u_cor, +precip_l, +precip_l_cor, +t_i_1,TS1 +t_i_2,TS2 +t_i_3,TS3 +t_i_4,TS4 +t_i_5,TS5 +t_i_6,TS6 +t_i_7,TS7 +t_i_8,TS8 +t_i_9,TS9 +t_i_10,TS10 +t_i_11, +tilt_x, +tilt_y, +rot, +gps_lat,latitude +gps_lon,longitude +gps_alt,elevation +gps_time, +gps_geounit, +gps_hdop, +batt_v,V +fan_dc_u, +fan_dc_l, +t_rad, +msg_lat, +msg_lon, +z_surf_1,HS1 +z_surf_2,HS2 +z_surf_1_adj_flag,HS1_adj_flag +z_surf_2_adj_flag,HS2_adj_flag +z_surf_combined,HS_combined +depth_t_i_1,DTS1 +depth_t_i_2,DTS2 +depth_t_i_3,DTS3 +depth_t_i_4,DTS4 +depth_t_i_5,DTS5 +depth_t_i_6,DTS6 +depth_t_i_7,DTS7 +depth_t_i_8,DTS8 +depth_t_i_9,DTS9 +depth_t_i_10,DTS10 +depth_t_i_11, +t_i_10m,TS_10m diff --git a/src/pypromice/resources/variables.csv b/src/pypromice/resources/variables.csv new file mode 100644 index 00000000..ab8e4317 --- /dev/null +++ b/src/pypromice/resources/variables.csv @@ -0,0 +1,91 @@ +field,standard_name,long_name,units,coverage_content_type,coordinates,instantaneous_hourly,where_to_find,lo,hi,OOL,station_type,L0,L2,L3,max_decimals +time,time,Time,yyyy-mm-dd HH:MM:SS,physicalMeasurement,time lat lon alt,,,,,,all,1,1,1, +rec,record,Record,-,referenceInformation,time lat lon alt,,L0 or L2,,,,all,1,1,0,0 +p_u,air_pressure,Air pressure (upper boom),hPa,physicalMeasurement,time lat lon alt,FALSE,,650,1100,z_pt z_pt_cor dshf_u dlhf_u qh_u,all,1,1,1,4 +p_l,air_pressure,Air pressure (lower boom),hPa,physicalMeasurement,time lat lon alt,FALSE,,650,1100,dshf_l dlhf_l qh_l,two-boom,1,1,1,4 +t_u,air_temperature,Air temperature (upper boom),degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,40,rh_u_cor cc dsr_cor usr_cor z_boom z_stake dshf_u dlhf_u qh_u,all,1,1,1,4 +t_l,air_temperature,Air temperature (lower boom),degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,40,rh_l_cor z_boom_l dshf_l dlhf_l qh_l,two-boom,1,1,1,4 +rh_u,relative_humidity,Relative humidity (upper boom),%,physicalMeasurement,time lat lon alt,FALSE,,0,100,rh_u_cor,all,1,1,1,4 +rh_u_cor,relative_humidity_corrected,Relative humidity (upper boom) - corrected,%,modelResult,time lat lon alt,FALSE,L2 or later,0,150,dshf_u dlhf_u qh_u,all,0,1,1,4 +qh_u,specific_humidity,Specific humidity (upper boom),kg/kg,modelResult,time lat lon alt,FALSE,L2 or later,0,100,,all,0,1,1,4 +rh_l,relative_humidity,Relative humidity (lower boom),%,physicalMeasurement,time lat lon alt,FALSE,,0,100,rh_l_cor,two-boom,1,1,1,4 +rh_l_cor,relative_humidity_corrected,Relative humidity (lower boom) - corrected,%,modelResult,time lat lon alt,FALSE,L2 or later,0,150,dshf_l dlhf_l qh_l,two-boom,0,1,1,4 +qh_l,specific_humidity,Specific humidity (lower boom),kg/kg,modelResult,time lat lon alt,FALSE,L2 or later,0,100,,two-boom,0,1,1,4 +wspd_u,wind_speed,Wind speed (upper boom),m s-1,physicalMeasurement,time lat lon alt,FALSE,,0,100,"wdir_u wspd_x_u wspd_y_u dshf_u dlhf_u qh_u, precip_u",all,1,1,1,4 +wspd_l,wind_speed,Wind speed (lower boom),m s-1,physicalMeasurement,time lat lon alt,FALSE,,0,100,"wdir_l wspd_x_l wspd_y_l dshf_l dlhf_l qh_l , precip_l",two-boom,1,1,1,4 +wdir_u,wind_from_direction,Wind from direction (upper boom),degrees,physicalMeasurement,time lat lon alt,FALSE,,1,360,wspd_x_u wspd_y_u,all,1,1,1,4 +wdir_std_u,wind_from_direction_standard_deviation,Wind from direction (standard deviation),degrees,qualityInformation,time lat lon alt,FALSE,L0 or L2,,,,one-boom,1,1,0,4 +wdir_l,wind_from_direction,Wind from direction (lower boom),degrees,physicalMeasurement,time lat lon alt,FALSE,,1,360,wspd_x_l wspd_y_l,two-boom,1,1,1,4 +wspd_x_u,wind_speed_from_x_direction,Wind speed from x direction (upper boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4 +wspd_y_u,wind_speed_from_y_direction,Wind speed from y direction (upper boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2,-100,100,wdir_u wspd_u,all,0,1,1,4 +wspd_x_l,wind_speed_from_x_direction,Wind speed from x direction (lower boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4 +wspd_y_l,wind_speed_from_y_direction,Wind speed from y direction (lower boom),m s-1,modelResult,time lat lon alt,FALSE,L0 or L2,-100,100,wdir_l wspd_l,two-boom,0,1,1,4 +dsr,surface_downwelling_shortwave_flux,Downwelling shortwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,,-10,1500,albedo dsr_cor usr_cor,all,1,1,1,4 +dsr_cor,surface_downwelling_shortwave_flux_corrected,Downwelling shortwave radiation - corrected,W m-2,modelResult,time lat lon alt,FALSE,L2 or later,,,,all,0,1,1,4 +usr,surface_upwelling_shortwave_flux,Upwelling shortwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,,-10,1000,albedo dsr_cor usr_cor,all,1,1,1,4 +usr_cor,surface_upwelling_shortwave_flux_corrected,Upwelling shortwave radiation - corrected,W m-2,modelResult,time lat lon alt,FALSE,L2 or later,0,1000,,all,0,1,1,4 +albedo,surface_albedo,Albedo,-,modelResult,time lat lon alt,FALSE,L2 or later,,,,all,0,1,1,4 +dlr,surface_downwelling_longwave_flux,Downwelling longwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,,50,500,albedo dsr_cor usr_cor cc t_surf,all,1,1,1,4 +ulr,surface_upwelling_longwave_flux,Upwelling longwave radiation,W m-2,physicalMeasurement,time lat lon alt,FALSE,,50,500,t_surf,all,1,1,1,4 +cc,cloud_area_fraction,Cloud cover,%,modelResult,time lat lon alt,FALSE,L2 or later,,,,all,0,1,1,4 +t_surf,surface_temperature,Surface temperature,C,modelResult,time lat lon alt,FALSE,L2 or later,-80,40,dshf_u dlhf_u qh_u,all,0,1,1,4 +dlhf_u,surface_downward_latent_heat_flux,Latent heat flux (upper boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later,,,,all,0,0,1,4 +dlhf_l,surface_downward_latent_heat_flux,Latent heat flux (lower boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later,,,,two-boom,0,0,1,4 +dshf_u,surface_downward_sensible_heat_flux,Sensible heat flux (upper boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later,,,,all,0,0,1,4 +dshf_l,surface_downward_sensible_heat_flux,Sensible heat flux (lower boom),W m-2,modelResult,time lat lon alt,FALSE,L3 or later,,,,two-boom,0,0,1,4 +z_boom_u,distance_to_surface_from_boom,Upper boom height,m,physicalMeasurement,time lat lon alt,TRUE,,0.3,10,dshf_u dlhf_u qh_u,all,1,1,1,4 +z_boom_q_u,distance_to_surface_from_boom_quality,Upper boom height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,all,1,1,0,4 +z_boom_l,distance_to_surface_from_boom,Lower boom height,m,physicalMeasurement,time lat lon alt,TRUE,,0.3,5,dshf_l dlhf_l qh_l,two-boom,1,1,1,4 +z_boom_q_l,distance_to_surface_from_boom_quality,Lower boom height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,two-boom,1,1,0,4 +z_stake,distance_to_surface_from_stake_assembly,Stake height,m,physicalMeasurement,time lat lon alt,TRUE,,0.3,8,,one-boom,1,1,1,4 +z_stake_q,distance_to_surface_from_stake_assembly_quality,Stake height (quality),-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,one-boom,1,1,0,4 +z_pt,depth_of_pressure_transducer_in_ice,Depth of pressure transducer in ice,m,physicalMeasurement,time lat lon alt,FALSE,,0,30,z_pt_cor,one-boom,1,1,1,4 +z_pt_cor,depth_of_pressure_transducer_in_ice_corrected,Depth of pressure transducer in ice - corrected,m,modelResult,time lat lon alt,FALSE,L2 or later,0,30,,one-boom,0,1,1,4 +precip_u,precipitation,Precipitation (upper boom) (cumulative solid & liquid),mm,physicalMeasurement,time lat lon alt,TRUE,,0,,precip_u_cor precip_u_rate,all,1,1,1,4 +precip_u_cor,precipitation_corrected,Precipitation (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,0,,,all,0,1,1,4 +precip_u_rate,precipitation_rate,Precipitation rate (upper boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,0,,,all,0,1,1,4 +precip_l,precipitation,Precipitation (lower boom) (cumulative solid & liquid),mm,physicalMeasurement,time lat lon alt,TRUE,,0,,precip_l_cor precip_l_rate,two-boom,1,1,1,4 +precip_l_cor,precipitation_corrected,Precipitation (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,0,,,two-boom,0,1,1,4 +precip_l_rate,precipitation_rate,Precipitation rate (lower boom) (cumulative solid & liquid) – corrected,mm,modelResult,time lat lon alt,TRUE,L2 or later,0,,,two-boom,0,1,1,4 +t_i_1,ice_temperature_at_t1,Ice temperature at sensor 1,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_2,ice_temperature_at_t2,Ice temperature at sensor 2,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_3,ice_temperature_at_t3,Ice temperature at sensor 3,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_4,ice_temperature_at_t4,Ice temperature at sensor 4,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_5,ice_temperature_at_t5,Ice temperature at sensor 5,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_6,ice_temperature_at_t6,Ice temperature at sensor 6,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_7,ice_temperature_at_t7,Ice temperature at sensor 7,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_8,ice_temperature_at_t8,Ice temperature at sensor 8,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,all,1,1,1,4 +t_i_9,ice_temperature_at_t9,Ice temperature at sensor 9,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,two-boom,1,1,1,4 +t_i_10,ice_temperature_at_t10,Ice temperature at sensor 10,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,two-boom,1,1,1,4 +t_i_11,ice_temperature_at_t11,Ice temperature at sensor 11,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,1,,two-boom,1,1,1,4 +tilt_x,platform_view_angle_x,Tilt to east,degrees,physicalMeasurement,time lat lon alt,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4 +tilt_y,platform_view_angle_y,Tilt to north,degrees,physicalMeasurement,time lat lon alt,FALSE,,-30,30,dsr_cor usr_cor albedo,all,1,1,1,4 +rot,platform_azimuth_angle,Station rotation from true North,degrees,physicalMeasurement,time lat lon alt,FALSE,,0,360,,all,1,1,1,2 +gps_lat,gps_latitude,Latitude,degrees_north,coordinate,time lat lon alt,TRUE,,50,83,,all,1,1,1,6 +gps_lon,gps_longitude,Longitude,degrees_east,coordinate,time lat lon alt,TRUE,,5,70,,all,1,1,1,6 +gps_alt,gps_altitude,Altitude,m,coordinate,time lat lon alt,TRUE,,0,3000,,all,1,1,1,2 +gps_time,gps_time,GPS time,s,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,240000,,all,1,1,0, +gps_geoid,gps_geoid_separation,Height of EGM96 geoid over WGS84 ellipsoid,m,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,,,,one-boom,1,1,0, +gps_geounit,gps_geounit,GeoUnit,-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,all,1,1,0, +gps_hdop,gps_hdop,GPS horizontal dillution of precision (HDOP),m,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,all,1,1,0,2 +gps_numsat,gps_numsat,GPS number of satellites,-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,,1,1,0,0 +gps_q,gps_q,Quality,-,qualityInformation,time lat lon alt,TRUE,L0 or L2,,,,,1,1,0, +lat,gps_mean_latitude,GPS mean latitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,,,,,all,1,1,1,6 +lon,gps_mean_longitude,GPS mean longitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,,,,,all,1,1,1,6 +alt,gps_mean_altitude,GPS mean altitude (from all time-series),degrees,modelResult,time lat lon alt,TRUE,,,,,all,1,1,1,6 +batt_v,battery_voltage,Battery voltage,V,physicalMeasurement,time lat lon alt,TRUE,,0,30,,all,1,1,1,2 +batt_v_ini,,,-,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,30,,,1,1,0,2 +batt_v_ss,battery_voltage_at_sample_start,Battery voltage (sample start),V,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,30,,,1,1,0,2 +fan_dc_u,fan_current,Fan current (upper boom),mA,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,200,,all,1,1,0,2 +fan_dc_l,fan_current,Fan current (lower boom),mA,physicalMeasurement,time lat lon alt,TRUE,,0,200,,two-boom,1,1,0,2 +freq_vw,frequency_of_precipitation_wire_vibration,Frequency of vibrating wire in precipitation gauge,Hz,physicalMeasurement,time lat lon alt,TRUE,L0 or L2,0,10000,precip_u,,1,1,0, +t_log,temperature_of_logger,Logger temperature,degrees_C,physicalMeasurement,time lat lon alt,TRUE,,-80,40,,one-boom,1,1,0,4 +t_rad,temperature_of_radiation_sensor,Radiation sensor temperature,degrees_C,physicalMeasurement,time lat lon alt,FALSE,,-80,40,t_surf dlr ulr,all,1,1,1,4 +p_i,air_pressure,Air pressure (instantaneous) minus 1000,hPa,physicalMeasurement,time lat lon alt,TRUE,,-350,100,,all,1,1,1,4 +t_i,air_temperature,Air temperature (instantaneous),degrees_C,physicalMeasurement,time lat lon alt,TRUE,,-80,40,,all,1,1,1,4 +rh_i,relative_humidity,Relative humidity (instantaneous),%,physicalMeasurement,time lat lon alt,TRUE,,0,150,rh_i_cor,all,1,1,1,4 +rh_i_cor,relative_humidity_corrected,Relative humidity (instantaneous) – corrected,%,modelResult,time lat lon alt,TRUE,L2 or later,0,100,,all,0,1,1,4 +wspd_i,wind_speed,Wind speed (instantaneous),m s-1,physicalMeasurement,time lat lon alt,TRUE,,0,100,wdir_i wspd_x_i wspd_y_i,all,1,1,1,4 +wdir_i,wind_from_direction,Wind from direction (instantaneous),degrees,physicalMeasurement,time lat lon alt,TRUE,,1,360,wspd_x_i wspd_y_i,all,1,1,1,4 +wspd_x_i,wind_speed_from_x_direction,Wind speed from x direction (instantaneous),m s-1,modelResult,time lat lon alt,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4 +wspd_y_i,wind_speed_from_y_direction,Wind speed from y direction (instantaneous),m s-1,modelResult,time lat lon alt,TRUE,L2 or later,-100,100,wdir_i wspd_i,all,0,1,1,4 \ No newline at end of file