bringing latest changes from #262 and #252

src/pypromice/process/L1toL2.py

@@ -86,7 +86,7 @@ def toL2(
 
     # filtering gps_lat, gps_lon and gps_alt based on the difference to a baseline elevation
     # right now baseline elevation is gapfilled monthly median elevation
-    baseline_elevation = (ds.gps_alt.to_series().resample('M').median()
+    baseline_elevation = (ds.gps_alt.to_series().resample('MS').median()
                           .reindex(ds.time.to_series().index, method='nearest')
                           .ffill().bfill())
     mask = (np.abs(ds.gps_alt - baseline_elevation) < 100) & ds.gps_alt.notnull()
@@ -327,7 +327,7 @@ def smoothTilt(da: xr.DataArray, threshold=0.2):
     # we calculate the moving standard deviation over a 3-day sliding window
     # hourly resampling is necessary to make sure the same threshold can be used 
     # for 10 min and hourly data
-    moving_std_gap_filled = da.to_series().resample('H').median().rolling(
+    moving_std_gap_filled = da.to_series().resample('h').median().rolling(
                     3*24, center=True, min_periods=2
                     ).std().reindex(da.time, method='bfill').values
     # we select the good timestamps and gapfill assuming that
@@ -354,7 +354,7 @@ def smoothRot(da: xr.DataArray, threshold=4):
     xarray.DataArray
         smoothed rotation measurements from inclinometer
     '''
-    moving_std_gap_filled = da.to_series().resample('H').median().rolling(
+    moving_std_gap_filled = da.to_series().resample('h').median().rolling(
                     3*24, center=True, min_periods=2
                     ).std().reindex(da.time, method='bfill').values
     # same as for tilt with, in addition:

src/pypromice/process/get_l2.py

@@ -3,6 +3,7 @@
 from argparse import ArgumentParser
 import pypromice
 from pypromice.process.aws import AWS
+from pypromice.process.load import getVars, getMeta
 from pypromice.process.write import prepare_and_write
 
 def parse_arguments_l2():
@@ -21,36 +22,41 @@ def parse_arguments_l2():
     args = parser.parse_args()
     return args
 
-def get_l2():
-    args = parse_arguments_l2()
 
+def get_l2(config_file, inpath, outpath, variables, metadata):
     logging.basicConfig(
         format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
         level=logging.INFO,
         stream=sys.stdout,
     )
 
     # Define input path
-    station_name = args.config_file.split('/')[-1].split('.')[0] 
-    station_path = os.path.join(args.inpath, station_name)
+    station_name = config_file.split('/')[-1].split('.')[0] 
+    station_path = os.path.join(inpath, station_name)
     if os.path.exists(station_path):
-        aws = AWS(args.config_file, station_path, args.variables, args.metadata)
+        aws = AWS(config_file, station_path, variables, metadata)
     else:
-        aws = AWS(args.config_file, args.inpath, args.variables, args.metadata)
+        aws = AWS(config_file, inpath, variables, metadata)
 
     # Perform level 1 and 2 processing
     aws.getL1()
     aws.getL2() 
-
+    v = getVars(variables)
+    m = getMeta(metadata)
     # Write out level 2
-    if args.outpath is not None:
-        if not os.path.isdir(args.outpath):
-            os.mkdir(args.outpath)
+    if outpath is not None:
+        if not os.path.isdir(outpath):
+            os.mkdir(outpath)
         if aws.L2.attrs['format'] == 'raw':
-            prepare_and_write(aws.L2, args.outpath, args.variables, args.metadata, '10min')
-        prepare_and_write(aws.L2, args.outpath, args.variables, args.metadata, '60min')
+            prepare_and_write(aws.L2, outpath, v, m, '10min')
+        prepare_and_write(aws.L2, outpath, v, m, '60min')
+    return aws
+
+def main():
+    args = parse_arguments_l2()
+    _ = get_l2(args.config_file, args.inpath, args.outpath, args.variables, args.metadata)
 
 
 if __name__ == "__main__":  
-    get_l2()
+    main()
 

src/pypromice/process/get_l2tol3.py

@@ -4,13 +4,13 @@
 from argparse import ArgumentParser
 import pypromice
 from pypromice.process.L2toL3 import toL3
+from pypromice.process.load import getVars, getMeta
 from pypromice.process.write import prepare_and_write
 logger = logging.getLogger(__name__)
 
 def parse_arguments_l2tol3(debug_args=None):
     parser = ArgumentParser(description="AWS L3 script for the processing L3 "+
-                            "data from L2 and merging the L3 data with its "+
-                            "historical site. An hourly, daily and monthly L3 "+
+                            "data from L2. An hourly, daily and monthly L3 "+
                             "data product is outputted to the defined output path")
     parser.add_argument('-i', '--inpath', type=str, required=True, 
                         help='Path to Level 2 .nc data file')
@@ -20,40 +20,46 @@ def parse_arguments_l2tol3(debug_args=None):
                         required=False, help='File path to variables look-up table')
     parser.add_argument('-m', '--metadata', default=None, type=str, 
                         required=False, help='File path to metadata')
-    parser.add_argument('-g', '--gcnet_historical', default=None, type=str, 
-                        required=False, help='File path to historical GC-Net data file')
 
-    # here will come additional arguments for the merging with historical stations
     args = parser.parse_args(args=debug_args)
     return args
 
-def get_l2tol3():
-    args = parse_arguments_l2tol3()
+def get_l2tol3(inpath, outpath, variables, metadata):
     logging.basicConfig(
         format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
         level=logging.INFO,
         stream=sys.stdout,
     )
 
     # Define Level 2 dataset from file
-    with xr.open_dataset(args.inpath) as l2:
+    with xr.open_dataset(inpath) as l2:
         l2.load()
+
+    # Remove encoding attributes from NetCDF
     for varname in l2.variables:
         if 'encoding' in l2[varname].attrs:
-            del l2[varname].attrs['encoding']        
+            del l2[varname].attrs['encoding']     
+
     if 'bedrock' in l2.attrs.keys():
         l2.attrs['bedrock'] = l2.attrs['bedrock'] == 'True'
     if 'number_of_booms' in l2.attrs.keys():
         l2.attrs['number_of_booms'] = int(l2.attrs['number_of_booms'])
 
     # Perform Level 3 processing
     l3 = toL3(l2)
-    
+
     # Write Level 3 dataset to file if output directory given
-    if args.outpath is not None:
-        prepare_and_write(l3, args.outpath, args.variables, args.metadata, '60min')
-        prepare_and_write(l3, args.outpath, args.variables, args.metadata, '1D')
-        prepare_and_write(l3, args.outpath, args.variables, args.metadata, 'M')
+    v = getVars(variables)
+    m = getMeta(metadata)
+    if outpath is not None:
+        prepare_and_write(l3, outpath, v, m, '60min')
+        prepare_and_write(l3, outpath, v, m, '1D')
+        prepare_and_write(l3, outpath, v, m, 'M')
+    return l3
 
+def main():
+    args = parse_arguments_l2tol3()
+    _ = get_l2tol3(args.inpath, args.outpath, args.variables, args.metadata)
+
 if __name__ == "__main__":  
-    get_l2tol3()
+    main()

src/pypromice/process/join_l2.py

@@ -3,9 +3,8 @@
 import pandas as pd
 import xarray as xr
 from argparse import ArgumentParser
-from pypromice.process.utilities import addMeta, roundValues
-from pypromice.process.write import prepare_and_write
 from pypromice.process.L1toL2 import correctPrecip
+from pypromice.process.write import prepare_and_write
 logger = logging.getLogger(__name__)
 
 def parse_arguments_join():
@@ -30,6 +29,7 @@ def loadArr(infile):
     elif infile.split('.')[-1].lower() == 'nc':
         with xr.open_dataset(infile) as ds:
             ds.load()
+         # Remove encoding attributes from NetCDF
         for varname in ds.variables:
             if 'encoding' in ds[varname].attrs:
                 del ds[varname].attrs['encoding']
@@ -48,7 +48,7 @@ def loadArr(infile):
     return ds, name
 
 
-def join_l2():
+def join_l2(file1,file2,outpath,variables,metadata):
     args = parse_arguments_join()
     logging.basicConfig(
         format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
@@ -100,7 +100,14 @@ def join_l2():
     all_ds.attrs['format'] = 'merged RAW and TX'
 
     # Resample to hourly, daily and monthly datasets and write to file
-    prepare_and_write(all_ds, args.outpath, args.variables, args.metadata, resample = False)
+    prepare_and_write(all_ds, outpath, variables, metadata, resample = False)
+
+    logger.info(f'Files saved to {os.path.join(outpath, name)}...')
+    return all_ds
 
+def main():
+    args = parse_arguments_join()
+    _ = join_l2(args.file1, args.file2, args.outpath, args.variables, args.metadata)
+
 if __name__ == "__main__":  
-    join_l2()
+    main()