Skip to content

Commit

Permalink
resampling frequency specified
Browse files Browse the repository at this point in the history
  • Loading branch information
PennyHow committed Jun 12, 2024
1 parent b9fb365 commit 9d0cb14
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 14 deletions.
18 changes: 12 additions & 6 deletions src/pypromice/process/aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ def getL3(self):
# logger.info('Resampling to hour')
# return resampled

def writeArr(self, dataset, outpath):
def writeArr(self, dataset, outpath, t=None):
'''Write L3 data to .nc and .csv hourly and daily files
Parameters
Expand All @@ -128,13 +128,19 @@ def writeArr(self, dataset, outpath):
outpath : str
Output directory
t : str
Resampling string
Resampling string. This is automatically defined based
on the data type if not given. The default is None.
'''
f = [l.attrs['format'] for l in self.L0]
if 'raw' in f or 'STM' in f:
write.prepare_and_write(dataset, outpath, self.vars, self.meta, '10min')
if t is not None:
write.prepare_and_write(dataset, outpath, self.vars, self.meta, t)
else:
write.prepare_and_write(dataset, outpath, self.vars, self.meta, '60min')
f = [l.attrs['format'] for l in self.L0]
if 'raw' in f or 'STM' in f:
write.prepare_and_write(dataset, outpath, self.vars,
self.meta, '10min')
else:
write.prepare_and_write(dataset, outpath, self.vars,
self.meta, '60min')

# def addAttributes(self, dataset):
# '''Add variable and attribute metadata
Expand Down
9 changes: 3 additions & 6 deletions src/pypromice/process/get_l2.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
from argparse import ArgumentParser
import pypromice
from pypromice.process.aws import AWS
from pypromice.process.write import prepare_and_write
from pypromice.process.load import getVars, getMeta

def parse_arguments_l2():
parser = ArgumentParser(description="AWS L2 processor")
Expand All @@ -19,6 +17,8 @@ def parse_arguments_l2():
required=False, help='File path to variables look-up table')
parser.add_argument('-m', '--metadata', default=None, type=str,
required=False, help='File path to metadata')
parser.add_argument('-t', '--time', default=None, type=str,
required=False, help='Resampling frequency')
args = parser.parse_args()
return args

Expand Down Expand Up @@ -59,10 +59,7 @@ def get_l2():
if args.outpath is not None:
if not os.path.isdir(args.outpath):
os.mkdir(args.outpath)
if aws.L2.attrs['format'] == 'raw':
prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '10min')
prepare_and_write(aws.L2, args.outpath, getVars(), getMeta(), '60min')

aws.writeArr(aws.L2, args.outpath, args.time)

This comment has been minimized.

Copy link
@BaptisteVandecrux

BaptisteVandecrux Jun 12, 2024

Member

Hi Penny,

Sorry but this edit does not produce the same result as what was before:

  • if working on l2_raw, then both 10min and hourly files are written
  • if working on l2_tx, then hourly file is written

also I'd like to keep calling directly prepare_and_write and not add an intermediate call to aws.writeArr which itself calls to prepare_and_write. This goes towards the simplification of the code where similare functionalities should not be maintained both within the class aws and within a toolbox like write.py. My preference goes towards calling the functions from the toolbox directly.

This comment has been minimized.

Copy link
@PennyHow

PennyHow Jun 12, 2024

Author Member

Ah sorry! I didn't realise - I just thought maybe you had not seen the class function. No problem, feel free to revert it back.

This comment has been minimized.

Copy link
@PennyHow

PennyHow Jun 12, 2024

Author Member

I think I have solved it. I've added a filter to only sort files that have been modified in the last hour. I've tested this locally and it is working fine, but might be worth testing it on an operational instance also.

This comment has been minimized.

Copy link
@BaptisteVandecrux

BaptisteVandecrux Jun 12, 2024

Member

thanks! I really like building lists of files to process within python and using isModified.


if __name__ == "__main__":
get_l2()
Expand Down
8 changes: 6 additions & 2 deletions src/pypromice/process/get_l3.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ def parse_arguments_l3():
required=False, help='File path to variables look-up table')
parser.add_argument('-m', '--metadata', default=None, type=str,
required=False, help='File path to metadata')
parser.add_argument('-t', '--time', default=None, type=str,
required=False, help='Resampling frequency')
args = parser.parse_args()
return args

Expand Down Expand Up @@ -54,9 +56,11 @@ def get_l3():
aws.getL2()
aws.getL3()

# Write out Level 3
# Write out level 3
if args.outpath is not None:
aws.writeL3(args.outpath)
if not os.path.isdir(args.outpath):
os.mkdir(args.outpath)
aws.writeArr(aws.L3, args.outpath, args.time)

if __name__ == "__main__":
get_l3()
Expand Down

0 comments on commit 9d0cb14

Please sign in to comment.