Skip to content

Check if files are found within requested time range #219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 8, 2017
Merged

Check if files are found within requested time range #219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
45b682e
Fix update of clim. start and end date in MOC
xylar Aug 4, 2017
c06a85c
Fix setup_and_check call chain for climatology maps
xylar Aug 4, 2017
a700369
Check that streams files were found
xylar Aug 4, 2017
0d051bd
Fix start/end year for MHT task
xylar Aug 4, 2017
49653bc
Fix calendar in error message in generalized_reader
xylar Aug 7, 2017
079c450
Remove file padding when getting file list from stream
xylar Aug 7, 2017
94e8402
Move MHT file check to setup_and_check
xylar Aug 7, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 23 additions & 18 deletions mpas_analysis/ocean/climatology_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,21 @@ def setup_and_check(self): # {{{
analysisOptionName='config_am_timeseriesstatsmonthly_enable',
raiseException=True)

# get a list of timeSeriesStats output files from the streams file,
# reading only those that are between the start and end dates
self.startDate = self.config.get('climatology', 'startDate')
self.endDate = self.config.get('climatology', 'endDate')
streamName = \
self.historyStreams.find_stream(self.streamMap['timeSeriesStats'])
self.inputFiles = self.historyStreams.readpath(
streamName, startDate=self.startDate, endDate=self.endDate,
calendar=self.calendar)

if len(self.inputFiles) == 0:
raise IOError('No files were found in stream {} between {} and '
'{}.'.format(streamName, self.startDate,
self.endDate))

# }}}

def run(self): # {{{
Expand All @@ -93,20 +108,10 @@ def run(self): # {{{

simulationStartTime = get_simulation_start_time(self.runStreams)

# get a list of timeSeriesStats output files from the streams file,
# reading only those that are between the start and end dates
startDate = config.get('climatology', 'startDate')
endDate = config.get('climatology', 'endDate')
streamName = \
self.historyStreams.find_stream(self.streamMap['timeSeriesStats'])
inputFiles = self.historyStreams.readpath(streamName,
startDate=startDate,
endDate=endDate,
calendar=calendar)
print '\n Reading files:\n' \
' {} through\n {}'.format(
os.path.basename(inputFiles[0]),
os.path.basename(inputFiles[-1]))
os.path.basename(self.inputFiles[0]),
os.path.basename(self.inputFiles[-1]))

mainRunName = config.get('runs', 'mainRunName')

Expand All @@ -128,16 +133,16 @@ def run(self): # {{{

varList = [fieldName]

ds = open_multifile_dataset(fileNames=inputFiles,
ds = open_multifile_dataset(fileNames=self.inputFiles,
calendar=calendar,
config=config,
simulationStartTime=simulationStartTime,
timeVariableName='Time',
variableList=varList,
iselValues=self.iselValues,
variableMap=self.variableMap,
startDate=startDate,
endDate=endDate)
startDate=self.startDate,
endDate=self.endDate)

changed, startYear, endYear = update_start_end_year(ds, config,
calendar)
Expand Down Expand Up @@ -342,7 +347,7 @@ def setup_and_check(self): # {{{
# self.runDirectory , self.historyDirectory, self.plotsDirectory,
# self.namelist, self.runStreams, self.historyStreams,
# self.calendar, self.namelistMap, self.streamMap, self.variableMap
super(ClimatologyMapOcean, self).setup_and_check()
super(ClimatologyMapSST, self).setup_and_check()

observationsDirectory = build_config_full_path(
self.config, 'oceanObservations',
Expand Down Expand Up @@ -451,7 +456,7 @@ def setup_and_check(self): # {{{
# self.runDirectory , self.historyDirectory, self.plotsDirectory,
# self.namelist, self.runStreams, self.historyStreams,
# self.calendar, self.namelistMap, self.streamMap, self.variableMap
super(ClimatologyMapOcean, self).setup_and_check()
super(ClimatologyMapSSS, self).setup_and_check()

observationsDirectory = build_config_full_path(
self.config, 'oceanObservations',
Expand Down Expand Up @@ -543,7 +548,7 @@ def setup_and_check(self): # {{{
# self.runDirectory , self.historyDirectory, self.plotsDirectory,
# self.namelist, self.runStreams, self.historyStreams,
# self.calendar, self.namelistMap, self.streamMap, self.variableMap
super(ClimatologyMapOcean, self).setup_and_check()
super(ClimatologyMapMLD, self).setup_and_check()

observationsDirectory = build_config_full_path(
self.config, 'oceanObservations',
Expand Down
35 changes: 20 additions & 15 deletions mpas_analysis/ocean/index_nino34.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,21 @@ def setup_and_check(self): # {{{
# self.calendar, self.namelistMap, self.streamMap, self.variableMap
super(IndexNino34, self).setup_and_check()

# get a list of timeSeriesStats output files from the streams file,
# reading only those that are between the start and end dates
streamName = self.historyStreams.find_stream(
self.streamMap['timeSeriesStats'])
self.startDate = self.config.get('index', 'startDate')
self.endDate = self.config.get('index', 'endDate')
self.inputFiles = self.historyStreams.readpath(
streamName, startDate=self.startDate, endDate=self.endDate,
calendar=self.calendar)

if len(self.inputFiles) == 0:
raise IOError('No files were found in stream {} between {} and '
'{}.'.format(streamName, self.startDate,
self.endDate))

# }}}

def run(self): # {{{
Expand All @@ -87,10 +102,6 @@ def run(self): # {{{
config = self.config
calendar = self.calendar

# get a list of timeSeriesStats output files from the streams file,
# reading only those that are between the start and end dates
startDate = config.get('index', 'startDate')
endDate = config.get('index', 'endDate')
dataSource = config.get('indexNino34', 'observationData')

observationsDirectory = build_config_full_path(
Expand All @@ -105,16 +116,10 @@ def run(self): # {{{
dataPath = "{}/ERS_SSTv4_nino34.nc".format(observationsDirectory)
obsTitle = 'ERS SSTv4'

streamName = self.historyStreams.find_stream(
self.streamMap['timeSeriesStats'])
fileNames = self.historyStreams.readpath(streamName,
startDate=startDate,
endDate=endDate,
calendar=calendar)
print '\n Reading files:\n' \
' {} through\n {}'.format(
os.path.basename(fileNames[0]),
os.path.basename(fileNames[-1]))
os.path.basename(self.inputFiles[0]),
os.path.basename(self.inputFiles[-1]))
mainRunName = config.get('runs', 'mainRunName')

# regionIndex should correspond to NINO34 in surface weighted Average
Expand All @@ -123,15 +128,15 @@ def run(self): # {{{

# Load data:
varList = ['avgSurfaceTemperature']
ds = open_multifile_dataset(fileNames=fileNames,
ds = open_multifile_dataset(fileNames=self.inputFiles,
calendar=calendar,
config=config,
simulationStartTime=simulationStartTime,
timeVariableName='Time',
variableList=varList,
variableMap=self.variableMap,
startDate=startDate,
endDate=endDate)
startDate=self.startDate,
endDate=self.endDate)

# Observations have been processed to the nino34Index prior to reading
dsObs = xr.open_dataset(dataPath)
Expand Down
40 changes: 27 additions & 13 deletions mpas_analysis/ocean/meridional_heat_transport.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@

from ..shared.timekeeping.utility import get_simulation_start_time

from ..shared.climatology.climatology import cache_climatologies
from ..shared.climatology.climatology import update_start_end_year, \
cache_climatologies

from ..shared.analysis_task import AnalysisTask

Expand Down Expand Up @@ -82,7 +83,7 @@ def setup_and_check(self): # {{{

# Get a list of timeSeriesStats output files from the streams file,
# reading only those that are between the start and end dates
# First a list necessary for theMHT climatology
# First a list necessary for the MHT climatology
streamName = self.historyStreams.find_stream(
self.streamMap['timeSeriesStats'])
self.startDate = config.get('climatology', 'startDate')
Expand All @@ -92,6 +93,22 @@ def setup_and_check(self): # {{{
startDate=self.startDate,
endDate=self.endDate,
calendar=self.calendar)

if len(self.inputFiles) == 0:
raise IOError('No files were found in stream {} between {} and '
'{}.'.format(streamName, self.startDate,
self.endDate))

# Later, we will read in depth and MHT latitude points
# from mpaso.hist.am.meridionalHeatTransport.*.nc
mhtFiles = self.historyStreams.readpath(
'meridionalHeatTransportOutput')
if len(mhtFiles) == 0:
raise IOError('No MPAS-O MHT history file found: need at least '
'one ')

self.mhtFile = mhtFiles[0]

self.simulationStartTime = get_simulation_start_time(self.runStreams)

self.startYear = config.getint('climatology', 'startYear')
Expand Down Expand Up @@ -137,15 +154,9 @@ def run(self): # {{{
# mpaso.hist.am.meridionalHeatTransport.*.nc
# Depth is from refZMid, also in
# mpaso.hist.am.meridionalHeatTransport.*.nc
try:
mhtFile = self.historyStreams.readpath(
'meridionalHeatTransportOutput')[0]
except ValueError:
raise IOError('No MPAS-O MHT history file found: need at least '
'one ')

print ' Read in depth and latitude...'
ncFile = netCDF4.Dataset(mhtFile, mode='r')
ncFile = netCDF4.Dataset(self.mhtFile, mode='r')
# reference depth [m]
refZMid = ncFile.variables['refZMid'][:]
refBottomDepth = ncFile.variables['refBottomDepth'][:]
Expand Down Expand Up @@ -195,6 +206,9 @@ def run(self): # {{{
startDate=self.startDate,
endDate=self.endDate)

changed, startYear, endYear = update_start_end_year(ds, config,
self.calendar)

# Compute annual climatology
cachePrefix = '{}/meridionalHeatTransport'.format(outputDirectory)
annualClimatology = cache_climatologies(ds, monthDictionary['ANN'],
Expand All @@ -216,10 +230,10 @@ def run(self): # {{{
xLabel = 'latitude [deg]'
yLabel = 'meridional heat transport [PW]'
title = 'Global MHT (ANN, years {:04d}-{:04d})\n {}'.format(
self.startYear, self.endYear, mainRunName)
startYear, endYear, mainRunName)
figureName = '{}/mht_{}_years{:04d}-{:04d}.png'.format(
self.plotsDirectory, mainRunName,
self.startYear, self.endYear)
startYear, endYear)
if self.observationsFile is not None:
# Load in observations
dsObs = xr.open_dataset(self.observationsFile)
Expand Down Expand Up @@ -261,10 +275,10 @@ def run(self): # {{{
xLabel = 'latitude [deg]'
yLabel = 'depth [m]'
title = 'Global MHT (ANN, years {:04d}-{:04d})\n {}'.format(
self.startYear, self.endYear, mainRunName)
startYear, endYear, mainRunName)
figureName = '{}/mhtZ_{}_years{:04d}-{:04d}.png'.format(
self.plotsDirectory, mainRunName,
self.startYear, self.endYear)
startYear, endYear)
colorbarLabel = '[PW/m]'
contourLevels = config.getExpression(self.sectionName,
'contourLevelsGlobal',
Expand Down
11 changes: 11 additions & 0 deletions mpas_analysis/ocean/streamfunction_moc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,11 @@ def setup_and_check(self): # {{{
startDate=self.startDateClimo,
endDate=self.endDateClimo,
calendar=self.calendar)
if len(self.inputFilesClimo) == 0:
raise IOError('No files were found in stream {} between {} and '
'{}.'.format(streamName, self.startDateClimo,
self.endDateClimo))

self.simulationStartTime = get_simulation_start_time(self.runStreams)

self.startYearClimo = config.getint('climatology', 'startYear')
Expand All @@ -118,6 +123,10 @@ def setup_and_check(self): # {{{
startDate=self.startDateTseries,
endDate=self.endDateTseries,
calendar=self.calendar)
if len(self.inputFilesTseries) == 0:
raise IOError('No files were found in stream {} between {} and '
'{}.'.format(streamName, self.startDateTseries,
self.endDateTseries))

self.startYearTseries = config.getint('timeSeries', 'startYear')
self.endYearTseries = config.getint('timeSeries', 'endYear')
Expand Down Expand Up @@ -270,6 +279,8 @@ def _cache_velocity_climatologies(self): # {{{
# update the start and end year in config based on the real extend of
# ds
update_start_end_year(ds, config, self.calendar)
self.startYearClimo = config.getint('climatology', 'startYear')
self.endYearClimo = config.getint('climatology', 'endYear')

cachePrefix = '{}/meanVelocity'.format(outputDirectory)

Expand Down
42 changes: 24 additions & 18 deletions mpas_analysis/ocean/time_series_ohc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,22 @@ def setup_and_check(self): # {{{
if config.get('runs', 'preprocessedReferenceRunName') != 'None':
check_path_exists(config.get('oceanPreprocessedReference',
'baseDirectory'))

# get a list of timeSeriesStats output files from the streams file,
# reading only those that are between the start and end dates
self.streamName = self.historyStreams.find_stream(
self.streamMap['timeSeriesStats'])
self.startDate = self.config.get('timeSeries', 'startDate')
self.endDate = self.config.get('timeSeries', 'endDate')
self.inputFiles = self.historyStreams.readpath(
self.streamName, startDate=self.startDate,
endDate=self.endDate, calendar=self.calendar)

if len(self.inputFiles) == 0:
raise IOError('No files were found in stream {} between {} and '
'{}.'.format(self.streamName, self.startDate,
self.endDate))

return # }}}

def run(self): # {{{
Expand Down Expand Up @@ -131,20 +147,10 @@ def run(self): # {{{
raise IOError('No MPAS-O restart file found: need at least one '
'restart file for OHC calculation')

# get a list of timeSeriesStats output files from the streams file,
# reading only those that are between the start and end dates
startDate = config.get('timeSeries', 'startDate')
endDate = config.get('timeSeries', 'endDate')
streamName = self.historyStreams.find_stream(
self.streamMap['timeSeriesStats'])
fileNames = self.historyStreams.readpath(streamName,
startDate=startDate,
endDate=endDate,
calendar=calendar)
print '\n Reading files:\n' \
' {} through\n {}'.format(
os.path.basename(fileNames[0]),
os.path.basename(fileNames[-1]))
os.path.basename(self.inputFiles[0]),
os.path.basename(self.inputFiles[-1]))

# Define/read in general variables
print ' Read in depth and compute specific depth indexes...'
Expand All @@ -164,20 +170,20 @@ def run(self): # {{{
'sumLayerMaskValue',
'avgLayerArea',
'avgLayerThickness']
ds = open_multifile_dataset(fileNames=fileNames,
ds = open_multifile_dataset(fileNames=self.inputFiles,
calendar=calendar,
config=config,
simulationStartTime=simulationStartTime,
timeVariableName='Time',
variableList=variableList,
variableMap=self.variableMap,
startDate=startDate,
endDate=endDate)
startDate=self.startDate,
endDate=self.endDate)

ds = ds.isel(nOceanRegionsTmp=regionIndicesToPlot)

timeStart = string_to_datetime(startDate)
timeEnd = string_to_datetime(endDate)
timeStart = string_to_datetime(self.startDate)
timeEnd = string_to_datetime(self.endDate)

# Select year-1 data and average it (for later computing anomalies)
timeStartFirstYear = string_to_datetime(simulationStartTime)
Expand All @@ -186,7 +192,7 @@ def run(self): # {{{
firstYear = int(startDateFirstYear[0:4])
endDateFirstYear = '{:04d}-12-31_23:59:59'.format(firstYear)
filesFirstYear = \
self.historyStreams.readpath(streamName,
self.historyStreams.readpath(self.streamName,
startDate=startDateFirstYear,
endDate=endDateFirstYear,
calendar=calendar)
Expand Down
Loading