diff --git a/aodntools/timeseries_products/velocity_hourly_timeseries.py b/aodntools/timeseries_products/velocity_hourly_timeseries.py index e5621e3..35cc60f 100644 --- a/aodntools/timeseries_products/velocity_hourly_timeseries.py +++ b/aodntools/timeseries_products/velocity_hourly_timeseries.py @@ -57,13 +57,12 @@ def append_resampled_values(nc_cell, ds, slice_start, binning_functions): df_cell = nc_cell.squeeze().to_dataframe() # shift the index forward 30min to centre the bins on the hour df_cell.index = df_cell.index + pd.Timedelta(minutes=30) - # TODO: shift timestamps to centre of sampling interval df_cell_1H = df_cell.resample('1H') slice_end = len(df_cell_1H) + slice_start # set binned timestamps - time_slice = (np.fromiter(df_cell_1H.groups.keys(), dtype='M8[ns]') - TIME_EPOCH) / ONE_DAY + time_slice = (np.fromiter(df_cell_1H.min().index, dtype='M8[ns]') - TIME_EPOCH) / ONE_DAY ds['TIME'][slice_start:slice_end] = time_slice # take the mean of the variables diff --git a/test_aodntools/timeseries_products/IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV02_velocity-hourly-timeseries_END-20191018_C-20220502.nc b/test_aodntools/timeseries_products/IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV02_velocity-hourly-timeseries_END-20191018_C-20220502.nc new file mode 100644 index 0000000..48433ee Binary files /dev/null and b/test_aodntools/timeseries_products/IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV02_velocity-hourly-timeseries_END-20191018_C-20220502.nc differ diff --git a/test_aodntools/timeseries_products/test_velocity_aggregated_timeseries.py b/test_aodntools/timeseries_products/test_velocity_aggregated_timeseries.py index ac064b0..8001978 100644 --- a/test_aodntools/timeseries_products/test_velocity_aggregated_timeseries.py +++ b/test_aodntools/timeseries_products/test_velocity_aggregated_timeseries.py @@ -22,8 +22,14 @@ TEST_ROOT, 'IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV01_velocity-aggregated-timeseries_END-20191018_C-20200623.nc' ) +OBS_VARS = {'TIME', 'DEPTH', 'DEPTH_quality_control', 'UCUR', 'UCUR_quality_control', + 'VCUR', 'VCUR_quality_control', 'WCUR', 'WCUR_quality_control', 'instrument_index', 'CELL_INDEX'} +INST_VARS = {'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH', 'SECONDS_TO_MIDDLE'} +STR_VARS = {'source_file', 'instrument_id'} + + class TestVelocityAggregatedTimeseries(BaseTestCase): - def test_main_aggregator(self): + def test_velocity_aggregated(self): output_file, bad_files = velocity_aggregated(INPUT_FILES, 'NRSROT', input_dir=TEST_ROOT, output_dir='/tmp') self.assertEqual(1, len(bad_files)) @@ -34,19 +40,13 @@ def test_main_aggregator(self): # check dimensions and variables self.assertSetEqual(set(dataset.dimensions), {'OBSERVATION', 'INSTRUMENT', 'strlen'}) + self.assertSetEqual(set(dataset.variables.keys()), OBS_VARS | INST_VARS | STR_VARS) - obs_vars = {'TIME', 'DEPTH', 'DEPTH_quality_control', 'UCUR', 'UCUR_quality_control', - 'VCUR', 'VCUR_quality_control', 'WCUR', 'WCUR_quality_control', 'instrument_index', 'CELL_INDEX'} - inst_vars = {'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH', 'SECONDS_TO_MIDDLE'} - string_vars = {'source_file', 'instrument_id'} - - self.assertSetEqual(set(dataset.variables.keys()), obs_vars | inst_vars | string_vars) - - for var in obs_vars: + for var in OBS_VARS: self.assertEqual(dataset.variables[var].dimensions, ('OBSERVATION',)) - for var in inst_vars: + for var in INST_VARS: self.assertEqual(dataset.variables[var].dimensions, ('INSTRUMENT',)) - for var in string_vars: + for var in STR_VARS: self.assertEqual(dataset.variables[var].dimensions, ('INSTRUMENT', 'strlen')) for f in chartostring(dataset['source_file'][:]): @@ -58,8 +58,7 @@ def test_main_aggregator(self): # check aggregated variable values expected = Dataset(EXPECTED_OUTPUT_FILE) - compare_vars = ('TIME', 'UCUR', 'UCUR_quality_control', 'VCUR', 'VCUR_quality_control', 'NOMINAL_DEPTH', - 'CELL_INDEX', 'instrument_index', 'DEPTH', 'DEPTH_quality_control') + compare_vars = set(expected.variables.keys()) - STR_VARS non_match_vars = [var for var in compare_vars if not all(dataset[var][:] == expected[var][:]) ] diff --git a/test_aodntools/timeseries_products/test_velocity_hourly_timeseries.py b/test_aodntools/timeseries_products/test_velocity_hourly_timeseries.py new file mode 100644 index 0000000..5f734b7 --- /dev/null +++ b/test_aodntools/timeseries_products/test_velocity_hourly_timeseries.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +import os +import unittest + +import numpy as np +from netCDF4 import Dataset, chartostring + +from aodntools import __version__ +from aodntools.timeseries_products.common import NoInputFilesError +from aodntools.timeseries_products.velocity_hourly_timeseries import velocity_hourly_aggregated +from test_aodntools.base_test import BaseTestCase + +TEST_ROOT = os.path.dirname(__file__) +BAD_FILE = 'IMOS_ANMN-NRS_BAD_VELOCITY_FILE.nc' +INPUT_FILES = [ + 'IMOS_ANMN-NRS_AETVZ_20181213T080000Z_NRSROT-ADCP_FV01_NRSROT-ADCP-1812-Sentinel-or-Monitor-Workhorse-ADCP-44_END-20181215T100000Z_C-20200430T000000Z.nc', + 'IMOS_ANMN-NRS_AETVZ_20180816T080000Z_NRSROT-ADCP_FV01_NRSROT-ADCP-1808-Sentinel-or-Monitor-Workhorse-ADCP-44_END-20180822T053000Z_C-20200623T000000Z.nc', + 'IMOS_ANMN-NRS_AETVZ_20191016T080000Z_NRSROT-ADCP_FV01_NRSROT-ADCP-1910-Sentinel-or-Monitor-Workhorse-ADCP-44_END-20191018T100000Z_C-20200430T000000Z.nc', + BAD_FILE +] +EXPECTED_OUTPUT_FILE = os.path.join( + TEST_ROOT, 'IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV02_velocity-hourly-timeseries_END-20191018_C-20220502.nc' +) + +OBS_VARS = {'TIME', 'instrument_index', 'CELL_INDEX'} +INST_VARS = {'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH', 'SECONDS_TO_MIDDLE'} +STR_VARS = {'source_file', 'instrument_id'} +for v in ['DEPTH', 'UCUR', 'VCUR', 'WCUR']: + OBS_VARS.add(v) + for s in ['_min', '_max', '_std', '_count']: + OBS_VARS.add(v + s) + + +class TestVelocityHourlyTimeseries(BaseTestCase): + def test_velocity_hourly(self): + output_file, bad_files = velocity_hourly_aggregated(INPUT_FILES, 'NRSROT', + input_dir=TEST_ROOT, output_dir='/tmp') + + self.assertEqual(1, len(bad_files)) + for file, errors in bad_files.items(): + self.assertEqual(BAD_FILE, file) + + dataset = Dataset(output_file) + + # check dimensions and variables + self.assertSetEqual(set(dataset.dimensions), {'OBSERVATION', 'INSTRUMENT', 'strlen'}) + self.assertSetEqual(set(dataset.variables.keys()), OBS_VARS | INST_VARS | STR_VARS) + + for var in OBS_VARS: + self.assertEqual(dataset.variables[var].dimensions, ('OBSERVATION',)) + for var in INST_VARS: + self.assertEqual(dataset.variables[var].dimensions, ('INSTRUMENT',)) + for var in STR_VARS: + self.assertEqual(dataset.variables[var].dimensions, ('INSTRUMENT', 'strlen')) + + for f in chartostring(dataset['source_file'][:]): + self.assertIn(f, INPUT_FILES) + + # check attributes + self.assertEqual(__version__, dataset.generating_code_version) + self.assertIn(__version__, dataset.lineage) + + # check aggregated variable values + expected = Dataset(EXPECTED_OUTPUT_FILE) + self.assertEqual(len(expected['TIME']), len(dataset['TIME'])) + + non_match_vars = [] + for var in set(expected.variables.keys()) - STR_VARS: + if not all(np.isclose(dataset[var], expected[var], equal_nan=True)): + non_match_vars.append(var) + self.assertEqual(non_match_vars, []) + + def test_all_rejected(self): + self.assertRaises(NoInputFilesError, velocity_hourly_aggregated, [BAD_FILE], 'NRSROT', + input_dir=TEST_ROOT, output_dir='/tmp') + + +if __name__ == '__main__': + unittest.main()