diff --git a/docs/index.rst b/docs/index.rst index 980b965..1a57699 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,6 @@ Contents xarray_readers_tutorial.ipynb License Authors - Changelog Module Reference diff --git a/tests/get_path.py b/tests/get_path.py new file mode 100644 index 0000000..861860a --- /dev/null +++ b/tests/get_path.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, TU Wien +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of TU Wien, Department of Geodesy and Geoinformation +# nor the names of its contributors may be used to endorse or promote +# products derived from this software without specific prior written +# permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL TU WIEN DEPARTMENT OF GEODESY AND +# GEOINFORMATION BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from pathlib import Path + + +def get_current_path(): + """ + Get current file path. + """ + return Path(__file__).resolve().parent + + +def get_testdata_path(): + """ + Get test data path. + """ + return Path(__file__).resolve().parent / "ascat-test-data" diff --git a/tests/test_cgls.py b/tests/test_cgls.py index 7e80002..2e43697 100644 --- a/tests/test_cgls.py +++ b/tests/test_cgls.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, TU Wien, Department of Geodesy and Geoinformation +# Copyright (c) 2024, TU Wien # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -24,24 +24,23 @@ # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """ Tests for reading CGLOPS SWI data. """ -from ascat.cgls import SWI_TS -import os import pandas as pd import numpy as np import pytest +from get_path import get_testdata_path +from ascat.cgls import SWI_TS + def test_swi_ts_reader(): """ Test SWI time series reader. """ - data_path = os.path.join( - os.path.dirname(__file__), 'ascat_test_data', 'cglops', 'swi_ts') + data_path = get_testdata_path() / "cglops" / "swi_ts" rd = SWI_TS(data_path) data = rd.read(3002621, mask_frozen=False) @@ -68,8 +67,7 @@ def test_swi_ts_reader_no_data_in_folder(): """ Test SWI time series reader when no data is in folder. """ - data_path = os.path.join( - os.path.dirname(__file__), 'ascat_test_data', 'cglops', 'swi_ts_non_existing') + data_path = get_testdata_path() / "cglops" / "swi_ts_non_existing" with pytest.raises(IOError): SWI_TS(data_path) @@ -79,15 +77,14 @@ def test_swi_ts_qflag_reading(): """ Test SWI time series quality flag reader. """ - data_path = os.path.join( - os.path.dirname(__file__), 'ascat_test_data', 'cglops', 'swi_ts') + data_path = get_testdata_path() / "cglops" / "swi_ts" rd = SWI_TS(data_path, parameters=['SWI_001', 'QFLAG_001', 'SSF']) data = rd.read(3002621, mask_frozen=True) # check if QFLAG is correctly read. It should have as many NaN values as # SWI assert len(data[data.loc[:, 'QFLAG_001'] != np.nan]) > 0 - assert (len(data[data.loc[:, 'QFLAG_001'] == np.nan]) == - len(data[data.loc[:, 'SWI_001'] == np.nan])) + assert (len(data[data.loc[:, 'QFLAG_001'] == np.nan]) == len( + data[data.loc[:, 'SWI_001'] == np.nan])) if __name__ == "__main__": diff --git a/tests/test_download.py b/tests/test_download.py index ff1eead..ae0cd0e 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, TU Wien, Department of Geodesy and Geoinformation +# Copyright (c) 2024, TU Wien # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -24,7 +24,6 @@ # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """ Test download. """ @@ -37,10 +36,17 @@ from ascat.download.connectors import HsafConnector from ascat.download.connectors import EumConnector -credentials = {'EUM': {'consumer_key': os.getenv('EUM_CONSUMER_KEY'), - 'consumer_secret': os.getenv('EUM_CONSUMER_SECRET')}, - 'HSAF': {'user': os.getenv('HSAF_FTP_USER'), - 'password': os.getenv('HSAF_FTP_PASSWORD')}} +credentials = { + 'EUM': { + 'consumer_key': os.getenv('EUM_CONSUMER_KEY'), + 'consumer_secret': os.getenv('EUM_CONSUMER_SECRET') + }, + 'HSAF': { + 'user': os.getenv('HSAF_FTP_USER'), + 'password': os.getenv('HSAF_FTP_PASSWORD') + } +} + class TestDownload(unittest.TestCase): @@ -74,8 +80,8 @@ def test_eumetsat_download(self): connector = EumConnector() connector.connect(credentials['EUM']) - connector.download(product, self.local_path, self.start_date, - self.end_date, limit=1) + connector.download( + product, self.local_path, self.start_date, self.end_date, limit=1) @unittest.skipIf(credentials['HSAF']['user'] is None, "Skip H SAF connection test") @@ -97,9 +103,14 @@ def test_hsaf_download(self): connector = HsafConnector() connector.connect(credentials['HSAF']) - connector.download(remote_path, self.local_path, self.start_date, - self.end_date, limit=1) + connector.download( + remote_path, + self.local_path, + self.start_date, + self.end_date, + limit=1) connector.close() + if __name__ == '__main__': unittest.main() diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index 9b052b3..9479532 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, TU Wien, Department of Geodesy and Geoinformation +# Copyright (c) 2024, TU Wien # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -52,12 +52,10 @@ def generate_test_data(): for num in ["01", "02", "03"]: for tile in ["EN01212", "EN01234", "EN01256"]: - folder = tmpdir / Path(f"temperature/{tile}") - + folder = tmpdir / "temperature" / tile folder.mkdir(parents=True, exist_ok=True) - filename = tmpdir / Path( - f"temperature/{tile}/202201{num}_ascat.csv") + filename = tmpdir / "temperature" / tile / f"202201{num}_ascat.csv" csv_file = Csv(filename, mode="w") file_dates = [ @@ -69,8 +67,8 @@ def generate_test_data(): file_temps = random.choices(range(20, 35), k=15) dtype = np.dtype([("date", "datetime64[s]"), ("temperature", "float32")]) - write_data = np.array(list(zip(file_dates, file_temps)), - dtype=dtype) + write_data = np.array( + list(zip(file_dates, file_temps)), dtype=dtype) csv_file.write(write_data) for num in ["01", "02", "03"]: @@ -78,7 +76,7 @@ def generate_test_data(): prev_time = datetime.strptime(f"202201{num}0000", "%Y%m%d%H%M") next_day = prev_time + timedelta(days=1) - folder = tmpdir / Path(f"precipitation/{tile}") + folder = tmpdir / "precipitation" / tile folder.mkdir(parents=True, exist_ok=True) while prev_time < next_day: @@ -102,12 +100,12 @@ def generate_test_data(): filename = folder / Path(f"ascat_{d1}_{t1}-{d2}_{t2}.csv") csv_file = Csv(filename, mode="w") - file_precips = random.choices(range(0, 5), - k=len(file_dates) - 1) + file_precips = random.choices( + range(0, 5), k=len(file_dates) - 1) dtype = np.dtype([("date", "datetime64[s]"), ("precipitation", "float32")]) - write_data = np.array(list(zip(file_dates[:-1], file_precips)), - dtype=dtype) + write_data = np.array( + list(zip(file_dates[:-1], file_precips)), dtype=dtype) csv_file.write(write_data) prev_time = file_dates[-1] @@ -148,7 +146,7 @@ def test_template_property(self): """ self.assertEqual( str(self.template.template), - str(self.tmpdir / "{variable}/{tile}/{date}_*.{suffix}")) + str(self.tmpdir / "{variable}" / "{tile}" / "{date}_*.{suffix}")) def test_build_filename(self): """ @@ -165,7 +163,7 @@ def test_build_filename(self): } self.assertEqual( self.template.build_filename(fn_fmt, sf_fmt), - str(self.tmpdir / "temperature/EN01234/20220101_*.csv")) + str(self.tmpdir / "temperature" / "EN01234" / "20220101_*.csv")) def test_build_basename(self): """ @@ -186,8 +184,8 @@ def test_build_subfolder(self): "tile": "EN01234" } } - self.assertEqual(self.template.build_subfolder(fmt), - ["temperature", "EN01234"]) + self.assertEqual( + self.template.build_subfolder(fmt), ["temperature", "EN01234"]) class TestFileSearch(CustomTestCase): @@ -219,7 +217,7 @@ def test_search(self): recursive = False search_result = self.filesearch.search(fn_fmt, sf_fmt, recursive) expected_result = [ - str(self.tmpdir / "temperature/EN01234/20220101_ascat.csv") + str(self.tmpdir / "temperature" / "EN01234" / "20220101_ascat.csv") ] self.assertEqual(search_result, expected_result) @@ -240,16 +238,15 @@ def test_search_wc(self): search_result = self.filesearch.search(fn_fmt, sf_fmt, recursive) expected_result = [ str(item) for item in [ - self.tmpdir / - "temperature/EN01234/20220101_ascat.csv", self.tmpdir / - "temperature/EN01234/20220102_ascat.csv", self.tmpdir / - "temperature/EN01234/20220103_ascat.csv", self.tmpdir / - "temperature/EN01212/20220101_ascat.csv", self.tmpdir / - "temperature/EN01212/20220102_ascat.csv", self.tmpdir / - "temperature/EN01212/20220103_ascat.csv", self.tmpdir / - "temperature/EN01256/20220101_ascat.csv", self.tmpdir / - "temperature/EN01256/20220102_ascat.csv", self.tmpdir / - "temperature/EN01256/20220103_ascat.csv" + self.tmpdir / "temperature" / "EN01234" / "20220101_ascat.csv", + self.tmpdir / "temperature" / "EN01234" / "20220102_ascat.csv", + self.tmpdir / "temperature" / "EN01234" / "20220103_ascat.csv", + self.tmpdir / "temperature" / "EN01212" / "20220101_ascat.csv", + self.tmpdir / "temperature" / "EN01212" / "20220102_ascat.csv", + self.tmpdir / "temperature" / "EN01212" / "20220103_ascat.csv", + self.tmpdir / "temperature" / "EN01256" / "20220101_ascat.csv", + self.tmpdir / "temperature" / "EN01256" / "20220102_ascat.csv", + self.tmpdir / "temperature" / "EN01256" / "20220103_ascat.csv" ] ] self.assertEqual(sorted(search_result), sorted(expected_result)) @@ -270,7 +267,7 @@ def test_isearch(self): recursive = False search_result = self.filesearch.isearch(fn_fmt, sf_fmt, recursive) expected_result = iter( - [str(self.tmpdir / "temperature/EN01234/20220101_ascat.csv")]) + [str(self.tmpdir / "temperature" / "EN01234" / "20220101_ascat.csv")]) self.assertEqual(list(search_result), list(expected_result)) @@ -291,16 +288,15 @@ def test_isearch_wc(self): search_result = self.filesearch.isearch(fn_fmt, sf_fmt, recursive) expected_result = iter([ str(item) for item in [ - self.tmpdir / - "temperature/EN01212/20220101_ascat.csv", self.tmpdir / - "temperature/EN01234/20220101_ascat.csv", self.tmpdir / - "temperature/EN01256/20220101_ascat.csv", self.tmpdir / - "temperature/EN01212/20220102_ascat.csv", self.tmpdir / - "temperature/EN01234/20220102_ascat.csv", self.tmpdir / - "temperature/EN01256/20220102_ascat.csv", self.tmpdir / - "temperature/EN01212/20220103_ascat.csv", self.tmpdir / - "temperature/EN01234/20220103_ascat.csv", self.tmpdir / - "temperature/EN01256/20220103_ascat.csv" + self.tmpdir / "temperature" /"EN01212"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01234"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01212"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01234"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01212"/ "20220103_ascat.csv", + self.tmpdir / "temperature" /"EN01234"/ "20220103_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220103_ascat.csv" ] ]) self.assertEqual(sorted(search_result), sorted(expected_result)) @@ -327,7 +323,7 @@ def custom_func(arg1, arg2): custom_func, recursive) search_result = custom_search_func("20220101", "temperature") expected_result = [ - str(self.tmpdir / "temperature/EN01234/20220101_ascat.csv") + str(self.tmpdir / "temperature" / "EN01234" / "20220101_ascat.csv") ] self.assertEqual(search_result, expected_result) @@ -354,16 +350,15 @@ def custom_func(arg1, arg2): search_result = custom_search_func("202201*", "temperature") expected_result = [ str(item) for item in [ - self.tmpdir / - "temperature/EN01234/20220101_ascat.csv", self.tmpdir / - "temperature/EN01234/20220102_ascat.csv", self.tmpdir / - "temperature/EN01234/20220103_ascat.csv", self.tmpdir / - "temperature/EN01212/20220101_ascat.csv", self.tmpdir / - "temperature/EN01212/20220102_ascat.csv", self.tmpdir / - "temperature/EN01212/20220103_ascat.csv", self.tmpdir / - "temperature/EN01256/20220101_ascat.csv", self.tmpdir / - "temperature/EN01256/20220102_ascat.csv", self.tmpdir / - "temperature/EN01256/20220103_ascat.csv" + self.tmpdir / "temperature" /"EN01234"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01234"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01234"/ "20220103_ascat.csv", + self.tmpdir / "temperature" /"EN01212"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01212"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01212"/ "20220103_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220103_ascat.csv" ] ] self.assertEqual(sorted(search_result), sorted(expected_result)) @@ -390,7 +385,7 @@ def custom_func(arg1, arg2): custom_func, recursive) search_result = custom_isearch_func("20220101", "temperature") expected_result = iter( - [str(self.tmpdir / "temperature/EN01234/20220101_ascat.csv")]) + [str(self.tmpdir / "temperature" / "EN01234" / "20220101_ascat.csv")]) self.assertEqual(list(search_result), list(expected_result)) @@ -417,16 +412,15 @@ def custom_func(arg1, arg2): search_result = custom_isearch_func("202201*", "temperature") expected_result = iter([ str(item) for item in [ - self.tmpdir / - "temperature/EN01234/20220101_ascat.csv", self.tmpdir / - "temperature/EN01234/20220102_ascat.csv", self.tmpdir / - "temperature/EN01234/20220103_ascat.csv", self.tmpdir / - "temperature/EN01212/20220101_ascat.csv", self.tmpdir / - "temperature/EN01212/20220102_ascat.csv", self.tmpdir / - "temperature/EN01212/20220103_ascat.csv", self.tmpdir / - "temperature/EN01256/20220101_ascat.csv", self.tmpdir / - "temperature/EN01256/20220102_ascat.csv", self.tmpdir / - "temperature/EN01256/20220103_ascat.csv" + self.tmpdir / "temperature" /"EN01234"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01234"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01234"/ "20220103_ascat.csv", + self.tmpdir / "temperature" /"EN01212"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01212"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01212"/ "20220103_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220101_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220102_ascat.csv", + self.tmpdir / "temperature" /"EN01256"/ "20220103_ascat.csv" ] ]) self.assertEqual(sorted(search_result), sorted(expected_result)) @@ -464,11 +458,10 @@ def test_search_date(self): Test search date. """ timestamp = datetime(2022, 1, 1) - filenames = self.chron_files.search_date(timestamp, - date_field_fmt="%Y%m%d", - date_field="date") + filenames = self.chron_files.search_date( + timestamp, date_field_fmt="%Y%m%d", date_field="date") expected_filenames = [ - str(self.tmpdir / "temperature/EN01234/20220101_ascat.csv") + str(self.tmpdir / "temperature" / "EN01234" / "20220101_ascat.csv") ] self.assertEqual(filenames, expected_filenames) @@ -478,13 +471,12 @@ def test_search_period(self): """ dt_start = datetime(2022, 1, 1) dt_end = datetime(2022, 1, 3) - filenames = self.chron_files.search_period(dt_start, - dt_end, - dt_delta=timedelta(days=1)) + filenames = self.chron_files.search_period( + dt_start, dt_end, dt_delta=timedelta(days=1)) expected_filenames = [ - str(self.tmpdir / "temperature/EN01234/20220101_ascat.csv"), - str(self.tmpdir / "temperature/EN01234/20220102_ascat.csv"), - str(self.tmpdir / "temperature/EN01234/20220103_ascat.csv") + str(self.tmpdir / "temperature" / "EN01234" / "20220101_ascat.csv"), + str(self.tmpdir / "temperature" / "EN01234" / "20220102_ascat.csv"), + str(self.tmpdir / "temperature" / "EN01234" / "20220103_ascat.csv") ] self.assertTrue(filenames) self.assertEqual(filenames, expected_filenames) @@ -495,13 +487,11 @@ def test_search_period_exclusive(self): """ dt_start = datetime(2022, 1, 1) dt_end = datetime(2022, 1, 3) - filenames = self.chron_files.search_period(dt_start, - dt_end, - dt_delta=timedelta(days=1), - end_inclusive=False) + filenames = self.chron_files.search_period( + dt_start, dt_end, dt_delta=timedelta(days=1), end_inclusive=False) expected_filenames = [ - str(self.tmpdir / "temperature/EN01234/20220101_ascat.csv"), - str(self.tmpdir / "temperature/EN01234/20220102_ascat.csv"), + str(self.tmpdir / "temperature" / "EN01234" / "20220101_ascat.csv"), + str(self.tmpdir / "temperature" / "EN01234" / "20220102_ascat.csv"), ] self.assertTrue(filenames) self.assertEqual(filenames, expected_filenames) @@ -512,9 +502,8 @@ def test_read_period(self): """ dt_start = datetime(2022, 1, 1, hour=12, minute=30) dt_end = datetime(2022, 1, 2, hour=12, minute=30) - data = self.chron_files.read_period(dt_start, - dt_end, - dt_delta=timedelta(days=1)) + data = self.chron_files.read_period( + dt_start, dt_end, dt_delta=timedelta(days=1)) self.assertTrue(data["date"].min() >= dt_start) self.assertTrue(data["date"].max() <= dt_end) @@ -544,9 +533,8 @@ def test_read_write(self): tmp_data = {} for date in dates: - dt = np.arange(date, - date + np.timedelta64(1, "D"), - dtype="datetime64[h]") + dt = np.arange( + date, date + np.timedelta64(1, "D"), dtype="datetime64[h]") temperature = random.choices(range(0, 40), k=dt.size) data = np.array(list(zip(dt, temperature)), dtype=dtype) diff --git a/tests/test_h_saf.py b/tests/test_h_saf.py index 055dbc6..1fdde07 100644 --- a/tests/test_h_saf.py +++ b/tests/test_h_saf.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, TU Wien, Department of Geodesy and Geoinformation +# Copyright (c) 2024, TU Wien # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -25,7 +25,6 @@ # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os import sys import pytest import unittest @@ -38,6 +37,8 @@ from ascat.h_saf import AscatNrtBufrFileList from ascat.h_saf import AscatSsmDataRecord +from get_path import get_testdata_path + @pytest.mark.skipif(sys.platform == 'win32', reason="Does not work on Windows") class Test_AscatNrtSsm(unittest.TestCase): @@ -46,15 +47,14 @@ def setUp(self): """ Setup test data. """ - self.root_path = os.path.join(os.path.dirname(__file__), - 'ascat_test_data', 'hsaf') + self.root_path = get_testdata_path() / "hsaf" def test_h16_read(self): """ Test read file. """ dt = datetime(2017, 2, 20, 11, 15, 0) - path = os.path.join(self.root_path, 'h16') + path = self.root_path / 'h16' product = 'h16' h16 = AscatNrtBufrFileList(path, product) @@ -90,16 +90,15 @@ def test_h16_read(self): nptest.assert_allclose(data['lat'][798:813], lats_should, atol=1e-5) nptest.assert_allclose(data['sm'][798:813], sm_should, atol=0.01) - nptest.assert_allclose(data['sm_mean'][798:813], - sm_mean_should, - atol=0.01) + nptest.assert_allclose( + data['sm_mean'][798:813], sm_mean_should, atol=0.01) def test_h101_read(self): """ Test read file. """ dt = datetime(2017, 2, 20, 10, 42, 0) - path = os.path.join(self.root_path, 'h101') + path = self.root_path / 'h101' product_id = 'h101' h101 = AscatNrtBufrFileList(path, product_id) @@ -124,16 +123,15 @@ def test_h101_read(self): nptest.assert_allclose(data['lat'][1800:1820], lats_should, atol=1e-5) nptest.assert_allclose(data['sm'][1800:1820], sm_should, atol=0.01) - nptest.assert_allclose(data['sm_mean'][1800:1820], - sm_mean_should, - atol=0.01) + nptest.assert_allclose( + data['sm_mean'][1800:1820], sm_mean_should, atol=0.01) def test_h102_read(self): """ Test read file. """ dt = datetime(2017, 2, 20, 10, 42, 0) - path = os.path.join(self.root_path, 'h102') + path = self.root_path / 'h102' product_id = 'h102' h102 = AscatNrtBufrFileList(path, product_id) @@ -157,16 +155,15 @@ def test_h102_read(self): nptest.assert_allclose(data['lat'][0:19], lats_should, atol=1e-5) nptest.assert_allclose(data['sm'][0:19], sm_should, atol=0.01) - nptest.assert_allclose(data['sm_mean'][0:19], - sm_mean_should, - atol=0.01) + nptest.assert_allclose( + data['sm_mean'][0:19], sm_mean_should, atol=0.01) def test_h103_read(self): """ Test read file. """ dt = datetime(2017, 2, 20, 10, 30, 0) - path = os.path.join(self.root_path, 'h103') + path = self.root_path / 'h103' product_id = 'h103' h103 = AscatNrtBufrFileList(path, product_id) @@ -190,9 +187,8 @@ def test_h103_read(self): nptest.assert_allclose(data['lat'][0:19], lats_should, atol=1e-5) nptest.assert_allclose(data['sm'][0:19], sm_should, atol=0.01) - nptest.assert_allclose(data['sm_mean'][0:19], - sm_mean_should, - atol=0.01) + nptest.assert_allclose( + data['sm_mean'][0:19], sm_mean_should, atol=0.01) @pytest.mark.skipif(sys.platform == 'win32', reason="Does not work on Windows") @@ -202,8 +198,7 @@ def setUp(self): """ Setup test data. """ - self.root_path = os.path.join(os.path.dirname(__file__), - 'ascat_test_data', 'hsaf', 'h14') + self.root_path = get_testdata_path() / "hsaf" / "h14" def test_read(self): """ @@ -225,32 +220,29 @@ def test_read(self): class Test_AscatSsmDataRecord(unittest.TestCase): def setUp(self): - - path = os.path.dirname(__file__) + path = get_testdata_path() self.gpi = 3066159 - self.cdr_path = os.path.join(path, 'ascat_test_data', 'hsaf') - self.grid_path = os.path.join(path, 'ascat_test_data', 'hsaf', 'grid') - self.static_layer_path = os.path.join(path, 'ascat_test_data', 'hsaf', - 'static_layer') + self.cdr_path = path / "hsaf" + self.grid_path = str(path / "hsaf" / "grid") + self.static_layer_path = str(path / "hsaf" / "static_layer") def test_read_h25(self): """ Test read H25. """ - self.h25 = AscatSsmDataRecord(os.path.join(self.cdr_path, 'h25'), - self.grid_path, - static_layer_path=self.static_layer_path) + self.h25 = AscatSsmDataRecord( + str(self.cdr_path / 'h25'), + self.grid_path, + static_layer_path=self.static_layer_path) data = self.h25.read(self.gpi, absolute_sm=True) assert data.attrs['gpi'] == self.gpi - np.testing.assert_approx_equal(data.attrs['lon'], - 19.03533, - significant=4) - np.testing.assert_approx_equal(data.attrs['lat'], - 70.05438, - significant=4) + np.testing.assert_approx_equal( + data.attrs['lon'], 19.03533, significant=4) + np.testing.assert_approx_equal( + data.attrs['lat'], 70.05438, significant=4) assert len(data) == 7737 assert data.iloc[15].name.to_pydatetime() == datetime( @@ -267,41 +259,35 @@ def test_read_h25(self): np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_gldas'], np.nan) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_hwsd'], - 0.1078, - significant=6) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_hwsd'], - 0.0294, - significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_hwsd'], 0.1078, significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_noise_hwsd'], 0.0294, significant=6) assert data.attrs['topo_complex'] == 9 assert data.attrs['wetland_frac'] == 41 - np.testing.assert_approx_equal(data.attrs['porosity_gldas'], - np.nan, - significant=5) - np.testing.assert_approx_equal(data.attrs['porosity_hwsd'], - 0.49000001, - significant=5) + np.testing.assert_approx_equal( + data.attrs['porosity_gldas'], np.nan, significant=5) + np.testing.assert_approx_equal( + data.attrs['porosity_hwsd'], 0.49000001, significant=5) def test_read_h108(self): """ Test read H108. """ self.h108 = AscatSsmDataRecord( - os.path.join(self.cdr_path, 'h108'), + str(self.cdr_path / 'h108'), self.grid_path, static_layer_path=self.static_layer_path) data = self.h108.read(self.gpi, absolute_sm=True) assert data.attrs['gpi'] == self.gpi - np.testing.assert_approx_equal(data.attrs['lon'], - 19.03533, - significant=4) - np.testing.assert_approx_equal(data.attrs['lat'], - 70.05438, - significant=4) + np.testing.assert_approx_equal( + data.attrs['lon'], 19.03533, significant=4) + np.testing.assert_approx_equal( + data.attrs['lat'], 70.05438, significant=4) assert len(data) == 8222 assert data.iloc[15].name.to_pydatetime() == datetime( @@ -318,39 +304,34 @@ def test_read_h108(self): np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_gldas'], np.nan) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_hwsd'], - 0.1078, - significant=6) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_hwsd'], - 0.0294, - significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_hwsd'], 0.1078, significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_noise_hwsd'], 0.0294, significant=6) assert data.attrs['topo_complex'] == 9 assert data.attrs['wetland_frac'] == 41 np.testing.assert_equal(data.attrs['porosity_gldas'], np.nan) - np.testing.assert_approx_equal(data.attrs['porosity_hwsd'], - 0.49000001, - significant=5) + np.testing.assert_approx_equal( + data.attrs['porosity_hwsd'], 0.49000001, significant=5) def test_read_h109(self): """ Test read H109. """ self.h109 = AscatSsmDataRecord( - os.path.join(self.cdr_path, 'h109'), + str(self.cdr_path / 'h109'), self.grid_path, static_layer_path=self.static_layer_path) data = self.h109.read(self.gpi, absolute_sm=True) assert data.attrs['gpi'] == self.gpi - np.testing.assert_approx_equal(data.attrs['lon'], - 19.03533, - significant=4) - np.testing.assert_approx_equal(data.attrs['lat'], - 70.05438, - significant=4) + np.testing.assert_approx_equal( + data.attrs['lon'], 19.03533, significant=4) + np.testing.assert_approx_equal( + data.attrs['lat'], 70.05438, significant=4) assert len(data) == 11736 assert data.iloc[15].name.to_pydatetime() == \ @@ -369,39 +350,34 @@ def test_read_h109(self): np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_gldas'], np.nan) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_hwsd'], - 0.1323, - significant=6) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_hwsd'], - 0.0245, - significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_hwsd'], 0.1323, significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_noise_hwsd'], 0.0245, significant=6) assert data.attrs['topo_complex'] == 9 assert data.attrs['wetland_frac'] == 41 np.testing.assert_equal(data.attrs['porosity_gldas'], np.nan) - np.testing.assert_approx_equal(data.attrs['porosity_hwsd'], - 0.49000001, - significant=5) + np.testing.assert_approx_equal( + data.attrs['porosity_hwsd'], 0.49000001, significant=5) def test_read_h110(self): """ Test read H110. """ self.h110 = AscatSsmDataRecord( - os.path.join(self.cdr_path, 'h110'), + str(self.cdr_path / 'h110'), self.grid_path, static_layer_path=self.static_layer_path) data = self.h110.read(self.gpi, absolute_sm=True) assert data.attrs['gpi'] == self.gpi - np.testing.assert_approx_equal(data.attrs['lon'], - 19.03533, - significant=4) - np.testing.assert_approx_equal(data.attrs['lat'], - 70.05438, - significant=4) + np.testing.assert_approx_equal( + data.attrs['lon'], 19.03533, significant=4) + np.testing.assert_approx_equal( + data.attrs['lat'], 70.05438, significant=4) assert len(data) == 1148 assert data.iloc[15].name.to_pydatetime() == datetime( @@ -420,39 +396,34 @@ def test_read_h110(self): np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_gldas'], np.nan) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_hwsd'], - 0.2352, - significant=6) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_hwsd'], - 0.0245, - significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_hwsd'], 0.2352, significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_noise_hwsd'], 0.0245, significant=6) assert data.attrs['topo_complex'] == 9 assert data.attrs['wetland_frac'] == 41 np.testing.assert_equal(data.attrs['porosity_gldas'], np.nan) - np.testing.assert_approx_equal(data.attrs['porosity_hwsd'], - 0.49000001, - significant=5) + np.testing.assert_approx_equal( + data.attrs['porosity_hwsd'], 0.49000001, significant=5) def test_read_h111(self): """ Test read H111. """ self.h111 = AscatSsmDataRecord( - os.path.join(self.cdr_path, 'h111'), + str(self.cdr_path / 'h111'), self.grid_path, static_layer_path=self.static_layer_path) data = self.h111.read(self.gpi, absolute_sm=True) assert data.attrs['gpi'] == self.gpi - np.testing.assert_approx_equal(data.attrs['lon'], - 19.03533, - significant=4) - np.testing.assert_approx_equal(data.attrs['lat'], - 70.05438, - significant=4) + np.testing.assert_approx_equal( + data.attrs['lon'], 19.03533, significant=4) + np.testing.assert_approx_equal( + data.attrs['lat'], 70.05438, significant=4) assert len(data) == 13715 assert data.iloc[15].name.to_pydatetime() == datetime( @@ -471,20 +442,17 @@ def test_read_h111(self): np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_gldas'], np.nan) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_hwsd'], - 0.1372, - significant=6) - np.testing.assert_approx_equal(data.iloc[15]['abs_sm_noise_hwsd'], - 0.0245, - significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_hwsd'], 0.1372, significant=6) + np.testing.assert_approx_equal( + data.iloc[15]['abs_sm_noise_hwsd'], 0.0245, significant=6) assert data.attrs['topo_complex'] == 9 assert data.attrs['wetland_frac'] == 41 np.testing.assert_equal(data.attrs['porosity_gldas'], np.nan) - np.testing.assert_approx_equal(data.attrs['porosity_hwsd'], - 0.49000001, - significant=5) + np.testing.assert_approx_equal( + data.attrs['porosity_hwsd'], 0.49000001, significant=5) def test_read_2points_cell_switch(self): """ @@ -493,7 +461,7 @@ def test_read_2points_cell_switch(self): was closed too soon. """ self.h111 = AscatSsmDataRecord( - os.path.join(self.cdr_path, 'h111'), + str(self.cdr_path / 'h111'), self.grid_path, static_layer_path=self.static_layer_path) diff --git a/tests/test_level1.py b/tests/test_level1.py index acfdb8b..8ccf824 100644 --- a/tests/test_level1.py +++ b/tests/test_level1.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, TU Wien, Department of Geodesy and Geoinformation +# Copyright (c) 2024, TU Wien # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -28,7 +28,6 @@ Test ASCAT Level 1 reader. """ -import os import sys import pytest import unittest @@ -43,6 +42,8 @@ from ascat.eumetsat.level1 import AscatL1bBufrFileList from ascat.eumetsat.level1 import AscatL1bHdf5FileList +from get_path import get_testdata_path + float32_nan = -999999. @@ -50,36 +51,15 @@ class Test_AscatL1bFile(unittest.TestCase): def setUp(self): - data_path = os.path.join(os.path.dirname(__file__), "ascat_test_data", - "eumetsat", "ASCAT_generic_reader_data") - - name_b = os.path.join( - data_path, "bufr", - "M02-ASCA-ASCSZR1B0200-NA-9.1-20100609013900.000000000Z-20130824233100-1280350.bfr" - ) - name_e = os.path.join( - data_path, "eps_nat", - "ASCA_SZR_1B_M02_20100609013900Z_20100609032058Z_R_O_20130824233100Z.nat" - ) - name_n = os.path.join( - data_path, "nc", - "W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,METOPA+ASCAT_C_EUMP_20100609013900_18872_eps_o_125_l1.nc" - ) - - name_e11 = os.path.join( - data_path, "eps_nat", - "ASCA_SZR_1B_M02_20071212071500Z_20071212085659Z_R_O_20081225063118Z.nat" - ) - - name_e_szf = os.path.join( - data_path, "eps_nat", - "ASCA_SZF_1B_M01_20180611041800Z_20180611055959Z_N_O_20180611050637Z.nat" - ) - - name_h = os.path.join( - data_path, "hdf5", - "ASCA_SZF_1B_M01_20180611041800Z_20180611055959Z_N_O_20180611050637Z.h5" - ) + data_path = get_testdata_path() / "eumetsat" / "ASCAT_generic_reader_data" + + name_b = data_path / "bufr" / "M02-ASCA-ASCSZR1B0200-NA-9.1-20100609013900.000000000Z-20130824233100-1280350.bfr" + name_e = data_path / "eps_nat" / "ASCA_SZR_1B_M02_20100609013900Z_20100609032058Z_R_O_20130824233100Z.nat" + name_n = data_path / "nc" / "W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,METOPA+ASCAT_C_EUMP_20100609013900_18872_eps_o_125_l1.nc" + + name_e11 = data_path / "eps_nat" / "ASCA_SZR_1B_M02_20071212071500Z_20071212085659Z_R_O_20081225063118Z.nat" + name_e_szf = data_path / "eps_nat" / "ASCA_SZF_1B_M01_20180611041800Z_20180611055959Z_N_O_20180611050637Z.nat" + name_h = data_path / "hdf5" / "ASCA_SZF_1B_M01_20180611041800Z_20180611055959Z_N_O_20180611050637Z.h5" self.bufr = AscatL1bFile(name_b) self.nc = AscatL1bFile(name_n) @@ -324,32 +304,31 @@ def setUp(self): """ Setup test data. """ - root_path = os.path.join(os.path.dirname(__file__), "ascat_test_data", - "eumetsat", "ASCAT_generic_reader_data") + root_path = get_testdata_path() / "eumetsat" / "ASCAT_generic_reader_data" - path = os.path.join(root_path, "bufr") + path = root_path / "bufr" sat = "a" product = "szr" self.bufr_szr = AscatL1bBufrFileList(path, sat, product) - path = os.path.join(root_path, "nc") + path = root_path / "nc" sat = "a" product = "szr" self.nc_szr = AscatL1bNcFileList(path, sat, product) - path = os.path.join(root_path, "eps_nat") + path = root_path / "eps_nat" sat = "a" product = "szr" self.eps_szr = AscatL1bEpsFileList(path, sat, product) sat = "b" product = "szf" - path = os.path.join(root_path, "eps_nat") + path = root_path / "eps_nat" self.eps_szf = AscatL1bEpsFileList(path, sat, product) sat = "b" product = "szf" - path = os.path.join(root_path, "hdf5") + path = root_path / "hdf5" self.hdf5_szf = AscatL1bHdf5FileList(path, sat, product) def test_szr_read_date(self): diff --git a/tests/test_level2.py b/tests/test_level2.py index c370b32..3cb16ab 100644 --- a/tests/test_level2.py +++ b/tests/test_level2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, TU Wien, Department of Geodesy and Geoinformation +# Copyright (c) 2024, TU Wien # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -44,6 +44,8 @@ from ascat.eumetsat.level2 import AscatL2BufrFileList from ascat.eumetsat.level2 import AscatL2EpsFileList +from get_path import get_testdata_path + eps_float_nan = -2147483648. bufr_float_nan = 1.7e+38 uint8_nan = np.iinfo(np.uint8).max @@ -58,15 +60,8 @@ def setUp(self): """ Setup test files. """ - data_path = os.path.join(os.path.dirname(__file__), 'ascat_test_data', - 'eumetsat', 'ASCAT_L2_SM_125', 'bufr', - 'Metop_B') - - fname = os.path.join( - data_path, - 'M01-ASCA-ASCSMR02-NA-5.0-20170220050900.000000000Z-20170220055833-1207110.bfr' - ) - + data_path = get_testdata_path() / "eumetsat/ASCAT_L2_SM_125/bufr/Metop_B" + fname = data_path / 'M01-ASCA-ASCSMR02-NA-5.0-20170220050900.000000000Z-20170220055833-1207110.bfr' self.reader = AscatL2BufrFile(fname) def test_read(self): @@ -109,13 +104,8 @@ def setUp(self): """ Setup test files. """ - data_path = os.path.join(os.path.dirname(__file__), 'ascat_test_data', - 'eumetsat', 'ASCAT_L2_SM_125', 'nc', - 'Metop_A') - fname = os.path.join( - data_path, - 'W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,METOPA+ASCAT_C_EUMP_20170220041500_53652_eps_o_125_ssm_l2.nc' - ) + data_path = get_testdata_path() / "eumetsat/ASCAT_L2_SM_125/nc/Metop_A" + fname = data_path / 'W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,METOPA+ASCAT_C_EUMP_20170220041500_53652_eps_o_125_ssm_l2.nc' self.reader = AscatL2NcFile(fname) def test_read(self): @@ -162,18 +152,12 @@ def setUp(self): """ Setup test files. """ - data_path = os.path.join(os.path.dirname(__file__), 'ascat_test_data', - 'eumetsat', 'ASCAT_L2_SM_125') - fname_nc = os.path.join( - data_path, 'nc', 'Metop_A', - 'W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,METOPA+ASCAT_C_EUMP_20170220041500_53652_eps_o_125_ssm_l2.nc' - ) + data_path = get_testdata_path() / "eumetsat/ASCAT_L2_SM_125" + + fname_nc = data_path / "nc/Metop_A/W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,METOPA+ASCAT_C_EUMP_20170220041500_53652_eps_o_125_ssm_l2.nc" self.reader_nc = AscatL2NcFile(fname_nc) - fname_bufr = os.path.join( - data_path, 'bufr', 'Metop_A', - 'M02-ASCA-ASCSMR02-NA-5.0-20170220041500.000000000Z-20170220055656-1207110.bfr' - ) + fname_bufr = data_path / "bufr/Metop_A/M02-ASCA-ASCSMR02-NA-5.0-20170220041500.000000000Z-20170220055656-1207110.bfr" self.reader_bufr = AscatL2BufrFile(fname_bufr) def test_read(self): @@ -254,21 +238,11 @@ def setUp(self): """ Setup test files. """ - data_path = os.path.join(os.path.dirname(__file__), 'ascat_test_data', - 'eumetsat', 'ASCAT_generic_reader_data') - - name_b = os.path.join( - data_path, 'bufr', - 'M01-ASCA-ASCSMO02-NA-5.0-20180612035700.000000000Z-20180612044530-1281300.bfr' - ) - name_e = os.path.join( - data_path, 'eps_nat', - 'ASCA_SMO_02_M01_20180612035700Z_20180612053856Z_N_O_20180612044530Z.nat' - ) - name_n = os.path.join( - data_path, 'nc', - 'W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,METOPB+ASCAT_C_EUMP_20180612035700_29742_eps_o_250_ssm_l2.nc' - ) + data_path = get_testdata_path() / "eumetsat/ASCAT_generic_reader_data" + + name_b = data_path / "bufr/M01-ASCA-ASCSMO02-NA-5.0-20180612035700.000000000Z-20180612044530-1281300.bfr" + name_e = data_path / "eps_nat/ASCA_SMO_02_M01_20180612035700Z_20180612053856Z_N_O_20180612044530Z.nat" + name_n = data_path / "nc/W_XX-EUMETSAT-Darmstadt,SURFACE+SATELLITE,METOPB+ASCAT_C_EUMP_20180612035700_29742_eps_o_250_ssm_l2.nc" self.bufr = AscatL2File(name_b) self.eps = AscatL2File(name_e) @@ -378,20 +352,19 @@ def setUp(self): """ Setup test data. """ - root_path = os.path.join(os.path.dirname(__file__), 'ascat_test_data', - 'eumetsat', 'ASCAT_generic_reader_data') + root_path = get_testdata_path() / "eumetsat/ASCAT_generic_reader_data" - path = os.path.join(root_path, 'bufr') + path = root_path / 'bufr' sat = 'b' product = 'smo' self.bufr_smo = AscatL2BufrFileList(path, sat, product) - path = os.path.join(root_path, 'nc') + path = root_path / 'nc' sat = 'b' product = 'smo' self.nc_smo = AscatL2NcFileList(path, sat, product) - path = os.path.join(root_path, 'eps_nat') + path = root_path / 'eps_nat' sat = 'b' product = 'smo' self.eps_smo = AscatL2EpsFileList(path, sat, product) diff --git a/tests/test_ragged_array.py b/tests/test_ragged_array.py index 832a693..24f1db6 100644 --- a/tests/test_ragged_array.py +++ b/tests/test_ragged_array.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, TU Wien, Department of Geodesy and Geoinformation +# Copyright (c) 2024, TU Wien # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -29,21 +29,15 @@ from pathlib import Path from datetime import datetime from tempfile import TemporaryDirectory -from copy import deepcopy import numpy as np import xarray as xr -from fibgrid.realization import FibGrid - -from ascat.file_handling import ChronFiles - from ascat.read_native.ragged_array_ts import CellFileCollection from ascat.read_native.ragged_array_ts import SwathFileCollection from ascat.read_native.ragged_array_ts import CellFileCollectionStack from ascat.read_native.xarray_io import AscatH129Cell -from ascat.read_native.xarray_io import AscatSIG0Cell6250m from ascat.read_native.xarray_io import AscatSIG0Cell12500m from ascat.read_native.xarray_io import AscatH129Swath