From 88f41d6fcbd40834cfa5b206393ac3cc7d8357ba Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 17 Feb 2021 21:15:28 +0100 Subject: [PATCH 1/5] Add option to fetch recent TLEs from another/central read-only place Signed-off-by: Adam.Dybbroe --- aapp_runner/tle_satpos_prepare.py | 59 +++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/aapp_runner/tle_satpos_prepare.py b/aapp_runner/tle_satpos_prepare.py index 3a3d5f3..10be18a 100644 --- a/aapp_runner/tle_satpos_prepare.py +++ b/aapp_runner/tle_satpos_prepare.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015, 2018 Pytroll developers +# Copyright (c) 2015, 2018, 2021 Pytroll developers # Author(s): @@ -38,9 +38,11 @@ from datetime import datetime from shutil import copy from aapp_runner.helper_functions import run_shell_command -from trollsift.parser import compose +from trollsift.parser import compose, globify, Parser import re import time +import tempfile +import shutil LOG = logging.getLogger(__name__) @@ -64,12 +66,14 @@ def _do_3_matches(m): def _do_3_matchesYY(m): return datetime.strptime(m.group(1) + m.group(2) + m.group(3), "%y%m%d") + tle_match_tests = ((r'.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2})(\d{2}).*', _do_6_matches), (r'.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2}).*', _do_5_matches), (r'.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2}).*', _do_4_matches), (r'.*(\d{4})(\d{2})(\d{2}).*', _do_3_matches), (r'.*(\d{2})(\d{2})(\d{2}).*', _do_3_matchesYY)) + def download_tle(config, timestamp, dir_data_tle): user = os.getenv("PAR_NAVIGATION_TLE_USER", "xxxxxx") @@ -178,12 +182,46 @@ def download_tle(config, timestamp, dir_data_tle): return tle_file_list +def fetch_realtime_tles(tle_input_path, tle_output_path, tle_infile_format): + """Get the recent TLEs and copy them into the AAPP data structure""" + # tle_infile_format + + infiles = glob(os.path.join(tle_input_path, globify(tle_infile_format))) + p__ = Parser(tle_infile_format) + for filepath in infiles: + filename = os.path.basename(filepath) + res = p__.parse(filename) + dobj = res['timestamp'] + + subdirname = dtobj.strftime('%Y_%m') + subdirpath = os.path.join(tle_output_path, subdirname) + outfile = os.path.join(subdirpath, filename) + LOG.debug("OUTPUT file = %s", str(outfile)) + + if not os.path.exists(subdirpath): + os.mkdir(subdirpath) + tmp_filepath = tempfile.mktemp(suffix='_' + os.path.basename(outfile), + dir=os.path.dirname(outfile)) + LOG.debug("tmp-filepath = %s", tmp_filepath) + shutil.copy(filepath, tmp_filepath) + LOG.debug("File copied: %s -> %s", filepath, tmp_filepath) + os.rename(tmp_filepath, outfile) + LOG.debug("Rename: %s -> %s", tmp_filepath, outfile) + + def do_tleing(config, timestamp, satellite): """Get the tle-file and copy them to the AAPP data structure and run the AAPP tleing script and executable""" return_status = True + # Fetch TLE files from central real-time repo and place them under the AAPP orbelems structure: + if 'recent_tlefiles_ext_dir' in config['aapp_processes'][config.process_name]: + extdir = config['aapp_processes'][config.process_name]['recent_tlefiles_ext_dir'] + LOG.debug("Fetch TLEs from %s to %s", extdir, DIR_DATA_TLE) + tle_file_format = config['aapp_processes'][config.process_name]['tle_infile_format'] + fetch_realtime_tles(extdir, DIR_DATA_TLE, tle_file_format) + # This function relies on beeing in a working directory try: current_dir = os.getcwd() # Store the dir to change back to after function complete @@ -203,21 +241,21 @@ def do_tleing(config, timestamp, satellite): TLE_INDEX = os.path.join(DIR_DATA_TLE, "tle_{}.index".format(satellite)) (tle_dict, tle_file_list, tle_search_dir) = _search_tle_files(config, DIR_DATA_TLE, TLE_INDEX, - timestamp) + timestamp) if not tle_file_list and config['aapp_processes'][config.process_name]['download_tle_files']: LOG.warning("Found no tle files. Try to download ... ") tle_file_list = download_tle(config, timestamp, DIR_DATA_TLE) _ingest_and_archive_tle_files(config, tle_file_list, DIR_DATA_TLE, tle_dict, - tle_search_dir, satellite, TLE_INDEX) - + tle_search_dir, satellite, TLE_INDEX) # Change back after this is done os.chdir(current_dir) return return_status + def _maybe_update_env(config): """Potentially update environment based on config. @@ -301,9 +339,9 @@ def _search_tle_files(config, tle_dir, tle_index, timestamp): " otherwise it is a bit suspicious.") try: tle_files = [s for s in os.listdir(tle_dir) if - os.path.isfile(os.path.join(tle_dir, s))] + os.path.isfile(os.path.join(tle_dir, s))] tle_files.sort(key=lambda s: - os.path.getctime(os.path.join(tle_dir, s))) + os.path.getctime(os.path.join(tle_dir, s))) tle_file_list = tle_files except OSError: LOG.warning("Found no tle files .... ") @@ -329,7 +367,7 @@ def _search_tle_files(config, tle_dir, tle_index, timestamp): # FIXME: In AAPP default get_tle script direcory timestamp is TLE_MONTH=`date +%Y-\%m` for tle_search_dir in [compose(os.path.join(tle_dir, - "{timestamp:%Y_%m}"), tle_dict), tle_dir]: + "{timestamp:%Y_%m}"), tle_dict), tle_dir]: if not os.path.exists(tle_search_dir): LOG.debug("tle_search_dir {} does not exists.".format(tle_search_dir)) continue @@ -383,7 +421,7 @@ def _search_tle_files(config, tle_dir, tle_index, timestamp): def _ingest_and_archive_tle_files(config, tle_file_list, tle_dir, tle_dict, - tle_search_dir, satellite, tle_index): + tle_search_dir, satellite, tle_index): for tle_file in tle_file_list: archive = False @@ -402,7 +440,7 @@ def _ingest_and_archive_tle_files(config, tle_file_list, tle_dir, tle_dict, stderr = "" cmd = "tleing.exe" stdin = "{}\n{}\n{}\n{}\n".format(tle_dir, tle_filename, satellite, - tle_index) + tle_index) LOG.debug('stdin arguments to command: ' + str(stdin)) try: status, returncode, stdout, stderr = run_shell_command(cmd, stdin=stdin) @@ -531,7 +569,6 @@ def _archive_tles(config, tle_file_list): break - def do_tle_satpos(config, timestamp, satellite): return_status = True From b5aa6070d746d3a4d558f004603cf85ff170df09 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 17 Feb 2021 21:34:17 +0100 Subject: [PATCH 2/5] Bugfixes Signed-off-by: Adam.Dybbroe --- aapp_runner/tle_satpos_prepare.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/aapp_runner/tle_satpos_prepare.py b/aapp_runner/tle_satpos_prepare.py index 10be18a..44584d2 100644 --- a/aapp_runner/tle_satpos_prepare.py +++ b/aapp_runner/tle_satpos_prepare.py @@ -191,7 +191,7 @@ def fetch_realtime_tles(tle_input_path, tle_output_path, tle_infile_format): for filepath in infiles: filename = os.path.basename(filepath) res = p__.parse(filename) - dobj = res['timestamp'] + dtobj = res['timestamp'] subdirname = dtobj.strftime('%Y_%m') subdirpath = os.path.join(tle_output_path, subdirname) @@ -215,13 +215,6 @@ def do_tleing(config, timestamp, satellite): return_status = True - # Fetch TLE files from central real-time repo and place them under the AAPP orbelems structure: - if 'recent_tlefiles_ext_dir' in config['aapp_processes'][config.process_name]: - extdir = config['aapp_processes'][config.process_name]['recent_tlefiles_ext_dir'] - LOG.debug("Fetch TLEs from %s to %s", extdir, DIR_DATA_TLE) - tle_file_format = config['aapp_processes'][config.process_name]['tle_infile_format'] - fetch_realtime_tles(extdir, DIR_DATA_TLE, tle_file_format) - # This function relies on beeing in a working directory try: current_dir = os.getcwd() # Store the dir to change back to after function complete @@ -238,6 +231,13 @@ def do_tleing(config, timestamp, satellite): _ensure_tledir(DIR_DATA_TLE) + # Fetch TLE files from central real-time repo and place them under the AAPP orbelems structure: + if 'recent_tlefiles_ext_dir' in config['aapp_processes'][config.process_name]: + extdir = config['aapp_processes'][config.process_name]['recent_tlefiles_ext_dir'] + LOG.debug("Fetch TLEs from %s to %s", extdir, DIR_DATA_TLE) + tle_file_format = config['aapp_processes'][config.process_name]['tle_infile_format'] + fetch_realtime_tles(extdir, DIR_DATA_TLE, tle_file_format) + TLE_INDEX = os.path.join(DIR_DATA_TLE, "tle_{}.index".format(satellite)) (tle_dict, tle_file_list, tle_search_dir) = _search_tle_files(config, DIR_DATA_TLE, TLE_INDEX, From 51fa0c13476b52a27a6f766987862216e07e4b01 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 1 Apr 2021 17:21:55 +0200 Subject: [PATCH 3/5] Add header to file Signed-off-by: Adam.Dybbroe --- aapp_runner/tests/test_tle_satpos_prepare.py | 33 ++++++++++++++++++-- aapp_runner/tle_satpos_prepare.py | 2 +- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/aapp_runner/tests/test_tle_satpos_prepare.py b/aapp_runner/tests/test_tle_satpos_prepare.py index fd69091..02cafb5 100644 --- a/aapp_runner/tests/test_tle_satpos_prepare.py +++ b/aapp_runner/tests/test_tle_satpos_prepare.py @@ -1,3 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2015 - 2021 Pytroll developers + +# Author(s): + +# Adam Dybbroe +# Trygve Aspenes +# Gerrit Holl +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Testing the preparation of TLE and Satpos files. +""" + import inspect import pathlib import unittest.mock @@ -23,7 +50,7 @@ def get_config(pth): 'download_tle_files': False, 'tle_file_to_data_diff_limit_days': 3000, 'working_dir': str(pth)}}, - }, "test") + }, "test") def mk_tle_files(pth): @@ -72,8 +99,8 @@ def fake_run_tleing(cmd, stdin="", stdout_logfile=None): with caplog.at_level(logging.ERROR): atr.side_effect = fake_run_tleing aapp_runner.tle_satpos_prepare.do_tleing( - config, datetime.datetime(2021, 1, 19, 14, 8, 26), - "noaa19") + config, datetime.datetime(2021, 1, 19, 14, 8, 26), + "noaa19") assert caplog.text == "" exp_d = (p / "archive" / "tle-20210119") assert exp_d.exists() diff --git a/aapp_runner/tle_satpos_prepare.py b/aapp_runner/tle_satpos_prepare.py index 44584d2..14e325d 100644 --- a/aapp_runner/tle_satpos_prepare.py +++ b/aapp_runner/tle_satpos_prepare.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015, 2018, 2021 Pytroll developers +# Copyright (c) 2015 - 2021 Pytroll developers # Author(s): From aac6cb0ed4a8c21a1ead93447fbca40074b52585 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 7 Jan 2022 19:08:05 +0100 Subject: [PATCH 4/5] Add unittesting of the real-time tle fetch and file copy Signed-off-by: Adam.Dybbroe --- aapp_runner/tests/test_tle_satpos_prepare.py | 40 ++++++++++++++++++-- aapp_runner/tle_satpos_prepare.py | 20 +++++----- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/aapp_runner/tests/test_tle_satpos_prepare.py b/aapp_runner/tests/test_tle_satpos_prepare.py index 02cafb5..24837df 100644 --- a/aapp_runner/tests/test_tle_satpos_prepare.py +++ b/aapp_runner/tests/test_tle_satpos_prepare.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015 - 2021 Pytroll developers +# Copyright (c) 2015 - 2022 Pytroll developers # Author(s): @@ -25,13 +25,16 @@ """Testing the preparation of TLE and Satpos files. """ +import datetime import inspect +import logging +import os import pathlib +import sys import unittest.mock -import logging +from glob import glob -import datetime -import sys +from aapp_runner.tle_satpos_prepare import fetch_realtime_tles def get_config(pth): @@ -108,3 +111,32 @@ def fake_run_tleing(cmd, stdin="", stdout_logfile=None): # confirm no other directories created assert len(list(exp_d.parent.iterdir())) == 1 assert [f.name for f in exp_d.iterdir()] == ["weather202101190616.tle"] + + +def test_fetch_realtime_tles(tmp_path): + """Test fetching TLE files and copy them in under the data dir structure as expected by AAPP.""" + mypath = tmp_path / "input" + mk_tle_files(mypath) + + outpath = tmp_path / "output" + outpath.mkdir() + exp_subdir = os.path.join(outpath, "2021_01") + + tle_infile_format = 'weather{timestamp:%Y%m%d%H%M}.tle' + + fetch_realtime_tles(mypath, outpath, tle_infile_format) + + assert os.path.exists(exp_subdir) + assert os.path.isdir(exp_subdir) + # confirm no other directories created + assert len(list(outpath.iterdir())) == 1 + + result_files = [os.path.basename(f) for f in glob(os.path.join(exp_subdir, 'weather*'))] + expected_file_names = ["weather202101180325.tle", + "weather202101180008.tle", + "weather202101190616.tle"] + + for item in result_files: + assert item in expected_file_names + + assert len(result_files) == len(expected_file_names) diff --git a/aapp_runner/tle_satpos_prepare.py b/aapp_runner/tle_satpos_prepare.py index 14e325d..7578133 100644 --- a/aapp_runner/tle_satpos_prepare.py +++ b/aapp_runner/tle_satpos_prepare.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015 - 2021 Pytroll developers +# Copyright (c) 2015 - 2022 Pytroll developers # Author(s): @@ -33,16 +33,18 @@ """ import logging -from glob import glob import os +import re +import shutil +import tempfile +import time from datetime import datetime +from glob import glob from shutil import copy + +from trollsift.parser import Parser, compose, globify + from aapp_runner.helper_functions import run_shell_command -from trollsift.parser import compose, globify, Parser -import re -import time -import tempfile -import shutil LOG = logging.getLogger(__name__) @@ -183,9 +185,7 @@ def download_tle(config, timestamp, dir_data_tle): def fetch_realtime_tles(tle_input_path, tle_output_path, tle_infile_format): - """Get the recent TLEs and copy them into the AAPP data structure""" - # tle_infile_format - + """Get the recent TLEs and copy them into the AAPP data structure.""" infiles = glob(os.path.join(tle_input_path, globify(tle_infile_format))) p__ = Parser(tle_infile_format) for filepath in infiles: From 8632975979d640a99a5abb225ff2824f94777c20 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 7 Jan 2022 22:23:20 +0100 Subject: [PATCH 5/5] Add explanation of the tle_indir and recent_tlefiles_ext_dir Signed-off-by: Adam.Dybbroe --- examples/aapp-processing.yaml-template | 34 ++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/examples/aapp-processing.yaml-template b/examples/aapp-processing.yaml-template index 2147e7e..12324ac 100644 --- a/examples/aapp-processing.yaml-template +++ b/examples/aapp-processing.yaml-template @@ -8,7 +8,7 @@ logging: # Static configuration of various elements needed in the AAPP processing aapp_static_configuration: - # How to name the output files from + # How to name the output files from # the decommutation processing decommutation_files: hirs_file: hrsn.l1b @@ -17,7 +17,7 @@ aapp_static_configuration: mhs_file: ambn.l1b avhrr_file: hrpt.l1b msu_file: msun.l1b - + # Valid NOAA(POES)satellite names to process supported_noaa_satellites: - NOAA-15 @@ -28,7 +28,7 @@ aapp_static_configuration: supported_metop_satellites: - Metop-B - Metop-A - + # Unfortunately satellite names comes in a few combinations # This is how aapp names the satellite internally # and all other names needs to be translated. @@ -50,7 +50,7 @@ aapp_static_configuration: M01: metop01 M02: metop02 M03: metop03 - + # Aliases of satellite sensor names satellite_sensor_name_aliases: 'amsua': 'amsu-a' @@ -71,7 +71,7 @@ aapp_static_configuration: 'Metop-C': 'METOP-C' # Satellite sensors are named differently. - # Here is how to translate various names depending + # Here is how to translate various names depending # on the processing sensor_name_converter: 'amsua': 'amsu-a' @@ -94,8 +94,12 @@ aapp_processes: - /XLBANDANTENNA/HRPT/L0 - /XLBANDANTENNA/METOP/L0 - # Base dir of your TLE files. - tle_indir: /disk2/AAPP/orbelems + # Base dir of the TLE files for AAPP. Will override the (AAPP) environment variable DIR_DATA_TLE + tle_indir: /base/dir/under/which/aapp/expects/the/tle/files + # Directory where you keep your recent (real-time) TLE files. Independent from where AAPP expects them to be. + # If not specified, the TLE files are expected to be (already) landing in tle_indir + recent_tlefiles_ext_dir: /path/to/where/your/tle/files/are/located + # Sift format of your TLE file archive tle_archive_dir: '{tle_indir:s}/tle-archive/{timestamp:%Y%m}' # Sift format of your TLE files @@ -105,11 +109,11 @@ aapp_processes: tle_download: - {url: 'http://www.celestrak.com/NORAD/elements/weather.txt'} - {url: 'http://oiswww.eumetsat.org/metopTLEs/html/data_out/latest_m01_tle.txt'} - + # Search for the closest tle file from data timestamp # but maximum difference can not be larger than this value tle_file_to_data_diff_limit_days: 3 - + # Minutes to lock for similar passes in minutes locktime_before_rerun: 10 @@ -162,7 +166,7 @@ aapp_processes: # Do avhrr to hirs processing. Means running MAIA. And MAIA need NWP fields to work properly. # Please look in the AAPP documentation before you use this. do_avh2hirs: True - + # Skip certain Platforms and/or instruments. instrument_skipped_in_processing: - NOAA-15: @@ -171,9 +175,9 @@ aapp_processes: - amsu-b - METOP-C: - hirs/4 - + # Sift format of the resulting file names of the AAPP processing - rename_aapp_compose: '{data_type:s}_{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:5d}.{data_level:s}' + rename_aapp_compose: '{data_type:s}_{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:5d}.{data_level:s}' # Data using the rename_aapp_compose. This is how the aapp runner does it. rename_aapp_files: - avhrr: {aapp_file: hrpt.l1b, data_type: hrpt, data_level: l1b} @@ -183,14 +187,14 @@ aapp_processes: - amsua: {aapp_file: aman.l1b, data_type: amsual1b, data_level: l1b} - amsua: {aapp_file: aman.l1c, data_type: amsual1c, data_level: l1c} - amsua: {aapp_file: aman.l1d, data_type: amsual1d, data_level: l1d} - - amsub: {aapp_file: ambn.l1b, data_type: amsubl1b, data_level: l1b} - - amsub: {aapp_file: ambn.l1c, data_type: amsubl1c, data_level: l1c} + - amsub: {aapp_file: ambn.l1b, data_type: amsubl1b, data_level: l1b} + - amsub: {aapp_file: ambn.l1c, data_type: amsubl1c, data_level: l1c} - mhs: {aapp_file: ambn.l1b, data_type: mhsl1b, data_level: l1b} - mhs: {aapp_file: ambn.l1c, data_type: mhsl1c, data_level: l1c} - msu: {aapp_file: msu.l1b, data_type: msul1b, data_level: l1b} - msu: {aapp_file: msu.l1c, data_type: msul1c, data_level: l1c} - dcs: {aapp_file: dcs.l1b, data_type: dcsl1b, data_level: l1b} - + # The idea is to send posttroll messages during the processing for various status # information. # NOT Implemented.