Merge pull request #20 from adybbroe/refactor-tle-handling

Refactor tle handling

aapp_runner/tests/test_tle_satpos_prepare.py

@@ -1,10 +1,40 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2015 - 2022 Pytroll developers
+
+# Author(s):
+
+#   Adam Dybbroe <Firstname.Lastname at smhi.se>
+#   Trygve Aspenes <trygveas@met.no>
+#   Gerrit Holl
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Testing the preparation of TLE and Satpos files.
+"""
+
+import datetime
 import inspect
+import logging
+import os
 import pathlib
+import sys
 import unittest.mock
-import logging
+from glob import glob
 
-import datetime
-import sys
+from aapp_runner.tle_satpos_prepare import fetch_realtime_tles
 
 
 def get_config(pth):
@@ -23,7 +53,7 @@ def get_config(pth):
                 'download_tle_files': False,
                 'tle_file_to_data_diff_limit_days': 3000,
                 'working_dir': str(pth)}},
-            }, "test")
+    }, "test")
 
 
 def mk_tle_files(pth):
@@ -72,12 +102,41 @@ def fake_run_tleing(cmd, stdin="", stdout_logfile=None):
         with caplog.at_level(logging.ERROR):
             atr.side_effect = fake_run_tleing
             aapp_runner.tle_satpos_prepare.do_tleing(
-                    config, datetime.datetime(2021, 1, 19, 14, 8, 26),
-                    "noaa19")
+                config, datetime.datetime(2021, 1, 19, 14, 8, 26),
+                "noaa19")
             assert caplog.text == ""
     exp_d = (p / "archive" / "tle-20210119")
     assert exp_d.exists()
     assert exp_d.is_dir()
     # confirm no other directories created
     assert len(list(exp_d.parent.iterdir())) == 1
     assert [f.name for f in exp_d.iterdir()] == ["weather202101190616.tle"]
+
+
+def test_fetch_realtime_tles(tmp_path):
+    """Test fetching TLE files and copy them in under the data dir structure as expected by AAPP."""
+    mypath = tmp_path / "input"
+    mk_tle_files(mypath)
+
+    outpath = tmp_path / "output"
+    outpath.mkdir()
+    exp_subdir = os.path.join(outpath, "2021_01")
+
+    tle_infile_format = 'weather{timestamp:%Y%m%d%H%M}.tle'
+
+    fetch_realtime_tles(mypath, outpath, tle_infile_format)
+
+    assert os.path.exists(exp_subdir)
+    assert os.path.isdir(exp_subdir)
+    # confirm no other directories created
+    assert len(list(outpath.iterdir())) == 1
+
+    result_files = [os.path.basename(f) for f in glob(os.path.join(exp_subdir, 'weather*'))]
+    expected_file_names = ["weather202101180325.tle",
+                           "weather202101180008.tle",
+                           "weather202101190616.tle"]
+
+    for item in result_files:
+        assert item in expected_file_names
+
+    assert len(result_files) == len(expected_file_names)

aapp_runner/tle_satpos_prepare.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2015, 2018 Pytroll developers
+# Copyright (c) 2015 - 2022 Pytroll developers
 
 # Author(s):
 
@@ -33,14 +33,18 @@
 """
 
 import logging
-from glob import glob
 import os
+import re
+import shutil
+import tempfile
+import time
 from datetime import datetime
+from glob import glob
 from shutil import copy
+
+from trollsift.parser import Parser, compose, globify
+
 from aapp_runner.helper_functions import run_shell_command
-from trollsift.parser import compose
-import re
-import time
 
 LOG = logging.getLogger(__name__)
 
@@ -64,12 +68,14 @@ def _do_3_matches(m):
 def _do_3_matchesYY(m):
     return datetime.strptime(m.group(1) + m.group(2) + m.group(3), "%y%m%d")
 
+
 tle_match_tests = ((r'.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2})(\d{2}).*', _do_6_matches),
                    (r'.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2}).*', _do_5_matches),
                    (r'.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2}).*', _do_4_matches),
                    (r'.*(\d{4})(\d{2})(\d{2}).*', _do_3_matches),
                    (r'.*(\d{2})(\d{2})(\d{2}).*', _do_3_matchesYY))
 
+
 def download_tle(config, timestamp, dir_data_tle):
 
     user = os.getenv("PAR_NAVIGATION_TLE_USER", "xxxxxx")
@@ -178,6 +184,31 @@ def download_tle(config, timestamp, dir_data_tle):
     return tle_file_list
 
 
+def fetch_realtime_tles(tle_input_path, tle_output_path, tle_infile_format):
+    """Get the recent TLEs and copy them into the AAPP data structure."""
+    infiles = glob(os.path.join(tle_input_path, globify(tle_infile_format)))
+    p__ = Parser(tle_infile_format)
+    for filepath in infiles:
+        filename = os.path.basename(filepath)
+        res = p__.parse(filename)
+        dtobj = res['timestamp']
+
+        subdirname = dtobj.strftime('%Y_%m')
+        subdirpath = os.path.join(tle_output_path, subdirname)
+        outfile = os.path.join(subdirpath, filename)
+        LOG.debug("OUTPUT file = %s", str(outfile))
+
+        if not os.path.exists(subdirpath):
+            os.mkdir(subdirpath)
+        tmp_filepath = tempfile.mktemp(suffix='_' + os.path.basename(outfile),
+                                       dir=os.path.dirname(outfile))
+        LOG.debug("tmp-filepath = %s", tmp_filepath)
+        shutil.copy(filepath, tmp_filepath)
+        LOG.debug("File copied: %s -> %s", filepath, tmp_filepath)
+        os.rename(tmp_filepath, outfile)
+        LOG.debug("Rename: %s -> %s", tmp_filepath, outfile)
+
+
 def do_tleing(config, timestamp, satellite):
     """Get the tle-file and copy them to the AAPP data structure
        and run the AAPP tleing script and executable"""
@@ -200,24 +231,31 @@ def do_tleing(config, timestamp, satellite):
 
     _ensure_tledir(DIR_DATA_TLE)
 
+    # Fetch TLE files from central real-time repo and place them under the AAPP orbelems structure:
+    if 'recent_tlefiles_ext_dir' in config['aapp_processes'][config.process_name]:
+        extdir = config['aapp_processes'][config.process_name]['recent_tlefiles_ext_dir']
+        LOG.debug("Fetch TLEs from %s to %s", extdir, DIR_DATA_TLE)
+        tle_file_format = config['aapp_processes'][config.process_name]['tle_infile_format']
+        fetch_realtime_tles(extdir, DIR_DATA_TLE, tle_file_format)
+
     TLE_INDEX = os.path.join(DIR_DATA_TLE, "tle_{}.index".format(satellite))
 
     (tle_dict, tle_file_list, tle_search_dir) = _search_tle_files(config, DIR_DATA_TLE, TLE_INDEX,
-            timestamp)
+                                                                  timestamp)
 
     if not tle_file_list and config['aapp_processes'][config.process_name]['download_tle_files']:
         LOG.warning("Found no tle files. Try to download ... ")
         tle_file_list = download_tle(config, timestamp, DIR_DATA_TLE)
 
     _ingest_and_archive_tle_files(config, tle_file_list, DIR_DATA_TLE, tle_dict,
-            tle_search_dir, satellite, TLE_INDEX)
-
+                                  tle_search_dir, satellite, TLE_INDEX)
 
     # Change back after this is done
     os.chdir(current_dir)
 
     return return_status
 
+
 def _maybe_update_env(config):
     """Potentially update environment based on config.
 
@@ -301,9 +339,9 @@ def _search_tle_files(config, tle_dir, tle_index, timestamp):
                         " otherwise it is a bit suspicious.")
             try:
                 tle_files = [s for s in os.listdir(tle_dir) if
-                        os.path.isfile(os.path.join(tle_dir, s))]
+                             os.path.isfile(os.path.join(tle_dir, s))]
                 tle_files.sort(key=lambda s:
-                        os.path.getctime(os.path.join(tle_dir, s)))
+                               os.path.getctime(os.path.join(tle_dir, s)))
                 tle_file_list = tle_files
             except OSError:
                 LOG.warning("Found no tle files .... ")
@@ -329,7 +367,7 @@ def _search_tle_files(config, tle_dir, tle_index, timestamp):
 
         # FIXME: In AAPP default get_tle script direcory timestamp is TLE_MONTH=`date +%Y-\%m`
         for tle_search_dir in [compose(os.path.join(tle_dir,
-            "{timestamp:%Y_%m}"), tle_dict), tle_dir]:
+                                                    "{timestamp:%Y_%m}"), tle_dict), tle_dir]:
             if not os.path.exists(tle_search_dir):
                 LOG.debug("tle_search_dir {} does not exists.".format(tle_search_dir))
                 continue
@@ -383,7 +421,7 @@ def _search_tle_files(config, tle_dir, tle_index, timestamp):
 
 
 def _ingest_and_archive_tle_files(config, tle_file_list, tle_dir, tle_dict,
-        tle_search_dir, satellite, tle_index):
+                                  tle_search_dir, satellite, tle_index):
     for tle_file in tle_file_list:
         archive = False
 
@@ -402,7 +440,7 @@ def _ingest_and_archive_tle_files(config, tle_file_list, tle_dir, tle_dict,
         stderr = ""
         cmd = "tleing.exe"
         stdin = "{}\n{}\n{}\n{}\n".format(tle_dir, tle_filename, satellite,
-                tle_index)
+                                          tle_index)
         LOG.debug('stdin arguments to command: ' + str(stdin))
         try:
             status, returncode, stdout, stderr = run_shell_command(cmd, stdin=stdin)
@@ -531,7 +569,6 @@ def _archive_tles(config, tle_file_list):
                     break
 
 
-
 def do_tle_satpos(config, timestamp, satellite):
 
     return_status = True

examples/aapp-processing.yaml-template

@@ -8,7 +8,7 @@ logging:
 
 # Static configuration of various elements needed in the AAPP processing
 aapp_static_configuration:
-  # How to name the output files from 
+  # How to name the output files from
   # the decommutation processing
   decommutation_files:
     hirs_file: hrsn.l1b
@@ -17,7 +17,7 @@ aapp_static_configuration:
     mhs_file: ambn.l1b
     avhrr_file: hrpt.l1b
     msu_file: msun.l1b
-     
+
   # Valid NOAA(POES)satellite names to process
   supported_noaa_satellites:
     - NOAA-15
@@ -28,7 +28,7 @@ aapp_static_configuration:
   supported_metop_satellites:
     - Metop-B
     - Metop-A
-    
+
   # Unfortunately satellite names comes in a few combinations
   # This is how aapp names the satellite internally
   # and all other names needs to be translated.
@@ -50,7 +50,7 @@ aapp_static_configuration:
     M01: metop01
     M02: metop02
     M03: metop03
-    
+
   # Aliases of satellite sensor names
   satellite_sensor_name_aliases:
     'amsua': 'amsu-a'
@@ -71,7 +71,7 @@ aapp_static_configuration:
     'Metop-C': 'METOP-C'
 
   # Satellite sensors are named differently.
-  # Here is how to translate various names depending 
+  # Here is how to translate various names depending
   # on the processing
   sensor_name_converter:
     'amsua': 'amsu-a'
@@ -94,11 +94,15 @@ aapp_processes:
       - /XLBANDANTENNA/HRPT/L0
       - /XLBANDANTENNA/METOP/L0
 
+    # Base dir of the TLE files for AAPP. Will override the (AAPP) environment variable DIR_DATA_TLE
+    tle_indir: /base/dir/under/which/aapp/expects/the/tle/files
+    # Directory where you keep your recent (real-time) TLE files. Independent from where AAPP expects them to be.
+    # If not specified, the TLE files are expected to be (already) landing in tle_indir
+    recent_tlefiles_ext_dir: /path/to/where/your/tle/files/are/located
+
     # Default collection area id:
     collection_area_id: euron1
 
-    # Base dir of your TLE files.
-    tle_indir: /disk2/AAPP/orbelems
     # Sift format of your TLE file archive
     tle_archive_dir: '{tle_indir:s}/tle-archive/{timestamp:%Y%m}'
     # Sift format of your TLE files
@@ -108,11 +112,11 @@ aapp_processes:
     tle_download:
       - {url: 'http://www.celestrak.com/NORAD/elements/weather.txt'}
       - {url: 'http://oiswww.eumetsat.org/metopTLEs/html/data_out/latest_m01_tle.txt'}
-    
+
     # Search for the closest tle file from data timestamp
     # but maximum difference can not be larger than this value
     tle_file_to_data_diff_limit_days: 3
-    
+
     # Minutes to lock for similar passes in minutes
     locktime_before_rerun: 10
 
@@ -165,7 +169,7 @@ aapp_processes:
     # Do avhrr to hirs processing. Means running MAIA. And MAIA need NWP fields to work properly.
     # Please look in the AAPP documentation before you use this.
     do_avh2hirs: True
-    
+
     # Skip certain Platforms and/or instruments.
     instrument_skipped_in_processing:
       - NOAA-15:
@@ -174,9 +178,9 @@ aapp_processes:
         - amsu-b
       - METOP-C:
         - hirs/4
- 
+
     # Sift format of the resulting file names of the AAPP processing
-    rename_aapp_compose: '{data_type:s}_{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:5d}.{data_level:s}' 
+    rename_aapp_compose: '{data_type:s}_{satellite_name:s}_{start_time:%Y%m%d}_{start_time:%H%M}_{orbit_number:5d}.{data_level:s}'
     # Data using the rename_aapp_compose. This is how the aapp runner does it.
     rename_aapp_files:
       - avhrr: {aapp_file: hrpt.l1b, data_type: hrpt, data_level: l1b}
@@ -186,14 +190,14 @@ aapp_processes:
       - amsua: {aapp_file: aman.l1b, data_type: amsual1b, data_level: l1b}
       - amsua: {aapp_file: aman.l1c, data_type: amsual1c, data_level: l1c}
       - amsua: {aapp_file: aman.l1d, data_type: amsual1d, data_level: l1d}
-      - amsub: {aapp_file: ambn.l1b, data_type: amsubl1b, data_level: l1b}        
-      - amsub: {aapp_file: ambn.l1c, data_type: amsubl1c, data_level: l1c}        
+      - amsub: {aapp_file: ambn.l1b, data_type: amsubl1b, data_level: l1b}
+      - amsub: {aapp_file: ambn.l1c, data_type: amsubl1c, data_level: l1c}
       - mhs: {aapp_file: ambn.l1b, data_type: mhsl1b, data_level: l1b}
       - mhs: {aapp_file: ambn.l1c, data_type: mhsl1c, data_level: l1c}
       - msu: {aapp_file: msu.l1b, data_type: msul1b, data_level: l1b}
       - msu: {aapp_file: msu.l1c, data_type: msul1c, data_level: l1c}
       - dcs: {aapp_file: dcs.l1b, data_type: dcsl1b, data_level: l1b}
-      
+
     # The idea is to send posttroll messages during the processing for various status
     # information.
     # NOT Implemented.