Clean up a bit and prepare template yaml config for control of suppor…

…ted sats

Signed-off-by: Adam.Dybbroe <a000680@c21856.ad.smhi.se>

bin/viirs_af_runner.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2019, 2020, 2021 Pytroll
+# Copyright (c) 2019, 2020, 2021, 2023 Pytroll
 
 # Author(s):
 
@@ -20,8 +20,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Level-2 CSPP VIIRS Active Fire runner. From VIIRS SDRs it generates Active
-Fire outputs on I- and/or M-bands.
+"""Level-2 CSPP VIIRS Active Fire runner.
+
+From VIIRS SDRs it generates Active Fire outputs on I- and/or M-bands.
 
 """
 
@@ -39,20 +40,10 @@
 from viirs_active_fires.utils import (deliver_output_files, cleanup_cspp_workdir,
                                       get_edr_times, get_active_fire_result_files)
 
-# from posttroll.adress_receiver import get_local_ips
-# NOT used at the moment...
-
-if six.PY2:
-    from urlparse import urlparse
-    from urlparse import urlunsplit
-elif six.PY3:
-    from urllib.parse import urlparse
-    from urllib.parse import urlunsplit
+from urllib.parse import urlparse
+from urllib.parse import urlunsplit
 
-if six.PY2:
-    ptimer = time.clock
-elif six.PY3:
-    ptimer = time.perf_counter
+ptimer = time.perf_counter
 
 #: Default time format
 _DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
@@ -65,15 +56,17 @@
 CSPP_AF_HOME = os.environ.get("CSPP_ACTIVE_FIRE_HOME", '')
 CSPP_AF_WORKDIR = os.environ.get("CSPP_ACTIVE_FIRE_WORKDIR", '')
 
+VIIRS_SATELLITES = ['Suomi-NPP', 'NOAA-20']
+
 
 class ViirsActiveFiresProcessor(object):
 
     """
-    Container for the VIIRS Active Fires processing based on CSPP
+    Container for the VIIRS Active Fires processing based on CSPP.
 
     """
 
-    def __init__(self, ncpus):
+    def __init__(self, ncpus, options):
         from multiprocessing.pool import ThreadPool
         self.pool = ThreadPool(ncpus)
         self.ncpus = ncpus
@@ -84,10 +77,11 @@ def __init__(self, ncpus):
         self.pass_start_time = None
         self.result_files = []
         self.sdr_files = []
-        self.result_home = OPTIONS.get('output_dir', '/tmp')
-        self.publish_topic = OPTIONS.get('publish_topic')
-        self.site = OPTIONS.get('site', 'unknown')
-        self.environment = OPTIONS.get('environment')
+        self.result_home = options.get('output_dir', '/tmp')
+        self.publish_topic = options.get('publish_topic')
+        self.site = options.get('site', 'unknown')
+        self.environment = options.get('environment')
+        self.viirs_satellites = options.get('platform_names', VIIRS_SATELLITES)
         self.message_data = None
         self.service = None
 
@@ -106,16 +100,17 @@ def deliver_output_files(self, subd=None):
         return deliver_output_files(self.result_files, self.result_home, subd)
 
     def run(self, msg):
-        """Start the VIIRS Active Fires processing using CSPP on one sdr granule"""
-
+        """Start the VIIRS Active Fires processing using CSPP on one sdr granule."""
         if msg:
             LOG.debug("Received message: " + str(msg))
-        elif msg and ('platform_name' not in msg.data or 'sensor' not in msg.data):
+
+        if msg and ('platform_name' not in msg.data or 'sensor' not in msg.data):
             LOG.debug("No platform_name or sensor in message. Continue...")
             return True
-        elif msg and not (msg.data['platform_name'] in VIIRS_SATELLITES and
+        elif msg and not (msg.data['platform_name'] in self.viirs_satellites and
                           msg.data['sensor'] == 'viirs'):
-            LOG.info("Not a VIIRS scene. Continue...")
+            LOG.info("Not a supported VIIRS scene. Satellite = %s - Continue...",
+                     str(msg.data['platform_name']))
             return True
 
         self.platform_name = str(msg.data['platform_name'])
@@ -127,20 +122,9 @@ def run(self, msg):
             return True
 
         sdr_dataset = msg.data['dataset']
-
         if len(sdr_dataset) < 1:
             return True
 
-        # sdr = sdr_dataset[0]
-        # urlobj = urlparse(sdr['uri'])
-        # LOG.debug("Server = " + str(urlobj.netloc))
-        # url_ip = socket.gethostbyname(urlobj.netloc)
-        # if url_ip not in get_local_ips():
-        #     LOG.warning(
-        #         "Server %s not the current one: %s" % (str(urlobj.netloc),
-        #                                                socket.gethostname()))
-        #     return True
-
         sdr_files = []
         for sdr in sdr_dataset:
             urlobj = urlparse(sdr['uri'])
@@ -184,7 +168,7 @@ def publish_af(publisher, result_files, mda, **kwargs):
     to_send = mda.copy()
     # Delete the SDR dataset from the message:
     try:
-        del(to_send['dataset'])
+        del (to_send['dataset'])
     except KeyError:
         LOG.warning("Couldn't remove dataset from message")
 
@@ -240,16 +224,15 @@ def viirs_active_fire_runner(options, service_name):
 
     ncpus_available = cpu_count()
     LOG.info("Number of CPUs available = " + str(ncpus_available))
-    ncpus = int(OPTIONS.get('ncpus', 1))
+    ncpus = int(options.get('ncpus', 1))
     LOG.info("Will use %d CPUs when running the CSPP VIIRS Active Fires instances", ncpus)
-    viirs_af_proc = ViirsActiveFiresProcessor(ncpus)
+    viirs_af_proc = ViirsActiveFiresProcessor(ncpus, options)
 
     with posttroll.subscriber.Subscribe('', options['message_types'], True) as subscr:
         with Publish('viirs_active_fire_runner', 0) as publisher:
 
             while True:
                 viirs_af_proc.initialise(service_name)
-                # for msg in subscr.recv(timeout=300):
                 for msg in subscr.recv():
                     status = viirs_af_proc.run(msg)
                     if not status:
@@ -272,7 +255,7 @@ def viirs_active_fire_runner(options, service_name):
                                environment=viirs_af_proc.environment,
                                site=viirs_af_proc.site)
 
-                LOG.info("SDR processing has completed.")
+                LOG.info("Active Fires EDR processing has completed.")
 
     return
 

etc/viirs_af_config.yaml.template

@@ -5,6 +5,10 @@ tle_dir: /data/24/saf/polar_in/tle
 tlefilename: tle-{time:%Y%m%d%H%M}.txt
 site: nrk
 
+platform_names:
+  - NOAA-20
+  - Suomi-NPP
+
 viirs-ibands:
 
   areas_of_interest: [euron1]
@@ -45,4 +49,3 @@ viirs-mbands:
 
   utv:
     output_dir: /data/proj/safutv/polar_out/viirs_active_fires
-

viirs_active_fires/tests/test_utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2021 Adam.Dybbroe
+# Copyright (c) 2021, 2023 Adam.Dybbroe
 
 # Author(s):
 
@@ -20,8 +20,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Unit testing the utility functions.
-"""
+"""Unit testing the utility functions."""
 
 import os.path
 import pytest