From a44b3762a95e48c640fa8f8d578a635aee0303d3 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 4 May 2023 14:11:53 +0200 Subject: [PATCH 1/3] Create publish topic in scisys-receiver from config-pattern and message Signed-off-by: Adam.Dybbroe --- bin/scisys_receiver.py | 29 +++++--- examples/scisys_receiver.yaml_template | 13 ++++ pytroll_collectors/config.py | 34 +++++++++ pytroll_collectors/scisys.py | 41 +++++------ pytroll_collectors/tests/conftest.py | 53 ++++++++++++++ pytroll_collectors/tests/test_config.py | 40 +++++++++++ pytroll_collectors/tests/test_scisys.py | 92 +++++++------------------ 7 files changed, 202 insertions(+), 100 deletions(-) create mode 100644 examples/scisys_receiver.yaml_template create mode 100644 pytroll_collectors/config.py create mode 100644 pytroll_collectors/tests/conftest.py create mode 100644 pytroll_collectors/tests/test_config.py diff --git a/bin/scisys_receiver.py b/bin/scisys_receiver.py index 99340a15..697ff56a 100755 --- a/bin/scisys_receiver.py +++ b/bin/scisys_receiver.py @@ -1,13 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2012 - 2021 Pytroll developers +# Copyright (c) 2012 - 2023 Pytroll developers # # Author(s): # # Martin Raspaud # Janne Kotro # Panu Lahtinen +# Adam Dybbroe # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -54,6 +55,8 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("host", help="GMC host") parser.add_argument("port", help="Port to listen to", type=int) + parser.add_argument("-c", "--config", + help="YAML config file to use.") parser.add_argument("-P", "--publish-port", type=int, default=0, dest="publish_port", help="Publish port") parser.add_argument("-n", "--nameserver", nargs='+', default=[], @@ -82,11 +85,11 @@ def parse_args(): help="IP of the target server." "In case of multiple dispatches in GMC." "Defaults to the local host.") - parser.add_argument("-T", "--topic_postfix", - dest="topic_postfix", - type=str, - help="Publish topic postfix. " - "Prefix will be /format/data_processing_level/") + # parser.add_argument("-T", "--topic_postfix", + # dest="topic_postfix", + # type=str, + # help="Publish topic postfix. " + # "Prefix will be /format/data_processing_level/") return parser.parse_args() @@ -115,15 +118,21 @@ def main(): """Run scisys receiver.""" opts = parse_args() - no_sats = opts.excluded_satellites + configfile = opts.config + + # no_sats = opts.excluded_satellites setup_logging(log_file=opts.log) try: - receive_from_zmq(opts.host, opts.port, - opts.station, opts.environment, no_sats, + # receive_from_zmq(opts.host, opts.port, + # opts.station, opts.environment, no_sats, + # opts.target_server, opts.ftp_prefix, + # opts.topic_postfix, publish_port=opts.publish_port, + # nameservers=opts.nameservers, days=1) + receive_from_zmq(configfile, opts.target_server, opts.ftp_prefix, - opts.topic_postfix, publish_port=opts.publish_port, + publish_port=opts.publish_port, nameservers=opts.nameservers, days=1) except KeyboardInterrupt: pass diff --git a/examples/scisys_receiver.yaml_template b/examples/scisys_receiver.yaml_template new file mode 100644 index 00000000..18f65947 --- /dev/null +++ b/examples/scisys_receiver.yaml_template @@ -0,0 +1,13 @@ +# Publish topic +publish_topic_pattern: "/{sensor}/{format}/{data_processing_level}/{platform_name}" + +# It is possible to here add a static postfix topic if needed: +topic_postfix: "my/cool/postfix/topic" + +host: merlin +port: 10600 +station: nrk +environment: dev + +excluded_satellites: + - fy3d diff --git a/pytroll_collectors/config.py b/pytroll_collectors/config.py new file mode 100644 index 00000000..1e8aff4c --- /dev/null +++ b/pytroll_collectors/config.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Pytroll Developers + +# Author(s): + +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Handling the yaml configurations.""" + +import yaml +from yaml import UnsafeLoader + + +def read_config(config_filepath): + """Read and extract config information.""" + with open(config_filepath, 'r') as fp_: + config = yaml.load(fp_, Loader=UnsafeLoader) + + return config diff --git a/pytroll_collectors/scisys.py b/pytroll_collectors/scisys.py index a7403884..e606a421 100755 --- a/pytroll_collectors/scisys.py +++ b/pytroll_collectors/scisys.py @@ -44,6 +44,9 @@ from time import sleep from urllib.parse import SplitResult, urlsplit, urlunsplit +from trollsift import compose + +from pytroll_collectors.config import read_config from posttroll.message import Message from posttroll.publisher import Publish from pytroll_collectors.helper_functions import is_uri_on_server @@ -493,35 +496,25 @@ def stop(self): self.loop = False -def get_subject_from_msg2send(to_send, station, env, topic_postfix): - """Get the publish topic from the message to be send.""" - if isinstance(to_send['sensor'], str): - prefix = '/' + to_send['sensor'].replace('/', '-') - else: - prefix = '' - if topic_postfix is not None: - subject = "/".join((prefix, to_send['format'], - to_send['data_processing_level'], - topic_postfix)) - else: - subject = "/".join((prefix, to_send['format'], - to_send['data_processing_level'], - station, env, - "polar", "direct_readout")) - return subject - - -def receive_from_zmq(host, port, station, environment, excluded_platforms, - target_server, ftp_prefix, topic_postfix, +def get_subject_from_message_and_config(to_send, config): + """Get the publish topic from the message and the yaml configuration settings.""" + return compose(config['publish_topic_pattern'], to_send) + + +def receive_from_zmq(config_filename, + target_server, ftp_prefix, publish_port=0, nameservers=None, days=1): """Receive 2met! messages from zeromq.""" logger.debug("target_server: %s", str(target_server)) logger.debug("ftp_prefix: %s", str(ftp_prefix)) - logger.debug("topic_postfix: %s", str(topic_postfix)) - logger.debug("station %s", str(station)) logger.debug(type(target_server)) - # socket = Subscriber(["tcp://localhost:9331"], ["2met!"]) + config = read_config(config_filename) + + excluded_platforms = config['excluded_satellites'] + host = config['host'] + port = config['port'] + sock = GMCSubscriber(host, port) msg_rec = MessageReceiver(host, excluded_platforms, target_server, ftp_prefix) @@ -539,7 +532,7 @@ def receive_from_zmq(host, port, station, environment, excluded_platforms, if to_send is None: continue - subject = get_subject_from_msg2send(to_send, station, environment, topic_postfix) + subject = get_subject_from_message_and_config(to_send, config) logger.debug("Subject: %s", str(subject)) msg = Message(subject, "file", to_send).encode() logger.debug("publishing %s", str(msg)) diff --git a/pytroll_collectors/tests/conftest.py b/pytroll_collectors/tests/conftest.py new file mode 100644 index 00000000..e70398ed --- /dev/null +++ b/pytroll_collectors/tests/conftest.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Adam.Dybbroe + +# Author(s): + +# Adam.Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Fixtures for unittests.""" + +import pytest + + +TEST_YAML_CONFIG_CONTENT_SCISYS_RECEIVER = """ +# Publish topic +publish_topic_pattern: '/{sensor}/{format}/{data_processing_level}/{platform_name}' + +# It is possible to here add a static postfix topic if needed: +topic_postfix: "my/cool/postfix/topic" + +host: merlin +port: 10600 +station: nrk +environment: dev + +excluded_satellites: + - fy3d + +""" + + +@pytest.fixture +def fake_yamlconfig_file_for_scisys_receiver(tmp_path): + """Write fake yaml config file for the SCISYS receiver.""" + file_path = tmp_path / 'test_scisys_receiver_config.yaml' + with open(file_path, 'w') as fpt: + fpt.write(TEST_YAML_CONFIG_CONTENT_SCISYS_RECEIVER) + + yield file_path diff --git a/pytroll_collectors/tests/test_config.py b/pytroll_collectors/tests/test_config.py new file mode 100644 index 00000000..031cc318 --- /dev/null +++ b/pytroll_collectors/tests/test_config.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2023 Adam.Dybbroe + +# Author(s): + +# Adam.Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test getting the yaml configurations from file.""" + +from pytroll_collectors.config import read_config + + +def test_get_yaml_configuration(fake_yamlconfig_file_for_scisys_receiver): + """Test read and get the yaml configuration for the scisys receiver from file.""" + config = read_config(fake_yamlconfig_file_for_scisys_receiver) + + assert config['publish_topic_pattern'] == '/{sensor}/{format}/{data_processing_level}/{platform_name}' + assert config['topic_postfix'] == 'my/cool/postfix/topic' + assert config['host'] == 'merlin' + assert isinstance(config['port'], int) + assert config['port'] == 10600 + assert config['station'] == 'nrk' + assert config['environment'] == 'dev' + assert len(config['excluded_satellites']) == 1 + assert config['excluded_satellites'][0] == 'fy3d' diff --git a/pytroll_collectors/tests/test_scisys.py b/pytroll_collectors/tests/test_scisys.py index 5e37abda..728a7998 100644 --- a/pytroll_collectors/tests/test_scisys.py +++ b/pytroll_collectors/tests/test_scisys.py @@ -31,7 +31,7 @@ from copy import deepcopy from pytroll_collectors.scisys import MessageReceiver, TwoMetMessage -from pytroll_collectors.scisys import get_subject_from_msg2send +from pytroll_collectors.scisys import get_subject_from_message_and_config hostname = 'localhost' @@ -257,10 +257,28 @@ def test_twomet_messages_with_stop_reception_message(platform_name, tmp_path): TestCase().assertDictEqual(to_send, CREATE_MESSAGES[platform_name](tmp_path)) -@pytest.mark.parametrize("sensor, sensor_name", [(VIIRS, 'viirs'), - (ATMS, 'atms')]) -def test_get_subject_from_msg2send_postfix_topic_is_none(sensor, sensor_name, tmp_path): - """Test the get the subject from the message being send.""" +@pytest.mark.parametrize("sensor, sensor_name, topic_pattern, topic_result", + [(VIIRS, 'viirs', + '/{sensor}/{format}/{data_processing_level}/{platform_name}', + '/viirs/RDR/0/Suomi-NPP'), + (VIIRS, 'viirs', + '/RDR/nrk/dev', + '/RDR/nrk/dev'), + (ATMS, 'atms', + '/{sensor}/{data_processing_level}/{format}/{platform_name}/{type}', + '/atms/0/RDR/Suomi-NPP/HDF5'), + (ATMS, 'atms', + '/{sensor}/{data_processing_level}/{format}/{platform_name}/{type}/nrk/dev', + '/atms/0/RDR/Suomi-NPP/HDF5/nrk/dev'), + ]) +def test_create_message_topic_from_message_and_config_pattern(sensor, sensor_name, + topic_pattern, topic_result, tmp_path): + """Test create a message topic from the config pattern and the message.""" + config = {'publish_topic_pattern': 'some-pattern-for-a-publish-topic', + 'topic_postfix': 'my/cool/postfix/topic', + 'host': 'merlin', 'port': 10600, + 'station': 'nrk', 'environment': 'dev', + 'excluded_satellites': ['fy3d']} msg_rec = MessageReceiver("nimbus") filename = tmp_path / sensor['uid'] @@ -269,65 +287,7 @@ def test_get_subject_from_msg2send_postfix_topic_is_none(sensor, sensor_name, tm string = TwoMetMessage(INPUT_DISPATCH[sensor_name](str(filename.parent))) to_send = msg_rec.receive(string) - topic_postfix = None - station = 'nrk' - environment = 'dev' - subject = get_subject_from_msg2send(to_send, station, environment, topic_postfix) - - assert subject == "/{sensor}/RDR/0/nrk/dev/polar/direct_readout".format(sensor=sensor_name) - - -@pytest.mark.parametrize("sensor, sensor_name", [(VIIRS, 'viirs'), - (ATMS, 'atms')]) -def test_get_subject_from_msg2send_with_postfix_topic(sensor, sensor_name, tmp_path): - """Test the get the subject from the message being send.""" - msg_rec = MessageReceiver("nimbus") - - filename = tmp_path / sensor['uid'] - create_empty_file(filename) - - string = TwoMetMessage(INPUT_DISPATCH[sensor_name](str(filename.parent))) - to_send = msg_rec.receive(string) - - station = 'nrk' - environment = 'dev' - topic_postfix = 'my_topic' - subject = get_subject_from_msg2send(to_send, station, environment, topic_postfix) - - assert subject == "/{sensor}/RDR/0/my_topic".format(sensor=sensor_name) - - -def test_get_subject_from_msg2send_empty_postfix_topic(tmp_path): - """Test get the subject from the message being send - empty postfix topic.""" - msg_rec = MessageReceiver("nimbus") - - filename = tmp_path / ATMS['uid'] - create_empty_file(filename) - - string = TwoMetMessage(INPUT_DISPATCH['atms'](str(filename.parent))) - to_send = msg_rec.receive(string) - - station = 'nrk' - environment = 'dev' - topic_postfix = '' - subject = get_subject_from_msg2send(to_send, station, environment, topic_postfix) - - assert subject == "/atms/RDR/0/" - - -def test_get_subject_from_msg2send_avhrr3(tmp_path): - """Test get the subject from the message being send - avhrr data.""" - msg_rec = MessageReceiver("nimbus") - - filename = tmp_path / create_msg_n19(tmp_path)['uid'] - create_empty_file(filename) - - string = TwoMetMessage(create_fildis_n19(tmp_path)) - to_send = msg_rec.receive(string) - - station = 'nrk' - environment = 'dev' - topic_postfix = '' + config.update({'publish_topic_pattern': topic_pattern}) + subject = get_subject_from_message_and_config(to_send, config) - subject = get_subject_from_msg2send(to_send, station, environment, topic_postfix) - assert subject == "/HRPT/0/" + assert subject == topic_result From 3ab365ee185fcbed73bb0223724f3dbfbd74765f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 4 May 2023 14:54:55 +0200 Subject: [PATCH 2/3] Bugfix: Remove unused command line arguments Signed-off-by: Adam.Dybbroe --- bin/scisys_receiver.py | 23 ----------------------- examples/scisys_receiver.yaml_template | 8 ++++---- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/bin/scisys_receiver.py b/bin/scisys_receiver.py index 697ff56a..af02ff83 100755 --- a/bin/scisys_receiver.py +++ b/bin/scisys_receiver.py @@ -53,8 +53,6 @@ def parse_args(): local_ips.remove('127.0.0.1') parser = argparse.ArgumentParser() - parser.add_argument("host", help="GMC host") - parser.add_argument("port", help="Port to listen to", type=int) parser.add_argument("-c", "--config", help="YAML config file to use.") parser.add_argument("-P", "--publish-port", type=int, default=0, @@ -62,17 +60,6 @@ def parse_args(): parser.add_argument("-n", "--nameserver", nargs='+', default=[], dest="nameservers", help="Nameserver(s) to connect to") - parser.add_argument("-s", "--station", help="Name of the station", - dest="station", - default="unknown") - parser.add_argument("-x", "--excluded_satellites", nargs='*', - help="List of platform names to exclude", - dest="excluded_satellites", - default=[]) - parser.add_argument("-e", "--environment", - help="Name of the environment (e.g. dev, test, oper)", - dest="environment", - default="dev") parser.add_argument("-l", "--log", help="File to log to", dest="log", default=None) parser.add_argument("-f", "--ftp_prefix", dest="ftp_prefix", @@ -85,11 +72,6 @@ def parse_args(): help="IP of the target server." "In case of multiple dispatches in GMC." "Defaults to the local host.") - # parser.add_argument("-T", "--topic_postfix", - # dest="topic_postfix", - # type=str, - # help="Publish topic postfix. " - # "Prefix will be /format/data_processing_level/") return parser.parse_args() @@ -125,11 +107,6 @@ def main(): setup_logging(log_file=opts.log) try: - # receive_from_zmq(opts.host, opts.port, - # opts.station, opts.environment, no_sats, - # opts.target_server, opts.ftp_prefix, - # opts.topic_postfix, publish_port=opts.publish_port, - # nameservers=opts.nameservers, days=1) receive_from_zmq(configfile, opts.target_server, opts.ftp_prefix, publish_port=opts.publish_port, diff --git a/examples/scisys_receiver.yaml_template b/examples/scisys_receiver.yaml_template index 18f65947..cf425658 100644 --- a/examples/scisys_receiver.yaml_template +++ b/examples/scisys_receiver.yaml_template @@ -1,11 +1,11 @@ # Publish topic publish_topic_pattern: "/{sensor}/{format}/{data_processing_level}/{platform_name}" -# It is possible to here add a static postfix topic if needed: -topic_postfix: "my/cool/postfix/topic" +# GMC Server host +host: my-reception-server-name +# Port to listen to +port: portnumber -host: merlin -port: 10600 station: nrk environment: dev From b752d382311c8035ea62e73943c34eea59b96c8f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 4 May 2023 16:16:59 +0200 Subject: [PATCH 3/3] Fix unsafe yaml load and small refactoring Signed-off-by: Adam.Dybbroe --- pytroll_collectors/config.py | 3 +-- pytroll_collectors/scisys.py | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pytroll_collectors/config.py b/pytroll_collectors/config.py index 1e8aff4c..d49c77e3 100644 --- a/pytroll_collectors/config.py +++ b/pytroll_collectors/config.py @@ -23,12 +23,11 @@ """Handling the yaml configurations.""" import yaml -from yaml import UnsafeLoader def read_config(config_filepath): """Read and extract config information.""" with open(config_filepath, 'r') as fp_: - config = yaml.load(fp_, Loader=UnsafeLoader) + config = yaml.safe_load(fp_) return config diff --git a/pytroll_collectors/scisys.py b/pytroll_collectors/scisys.py index e606a421..ac3083fe 100755 --- a/pytroll_collectors/scisys.py +++ b/pytroll_collectors/scisys.py @@ -511,12 +511,8 @@ def receive_from_zmq(config_filename, config = read_config(config_filename) - excluded_platforms = config['excluded_satellites'] - host = config['host'] - port = config['port'] - - sock = GMCSubscriber(host, port) - msg_rec = MessageReceiver(host, excluded_platforms, + sock = GMCSubscriber(config['host'], config['port']) + msg_rec = MessageReceiver(config['host'], config['excluded_platforms'], target_server, ftp_prefix) with Publish("receiver", port=publish_port,