FIX: Refactored upgrades to a more generic solution

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

intelmq/bin/intelmqctl.py

@@ -30,6 +30,7 @@
 from intelmq.lib.pipeline import PipelineFactory
 import intelmq.lib.upgrades as upgrades
 
+
 yaml = YAML(typ="safe", pure=True)
 
 try:

intelmq/lib/upgrade/__init__.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2022 CERT.at GmbH <intelmq-team@cert.at>
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+from os.path import dirname, basename, isfile, join
+import glob
+
+
+modules = glob.glob(join(dirname(__file__), "*.py"))
+__all__ = sorted([basename(f)[:-3] for f in modules if isfile(f) and not basename(f).startswith('_')])

intelmq/lib/upgrade/harmonization.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: 2022 Sebastian Wagner
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+from pkg_resources import resource_filename
+from intelmq.lib.utils import load_configuration
+
+
+def harmonization(configuration, harmonization, dry_run, **kwargs):
+    """
+    Checks if all harmonization fields and types are correct
+    """
+    changed = None
+    original = load_configuration(resource_filename('intelmq',
+                                                    'etc/harmonization.conf'))
+    for msg_type, msg in original.items():
+        if msg_type not in harmonization:
+            harmonization[msg_type] = msg
+            changed = True
+            continue
+        for fieldname, field in msg.items():
+            if fieldname not in harmonization[msg_type]:
+                harmonization[msg_type][fieldname] = field
+                changed = True
+                continue
+            if harmonization[msg_type][fieldname]['type'] != original[msg_type][fieldname]['type']:
+                harmonization[msg_type][fieldname]['type'] = original[msg_type][fieldname]['type']
+                changed = True
+            installed_regex = harmonization[msg_type][fieldname].get('regex')
+            original_regex = original[msg_type][fieldname].get('regex')
+            if original_regex and original_regex != installed_regex:
+                harmonization[msg_type][fieldname]['regex'] = original[msg_type][fieldname]['regex']
+                changed = True
+            installed_regex = harmonization[msg_type][fieldname].get('iregex')
+            original_regex = original[msg_type][fieldname].get('iregex')
+            if original_regex and original_regex != installed_regex:
+                harmonization[msg_type][fieldname]['iregex'] = original[msg_type][fieldname]['iregex']
+                changed = True
+    return changed, configuration, harmonization

intelmq/lib/upgrade/v100_dev7.py

@@ -0,0 +1,48 @@
+# SPDX-FileCopyrightText: 2022 Sebastian Wagner
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+from intelmq.lib.utils import load_configuration, write_configuration
+
+
+def modify_expert_convert_config(old):
+    """
+    Also used in the modify expert
+    """
+    config = []
+    for groupname, group in old.items():
+        for rule_name, rule in group.items():
+            config.append({"rulename": groupname + ' ' + rule_name,
+                           "if": rule[0],
+                           "then": rule[1]})
+    return config
+
+
+def modify_syntax(configuration, harmonization, dry_run, **kwargs):
+    """
+    Migrate modify bot configuration format
+    """
+    changed = None
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] == "intelmq.bots.experts.modify.expert":
+            if "configuration_path" in bot["parameters"]:
+                config = load_configuration(bot["parameters"]["configuration_path"])
+                if type(config) is dict:
+                    new_config = modify_expert_convert_config(config)
+                    if len(config) != len(new_config):
+                        return 'Error converting modify expert syntax. Different size of configurations. Please report this.'
+                    changed = True
+                    if dry_run:
+                        print('Would now convert file %r syntax.',
+                              bot["parameters"]["configuration_path"])
+                        continue
+                    try:
+                        write_configuration(bot["parameters"]["configuration_path"],
+                                            new_config)
+                    except PermissionError:
+                        return ('Can\'t update %s\'s configuration: Permission denied.' % bot_id,
+                                configuration, harmonization)
+
+    return changed, configuration, harmonization

intelmq/lib/upgrade/v110.py

@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText: 2022 Sebastian Wagner
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+
+
+def shadowserver_feednames(configuration, harmonization, dry_run, **kwargs):
+    """
+    Replace deprecated Shadowserver feednames
+    """
+    mapping = {
+        "Botnet-Drone-Hadoop": "Drone",
+        "DNS-open-resolvers": "DNS-Open-Resolvers",
+        "Open-NetBIOS": "Open-NetBIOS-Nameservice",
+        "Ssl-Freak-Scan": "SSL-FREAK-Vulnerable-Servers",
+        "Ssl-Scan": "SSL-POODLE-Vulnerable-Servers",
+    }
+    changed = None
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] == "intelmq.bots.parsers.shadowserver.parser":
+            if bot["parameters"]["feedname"] in mapping:
+                changed = True
+                bot["parameters"]["feedname"] = mapping[bot["parameters"]["feedname"]]
+
+    return changed, configuration, harmonization
+
+
+def deprecations(configuration, harmonization, dry_run, **kwargs):
+    """
+    Checking for deprecated runtime configurations (stomp collector, cymru parser, ripe expert, collector feed parameter)
+    """
+    mapping = {
+        "intelmq.bots.collectors.n6.collector_stomp": "intelmq.bots.collectors.stomp.collector",
+        "intelmq.bots.parsers.cymru_full_bogons.parser": "intelmq.bots.parsers.cymru.parser_full_bogons",
+    }
+    changed = None
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] in mapping:
+            bot["module"] = mapping[bot["module"]]
+            changed = True
+        if bot["module"] == "intelmq.bots.experts.ripencc_abuse_contact.expert":
+            bot["module"] = "intelmq.bots.experts.ripe.expert"
+            changed = True
+        if bot["module"] == "intelmq.bots.experts.ripe.expert":
+            if bot["parameters"].get("query_ripe_stat"):
+                if "query_ripe_stat_asn" not in bot["parameters"]:
+                    bot["parameters"]["query_ripe_stat_asn"] = bot["parameters"]["query_ripe_stat"]
+                if "query_ripe_stat_ip" not in bot["parameters"]:
+                    bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat"]
+                del bot["parameters"]["query_ripe_stat"]
+                changed = True
+        if bot["group"] == 'Collector' and bot["parameters"].get("feed") and not bot["parameters"].get("name"):
+            try:
+                bot["parameters"]["name"] = bot["parameters"]["feed"]
+                del bot["parameters"]["feed"]
+            except KeyError:
+                pass
+            else:
+                changed = True
+
+    return changed, configuration, harmonization

intelmq/lib/upgrade/v111.py

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: 2022 Sebastian Wagner
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+
+
+def defaults_process_manager(configuration, harmonization, dry_run, **kwargs):
+    """
+    Fix typo in proccess_manager parameter
+    """
+    changed = None
+    if "proccess_manager" in configuration['global']:
+        if "process_manager" in configuration['global']:
+            del configuration['global']["proccess_manager"]
+        elif "process_manager" not in configuration['global']:
+            configuration['global']["process_manager"] = configuration['global']["proccess_manager"]
+            del configuration['global']["proccess_manager"]
+        changed = True
+    else:
+        if "process_manager" not in configuration['global']:
+            configuration['global']["process_manager"] = "intelmq"
+            changed = True
+
+    return changed, configuration, harmonization

intelmq/lib/upgrade/v112.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: 2022 Sebastian Wagner
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+
+
+def feodo_tracker_ips(configuration, harmonization, dry_run, **kwargs):
+    """
+    Fix URL of feodotracker IPs feed in runtime configuration
+    """
+    changed = None
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["parameters"].get("http_url") == "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist":
+            bot["parameters"]["http_url"] = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv"
+            changed = True
+
+    return changed, configuration, harmonization
+
+
+def feodo_tracker_domains(configuration, harmonization, dry_run, **kwargs):
+    """
+    Search for discontinued feodotracker domains feed
+    """
+    found = False
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["parameters"].get("http_url") == "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist":
+            found = bot_id
+
+    if not found:
+        return None, configuration, harmonization
+    else:
+        return ('The discontinued feed "Feodo Tracker Domains" has been found '
+                'as bot %r. Remove it yourself please.' % found,
+                configuration, harmonization)

intelmq/lib/upgrade/v200.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2022 Sebastian Wagner
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+
+
+def defaults_statistics(configuration, harmonization, dry_run, **kwargs):
+    """
+    Inserting `statistics_*` parameters into defaults configuration file
+    """
+    values = {"statistics_database": 3,
+              "statistics_host": "127.0.0.1",
+              "statistics_password": configuration['global'].get('source_pipeline_password', None),
+              "statistics_port": 6379
+              }
+    changed = None
+    for key, value in values.items():
+        if key not in configuration['global']:
+            configuration['global'][key] = value
+            changed = True
+    return changed, configuration, harmonization
+
+
+def defaults_broker(configuration, harmonization, dry_run, **kwargs):
+    """
+    Inserting `*_pipeline_broker` and deleting broker into/from defaults configuration
+    """
+    changed = None
+    values = {"destination_pipeline_broker": configuration['global'].get("broker", "redis"),
+              "source_pipeline_broker": configuration['global'].get("broker", "redis"),
+              }
+    for key, value in values.items():
+        if key not in configuration['global']:
+            configuration['global'][key] = value
+            changed = True
+    if "broker" in configuration['global']:
+        del configuration['global']["broker"]
+        changed = True
+
+    return changed, configuration, harmonization
+
+
+def defaults_ssl_ca_certificate(configuration, harmonization, dry_run, **kwargs):
+    """
+    Add ssl_ca_certificate to defaults
+    """
+    if "ssl_ca_certificate" not in configuration['global']:
+        configuration['global']["ssl_ca_certificate"] = None
+        return True, configuration, harmonization
+    else:
+        return None, configuration, harmonization

intelmq/lib/upgrade/v202.py

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: 2022 Sebastian Wagner
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+
+
+def fixes(configuration, harmonization, dry_run, **kwargs):
+    """
+    Migrate Collector parameter `feed` to `name`. RIPE expert set `query_ripe_stat_ip` with `query_ripe_stat_asn` as default.
+    Set cymru whois expert `overwrite` to true.
+    """
+    changed = None
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["group"] == 'Collector' and bot["parameters"].get("feed"):
+            try:
+                bot["parameters"]["name"] = bot["parameters"]["feed"]
+                del bot["parameters"]["feed"]
+            except KeyError:
+                pass
+            else:
+                changed = True
+        if bot["module"] == "intelmq.bots.experts.ripe.expert":
+            if "query_ripe_stat_asn" in bot["parameters"]:
+                if "query_ripe_stat_ip" not in bot["parameters"]:
+                    bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat_asn"]
+                    changed = True
+        if bot["module"] in ("intelmq.bots.experts.cymru_whois.expert",
+                             "intelmq.bots.experts.reverse_dns.expert",
+                             "intelmq.bots.experts.modify.expert"):
+            if "overwrite" not in bot["parameters"]:
+                bot["parameters"]["overwrite"] = True
+                changed = True
+
+    return changed, configuration, harmonization

intelmq/lib/upgrade/v210.py

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: 2022 Sebastian Wagner
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# -*- coding: utf-8 -*-
+
+
+def deprecations(configuration, harmonization, dry_run, **kwargs):
+    """
+    Migrating configuration
+    """
+    changed = None
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] == "intelmq.bots.collectors.rt.collector_rt":
+            # from 29c4b2c42b126ef51ac7287edc1a9fee28ab27fd to ce96e6d995d420e117a49a22d3bfdea762d899ec
+            if "extract_files" in bot["parameters"]:
+                bot["parameters"]["extract_attachment"] = bot["parameters"]["extract_files"]
+                del bot["parameters"]["extract_files"]
+                changed = True
+            if "unzip_attachment" not in bot["parameters"]:
+                continue
+            if "extract_files" not in bot["parameters"]:
+                bot["parameters"]["extract_attachment"] = bot["parameters"]["unzip_attachment"]
+            del bot["parameters"]["unzip_attachment"]
+            changed = True
+        if bot["module"] in ("intelmq.bots.experts.generic_db_lookup.expert",
+                             "intelmq.bots.outputs.postgresql.output"):
+            if "engine" not in bot["parameters"]:
+                bot["parameters"]["engine"] = "postgresql"
+                changed = True
+            if bot["module"] == "intelmq.bots.outputs.postgresql.output":
+                bot["module"] = "intelmq.bots.outputs.sql.output"
+                changed = True
+    return changed, configuration, harmonization