From e051d26c9e08331ce34cf17232d7b6520e8bea2f Mon Sep 17 00:00:00 2001 From: Sebastian Waldbauer Date: Fri, 15 Jul 2022 17:34:10 +0200 Subject: [PATCH] FIX: Refactored upgrades to a more generic solution Signed-off-by: Sebastian Waldbauer --- intelmq/bin/intelmqctl.py | 1 + intelmq/lib/upgrade/__init__.py | 10 + intelmq/lib/upgrade/harmonization.py | 39 + intelmq/lib/upgrade/v100_dev7.py | 48 ++ intelmq/lib/upgrade/v110.py | 65 ++ intelmq/lib/upgrade/v111.py | 24 + intelmq/lib/upgrade/v112.py | 38 + intelmq/lib/upgrade/v200.py | 51 ++ intelmq/lib/upgrade/v202.py | 36 + intelmq/lib/upgrade/v210.py | 35 + intelmq/lib/upgrade/v213.py | 95 +++ intelmq/lib/upgrade/v220.py | 69 ++ intelmq/lib/upgrade/v221.py | 43 + intelmq/lib/upgrade/v222.py | 19 + intelmq/lib/upgrade/v230.py | 62 ++ intelmq/lib/upgrade/v233.py | 25 + intelmq/lib/upgrade/v300.py | 75 ++ intelmq/lib/upgrade/v301.py | 32 + intelmq/lib/upgrades.py | 763 +----------------- intelmq/tests/lib/test_upgrades.py | 727 ----------------- intelmq/tests/lib/upgrades/__init__.py | 0 .../tests/lib/upgrades/test_harmonization.py | 41 + intelmq/tests/lib/upgrades/test_v110.py | 61 ++ intelmq/tests/lib/upgrades/test_v202.py | 63 ++ intelmq/tests/lib/upgrades/test_v210.py | 133 +++ intelmq/tests/lib/upgrades/test_v213.py | 152 ++++ intelmq/tests/lib/upgrades/test_v220.py | 67 ++ intelmq/tests/lib/upgrades/test_v221.py | 97 +++ intelmq/tests/lib/upgrades/test_v222.py | 29 + intelmq/tests/lib/upgrades/test_v230.py | 87 ++ intelmq/tests/lib/upgrades/test_v233.py | 41 + intelmq/tests/lib/upgrades/test_v301.py | 34 + 32 files changed, 1594 insertions(+), 1468 deletions(-) create mode 100644 intelmq/lib/upgrade/__init__.py create mode 100644 intelmq/lib/upgrade/harmonization.py create mode 100644 intelmq/lib/upgrade/v100_dev7.py create mode 100644 intelmq/lib/upgrade/v110.py create mode 100644 intelmq/lib/upgrade/v111.py create mode 100644 intelmq/lib/upgrade/v112.py create mode 100644 intelmq/lib/upgrade/v200.py create mode 100644 intelmq/lib/upgrade/v202.py create mode 100644 intelmq/lib/upgrade/v210.py create mode 100644 intelmq/lib/upgrade/v213.py create mode 100644 intelmq/lib/upgrade/v220.py create mode 100644 intelmq/lib/upgrade/v221.py create mode 100644 intelmq/lib/upgrade/v222.py create mode 100644 intelmq/lib/upgrade/v230.py create mode 100644 intelmq/lib/upgrade/v233.py create mode 100644 intelmq/lib/upgrade/v300.py create mode 100644 intelmq/lib/upgrade/v301.py delete mode 100644 intelmq/tests/lib/test_upgrades.py create mode 100644 intelmq/tests/lib/upgrades/__init__.py create mode 100644 intelmq/tests/lib/upgrades/test_harmonization.py create mode 100644 intelmq/tests/lib/upgrades/test_v110.py create mode 100644 intelmq/tests/lib/upgrades/test_v202.py create mode 100644 intelmq/tests/lib/upgrades/test_v210.py create mode 100644 intelmq/tests/lib/upgrades/test_v213.py create mode 100644 intelmq/tests/lib/upgrades/test_v220.py create mode 100644 intelmq/tests/lib/upgrades/test_v221.py create mode 100644 intelmq/tests/lib/upgrades/test_v222.py create mode 100644 intelmq/tests/lib/upgrades/test_v230.py create mode 100644 intelmq/tests/lib/upgrades/test_v233.py create mode 100644 intelmq/tests/lib/upgrades/test_v301.py diff --git a/intelmq/bin/intelmqctl.py b/intelmq/bin/intelmqctl.py index 059802542..bd384c808 100644 --- a/intelmq/bin/intelmqctl.py +++ b/intelmq/bin/intelmqctl.py @@ -30,6 +30,7 @@ from intelmq.lib.pipeline import PipelineFactory import intelmq.lib.upgrades as upgrades + yaml = YAML(typ="safe", pure=True) try: diff --git a/intelmq/lib/upgrade/__init__.py b/intelmq/lib/upgrade/__init__.py new file mode 100644 index 000000000..86b818ce2 --- /dev/null +++ b/intelmq/lib/upgrade/__init__.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2022 CERT.at GmbH +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +from os.path import dirname, basename, isfile, join +import glob + + +modules = glob.glob(join(dirname(__file__), "*.py")) +__all__ = sorted([basename(f)[:-3] for f in modules if isfile(f) and not basename(f).startswith('_')]) diff --git a/intelmq/lib/upgrade/harmonization.py b/intelmq/lib/upgrade/harmonization.py new file mode 100644 index 000000000..d6c93cbc4 --- /dev/null +++ b/intelmq/lib/upgrade/harmonization.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +from pkg_resources import resource_filename +from intelmq.lib.utils import load_configuration + + +def harmonization(configuration, harmonization, dry_run, **kwargs): + """ + Checks if all harmonization fields and types are correct + """ + changed = None + original = load_configuration(resource_filename('intelmq', + 'etc/harmonization.conf')) + for msg_type, msg in original.items(): + if msg_type not in harmonization: + harmonization[msg_type] = msg + changed = True + continue + for fieldname, field in msg.items(): + if fieldname not in harmonization[msg_type]: + harmonization[msg_type][fieldname] = field + changed = True + continue + if harmonization[msg_type][fieldname]['type'] != original[msg_type][fieldname]['type']: + harmonization[msg_type][fieldname]['type'] = original[msg_type][fieldname]['type'] + changed = True + installed_regex = harmonization[msg_type][fieldname].get('regex') + original_regex = original[msg_type][fieldname].get('regex') + if original_regex and original_regex != installed_regex: + harmonization[msg_type][fieldname]['regex'] = original[msg_type][fieldname]['regex'] + changed = True + installed_regex = harmonization[msg_type][fieldname].get('iregex') + original_regex = original[msg_type][fieldname].get('iregex') + if original_regex and original_regex != installed_regex: + harmonization[msg_type][fieldname]['iregex'] = original[msg_type][fieldname]['iregex'] + changed = True + return changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v100_dev7.py b/intelmq/lib/upgrade/v100_dev7.py new file mode 100644 index 000000000..69132a5d1 --- /dev/null +++ b/intelmq/lib/upgrade/v100_dev7.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +from intelmq.lib.utils import load_configuration, write_configuration + + +def modify_expert_convert_config(old): + """ + Also used in the modify expert + """ + config = [] + for groupname, group in old.items(): + for rule_name, rule in group.items(): + config.append({"rulename": groupname + ' ' + rule_name, + "if": rule[0], + "then": rule[1]}) + return config + + +def modify_syntax(configuration, harmonization, dry_run, **kwargs): + """ + Migrate modify bot configuration format + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.experts.modify.expert": + if "configuration_path" in bot["parameters"]: + config = load_configuration(bot["parameters"]["configuration_path"]) + if type(config) is dict: + new_config = modify_expert_convert_config(config) + if len(config) != len(new_config): + return 'Error converting modify expert syntax. Different size of configurations. Please report this.' + changed = True + if dry_run: + print('Would now convert file %r syntax.', + bot["parameters"]["configuration_path"]) + continue + try: + write_configuration(bot["parameters"]["configuration_path"], + new_config) + except PermissionError: + return ('Can\'t update %s\'s configuration: Permission denied.' % bot_id, + configuration, harmonization) + + return changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v110.py b/intelmq/lib/upgrade/v110.py new file mode 100644 index 000000000..b75825c74 --- /dev/null +++ b/intelmq/lib/upgrade/v110.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def shadowserver_feednames(configuration, harmonization, dry_run, **kwargs): + """ + Replace deprecated Shadowserver feednames + """ + mapping = { + "Botnet-Drone-Hadoop": "Drone", + "DNS-open-resolvers": "DNS-Open-Resolvers", + "Open-NetBIOS": "Open-NetBIOS-Nameservice", + "Ssl-Freak-Scan": "SSL-FREAK-Vulnerable-Servers", + "Ssl-Scan": "SSL-POODLE-Vulnerable-Servers", + } + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.parsers.shadowserver.parser": + if bot["parameters"]["feedname"] in mapping: + changed = True + bot["parameters"]["feedname"] = mapping[bot["parameters"]["feedname"]] + + return changed, configuration, harmonization + + +def deprecations(configuration, harmonization, dry_run, **kwargs): + """ + Checking for deprecated runtime configurations (stomp collector, cymru parser, ripe expert, collector feed parameter) + """ + mapping = { + "intelmq.bots.collectors.n6.collector_stomp": "intelmq.bots.collectors.stomp.collector", + "intelmq.bots.parsers.cymru_full_bogons.parser": "intelmq.bots.parsers.cymru.parser_full_bogons", + } + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] in mapping: + bot["module"] = mapping[bot["module"]] + changed = True + if bot["module"] == "intelmq.bots.experts.ripencc_abuse_contact.expert": + bot["module"] = "intelmq.bots.experts.ripe.expert" + changed = True + if bot["module"] == "intelmq.bots.experts.ripe.expert": + if bot["parameters"].get("query_ripe_stat"): + if "query_ripe_stat_asn" not in bot["parameters"]: + bot["parameters"]["query_ripe_stat_asn"] = bot["parameters"]["query_ripe_stat"] + if "query_ripe_stat_ip" not in bot["parameters"]: + bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat"] + del bot["parameters"]["query_ripe_stat"] + changed = True + if bot["group"] == 'Collector' and bot["parameters"].get("feed") and not bot["parameters"].get("name"): + try: + bot["parameters"]["name"] = bot["parameters"]["feed"] + del bot["parameters"]["feed"] + except KeyError: + pass + else: + changed = True + + return changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v111.py b/intelmq/lib/upgrade/v111.py new file mode 100644 index 000000000..9a0e7b91b --- /dev/null +++ b/intelmq/lib/upgrade/v111.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def defaults_process_manager(configuration, harmonization, dry_run, **kwargs): + """ + Fix typo in proccess_manager parameter + """ + changed = None + if "proccess_manager" in configuration['global']: + if "process_manager" in configuration['global']: + del configuration['global']["proccess_manager"] + elif "process_manager" not in configuration['global']: + configuration['global']["process_manager"] = configuration['global']["proccess_manager"] + del configuration['global']["proccess_manager"] + changed = True + else: + if "process_manager" not in configuration['global']: + configuration['global']["process_manager"] = "intelmq" + changed = True + + return changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v112.py b/intelmq/lib/upgrade/v112.py new file mode 100644 index 000000000..7bcbf4870 --- /dev/null +++ b/intelmq/lib/upgrade/v112.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def feodo_tracker_ips(configuration, harmonization, dry_run, **kwargs): + """ + Fix URL of feodotracker IPs feed in runtime configuration + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["parameters"].get("http_url") == "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist": + bot["parameters"]["http_url"] = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv" + changed = True + + return changed, configuration, harmonization + + +def feodo_tracker_domains(configuration, harmonization, dry_run, **kwargs): + """ + Search for discontinued feodotracker domains feed + """ + found = False + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["parameters"].get("http_url") == "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist": + found = bot_id + + if not found: + return None, configuration, harmonization + else: + return ('The discontinued feed "Feodo Tracker Domains" has been found ' + 'as bot %r. Remove it yourself please.' % found, + configuration, harmonization) diff --git a/intelmq/lib/upgrade/v200.py b/intelmq/lib/upgrade/v200.py new file mode 100644 index 000000000..05bf7fc08 --- /dev/null +++ b/intelmq/lib/upgrade/v200.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def defaults_statistics(configuration, harmonization, dry_run, **kwargs): + """ + Inserting `statistics_*` parameters into defaults configuration file + """ + values = {"statistics_database": 3, + "statistics_host": "127.0.0.1", + "statistics_password": configuration['global'].get('source_pipeline_password', None), + "statistics_port": 6379 + } + changed = None + for key, value in values.items(): + if key not in configuration['global']: + configuration['global'][key] = value + changed = True + return changed, configuration, harmonization + + +def defaults_broker(configuration, harmonization, dry_run, **kwargs): + """ + Inserting `*_pipeline_broker` and deleting broker into/from defaults configuration + """ + changed = None + values = {"destination_pipeline_broker": configuration['global'].get("broker", "redis"), + "source_pipeline_broker": configuration['global'].get("broker", "redis"), + } + for key, value in values.items(): + if key not in configuration['global']: + configuration['global'][key] = value + changed = True + if "broker" in configuration['global']: + del configuration['global']["broker"] + changed = True + + return changed, configuration, harmonization + + +def defaults_ssl_ca_certificate(configuration, harmonization, dry_run, **kwargs): + """ + Add ssl_ca_certificate to defaults + """ + if "ssl_ca_certificate" not in configuration['global']: + configuration['global']["ssl_ca_certificate"] = None + return True, configuration, harmonization + else: + return None, configuration, harmonization diff --git a/intelmq/lib/upgrade/v202.py b/intelmq/lib/upgrade/v202.py new file mode 100644 index 000000000..dce60a23b --- /dev/null +++ b/intelmq/lib/upgrade/v202.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def fixes(configuration, harmonization, dry_run, **kwargs): + """ + Migrate Collector parameter `feed` to `name`. RIPE expert set `query_ripe_stat_ip` with `query_ripe_stat_asn` as default. + Set cymru whois expert `overwrite` to true. + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["group"] == 'Collector' and bot["parameters"].get("feed"): + try: + bot["parameters"]["name"] = bot["parameters"]["feed"] + del bot["parameters"]["feed"] + except KeyError: + pass + else: + changed = True + if bot["module"] == "intelmq.bots.experts.ripe.expert": + if "query_ripe_stat_asn" in bot["parameters"]: + if "query_ripe_stat_ip" not in bot["parameters"]: + bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat_asn"] + changed = True + if bot["module"] in ("intelmq.bots.experts.cymru_whois.expert", + "intelmq.bots.experts.reverse_dns.expert", + "intelmq.bots.experts.modify.expert"): + if "overwrite" not in bot["parameters"]: + bot["parameters"]["overwrite"] = True + changed = True + + return changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v210.py b/intelmq/lib/upgrade/v210.py new file mode 100644 index 000000000..62121ad65 --- /dev/null +++ b/intelmq/lib/upgrade/v210.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def deprecations(configuration, harmonization, dry_run, **kwargs): + """ + Migrating configuration + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.rt.collector_rt": + # from 29c4b2c42b126ef51ac7287edc1a9fee28ab27fd to ce96e6d995d420e117a49a22d3bfdea762d899ec + if "extract_files" in bot["parameters"]: + bot["parameters"]["extract_attachment"] = bot["parameters"]["extract_files"] + del bot["parameters"]["extract_files"] + changed = True + if "unzip_attachment" not in bot["parameters"]: + continue + if "extract_files" not in bot["parameters"]: + bot["parameters"]["extract_attachment"] = bot["parameters"]["unzip_attachment"] + del bot["parameters"]["unzip_attachment"] + changed = True + if bot["module"] in ("intelmq.bots.experts.generic_db_lookup.expert", + "intelmq.bots.outputs.postgresql.output"): + if "engine" not in bot["parameters"]: + bot["parameters"]["engine"] = "postgresql" + changed = True + if bot["module"] == "intelmq.bots.outputs.postgresql.output": + bot["module"] = "intelmq.bots.outputs.sql.output" + changed = True + return changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v213.py b/intelmq/lib/upgrade/v213.py new file mode 100644 index 000000000..6a3232de5 --- /dev/null +++ b/intelmq/lib/upgrade/v213.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def deprecations(configuration, harmonization, dry_run, **kwargs): + """ + migrate attach_unzip to extract_files for mail attachment collector + + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.mail.collector_mail_attach": + if "attach_unzip" not in bot["parameters"]: + continue + if "extract_files" in bot["parameters"] and "attach_unzip" in bot["parameters"]: + del bot["parameters"]["attach_unzip"] + changed = True + elif "extract_files" not in bot["parameters"] and "attach_unzip" in bot["parameters"]: + bot["parameters"]["extract_files"] = bot["parameters"]["attach_unzip"] + del bot["parameters"]["attach_unzip"] + changed = True + return changed, configuration, harmonization + + +def feed_changes(configuration, harmonization, dry_run, **kwargs): + """ + Migrates feed configuration for changed feed parameters. + """ + found_zeus = [] + found_bitcash = [] + found_ddos_attack = [] + found_ransomware = [] + found_bambenek = [] + found_nothink = [] + found_nothink_parser = [] + changed = None + messages = [] + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.http.collector_http": + if "http_url" not in bot["parameters"]: + continue + if bot["parameters"]["http_url"] == 'https://www.tc.edu.tw/net/netflow/lkout/recent/30': + bot["parameters"]["http_url"] = "https://www.tc.edu.tw/net/netflow/lkout/recent/" + changed = True + if bot["parameters"]["http_url"].startswith("https://zeustracker.abuse.ch/"): + found_zeus.append(bot_id) + elif bot["parameters"]["http_url"].startswith("https://bitcash.cz/misc/log/blacklist"): + found_bitcash.append(bot_id) + elif bot["parameters"]["http_url"].startswith("https://ransomwaretracker.abuse.ch/feeds/csv/"): + found_ransomware.append(bot_id) + elif bot["parameters"]["http_url"] == "https://osint.bambenekconsulting.com/feeds/dga-feed.txt": + bot["parameters"]["http_url"] = "https://faf.bambenekconsulting.com/feeds/dga-feed.txt" + changed = True + elif bot["parameters"]["http_url"] in ("http://osing.bambenekconsulting.com/feeds/dga/c2-ipmasterlist.txt", + "https://osing.bambenekconsulting.com/feeds/dga/c2-ipmasterlist.txt", + "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt", + "https://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt"): + found_bambenek.append(bot_id) + elif (bot["parameters"]["http_url"].startswith("http://www.nothink.org/") or + bot["parameters"]["http_url"].startswith("https://www.nothink.org/")): + found_nothink.append(bot_id) + elif bot["module"] == "intelmq.bots.collectors.http.collector_http_stream": + if bot["parameters"].get("http_url", "").startswith("https://feed.caad.fkie.fraunhofer.de/ddosattackfeed"): + found_ddos_attack.append(bot_id) + elif bot['module'] == "intelmq.bots.parsers.nothink.parser": + found_nothink_parser.append(bot_id) + if found_zeus: + messages.append('A discontinued feed "Zeus Tracker" has been found ' + 'as bot %s.' % ', '.join(sorted(found_zeus))) + if found_bitcash: + messages.append('The discontinued feed "Bitcash.cz" has been found ' + 'as bot %s.' % ', '.join(sorted(found_bitcash))) + if found_ddos_attack: + messages.append('The discontinued feed "Fraunhofer DDos Attack" has been found ' + 'as bot %s.' % ', '.join(sorted(found_ddos_attack))) + if found_ransomware: + messages.append('The discontinued feed "Abuse.ch Ransomware Tracker" has been found ' + 'as bot %s.' % ', '.join(sorted(found_ransomware))) + if found_bambenek: + messages.append('Many Bambenek feeds now require a license, see https://osint.bambenekconsulting.com/feeds/' + ' potentially affected bots are %s.' % ', '.join(sorted(found_bambenek))) + if found_nothink: + messages.append('All Nothink Honeypot feeds are discontinued, ' + 'potentially affected bots are %s.' % ', '.join(sorted(found_nothink))) + if found_nothink_parser: + messages.append('The Nothink Parser has been removed, ' + 'affected bots are %s.' % ', '.join(sorted(found_nothink_parser))) + messages = ' '.join(messages) + return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v220.py b/intelmq/lib/upgrade/v220.py new file mode 100644 index 000000000..e7178b1b4 --- /dev/null +++ b/intelmq/lib/upgrade/v220.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def configuration(configuration, harmonization, dry_run, **kwargs): + """ + Migrating configuration + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.misp.collector": + if "misp_verify" not in bot["parameters"]: + continue + if bot["parameters"]["misp_verify"] != configuration['global']["http_verify_cert"]: + bot["parameters"]["http_verify_cert"] = bot["parameters"]["misp_verify"] + del bot["parameters"]["misp_verify"] + changed = True + elif bot["module"] == "intelmq.bots.outputs.elasticsearch.output": + if "elastic_doctype" in bot["parameters"]: + del bot["parameters"]["elastic_doctype"] + return changed, configuration, harmonization + + +def azure_collector(configuration, harmonization, dry_run, **kwargs): + """ + Checking for the Microsoft Azure collector + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.microsoft.collector_azure": + if "connection_string" not in bot["parameters"]: + changed = ("The Microsoft Azure collector changed backwards-" + "incompatible in IntelMQ 2.2.0. Look at the bot's " + "documentation and NEWS file to adapt the " + "configuration.") + return changed, configuration, harmonization + + +def feed_changes(configuration, harmonization, dry_run, **kwargs): + """ + Migrates feed configuration for changed feed parameters. + """ + found_urlvir_feed = [] + found_urlvir_parser = [] + messages = [] + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.http.collector_http": + if "http_url" not in bot["parameters"]: + continue + if bot["parameters"]["http_url"].startswith("http://www.urlvir.com/export-"): + found_urlvir_feed.append(bot_id) + elif bot['module'] == "intelmq.bots.parsers.urlvir.parser": + found_urlvir_parser.append(bot_id) + if found_urlvir_feed: + messages.append('A discontinued feed "URLVir" has been found ' + 'as bot %s.' % ', '.join(sorted(found_urlvir_feed))) + if found_urlvir_parser: + messages.append('The removed parser "URLVir" has been found ' + 'as bot %s.' % ', '.join(sorted(found_urlvir_parser))) + messages = ' '.join(messages) + return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization diff --git a/intelmq/lib/upgrade/v221.py b/intelmq/lib/upgrade/v221.py new file mode 100644 index 000000000..751359778 --- /dev/null +++ b/intelmq/lib/upgrade/v221.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def feed_changes(configuration, harmonization, dry_run, **kwargs): + """ + Migrates feeds' configuration for changed/fixed parameters. Deprecation of HP Hosts file feed & parser. + """ + found_hphosts_collector = [] + found_hphosts_parser = [] + messages = [] + ULRHAUS_OLD = ['time.source', 'source.url', 'status', 'extra.urlhaus.threat_type', 'source.fqdn', 'source.ip', 'source.asn', 'source.geolocation.cc'] + URLHAUS_NEW = ['time.source', 'source.url', 'status', 'classification.type|__IGNORE__', 'source.fqdn|__IGNORE__', 'source.ip', 'source.asn', 'source.geolocation.cc'] + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.http.collector_http": + if bot["parameters"].get("http_url", None) == "http://hosts-file.net/download/hosts.txt": + found_hphosts_collector.append(bot_id) + elif bot['module'] == "intelmq.bots.parsers.hphosts.parser": + found_hphosts_parser.append(bot_id) + if bot["module"] == "intelmq.bots.parsers.generic.parser_csv": + if "columns" not in bot["parameters"]: + continue + columns = bot["parameters"]["columns"] + # convert columns to an array + if type(columns) is str: + columns = [column.strip() for column in columns.split(",")] + if columns == ULRHAUS_OLD: + changed = True + bot["parameters"]["columns"] = URLHAUS_NEW + + if found_hphosts_collector: + messages.append('A discontinued feed "HP Hosts File" has been found ' + 'as bot %s.' % ', '.join(sorted(found_hphosts_collector))) + if found_hphosts_parser: + messages.append('The removed parser "HP Hosts" has been found ' + 'as bot %s.' % ', '.join(sorted(found_hphosts_parser))) + messages = ' '.join(messages) + return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v222.py b/intelmq/lib/upgrade/v222.py new file mode 100644 index 000000000..d11f826fc --- /dev/null +++ b/intelmq/lib/upgrade/v222.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def feed_changes(configuration, harmonization, dry_run, **kwargs): + """ + Migrate Shadowserver feed name + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.parsers.shadowserver.parser": + if bot["parameters"].get("feedname", None) == "Blacklisted-IP": + bot["parameters"]["feedname"] = "Blocklist" + changed = True + return changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v230.py b/intelmq/lib/upgrade/v230.py new file mode 100644 index 000000000..31585dadc --- /dev/null +++ b/intelmq/lib/upgrade/v230.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def csv_parser_parameter_fix(configuration, harmonization, dry_run, **kwargs): + """ + Fix CSV parser parameter misspelling + """ + changed = None + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.parsers.generic.parser_csv": + if "delimeter" in bot["parameters"] and "delimiter" in bot["parameters"]: + del bot["parameters"]["delimeter"] + changed = True + elif "delimeter" in bot["parameters"]: + bot["parameters"]["delimiter"] = bot["parameters"]["delimeter"] + del bot["parameters"]["delimeter"] + changed = True + return changed, configuration, harmonization + + +def deprecations(configuration, harmonization, dry_run, **kwargs): + """ + Deprecate malwaredomainlist parser + """ + found_malwaredomainlistparser = [] + messages = [] + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.parsers.malwaredomainlist.parser": + found_malwaredomainlistparser.append(bot_id) + if found_malwaredomainlistparser: + messages.append('A discontinued bot "Malware Domain List Parser" has been found ' + 'as bot %s.' % ', '.join(sorted(found_malwaredomainlistparser))) + messages = ' '.join(messages) + return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization + + +def feed_changes(configuration, harmonization, dry_run, **kwargs): + """ + Migrates feeds' configuration for changed/fixed parameter + """ + found_malwaredomainlist = [] + messages = [] + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.http.collector_http": + if "http_url" not in bot["parameters"]: + continue + if bot["parameters"]["http_url"].startswith("http://www.malwaredomainlist.com/updatescsv.php"): + found_malwaredomainlist.append(bot_id) + if found_malwaredomainlist: + messages.append('A discontinued feed "Malware Domain List" has been found ' + 'as bot %s.' % ', '.join(sorted(found_malwaredomainlist))) + messages = ' '.join(messages) + return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization diff --git a/intelmq/lib/upgrade/v233.py b/intelmq/lib/upgrade/v233.py new file mode 100644 index 000000000..8897699d5 --- /dev/null +++ b/intelmq/lib/upgrade/v233.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def feodotracker_browse(configuration, harmonization, dry_run, **kwargs): + """ + Migrate Abuse.ch Feodotracker Browser feed parsing parameters + """ + changed = None + old_feodo_columns = 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc' + old_ignore_values = ',,,,Not listed,,' + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + # The parameters can be given as string or list of strings + if (bot["module"] == "intelmq.bots.parsers.html_table.parser" and 'feodo' in bot_id.lower() and + "columns" in bot["parameters"] and "ignore_values" in bot["parameters"] and + (bot["parameters"]["columns"] == old_feodo_columns or bot["parameters"]["columns"] == old_feodo_columns.split(',')) and + (bot["parameters"]["ignore_values"] == old_ignore_values or bot["parameters"]["ignore_values"] == old_ignore_values.split(','))): + bot["parameters"]["columns"] = 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc' + bot["parameters"]['ignore_values'] = ',,,,,' + changed = True + return changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v300.py b/intelmq/lib/upgrade/v300.py new file mode 100644 index 000000000..47f41bd23 --- /dev/null +++ b/intelmq/lib/upgrade/v300.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +from pathlib import Path +from intelmq import CONFIG_DIR +from intelmq.lib.utils import load_configuration + + +def bots_file_removal(configuration, harmonization, dry_run, **kwargs): + """ + Remove BOTS file + """ + changed = None + messages = [] + bots_file = Path(CONFIG_DIR) / "BOTS" + if bots_file.exists(): + if dry_run: + print(f'Would now remove file {bots_file!r}.') + else: + bots_file.unlink() + changed = True + messages = ' '.join(messages) + return messages if messages else changed, configuration, harmonization + + +def defaults_file_removal(configuration, harmonization, dry_run, **kwargs): + """ + Remove the defaults.conf file + """ + changed = None + messages = [] + defaults_file = Path(CONFIG_DIR) / "defaults.conf" + if defaults_file.exists(): + if dry_run: + print(f'Would now remove file {defaults_file!r}.') + else: + configuration['global'] = load_configuration(defaults_file) + defaults_file.unlink() + changed = True + messages = ' '.join(messages) + return messages if messages else changed, configuration, harmonization + + +def pipeline_file_removal(configuration, harmonization, dry_run, **kwargs): + """ + Remove the pipeline.conf file + """ + changed = None + messages = [] + pipeline_file = Path(CONFIG_DIR) / "pipeline.conf" + if pipeline_file.exists(): + pipelines = load_configuration(pipeline_file) + for bot in configuration: + if bot == 'global': + continue + if bot in pipelines: + if 'destination-queues' in pipelines[bot]: + destination_queues = pipelines[bot]['destination-queues'] + if isinstance(destination_queues, dict): + configuration[bot]['parameters']['destination_queues'] = destination_queues + if isinstance(destination_queues, list): + configuration[bot]['parameters']['destination_queues'] = {'_default': destination_queues} + if isinstance(destination_queues, str): + configuration[bot]['parameters']['destination_queues'] = {'_default': [destination_queues]} + if 'source-queue' in pipelines[bot]: + if pipelines[bot]['source-queue'] != f"{bot}-queue": + configuration[bot]['parameters']['source_queue'] = pipelines[bot]['source-queue'] + if dry_run: + print(f'Would now remove file {pipeline_file!r}.') + else: + pipeline_file.unlink() + changed = True + messages = ' '.join(messages) + return messages if messages else changed, configuration, harmonization diff --git a/intelmq/lib/upgrade/v301.py b/intelmq/lib/upgrade/v301.py new file mode 100644 index 000000000..89ddb6038 --- /dev/null +++ b/intelmq/lib/upgrade/v301.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2022 Birger Schacht +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- + + +def deprecations(configuration, harmonization, dry_run, **kwargs): + """ + Deprecate malwaredomains parser and collector + """ + found_malwaredomainsparser = [] + found_malwaredomainscollector = [] + changed = None + messages = [] + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.parsers.malwaredomains.parser": + found_malwaredomainsparser.append(bot_id) + if bot["module"] == "intelmq.bots.collectors.http.collector": + if "http_url" not in bot["parameters"]: + continue + if bot["parameters"]["http_url"] == 'http://mirror1.malwaredomains.com/files/domains.txt': + found_malwaredomainscollector.append(bot_id) + if found_malwaredomainsparser: + messages.append('A discontinued bot "Malware Domains Parser" has been found ' + 'as bot %s.' % ', '.join(sorted(found_malwaredomainsparser))) + if found_malwaredomainscollector: + messages.append('A discontinued bot "Malware Domains Collector" has been found ' + 'as bot %s.' % ', '.join(sorted(found_malwaredomainscollector))) + messages = ' '.join(messages) + return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 83dd28923..bd96d9e31 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -1,755 +1,36 @@ -""" -© 2020 Sebastian Wagner - -SPDX-License-Identifier: AGPL-3.0-or-later -""" +# SPDX-FileCopyrightText: 2020 Sebastian Wagner , 2022 CERT.at GmbH +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- from collections import OrderedDict -from pkg_resources import resource_filename -from pathlib import Path -from intelmq import CONFIG_DIR - -from intelmq.lib.utils import load_configuration, write_configuration - -__all__ = ['v100_dev7_modify_syntax', - 'v110_shadowserver_feednames', - 'v110_deprecations', - 'v200_defaults_statistics', - 'v200_defaults_broker', - 'v112_feodo_tracker_ips', - 'v112_feodo_tracker_domains', - 'v200_defaults_ssl_ca_certificate', - 'v111_defaults_process_manager', - 'v202_fixes', - 'v210_deprecations', - 'v213_deprecations', - 'v213_feed_changes', - 'v220_configuration', - 'v220_azure_collector', - 'v220_feed_changes', - 'v221_feed_changes', - 'v222_feed_changes', - 'v230_csv_parser_parameter_fix', - 'v230_deprecations', - 'v230_feed_changes', - 'v233_feodotracker_browse', - 'v300_bots_file_removal', - 'v300_defaults_file_removal', - 'v300_pipeline_file_removal', - 'v301_deprecations', - ] - - -def v200_defaults_statistics(configuration, harmonization, dry_run, **kwargs): - """ - Inserting `statistics_*` parameters into defaults configuration file - """ - values = {"statistics_database": 3, - "statistics_host": "127.0.0.1", - "statistics_password": configuration['global'].get('source_pipeline_password', None), - "statistics_port": 6379 - } - changed = None - for key, value in values.items(): - if key not in configuration['global']: - configuration['global'][key] = value - changed = True - return changed, configuration, harmonization - - -def v200_defaults_broker(configuration, harmonization, dry_run, **kwargs): - """ - Inserting `*_pipeline_broker` and deleting broker into/from defaults configuration - """ - changed = None - values = {"destination_pipeline_broker": configuration['global'].get("broker", "redis"), - "source_pipeline_broker": configuration['global'].get("broker", "redis"), - } - for key, value in values.items(): - if key not in configuration['global']: - configuration['global'][key] = value - changed = True - if "broker" in configuration['global']: - del configuration['global']["broker"] - changed = True - - return changed, configuration, harmonization - - -def v112_feodo_tracker_ips(configuration, harmonization, dry_run, **kwargs): - """ - Fix URL of feodotracker IPs feed in runtime configuration - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["parameters"].get("http_url") == "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist": - bot["parameters"]["http_url"] = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv" - changed = True - - return changed, configuration, harmonization - - -def v112_feodo_tracker_domains(configuration, harmonization, dry_run, **kwargs): - """ - Search for discontinued feodotracker domains feed - """ - found = False - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["parameters"].get("http_url") == "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist": - found = bot_id - - if not found: - return None, configuration, harmonization - else: - return ('The discontinued feed "Feodo Tracker Domains" has been found ' - 'as bot %r. Remove it yourself please.' % found, - configuration, harmonization) - - -def v110_shadowserver_feednames(configuration, harmonization, dry_run, **kwargs): - """ - Replace deprecated Shadowserver feednames - """ - mapping = { - "Botnet-Drone-Hadoop": "Drone", - "DNS-open-resolvers": "DNS-Open-Resolvers", - "Open-NetBIOS": "Open-NetBIOS-Nameservice", - "Ssl-Freak-Scan": "SSL-FREAK-Vulnerable-Servers", - "Ssl-Scan": "SSL-POODLE-Vulnerable-Servers", - } - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.parsers.shadowserver.parser": - if bot["parameters"]["feedname"] in mapping: - changed = True - bot["parameters"]["feedname"] = mapping[bot["parameters"]["feedname"]] - - return changed, configuration, harmonization - - -def v110_deprecations(configuration, harmonization, dry_run, **kwargs): - """ - Checking for deprecated runtime configurations (stomp collector, cymru parser, ripe expert, collector feed parameter) - """ - mapping = { - "intelmq.bots.collectors.n6.collector_stomp": "intelmq.bots.collectors.stomp.collector", - "intelmq.bots.parsers.cymru_full_bogons.parser": "intelmq.bots.parsers.cymru.parser_full_bogons", - } - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] in mapping: - bot["module"] = mapping[bot["module"]] - changed = True - if bot["module"] == "intelmq.bots.experts.ripencc_abuse_contact.expert": - bot["module"] = "intelmq.bots.experts.ripe.expert" - changed = True - if bot["module"] == "intelmq.bots.experts.ripe.expert": - if bot["parameters"].get("query_ripe_stat"): - if "query_ripe_stat_asn" not in bot["parameters"]: - bot["parameters"]["query_ripe_stat_asn"] = bot["parameters"]["query_ripe_stat"] - if "query_ripe_stat_ip" not in bot["parameters"]: - bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat"] - del bot["parameters"]["query_ripe_stat"] - changed = True - if bot["group"] == 'Collector' and bot["parameters"].get("feed") and not bot["parameters"].get("name"): - try: - bot["parameters"]["name"] = bot["parameters"]["feed"] - del bot["parameters"]["feed"] - except KeyError: - pass - else: - changed = True - - return changed, configuration, harmonization - - -def modify_expert_convert_config(old): - """ - Also used in the modify expert - """ - config = [] - for groupname, group in old.items(): - for rule_name, rule in group.items(): - config.append({"rulename": groupname + ' ' + rule_name, - "if": rule[0], - "then": rule[1]}) - return config - - -def v100_dev7_modify_syntax(configuration, harmonization, dry_run, **kwargs): - """ - Migrate modify bot configuration format - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.experts.modify.expert": - if "configuration_path" in bot["parameters"]: - config = load_configuration(bot["parameters"]["configuration_path"]) - if type(config) is dict: - new_config = modify_expert_convert_config(config) - if len(config) != len(new_config): - return 'Error converting modify expert syntax. Different size of configurations. Please report this.' - changed = True - if dry_run: - print('Would now convert file %r syntax.', - bot["parameters"]["configuration_path"]) - continue - try: - write_configuration(bot["parameters"]["configuration_path"], - new_config) - except PermissionError: - return ('Can\'t update %s\'s configuration: Permission denied.' % bot_id, - configuration, harmonization) - - return changed, configuration, harmonization - - -def v200_defaults_ssl_ca_certificate(configuration, harmonization, dry_run, **kwargs): - """ - Add ssl_ca_certificate to defaults - """ - if "ssl_ca_certificate" not in configuration['global']: - configuration['global']["ssl_ca_certificate"] = None - return True, configuration, harmonization - else: - return None, configuration, harmonization - - -def v111_defaults_process_manager(configuration, harmonization, dry_run, **kwargs): - """ - Fix typo in proccess_manager parameter - """ - changed = None - if "proccess_manager" in configuration['global']: - if "process_manager" in configuration['global']: - del configuration['global']["proccess_manager"] - elif "process_manager" not in configuration['global']: - configuration['global']["process_manager"] = configuration['global']["proccess_manager"] - del configuration['global']["proccess_manager"] - changed = True - else: - if "process_manager" not in configuration['global']: - configuration['global']["process_manager"] = "intelmq" - changed = True - - return changed, configuration, harmonization - - -def v202_fixes(configuration, harmonization, dry_run, **kwargs): - """ - Migrate Collector parameter `feed` to `name`. RIPE expert set `query_ripe_stat_ip` with `query_ripe_stat_asn` as default. - Set cymru whois expert `overwrite` to true. - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["group"] == 'Collector' and bot["parameters"].get("feed"): - try: - bot["parameters"]["name"] = bot["parameters"]["feed"] - del bot["parameters"]["feed"] - except KeyError: - pass - else: - changed = True - if bot["module"] == "intelmq.bots.experts.ripe.expert": - if "query_ripe_stat_asn" in bot["parameters"]: - if "query_ripe_stat_ip" not in bot["parameters"]: - bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat_asn"] - changed = True - if bot["module"] in ("intelmq.bots.experts.cymru_whois.expert", - "intelmq.bots.experts.reverse_dns.expert", - "intelmq.bots.experts.modify.expert"): - if "overwrite" not in bot["parameters"]: - bot["parameters"]["overwrite"] = True - changed = True - - return changed, configuration, harmonization - - -def v210_deprecations(configuration, harmonization, dry_run, **kwargs): - """ - Migrating configuration - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.collectors.rt.collector_rt": - # from 29c4b2c42b126ef51ac7287edc1a9fee28ab27fd to ce96e6d995d420e117a49a22d3bfdea762d899ec - if "extract_files" in bot["parameters"]: - bot["parameters"]["extract_attachment"] = bot["parameters"]["extract_files"] - del bot["parameters"]["extract_files"] - changed = True - if "unzip_attachment" not in bot["parameters"]: - continue - if "extract_files" not in bot["parameters"]: - bot["parameters"]["extract_attachment"] = bot["parameters"]["unzip_attachment"] - del bot["parameters"]["unzip_attachment"] - changed = True - if bot["module"] in ("intelmq.bots.experts.generic_db_lookup.expert", - "intelmq.bots.outputs.postgresql.output"): - if "engine" not in bot["parameters"]: - bot["parameters"]["engine"] = "postgresql" - changed = True - if bot["module"] == "intelmq.bots.outputs.postgresql.output": - bot["module"] = "intelmq.bots.outputs.sql.output" - changed = True - return changed, configuration, harmonization - - -def v213_deprecations(configuration, harmonization, dry_run, **kwargs): - """ - migrate attach_unzip to extract_files for mail attachment collector - - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.collectors.mail.collector_mail_attach": - if "attach_unzip" not in bot["parameters"]: - continue - if "extract_files" in bot["parameters"] and "attach_unzip" in bot["parameters"]: - del bot["parameters"]["attach_unzip"] - changed = True - elif "extract_files" not in bot["parameters"] and "attach_unzip" in bot["parameters"]: - bot["parameters"]["extract_files"] = bot["parameters"]["attach_unzip"] - del bot["parameters"]["attach_unzip"] - changed = True - return changed, configuration, harmonization - - -def v220_configuration(configuration, harmonization, dry_run, **kwargs): - """ - Migrating configuration - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.collectors.misp.collector": - if "misp_verify" not in bot["parameters"]: - continue - if bot["parameters"]["misp_verify"] != configuration['global']["http_verify_cert"]: - bot["parameters"]["http_verify_cert"] = bot["parameters"]["misp_verify"] - del bot["parameters"]["misp_verify"] - changed = True - elif bot["module"] == "intelmq.bots.outputs.elasticsearch.output": - if "elastic_doctype" in bot["parameters"]: - del bot["parameters"]["elastic_doctype"] - return changed, configuration, harmonization - - -def v220_azure_collector(configuration, harmonization, dry_run, **kwargs): - """ - Checking for the Microsoft Azure collector - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.collectors.microsoft.collector_azure": - if "connection_string" not in bot["parameters"]: - changed = ("The Microsoft Azure collector changed backwards-" - "incompatible in IntelMQ 2.2.0. Look at the bot's " - "documentation and NEWS file to adapt the " - "configuration.") - return changed, configuration, harmonization - - -def harmonization(configuration, harmonization, dry_run, **kwargs): - """ - Checks if all harmonization fields and types are correct - """ - changed = None - original = load_configuration(resource_filename('intelmq', - 'etc/harmonization.conf')) - for msg_type, msg in original.items(): - if msg_type not in harmonization: - harmonization[msg_type] = msg - changed = True - continue - for fieldname, field in msg.items(): - if fieldname not in harmonization[msg_type]: - harmonization[msg_type][fieldname] = field - changed = True - continue - if harmonization[msg_type][fieldname]['type'] != original[msg_type][fieldname]['type']: - harmonization[msg_type][fieldname]['type'] = original[msg_type][fieldname]['type'] - changed = True - installed_regex = harmonization[msg_type][fieldname].get('regex') - original_regex = original[msg_type][fieldname].get('regex') - if original_regex and original_regex != installed_regex: - harmonization[msg_type][fieldname]['regex'] = original[msg_type][fieldname]['regex'] - changed = True - installed_regex = harmonization[msg_type][fieldname].get('iregex') - original_regex = original[msg_type][fieldname].get('iregex') - if original_regex and original_regex != installed_regex: - harmonization[msg_type][fieldname]['iregex'] = original[msg_type][fieldname]['iregex'] - changed = True - return changed, configuration, harmonization - - -def v213_feed_changes(configuration, harmonization, dry_run, **kwargs): - """ - Migrates feed configuration for changed feed parameters. - """ - found_zeus = [] - found_bitcash = [] - found_ddos_attack = [] - found_ransomware = [] - found_bambenek = [] - found_nothink = [] - found_nothink_parser = [] - changed = None - messages = [] - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.collectors.http.collector_http": - if "http_url" not in bot["parameters"]: - continue - if bot["parameters"]["http_url"] == 'https://www.tc.edu.tw/net/netflow/lkout/recent/30': - bot["parameters"]["http_url"] = "https://www.tc.edu.tw/net/netflow/lkout/recent/" - changed = True - if bot["parameters"]["http_url"].startswith("https://zeustracker.abuse.ch/"): - found_zeus.append(bot_id) - elif bot["parameters"]["http_url"].startswith("https://bitcash.cz/misc/log/blacklist"): - found_bitcash.append(bot_id) - elif bot["parameters"]["http_url"].startswith("https://ransomwaretracker.abuse.ch/feeds/csv/"): - found_ransomware.append(bot_id) - elif bot["parameters"]["http_url"] == "https://osint.bambenekconsulting.com/feeds/dga-feed.txt": - bot["parameters"]["http_url"] = "https://faf.bambenekconsulting.com/feeds/dga-feed.txt" - changed = True - elif bot["parameters"]["http_url"] in ("http://osing.bambenekconsulting.com/feeds/dga/c2-ipmasterlist.txt", - "https://osing.bambenekconsulting.com/feeds/dga/c2-ipmasterlist.txt", - "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt", - "https://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt"): - found_bambenek.append(bot_id) - elif (bot["parameters"]["http_url"].startswith("http://www.nothink.org/") or - bot["parameters"]["http_url"].startswith("https://www.nothink.org/")): - found_nothink.append(bot_id) - elif bot["module"] == "intelmq.bots.collectors.http.collector_http_stream": - if bot["parameters"].get("http_url", "").startswith("https://feed.caad.fkie.fraunhofer.de/ddosattackfeed"): - found_ddos_attack.append(bot_id) - elif bot['module'] == "intelmq.bots.parsers.nothink.parser": - found_nothink_parser.append(bot_id) - if found_zeus: - messages.append('A discontinued feed "Zeus Tracker" has been found ' - 'as bot %s.' % ', '.join(sorted(found_zeus))) - if found_bitcash: - messages.append('The discontinued feed "Bitcash.cz" has been found ' - 'as bot %s.' % ', '.join(sorted(found_bitcash))) - if found_ddos_attack: - messages.append('The discontinued feed "Fraunhofer DDos Attack" has been found ' - 'as bot %s.' % ', '.join(sorted(found_ddos_attack))) - if found_ransomware: - messages.append('The discontinued feed "Abuse.ch Ransomware Tracker" has been found ' - 'as bot %s.' % ', '.join(sorted(found_ransomware))) - if found_bambenek: - messages.append('Many Bambenek feeds now require a license, see https://osint.bambenekconsulting.com/feeds/' - ' potentially affected bots are %s.' % ', '.join(sorted(found_bambenek))) - if found_nothink: - messages.append('All Nothink Honeypot feeds are discontinued, ' - 'potentially affected bots are %s.' % ', '.join(sorted(found_nothink))) - if found_nothink_parser: - messages.append('The Nothink Parser has been removed, ' - 'affected bots are %s.' % ', '.join(sorted(found_nothink_parser))) - messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization - - -def v220_feed_changes(configuration, harmonization, dry_run, **kwargs): - """ - Migrates feed configuration for changed feed parameters. - """ - found_urlvir_feed = [] - found_urlvir_parser = [] - messages = [] - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.collectors.http.collector_http": - if "http_url" not in bot["parameters"]: - continue - if bot["parameters"]["http_url"].startswith("http://www.urlvir.com/export-"): - found_urlvir_feed.append(bot_id) - elif bot['module'] == "intelmq.bots.parsers.urlvir.parser": - found_urlvir_parser.append(bot_id) - if found_urlvir_feed: - messages.append('A discontinued feed "URLVir" has been found ' - 'as bot %s.' % ', '.join(sorted(found_urlvir_feed))) - if found_urlvir_parser: - messages.append('The removed parser "URLVir" has been found ' - 'as bot %s.' % ', '.join(sorted(found_urlvir_parser))) - messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization - - -def v221_feed_changes(configuration, harmonization, dry_run, **kwargs): - """ - Migrates feeds' configuration for changed/fixed parameters. Deprecation of HP Hosts file feed & parser. - """ - found_hphosts_collector = [] - found_hphosts_parser = [] - messages = [] - ULRHAUS_OLD = ['time.source', 'source.url', 'status', 'extra.urlhaus.threat_type', 'source.fqdn', 'source.ip', 'source.asn', 'source.geolocation.cc'] - URLHAUS_NEW = ['time.source', 'source.url', 'status', 'classification.type|__IGNORE__', 'source.fqdn|__IGNORE__', 'source.ip', 'source.asn', 'source.geolocation.cc'] - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.collectors.http.collector_http": - if bot["parameters"].get("http_url", None) == "http://hosts-file.net/download/hosts.txt": - found_hphosts_collector.append(bot_id) - elif bot['module'] == "intelmq.bots.parsers.hphosts.parser": - found_hphosts_parser.append(bot_id) - if bot["module"] == "intelmq.bots.parsers.generic.parser_csv": - if "columns" not in bot["parameters"]: - continue - columns = bot["parameters"]["columns"] - # convert columns to an array - if type(columns) is str: - columns = [column.strip() for column in columns.split(",")] - if columns == ULRHAUS_OLD: - changed = True - bot["parameters"]["columns"] = URLHAUS_NEW - - if found_hphosts_collector: - messages.append('A discontinued feed "HP Hosts File" has been found ' - 'as bot %s.' % ', '.join(sorted(found_hphosts_collector))) - if found_hphosts_parser: - messages.append('The removed parser "HP Hosts" has been found ' - 'as bot %s.' % ', '.join(sorted(found_hphosts_parser))) - messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization - - -def v222_feed_changes(configuration, harmonization, dry_run, **kwargs): - """ - Migrate Shadowserver feed name - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.parsers.shadowserver.parser": - if bot["parameters"].get("feedname", None) == "Blacklisted-IP": - bot["parameters"]["feedname"] = "Blocklist" - changed = True - return changed, configuration, harmonization - - -def v230_csv_parser_parameter_fix(configuration, harmonization, dry_run, **kwargs): - """ - Fix CSV parser parameter misspelling - """ - changed = None - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.parsers.generic.parser_csv": - if "delimeter" in bot["parameters"] and "delimiter" in bot["parameters"]: - del bot["parameters"]["delimeter"] - changed = True - elif "delimeter" in bot["parameters"]: - bot["parameters"]["delimiter"] = bot["parameters"]["delimeter"] - del bot["parameters"]["delimeter"] - changed = True - return changed, configuration, harmonization - - -def v230_deprecations(configuration, harmonization, dry_run, **kwargs): - """ - Deprecate malwaredomainlist parser - """ - found_malwaredomainlistparser = [] - messages = [] - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.parsers.malwaredomainlist.parser": - found_malwaredomainlistparser.append(bot_id) - if found_malwaredomainlistparser: - messages.append('A discontinued bot "Malware Domain List Parser" has been found ' - 'as bot %s.' % ', '.join(sorted(found_malwaredomainlistparser))) - messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization - - -def v230_feed_changes(configuration, harmonization, dry_run, **kwargs): - """ - Migrates feeds' configuration for changed/fixed parameter - """ - found_malwaredomainlist = [] - messages = [] - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.collectors.http.collector_http": - if "http_url" not in bot["parameters"]: - continue - if bot["parameters"]["http_url"].startswith("http://www.malwaredomainlist.com/updatescsv.php"): - found_malwaredomainlist.append(bot_id) - if found_malwaredomainlist: - messages.append('A discontinued feed "Malware Domain List" has been found ' - 'as bot %s.' % ', '.join(sorted(found_malwaredomainlist))) - messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization - - -def v300_bots_file_removal(configuration, harmonization, dry_run, **kwargs): - """ - Remove BOTS file - """ - changed = None - messages = [] - bots_file = Path(CONFIG_DIR) / "BOTS" - if bots_file.exists(): - if dry_run: - print(f'Would now remove file {bots_file!r}.') - else: - bots_file.unlink() - changed = True - messages = ' '.join(messages) - return messages if messages else changed, configuration, harmonization - - -def v300_defaults_file_removal(configuration, harmonization, dry_run, **kwargs): - """ - Remove the defaults.conf file - """ - changed = None - messages = [] - defaults_file = Path(CONFIG_DIR) / "defaults.conf" - if defaults_file.exists(): - if dry_run: - print(f'Would now remove file {defaults_file!r}.') - else: - configuration['global'] = load_configuration(defaults_file) - defaults_file.unlink() - changed = True - messages = ' '.join(messages) - return messages if messages else changed, configuration, harmonization - - -def v233_feodotracker_browse(configuration, harmonization, dry_run, **kwargs): - """ - Migrate Abuse.ch Feodotracker Browser feed parsing parameters - """ - changed = None - old_feodo_columns = 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc' - old_ignore_values = ',,,,Not listed,,' - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - # The parameters can be given as string or list of strings - if (bot["module"] == "intelmq.bots.parsers.html_table.parser" and 'feodo' in bot_id.lower() and - "columns" in bot["parameters"] and "ignore_values" in bot["parameters"] and - (bot["parameters"]["columns"] == old_feodo_columns or bot["parameters"]["columns"] == old_feodo_columns.split(',')) and - (bot["parameters"]["ignore_values"] == old_ignore_values or bot["parameters"]["ignore_values"] == old_ignore_values.split(','))): - bot["parameters"]["columns"] = 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc' - bot["parameters"]['ignore_values'] = ',,,,,' - changed = True - return changed, configuration, harmonization - - -def v300_pipeline_file_removal(configuration, harmonization, dry_run, **kwargs): - """ - Remove the pipeline.conf file - """ - changed = None - messages = [] - pipeline_file = Path(CONFIG_DIR) / "pipeline.conf" - if pipeline_file.exists(): - pipelines = load_configuration(pipeline_file) - for bot in configuration: - if bot == 'global': - continue - if bot in pipelines: - if 'destination-queues' in pipelines[bot]: - destination_queues = pipelines[bot]['destination-queues'] - if isinstance(destination_queues, dict): - configuration[bot]['parameters']['destination_queues'] = destination_queues - if isinstance(destination_queues, list): - configuration[bot]['parameters']['destination_queues'] = {'_default': destination_queues} - if isinstance(destination_queues, str): - configuration[bot]['parameters']['destination_queues'] = {'_default': [destination_queues]} - if 'source-queue' in pipelines[bot]: - if pipelines[bot]['source-queue'] != f"{bot}-queue": - configuration[bot]['parameters']['source_queue'] = pipelines[bot]['source-queue'] - if dry_run: - print(f'Would now remove file {pipeline_file!r}.') - else: - pipeline_file.unlink() - changed = True - messages = ' '.join(messages) - return messages if messages else changed, configuration, harmonization - - -def v301_deprecations(configuration, harmonization, dry_run, **kwargs): - """ - Deprecate malwaredomains parser and collector - """ - found_malwaredomainsparser = [] - found_malwaredomainscollector = [] - changed = None - messages = [] - for bot_id, bot in configuration.items(): - if bot_id == 'global': - continue - if bot["module"] == "intelmq.bots.parsers.malwaredomains.parser": - found_malwaredomainsparser.append(bot_id) - if bot["module"] == "intelmq.bots.collectors.http.collector": - if "http_url" not in bot["parameters"]: - continue - if bot["parameters"]["http_url"] == 'http://mirror1.malwaredomains.com/files/domains.txt': - found_malwaredomainscollector.append(bot_id) - if found_malwaredomainsparser: - messages.append('A discontinued bot "Malware Domains Parser" has been found ' - 'as bot %s.' % ', '.join(sorted(found_malwaredomainsparser))) - if found_malwaredomainscollector: - messages.append('A discontinued bot "Malware Domains Collector" has been found ' - 'as bot %s.' % ', '.join(sorted(found_malwaredomainscollector))) - messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization +from intelmq.lib.upgrade import harmonization, v100_dev7, v110, v111, v112, v200, v202, v210, v213, v220, v221, v222, v230, v233, v300, v301 UPGRADES = OrderedDict([ - ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax, )), - ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)), - ((1, 1, 1), (v111_defaults_process_manager, )), - ((1, 1, 2), (v112_feodo_tracker_ips, v112_feodo_tracker_domains, )), - ((2, 0, 0), (v200_defaults_statistics, v200_defaults_broker, - v200_defaults_ssl_ca_certificate)), + ((1, 0, 0, 'dev7'), (v100_dev7.modify_syntax)), + ((1, 1, 0), (v110.shadowserver_feednames, v110.deprecations)), + ((1, 1, 1), (v111.defaults_process_manager, )), + ((1, 1, 2), (v112.feodo_tracker_ips, v112.feodo_tracker_domains, )), + ((2, 0, 0), (v200.defaults_statistics, v200.defaults_broker, + v200.defaults_ssl_ca_certificate)), ((2, 0, 1), ()), - ((2, 0, 2), (v202_fixes, )), - ((2, 1, 0), (v210_deprecations, )), + ((2, 0, 2), (v202.fixes, )), + ((2, 1, 0), (v210.deprecations, )), ((2, 1, 1), ()), ((2, 1, 2), ()), - ((2, 1, 3), (v213_deprecations, v213_feed_changes)), - ((2, 2, 0), (v220_configuration, v220_azure_collector, v220_feed_changes)), - ((2, 2, 1), (v221_feed_changes, )), - ((2, 2, 2), (v222_feed_changes, )), + ((2, 1, 3), (v213.deprecations, v213.deprecations)), + ((2, 2, 0), (v220.configuration, v220.azure_collector, v220.feed_changes)), + ((2, 2, 1), (v221.feed_changes, )), + ((2, 2, 2), (v222.feed_changes, )), ((2, 2, 3), ()), - ((2, 3, 0), (v230_csv_parser_parameter_fix, v230_feed_changes, v230_deprecations,)), + ((2, 3, 0), (v230.csv_parser_parameter_fix, v230.feed_changes, v230.deprecations,)), ((2, 3, 1), ()), ((2, 3, 2), ()), - ((2, 3, 3), (v233_feodotracker_browse, )), - ((3, 0, 0), (v300_bots_file_removal, v300_defaults_file_removal, v300_pipeline_file_removal, )), - ((3, 0, 1), (v301_deprecations, )), + ((2, 3, 3), (v233.feodotracker_browse, )), + ((3, 0, 0), (v300.bots_file_removal, v300.defaults_file_removal, v300.pipeline_file_removal, )), + ((3, 0, 1), (v301.deprecations, )), ((3, 0, 2), ()), ((3, 1, 0), ()), ]) -ALWAYS = (harmonization, ) +ALWAYS = (harmonization.harmonization, ) diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py deleted file mode 100644 index 8a6e13b21..000000000 --- a/intelmq/tests/lib/test_upgrades.py +++ /dev/null @@ -1,727 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -""" -Tests the upgrade functions. -""" -import unittest -import pkg_resources -from copy import deepcopy - -import intelmq.lib.upgrades as upgrades -from intelmq.lib.utils import load_configuration - - -V202 = {"global": {}, -"test-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "feed": "Feed" - } -}, - "ripe-expert": { - "group": "Expert", - "module": "intelmq.bots.experts.ripe.expert", - "parameters": { - "query_ripe_stat_asn": True, - }, -}, - "reversedns-expert": { - "group": "Expert", - "module": "intelmq.bots.experts.reverse_dns.expert", - "parameters": { - }, -}, -} -V202_EXP = {"global": {}, -"test-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "name": "Feed" - } -}, - "ripe-expert": { - "group": "Expert", - "module": "intelmq.bots.experts.ripe.expert", - "parameters": { - "query_ripe_stat_asn": True, - "query_ripe_stat_ip": True, - }, -}, - "reversedns-expert": { - "group": "Expert", - "module": "intelmq.bots.experts.reverse_dns.expert", - "parameters": { - "overwrite": True, - }, -}, -} - -DEP_110 = {"global": {}, -"n6-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.n6.collector_stomp", - "parameters": { - "feed": "Feed", - }, -}, - "cymru-full-bogons-parser": { - "group": "Parser", - "module": "intelmq.bots.parsers.cymru_full_bogons.parser", - "parameters": { - }, -}, - "ripe-expert": { - "group": "Expert", - "module": "intelmq.bots.experts.ripencc_abuse_contact.expert", - "parameters": { - "query_ripe_stat": True, - }, -} -} -DEP_110_EXP = {"global": {}, -"n6-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.stomp.collector", - "parameters": { - "name": "Feed", - }, -}, - "cymru-full-bogons-parser": { - "group": "Parser", - "module": "intelmq.bots.parsers.cymru.parser_full_bogons", - "parameters": { - }}, - "ripe-expert": { - "group": "Expert", - "module": "intelmq.bots.experts.ripe.expert", - "parameters": { - "query_ripe_stat_asn": True, - "query_ripe_stat_ip": True, - }, -}} -V210 = {"global": {}, -"test-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.rt.collector_rt", - "parameters": { - "unzip_attachment": True, - } -}, - "test-collector2": { - "group": "Collector", - "module": "intelmq.bots.collectors.rt.collector_rt", - "parameters": { - }, -}, - "postgresql-output": { - "group": "Output", - "module": "intelmq.bots.outputs.postgresql.output", - "parameters": { - "autocommit": True, - "database": "intelmq-events", - "host": "localhost", - "jsondict_as_string": True, - "password": "", - "port": "5432", - "sslmode": "require", - "table": "events", - "user": "intelmq" - }, -}, - "db-lookup": { - "module": "intelmq.bots.experts.generic_db_lookup.expert", - "parameters": { - "database": "intelmq", - "host": "localhost", - "match_fields": { - "source.asn": "asn" - }, - "overwrite": False, - "password": "", - "port": "5432", - "replace_fields": { - "contact": "source.abuse_contact", - "note": "comment" - }, - "sslmode": "require", - "table": "contacts", - "user": "intelmq" - } -} -} -V210_EXP = {"global": {}, -"test-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.rt.collector_rt", - "parameters": { - "extract_attachment": True, - } -}, - "test-collector2": { - "group": "Collector", - "module": "intelmq.bots.collectors.rt.collector_rt", - "parameters": { - }, -}, - "postgresql-output": { - "group": "Output", - "module": "intelmq.bots.outputs.sql.output", - "parameters": { - "autocommit": True, - "database": "intelmq-events", - "engine": "postgresql", - "host": "localhost", - "jsondict_as_string": True, - "password": "", - "port": "5432", - "sslmode": "require", - "table": "events", - "user": "intelmq" - }, -}, - "db-lookup": { - "module": "intelmq.bots.experts.generic_db_lookup.expert", - "parameters": { - "engine": "postgresql", - "database": "intelmq", - "host": "localhost", - "match_fields": { - "source.asn": "asn" - }, - "overwrite": False, - "password": "", - "port": "5432", - "replace_fields": { - "contact": "source.abuse_contact", - "note": "comment" - }, - "sslmode": "require", - "table": "contacts", - "user": "intelmq" - } -} -} -V213 = {"global": {}, -"mail-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.mail.collector_mail_attach", - "parameters": { - "attach_unzip": True, - } -}, - "mail-collector2": { - "group": "Collector", - "module": "intelmq.bots.collectors.mail.collector_mail_attach", - "parameters": { - "attach_unzip": False, - "extract_files": True, - } -} -} -V213_EXP = {"global": {}, -"mail-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.mail.collector_mail_attach", - "parameters": { - "extract_files": True, - } -}, - "mail-collector2": { - "group": "Collector", - "module": "intelmq.bots.collectors.mail.collector_mail_attach", - "parameters": { - "extract_files": True, - }, -} -} -V220_MISP_VERIFY_FALSE = { -"global": {"http_verify_cert": True}, -"misp-collector": { - "module": "intelmq.bots.collectors.misp.collector", - "parameters": { - "misp_verify": False}}} -V220_MISP_VERIFY_NULL = { -"global": {"http_verify_cert": True}, -"misp-collector": { - "module": "intelmq.bots.collectors.misp.collector", - "parameters": {}}} -V220_MISP_VERIFY_TRUE = { -"global": {"http_verify_cert": True}, -"misp-collector": { - "module": "intelmq.bots.collectors.misp.collector", - "parameters": { - "misp_verify": True}}} -V220_HTTP_VERIFY_FALSE = { -"global": {"http_verify_cert": True}, -"misp-collector": { - "module": "intelmq.bots.collectors.misp.collector", - "parameters": { - "http_verify_cert": False}}} -HARM = load_configuration(pkg_resources.resource_filename('intelmq', - 'etc/harmonization.conf')) -V210_HARM = deepcopy(HARM) -del V210_HARM['report']['extra'] -MISSING_REPORT = deepcopy(HARM) -del MISSING_REPORT['report'] -WRONG_TYPE = deepcopy(HARM) -WRONG_TYPE['event']['source.asn']['type'] = 'String' -WRONG_REGEX = deepcopy(HARM) -WRONG_REGEX['event']['protocol.transport']['iregex'] = 'foobar' -V213_FEED = {"global": {}, -"zeus-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "https://zeustracker.abuse.ch/blocklist.php?download=badips", - } -}, -"bitcash-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "https://bitcash.cz/misc/log/blacklist", - } -}, -"ddos-attack-c2-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http_stream", - "parameters": { - "http_url": "https://feed.caad.fkie.fraunhofer.de/ddosattackfeed/", - } -}, -"ddos-attack-targets-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http_stream", - "parameters": { - "http_url": "https://feed.caad.fkie.fraunhofer.de/ddosattackfeed/", - } -}, -"taichung-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "https://www.tc.edu.tw/net/netflow/lkout/recent/30", - }, -}, -"ransomware-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "https://ransomwaretracker.abuse.ch/feeds/csv/", - }, -}, -"bambenek-dga-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "https://osint.bambenekconsulting.com/feeds/dga-feed.txt", - }, -}, -"bambenek-c2dommasterlist-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt", - }, -}, -"nothink-dns-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "http://www.nothink.org/honeypot_dns_attacks.txt", - }, -}, -"nothink-ssh-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "http://www.nothink.org/blacklist/blacklist_ssh_day.txt", - }, -}, -"nothink-parser": { - "group": "Parser", - "module": "intelmq.bots.parsers.nothink.parser", -}, -} -V220_FEED = {"global": {}, -"urlvir-hosts-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "http://www.urlvir.com/export-hosts/", - }, -}, -"urlvir-parser": { - "group": "Parser", - "module": "intelmq.bots.parsers.urlvir.parser", -}, -} -V221_FEED = {"global": {}, -"abusech-urlhaus-columns-string-parser": { - "parameters": { - "column_regex_search": {}, - "columns": "time.source,source.url,status,extra.urlhaus.threat_type,source.fqdn,source.ip,source.asn,source.geolocation.cc", - "default_url_protocol": "http://", - "delimiter": ",", - "filter_text": None, - "filter_type": None, - "skip_header": False, - "time_format": None, - "type": "c2server", - "type_translation": { - "malware_download": "malware-distribution" - } - }, - "module": "intelmq.bots.parsers.generic.parser_csv", -}, -"abusech-urlhaus-columns-dict-parser": { - "parameters": { - "column_regex_search": {}, - "columns": ["time.source", "source.url","status","extra.urlhaus.threat_type","source.fqdn","source.ip","source.asn","source.geolocation.cc"], - "default_url_protocol": "http://", - "delimiter": ",", - "filter_text": None, - "filter_type": None, - "skip_header": False, - "time_format": None, - "type": "c2server", - "type_translation": { - "malware_download": "malware-distribution" - } - }, - "module": "intelmq.bots.parsers.generic.parser_csv", -} -} -V221_FEED_OUT = {"global": {}, -"abusech-urlhaus-columns-string-parser": { - "parameters": { - "column_regex_search": {}, - "columns": ['time.source', 'source.url', 'status', 'classification.type|__IGNORE__', 'source.fqdn|__IGNORE__', 'source.ip', 'source.asn', 'source.geolocation.cc'], - "default_url_protocol": "http://", - "delimiter": ",", - "filter_text": None, - "filter_type": None, - "skip_header": False, - "time_format": None, - "type": "c2server", - "type_translation": { - "malware_download": "malware-distribution" - } - }, - "module": "intelmq.bots.parsers.generic.parser_csv", -} -} -V221_FEED_OUT['abusech-urlhaus-columns-dict-parser'] = V221_FEED_OUT['abusech-urlhaus-columns-string-parser'] -V221_FEED_2 = {"global": {}, -"hphosts-collector": { - "group": "Collector", - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "http://hosts-file.net/download/hosts.txt", - }, -}, -"hphosts-parser": { - "group": "Parser", - "module": "intelmq.bots.parsers.hphosts.parser", -}, -} -V222 = { -"global": {}, -"shadowserver-parser": { - "module": "intelmq.bots.parsers.shadowserver.parser", - "parameters": { - "feedname": "Blacklisted-IP"}}} -V222_OUT = { -"global": {}, -"shadowserver-parser": { - "module": "intelmq.bots.parsers.shadowserver.parser", - "parameters": { - "feedname": "Blocklist"}}} - -V230_IN = { -"global": {}, -"urlhaus-parser": { - "module": "intelmq.bots.parsers.generic.parser_csv", - "parameters": { - "delimeter": "," - } -} -} -V230_IN_BOTH = { -"global": {}, -"urlhaus-parser": { - "module": "intelmq.bots.parsers.generic.parser_csv", - "parameters": { - "delimeter": ",", - "delimiter": "," - } -} -} -V230_OUT = { -"global": {}, -"urlhaus-parser": { - "module": "intelmq.bots.parsers.generic.parser_csv", - "parameters": { - "delimiter": "," - } -} -} -V230_MALWAREDOMAINLIST_IN = { -"global": {}, -"malwaredomainlist-parser": { - "module": "intelmq.bots.parsers.malwaredomainlist.parser", - "parameters": { - } -}, -"malwaredomainlist-collector": { - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "http://www.malwaredomainlist.com/updatescsv.php" - } - } -} -V233_FEODOTRACKER_BROWSE_IN = { -"global": {}, -'Feodo-tracker-browse-parser': { - 'module': "intelmq.bots.parsers.html_table.parser", - 'parameters': { - 'columns': 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc'.split(','), - 'type': 'c2server', - 'ignore_values': ',,,,Not listed,,', - 'skip_table_head': True, - } -} -} -V233_FEODOTRACKER_BROWSE_OUT = { -"global": {}, -'Feodo-tracker-browse-parser': { - 'module': "intelmq.bots.parsers.html_table.parser", - 'parameters': { - 'columns': 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc', - 'type': 'c2server', - 'ignore_values': ',,,,,', - 'skip_table_head': True, - } -} -} -V301_MALWAREDOMAINS_IN = { -"global": {}, -"malwaredomains-parser": { - "module": "intelmq.bots.parsers.malwaredomains.parser", - "parameters": { - } -}, -"malwaredomains-collector": { - "module": "intelmq.bots.collectors.http.collector", - "parameters": { - "http_url": "http://mirror1.malwaredomains.com/files/domains.txt" - } - } -} -def generate_function(function): - def test_function(self): - """ Test if no errors happen for upgrade function %s. """ % function.__name__ - function({'global': {}}, {}, dry_run=True, - version_history=()) - return test_function - - -class TestUpgradeLib(unittest.TestCase): - def setUp(self): - self.allfs = upgrades.__all__ - self.modulefs = [f for f in dir(upgrades) if f.startswith('v')] - self.mapping_list = [] - self.mapping_list_name = [] - for values in upgrades.UPGRADES.values(): - self.mapping_list.extend(x for x in values) - self.mapping_list_name.extend(x.__name__ for x in values) - - def test_all_functions_used(self): - self.assertEqual(len(self.mapping_list_name), - len(set(self.mapping_list_name)), - msg='Some function is assigned to multiple versions.') - self.assertEqual(set(self.allfs), - set(self.mapping_list_name), - msg='v* functions in the module do not ' - 'match the mapping.') - self.assertEqual(set(self.allfs), set(self.modulefs), - msg='v* functions in the module do not ' - 'match functions in __all__.') - - def test_v110_deprecations(self): - """ Test v110_deprecations """ - result = upgrades.v110_deprecations(DEP_110, {}, False) - self.assertTrue(result[0]) - self.assertEqual(DEP_110_EXP, result[1]) - - def test_v202_fixes(self): - """ Test v202_feed_name """ - result = upgrades.v202_fixes(V202, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V202_EXP, result[1]) - - def test_v210_deprecations(self): - """ Test v210_deprecations """ - result = upgrades.v210_deprecations(V210, {}, True) - self.assertTrue(result[0]) - self.assertEqual(V210_EXP, result[1]) - - def test_harmonization(self): - """ Test harmonization: Addition of extra to report """ - result = upgrades.harmonization({}, V210_HARM, False) - self.assertTrue(result[0]) - self.assertEqual(HARM, result[2]) - - def test_v220_configuration(self): - """ Test v220_configuration. """ - result = upgrades.v220_configuration(V220_MISP_VERIFY_TRUE, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V220_MISP_VERIFY_NULL, result[1]) - result = upgrades.v220_configuration(V220_MISP_VERIFY_FALSE, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V220_HTTP_VERIFY_FALSE, result[1]) - - def test_missing_report_harmonization(self): - """ Test missing report in harmonization """ - result = upgrades.harmonization({}, MISSING_REPORT, False) - self.assertTrue(result[0]) - self.assertEqual(HARM, result[2]) - - def test_wrong_type_harmonization(self): - """ Test wrong type in harmonization """ - result = upgrades.harmonization({}, WRONG_TYPE, False) - self.assertTrue(result[0]) - self.assertEqual(HARM, result[2]) - - def test_wrong_regex_harmonization(self): - """ Test wrong regex in harmonization """ - result = upgrades.harmonization({}, WRONG_REGEX, False) - self.assertTrue(result[0]) - self.assertEqual(HARM, result[2]) - - def test_v213_deprecations(self): - """ Test v213_fixes """ - result = upgrades.v213_deprecations(V213, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V213_EXP, result[1]) - - def test_v213_feed_changes(self): - """ Test v213_feed_changes """ - result = upgrades.v213_feed_changes(V213_FEED, {}, False) - self.assertEqual('A discontinued feed "Zeus Tracker" has been found ' - 'as bot zeus-collector. ' - 'The discontinued feed "Bitcash.cz" has been found ' - 'as bot bitcash-collector. ' - 'The discontinued feed "Fraunhofer DDos Attack" has ' - 'been found as bot ddos-attack-c2-collector, ' - 'ddos-attack-targets-collector. ' - 'The discontinued feed "Abuse.ch Ransomware Tracker" ' - 'has been found as bot ransomware-collector. ' - 'Many Bambenek feeds now require a license, see ' - 'https://osint.bambenekconsulting.com/feeds/ ' - 'potentially affected bots are ' - 'bambenek-c2dommasterlist-collector. ' - 'All Nothink Honeypot feeds are discontinued, ' - 'potentially affected bots are nothink-dns-collector, ' - 'nothink-ssh-collector. ' - 'The Nothink Parser has been removed, ' - 'affected bots are nothink-parser. ' - 'Remove affected bots yourself.', - result[0]) - self.assertEqual(V213_FEED, result[1]) - - def test_v220_feed_changes(self): - """ Test v213_feed_changes """ - result = upgrades.v220_feed_changes(V220_FEED, {}, False) - self.assertEqual('A discontinued feed "URLVir" has been found ' - 'as bot urlvir-hosts-collector. ' - 'The removed parser "URLVir" has been found ' - 'as bot urlvir-parser. ' - 'Remove affected bots yourself.', - result[0]) - self.assertEqual(V220_FEED, result[1]) - - def test_v221_feed_changes(self): - """ Test v221_feeds_1 """ - result = upgrades.v221_feed_changes(V221_FEED, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V221_FEED_OUT, result[1]) - - def test_v221_feed_changes_2(self): - """ Test v213_feed_changes """ - result = upgrades.v221_feed_changes(V221_FEED_2, {}, False) - self.assertEqual('A discontinued feed "HP Hosts File" has been found ' - 'as bot hphosts-collector. ' - 'The removed parser "HP Hosts" has been found ' - 'as bot hphosts-parser. ' - 'Remove affected bots yourself.', - result[0]) - self.assertEqual(V221_FEED_2, result[1]) - - def test_v222_feed_changes(self): - """ Test v222_feed_changes """ - result = upgrades.v222_feed_changes(V222, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V222_OUT, result[1]) - - def test_v230_csv_parser_parameter_fix(self): - """ Test v230_feed_fix """ - result = upgrades.v230_csv_parser_parameter_fix(V230_IN, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V230_OUT, result[1]) - - # with also the new fixed parameter - result = upgrades.v230_csv_parser_parameter_fix(V230_IN_BOTH, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V230_OUT, result[1]) - - # with new parameter, no change - result = upgrades.v230_csv_parser_parameter_fix(V230_OUT, {}, False) - self.assertIsNone(result[0]) - self.assertEqual(V230_OUT, result[1]) - - def test_v230_deprecations(self): - """ Test v230_deprecations """ - result = upgrades.v230_deprecations(V230_MALWAREDOMAINLIST_IN, {}, False) - self.assertTrue(result[0]) - self.assertEqual('A discontinued bot "Malware Domain List Parser" has been found as bot ' - 'malwaredomainlist-parser. Remove affected bots yourself.', - result[0]) - self.assertEqual(V230_MALWAREDOMAINLIST_IN, result[1]) - - def test_v230_feed_changes(self): - """ Test v230_feed_changes """ - result = upgrades.v230_feed_changes(V230_MALWAREDOMAINLIST_IN, {}, False) - self.assertTrue(result[0]) - self.assertEqual('A discontinued feed "Malware Domain List" has been found as bot ' - 'malwaredomainlist-collector. Remove affected bots yourself.', - result[0]) - self.assertEqual(V230_MALWAREDOMAINLIST_IN, result[1]) - - def test_v233_feodotracker_browse(self): - """ Test v233_feodotracker_browse """ - result = upgrades.v233_feodotracker_browse(V233_FEODOTRACKER_BROWSE_IN, {}, False) - self.assertTrue(result[0]) - self.assertEqual(V233_FEODOTRACKER_BROWSE_OUT, result[1]) - - def test_v301_feed_changes(self): - """ Test v301_feed_changes """ - result = upgrades.v301_deprecations(V301_MALWAREDOMAINS_IN, {}, False) - self.assertTrue(result[0]) - self.assertEqual('A discontinued bot "Malware Domains Parser" has been found as bot ' - 'malwaredomains-parser. A discontinued bot "Malware Domains Collector" ' - 'has been found as bot malwaredomains-collector. Remove affected bots yourself.', - result[0]) - self.assertEqual(V301_MALWAREDOMAINS_IN, result[1]) - -for name in upgrades.__all__: - setattr(TestUpgradeLib, 'test_function_%s' % name, - generate_function(getattr(upgrades, name))) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/lib/upgrades/__init__.py b/intelmq/tests/lib/upgrades/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/tests/lib/upgrades/test_harmonization.py b/intelmq/tests/lib/upgrades/test_harmonization.py new file mode 100644 index 000000000..93be48e2f --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_harmonization.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +from copy import deepcopy +import unittest + +import pkg_resources + +from intelmq.lib.upgrade.harmonization import harmonization +from intelmq.lib.utils import load_configuration + + +HARM = load_configuration(pkg_resources.resource_filename('intelmq', + 'etc/harmonization.conf')) +MISSING_REPORT = deepcopy(HARM) +del MISSING_REPORT['report'] +WRONG_TYPE = deepcopy(HARM) +WRONG_TYPE['event']['source.asn']['type'] = 'String' +WRONG_REGEX = deepcopy(HARM) +WRONG_REGEX['event']['protocol.transport']['iregex'] = 'foobar' + + +class TestUpgradeHarmonization(unittest.TestCase): + def test_missing_report_harmonization(self): + """ Test missing report in harmonization """ + result = harmonization({}, MISSING_REPORT, False) + self.assertTrue(result[0]) + self.assertEqual(HARM, result[2]) + + def test_wrong_type_harmonization(self): + """ Test wrong type in harmonization """ + result = harmonization({}, WRONG_TYPE, False) + self.assertTrue(result[0]) + self.assertEqual(HARM, result[2]) + + def test_wrong_regex_harmonization(self): + """ Test wrong regex in harmonization """ + result = harmonization({}, WRONG_REGEX, False) + self.assertTrue(result[0]) + self.assertEqual(HARM, result[2]) diff --git a/intelmq/tests/lib/upgrades/test_v110.py b/intelmq/tests/lib/upgrades/test_v110.py new file mode 100644 index 000000000..246d03487 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v110.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest +from intelmq.lib.upgrade.v110 import deprecations + + +DEP_110 = {"global": {}, + "n6-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.n6.collector_stomp", + "parameters": { + "feed": "Feed", + }, +}, + "cymru-full-bogons-parser": { + "group": "Parser", + "module": "intelmq.bots.parsers.cymru_full_bogons.parser", + "parameters": { + }, +}, + "ripe-expert": { + "group": "Expert", + "module": "intelmq.bots.experts.ripencc_abuse_contact.expert", + "parameters": { + "query_ripe_stat": True, + }, +} +} + +DEP_110_EXP = {"global": {}, + "n6-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.stomp.collector", + "parameters": { + "name": "Feed", + }, +}, + "cymru-full-bogons-parser": { + "group": "Parser", + "module": "intelmq.bots.parsers.cymru.parser_full_bogons", + "parameters": { + }}, + "ripe-expert": { + "group": "Expert", + "module": "intelmq.bots.experts.ripe.expert", + "parameters": { + "query_ripe_stat_asn": True, + "query_ripe_stat_ip": True, + }, +} +} + + +class TestUpgradeV110(unittest.TestCase): + def test_deprecations(self): + """ Test deprecations """ + result = deprecations(DEP_110, {}, False) + self.assertTrue(result[0]) + self.assertEqual(DEP_110_EXP, result[1]) diff --git a/intelmq/tests/lib/upgrades/test_v202.py b/intelmq/tests/lib/upgrades/test_v202.py new file mode 100644 index 000000000..5730512d1 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v202.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest + +from intelmq.lib.upgrade.v202 import fixes + + +V202 = {"global": {}, + "test-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "feed": "Feed" + } +}, + "ripe-expert": { + "group": "Expert", + "module": "intelmq.bots.experts.ripe.expert", + "parameters": { + "query_ripe_stat_asn": True, + }, +}, + "reversedns-expert": { + "group": "Expert", + "module": "intelmq.bots.experts.reverse_dns.expert", + "parameters": { + }, +}, +} +V202_EXP = {"global": {}, + "test-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "name": "Feed" + } +}, + "ripe-expert": { + "group": "Expert", + "module": "intelmq.bots.experts.ripe.expert", + "parameters": { + "query_ripe_stat_asn": True, + "query_ripe_stat_ip": True, + }, +}, + "reversedns-expert": { + "group": "Expert", + "module": "intelmq.bots.experts.reverse_dns.expert", + "parameters": { + "overwrite": True, + }, +}, +} + + +class TestUpgradeV202(unittest.TestCase): + def test_fixes(self): + """ Test fixes """ + result = fixes(V202, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V202_EXP, result[1]) diff --git a/intelmq/tests/lib/upgrades/test_v210.py b/intelmq/tests/lib/upgrades/test_v210.py new file mode 100644 index 000000000..52d8ea5d1 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v210.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +from copy import deepcopy +import unittest +import pkg_resources +from intelmq.lib.upgrade.harmonization import harmonization +from intelmq.lib.upgrade.v210 import deprecations +from intelmq.lib.utils import load_configuration + + +V210 = {"global": {}, + "test-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.rt.collector_rt", + "parameters": { + "unzip_attachment": True, + } +}, + "test-collector2": { + "group": "Collector", + "module": "intelmq.bots.collectors.rt.collector_rt", + "parameters": { + }, +}, + "postgresql-output": { + "group": "Output", + "module": "intelmq.bots.outputs.postgresql.output", + "parameters": { + "autocommit": True, + "database": "intelmq-events", + "host": "localhost", + "jsondict_as_string": True, + "password": "", + "port": "5432", + "sslmode": "require", + "table": "events", + "user": "intelmq" + }, +}, + "db-lookup": { + "module": "intelmq.bots.experts.generic_db_lookup.expert", + "parameters": { + "database": "intelmq", + "host": "localhost", + "match_fields": { + "source.asn": "asn" + }, + "overwrite": False, + "password": "", + "port": "5432", + "replace_fields": { + "contact": "source.abuse_contact", + "note": "comment" + }, + "sslmode": "require", + "table": "contacts", + "user": "intelmq" + } +} +} +V210_EXP = {"global": {}, + "test-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.rt.collector_rt", + "parameters": { + "extract_attachment": True, + } +}, + "test-collector2": { + "group": "Collector", + "module": "intelmq.bots.collectors.rt.collector_rt", + "parameters": { + }, +}, + "postgresql-output": { + "group": "Output", + "module": "intelmq.bots.outputs.sql.output", + "parameters": { + "autocommit": True, + "database": "intelmq-events", + "engine": "postgresql", + "host": "localhost", + "jsondict_as_string": True, + "password": "", + "port": "5432", + "sslmode": "require", + "table": "events", + "user": "intelmq" + }, +}, + "db-lookup": { + "module": "intelmq.bots.experts.generic_db_lookup.expert", + "parameters": { + "engine": "postgresql", + "database": "intelmq", + "host": "localhost", + "match_fields": { + "source.asn": "asn" + }, + "overwrite": False, + "password": "", + "port": "5432", + "replace_fields": { + "contact": "source.abuse_contact", + "note": "comment" + }, + "sslmode": "require", + "table": "contacts", + "user": "intelmq" + } +} +} + +HARM = load_configuration(pkg_resources.resource_filename('intelmq', + 'etc/harmonization.conf')) +V210_HARM = deepcopy(HARM) +del V210_HARM['report']['extra'] + + +class TestUpgradeV210(unittest.TestCase): + def test_deprecations(self): + """ Test deprecations """ + result = deprecations(V210, {}, True) + self.assertTrue(result[0]) + self.assertEqual(V210_EXP, result[1]) + + def test_harmonization(self): + """ Test harmonization: Addition of extra to report """ + result = harmonization({}, V210_HARM, False) + self.assertTrue(result[0]) + self.assertEqual(HARM, result[2]) diff --git a/intelmq/tests/lib/upgrades/test_v213.py b/intelmq/tests/lib/upgrades/test_v213.py new file mode 100644 index 000000000..15eb30ba5 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v213.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest + +from intelmq.lib.upgrade.v213 import deprecations, feed_changes + + +V213 = {"global": {}, + "mail-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.mail.collector_mail_attach", + "parameters": { + "attach_unzip": True, + } +}, + "mail-collector2": { + "group": "Collector", + "module": "intelmq.bots.collectors.mail.collector_mail_attach", + "parameters": { + "attach_unzip": False, + "extract_files": True, + } +} +} +V213_EXP = {"global": {}, + "mail-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.mail.collector_mail_attach", + "parameters": { + "extract_files": True, + } +}, + "mail-collector2": { + "group": "Collector", + "module": "intelmq.bots.collectors.mail.collector_mail_attach", + "parameters": { + "extract_files": True, + }, +} +} + +V213_FEED = {"global": {}, + "zeus-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "https://zeustracker.abuse.ch/blocklist.php?download=badips", + } +}, + "bitcash-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "https://bitcash.cz/misc/log/blacklist", + } +}, + "ddos-attack-c2-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http_stream", + "parameters": { + "http_url": "https://feed.caad.fkie.fraunhofer.de/ddosattackfeed/", + } +}, + "ddos-attack-targets-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http_stream", + "parameters": { + "http_url": "https://feed.caad.fkie.fraunhofer.de/ddosattackfeed/", + } +}, + "taichung-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "https://www.tc.edu.tw/net/netflow/lkout/recent/30", + }, +}, + "ransomware-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "https://ransomwaretracker.abuse.ch/feeds/csv/", + }, +}, + "bambenek-dga-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "https://osint.bambenekconsulting.com/feeds/dga-feed.txt", + }, +}, + "bambenek-c2dommasterlist-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt", + }, +}, + "nothink-dns-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "http://www.nothink.org/honeypot_dns_attacks.txt", + }, +}, + "nothink-ssh-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "http://www.nothink.org/blacklist/blacklist_ssh_day.txt", + }, +}, + "nothink-parser": { + "group": "Parser", + "module": "intelmq.bots.parsers.nothink.parser", +}, +} + + +class TestUpgradeV213(unittest.TestCase): + def test_deprecations(self): + """ Test fixes """ + result = deprecations(V213, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V213_EXP, result[1]) + + def test_feed_changes(self): + """ Test feed_changes """ + result = feed_changes(V213_FEED, {}, False) + self.assertEqual('A discontinued feed "Zeus Tracker" has been found ' + 'as bot zeus-collector. ' + 'The discontinued feed "Bitcash.cz" has been found ' + 'as bot bitcash-collector. ' + 'The discontinued feed "Fraunhofer DDos Attack" has ' + 'been found as bot ddos-attack-c2-collector, ' + 'ddos-attack-targets-collector. ' + 'The discontinued feed "Abuse.ch Ransomware Tracker" ' + 'has been found as bot ransomware-collector. ' + 'Many Bambenek feeds now require a license, see ' + 'https://osint.bambenekconsulting.com/feeds/ ' + 'potentially affected bots are ' + 'bambenek-c2dommasterlist-collector. ' + 'All Nothink Honeypot feeds are discontinued, ' + 'potentially affected bots are nothink-dns-collector, ' + 'nothink-ssh-collector. ' + 'The Nothink Parser has been removed, ' + 'affected bots are nothink-parser. ' + 'Remove affected bots yourself.', + result[0]) + self.assertEqual(V213_FEED, result[1]) diff --git a/intelmq/tests/lib/upgrades/test_v220.py b/intelmq/tests/lib/upgrades/test_v220.py new file mode 100644 index 000000000..8b5936cd6 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v220.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest + +from intelmq.lib.upgrade.v220 import configuration, feed_changes + + +V220_MISP_VERIFY_FALSE = { + "global": {"http_verify_cert": True}, + "misp-collector": { + "module": "intelmq.bots.collectors.misp.collector", + "parameters": { + "misp_verify": False}}} +V220_MISP_VERIFY_NULL = { + "global": {"http_verify_cert": True}, + "misp-collector": { + "module": "intelmq.bots.collectors.misp.collector", + "parameters": {}}} +V220_MISP_VERIFY_TRUE = { + "global": {"http_verify_cert": True}, + "misp-collector": { + "module": "intelmq.bots.collectors.misp.collector", + "parameters": { + "misp_verify": True}}} +V220_HTTP_VERIFY_FALSE = { + "global": {"http_verify_cert": True}, + "misp-collector": { + "module": "intelmq.bots.collectors.misp.collector", + "parameters": { + "http_verify_cert": False}}} +V220_FEED = {"global": {}, + "urlvir-hosts-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "http://www.urlvir.com/export-hosts/", + }, +}, + "urlvir-parser": { + "group": "Parser", + "module": "intelmq.bots.parsers.urlvir.parser", +}, +} + + +class TestUpgradeV220(unittest.TestCase): + def test_configuration(self): + """ Test configuration. """ + result = configuration(V220_MISP_VERIFY_TRUE, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V220_MISP_VERIFY_NULL, result[1]) + result = configuration(V220_MISP_VERIFY_FALSE, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V220_HTTP_VERIFY_FALSE, result[1]) + + def test_feed_changes(self): + """ Test feed_changes """ + result = feed_changes(V220_FEED, {}, False) + self.assertEqual('A discontinued feed "URLVir" has been found ' + 'as bot urlvir-hosts-collector. ' + 'The removed parser "URLVir" has been found ' + 'as bot urlvir-parser. ' + 'Remove affected bots yourself.', + result[0]) + self.assertEqual(V220_FEED, result[1]) diff --git a/intelmq/tests/lib/upgrades/test_v221.py b/intelmq/tests/lib/upgrades/test_v221.py new file mode 100644 index 000000000..0fc7c9764 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v221.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest + +from intelmq.lib.upgrade.v221 import feed_changes + + +V221_FEED = {"global": {}, + "abusech-urlhaus-columns-string-parser": { + "parameters": { + "column_regex_search": {}, + "columns": "time.source,source.url,status,extra.urlhaus.threat_type,source.fqdn,source.ip,source.asn,source.geolocation.cc", + "default_url_protocol": "http://", + "delimiter": ",", + "filter_text": None, + "filter_type": None, + "skip_header": False, + "time_format": None, + "type": "c2server", + "type_translation": { + "malware_download": "malware-distribution" + } + }, + "module": "intelmq.bots.parsers.generic.parser_csv", +}, + "abusech-urlhaus-columns-dict-parser": { + "parameters": { + "column_regex_search": {}, + "columns": ["time.source", "source.url", "status", "extra.urlhaus.threat_type", "source.fqdn", "source.ip", "source.asn", "source.geolocation.cc"], + "default_url_protocol": "http://", + "delimiter": ",", + "filter_text": None, + "filter_type": None, + "skip_header": False, + "time_format": None, + "type": "c2server", + "type_translation": { + "malware_download": "malware-distribution" + } + }, + "module": "intelmq.bots.parsers.generic.parser_csv", +} +} +V221_FEED_OUT = {"global": {}, + "abusech-urlhaus-columns-string-parser": { + "parameters": { + "column_regex_search": {}, + "columns": ['time.source', 'source.url', 'status', 'classification.type|__IGNORE__', 'source.fqdn|__IGNORE__', 'source.ip', 'source.asn', 'source.geolocation.cc'], + "default_url_protocol": "http://", + "delimiter": ",", + "filter_text": None, + "filter_type": None, + "skip_header": False, + "time_format": None, + "type": "c2server", + "type_translation": { + "malware_download": "malware-distribution" + } + }, + "module": "intelmq.bots.parsers.generic.parser_csv", +} +} +V221_FEED_OUT['abusech-urlhaus-columns-dict-parser'] = V221_FEED_OUT['abusech-urlhaus-columns-string-parser'] +V221_FEED_2 = {"global": {}, + "hphosts-collector": { + "group": "Collector", + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "http://hosts-file.net/download/hosts.txt", + }, +}, + "hphosts-parser": { + "group": "Parser", + "module": "intelmq.bots.parsers.hphosts.parser", +}, +} + + +class TestUpgradeV221(unittest.TestCase): + def test_feed_changes(self): + """ Test feeds_1 """ + result = feed_changes(V221_FEED, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V221_FEED_OUT, result[1]) + + def test_feed_changes_2(self): + """ Test feed_changes """ + result = feed_changes(V221_FEED_2, {}, False) + self.assertEqual('A discontinued feed "HP Hosts File" has been found ' + 'as bot hphosts-collector. ' + 'The removed parser "HP Hosts" has been found ' + 'as bot hphosts-parser. ' + 'Remove affected bots yourself.', + result[0]) + self.assertEqual(V221_FEED_2, result[1]) diff --git a/intelmq/tests/lib/upgrades/test_v222.py b/intelmq/tests/lib/upgrades/test_v222.py new file mode 100644 index 000000000..c9180dd56 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v222.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest + +from intelmq.lib.upgrade.v222 import feed_changes + + +V222 = { + "global": {}, + "shadowserver-parser": { + "module": "intelmq.bots.parsers.shadowserver.parser", + "parameters": { + "feedname": "Blacklisted-IP"}}} +V222_OUT = { + "global": {}, + "shadowserver-parser": { + "module": "intelmq.bots.parsers.shadowserver.parser", + "parameters": { + "feedname": "Blocklist"}}} + + +class TestUpgradeV222(unittest.TestCase): + def test_feed_changes(self): + """ Test v222_feed_changes """ + result = feed_changes(V222, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V222_OUT, result[1]) diff --git a/intelmq/tests/lib/upgrades/test_v230.py b/intelmq/tests/lib/upgrades/test_v230.py new file mode 100644 index 000000000..1a1835c01 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v230.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest + +from intelmq.lib.upgrade.v230 import csv_parser_parameter_fix, deprecations, feed_changes + + +V230_IN = { + "global": {}, + "urlhaus-parser": { + "module": "intelmq.bots.parsers.generic.parser_csv", + "parameters": { + "delimeter": "," + } + } +} +V230_IN_BOTH = { + "global": {}, + "urlhaus-parser": { + "module": "intelmq.bots.parsers.generic.parser_csv", + "parameters": { + "delimeter": ",", + "delimiter": "," + } + } +} +V230_OUT = { + "global": {}, + "urlhaus-parser": { + "module": "intelmq.bots.parsers.generic.parser_csv", + "parameters": { + "delimiter": "," + } + } +} +V230_MALWAREDOMAINLIST_IN = { + "global": {}, + "malwaredomainlist-parser": { + "module": "intelmq.bots.parsers.malwaredomainlist.parser", + "parameters": { + } + }, + "malwaredomainlist-collector": { + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "http://www.malwaredomainlist.com/updatescsv.php" + } + } +} + + +class TestUpgradeV230(unittest.TestCase): + def test_csv_parser_parameter_fix(self): + """ Test feed_fix """ + result = csv_parser_parameter_fix(V230_IN, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V230_OUT, result[1]) + + # with also the new fixed parameter + result = csv_parser_parameter_fix(V230_IN_BOTH, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V230_OUT, result[1]) + + # with new parameter, no change + result = csv_parser_parameter_fix(V230_OUT, {}, False) + self.assertIsNone(result[0]) + self.assertEqual(V230_OUT, result[1]) + + def test_deprecations(self): + """ Test deprecations """ + result = deprecations(V230_MALWAREDOMAINLIST_IN, {}, False) + self.assertTrue(result[0]) + self.assertEqual('A discontinued bot "Malware Domain List Parser" has been found as bot ' + 'malwaredomainlist-parser. Remove affected bots yourself.', + result[0]) + self.assertEqual(V230_MALWAREDOMAINLIST_IN, result[1]) + + def test_feed_changes(self): + """ Test feed_changes """ + result = feed_changes(V230_MALWAREDOMAINLIST_IN, {}, False) + self.assertTrue(result[0]) + self.assertEqual('A discontinued feed "Malware Domain List" has been found as bot ' + 'malwaredomainlist-collector. Remove affected bots yourself.', + result[0]) + self.assertEqual(V230_MALWAREDOMAINLIST_IN, result[1]) diff --git a/intelmq/tests/lib/upgrades/test_v233.py b/intelmq/tests/lib/upgrades/test_v233.py new file mode 100644 index 000000000..b36cb3189 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v233.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2022 Sebastian Wagner +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest + +from intelmq.lib.upgrade.v233 import feodotracker_browse + + +V233_FEODOTRACKER_BROWSE_IN = { + "global": {}, + 'Feodo-tracker-browse-parser': { + 'module': "intelmq.bots.parsers.html_table.parser", + 'parameters': { + 'columns': 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc'.split(','), + 'type': 'c2server', + 'ignore_values': ',,,,Not listed,,', + 'skip_table_head': True, + } + } +} +V233_FEODOTRACKER_BROWSE_OUT = { + "global": {}, + 'Feodo-tracker-browse-parser': { + 'module': "intelmq.bots.parsers.html_table.parser", + 'parameters': { + 'columns': 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc', + 'type': 'c2server', + 'ignore_values': ',,,,,', + 'skip_table_head': True, + } + } +} + + +class TestUpgradeV233(unittest.TestCase): + def test_feodotracker_browse(self): + """ Test feodotracker_browse """ + result = feodotracker_browse(V233_FEODOTRACKER_BROWSE_IN, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V233_FEODOTRACKER_BROWSE_OUT, result[1]) diff --git a/intelmq/tests/lib/upgrades/test_v301.py b/intelmq/tests/lib/upgrades/test_v301.py new file mode 100644 index 000000000..51218f684 --- /dev/null +++ b/intelmq/tests/lib/upgrades/test_v301.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: 2022 Birger Schacht +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# -*- coding: utf-8 -*- +import unittest +from intelmq.lib.upgrade.v301 import deprecations + + +V301_MALWAREDOMAINS_IN = { + "global": {}, + "malwaredomains-parser": { + "module": "intelmq.bots.parsers.malwaredomains.parser", + "parameters": { + } + }, + "malwaredomains-collector": { + "module": "intelmq.bots.collectors.http.collector", + "parameters": { + "http_url": "http://mirror1.malwaredomains.com/files/domains.txt" + } + } +} + + +class TestUpgradeV301(unittest.TestCase): + def test_feed_changes(self): + """ Test feed_changes """ + result = deprecations(V301_MALWAREDOMAINS_IN, {}, False) + self.assertTrue(result[0]) + self.assertEqual('A discontinued bot "Malware Domains Parser" has been found as bot ' + 'malwaredomains-parser. A discontinued bot "Malware Domains Collector" ' + 'has been found as bot malwaredomains-collector. Remove affected bots yourself.', + result[0]) + self.assertEqual(V301_MALWAREDOMAINS_IN, result[1])