diff --git a/CHANGELOG.md b/CHANGELOG.md index d9d464bb5..f737e4fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ CHANGELOG ### Configuration ### Core +- `intelmq.lib.upgrades`: + - Refactor upgrade functions global configuration handling removing the old-style defaults configuration (PR#2058 by Sebastian Wagner). ### Development diff --git a/intelmq/bin/intelmqctl.py b/intelmq/bin/intelmqctl.py index 600049509..56b1d31d0 100644 --- a/intelmq/bin/intelmqctl.py +++ b/intelmq/bin/intelmqctl.py @@ -1649,13 +1649,8 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, self.logger.info('Successfully wrote initial state file.') runtime = utils.load_configuration(RUNTIME_CONF_FILE) - try: - # remove global defaults, handled by 'defaults' - del runtime['global'] - except KeyError: - # no global parameters is ok - pass - defaults = utils.get_global_settings() + if 'global' not in runtime: + runtime['global'] = {} harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE) if dry_run: self.logger.info('Doing a dry run, not writing anything now.') @@ -1675,11 +1670,10 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, ', '.join(upgrades.__all__)) return 1, 'error' try: - retval, defaults_new, runtime_new, harmonization_new = getattr( - upgrades, function)(defaults, runtime, harmonization, dry_run) + retval, runtime_new, harmonization_new = getattr( + upgrades, function)(runtime, harmonization, dry_run) # Handle changed configurations if retval is True and not dry_run: - runtime_new['global'] = defaults_new utils.write_configuration(RUNTIME_CONF_FILE, runtime_new, backup=not no_backup) utils.write_configuration(HARMONIZATION_CONF_FILE, harmonization_new, @@ -1783,7 +1777,7 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, "time": datetime.datetime.now().isoformat() } try: - retval, defaults, runtime, harmonization = function(defaults, runtime, harmonization, dry_run) + retval, runtime, harmonization = function(runtime, harmonization, dry_run) except Exception: self.logger.exception('%s: Upgrade failed, please report this bug ' 'with the traceback.', docstring) @@ -1838,7 +1832,6 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, try: if not dry_run: - runtime['global'] = defaults utils.write_configuration(RUNTIME_CONF_FILE, runtime, backup=not no_backup) utils.write_configuration(HARMONIZATION_CONF_FILE, harmonization, diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 5dfc7ba52..ce73b3712 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -40,73 +40,77 @@ ] -def v200_defaults_statistics(defaults, runtime, harmonization, dry_run): +def v200_defaults_statistics(configuration, harmonization, dry_run, **kwargs): """ Inserting `statistics_*` parameters into defaults configuration file """ values = {"statistics_database": 3, "statistics_host": "127.0.0.1", - "statistics_password": defaults.get('source_pipeline_password', None), + "statistics_password": configuration['global'].get('source_pipeline_password', None), "statistics_port": 6379 } changed = None for key, value in values.items(): - if key not in defaults: - defaults[key] = value + if key not in configuration['global']: + configuration['global'][key] = value changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v200_defaults_broker(defaults, runtime, harmonization, dry_run): +def v200_defaults_broker(configuration, harmonization, dry_run, **kwargs): """ Inserting `*_pipeline_broker` and deleting broker into/from defaults configuration """ changed = None - values = {"destination_pipeline_broker": defaults.get("broker", "redis"), - "source_pipeline_broker": defaults.get("broker", "redis"), + values = {"destination_pipeline_broker": configuration['global'].get("broker", "redis"), + "source_pipeline_broker": configuration['global'].get("broker", "redis"), } for key, value in values.items(): - if key not in defaults: - defaults[key] = value + if key not in configuration['global']: + configuration['global'][key] = value changed = True - if "broker" in defaults: - del defaults["broker"] + if "broker" in configuration['global']: + del configuration['global']["broker"] changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v112_feodo_tracker_ips(defaults, runtime, harmonization, dry_run): +def v112_feodo_tracker_ips(configuration, harmonization, dry_run, **kwargs): """ Fix URL of feodotracker IPs feed in runtime configuration """ changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["parameters"].get("http_url") == "https://feodotracker.abuse.ch/blocklist/?download=ipblocklist": bot["parameters"]["http_url"] = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv" changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v112_feodo_tracker_domains(defaults, runtime, harmonization, dry_run): +def v112_feodo_tracker_domains(configuration, harmonization, dry_run, **kwargs): """ Search for discontinued feodotracker domains feed """ found = False - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["parameters"].get("http_url") == "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist": found = bot_id if not found: - return None, defaults, runtime, harmonization + return None, configuration, harmonization else: return ('The discontinued feed "Feodo Tracker Domains" has been found ' 'as bot %r. Remove it yourself please.' % found, - defaults, runtime, harmonization) + configuration, harmonization) -def v110_shadowserver_feednames(defaults, runtime, harmonization, dry_run): +def v110_shadowserver_feednames(configuration, harmonization, dry_run, **kwargs): """ Replace deprecated Shadowserver feednames """ @@ -118,16 +122,18 @@ def v110_shadowserver_feednames(defaults, runtime, harmonization, dry_run): "Ssl-Scan": "SSL-POODLE-Vulnerable-Servers", } changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.parsers.shadowserver.parser": if bot["parameters"]["feedname"] in mapping: changed = True bot["parameters"]["feedname"] = mapping[bot["parameters"]["feedname"]] - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v110_deprecations(defaults, runtime, harmonization, dry_run): +def v110_deprecations(configuration, harmonization, dry_run, **kwargs): """ Checking for deprecated runtime configurations (stomp collector, cymru parser, ripe expert, collector feed parameter) """ @@ -136,7 +142,9 @@ def v110_deprecations(defaults, runtime, harmonization, dry_run): "intelmq.bots.parsers.cymru_full_bogons.parser": "intelmq.bots.parsers.cymru.parser_full_bogons", } changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] in mapping: bot["module"] = mapping[bot["module"]] changed = True @@ -160,7 +168,7 @@ def v110_deprecations(defaults, runtime, harmonization, dry_run): else: changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization def modify_expert_convert_config(old): @@ -176,12 +184,14 @@ def modify_expert_convert_config(old): return config -def v100_dev7_modify_syntax(defaults, runtime, harmonization, dry_run): +def v100_dev7_modify_syntax(configuration, harmonization, dry_run, **kwargs): """ Migrate modify bot configuration format """ changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.experts.modify.expert": if "configuration_path" in bot["parameters"]: config = load_configuration(bot["parameters"]["configuration_path"]) @@ -199,49 +209,51 @@ def v100_dev7_modify_syntax(defaults, runtime, harmonization, dry_run): new_config) except PermissionError: return ('Can\'t update %s\'s configuration: Permission denied.' % bot_id, - defaults, runtime, harmonization) + configuration, harmonization) - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v200_defaults_ssl_ca_certificate(defaults, runtime, harmonization, dry_run): +def v200_defaults_ssl_ca_certificate(configuration, harmonization, dry_run, **kwargs): """ Add ssl_ca_certificate to defaults """ - if "ssl_ca_certificate" not in defaults: - defaults["ssl_ca_certificate"] = None - return True, defaults, runtime, harmonization + if "ssl_ca_certificate" not in configuration['global']: + configuration['global']["ssl_ca_certificate"] = None + return True, configuration, harmonization else: - return None, defaults, runtime, harmonization + return None, configuration, harmonization -def v111_defaults_process_manager(defaults, runtime, harmonization, dry_run): +def v111_defaults_process_manager(configuration, harmonization, dry_run, **kwargs): """ Fix typo in proccess_manager parameter """ changed = None - if "proccess_manager" in defaults: - if "process_manager" in defaults: - del defaults["proccess_manager"] - elif "process_manager" not in defaults: - defaults["process_manager"] = defaults["proccess_manager"] - del defaults["proccess_manager"] + if "proccess_manager" in configuration['global']: + if "process_manager" in configuration['global']: + del configuration['global']["proccess_manager"] + elif "process_manager" not in configuration['global']: + configuration['global']["process_manager"] = configuration['global']["proccess_manager"] + del configuration['global']["proccess_manager"] changed = True else: - if "process_manager" not in defaults: - defaults["process_manager"] = "intelmq" + if "process_manager" not in configuration['global']: + configuration['global']["process_manager"] = "intelmq" changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v202_fixes(defaults, runtime, harmonization, dry_run): +def v202_fixes(configuration, harmonization, dry_run, **kwargs): """ Migrate Collector parameter `feed` to `name`. RIPE expert set `query_ripe_stat_ip` with `query_ripe_stat_asn` as default. Set cymru whois expert `overwrite` to true. """ changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["group"] == 'Collector' and bot["parameters"].get("feed"): try: bot["parameters"]["name"] = bot["parameters"]["feed"] @@ -262,15 +274,17 @@ def v202_fixes(defaults, runtime, harmonization, dry_run): bot["parameters"]["overwrite"] = True changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v210_deprecations(defaults, runtime, harmonization, dry_run): +def v210_deprecations(configuration, harmonization, dry_run, **kwargs): """ Migrating configuration """ changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.collectors.rt.collector_rt": # from 29c4b2c42b126ef51ac7287edc1a9fee28ab27fd to ce96e6d995d420e117a49a22d3bfdea762d899ec if "extract_files" in bot["parameters"]: @@ -291,16 +305,18 @@ def v210_deprecations(defaults, runtime, harmonization, dry_run): if bot["module"] == "intelmq.bots.outputs.postgresql.output": bot["module"] = "intelmq.bots.outputs.sql.output" changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v213_deprecations(defaults, runtime, harmonization, dry_run): +def v213_deprecations(configuration, harmonization, dry_run, **kwargs): """ migrate attach_unzip to extract_files for mail attachment collector """ changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.collectors.mail.collector_mail_attach": if "attach_unzip" not in bot["parameters"]: continue @@ -311,44 +327,48 @@ def v213_deprecations(defaults, runtime, harmonization, dry_run): bot["parameters"]["extract_files"] = bot["parameters"]["attach_unzip"] del bot["parameters"]["attach_unzip"] changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v220_configuration(defaults, runtime, harmonization, dry_run): +def v220_configuration(configuration, harmonization, dry_run, **kwargs): """ Migrating configuration """ changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.collectors.misp.collector": if "misp_verify" not in bot["parameters"]: continue - if bot["parameters"]["misp_verify"] != defaults["http_verify_cert"]: + if bot["parameters"]["misp_verify"] != configuration['global']["http_verify_cert"]: bot["parameters"]["http_verify_cert"] = bot["parameters"]["misp_verify"] del bot["parameters"]["misp_verify"] changed = True elif bot["module"] == "intelmq.bots.outputs.elasticsearch.output": if "elastic_doctype" in bot["parameters"]: del bot["parameters"]["elastic_doctype"] - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v220_azure_collector(defaults, runtime, harmonization, dry_run): +def v220_azure_collector(configuration, harmonization, dry_run, **kwargs): """ Checking for the Microsoft Azure collector """ changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.collectors.microsoft.collector_azure": if "connection_string" not in bot["parameters"]: changed = ("The Microsoft Azure collector changed backwards-" "incompatible in IntelMQ 2.2.0. Look at the bot's " "documentation and NEWS file to adapt the " "configuration.") - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def harmonization(defaults, runtime, harmonization, dry_run): +def harmonization(configuration, harmonization, dry_run, **kwargs): """ Checks if all harmonization fields and types are correct """ @@ -378,10 +398,10 @@ def harmonization(defaults, runtime, harmonization, dry_run): if original_regex and original_regex != installed_regex: harmonization[msg_type][fieldname]['iregex'] = original[msg_type][fieldname]['iregex'] changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v213_feed_changes(defaults, runtime, harmonization, dry_run): +def v213_feed_changes(configuration, harmonization, dry_run, **kwargs): """ Migrates feed configuration for changed feed parameters. """ @@ -394,7 +414,9 @@ def v213_feed_changes(defaults, runtime, harmonization, dry_run): found_nothink_parser = [] changed = None messages = [] - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.collectors.http.collector_http": if "http_url" not in bot["parameters"]: continue @@ -445,18 +467,19 @@ def v213_feed_changes(defaults, runtime, harmonization, dry_run): messages.append('The Nothink Parser has been removed, ' 'affected bots are %s.' % ', '.join(sorted(found_nothink_parser))) messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization + return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization -def v220_feed_changes(defaults, runtime, harmonization, dry_run): +def v220_feed_changes(configuration, harmonization, dry_run, **kwargs): """ Migrates feed configuration for changed feed parameters. """ found_urlvir_feed = [] found_urlvir_parser = [] - changed = None messages = [] - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.collectors.http.collector_http": if "http_url" not in bot["parameters"]: continue @@ -471,10 +494,10 @@ def v220_feed_changes(defaults, runtime, harmonization, dry_run): messages.append('The removed parser "URLVir" has been found ' 'as bot %s.' % ', '.join(sorted(found_urlvir_parser))) messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization + return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization -def v221_feed_changes(defaults, runtime, harmonization, dry_run): +def v221_feed_changes(configuration, harmonization, dry_run, **kwargs): """ Migrates feeds' configuration for changed/fixed parameters. Deprecation of HP Hosts file feed & parser. """ @@ -484,7 +507,9 @@ def v221_feed_changes(defaults, runtime, harmonization, dry_run): ULRHAUS_OLD = ['time.source', 'source.url', 'status', 'extra.urlhaus.threat_type', 'source.fqdn', 'source.ip', 'source.asn', 'source.geolocation.cc'] URLHAUS_NEW = ['time.source', 'source.url', 'status', 'classification.type|__IGNORE__', 'source.fqdn|__IGNORE__', 'source.ip', 'source.asn', 'source.geolocation.cc'] changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.collectors.http.collector_http": if bot["parameters"].get("http_url", None) == "http://hosts-file.net/download/hosts.txt": found_hphosts_collector.append(bot_id) @@ -508,28 +533,32 @@ def v221_feed_changes(defaults, runtime, harmonization, dry_run): messages.append('The removed parser "HP Hosts" has been found ' 'as bot %s.' % ', '.join(sorted(found_hphosts_parser))) messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization + return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization -def v222_feed_changes(defaults, runtime, harmonization, dry_run): +def v222_feed_changes(configuration, harmonization, dry_run, **kwargs): """ Migrate Shadowserver feed name """ changed = None - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.parsers.shadowserver.parser": if bot["parameters"].get("feedname", None) == "Blacklisted-IP": bot["parameters"]["feedname"] = "Blocklist" changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v230_csv_parser_parameter_fix(defaults, runtime, harmonization, dry_run): +def v230_csv_parser_parameter_fix(configuration, harmonization, dry_run, **kwargs): """ Fix CSV parser parameter misspelling """ changed = None - for bot in runtime.values(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.parsers.generic.parser_csv": if "delimeter" in bot["parameters"] and "delimiter" in bot["parameters"]: del bot["parameters"]["delimeter"] @@ -538,34 +567,36 @@ def v230_csv_parser_parameter_fix(defaults, runtime, harmonization, dry_run): bot["parameters"]["delimiter"] = bot["parameters"]["delimeter"] del bot["parameters"]["delimeter"] changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v230_deprecations(defaults, runtime, harmonization, dry_run): +def v230_deprecations(configuration, harmonization, dry_run, **kwargs): """ Deprecate malwaredomainlist parser """ found_malwaredomainlistparser = [] - changed = None messages = [] - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.parsers.malwaredomainlist.parser": found_malwaredomainlistparser.append(bot_id) if found_malwaredomainlistparser: messages.append('A discontinued bot "Malware Domain List Parser" has been found ' 'as bot %s.' % ', '.join(sorted(found_malwaredomainlistparser))) messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization + return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization -def v230_feed_changes(defaults, runtime, harmonization, dry_run): +def v230_feed_changes(configuration, harmonization, dry_run, **kwargs): """ Migrates feeds' configuration for changed/fixed parameter """ found_malwaredomainlist = [] - changed = None messages = [] - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.collectors.http.collector_http": if "http_url" not in bot["parameters"]: continue @@ -575,10 +606,10 @@ def v230_feed_changes(defaults, runtime, harmonization, dry_run): messages.append('A discontinued feed "Malware Domain List" has been found ' 'as bot %s.' % ', '.join(sorted(found_malwaredomainlist))) messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization + return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization -def v300_bots_file_removal(defaults, runtime, harmonization, dry_run): +def v300_bots_file_removal(configuration, harmonization, dry_run, **kwargs): """ Remove BOTS file """ @@ -592,10 +623,10 @@ def v300_bots_file_removal(defaults, runtime, harmonization, dry_run): bots_file.unlink() changed = True messages = ' '.join(messages) - return messages if messages else changed, defaults, runtime, harmonization + return messages if messages else changed, configuration, harmonization -def v300_defaults_file_removal(defaults, runtime, harmonization, dry_run): +def v300_defaults_file_removal(configuration, harmonization, dry_run, **kwargs): """ Remove the defaults.conf file """ @@ -606,21 +637,23 @@ def v300_defaults_file_removal(defaults, runtime, harmonization, dry_run): if dry_run: print(f'Would now remove file {defaults_file!r}.') else: - defaults = load_configuration(defaults_file) + configuration['global'] = load_configuration(defaults_file) defaults_file.unlink() changed = True messages = ' '.join(messages) - return messages if messages else changed, defaults, runtime, harmonization + return messages if messages else changed, configuration, harmonization -def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run): +def v233_feodotracker_browse(configuration, harmonization, dry_run, **kwargs): """ Migrate Abuse.ch Feodotracker Browser feed parsing parameters """ changed = None old_feodo_columns = 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc' old_ignore_values = ',,,,Not listed,,' - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue # The parameters can be given as string or list of strings if (bot["module"] == "intelmq.bots.parsers.html_table.parser" and 'feodo' in bot_id.lower() and "columns" in bot["parameters"] and "ignore_values" in bot["parameters"] and @@ -629,10 +662,10 @@ def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run): bot["parameters"]["columns"] = 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc' bot["parameters"]['ignore_values'] = ',,,,,' changed = True - return changed, defaults, runtime, harmonization + return changed, configuration, harmonization -def v300_pipeline_file_removal(defaults, runtime, harmonization, dry_run): +def v300_pipeline_file_removal(configuration, harmonization, dry_run, **kwargs): """ Remove the pipeline.conf file """ @@ -641,29 +674,31 @@ def v300_pipeline_file_removal(defaults, runtime, harmonization, dry_run): pipeline_file = Path(CONFIG_DIR) / "pipeline.conf" if pipeline_file.exists(): pipelines = load_configuration(pipeline_file) - for bot in runtime: + for bot in configuration: + if bot_id == 'global': + continue if bot in pipelines: if 'destination-queues' in pipelines[bot]: destination_queues = pipelines[bot]['destination-queues'] if isinstance(destination_queues, dict): - runtime[bot]['parameters']['destination_queues'] = destination_queues + configuration[bot]['parameters']['destination_queues'] = destination_queues if isinstance(destination_queues, list): - runtime[bot]['parameters']['destination_queues'] = {'_default': destination_queues} + configuration[bot]['parameters']['destination_queues'] = {'_default': destination_queues} if isinstance(destination_queues, str): - runtime[bot]['parameters']['destination_queues'] = {'_default': [destination_queues]} + configuration[bot]['parameters']['destination_queues'] = {'_default': [destination_queues]} if 'source-queue' in pipelines[bot]: if pipelines[bot]['source-queue'] != f"{bot}-queue": - runtime[bot]['parameters']['source_queue'] = pipelines[bot]['source-queue'] + configuration[bot]['parameters']['source_queue'] = pipelines[bot]['source-queue'] if dry_run: print(f'Would now remove file {pipeline_file!r}.') else: pipeline_file.unlink() changed = True messages = ' '.join(messages) - return messages if messages else changed, defaults, runtime, harmonization + return messages if messages else changed, configuration, harmonization -def v301_deprecations(defaults, runtime, harmonization, dry_run): +def v301_deprecations(configuration, harmonization, dry_run, **kwargs): """ Deprecate malwaredomains parser and collector """ @@ -671,7 +706,9 @@ def v301_deprecations(defaults, runtime, harmonization, dry_run): found_malwaredomainscollector = [] changed = None messages = [] - for bot_id, bot in runtime.items(): + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue if bot["module"] == "intelmq.bots.parsers.malwaredomains.parser": found_malwaredomainsparser.append(bot_id) if bot["module"] == "intelmq.bots.collectors.http.collector": @@ -686,7 +723,7 @@ def v301_deprecations(defaults, runtime, harmonization, dry_run): messages.append('A discontinued bot "Malware Domains Collector" has been found ' 'as bot %s.' % ', '.join(sorted(found_malwaredomainscollector))) messages = ' '.join(messages) - return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization + return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization UPGRADES = OrderedDict([ diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 8d319d167..a4d78e7f8 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -13,7 +13,8 @@ from intelmq.lib.utils import load_configuration -V202 = {"test-collector": { +V202 = {"global": {}, +"test-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { @@ -34,7 +35,8 @@ }, }, } -V202_EXP = {"test-collector": { +V202_EXP = {"global": {}, +"test-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { @@ -58,7 +60,8 @@ }, } -DEP_110 = {"n6-collector": { +DEP_110 = {"global": {}, +"n6-collector": { "group": "Collector", "module": "intelmq.bots.collectors.n6.collector_stomp", "parameters": { @@ -79,7 +82,8 @@ }, } } -DEP_110_EXP = {"n6-collector": { +DEP_110_EXP = {"global": {}, +"n6-collector": { "group": "Collector", "module": "intelmq.bots.collectors.stomp.collector", "parameters": { @@ -99,7 +103,8 @@ "query_ripe_stat_ip": True, }, }} -V210 = {"test-collector": { +V210 = {"global": {}, +"test-collector": { "group": "Collector", "module": "intelmq.bots.collectors.rt.collector_rt", "parameters": { @@ -148,7 +153,8 @@ } } } -V210_EXP = {"test-collector": { +V210_EXP = {"global": {}, +"test-collector": { "group": "Collector", "module": "intelmq.bots.collectors.rt.collector_rt", "parameters": { @@ -199,7 +205,8 @@ } } } -V213 = {"mail-collector": { +V213 = {"global": {}, +"mail-collector": { "group": "Collector", "module": "intelmq.bots.collectors.mail.collector_mail_attach", "parameters": { @@ -215,7 +222,8 @@ } } } -V213_EXP = {"mail-collector": { +V213_EXP = {"global": {}, +"mail-collector": { "group": "Collector", "module": "intelmq.bots.collectors.mail.collector_mail_attach", "parameters": { @@ -231,26 +239,28 @@ } } V220_MISP_VERIFY_FALSE = { +"global": {"http_verify_cert": True}, "misp-collector": { "module": "intelmq.bots.collectors.misp.collector", "parameters": { "misp_verify": False}}} V220_MISP_VERIFY_NULL = { +"global": {"http_verify_cert": True}, "misp-collector": { "module": "intelmq.bots.collectors.misp.collector", "parameters": {}}} V220_MISP_VERIFY_TRUE = { +"global": {"http_verify_cert": True}, "misp-collector": { "module": "intelmq.bots.collectors.misp.collector", "parameters": { "misp_verify": True}}} V220_HTTP_VERIFY_FALSE = { +"global": {"http_verify_cert": True}, "misp-collector": { "module": "intelmq.bots.collectors.misp.collector", "parameters": { "http_verify_cert": False}}} -DEFAULTS_HTTP_VERIFY_TRUE = { - "http_verify_cert": True} HARM = load_configuration(pkg_resources.resource_filename('intelmq', 'etc/harmonization.conf')) V210_HARM = deepcopy(HARM) @@ -261,7 +271,8 @@ WRONG_TYPE['event']['source.asn']['type'] = 'String' WRONG_REGEX = deepcopy(HARM) WRONG_REGEX['event']['protocol.transport']['iregex'] = 'foobar' -V213_FEED = {"zeus-collector": { +V213_FEED = {"global": {}, +"zeus-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { @@ -336,7 +347,7 @@ "module": "intelmq.bots.parsers.nothink.parser", }, } -V220_FEED = { +V220_FEED = {"global": {}, "urlvir-hosts-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", @@ -349,7 +360,7 @@ "module": "intelmq.bots.parsers.urlvir.parser", }, } -V221_FEED = { +V221_FEED = {"global": {}, "abusech-urlhaus-columns-string-parser": { "parameters": { "column_regex_search": {}, @@ -385,7 +396,7 @@ "module": "intelmq.bots.parsers.generic.parser_csv", } } -V221_FEED_OUT = { +V221_FEED_OUT = {"global": {}, "abusech-urlhaus-columns-string-parser": { "parameters": { "column_regex_search": {}, @@ -405,7 +416,7 @@ } } V221_FEED_OUT['abusech-urlhaus-columns-dict-parser'] = V221_FEED_OUT['abusech-urlhaus-columns-string-parser'] -V221_FEED_2 = { +V221_FEED_2 = {"global": {}, "hphosts-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", @@ -419,17 +430,20 @@ }, } V222 = { +"global": {}, "shadowserver-parser": { "module": "intelmq.bots.parsers.shadowserver.parser", "parameters": { "feedname": "Blacklisted-IP"}}} V222_OUT = { +"global": {}, "shadowserver-parser": { "module": "intelmq.bots.parsers.shadowserver.parser", "parameters": { "feedname": "Blocklist"}}} V230_IN = { +"global": {}, "urlhaus-parser": { "module": "intelmq.bots.parsers.generic.parser_csv", "parameters": { @@ -438,6 +452,7 @@ } } V230_IN_BOTH = { +"global": {}, "urlhaus-parser": { "module": "intelmq.bots.parsers.generic.parser_csv", "parameters": { @@ -447,6 +462,7 @@ } } V230_OUT = { +"global": {}, "urlhaus-parser": { "module": "intelmq.bots.parsers.generic.parser_csv", "parameters": { @@ -455,6 +471,7 @@ } } V230_MALWAREDOMAINLIST_IN = { +"global": {}, "malwaredomainlist-parser": { "module": "intelmq.bots.parsers.malwaredomainlist.parser", "parameters": { @@ -468,6 +485,7 @@ } } V233_FEODOTRACKER_BROWSE_IN = { +"global": {}, 'Feodo-tracker-browse-parser': { 'module': "intelmq.bots.parsers.html_table.parser", 'parameters': { @@ -479,6 +497,7 @@ } } V233_FEODOTRACKER_BROWSE_OUT = { +"global": {}, 'Feodo-tracker-browse-parser': { 'module': "intelmq.bots.parsers.html_table.parser", 'parameters': { @@ -490,6 +509,7 @@ } } V301_MALWAREDOMAINS_IN = { +"global": {}, "malwaredomains-parser": { "module": "intelmq.bots.parsers.malwaredomains.parser", "parameters": { @@ -505,7 +525,7 @@ def generate_function(function): def test_function(self): """ Test if no errors happen for upgrade function %s. """ % function.__name__ - function({}, {}, {}, dry_run=True) + function({'global': {}}, {}, dry_run=True) return test_function @@ -533,66 +553,64 @@ def test_all_functions_used(self): def test_v110_deprecations(self): """ Test v110_deprecations """ - result = upgrades.v110_deprecations({}, DEP_110, {}, False) + result = upgrades.v110_deprecations(DEP_110, {}, False) self.assertTrue(result[0]) - self.assertEqual(DEP_110_EXP, result[2]) + self.assertEqual(DEP_110_EXP, result[1]) def test_v202_fixes(self): """ Test v202_feed_name """ - result = upgrades.v202_fixes({}, V202, {}, False) + result = upgrades.v202_fixes(V202, {}, False) self.assertTrue(result[0]) - self.assertEqual(V202_EXP, result[2]) + self.assertEqual(V202_EXP, result[1]) def test_v210_deprecations(self): """ Test v210_deprecations """ - result = upgrades.v210_deprecations({}, V210, {}, True) + result = upgrades.v210_deprecations(V210, {}, True) self.assertTrue(result[0]) - self.assertEqual(V210_EXP, result[2]) + self.assertEqual(V210_EXP, result[1]) def test_harmonization(self): """ Test harmonization: Addition of extra to report """ - result = upgrades.harmonization({}, {}, V210_HARM, False) + result = upgrades.harmonization({}, V210_HARM, False) self.assertTrue(result[0]) - self.assertEqual(HARM, result[3]) + self.assertEqual(HARM, result[2]) def test_v220_configuration(self): """ Test v220_configuration. """ - result = upgrades.v220_configuration(DEFAULTS_HTTP_VERIFY_TRUE, - V220_MISP_VERIFY_TRUE, {}, False) + result = upgrades.v220_configuration(V220_MISP_VERIFY_TRUE, {}, False) self.assertTrue(result[0]) - self.assertEqual(V220_MISP_VERIFY_NULL, result[2]) - result = upgrades.v220_configuration(DEFAULTS_HTTP_VERIFY_TRUE, - V220_MISP_VERIFY_FALSE, {}, False) + self.assertEqual(V220_MISP_VERIFY_NULL, result[1]) + result = upgrades.v220_configuration(V220_MISP_VERIFY_FALSE, {}, False) self.assertTrue(result[0]) - self.assertEqual(V220_HTTP_VERIFY_FALSE, result[2]) + self.assertEqual(V220_HTTP_VERIFY_FALSE, result[1]) def test_missing_report_harmonization(self): """ Test missing report in harmonization """ - result = upgrades.harmonization({}, {}, MISSING_REPORT, False) + result = upgrades.harmonization({}, MISSING_REPORT, False) self.assertTrue(result[0]) - self.assertEqual(HARM, result[3]) + self.assertEqual(HARM, result[2]) def test_wrong_type_harmonization(self): """ Test wrong type in harmonization """ - result = upgrades.harmonization({}, {}, WRONG_TYPE, False) + result = upgrades.harmonization({}, WRONG_TYPE, False) self.assertTrue(result[0]) - self.assertEqual(HARM, result[3]) + self.assertEqual(HARM, result[2]) def test_wrong_regex_harmonization(self): """ Test wrong regex in harmonization """ - result = upgrades.harmonization({}, {}, WRONG_REGEX, False) + result = upgrades.harmonization({}, WRONG_REGEX, False) self.assertTrue(result[0]) - self.assertEqual(HARM, result[3]) + self.assertEqual(HARM, result[2]) def test_v213_deprecations(self): """ Test v213_fixes """ - result = upgrades.v213_deprecations({}, V213, {}, False) + result = upgrades.v213_deprecations(V213, {}, False) self.assertTrue(result[0]) - self.assertEqual(V213_EXP, result[2]) + self.assertEqual(V213_EXP, result[1]) def test_v213_feed_changes(self): """ Test v213_feed_changes """ - result = upgrades.v213_feed_changes({}, V213_FEED, {}, False) + result = upgrades.v213_feed_changes(V213_FEED, {}, False) self.assertEqual('A discontinued feed "Zeus Tracker" has been found ' 'as bot zeus-collector. ' 'The discontinued feed "Bitcash.cz" has been found ' @@ -613,91 +631,91 @@ def test_v213_feed_changes(self): 'affected bots are nothink-parser. ' 'Remove affected bots yourself.', result[0]) - self.assertEqual(V213_FEED, result[2]) + self.assertEqual(V213_FEED, result[1]) def test_v220_feed_changes(self): """ Test v213_feed_changes """ - result = upgrades.v220_feed_changes({}, V220_FEED, {}, False) + result = upgrades.v220_feed_changes(V220_FEED, {}, False) self.assertEqual('A discontinued feed "URLVir" has been found ' 'as bot urlvir-hosts-collector. ' 'The removed parser "URLVir" has been found ' 'as bot urlvir-parser. ' 'Remove affected bots yourself.', result[0]) - self.assertEqual(V220_FEED, result[2]) + self.assertEqual(V220_FEED, result[1]) def test_v221_feed_changes(self): """ Test v221_feeds_1 """ - result = upgrades.v221_feed_changes({}, V221_FEED, {}, False) + result = upgrades.v221_feed_changes(V221_FEED, {}, False) self.assertTrue(result[0]) - self.assertEqual(V221_FEED_OUT, result[2]) + self.assertEqual(V221_FEED_OUT, result[1]) def test_v221_feed_changes_2(self): """ Test v213_feed_changes """ - result = upgrades.v221_feed_changes({}, V221_FEED_2, {}, False) + result = upgrades.v221_feed_changes(V221_FEED_2, {}, False) self.assertEqual('A discontinued feed "HP Hosts File" has been found ' 'as bot hphosts-collector. ' 'The removed parser "HP Hosts" has been found ' 'as bot hphosts-parser. ' 'Remove affected bots yourself.', result[0]) - self.assertEqual(V221_FEED_2, result[2]) + self.assertEqual(V221_FEED_2, result[1]) def test_v222_feed_changes(self): """ Test v222_feed_changes """ - result = upgrades.v222_feed_changes({}, V222, {}, False) + result = upgrades.v222_feed_changes(V222, {}, False) self.assertTrue(result[0]) - self.assertEqual(V222_OUT, result[2]) + self.assertEqual(V222_OUT, result[1]) def test_v230_csv_parser_parameter_fix(self): """ Test v230_feed_fix """ - result = upgrades.v230_csv_parser_parameter_fix({}, V230_IN, {}, False) + result = upgrades.v230_csv_parser_parameter_fix(V230_IN, {}, False) self.assertTrue(result[0]) - self.assertEqual(V230_OUT, result[2]) + self.assertEqual(V230_OUT, result[1]) # with also the new fixed parameter - result = upgrades.v230_csv_parser_parameter_fix({}, V230_IN_BOTH, {}, False) + result = upgrades.v230_csv_parser_parameter_fix(V230_IN_BOTH, {}, False) self.assertTrue(result[0]) - self.assertEqual(V230_OUT, result[2]) + self.assertEqual(V230_OUT, result[1]) # with new parameter, no change - result = upgrades.v230_csv_parser_parameter_fix({}, V230_OUT, {}, False) + result = upgrades.v230_csv_parser_parameter_fix(V230_OUT, {}, False) self.assertIsNone(result[0]) - self.assertEqual(V230_OUT, result[2]) + self.assertEqual(V230_OUT, result[1]) def test_v230_deprecations(self): """ Test v230_deprecations """ - result = upgrades.v230_deprecations({}, V230_MALWAREDOMAINLIST_IN, {}, False) + result = upgrades.v230_deprecations(V230_MALWAREDOMAINLIST_IN, {}, False) self.assertTrue(result[0]) self.assertEqual('A discontinued bot "Malware Domain List Parser" has been found as bot ' 'malwaredomainlist-parser. Remove affected bots yourself.', result[0]) - self.assertEqual(V230_MALWAREDOMAINLIST_IN, result[2]) + self.assertEqual(V230_MALWAREDOMAINLIST_IN, result[1]) def test_v230_feed_changes(self): """ Test v230_feed_changes """ - result = upgrades.v230_feed_changes({}, V230_MALWAREDOMAINLIST_IN, {}, False) + result = upgrades.v230_feed_changes(V230_MALWAREDOMAINLIST_IN, {}, False) self.assertTrue(result[0]) self.assertEqual('A discontinued feed "Malware Domain List" has been found as bot ' 'malwaredomainlist-collector. Remove affected bots yourself.', result[0]) - self.assertEqual(V230_MALWAREDOMAINLIST_IN, result[2]) + self.assertEqual(V230_MALWAREDOMAINLIST_IN, result[1]) def test_v233_feodotracker_browse(self): """ Test v233_feodotracker_browse """ - result = upgrades.v233_feodotracker_browse({}, V233_FEODOTRACKER_BROWSE_IN, {}, False) + result = upgrades.v233_feodotracker_browse(V233_FEODOTRACKER_BROWSE_IN, {}, False) self.assertTrue(result[0]) - self.assertEqual(V233_FEODOTRACKER_BROWSE_OUT, result[2]) + self.assertEqual(V233_FEODOTRACKER_BROWSE_OUT, result[1]) def test_v301_feed_changes(self): """ Test v301_feed_changes """ - result = upgrades.v301_deprecations({}, V301_MALWAREDOMAINS_IN, {}, False) + result = upgrades.v301_deprecations(V301_MALWAREDOMAINS_IN, {}, False) self.assertTrue(result[0]) self.assertEqual('A discontinued bot "Malware Domains Parser" has been found as bot ' 'malwaredomains-parser. A discontinued bot "Malware Domains Collector" ' 'has been found as bot malwaredomains-collector. Remove affected bots yourself.', result[0]) - self.assertEqual(V301_MALWAREDOMAINS_IN, result[2]) + self.assertEqual(V301_MALWAREDOMAINS_IN, result[1]) for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name,