Skip to content
Permalink
Browse files

Merge branch 'maintenance' into develop

  • Loading branch information...
wagner-certat committed Nov 5, 2019
2 parents f587dde + 28f7aae commit 1aa5798a1446f548ad45c5a3e6b5d71aeca758b6
Showing with 60 additions and 34 deletions.
  1. +4 −0 CHANGELOG.md
  2. +2 −1 NEWS.md
  3. +27 −6 intelmq/bin/intelmqctl.py
  4. +27 −27 intelmq/lib/upgrades.py
@@ -46,6 +46,7 @@ CHANGELOG

### Core
- `intelmq.lib.pipeline`: AMQP: using port 15672 as default (like RabbitMQ's defaults) for the monitoring interface for getting statistical data (`intelmqctl_rabbitmq_monitoring_url`).
- `intelmq.lib.upgrades`: Added a generic upgrade function for harmonization, checking of all message types, it's fields and their types.

### Development

@@ -78,6 +79,9 @@ CHANGELOG
### Tests

### Tools
- `intelmq.bin.intelmqctl`:
- check: Check if running the upgrade function for harmonization is necessary.
- upgrade-config: Run the upgrade function for harmonization.

### Contrib

@@ -25,9 +25,10 @@ See the changelog for a full list of changes.
### Requirements

### Tools
`intelmqctl check` and `intelmqctl upgrade-config` checks/applies a generic harmonization upgrade, checking for it's completeness.

### Harmonization
In version 2.1.0 a new field `extra` has been added to *Reports*. You need to add this to the *report* section in your harmonization configuration:
In version 2.1.0 a new field `extra` has been added to *Reports*. You need to add this to the *report* section in your harmonization configuration, or run `intelmqctl upgrade-config`:
```json
"extra": {
"description": "All anecdotal information of the report, which cannot be parsed into the data harmonization elements. E.g. subject of mails, etc. This is data is not automatically propagated to the events.",
@@ -1434,6 +1434,10 @@ def check(self, no_connections=False):
if extra_type != 'JSONDict':
check_logger.warning("'extra' field needs to be of type 'JSONDict'.")
retval = 1
if upgrades.harmonization({}, {}, files[HARMONIZATION_CONF_FILE],
dry_run=True)[0]:
check_logger.warning("Harmonization needs an upgrade, call "
"intelmqctl upgrade-config.")

check_logger.info('Checking for bots.')
for bot_id, bot_config in files[RUNTIME_CONF_FILE].items():
@@ -1543,6 +1547,7 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, force=None):

defaults = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
if dry_run:
self.logger.info('Doing a dry run, not writing anything now.')

@@ -1561,11 +1566,13 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, force=None):
', '.join(upgrades.__all__))
return 1, 'error'
try:
retval, defaults_new, runtime_new = getattr(upgrades, function)(defaults, runtime, dry_run)
retval, defaults_new, runtime_new, harmonization_new = getattr(
upgrades, function)(defaults, runtime, harmonization, dry_run)
# Handle changed configurations
if retval is True and not dry_run:
utils.write_configuration(DEFAULTS_CONF_FILE, defaults_new)
utils.write_configuration(RUNTIME_CONF_FILE, runtime_new)
utils.write_configuration(HARMONIZATION_CONF_FILE, harmonization_new)
except Exception:
self.logger.exception('Upgrade %r failed, please report this bug '
'with the shown traceback.',
@@ -1633,6 +1640,9 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, force=None):
self.logger.info("Found no previous version, doing all upgrades.")
todo = [(version, bunch, True) for version, bunch in upgrades.UPGRADES.items()]

todo.extend([(None, (function, ), False)
for function in upgrades.ALWAYS])

"""
todo is now a list of tuples of functions.
todo = [
@@ -1647,7 +1657,7 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, force=None):

error = False
for version, bunch, version_new in todo:
if version_new:
if version and version_new:
self.logger.info('Upgrading to version %s.',
'.'.join(map(str, version)))
for function in bunch:
@@ -1660,7 +1670,7 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, force=None):
"time": datetime.datetime.now().isoformat()
}
try:
retval, defaults, runtime = function(defaults, runtime, dry_run)
retval, defaults, runtime, harmonization = function(defaults, runtime, harmonization, dry_run)
except Exception:
self.logger.exception('%s: Upgrade failed, please report this bug '
'with the traceback.', docstring)
@@ -1682,15 +1692,25 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, force=None):
'as bug.', docstring, retval)
result['success'] = False
result['retval'] = retval
state['results'].append(result)
state['upgrades'][function.__name__] = result['success']

if version or retval is not None:
"""
do not add it to the results if it is run always
and was a no-op.
"""
state['results'].append(result)
if version:
"""
Only add it to the upgrades list if it is specific to a function
"""
state['upgrades'][function.__name__] = result['success']

if not result['success']:
error = True
break
if error:
break
if version_new:
if version and version_new:
state['version_history'].append(version)

if error:
@@ -1706,6 +1726,7 @@ def upgrade_conf(self, previous=None, dry_run=None, function=None, force=None):
if not dry_run:
utils.write_configuration(DEFAULTS_CONF_FILE, defaults)
utils.write_configuration(RUNTIME_CONF_FILE, runtime)
utils.write_configuration(HARMONIZATION_CONF_FILE, harmonization)
except Exception as exc:
self.logger.error('Writing defaults or runtime configuration '
'did not succeed: %s\nFix the problem and '
@@ -23,7 +23,7 @@
]


def v200_defaults_statistics(defaults, runtime, dry_run):
def v200_defaults_statistics(defaults, runtime, harmonization, dry_run):
"""
Inserting `statistics_*` parameters into defaults configuration file
"""
@@ -37,10 +37,10 @@ def v200_defaults_statistics(defaults, runtime, dry_run):
if key not in defaults:
defaults[key] = value
changed = True
return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def v200_defaults_broker(defaults, runtime, dry_run):
def v200_defaults_broker(defaults, runtime, harmonization, dry_run):
"""
Inserting `*_pipeline_broker` and deleting broker into/from defaults configuration
"""
@@ -56,10 +56,10 @@ def v200_defaults_broker(defaults, runtime, dry_run):
del defaults["broker"]
changed = True

return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def v112_feodo_tracker_ips(defaults, runtime, dry_run):
def v112_feodo_tracker_ips(defaults, runtime, harmonization, dry_run):
"""
Fix URL of feodotracker IPs feed in runtime configuration
"""
@@ -69,10 +69,10 @@ def v112_feodo_tracker_ips(defaults, runtime, dry_run):
bot["parameters"]["http_url"] = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv"
changed = True

return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def v112_feodo_tracker_domains(defaults, runtime, dry_run):
def v112_feodo_tracker_domains(defaults, runtime, harmonization, dry_run):
"""
Search for discontinued feodotracker domains feed
"""
@@ -82,14 +82,14 @@ def v112_feodo_tracker_domains(defaults, runtime, dry_run):
found = bot_id

if not found:
return None, defaults, runtime
return None, defaults, runtime, harmonization
else:
return ('The discontinued feed "Feodo Tracker Domains" has been found '
'as bot %r. Remove it yourself please.' % found,
defaults, runtime)
defaults, runtime, harmonization)


def v110_shadowserver_feednames(defaults, runtime, dry_run):
def v110_shadowserver_feednames(defaults, runtime, harmonization, dry_run):
"""
Replace deprecated Shadowserver feednames
"""
@@ -107,10 +107,10 @@ def v110_shadowserver_feednames(defaults, runtime, dry_run):
changed = True
bot["parameters"]["feedname"] = mapping[bot["parameters"]["feedname"]]

return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def v110_deprecations(defaults, runtime, dry_run):
def v110_deprecations(defaults, runtime, harmonization, dry_run):
"""
Checking for deprecated runtime configurations (stomp collector, cymru parser, ripe expert)
"""
@@ -143,7 +143,7 @@ def v110_deprecations(defaults, runtime, dry_run):
else:
changed = True

return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def modify_expert_convert_config(old):
@@ -159,7 +159,7 @@ def modify_expert_convert_config(old):
return config


def v100_dev7_modify_syntax(defaults, runtime, dry_run):
def v100_dev7_modify_syntax(defaults, runtime, harmonization, dry_run):
"""
Migrate modify bot configuration format
"""
@@ -182,23 +182,23 @@ def v100_dev7_modify_syntax(defaults, runtime, dry_run):
new_config)
except PermissionError:
return ('Can\'t update %s\'s configuration: Permission denied.' % bot_id,
defaults, runtime)
defaults, runtime, harmonization)

return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def v200_defaults_ssl_ca_certificate(defaults, runtime, dry_run):
def v200_defaults_ssl_ca_certificate(defaults, runtime, harmonization, dry_run):
"""
Add ssl_ca_certificate to defaults
"""
if "ssl_ca_certificate" not in defaults:
defaults["ssl_ca_certificate"] = None
return True, defaults, runtime
return True, defaults, runtime, harmonization
else:
return None, defaults, runtime
return None, defaults, runtime, harmonization


def v111_defaults_process_manager(defaults, runtime, dry_run):
def v111_defaults_process_manager(defaults, runtime, harmonization, dry_run):
"""
Fix typo in proccess_manager parameter
"""
@@ -215,10 +215,10 @@ def v111_defaults_process_manager(defaults, runtime, dry_run):
defaults["process_manager"] = "intelmq"
changed = True

return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def v202_fixes(defaults, runtime, dry_run):
def v202_fixes(defaults, runtime, harmonization, dry_run):
"""
Migrating collector parameter `feed` to `name`. RIPE expert set: `query_ripe_stat_ip` with `query_ripe_stat_asn` as default
"""
@@ -244,12 +244,12 @@ def v202_fixes(defaults, runtime, dry_run):
bot["parameters"]["overwrite"] = True
changed = True

return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def v210_deprecations(defaults, runtime, dry_run):
def v210_deprecations(defaults, runtime, harmonization, dry_run):
"""
Migrating configuration.
Migrating configuration
"""
changed = None
for bot_id, bot in runtime.items():
@@ -273,12 +273,12 @@ def v210_deprecations(defaults, runtime, dry_run):
if bot["module"] == "intelmq.bots.outputs.postgresql.output":
bot["module"] = "intelmq.bots.outputs.sql.output"
changed = True
return changed, defaults, runtime
return changed, defaults, runtime, harmonization


def harmonization(defaults, runtime, harmonization, dry_run):
"""
Checks if all harmonization fields and types are correct.
Checks if all harmonization fields and types are correct
"""
changed = None
original = load_configuration(resource_filename('intelmq',

0 comments on commit 1aa5798

Please sign in to comment.
You can’t perform that action at this time.