Skip to content
Permalink
Browse files

Merge branch 'maintenance' into develop

  • Loading branch information...
wagner-certat committed Sep 4, 2019
2 parents 1249c7b + 0d12561 commit b20ef9643c01621eabe72d807ff472c21e3efcd7
@@ -95,6 +95,11 @@ CHANGELOG
### Configuration

### Core
- `intelmq.lib.bot.CollectorBot`: Support the deprecated parameter `feed` until version 2.2 as the documentation was not properly updated (#1445).
- `intelmq.lib.upgrades.v202_fixes`
- Migration of deprecated parameter `feed` for Collectors.
- Ripe expert parameter `query_ripe_stat_ip` was not correctly configured in `v110_deprecations`, now use `query_ripe_stat_asn` as default if it does not exist.
- `intelmq.lib.upgrades.v110_deprecations`: Fix upgrade of ripe expert configuration.

### Development

@@ -121,6 +126,8 @@ CHANGELOG
- Include update-rfiprisk-data in packages

### Tests
- Tests for `intelmq.lib.upgrades.v202_fixes`.
- Tests for `intelmq.lib.upgrades.v110_deprecations`.

### Tools
- intelmqctl:
@@ -132,6 +139,7 @@ CHANGELOG
- Check MK Statistics Cronjob:
- Use `statistics_*` parameters.
- Make file executable
- Handle None values in `*.temporary.*` keys and treat them as 0.

### Known issues

@@ -29,6 +29,10 @@ See the changelog for a full list of changes.
### Harmonization

### Configuration
The deprecated parameter `feed` for collectors is again supported as the documentation as not properly updated. The support will be removed before version 2.2.

#### RIPE expert
In the upgrade function for version 1.1.0 (in effect in version 2.0.1) the addition of the parameter `query_ripe_stat_ip` was not correctly done and is maybe missing. A new upgrade function re-adds it with the value of `query_ripe_stat_ip`.

### Libraries

@@ -22,6 +22,11 @@
handle.write("<<<local>>>\nP intelmq-statistics ")
stats = []
for key in db.keys():
stats.append("%s=%s" % (key.decode(), db.get(key).decode()))
value = db.get(key)
if value is None:
value = '0'
else:
value = value.decode()
stats.append("%s=%s" % (key.decode(), value))
handle.write("|".join(stats))
handle.write('\n')
@@ -121,7 +121,7 @@ For example:
"abusech-feodo-domains-collector": {
"parameters": {
"provider": "Abuse.ch",
"feed": "Abuse.ch Feodo Domains",
"name": "Abuse.ch Feodo Domains",
"http_url": "http://example.org/feodo-domains.txt"
},
"name": "Generic URL Fetcher",
@@ -148,7 +148,7 @@ This configuration resides in the file `runtime.conf` in your intelmq's configur

**Feed parameters**: Common configuration options for all collectors.

* `feed`: Name for the feed (`feed.name`).
* `name`: Name for the feed (`feed.name`). In IntelMQ versions smaller than 2.2 the parameter name `feed` is also supported.
* `accuracy`: Accuracy for the data of the feed (`feed.accuracy`).
* `code`: Code for the feed (`feed.code`).
* `documentation`: Link to documentation for the feed (`feed.documentation`).
@@ -222,9 +222,9 @@
"description": "Listens to McAfee openDXL fabric",
"module": "intelmq.bots.collectors.opendxl.collector",
"parameters": {
"feed": "<Feed Name>",
"dxl_config_file": "<insert /path/to/dxlclient.config>",
"dxl_topic": "/mcafee/event/atd/file/report"
"dxl_topic": "/mcafee/event/atd/file/report",
"name": "<Feed Name>"
}
},
"Microsoft Azure": {
@@ -17,11 +17,11 @@ A sample configuration for this bot is as follows (see also the BOTS file):
"description": "Collect events from a MISP server.",
"module": "intelmq.bots.collectors.misp.collector",
"parameters": {
"feed": "misp_generic",
"misp_url": "<URL of MISP server (with trailing '/')>",
"misp_key": "<MISP Authkey>",
"misp_tag_to_process": "<MISP tag for events to be processed>",
"misp_tag_processed": "<MISP tag for processed events>",
"name": "misp_generic",
"rate_limit": 3600
}
},
@@ -36,7 +36,7 @@ For temporary failures the status is not modified, instead the ticket will be sk

"request-tracker-collector": {
"attachment_regex": "\\.csv\\.zip$",
"feed": "Request Tracker",
"name": "Request Tracker",
"password": "intelmq",
"search_not_older_than": null,
"search_owner": "nobody",
@@ -1010,6 +1010,12 @@ def __filter_empty_report(self, message: dict):
def __add_report_fields(self, report: dict):
if hasattr(self.parameters, 'name'):
report.add("feed.name", self.parameters.name)
if hasattr(self.parameters, 'feed'):
warnings.warn("The parameter 'feed' is deprecated and will be "
"removed in version 2.2. Use 'name' instead.",
DeprecationWarning)
if "feed.name" not in report:
report.add("feed.name", self.parameters.feed)
if hasattr(self.parameters, 'code'):
report.add("feed.code", self.parameters.code)
if hasattr(self.parameters, 'documentation'):
@@ -17,6 +17,7 @@
'v112_feodo_tracker_domains',
'v200_defaults_ssl_ca_certificate',
'v111_defaults_process_manager',
'v202_fixes',
]


@@ -127,13 +128,14 @@ def v110_deprecations(defaults, runtime, dry_run):
if bot["parameters"].get("query_ripe_stat"):
if "query_ripe_stat_asn" not in bot["parameters"]:
bot["parameters"]["query_ripe_stat_asn"] = bot["parameters"]["query_ripe_stat"]
if "query_ripe_stat_asn" not in bot["parameters"]:
bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat_ip"]
if "query_ripe_stat_ip" not in bot["parameters"]:
bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat"]
del bot["parameters"]["query_ripe_stat"]
changed = True
if bot["group"] == 'Collector' and bot["parameters"].get("feed"):
if bot["group"] == 'Collector' and bot["parameters"].get("feed") and not bot["parameters"].get("name"):
try:
bot["parameters"]["feed"] = bot["parameters"]["name"]
bot["parameters"]["name"] = bot["parameters"]["feed"]
del bot["parameters"]["feed"]
except KeyError:
pass
else:
@@ -214,6 +216,32 @@ def v111_defaults_process_manager(defaults, runtime, dry_run):
return changed, defaults, runtime


def v202_fixes(defaults, runtime, dry_run):
"""
Migrating parameter `feed` to `name`.
ripe expert: query_ripe_stat_ip was not correctly set in v110_deprecations
Set query_ripe_stat_ip to value of query_ripe_stat_asn if query_ripe_stat_ip does not exist
"""
changed = None
for bot_id, bot in runtime.items():
if bot["group"] == 'Collector' and bot["parameters"].get("feed"):
try:
bot["parameters"]["name"] = bot["parameters"]["feed"]
del bot["parameters"]["feed"]
except KeyError:
pass
else:
changed = True
if bot["module"] == "intelmq.bots.experts.ripe.expert":
if "query_ripe_stat_asn" in bot["parameters"]:
if "query_ripe_stat_ip" not in bot["parameters"]:
bot["parameters"]["query_ripe_stat_ip"] = bot["parameters"]["query_ripe_stat_asn"]
changed = True

return changed, defaults, runtime


UPGRADES = OrderedDict([
((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax, )),
((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -222,5 +250,5 @@ def v111_defaults_process_manager(defaults, runtime, dry_run):
((2, 0, 0), (v200_defaults_statistics, v200_defaults_broker,
v200_defaults_ssl_ca_certificate)),
((2, 0, 1), ()),
((2, 0, 2), ()),
((2, 0, 2), (v202_fixes, )),
])
@@ -6,6 +6,81 @@
import intelmq.lib.upgrades as upgrades


V202 = {"test-collector": {
"group": "Collector",
"module": "intelmq.bots.collectors.http.collector_http",
"parameters": {
"feed": "Feed"
}
},
"ripe-expert": {
"group": "Expert",
"module": "intelmq.bots.experts.ripe.expert",
"parameters": {
"query_ripe_stat_asn": True,
},
}
}
V202_EXP = {"test-collector": {
"group": "Collector",
"module": "intelmq.bots.collectors.http.collector_http",
"parameters": {
"name": "Feed"
}
},
"ripe-expert": {
"group": "Expert",
"module": "intelmq.bots.experts.ripe.expert",
"parameters": {
"query_ripe_stat_asn": True,
"query_ripe_stat_ip": True,
},
}
}

DEP_110 = {"n6-collector": {
"group": "Collector",
"module": "intelmq.bots.collectors.n6.collector_stomp",
"parameters": {
"feed": "Feed",
},
},
"cymru-full-bogons-parser": {
"group": "Parser",
"module": "intelmq.bots.parsers.cymru_full_bogons.parser",
"parameters": {
},
},
"ripe-expert": {
"group": "Expert",
"module": "intelmq.bots.experts.ripencc_abuse_contact.expert",
"parameters": {
"query_ripe_stat": True,
},
}
}
DEP_110_EXP = {"n6-collector": {
"group": "Collector",
"module": "intelmq.bots.collectors.stomp.collector",
"parameters": {
"name": "Feed",
},
},
"cymru-full-bogons-parser": {
"group": "Parser",
"module": "intelmq.bots.parsers.cymru.parser_full_bogons",
"parameters": {
}},
"ripe-expert": {
"group": "Expert",
"module": "intelmq.bots.experts.ripe.expert",
"parameters": {
"query_ripe_stat_asn": True,
"query_ripe_stat_ip": True,
},
}}


def generate_function(function):
def test_function(self):
""" Test if no errors happen for upgrade function %s. """ % function.__name__
@@ -23,7 +98,6 @@ def setUp(self):
self.mapping_list.extend((x for x in values))
self.mapping_list_name.extend((x.__name__ for x in values))


def test_all_functions_used(self):
self.assertEqual(len(self.mapping_list_name),
len(set(self.mapping_list_name)),
@@ -36,6 +110,18 @@ def test_all_functions_used(self):
msg='v* functions in the module do not '
'match functions in __all__.')

def test_v110_deprecations(self):
""" Test v110_deprecations """
result = upgrades.v110_deprecations({}, DEP_110, False)
self.assertTrue(result[0])
self.assertEqual(DEP_110_EXP, result[2])

def test_v202_fixes(self):
""" Test v202_feed_name """
result = upgrades.v202_fixes({}, V202, False)
self.assertTrue(result[0])
self.assertEqual(V202_EXP, result[2])


for name in upgrades.__all__:
setattr(TestUpgradeLib, 'test_function_%s' % name,

0 comments on commit b20ef96

Please sign in to comment.
You can’t perform that action at this time.