Skip to content
Permalink
Browse files

Merge branch 'maintenance' into develop

  • Loading branch information
wagner-certat committed Jan 31, 2020
2 parents b0d4381 + be29590 commit 5328166fce5d0e2e34ebf818dc54a3eb6e71bb5e
@@ -75,15 +75,19 @@ CHANGELOG
### Configuration

### Core
- `intelmq.lib.upgrades`:
- Harmonization upgrade: Also check and update regular expressions

### Development

### Harmonization
- `protocol.transport`: Adapt regular expression to allow the value `nvp-ii` (protocol 11).

### Bots
#### Collectors

#### Parsers
- `intelmq.bots.parser.cymru.parser_cap_program`: Support for protocol 11 (`nvp-ii`).

#### Experts

@@ -28,6 +28,8 @@ See the changelog for a full list of changes.
### Tools

### Harmonization
The regular expression of the field `protocol.transport` has been updated to accommodate the value `nvp-ii`.
`intelmqctl upgrade-config` handles the change to automatically upgrade your configuration.

### Configuration

@@ -28,9 +28,12 @@
}
MAPPING_COMMENT = {'bruteforce': ('classification.identifier', 'protocol.application'),
'phishing': ('source.url', )}
PROTOCOL_MAPPING = {'6': 'tcp', # TODO: use getent in harmonization
PROTOCOL_MAPPING = { # TODO: use getent in harmonization
'1': 'icmp',
'6': 'tcp',
'11': 'nvp-ii',
'17': 'udp',
'1': 'icmp'}
}
BOGUS_HOSTNAME_PORT = re.compile('hostname: ([^:]+)port: ([0-9]+)')
DESTINATION_PORT_NUMBERS_TOTAL = re.compile(r' \(total_count:\d+\)$')

@@ -217,7 +217,7 @@
},
"protocol.transport": {
"description": "e.g. tcp, udp, icmp.",
"iregex": "^(ip|icmp|igmp|ggp|ipencap|st2|tcp|cbt|egp|igp|bbn-rcc|nvp|pup|argus|emcon|xnet|chaos|udp|mux|dcn|hmp|prm|xns-idp|trunk-1|trunk-2|leaf-1|leaf-2|rdp|irtp|iso-tp4|netblt|mfe-nsp|merit-inp|sep|3pc|idpr|xtp|ddp|idpr-cmtp|tp\\+\\+|il|ipv6|sdrp|ipv6-route|ipv6-frag|idrp|rsvp|gre|mhrp|bna|esp|ah|i-nlsp|swipe|narp|mobile|tlsp|skip|ipv6-icmp|ipv6-nonxt|ipv6-opts|cftp|sat-expak|kryptolan|rvd|ippc|sat-mon|visa|ipcv|cpnx|cphb|wsn|pvp|br-sat-mon|sun-nd|wb-mon|wb-expak|iso-ip|vmtp|secure-vmtp|vines|ttp|nsfnet-igp|dgp|tcf|eigrp|ospf|sprite-rpc|larp|mtp|ax.25|ipip|micp|scc-sp|etherip|encap|gmtp|ifmp|pnni|pim|aris|scps|qnx|a/n|ipcomp|snp|compaq-peer|ipx-in-ip|vrrp|pgm|l2tp|ddx|iatp|st|srp|uti|smp|sm|ptp|isis|fire|crtp|crdup|sscopmce|iplt|sps|pipe|sctp|fc|divert)$",
"iregex": "^(ip|icmp|igmp|ggp|ipencap|st2|tcp|cbt|egp|igp|bbn-rcc|nvp(-ii)?|pup|argus|emcon|xnet|chaos|udp|mux|dcn|hmp|prm|xns-idp|trunk-1|trunk-2|leaf-1|leaf-2|rdp|irtp|iso-tp4|netblt|mfe-nsp|merit-inp|sep|3pc|idpr|xtp|ddp|idpr-cmtp|tp\\+\\+|il|ipv6|sdrp|ipv6-route|ipv6-frag|idrp|rsvp|gre|mhrp|bna|esp|ah|i-nlsp|swipe|narp|mobile|tlsp|skip|ipv6-icmp|ipv6-nonxt|ipv6-opts|cftp|sat-expak|kryptolan|rvd|ippc|sat-mon|visa|ipcv|cpnx|cphb|wsn|pvp|br-sat-mon|sun-nd|wb-mon|wb-expak|iso-ip|vmtp|secure-vmtp|vines|ttp|nsfnet-igp|dgp|tcf|eigrp|ospf|sprite-rpc|larp|mtp|ax.25|ipip|micp|scc-sp|etherip|encap|gmtp|ifmp|pnni|pim|aris|scps|qnx|a/n|ipcomp|snp|compaq-peer|ipx-in-ip|vrrp|pgm|l2tp|ddx|iatp|st|srp|uti|smp|sm|ptp|isis|fire|crtp|crdup|sscopmce|iplt|sps|pipe|sctp|fc|divert)$",
"length": 11,
"type": "LowercaseString"
},
@@ -313,6 +313,16 @@ def harmonization(defaults, runtime, harmonization, dry_run):
if harmonization[msg_type][fieldname]['type'] != original[msg_type][fieldname]['type']:
harmonization[msg_type][fieldname]['type'] = original[msg_type][fieldname]['type']
changed = True
installed_regex = harmonization[msg_type][fieldname].get('regex')
original_regex = original[msg_type][fieldname].get('regex')
if original_regex and original_regex != installed_regex:
harmonization[msg_type][fieldname]['regex'] = original[msg_type][fieldname]['regex']
changed = True
installed_regex = harmonization[msg_type][fieldname].get('iregex')
original_regex = original[msg_type][fieldname].get('iregex')
if original_regex and original_regex != installed_regex:
harmonization[msg_type][fieldname]['iregex'] = original[msg_type][fieldname]['iregex']
changed = True
return changed, defaults, runtime, harmonization


@@ -26,3 +26,4 @@ scanner|172.16.0.21|64496|2019-09-19 00:03:13|destination_port_numbers: 57518;po
darknet|172.16.0.21|64496|2019-09-30 13:49:49|destination_port_numbers: 17875,24526,54449,9314,4903,1568,20749,30524,59316,60704 (total_count:19);port: 40434;protocol: 17;|Example AS Name, AT
spam|172.16.0.21|64496|2019-10-02 23:00:17||Example AS Name, AT
phishing|172.16.0.21|64496|2019-10-23 12:46:18||Example AS Name, AT
darknet|172.16.0.21|64496|2020-01-10 09:17:17|destination_port_numbers: 0;protocol: 11;|Example AS Name, AT
@@ -180,10 +180,16 @@
'classification.type': 'phishing',
'classification.identifier': 'phishing',
},
{'classification.type': 'scanner',
'classification.identifier': 'darknet',
'protocol.transport': 'nvp-ii',
'destination.port': 0,
'time.source': '2020-01-10T09:17:17+00:00',
},
]
# The number of events a single line in the raw data produces
NUM_EVENTS = [1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 10, 1, 1]
1, 10, 1, 1, 1]
RAWS = []
for i, line in enumerate(RAW_LINES[3:]):
for count in range(NUM_EVENTS[i]):
@@ -224,6 +224,8 @@
del MISSING_REPORT['report']
WRONG_TYPE = deepcopy(HARM)
WRONG_TYPE['event']['source.asn']['type'] = 'String'
WRONG_REGEX = deepcopy(HARM)
WRONG_REGEX['event']['protocol.transport']['iregex'] = 'foobar'


def generate_function(function):
@@ -302,6 +304,12 @@ def test_wrong_type_harmonization(self):
self.assertTrue(result[0])
self.assertEqual(HARM, result[3])

def test_wrong_regex_harmonization(self):
""" Test wrong regex in harmonization """
result = upgrades.harmonization({}, {}, WRONG_REGEX, False)
self.assertTrue(result[0])
self.assertEqual(HARM, result[3])


for name in upgrades.__all__:
setattr(TestUpgradeLib, 'test_function_%s' % name,

0 comments on commit 5328166

Please sign in to comment.
You can’t perform that action at this time.