BUG: reverse dns expert: ignore all invalid results

fixes #1264

CHANGELOG.md

@@ -15,6 +15,7 @@ CHANGELOG
 #### Parsers
 
 #### Experts
+- Reverse DNS Expert: ignore all invalid results and use first valid one (#1264).
 
 #### Outputs
 

docs/Bots.md

@@ -676,6 +676,8 @@ If the rule is a string, a regex-search is performed, also for numeric values (`
 
 ### Reverse DNS
 
+For both `source.ip` and `destination.ip` the PTR record is fetched and the first valid result is used for `source.reverse_dns`/`destination.reverse_dns`.
+
 #### Information:
 * `name:` reverse-dns
 * `lookup:` dns
@@ -685,7 +687,8 @@ If the rule is a string, a regex-search is performed, also for numeric values (`
 
 #### Configuration Parameters:
 
-FIXME
+* **Cache parameters** (see above)
+* `cache_ttl_invalid_response`: The TTL for cached invalid responses.
 
 * * *
 

intelmq/bots/experts/reverse_dns/expert.py

@@ -14,6 +14,10 @@
 DNS_EXCEPTION_VALUE = "__dns-exception"
 
 
+class InvalidPTRResult(ValueError):
+    pass
+
+
 class ReverseDnsExpertBot(Bot):
 
     def init(self):
@@ -57,19 +61,21 @@ def process(self):
             else:
                 rev_name = reversename.from_address(ip)
                 try:
-                    result = resolver.query(rev_name, "PTR")
-                    expiration = result.expiration
-                    result = result[0]
-
-                    if str(result) == '.':
-                        result = None
-                        raise ValueError
-                except (dns.exception.DNSException, ValueError) as e:
+                    results = resolver.query(rev_name, "PTR")
+                    expiration = results.expiration
+                    for result in results:
+                        # use first valid result
+                        if event.is_valid('source.reverse_dns', str(result)):
+                            break
+                    else:
+                        raise InvalidPTRResult
+                except (dns.exception.DNSException, InvalidPTRResult) as e:
                     # Set default TTL for 'DNS query name does not exist' error
                     ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \
                         getattr(self.parameters, "cache_ttl_invalid_response",
                                 60)
                     self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl)
+                    result = None
 
                 else:
                     ttl = datetime.fromtimestamp(expiration) - datetime.now()

intelmq/tests/bots/experts/reverse_dns/test_expert.py

@@ -29,13 +29,22 @@
                    "time.observation": "2015-01-01T00:00:00+00:00",
                    }
 INVALID_PTR_INP = {"__type": "Event",
-                   "source.ip": "31.210.115.39",  # PTR is .
+                   "source.ip": "31.210.115.39",  # PTR is '.'
                    "time.observation": "2015-01-01T00:00:00+00:00",
                    }
 INVALID_PTR_OUT = {"__type": "Event",
                    "source.ip": "31.210.115.39",
                    "time.observation": "2015-01-01T00:00:00+00:00",
                    }
+INVALID_PTR_INP2 = {"__type": "Event",
+                    "source.ip": "5.157.80.221",  # PTR is '5.157.80.221.' and 'aliancys.peopleinc.nl.'
+                    "time.observation": "2015-01-01T00:00:00+00:00",
+                    }
+INVALID_PTR_OUT2 = {"__type": "Event",
+                    "source.ip": "5.157.80.221",
+                    "source.reverse_dns": "aliancys.peopleinc.nl",
+                    "time.observation": "2015-01-01T00:00:00+00:00",
+                    }
 
 
 @test.skip_redis()
@@ -65,6 +74,11 @@ def test_invalid_ptr(self):
         self.run_bot()
         self.assertMessageEqual(0, INVALID_PTR_OUT)
 
+    def test_invalid_ptr2(self):
+        self.input_message = INVALID_PTR_INP2
+        self.run_bot()
+        self.assertMessageEqual(0, INVALID_PTR_OUT2)
+
 
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()