BUG: FQDNs in IDN format

fixes #622

Signed-off-by: Sebastian Wagner <sebix@sebix.at>

CHANGELOG.md

@@ -33,6 +33,7 @@ v1.0 (in development, master branch)
 - ENH: Added descriptions and matching types to all fields
 - DOC: harmonization documentation has same fields as configuration, docs are generated from configuration
 - ENH: New type LowercaseString and UppercaseString
+- BUG: FQDNs are only allowed in IDN representation
 
 #### Most important changes:
 - `(source|destination).bgp_prefix` is now `(source|destination).network`

intelmq/lib/harmonization.py

@@ -25,6 +25,7 @@
 
 import dateutil.parser
 import dns.resolver
+import encodings.idna
 import pytz
 
 import intelmq.lib.utils as utils
@@ -314,6 +315,9 @@ def is_valid(value, sanitize=False):
         if URL().is_valid(value):
             return False
 
+        if encodings.idna.ToASCII(value).decode() != value:
+            return False
+
         """
         # "localhost" is a valid hostname
         if len(value.split('.')) <= 1:
@@ -327,7 +331,7 @@ def is_valid(value, sanitize=False):
 
     @staticmethod
     def sanitize(value):
-        return value.rstrip('.').lower()
+        return encodings.idna.ToASCII(value).decode().rstrip('.').lower()
 
     @staticmethod
     def to_ip(value):

intelmq/tests/lib/test_harmonization.py

@@ -194,11 +194,13 @@ def test_fqdn_valid(self):
         self.assertTrue(harmonization.FQDN.is_valid('ex-am.ple.example'))
         self.assertTrue(harmonization.FQDN.is_valid('intelmq.org'))
         self.assertTrue(harmonization.FQDN.is_valid('sub_sub2.example.net'))
+        self.assertTrue(harmonization.FQDN.is_valid('xn--1.at-4qa'))
 
     def test_fqdn_invalid(self):
         """ Test FQDN.is_valid with invalid arguments. """
         self.assertFalse(harmonization.FQDN.is_valid('ex-am.ple.example.'))
         self.assertFalse(harmonization.FQDN.is_valid('exAmple.com'))
+        self.assertFalse(harmonization.FQDN.is_valid('ö1.at'))
 
     def test_fqdn_sanitize(self):
         """ Test FQDN.sanitize with valid arguments. """
@@ -208,6 +210,7 @@ def test_fqdn_sanitize(self):
                                                     sanitize=True))
         self.assertTrue(harmonization.FQDN.is_valid('exAmple.net',
                                                     sanitize=True))
+        self.assertTrue(harmonization.FQDN.is_valid('ö1.at', sanitize=True))
 
     def test_fqdn_to_ip(self):
         """ Test FQDN.to_ip """