Merge pull request #67 from csirtgadgets/fix/xn

fixes #66 #65 #64

csirtg_indicator/utils/__init__.py

@@ -15,8 +15,11 @@
 
 # http://stackoverflow.com/a/17871737
 RE_IPV6 = re.compile('(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))')
+
 # http://goo.gl/Cztyn2 -- probably needs more work
-RE_FQDN = re.compile('^((xn--)?(--)?[a-zA-Z0-9-_@]+(-[a-zA-Z0-9]+)*\.)+[a-zA-Z]{2,}(--p1ai)?$')
+# http://stackoverflow.com/a/26987741/7205341
+# ^((xn--)?(--)?[a-zA-Z0-9-_@]+(-[a-zA-Z0-9]+)*\.)+[a-zA-Z]{2,}(--p1ai)?$
+RE_FQDN = re.compile('^((?!-))(xn--)?[a-z0-9][a-z0-9-_\.]{0,61}[a-z0-9]{0,1}\.(xn--)?([a-z0-9\-]{1,61}|[a-z0-9-]{1,30}\.[a-z]{2,})$')
 RE_URI_SCHEMES = re.compile('^(https?|ftp)$')
 RE_EMAIL = re.compile('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$')
 
@@ -134,18 +137,18 @@ def _email(s):
     elif _hash(indicator):
         return _hash(indicator)
 
-    elif _email(indicator):
-        return 'email'
-
-    elif _fqdn(indicator):
-        return 'fqdn'
-
     elif _ipv4(indicator) or _ipv4_cidr(indicator):
         return 'ipv4'
 
     elif _ipv6(indicator):
         return 'ipv6'
 
+    elif _email(indicator):
+        return 'email'
+
+    elif _fqdn(indicator):
+        return 'fqdn'
+
     try:
         error = 'unknown itype for "{}"'.format(indicator)
     except UnicodeEncodeError:

requirements.txt

@@ -2,4 +2,4 @@ arrow>=0.7.0
 pytricia>=0.9.0
 ipaddress>=1.0.16
 pendulum>=0.5.2
-prettytable>=0.7.2
+prettytable>=0.7.2

test/test_fqdn.py

@@ -1,6 +1,14 @@
 from csirtg_indicator import Indicator
 from csirtg_indicator.utils import is_subdomain
 
+GOOD = [
+    'hdxturkceizle.xn--6frz82g',
+    'xn--1--blcfn0a0ai7a1a7e.xn--p1acf',
+    'example.org',
+    '1.2.3.4.com',
+    'xn----jtbbmekqknepg3a.xn--p1ai'
+]
+
 
 def _not(data):
     for d in data:
@@ -24,9 +32,8 @@ def test_fqdn_urls():
 
 
 def test_fqdn_ok():
-    data = ['example.org', '1.2.3.4.com', 'xn----jtbbmekqknepg3a.xn--p1ai']
 
-    for d in data:
+    for d in GOOD:
         d = Indicator(d)
         assert d.itype is 'fqdn'