Merge pull request #367 from NLnetLabs/hotfixes-20190301

Hotfixes 20190301

.gitignore

@@ -9,11 +9,10 @@
 /openssl/
 /build/
 /static/
-.htpasswd
+/.htpasswd
 /nbproject
 /dump.rdb
-celerybeat-schedule
-celerybeat-schedule.db
-celerybeat.pid
+/celerybeat-schedule*
+/celerybeat.pid
 /batch_results/
 /locale_files/

Changelog.md

@@ -0,0 +1,12 @@
+# Change Log
+
+## Unreleased next version
+
+New
+
+Bug Fixes
+
+
+## 1.0.0
+
+Initial public release.

README.md

@@ -52,11 +52,19 @@ based application.
 Current install base is Django 1.11 with Python 3.5+
 (Python 3.7+ is not currently supported).
 
+
+### Quick start
+
 Although the first open source release of the project is aimed to provide
 transparency on the tool and the way the tests are run, there is (currently) no
-trivial way to install the software.
+trivial way to install the software. Nonetheless, you could easily spin up a
+ready to use local dockerized environment for local development and testing
+purposes by following the [docker instructions](docker/README.md)
+
+
+### Slow start
 
-However if you feel brave enough you can follow the
+If you feel brave enough for a system install you can follow the
 [installation instructions](documentation/Installation.md).
 
 The [customize instructions](documentation/Customize.md) describe how you could

checks/categories.py

@@ -330,6 +330,9 @@ def result_good(self):
         self._status(STATUS_SUCCESS)
         self.verdict = "detail web ipv6 web-ipv46 verdict good"
 
+    def result_no_v4(self):
+        self.worst_status = STATUS_NOT_TESTED
+
 
 class MailIpv6MxAaaa(Subtest):
     def __init__(self):
@@ -1318,7 +1321,9 @@ def __init__(self):
             label="detail mail tls dane-exists label",
             explanation="detail mail tls dane-exists exp",
             tech_string="detail mail tls dane-exists tech table",
-            worst_status=STATUS_NOTICE)
+            worst_status=STATUS_NOTICE,
+            full_score=scoring.MAIL_TLS_DANE_VALIDATED,
+            model_score_field="dane_score")
 
     def was_tested(self):
         self.worst_status = scoring.MAIL_TLS_DANE_EXISTS_WORST_STATUS

checks/scoring.py

@@ -182,7 +182,7 @@
 
 MAIL_AUTH_DMARC_PASS = FULL_WEIGHT_POINTS
 MAIL_AUTH_DMARC_FAIL = NO_POINTS
-MAIL_AUTH_SPF_ERROR = NO_POINTS
+MAIL_AUTH_DMARC_ERROR = NO_POINTS
 MAIL_AUTH_DMARC_WORST_STATUS = STATUS_FAIL
 
 MAIL_AUTH_DMARC_POLICY_PASS = FULL_WEIGHT_POINTS

checks/tasks/dnssec.py

@@ -1,6 +1,7 @@
 # Copyright: 2019, NLnet Labs and the Internet.nl contributors
 # SPDX-License-Identifier: Apache-2.0
 from collections import OrderedDict
+import socket
 import time
 
 import pythonwhois
@@ -129,10 +130,13 @@ def registrar_lookup(addr):
         else:
             try:
                 whois = pythonwhois.get_whois(".".join(addr.split(".")[-2:]))
-            except Exception:
-                whois = None
-            if whois and isinstance(whois, dict) and whois.get("registrar"):
-                res = ", ".join(whois["registrar"])[:250]
+                if (whois and isinstance(whois, dict)
+                        and whois.get("registrar")):
+                    res = ", ".join(whois["registrar"])[:250]
+            except (
+                    socket.error, pythonwhois.shared.WhoisException,
+                    UnicodeDecodeError):
+                pass
 
             cache.set(cache_id, res, cache_ttl)
     return res

checks/tasks/http_headers.py

@@ -7,6 +7,17 @@
 from .. import scoring
 
 
+def get_multiple_values_from_header(header):
+    """
+    Get all the values for the header.
+
+    Multiple values of the same header are in a comma separated list; make sure
+    to ignore white space when splitting the values.
+
+    """
+    return [value.strip() for value in header.split(',')]
+
+
 class HeaderCheckerContentEncoding(object):
     """
     Class for checking the Content-Encoding HTTP header.
@@ -56,7 +67,7 @@ def check(self, value, results):
             score = scoring.WEB_APPSECPRIV_CONTENT_SECURITY_POLICY_BAD
             results['content_security_policy_score'] = score
         else:
-            values = value.split(",")
+            values = get_multiple_values_from_header(value)
             results['content_security_policy_values'].extend(values)
 
     def get_positive_values(self):
@@ -95,7 +106,7 @@ def check(self, value, results):
             results['hsts_score'] = scoring.WEB_TLS_HSTS_BAD
             self.first_time_seen = False
         elif value:
-            header_values = value.split(",")
+            header_values = get_multiple_values_from_header(value)
             try:
                 max_age = header_values[0].lower().split(
                     'max-age=')[1].split(';')[0]
@@ -143,7 +154,7 @@ def check(self, value, results):
             results['x_frame_options_score'] = score
             results['x_frame_options_enabled'] = False
         else:
-            values = value.split(",")
+            values = get_multiple_values_from_header(value)
             first_header = values[0].upper()
             if not (first_header == "DENY"
                     or first_header == "SAMEORIGIN"
@@ -188,7 +199,7 @@ def check(self, value, results):
             results['x_content_type_options_score'] = score
             results['x_content_type_options_enabled'] = False
         else:
-            values = value.split(",")
+            values = get_multiple_values_from_header(value)
             if not values[0].lower() == "nosniff":
                 score = scoring.WEB_APPSECPRIV_X_CONTENT_TYPE_OPTIONS_BAD
                 results['x_content_type_options_score'] = score
@@ -230,7 +241,7 @@ def check(self, value, results):
             results['x_xss_protection_score'] = score
             results['x_xss_protection_enabled'] = False
         else:
-            values = value.split(",")
+            values = get_multiple_values_from_header(value)
             enabled = values[0].split(";")[0]
             if enabled == "0":
                 score = scoring.WEB_APPSECPRIV_X_XSS_PROTECTION_BAD
@@ -278,7 +289,7 @@ def check(self, value, results):
             results['referrer_policy_enabled'] = False
 
         else:
-            values = value.split(",")
+            values = get_multiple_values_from_header(value)
             for value in values:
                 if value.lower() not in [
                         'no-referrer',

checks/tasks/ipv6.py

@@ -248,6 +248,8 @@ def callback(results, addr, parent, parent_name, category):
                         category.subtests['web_ipv46'].result_good()
                     elif web_simhash_distance >= 0:
                         category.subtests['web_ipv46'].result_bad()
+                    else:
+                        category.subtests['web_ipv46'].result_no_v4()
 
     parent.report = category.gen_report()
     parent.save()

checks/tasks/shared.py

@@ -307,22 +307,22 @@ def http_fetch(
                 conn.close()
             break
         # If we could not connect we can try again.
-        except socket.error as e:
+        except socket.error:
             try:
                 conn.close()
-            except Exception:
+            except (socket.error, http.client.HTTPException):
                 pass
             tries_left -= 1
             if tries_left <= 0:
-                raise e
+                raise
             time.sleep(1)
         # If we got another exception just raise it.
-        except Exception as e:
+        except http.client.HTTPException:
             try:
                 conn.close()
-            except Exception:
+            except (socket.error, http.client.HTTPException):
                 pass
-            raise e
+            raise
 
     if not ret_headers:
         ret_headers = {}

checks/tasks/tls.py

@@ -696,10 +696,14 @@ def get_common_name(cert):
     Get the commonName of the certificate.
 
     """
-    common_name = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0]
     value = "-"
-    if common_name:
-        value = common_name.value
+    try:
+        common_name = (
+            cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0])
+        if common_name:
+            value = common_name.value
+    except IndexError:
+        pass
     return value
 
 

documentation/Deployment.md

@@ -13,16 +13,17 @@ pushthe changes to the server:
    make frontend js
    ```
 
-2. Periodically update needed remote data with:
+2. _Periodically_ update needed remote data with:
    ```
    make update_cert_fingerprints
    make update_padded_macs
    make update_root_key_file
    ```
 On the server you need to:
 
-1. Make sure that you have the latest translation files compiled:
+1. Create the latest translation files and compile them:
    ```
+   make translations
    cd checks; python ../manage.py compilemessages; cd ..
    ```
 

documentation/Installation.md

@@ -91,8 +91,9 @@ Unbound (and pylibunbound) is used as a DNS resolver/nameserver for the various
 tests performed.
 
 Use the fork at https://github.com/ralphdolmans/unbound.
-Make sure to use the `internetnl` branch and follow the README instructions for
-installation.
+Make sure to use the `internetnl` branch and follow the
+[README.md](https://github.com/ralphdolmans/unbound/blob/internetnl/README.md)
+instructions for installation.
 
 If you setup a python virtual environment you should enable it for unbound's
 installation.
@@ -106,6 +107,10 @@ ldns-dane is an example application from the ldns library that validates a
 domain's DANE record(s) and is used for validating the DANE records in the
 website and mail tests.
 
+_Note that ldns-dane will use your locally configured DNS resolver to get
+and validate the TLSA records. This means that your locally configured DNS
+resolver needs to have DNSSEC enabled._
+
 If your system has ldns >= 1.7.0 **and** openssl >= 1.1.0 you are good to go.
 
 If not:
@@ -117,7 +122,7 @@ If not:
   - Get an ldns version >= 1.7.0 from [here](https://www.nlnetlabs.nl/projects/ldns/download/)
   - `./configure --with-examples --with-ssl=/path/to/above/ssl/build/dir`
   - `make`
-- Create a wrapper file for using the compiled ldns-dane binary
+- Create a wrapper file for using the compiled ldns-dane binary  
   `cat ldns-dane-wrapper`
   ```
   #!/bin/sh
@@ -148,10 +153,6 @@ PostgreSQL is used as the database.
   * `sudo -u postgres createuser <username> -P`
   * `sudo -u postgres createdb -O <username> <db_name>`
 
-Copy distributed config and edit:
- * `cp internetnl/settings.py-dist internetnl/setting.py`
- * change secret, database setting, redis settings, ldns-dane location
-
 If you expect high DB traffic the use of
 [PgBouncer](https://pgbouncer.github.io/) is recommended as a connection pooler
 for PostgreSQL.

internetnl/settings.py-dist

@@ -85,11 +85,11 @@ if ENABLE_BATCH:
 
 # --- Application definition
 #
+IPV6_TEST_ADDR = "::1"
 ALLOWED_HOSTS = [".internet.nl", "internet.nl", IPV6_TEST_ADDR, "[{}]".format(IPV6_TEST_ADDR)]
 ADMINS = (('Administrator', 'django@internet.nl'))
 SERVER_EMAIL = 'django@internet.nl'
 INTERNAL_IPS = [ "localhost", "127.0.0.1" ]
-IPV6_TEST_ADDR = "::1"
 CONN_TEST_DOMAIN = "internet.nl"
 
 INSTALLED_APPS = [