Merge branch 'develop' into bugfix/treat_multiple_dmarc_policy_record…

…s_as_an_error

.gitignore

@@ -95,3 +95,9 @@ ENV/
 
 # Rope project settings
 .ropeproject
+
+# PyCharm project settings
+.idea/
+
+# Cached Public Suffix List
+*.dat

README.md

@@ -30,10 +30,10 @@ install the requirements:
 pip install -r requirements.txt
 ```
 
-Then run it as a module via `python -m`:
+Then run the CLI:
 
 ```bash
-python -m trustymail.cli [options] example.com
+python scripts/trustymail [options] example.com
 ```
 
 
@@ -140,6 +140,18 @@ The following values are returned in `results.csv`:
 * `DMARC Policy` - An adjudication, based on any policies found in
   `DMARC Results` and `DMARC Results on Base Domain`, of the relevant
   DMARC policy that applies.
+* `DMARC Policy Percentage` - The percentage of mail that should be
+  subjected to the `DMARC Policy` according to the `DMARC Results`.
+* `DMARC Aggregate Report URIs` - A list of the DMARC aggregate report
+  URIs specified by the domain.
+* `DMARC Forensic Report URIs` - A list of the DMARC forensic report
+  URIs specified by the domain. 
+* `DMARC Has Aggregate Report URI` - A boolean value that indicates if
+  `DMARC Results` included `rua` URIs that tell recipients where to
+  send DMARC aggregate reports.
+* `DMARC Has Forensic Report URI` - A boolean value that indicates if
+  `DMARC Results` included `ruf` URIs that tell recipients where to
+  send DMARC forensic reports .
 
 #### Etc.
 

trustymail/cli.py → scripts/trustymail

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 """trustymail: A tool for scanning DNS mail records for evaluating security.
 Usage:
   trustymail (INPUT ...) [options]

setup.py

@@ -41,6 +41,8 @@
 
         # Specify the Python versions you support here. In particular, ensure
         # that you indicate whether you support Python 2, Python 3 or both.
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
@@ -69,9 +71,5 @@
         ],
     },
 
-    entry_points={
-        'console_scripts': [
-            'trustymail = trustymail.cli:main'
-        ]
-    }
+    scripts=['scripts/trustymail']
 )

trustymail/__init__.py

@@ -1 +1,3 @@
-__version__ = '0.4.0-dev'
+from __future__ import unicode_literals, absolute_import, print_function
+
+__version__ = '0.5.0-dev'

trustymail/domain.py

@@ -1,12 +1,58 @@
-from publicsuffix import PublicSuffixList
+from os import path, stat
+from datetime import datetime, timedelta
+from collections import OrderedDict
+
+import publicsuffix
 
 from trustymail import trustymail
 
-public_list = PublicSuffixList()
 
+def get_psl():
+    """
+    Gets the Public Suffix List - either new, or cached in the CWD for 24 hours
 
-class Domain:
+    Returns
+    -------
+    PublicSuffixList: An instance of PublicSuffixList loaded with a cached or updated list
+    """
+    psl_path = 'public_suffix_list.dat'
+
+    def download_psl():
+        fresh_psl = publicsuffix.fetch()
+        with open(psl_path, 'w', encoding='utf-8') as fresh_psl_file:
+            fresh_psl_file.write(fresh_psl.read())
+
+        return publicsuffix.PublicSuffixList(fresh_psl)
+
+    if not path.exists(psl_path):
+        psl = download_psl()
+    else:
+        psl_age = datetime.now() - datetime.fromtimestamp(stat(psl_path).st_mtime)
+        if psl_age > timedelta(hours=24):
+            psl = download_psl()
+        else:
+            with open(psl_path, encoding='utf-8') as psl_file:
+                psl = publicsuffix.PublicSuffixList(psl_file)
+
+    return psl
+
+
+public_list = get_psl()
 
+
+def format_list(record_list):
+    """Format a list into a string to increase readability in CSV"""
+    # record_list should only be a list, not an integer, None, or
+    # anything else.  Thus this if clause handles only empty
+    # lists.  This makes a "null" appear in the JSON output for
+    # empty lists, as expected.
+    if not record_list:
+        return None
+
+    return ', '.join(record_list)
+
+
+class Domain:
     base_domains = {}
 
     def __init__(self, domain_name, timeout, smtp_timeout, smtp_localhost, smtp_ports, smtp_cache, dns_hostnames):
@@ -31,6 +77,11 @@ def __init__(self, domain_name, timeout, smtp_timeout, smtp_localhost, smtp_port
         self.spf = []
         self.dmarc = []
         self.dmarc_policy = None
+        self.dmarc_pct = None
+        self.dmarc_aggregate_uris = []
+        self.dmarc_forensic_uris = []
+        self.dmarc_has_aggregate_uri = False
+        self.dmarc_has_forensic_uri = False
 
         # Syntax validity - default spf to false as the lack of an SPF is a bad thing.
         self.valid_spf = False
@@ -97,9 +148,9 @@ def parent_valid_dmarc(self):
         return ans
 
     def parent_dmarc_results(self):
-        ans = self.format_list(self.dmarc)
+        ans = format_list(self.dmarc)
         if self.base_domain:
-            ans = self.format_list(self.base_domain.dmarc)
+            ans = format_list(self.base_domain.dmarc)
         return ans
 
     def get_dmarc_policy(self):
@@ -114,52 +165,51 @@ def get_dmarc_policy(self):
         return ans
 
     def generate_results(self):
-        mail_servers_that_support_smtp = [x for x in self.starttls_results.keys() if self.starttls_results[x]['supports_smtp']]
-        mail_servers_that_support_starttls = [x for x in self.starttls_results.keys() if self.starttls_results[x]['starttls']]
+        mail_servers_that_support_smtp = [x for x in self.starttls_results.keys() if self.starttls_results[x][
+            'supports_smtp']]
+        mail_servers_that_support_starttls = [x for x in self.starttls_results.keys() if self.starttls_results[x][
+            'starttls']]
         domain_supports_smtp = bool(mail_servers_that_support_smtp)
 
-        results = {
-            'Domain': self.domain_name,
-            'Base Domain': self.base_domain_name,
-            'Live': self.is_live,
+        results = OrderedDict([
+            ('Domain', self.domain_name),
+            ('Base Domain', self.base_domain_name),
+            ('Live', self.is_live),
 
-            'MX Record': self.has_mail(),
-            'Mail Servers': self.format_list(self.mail_servers),
-            'Mail Server Ports Tested': self.format_list([str(port) for port in self.ports_tested]),
-            'Domain Supports SMTP Results': self.format_list(mail_servers_that_support_smtp),
+            ('MX Record', self.has_mail()),
+            ('Mail Servers', format_list(self.mail_servers)),
+            ('Mail Server Ports Tested', format_list([str(port) for port in self.ports_tested])),
+            ('Domain Supports SMTP Results', format_list(mail_servers_that_support_smtp)),
             # True if and only if at least one mail server speaks SMTP
-            'Domain Supports SMTP': domain_supports_smtp,
-            'Domain Supports STARTTLS Results': self.format_list(mail_servers_that_support_starttls),
+            ('Domain Supports SMTP', domain_supports_smtp),
+            ('Domain Supports STARTTLS Results', format_list(mail_servers_that_support_starttls)),
             # True if and only if all mail servers that speak SMTP
             # also support STARTTLS
-            'Domain Supports STARTTLS': domain_supports_smtp and all([self.starttls_results[x]['starttls'] for x in mail_servers_that_support_smtp]),
+            ('Domain Supports STARTTLS', domain_supports_smtp and all([self.starttls_results[x]['starttls'] for x in mail_servers_that_support_smtp])),
 
-            'SPF Record': self.has_spf(),
-            'Valid SPF': self.valid_spf,
-            'SPF Results': self.format_list(self.spf),
+            ('SPF Record', self.has_spf()),
+            ('Valid SPF', self.valid_spf),
+            ('SPF Results', format_list(self.spf)),
 
-            'DMARC Record': self.has_dmarc(),
-            'Valid DMARC': self.has_dmarc() and self.valid_dmarc,
-            'DMARC Results': self.format_list(self.dmarc),
+            ('DMARC Record', self.has_dmarc()),
+            ('Valid DMARC', self.has_dmarc() and self.valid_dmarc),
+            ('DMARC Results', format_list(self.dmarc)),
 
-            'DMARC Record on Base Domain': self.parent_has_dmarc(),
-            'Valid DMARC Record on Base Domain': self.parent_has_dmarc() and self.parent_valid_dmarc(),
-            'DMARC Results on Base Domain': self.parent_dmarc_results(),
-            'DMARC Policy': self.get_dmarc_policy(),
+            ('DMARC Record on Base Domain', self.parent_has_dmarc()),
+            ('Valid DMARC Record on Base Domain', self.parent_has_dmarc() and self.parent_valid_dmarc()),
+            ('DMARC Results on Base Domain', self.parent_dmarc_results()),
+            ('DMARC Policy', self.get_dmarc_policy()),
+            ('DMARC Policy Percentage', self.dmarc_pct),
 
-            'Syntax Errors': self.format_list(self.syntax_errors),
-            'Debug Info': self.format_list(self.debug_info)
-        }
+            ("DMARC Aggregate Report URIs", format_list(self.dmarc_aggregate_uris)),
+            ("DMARC Forensic Report URIs", format_list(self.dmarc_forensic_uris)),
 
-        return results
+            ('DMARC Has Aggregate Report URI', self.dmarc_has_aggregate_uri),
+            ('DMARC Has Forensic Report URI', self.dmarc_has_forensic_uri),
 
-    # Format a list into a string to increase readability in CSV.
-    def format_list(self, record_list):
-        # record_list should only be a list, not an integer, None, or
-        # anything else.  Thus this if clause handles only empty
-        # lists.  This makes a "null" appear in the JSON output for
-        # empty lists, as expected.
-        if not record_list:
-            return None
 
-        return ', '.join(record_list)
+            ('Syntax Errors', format_list(self.syntax_errors)),
+            ('Debug Info', format_list(self.debug_info))
+        ])
+
+        return results