Merge branch 'master' into 20171024-update-coverage

stats/dashboard.py

@@ -41,8 +41,13 @@ def add_years(d, years):
         return d + (date(d.year + years, 1, 1) - date(d.year, 1, 1))
 
 
-def all_and_not_empty(bool_iterable):
-    """ For a given list, check that all elements return true and that the list is not empty """
+def all_true_and_not_empty(bool_iterable):
+    """For a given list, check that all elements return true and that the list is not empty.
+
+    Args:
+        bool_iterable (iterable of bool): An iterable containing values that can be cast to a bool.
+
+    """
 
     # Ensure that the given list is indeed a simple list
     bool_list = list(bool_iterable)
@@ -1003,42 +1008,53 @@ def is_text_in_element(elementName):
             'description': is_text_in_element('description'),
             'activity-status': self.element.find('activity-status') is not None,
             'activity-date': self.element.find('activity-date') is not None,
-            'sector': self.element.find('sector') is not None or (self._major_version() != '1' and all_and_not_empty(
+            'sector': self.element.find('sector') is not None or (self._major_version() != '1' and all_true_and_not_empty(
                     (transaction.find('sector') is not None)
                         for transaction in self.element.findall('transaction')
                 )),
             'country_or_region': (
                 self.element.find('recipient-country') is not None
                 or self.element.find('recipient-region') is not None
-                or (self._major_version() != '1' and all_and_not_empty(
+                or (self._major_version() != '1' and all_true_and_not_empty(
                     (transaction.find('recipient-country') is not None or
                      transaction.find('recipient-region') is not None)
                         for transaction in self.element.findall('transaction')
                 ))),
             'transaction_commitment': self.element.xpath('transaction[transaction-type/@code="{}" or transaction-type/@code="11"]'.format(self._commitment_code())),
             'transaction_spend': self.element.xpath('transaction[transaction-type/@code="{}" or transaction-type/@code="{}"]'.format(self._disbursement_code(), self._expenditure_code())),
-            'transaction_currency': all_and_not_empty(x.xpath('value/@value-date') and x.xpath('../@default-currency|./value/@currency') for x in self.element.findall('transaction')),
-            'transaction_traceability': all_and_not_empty(x.xpath('provider-org/@provider-activity-id') for x in self.element.xpath('transaction[transaction-type/@code="{}"]'.format(self._incoming_funds_code())))
+            'transaction_currency': all_true_and_not_empty(x.xpath('value/@value-date') and x.xpath('../@default-currency|./value/@currency') for x in self.element.findall('transaction')),
+            'transaction_traceability': all_true_and_not_empty(x.xpath('provider-org/@provider-activity-id') for x in self.element.xpath('transaction[transaction-type/@code="{}"]'.format(self._incoming_funds_code())))
                                         or self._is_donor_publisher(),
             'budget': self.element.findall('budget'),
             'contact-info': self.element.findall('contact-info/email'),
             'location': self.element.xpath('location/point/pos|location/name|location/description|location/location-administrative'),
             'location_point_pos': self.element.xpath('location/point/pos'),
-            'sector_dac': self.element.xpath('sector[@vocabulary="{}" or @vocabulary="{}" or not(@vocabulary)]'.format(self._dac_5_code(), self._dac_3_code())) or
-            (self._major_version() != '1' and all_and_not_empty([transaction.xpath('sector[@vocabulary="{}" or @vocabulary="{}" or not(@vocabulary)]'.format(self._dac_5_code(), self._dac_3_code())) for transaction in self.element.xpath('transaction')])),
+            'sector_dac': self._is_sector_dac(),
             'capital-spend': self.element.xpath('capital-spend/@percentage'),
             'document-link': self.element.findall('document-link'),
             'activity-website': self.element.xpath('activity-website' if self._major_version() == '1' else 'document-link[category/@code="A12"]'),
             'recipient_language': self._is_recipient_language_used(),
             'conditions_attached': self.element.xpath('conditions/@attached'),
             'result_indicator': self.element.xpath('result/indicator'),
             'aid_type': (
-                all_and_not_empty(self.element.xpath('default-aid-type/@code'))
-                or all_and_not_empty([transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')])
+                all_true_and_not_empty(self.element.xpath('default-aid-type/@code'))
+                or all_true_and_not_empty([transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')])
                 )
-            # Alternative: all(map(all_and_not_empty, [transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')]))
+            # Alternative: all(map(all_true_and_not_empty, [transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')]))
         }
 
+    def _is_sector_dac(self):
+        """Determine whether an activity has comprehensive DAC sectors against the validation methodology."""
+        sector_dac_activity_level = self.element.xpath('sector[@vocabulary="{}" or @vocabulary="{}" or not(@vocabulary)]'.format(self._dac_5_code(), self._dac_3_code()))
+
+        if self._major_version() != '1':
+            sector_dac_transaction_level = [transaction.xpath('sector[@vocabulary="{}" or @vocabulary="{}" or not(@vocabulary)]'.format(self._dac_5_code(), self._dac_3_code())) for transaction in self.element.xpath('transaction')]
+            all_transactions_have_dac_sector_codes = all_true_and_not_empty(sector_dac_transaction_level)
+        else:
+            all_transactions_have_dac_sector_codes = False
+
+        return sector_dac_activity_level or all_transactions_have_dac_sector_codes
+
     def _comprehensiveness_with_validation_bools(self):
 
             def element_ref(element_obj):
@@ -1086,11 +1102,11 @@ def empty_or_percentage_sum_is_100(path, by_vocab=False):
                         if self._major_version() is not '1' else True
                     )),
                 'participating-org': bools['participating-org'] and self._funding_code() in self.element.xpath('participating-org/@role'),
-                'activity-status': bools['activity-status'] and all_and_not_empty(x in CODELISTS[self._major_version()]['ActivityStatus'] for x in self.element.xpath('activity-status/@code')),
+                'activity-status': bools['activity-status'] and all_true_and_not_empty(x in CODELISTS[self._major_version()]['ActivityStatus'] for x in self.element.xpath('activity-status/@code')),
                 'activity-date': (
                     bools['activity-date'] and
                     self.element.xpath('activity-date[@type="{}" or @type="{}"]'.format(self._planned_start_code(), self._actual_start_code())) and
-                    all_and_not_empty(map(valid_date, self.element.findall('activity-date')))
+                    all_true_and_not_empty(map(valid_date, self.element.findall('activity-date')))
                     ),
                 'sector': (
                     bools['sector'] and
@@ -1101,12 +1117,12 @@ def empty_or_percentage_sum_is_100(path, by_vocab=False):
                 'transaction_commitment': (
                     bools['transaction_commitment'] and
                     all([ valid_value(x.find('value')) for x in bools['transaction_commitment'] ]) and
-                    all_and_not_empty(any(valid_date(x) for x in t.xpath('transaction-date|value')) for t in bools['transaction_commitment'])
+                    all_true_and_not_empty(any(valid_date(x) for x in t.xpath('transaction-date|value')) for t in bools['transaction_commitment'])
                     ),
                 'transaction_spend': (
                     bools['transaction_spend'] and
                     all([ valid_value(x.find('value')) for x in bools['transaction_spend'] ]) and
-                    all_and_not_empty(any(valid_date(x) for x in t.xpath('transaction-date|value')) for t in bools['transaction_spend'])
+                    all_true_and_not_empty(any(valid_date(x) for x in t.xpath('transaction-date|value')) for t in bools['transaction_spend'])
                     ),
                 'transaction_currency': all(
                     all(map(valid_date, t.findall('value'))) and
@@ -1120,22 +1136,22 @@ def empty_or_percentage_sum_is_100(path, by_vocab=False):
                         valid_date(budget.find('value')) and
                         valid_value(budget.find('value'))
                         for budget in bools['budget'])),
-                'location_point_pos': all_and_not_empty(
+                'location_point_pos': all_true_and_not_empty(
                     valid_coords(x.text) for x in bools['location_point_pos']),
                 'sector_dac': (
                     bools['sector_dac'] and
                     all(x.attrib.get('code') in CODELISTS[self._major_version()]['Sector'] for x in self.element.xpath('sector[@vocabulary="{}" or not(@vocabulary)]'.format(self._dac_5_code()))) and
                     all(x.attrib.get('code') in CODELISTS[self._major_version()]['SectorCategory'] for x in self.element.xpath('sector[@vocabulary="{}"]'.format(self._dac_3_code())))
                     ),
-                'document-link': all_and_not_empty(
+                'document-link': all_true_and_not_empty(
                     valid_url(x) and x.find('category') is not None and x.find('category').attrib.get('code') in CODELISTS[self._major_version()]['DocumentCategory'] for x in bools['document-link']),
-                'activity-website': all_and_not_empty(map(valid_url, bools['activity-website'])),
+                'activity-website': all_true_and_not_empty(map(valid_url, bools['activity-website'])),
                 'aid_type': (
                     bools['aid_type'] and
                     # i) Value in default-aid-type/@code is found in the codelist
-                    (all_and_not_empty([code in CODELISTS[self._major_version()]['AidType'] for code in self.element.xpath('default-aid-type/@code')])
+                    (all_true_and_not_empty([code in CODELISTS[self._major_version()]['AidType'] for code in self.element.xpath('default-aid-type/@code')])
                      # Or ii) Each transaction has a aid-type/@code which is found in the codelist
-                     or all_and_not_empty(
+                     or all_true_and_not_empty(
                         [set(x).intersection(CODELISTS[self._major_version()]['AidType'])
                         for x in [transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')]]
                         )

stats/tests/test_comprehensiveness.py

@@ -49,9 +49,8 @@ def end_planned_date(datestring):
         Create an activity_stats element with a specified 'end-planned' date.
         Also sets the current date to 9990-06-01
 
-        Keyword arguments:
-        datestring -- An ISO date to be used as the 'end-planned' date for the
-            activity_stats element to be returned.
+        Args:
+            datestring (str): An ISO date to be used as the 'end-planned' date for the activity_stats element to be returned.
         """
         activity_stats = MockActivityStats(major_version)
         activity_stats.today = datetime.date(9990, 6, 1)

statsrunner/gitaggregate-publisher.py

@@ -1,3 +1,4 @@
+import datetime
 import decimal
 import json
 import os
@@ -43,7 +44,7 @@
     print "gitaggregate-publisher for commit {}".format(commit)
 
     for publisher in os.listdir(os.path.join(GITOUT_DIR, 'commits', commit, 'aggregated-publisher')):
-        print "Currently looping over publisher {}".format(publisher)
+        print "{0} Currently looping over publisher {1}".format(str(datetime.datetime.now()), publisher)
 
         # Set output directory for this publisher and attempt to make the directory. Pass if it already exists
         git_out_dir = os.path.join(GITOUT_DIR,'gitaggregate-publisher-dated' if dated else 'gitaggregate-publisher', publisher)

statsrunner/loop.py

@@ -14,6 +14,8 @@ def call_stats(this_stats, args):
 
     Args:
       this_stats (cls): stats_module that specifies calculations for relevant input, processed by internal methods of process_file().
+
+      args: Object containing program run options (set by CLI arguments at runtime. See __init__ for more details).
     """
     this_out = {}
     # For each method within this_stats object check the method is an enabled stat, if it is not enabled continue to next method.