Merge branch 'master' into blank-location

.gitignore

@@ -1,6 +1,7 @@
 data
 out
 gitout
+.coverage
 
 helpers/ckan/
 helpers/ckan.json

stats/dashboard.py

@@ -358,6 +358,7 @@ def element_versions(self):
     @returns_numberdict
     @memoize
     def _major_version(self):
+        # TODO: Refactor to use _version
         parent = self.element.getparent()
         if parent is None:
             print('No parent of iati-activity, is this a test? Assuming version 1.xx')
@@ -368,6 +369,20 @@ def _major_version(self):
         else:
             return '1'
 
+    @returns_numberdict
+    @memoize
+    def _version(self):
+        allowed_versions = CODELISTS['2']['Version']
+        parent = self.element.getparent()
+        if parent is None:
+            print('No parent of iati-activity, is this a test? Assuming version 1.01')
+            return '1.01'
+        version = self.element.getparent().attrib.get('version')
+        if version and version in allowed_versions:
+            return version
+        else:
+            return '1.01'
+
     @returns_numberdict
     def ruleset_passes(self):
         out = {}
@@ -1172,6 +1187,41 @@ def comprehensiveness_denominators(self):
                 'transaction_traceability': 0
             }
 
+    @returns_numberdict
+    def humanitarian(self):
+        humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010']
+        humanitarian_sectors_dac_3_digit = ['720', '730', '740']
+
+        # logic around use of the @humanitarian attribute
+        is_humanitarian_by_attrib_activity = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['1', 'true']) else 0
+        is_not_humanitarian_by_attrib_activity = 1 if ('humanitarian' in self.element.attrib) and (self.element.attrib['humanitarian'] in ['0', 'false']) else 0
+        is_humanitarian_by_attrib_transaction = 1 if set(self.element.xpath('transaction/@humanitarian')).intersection(['1', 'true']) else 0
+        is_not_humanitarian_by_attrib_transaction = 1 if not is_humanitarian_by_attrib_transaction and set(self.element.xpath('transaction/@humanitarian')).intersection(['0', 'false']) else 0
+        is_humanitarian_by_attrib = (self._version() in ['2.02']) and (is_humanitarian_by_attrib_activity or (is_humanitarian_by_attrib_transaction and not is_not_humanitarian_by_attrib_activity))
+
+        # logic around DAC sector codes deemed to be humanitarian
+        is_humanitarian_by_sector_5_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0
+        is_humanitarian_by_sector_5_digit_transaction = 1 if set(self.element.xpath('transaction[not(@humanitarian="0" or @humanitarian="false")]/sector[@vocabulary="{0}" or not(@vocabulary)]/@code'.format(self._dac_5_code()))).intersection(humanitarian_sectors_dac_5_digit) else 0
+        is_humanitarian_by_sector_3_digit_activity = 1 if set(self.element.xpath('sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0
+        is_humanitarian_by_sector_3_digit_transaction = 1 if set(self.element.xpath('transaction[not(@humanitarian="0" or @humanitarian="false")]/sector[@vocabulary="{0}"]/@code'.format(self._dac_3_code()))).intersection(humanitarian_sectors_dac_3_digit) else 0
+        # helper variables to help make logic easier to read
+        is_humanitarian_by_sector_activity = is_humanitarian_by_sector_5_digit_activity or is_humanitarian_by_sector_3_digit_activity
+        is_humanitarian_by_sector_transaction = is_humanitarian_by_sector_5_digit_transaction or is_humanitarian_by_sector_3_digit_transaction
+        is_humanitarian_by_sector = is_humanitarian_by_sector_activity or (is_humanitarian_by_sector_transaction and (self._major_version() in ['2']))
+
+        # combine the various ways in which an activity may be humanitarian
+        is_humanitarian = 1 if (is_humanitarian_by_attrib or is_humanitarian_by_sector) else 0
+        # deal with some edge cases that have veto
+        if is_not_humanitarian_by_attrib_activity:
+            is_humanitarian = 0
+
+        return {
+            'is_humanitarian': is_humanitarian,
+            'is_humanitarian_by_attrib': is_humanitarian_by_attrib,
+            'contains_humanitarian_scope': 1 if (self._version() in ['2.02']) and self.element.xpath('humanitarian-scope/@type') and self.element.xpath('humanitarian-scope/@code') else 0,
+            'uses_humanitarian_clusters_vocab': 1 if (self._version() in ['2.02']) and self.element.xpath('sector/@vocabulary="10"') else 0
+        }
+
     def _transaction_type_code(self, transaction):
         type_code = None
         transaction_type = transaction.find('transaction-type')
@@ -1285,8 +1335,6 @@ def sum_planned_disbursements_by_year(self):
         return out
 
 
-
-import json
 ckan = json.load(open('helpers/ckan.json'))
 publisher_re = re.compile('(.*)\-[^\-]')
 

stats/tests/test_generic_file_stats.py

@@ -0,0 +1,80 @@
+# coding=utf-8
+
+from lxml import etree
+import pytest
+
+from stats.dashboard import ActivityStats
+
+
+@pytest.mark.parametrize('version', ['1.01', '1.02', '1.03', '1.04', '1.05', '2.01', '2.02'])
+def test_version_detection_valid(version):
+    """
+    Tests that valid versions of the IATI Standard are detected.
+    """
+
+    activity = ActivityStats()
+
+    tree = etree.fromstring('''
+        <iati-activities version="{0}">
+           <iati-activity>
+           </iati-activity>
+        </iati-activities>
+    '''.format(version))
+
+    activity.element = tree.getchildren()[0]
+
+    assert activity._version() == version
+
+
+@pytest.mark.parametrize('version', ['1.06', '2.03', '3.01', '1', '', 'version 1.03', '1.03 version', '1.03 or 1.04', ' 2.01', '2 .01'])
+def test_version_detection_invalid(version):
+    """
+    Tests that invalid versions of the IATI Standard are detected.
+    """
+
+    activity = ActivityStats()
+
+    tree = etree.fromstring('''
+        <iati-activities version="{0}">
+           <iati-activity>
+           </iati-activity>
+        </iati-activities>
+    '''.format(version))
+
+    activity.element = tree.getchildren()[0]
+
+    assert activity._version() == '1.01'
+
+
+def test_version_detection_no_parent():
+    """
+    Tests that XML with no parent returns default version.
+    """
+
+    activity = ActivityStats()
+
+    activity.element = etree.fromstring('''
+        <iati-activity>
+        </iati-activity>
+    ''')
+
+    assert activity._version() == '1.01'
+
+
+def test_version_detection_no_version_attrib():
+    """
+    Tests that XML with no version attribute returns default version.
+    """
+
+    activity = ActivityStats()
+
+    tree = etree.fromstring('''
+        <iati-activities>
+           <iati-activity>
+           </iati-activity>
+        </iati-activities>
+    ''')
+
+    activity.element = tree.getchildren()[0]
+
+    assert activity._version() == '1.01'